fix(orchestration): fix set step failure (#10031)

What:
 - copy data before saving checkpoint
 - removed unused data format function
 - properly handle registerStepFailure to not throw
 - emit onFinish event even when execution failed

packages/modules/workflow-engine-redis/integration-tests/__fixtures__/workflow_2.ts

@@ -2,6 +2,7 @@ import {
   createStep,
   createWorkflow,
   StepResponse,
+  WorkflowResponse,
 } from "@medusajs/framework/workflows-sdk"
 
 const step_1 = createStep(
@@ -49,6 +50,14 @@ const step_3 = createStep(
   })
 )
 
+const broken_step_2 = createStep(
+  "broken_step_2",
+  jest.fn(() => {}),
+  jest.fn((_, context) => {
+    throw new Error("Broken compensation step")
+  })
+)
+
 createWorkflow(
   {
     name: "workflow_2",
@@ -67,3 +76,19 @@ createWorkflow(
     return step_3(ret2)
   }
 )
+
+createWorkflow(
+  {
+    name: "workflow_2_revert_fail",
+    retentionTime: 1000,
+  },
+  function (input) {
+    step_1(input)
+
+    broken_step_2().config({
+      async: true,
+    })
+
+    return new WorkflowResponse("done")
+  }
+)

packages/modules/workflow-engine-redis/integration-tests/__tests__/index.spec.ts

@@ -15,13 +15,13 @@ import {
   TransactionHandlerType,
   TransactionStepState,
 } from "@medusajs/framework/utils"
-import { asValue } from "awilix"
 import { moduleIntegrationTestRunner } from "@medusajs/test-utils"
+import { asValue } from "awilix"
 import { setTimeout } from "timers/promises"
+import { WorkflowsModuleService } from "../../src/services"
 import "../__fixtures__"
 import { createScheduled } from "../__fixtures__/workflow_scheduled"
 import { TestDatabase } from "../utils"
-import { WorkflowsModuleService } from "../../src/services"
 
 jest.setTimeout(999900000)
 
@@ -167,6 +167,98 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
           expect(executionsList).toHaveLength(1)
         })
 
+        it("should return a list of failed workflow executions and keep it saved when there is a retentionTime set", async () => {
+          await workflowOrcModule.run("workflow_2", {
+            input: {
+              value: "123",
+            },
+            transactionId: "transaction_1",
+          })
+
+          let executionsList = await query({
+            workflow_executions: {
+              fields: ["id"],
+            },
+          })
+
+          expect(executionsList).toHaveLength(1)
+
+          await workflowOrcModule.setStepFailure({
+            idempotencyKey: {
+              action: TransactionHandlerType.INVOKE,
+              stepId: "new_step_name",
+              workflowId: "workflow_2",
+              transactionId: "transaction_1",
+            },
+            stepResponse: { uhuuuu: "yeaah!" },
+          })
+
+          executionsList = await query({
+            workflow_executions: {
+              fields: ["id", "state"],
+            },
+          })
+
+          expect(executionsList).toHaveLength(1)
+          expect(executionsList[0].state).toEqual("reverted")
+        })
+
+        it("should throw if setStepFailure fails", async () => {
+          const { acknowledgement } = await workflowOrcModule.run(
+            "workflow_2_revert_fail",
+            {
+              input: {
+                value: "123",
+              },
+            }
+          )
+
+          let done = false
+          void workflowOrcModule.subscribe({
+            workflowId: "workflow_2_revert_fail",
+            transactionId: acknowledgement.transactionId,
+            subscriber: (event) => {
+              if (event.eventType === "onFinish") {
+                done = true
+              }
+            },
+          })
+
+          let executionsList = await query({
+            workflow_executions: {
+              fields: ["id"],
+            },
+          })
+
+          expect(executionsList).toHaveLength(1)
+
+          const setStepError = await workflowOrcModule
+            .setStepFailure({
+              idempotencyKey: {
+                action: TransactionHandlerType.INVOKE,
+                stepId: "broken_step_2",
+                workflowId: "workflow_2_revert_fail",
+                transactionId: acknowledgement.transactionId,
+              },
+              stepResponse: { uhuuuu: "yeaah!" },
+            })
+            .catch((e) => {
+              return e
+            })
+
+          expect(setStepError).toEqual({ uhuuuu: "yeaah!" })
+
+          executionsList = await query({
+            workflow_executions: {
+              fields: ["id", "state", "context"],
+            },
+          })
+
+          expect(executionsList).toHaveLength(1)
+          expect(executionsList[0].state).toEqual("failed")
+          expect(done).toBe(true)
+        })
+
         it("should revert the entire transaction when a step timeout expires", async () => {
           const { transaction, result, errors } = await workflowOrcModule.run(
             "workflow_step_timeout",

packages/modules/workflow-engine-redis/src/services/workflow-orchestrator.ts

@@ -175,17 +175,21 @@ export class WorkflowOrchestratorService {
     options?: WorkflowOrchestratorRunOptions<T>,
     @MedusaContext() sharedContext: Context = {}
   ) {
-    let {
+    const {
       input,
-      context,
       transactionId,
       resultFrom,
-      throwOnError,
       logOnError,
       events: eventHandlers,
       container,
     } = options ?? {}
 
+    let { throwOnError, context } = options ?? {}
+
+    throwOnError ??= true
+    context ??= {}
+    context.transactionId ??= transactionId ?? ulid()
+
     const workflowId = isString(workflowIdOrWorkflow)
       ? workflowIdOrWorkflow
       : workflowIdOrWorkflow.getName()
@@ -194,9 +198,6 @@ export class WorkflowOrchestratorService {
       throw new Error("Workflow ID is required")
     }
 
-    context ??= {}
-    context.transactionId ??= transactionId ?? ulid()
-
     const events: FlowRunOptions["events"] = this.buildWorkflowEvents({
       customEventHandlers: eventHandlers,
       workflowId,
@@ -210,7 +211,7 @@ export class WorkflowOrchestratorService {
 
     const ret = await exportedWorkflow.run({
       input,
-      throwOnError,
+      throwOnError: false,
       logOnError,
       resultFrom,
       context,
@@ -248,6 +249,10 @@ export class WorkflowOrchestratorService {
       await this.triggerParentStep(ret.transaction, result)
     }
 
+    if (throwOnError && ret.thrownError) {
+      throw ret.thrownError
+    }
+
     return { acknowledgement, ...ret }
   }
 
@@ -299,13 +304,15 @@ export class WorkflowOrchestratorService {
   ) {
     const {
       context,
-      throwOnError,
       logOnError,
       resultFrom,
       container,
       events: eventHandlers,
     } = options ?? {}
 
+    let { throwOnError } = options ?? {}
+    throwOnError ??= true
+
     const [idempotencyKey_, { workflowId, transactionId }] =
       this.buildIdempotencyKeyAndParts(idempotencyKey)
 
@@ -324,7 +331,7 @@ export class WorkflowOrchestratorService {
       idempotencyKey: idempotencyKey_,
       context,
       resultFrom,
-      throwOnError,
+      throwOnError: false,
       logOnError,
       events,
       response: stepResponse,
@@ -345,6 +352,10 @@ export class WorkflowOrchestratorService {
       await this.triggerParentStep(ret.transaction, result)
     }
 
+    if (throwOnError && ret.thrownError) {
+      throw ret.thrownError
+    }
+
     return ret
   }
 
@@ -363,13 +374,15 @@ export class WorkflowOrchestratorService {
   ) {
     const {
       context,
-      throwOnError,
       logOnError,
       resultFrom,
       container,
       events: eventHandlers,
     } = options ?? {}
 
+    let { throwOnError } = options ?? {}
+    throwOnError ??= true
+
     const [idempotencyKey_, { workflowId, transactionId }] =
       this.buildIdempotencyKeyAndParts(idempotencyKey)
 
@@ -388,7 +401,7 @@ export class WorkflowOrchestratorService {
       idempotencyKey: idempotencyKey_,
       context,
       resultFrom,
-      throwOnError,
+      throwOnError: false,
       logOnError,
       events,
       response: stepResponse,
@@ -409,6 +422,10 @@ export class WorkflowOrchestratorService {
       await this.triggerParentStep(ret.transaction, result)
     }
 
+    if (throwOnError && ret.thrownError) {
+      throw ret.thrownError
+    }
+
     return ret
   }