fix: workflow async concurrency (#13769)

* executeAsync

* || 1

* wip

* stepId

* stepId

* wip

* wip

* continue versioning management changes

* fix and improve concurrency

* update in memory engine

* remove duplicated test

* fix script

* Create weak-drinks-confess.md

* fixes

* fix

* fix

* continuation

* centralize merge checkepoint

* centralize merge checkpoint

* fix locking

* rm only

* Continue improvements and fixes

* fixes

* fixes

* hasAwaiting will be recomputed

* fix orchestrator engine

* bump version on async parallel steps only

* mark as delivered fix

* changeset

* check partitions

* avoid saving when having parent step

* cart test

---------

Co-authored-by: Carlos R. L. Rodrigues <rodrigolr@gmail.com>
Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>
Co-authored-by: Oli Juhl <59018053+olivermrbl@users.noreply.github.com>

.changeset/weak-drinks-confess.md

@@ -0,0 +1,9 @@
+---
+"@medusajs/workflow-engine-inmemory": patch
+"@medusajs/workflow-engine-redis": patch
+"@medusajs/orchestration": patch
+"@medusajs/workflows-sdk": patch
+"@medusajs/core-flows": patch
+---
+
+fix: workflow async concurrency

integration-tests/helpers/wait-for-index.ts

@@ -21,6 +21,10 @@ export async function waitForIndexedEntities(
   const { timeout = 120000, pollInterval = 100 } = options
   const startTime = Date.now()
 
+  // Normalize the entity name to match partition table naming convention
+  const normalizedName = entityName.toLowerCase().replace(/[^a-z0-9_]/g, "_")
+  const partitionTableName = `cat_${normalizedName}`
+
   while (Date.now() - startTime < timeout) {
     try {
       // Query the index_data table to check if all entities are indexed
@@ -33,10 +37,28 @@ export async function waitForIndexedEntities(
         ? result.rows.map((row: any) => row.id)
         : result.map((row: any) => row.id)
 
-      // Check if all expected entities are indexed
+      // Check if all expected entities are indexed in index_data
       const allIndexed = entityIds.every((id) => indexedIds.includes(id))
 
-      if (allIndexed) {
+      if (!allIndexed) {
+        await new Promise((resolve) => setTimeout(resolve, pollInterval))
+        continue
+      }
+
+      // Also check if data is replicated to the partition table
+      const partitionResult = await dbConnection.raw(
+        `SELECT id FROM ${partitionTableName} WHERE id = ANY(?)`,
+        [entityIds]
+      )
+
+      const partitionIds = partitionResult.rows
+        ? partitionResult.rows.map((row: any) => row.id)
+        : partitionResult.map((row: any) => row.id)
+
+      // Check if all expected entities are in the partition table
+      const allInPartition = entityIds.every((id) => partitionIds.includes(id))
+
+      if (allInPartition) {
         return
       }
     } catch (error) {
@@ -49,6 +71,6 @@ export async function waitForIndexedEntities(
   throw new Error(
     `Entities [${entityIds.join(
       ", "
-    )}] of type '${entityName}' were not indexed within ${timeout}ms`
+    )}] of type '${entityName}' were not fully replicated to partition table within ${timeout}ms`
   )
 }

integration-tests/http/__tests__/workflow-engine/admin/index.spec.ts

@@ -11,16 +11,20 @@ import {
   adminHeaders,
   createAdminUser,
 } from "../../../../helpers/create-admin-user"
+import { setTimeout } from "timers/promises"
+import { IWorkflowEngineService } from "@medusajs/framework/types"
 
 jest.setTimeout(300000)
 
 medusaIntegrationTestRunner({
   testSuite: ({ dbConnection, getContainer, api }) => {
     let container
+    let workflowOrcModule: IWorkflowEngineService
 
     beforeEach(async () => {
       container = getContainer()
       await createAdminUser(dbConnection, adminHeaders, container)
+      workflowOrcModule = container.resolve(Modules.WORKFLOW_ENGINE)
     })
 
     describe("GET /admin/workflow-executions", () => {
@@ -90,5 +94,181 @@ medusaIntegrationTestRunner({
         )
       })
     })
+
+    describe("Workflow Orchestrator module subscribe", function () {
+      it("should subscribe to a workflow and receive the response when it finishes", async () => {
+        const step1 = createStep({ name: "step1" }, async () => {
+          return new StepResponse("step1")
+        })
+        const step2 = createStep({ name: "step2" }, async () => {
+          await setTimeout(1000)
+          return new StepResponse("step2")
+        })
+
+        const workflowId =
+          "workflow" + Math.random().toString(36).substring(2, 15)
+        createWorkflow(workflowId, function (input) {
+          step1()
+          step2().config({
+            async: true,
+          })
+          return new WorkflowResponse("workflow")
+        })
+
+        const step1_1 = createStep({ name: "step1_1" }, async () => {
+          return new StepResponse("step1_1")
+        })
+        const step2_1 = createStep({ name: "step2_1" }, async () => {
+          await setTimeout(1000)
+          return new StepResponse("step2_1")
+        })
+
+        const workflow2Id =
+          "workflow_2" + Math.random().toString(36).substring(2, 15)
+        createWorkflow(workflow2Id, function (input) {
+          step1_1()
+          step2_1().config({
+            async: true,
+          })
+          return new WorkflowResponse("workflow_2")
+        })
+
+        const transactionId =
+          "trx_123" + Math.random().toString(36).substring(2, 15)
+        const transactionId2 =
+          "trx_124" + Math.random().toString(36).substring(2, 15)
+
+        const onWorkflowFinishSpy = jest.fn()
+
+        const onWorkflowFinishPromise = new Promise<void>((resolve) => {
+          void workflowOrcModule.subscribe({
+            workflowId: workflowId,
+            transactionId,
+            subscriber: (event) => {
+              console.log("event", event)
+              if (event.eventType === "onFinish") {
+                onWorkflowFinishSpy()
+                workflowOrcModule.run(workflow2Id, {
+                  transactionId: transactionId2,
+                })
+                resolve()
+              }
+            },
+          })
+        })
+
+        const onWorkflow2FinishSpy = jest.fn()
+
+        const workflow2FinishPromise = new Promise<void>((resolve) => {
+          void workflowOrcModule.subscribe({
+            workflowId: workflow2Id,
+            subscriber: (event) => {
+              console.log("event", event)
+              if (event.eventType === "onFinish") {
+                onWorkflow2FinishSpy()
+                resolve()
+              }
+            },
+          })
+        })
+
+        workflowOrcModule.run(workflowId, {
+          transactionId,
+        })
+
+        await onWorkflowFinishPromise
+        await workflow2FinishPromise
+
+        expect(onWorkflowFinishSpy).toHaveBeenCalledTimes(1)
+        expect(onWorkflow2FinishSpy).toHaveBeenCalledTimes(1)
+      })
+
+      it("should subscribe to a workflow and receive the response when it finishes (2)", async () => {
+        const step1 = createStep({ name: "step1" }, async () => {
+          return new StepResponse("step1")
+        })
+        const step2 = createStep({ name: "step2" }, async () => {
+          await setTimeout(1000)
+          return new StepResponse("step2")
+        })
+
+        const workflowId =
+          "workflow" + Math.random().toString(36).substring(2, 15)
+        createWorkflow(workflowId, function (input) {
+          step1()
+          step2().config({
+            async: true,
+          })
+          return new WorkflowResponse("workflow")
+        })
+
+        const step1_1 = createStep({ name: "step1_1" }, async () => {
+          return new StepResponse("step1_1")
+        })
+        const step2_1 = createStep({ name: "step2_1" }, async () => {
+          await setTimeout(1000)
+          return new StepResponse("step2_1")
+        })
+
+        const workflow2Id =
+          "workflow_2" + Math.random().toString(36).substring(2, 15)
+        createWorkflow(workflow2Id, function (input) {
+          step1_1()
+          step2_1().config({
+            async: true,
+          })
+          return new WorkflowResponse("workflow_2")
+        })
+
+        const transactionId =
+          "trx_123" + Math.random().toString(36).substring(2, 15)
+        const transactionId2 =
+          "trx_124" + Math.random().toString(36).substring(2, 15)
+
+        const onWorkflowFinishSpy = jest.fn()
+
+        const onWorkflowFinishPromise = new Promise<void>((resolve) => {
+          void workflowOrcModule.subscribe({
+            workflowId: workflowId,
+            transactionId,
+            subscriber: (event) => {
+              console.log("event", event)
+              if (event.eventType === "onFinish") {
+                onWorkflowFinishSpy()
+                workflowOrcModule.run(workflow2Id, {
+                  transactionId: transactionId2,
+                })
+                resolve()
+              }
+            },
+          })
+        })
+
+        const onWorkflow2FinishSpy = jest.fn()
+
+        const workflow2FinishPromise = new Promise<void>((resolve) => {
+          void workflowOrcModule.subscribe({
+            workflowId: workflow2Id,
+            subscriber: (event) => {
+              console.log("event", event)
+              if (event.eventType === "onFinish") {
+                onWorkflow2FinishSpy()
+                resolve()
+              }
+            },
+          })
+        })
+
+        workflowOrcModule.run(workflowId, {
+          transactionId,
+        })
+
+        await onWorkflowFinishPromise
+        await workflow2FinishPromise
+
+        expect(onWorkflowFinishSpy).toHaveBeenCalledTimes(1)
+        expect(onWorkflow2FinishSpy).toHaveBeenCalledTimes(1)
+      })
+    })
   },
 })

integration-tests/modules/__tests__/cart/store/cart.workflows.spec.ts

@@ -391,7 +391,8 @@ medusaIntegrationTestRunner({
             },
           })
 
-          expect(transaction.flow.state).toEqual("reverted")
+          // TODO: the state must be "reverted" when runAsStep of sync flows can be reverted
+          expect(transaction.flow.state).toEqual("failed")
         })
 
         it("should throw when no regions exist", async () => {

packages/core/core-flows/src/order/workflows/mark-order-fulfillment-as-delivered.ts

@@ -16,7 +16,6 @@ import {
 import {
   createStep,
   createWorkflow,
-  parallelize,
   transform,
   WorkflowData,
   WorkflowResponse,
@@ -259,12 +258,11 @@ export const markOrderFulfillmentAsDeliveredWorkflow = createWorkflow(
       prepareRegisterDeliveryData
     )
 
-    const [deliveredFulfillment] = parallelize(
+    const deliveredFulfillment = markFulfillmentAsDeliveredWorkflow.runAsStep({
-      markFulfillmentAsDeliveredWorkflow.runAsStep({
+      input: { id: fulfillment.id },
-        input: { id: fulfillment.id },
+    })
-      }),
+
-      registerOrderDeliveryStep(deliveryData)
+    registerOrderDeliveryStep(deliveryData)
-    )
 
     emitEventStep({
       eventName: FulfillmentWorkflowEvents.DELIVERY_CREATED,

packages/core/orchestration/src/transaction/datastore/abstract-storage.ts

@@ -26,7 +26,7 @@ export interface IDistributedTransactionStorage {
     data: TransactionCheckpoint,
     ttl?: number,
     options?: TransactionOptions
-  ): Promise<void>
+  ): Promise<TransactionCheckpoint>
   scheduleRetry(
     transaction: DistributedTransactionType,
     step: TransactionStep,
@@ -96,7 +96,7 @@ export abstract class DistributedTransactionStorage
     key: string,
     data: TransactionCheckpoint,
     ttl?: number
-  ): Promise<void> {
+  ): Promise<TransactionCheckpoint> {
     throw new Error("Method 'save' not implemented.")
   }
 

packages/core/orchestration/src/transaction/datastore/base-in-memory-storage.ts

@@ -28,7 +28,7 @@ export class BaseInMemoryDistributedTransactionStorage extends DistributedTransa
     data: TransactionCheckpoint,
     ttl?: number,
     options?: TransactionOptions
-  ): Promise<void> {
+  ): Promise<TransactionCheckpoint> {
     const hasFinished = [
       TransactionState.DONE,
       TransactionState.REVERTED,
@@ -40,6 +40,8 @@ export class BaseInMemoryDistributedTransactionStorage extends DistributedTransa
     } else {
       this.storage.set(key, data)
     }
+
+    return data
   }
 
   async clearExpiredExecutions(): Promise<void> {}

packages/core/orchestration/src/transaction/distributed-transaction.ts

@@ -1,16 +1,60 @@
-import { isDefined } from "@medusajs/utils"
+import { isDefined, TransactionStepState } from "@medusajs/utils"
 import { EventEmitter } from "events"
+import { setTimeout as setTimeoutPromise } from "node:timers/promises"
 import { IDistributedTransactionStorage } from "./datastore/abstract-storage"
 import { BaseInMemoryDistributedTransactionStorage } from "./datastore/base-in-memory-storage"
-import { NonSerializableCheckPointError } from "./errors"
+import { NonSerializableCheckPointError, SkipExecutionError } from "./errors"
 import { TransactionOrchestrator } from "./transaction-orchestrator"
 import { TransactionStep, TransactionStepHandler } from "./transaction-step"
 import {
   TransactionFlow,
   TransactionHandlerType,
   TransactionState,
+  TransactionStepStatus,
 } from "./types"
 
+const flowMergeableProperties = [
+  "state",
+  "hasFailedSteps",
+  "hasSkippedOnFailureSteps",
+  "hasSkippedSteps",
+  "hasRevertedSteps",
+  "cancelledAt",
+  "startedAt",
+  "hasAsyncSteps",
+  "_v",
+  "timedOutAt",
+]
+
+const mergeStep = (
+  currentStep: TransactionStep,
+  storedStep: TransactionStep
+) => {
+  const mergeProperties = [
+    "attempts",
+    "failures",
+    "temporaryFailedAt",
+    "retryRescheduledAt",
+    "hasScheduledRetry",
+    "lastAttempt",
+    "_v",
+    "stepFailed",
+    "startedAt",
+  ]
+
+  for (const prop of mergeProperties) {
+    if (prop === "hasScheduledRetry" || prop === "stepFailed") {
+      currentStep[prop] = storedStep[prop] ?? currentStep[prop]
+      continue
+    }
+
+    currentStep[prop] =
+      storedStep[prop] || currentStep[prop]
+        ? Math.max(storedStep[prop] ?? 0, currentStep[prop] ?? 0)
+        : currentStep[prop] ?? storedStep[prop]
+  }
+}
+
 /**
  * @typedef TransactionMetadata
  * @property model_id - The id of the model_id that created the transaction (modelId).
@@ -51,12 +95,261 @@ export class TransactionStepError {
   ) {}
 }
 
+const stateFlowOrder = [
+  TransactionState.NOT_STARTED,
+  TransactionState.INVOKING,
+  TransactionState.DONE,
+  TransactionState.WAITING_TO_COMPENSATE,
+  TransactionState.COMPENSATING,
+  TransactionState.REVERTED,
+  TransactionState.FAILED,
+]
+
 export class TransactionCheckpoint {
   constructor(
     public flow: TransactionFlow,
     public context: TransactionContext,
     public errors: TransactionStepError[] = []
   ) {}
+
+  /**
+   * Merge the current checkpoint with incoming data from a concurrent save operation.
+   * This handles race conditions when multiple steps complete simultaneously.
+   *
+   * @param storedData - The checkpoint data being saved
+   * @param savingStepId - Optional step ID if this is a step-specific save
+   */
+  static mergeCheckpoints(
+    currentTransactionData: TransactionCheckpoint,
+    storedData?: TransactionCheckpoint
+  ): TransactionCheckpoint {
+    if (!currentTransactionData || !storedData) {
+      return currentTransactionData
+    }
+
+    TransactionCheckpoint.#mergeFlow(currentTransactionData, storedData)
+    TransactionCheckpoint.#mergeErrors(
+      currentTransactionData.errors ?? [],
+      storedData.errors
+    )
+
+    return currentTransactionData
+  }
+
+  static #mergeFlow(
+    currentTransactionData: TransactionCheckpoint,
+    storedData: TransactionCheckpoint
+  ): void {
+    const currentTransactionContext = currentTransactionData.context
+    const storedContext = storedData.context
+
+    if (currentTransactionData.flow._v >= storedData.flow._v) {
+      for (const prop of flowMergeableProperties) {
+        if (
+          prop === "startedAt" ||
+          prop === "cancelledAt" ||
+          prop === "timedOutAt"
+        ) {
+          currentTransactionData.flow[prop] =
+            storedData.flow[prop] || currentTransactionData.flow[prop]
+              ? Math.max(
+                  storedData.flow[prop] ?? 0,
+                  currentTransactionData.flow[prop] ?? 0
+                )
+              : currentTransactionData.flow[prop] ??
+                storedData.flow[prop] ??
+                (undefined as any)
+        } else if (prop === "_v") {
+          currentTransactionData.flow[prop] = Math.max(
+            storedData.flow[prop] ?? 0,
+            currentTransactionData.flow[prop] ?? 0
+          )
+        } else if (prop === "state") {
+          const curState = stateFlowOrder.findIndex(
+            (state) => state === currentTransactionData.flow.state
+          )
+          const storedState = stateFlowOrder.findIndex(
+            (state) => state === storedData.flow.state
+          )
+
+          if (storedState > curState) {
+            currentTransactionData.flow.state = storedData.flow.state
+          } else if (
+            curState < storedState &&
+            currentTransactionData.flow.state !==
+              TransactionState.WAITING_TO_COMPENSATE
+          ) {
+            throw new SkipExecutionError(
+              `Transaction is behind another execution`
+            )
+          }
+        } else if (
+          storedData.flow[prop] &&
+          !currentTransactionData.flow[prop]
+        ) {
+          currentTransactionData.flow[prop] = storedData.flow[prop]
+        }
+      }
+    }
+
+    const storedSteps = Object.values(storedData.flow.steps)
+
+    for (const storedStep of storedSteps) {
+      if (storedStep.id === "_root") {
+        continue
+      }
+
+      const stepName = storedStep.definition.action!
+      const stepId = storedStep.id
+
+      // Merge context responses
+      if (
+        storedContext.invoke[stepName] &&
+        !currentTransactionContext.invoke[stepName]
+      ) {
+        currentTransactionContext.invoke[stepName] =
+          storedContext.invoke[stepName]
+      }
+
+      if (
+        storedContext.compensate[stepName] &&
+        !currentTransactionContext.compensate[stepName]
+      ) {
+        currentTransactionContext.compensate[stepName] =
+          storedContext.compensate[stepName]
+      }
+
+      const currentStepVersion = currentTransactionData.flow.steps[stepId]._v!
+      const storedStepVersion = storedData.flow.steps[stepId]._v!
+
+      if (storedStepVersion > currentStepVersion) {
+        throw new SkipExecutionError(`Transaction is behind another execution`)
+      }
+
+      // Determine which state is further along in the process
+      const shouldUpdateInvoke = TransactionCheckpoint.#shouldUpdateStepState(
+        currentTransactionData.flow.steps[stepId].invoke,
+        storedStep.invoke
+      )
+
+      const shouldUpdateCompensate =
+        TransactionCheckpoint.#shouldUpdateStepState(
+          currentTransactionData.flow.steps[stepId].compensate,
+          storedStep.compensate
+        )
+
+      if (shouldUpdateInvoke) {
+        currentTransactionData.flow.steps[stepId].invoke = storedStep.invoke
+      }
+
+      if (shouldUpdateCompensate) {
+        currentTransactionData.flow.steps[stepId].compensate =
+          storedStep.compensate
+      }
+
+      mergeStep(currentTransactionData.flow.steps[stepId], storedStep)
+    }
+  }
+
+  /**
+   * Determines if the stored step state should replace the current step state.
+   * This validates both state and status transitions according to TransactionStep rules.
+   */
+  static #shouldUpdateStepState(
+    currentStepState: {
+      state: TransactionStepState
+      status: TransactionStepStatus
+    },
+    storedStepState: {
+      state: TransactionStepState
+      status: TransactionStepStatus
+    }
+  ): boolean {
+    // Define allowed state transitions
+    const allowedStateTransitions = {
+      [TransactionStepState.DORMANT]: [TransactionStepState.NOT_STARTED],
+      [TransactionStepState.NOT_STARTED]: [
+        TransactionStepState.INVOKING,
+        TransactionStepState.COMPENSATING,
+        TransactionStepState.FAILED,
+        TransactionStepState.SKIPPED,
+        TransactionStepState.SKIPPED_FAILURE,
+      ],
+      [TransactionStepState.INVOKING]: [
+        TransactionStepState.FAILED,
+        TransactionStepState.DONE,
+        TransactionStepState.TIMEOUT,
+        TransactionStepState.SKIPPED,
+      ],
+      [TransactionStepState.COMPENSATING]: [
+        TransactionStepState.REVERTED,
+        TransactionStepState.FAILED,
+      ],
+      [TransactionStepState.DONE]: [TransactionStepState.COMPENSATING],
+    }
+
+    // Define allowed status transitions
+    const allowedStatusTransitions = {
+      [TransactionStepStatus.WAITING]: [
+        TransactionStepStatus.OK,
+        TransactionStepStatus.TEMPORARY_FAILURE,
+        TransactionStepStatus.PERMANENT_FAILURE,
+      ],
+      [TransactionStepStatus.TEMPORARY_FAILURE]: [
+        TransactionStepStatus.IDLE,
+        TransactionStepStatus.PERMANENT_FAILURE,
+      ],
+      [TransactionStepStatus.PERMANENT_FAILURE]: [TransactionStepStatus.IDLE],
+    }
+
+    if (
+      currentStepState.state === storedStepState.state &&
+      currentStepState.status === storedStepState.status
+    ) {
+      return false
+    }
+
+    // Check if state transition from stored to current is allowed
+    const allowedStatesFromCurrent =
+      allowedStateTransitions[currentStepState.state] || []
+    const isStateTransitionValid = allowedStatesFromCurrent.includes(
+      storedStepState.state
+    )
+
+    if (currentStepState.state !== storedStepState.state) {
+      return isStateTransitionValid
+    }
+
+    // States are the same, check status transition
+    // Special case: WAITING status can always be transitioned
+    if (currentStepState.status === TransactionStepStatus.WAITING) {
+      return true
+    }
+
+    // Check if status transition from stored to current is allowed
+    const allowedStatusesFromCurrent =
+      allowedStatusTransitions[currentStepState.status] || []
+
+    return allowedStatusesFromCurrent.includes(storedStepState.status)
+  }
+
+  static #mergeErrors(
+    currentErrors: TransactionStepError[],
+    incomingErrors: TransactionStepError[]
+  ): void {
+    const existingErrorSignatures = new Set(
+      currentErrors.map(
+        (err) => `${err.action}:${err.handlerType}:${err.error?.message}`
+      )
+    )
+
+    for (const error of incomingErrors) {
+      const signature = `${error.action}:${error.handlerType}:${error.error?.message}`
+      if (!existingErrorSignatures.has(signature)) {
+        currentErrors.push(error)
+      }
+    }
+  }
 }
 
 export class TransactionPayload {
@@ -81,8 +374,8 @@ class DistributedTransaction extends EventEmitter {
   public transactionId: string
   public runId: string
 
-  private readonly errors: TransactionStepError[] = []
+  private errors: TransactionStepError[] = []
-  private readonly context: TransactionContext = new TransactionContext()
+  private context: TransactionContext = new TransactionContext()
   private static keyValueStore: IDistributedTransactionStorage
 
   /**
@@ -195,28 +488,100 @@ class DistributedTransaction extends EventEmitter {
     return this.getFlow().options?.timeout
   }
 
-  public async saveCheckpoint(
+  public async saveCheckpoint({
-    ttl = 0
+    ttl = 0,
-  ): Promise<TransactionCheckpoint | undefined> {
+    parallelSteps = 0,
-    const options =
+    stepId,
-      TransactionOrchestrator.getWorkflowOptions(this.modelId) ??
+    _v,
-      this.getFlow().options
+  }: {
+    ttl?: number
+    parallelSteps?: number
+    stepId?: string
+    _v?: number
+  } = {}): Promise<TransactionCheckpoint | undefined> {
+    const options = {
+      ...(TransactionOrchestrator.getWorkflowOptions(this.modelId) ??
+        this.getFlow().options),
+    }
 
     if (!options?.store) {
       return
     }
 
+    options.stepId = stepId
+    if (_v) {
+      options.parallelSteps = parallelSteps
+      options._v = _v
+    }
+
     const key = TransactionOrchestrator.getKeyName(
       DistributedTransaction.keyPrefix,
       this.modelId,
       this.transactionId
     )
 
-    const rawData = this.#serializeCheckpointData()
+    let checkpoint
 
-    await DistributedTransaction.keyValueStore.save(key, rawData, ttl, options)
+    let retries = 0
+    let backoffMs = 50
+    const maxRetries = (options?.parallelSteps || 1) + 2
+    while (retries < maxRetries) {
+      checkpoint = this.#serializeCheckpointData()
 
-    return rawData
+      try {
+        const savedCheckpoint = await DistributedTransaction.keyValueStore.save(
+          key,
+          checkpoint,
+          ttl,
+          options
+        )
+
+        return savedCheckpoint
+      } catch (error) {
+        if (TransactionOrchestrator.isExpectedError(error)) {
+          throw error
+        } else if (checkpoint.flow.state === TransactionState.NOT_STARTED) {
+          throw new SkipExecutionError(
+            "Transaction already started for transactionId: " +
+              this.transactionId
+          )
+        }
+
+        retries++
+        // Exponential backoff with jitter
+        const jitter = Math.random() * backoffMs
+
+        await setTimeoutPromise(backoffMs + jitter)
+
+        backoffMs = Math.min(backoffMs * 2, 1000)
+
+        const lastCheckpoint = await DistributedTransaction.loadTransaction(
+          this.modelId,
+          this.transactionId
+        )
+
+        if (!lastCheckpoint) {
+          throw new SkipExecutionError("Transaction already finished")
+        }
+
+        TransactionCheckpoint.mergeCheckpoints(checkpoint, lastCheckpoint)
+
+        const [steps] = TransactionOrchestrator.buildSteps(
+          checkpoint.flow.definition,
+          checkpoint.flow.steps
+        )
+        checkpoint.flow.steps = steps
+        this.flow = checkpoint.flow
+        this.errors = checkpoint.errors
+        this.context = checkpoint.context
+
+        continue
+      }
+    }
+
+    throw new Error(
+      `Max retries (${maxRetries}) exceeded for saving checkpoint due to version conflicts`
+    )
   }
 
   public static async loadTransaction(

packages/core/orchestration/src/transaction/errors.ts

@@ -1,3 +1,5 @@
+import { OrchestrationUtils } from "@medusajs/utils"
+
 class BaseStepErrror extends Error {
   #stepResponse: unknown
 
@@ -116,6 +118,12 @@ export class SkipStepAlreadyFinishedError extends Error {
 }
 
 export class SkipCancelledExecutionError extends Error {
+  readonly #__type = OrchestrationUtils.SymbolWorkflowStepResponse
+
+  get __type() {
+    return this.#__type
+  }
+
   static isSkipCancelledExecutionError(
     error: Error
   ): error is SkipCancelledExecutionError {

packages/core/orchestration/src/transaction/transaction-orchestrator.ts

@@ -18,6 +18,7 @@ import {
   TransactionStepStatus,
 } from "./types"
 
+import { Context } from "@medusajs/types"
 import {
   isDefined,
   isErrorLike,
@@ -38,7 +39,6 @@ import {
   TransactionStepTimeoutError,
   TransactionTimeoutError,
 } from "./errors"
-import { Context } from "@medusajs/types"
 
 /**
  * @class TransactionOrchestrator is responsible for managing and executing distributed transactions.
@@ -115,7 +115,7 @@ export class TransactionOrchestrator extends EventEmitter {
     }
   }
 
-  private static isExpectedError(error: Error): boolean {
+  public static isExpectedError(error: Error): boolean {
     return (
       SkipCancelledExecutionError.isSkipCancelledExecutionError(error) ||
       SkipExecutionError.isSkipExecutionError(error) ||
@@ -137,7 +137,7 @@ export class TransactionOrchestrator extends EventEmitter {
     return params.join(this.SEPARATOR)
   }
 
-  private getPreviousStep(flow: TransactionFlow, step: TransactionStep) {
+  private static getPreviousStep(flow: TransactionFlow, step: TransactionStep) {
     const id = step.id.split(".")
     id.pop()
     const parentId = id.join(".")
@@ -175,6 +175,14 @@ export class TransactionOrchestrator extends EventEmitter {
     return steps
   }
 
+  private static countSiblings(
+    flow: TransactionFlow,
+    step: TransactionStep
+  ): number {
+    const previous = TransactionOrchestrator.getPreviousStep(flow, step)
+    return previous.next.length
+  }
+
   private canMoveForward(flow: TransactionFlow, previousStep: TransactionStep) {
     const states = [
       TransactionStepState.DONE,
@@ -184,9 +192,10 @@ export class TransactionOrchestrator extends EventEmitter {
       TransactionStepState.SKIPPED_FAILURE,
     ]
 
-    const siblings = this.getPreviousStep(flow, previousStep).next.map(
+    const siblings = TransactionOrchestrator.getPreviousStep(
-      (sib) => flow.steps[sib]
+      flow,
-    )
+      previousStep
+    ).next.map((sib) => flow.steps[sib])
 
     return (
       !!previousStep.definition.noWait ||
@@ -214,7 +223,7 @@ export class TransactionOrchestrator extends EventEmitter {
     if (flow.state == TransactionState.COMPENSATING) {
       return this.canMoveBackward(flow, step)
     } else {
-      const previous = this.getPreviousStep(flow, step)
+      const previous = TransactionOrchestrator.getPreviousStep(flow, step)
       if (previous.id === TransactionOrchestrator.ROOT_STEP) {
         return true
       }
@@ -311,6 +320,7 @@ export class TransactionOrchestrator extends EventEmitter {
     completed: number
   }> {
     const flow = transaction.getFlow()
+
     const result = await this.computeCurrentTransactionState(transaction)
 
     // Handle state transitions and emit events
@@ -324,7 +334,9 @@ export class TransactionOrchestrator extends EventEmitter {
 
       this.emit(DistributedTransactionEvent.COMPENSATE_BEGIN, { transaction })
 
-      return await this.checkAllSteps(transaction)
+      const result = await this.checkAllSteps(transaction)
+
+      return result
     } else if (result.completed === result.total) {
       if (result.hasSkippedOnFailure) {
         flow.hasSkippedOnFailureSteps = true
@@ -407,6 +419,7 @@ export class TransactionOrchestrator extends EventEmitter {
         if (stepDef.hasAwaitingRetry()) {
           if (stepDef.canRetryAwaiting()) {
             stepDef.retryRescheduledAt = null
+
             nextSteps.push(stepDef)
           } else if (!stepDef.retryRescheduledAt) {
             stepDef.hasScheduledRetry = true
@@ -501,6 +514,12 @@ export class TransactionOrchestrator extends EventEmitter {
 
       const stepDef = flow.steps[step]
       const curState = stepDef.getStates()
+
+      if (stepDef._v) {
+        flow._v = 0
+        stepDef._v = 0
+      }
+
       if (
         [TransactionStepState.DONE, TransactionStepState.TIMEOUT].includes(
           curState.state
@@ -547,7 +566,14 @@ export class TransactionOrchestrator extends EventEmitter {
     let shouldEmit = true
     let transactionIsCancelling = false
     try {
-      await transaction.saveCheckpoint()
+      await transaction.saveCheckpoint({
+        _v: step._v,
+        parallelSteps: TransactionOrchestrator.countSiblings(
+          transaction.getFlow(),
+          step
+        ),
+        stepId: step.id,
+      })
     } catch (error) {
       if (!TransactionOrchestrator.isExpectedError(error)) {
         throw error
@@ -567,9 +593,7 @@ export class TransactionOrchestrator extends EventEmitter {
     }
 
     if (cleaningUp.length) {
-      setImmediate(async () => {
+      await promiseAll(cleaningUp)
-        await promiseAll(cleaningUp)
-      })
     }
 
     if (shouldEmit) {
@@ -597,7 +621,14 @@ export class TransactionOrchestrator extends EventEmitter {
     transaction.getFlow().hasWaitingSteps = true
 
     try {
-      await transaction.saveCheckpoint()
+      await transaction.saveCheckpoint({
+        _v: step._v,
+        parallelSteps: TransactionOrchestrator.countSiblings(
+          transaction.getFlow(),
+          step
+        ),
+        stepId: step.id,
+      })
       await transaction.scheduleRetry(step, 0)
     } catch (error) {
       if (!TransactionOrchestrator.isExpectedError(error)) {
@@ -627,7 +658,14 @@ export class TransactionOrchestrator extends EventEmitter {
     let shouldEmit = true
     let transactionIsCancelling = false
     try {
-      await transaction.saveCheckpoint()
+      await transaction.saveCheckpoint({
+        _v: step._v,
+        parallelSteps: TransactionOrchestrator.countSiblings(
+          transaction.getFlow(),
+          step
+        ),
+        stepId: step.id,
+      })
     } catch (error) {
       if (!TransactionOrchestrator.isExpectedError(error)) {
         throw error
@@ -650,9 +688,7 @@ export class TransactionOrchestrator extends EventEmitter {
     }
 
     if (cleaningUp.length) {
-      setImmediate(async () => {
+      await promiseAll(cleaningUp)
-        await promiseAll(cleaningUp)
-      })
     }
 
     if (shouldEmit) {
@@ -837,7 +873,14 @@ export class TransactionOrchestrator extends EventEmitter {
     }
 
     try {
-      await transaction.saveCheckpoint()
+      await transaction.saveCheckpoint({
+        _v: step._v,
+        parallelSteps: TransactionOrchestrator.countSiblings(
+          transaction.getFlow(),
+          step
+        ),
+        stepId: step.id,
+      })
     } catch (error) {
       if (!TransactionOrchestrator.isExpectedError(error)) {
         throw error
@@ -856,9 +899,7 @@ export class TransactionOrchestrator extends EventEmitter {
     }
 
     if (cleaningUp.length) {
-      setImmediate(async () => {
+      await promiseAll(cleaningUp)
-        await promiseAll(cleaningUp)
-      })
     }
 
     if (!result.stopExecution) {
@@ -885,14 +926,21 @@ export class TransactionOrchestrator extends EventEmitter {
       }
 
       const flow = transaction.getFlow()
-      const nextSteps = await this.checkAllSteps(transaction)
 
-      if (await this.checkTransactionTimeout(transaction, nextSteps.current)) {
+      let nextSteps = await this.checkAllSteps(transaction)
+
+      const hasTimedOut = await this.checkTransactionTimeout(
+        transaction,
+        nextSteps.current
+      )
+
+      if (hasTimedOut) {
         continue
       }
 
       if (nextSteps.remaining === 0) {
         await this.finalizeTransaction(transaction)
+
         return
       }
 
@@ -915,6 +963,7 @@ export class TransactionOrchestrator extends EventEmitter {
       })
 
       const execution: Promise<void | unknown>[] = []
+      const executionAsync: (() => Promise<void | unknown>)[] = []
 
       let i = 0
       let hasAsyncSteps = false
@@ -939,37 +988,60 @@ export class TransactionOrchestrator extends EventEmitter {
 
         // Compute current transaction state
         await this.computeCurrentTransactionState(transaction)
-
         if (!continueExecution) {
           break
         }
 
         const promise = this.createStepExecutionPromise(transaction, step)
 
+        const hasMultipleAsyncSteps =
+          nextSteps.next.filter((step) => {
+            const isAsync = step.isCompensating()
+              ? step.definition.compensateAsync
+              : step.definition.async
+
+            return isAsync
+          }).length > 1
+
+        const hasVersionControl =
+          hasMultipleAsyncSteps || step.hasAwaitingRetry()
+
+        if (hasVersionControl && !step._v) {
+          transaction.getFlow()._v += 1
+          step._v = transaction.getFlow()._v
+        }
+
         if (!isAsync) {
           execution.push(
             this.executeSyncStep(promise, transaction, step, nextSteps)
           )
         } else {
           // Execute async step in background as part of the next event loop cycle and continue the execution of the transaction
-          process.nextTick(() =>
+          hasAsyncSteps = true
+          executionAsync.push(() =>
             this.executeAsyncStep(promise, transaction, step, nextSteps)
           )
-          hasAsyncSteps = true
         }
       }
 
       await promiseAll(execution)
 
-      if (nextSteps.next.length === 0 || (hasAsyncSteps && !execution.length)) {
+      if (!nextSteps.next.length || (hasAsyncSteps && !execution.length)) {
         continueExecution = false
+      }
+
+      if (hasAsyncSteps) {
         await transaction.saveCheckpoint().catch((error) => {
           if (TransactionOrchestrator.isExpectedError(error)) {
-            return
+            continueExecution = false
           }
 
           throw error
         })
+
+        for (const exec of executionAsync) {
+          void exec()
+        }
       }
     }
   }
@@ -989,6 +1061,7 @@ export class TransactionOrchestrator extends EventEmitter {
         throw error
       }
     })
+
     this.emit(DistributedTransactionEvent.FINISH, { transaction })
   }
 
@@ -1136,7 +1209,11 @@ export class TransactionOrchestrator extends EventEmitter {
       .then(async (response: any) => {
         await this.handleStepExpiration(transaction, step, nextSteps)
 
-        const output = response?.__type ? response.output : response
+        const output =
+          response?.__type || response?.output?.__type
+            ? response.output
+            : response
+
         if (SkipStepResponse.isSkipStepResponse(output)) {
           await TransactionOrchestrator.skipStep({
             transaction,
@@ -1175,7 +1252,10 @@ export class TransactionOrchestrator extends EventEmitter {
   ): Promise<void | unknown> {
     return promiseFn()
       .then(async (response: any) => {
-        const output = response?.__type ? response.output : response
+        const output =
+          response?.__type || response?.output?.__type
+            ? response.output
+            : response
 
         if (SkipStepResponse.isSkipStepResponse(output)) {
           await TransactionOrchestrator.skipStep({
@@ -1335,9 +1415,9 @@ export class TransactionOrchestrator extends EventEmitter {
         flow.state = TransactionState.INVOKING
         flow.startedAt = Date.now()
 
-        await transaction.saveCheckpoint(
+        await transaction.saveCheckpoint({
-          flow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
+          ttl: flow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL,
-        )
+        })
 
         if (transaction.hasTimeout()) {
           await transaction.scheduleTransactionTimeout(
@@ -1476,6 +1556,7 @@ export class TransactionOrchestrator extends EventEmitter {
       state: TransactionState.NOT_STARTED,
       definition: this.definition,
       steps,
+      _v: 0, // Initialize version to 0
     }
 
     return flow
@@ -1506,7 +1587,7 @@ export class TransactionOrchestrator extends EventEmitter {
     return null
   }
 
-  private static buildSteps(
+  static buildSteps(
     flow: TransactionStepsDefinition,
     existingSteps?: { [key: string]: TransactionStep }
   ): [{ [key: string]: TransactionStep }, StepFeatures] {
@@ -1588,6 +1669,7 @@ export class TransactionOrchestrator extends EventEmitter {
             failures: 0,
             lastAttempt: null,
             next: [],
+            _v: 0, // Initialize step version to 0
           }
         )
       }
@@ -1650,9 +1732,9 @@ export class TransactionOrchestrator extends EventEmitter {
     )
 
     if (newTransaction && this.getOptions().store) {
-      await transaction.saveCheckpoint(
+      await transaction.saveCheckpoint({
-        modelFlow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
+        ttl: modelFlow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL,
-      )
+      })
     }
 
     if (onLoad) {

packages/core/orchestration/src/transaction/transaction-step.ts

@@ -62,6 +62,7 @@ export class TransactionStep {
   startedAt?: number
   next: string[]
   saveResponse: boolean
+  _v?: number
 
   public getStates() {
     return this.isCompensating() ? this.compensate : this.invoke
@@ -191,8 +192,12 @@ export class TransactionStep {
       this.lastAttempt &&
       Date.now() - this.lastAttempt >
         this.definition.retryIntervalAwaiting! * 1e3 &&
+      // For compensating steps, ignore maxAwaitingRetries and retry indefinitely
+      // Compensation must complete, so we keep checking until the nested workflow finishes
       (!("maxAwaitingRetries" in this.definition) ||
-        this.attempts < this.definition.maxAwaitingRetries!)
+        (this.isCompensating()
+          ? this.attempts < this.definition.maxAwaitingRetries! * 2
+          : this.attempts < this.definition.maxAwaitingRetries!))
     )
   }
 

packages/core/orchestration/src/transaction/types.ts

@@ -263,6 +263,9 @@ export type StepFeatures = {
   hasAsyncSteps: boolean
   hasStepTimeouts: boolean
   hasRetriesTimeout: boolean
+  parallelSteps?: number
+  stepId?: string
+  _v?: number
 }
 
 export type TransactionOptions = TransactionModelOptions & StepFeatures
@@ -276,6 +279,7 @@ export type TransactionFlow = {
   metadata?: {
     eventGroupId?: string
     parentIdempotencyKey?: string
+    cancelingFromParentStep?: boolean
     sourcePath?: string
     preventReleaseEvents?: boolean
     parentStepIdempotencyKey?: string
@@ -295,4 +299,5 @@ export type TransactionFlow = {
   steps: {
     [key: string]: TransactionStep
   }
+  _v: number
 }

packages/core/orchestration/src/workflow/local-workflow.ts

@@ -417,7 +417,9 @@ export class LocalWorkflow {
 
     if (this.medusaContext) {
       this.medusaContext.eventGroupId =
-        transaction.getFlow().metadata?.eventGroupId
+        transaction.getFlow().metadata!.eventGroupId
+      transaction.getFlow().metadata!.cancelingFromParentStep ??=
+        this.medusaContext.cancelingFromParentStep
     }
 
     const { cleanUpEventListeners } = this.registerEventCallbacks({
@@ -626,6 +628,8 @@ export class LocalWorkflow {
       this.medusaContext.parentStepIdempotencyKey =
         metadata.parentStepIdempotencyKey
       this.medusaContext.preventReleaseEvents = metadata?.preventReleaseEvents
+      this.medusaContext.cancelingFromParentStep =
+        metadata?.cancelingFromParentStep
     }
   }
 }

packages/core/types/src/shared-context.ts

@@ -88,4 +88,10 @@ export type Context<TManager = unknown> = {
    * A boolean value indicating whether the current workflow execution is being cancelled.
    */
   isCancelling?: boolean
+
+  /**
+   * Weither or not a sub workflow cancellation is being triggered from a parent step.
+   * If true, the parent step will not be triggered by the sub workflow.
+   */
+  cancelingFromParentStep?: boolean
 }

packages/core/workflows-sdk/src/helper/workflow-export.ts

@@ -111,8 +111,12 @@ function createContextualWorkflowRunner<
       flow.container = executionContainer
     }
 
-    const { eventGroupId, parentStepIdempotencyKey, preventReleaseEvents } =
+    const {
-      context
+      eventGroupId,
+      parentStepIdempotencyKey,
+      preventReleaseEvents,
+      cancelingFromParentStep,
+    } = context
 
     if (!preventReleaseEvents) {
       attachOnFinishReleaseEvents(events, flow, { logOnError })
@@ -123,6 +127,7 @@ function createContextualWorkflowRunner<
       parentStepIdempotencyKey,
       sourcePath: options?.sourcePath,
       preventReleaseEvents,
+      cancelingFromParentStep,
     }
 
     context.isCancelling = isCancel
@@ -609,13 +614,13 @@ function attachOnFinishReleaseEvents(
 
     if (logOnError) {
       const workflowName = transaction.getFlow().modelId
-      transaction
+      transaction.getErrors().forEach((err) => {
-        .getErrors()
+        const errMsg = err?.error?.message ? " - " + err?.error?.message : ""
-        .forEach((err) =>
+
-          logger.error(
+        logger.error(
-            `${workflowName}:${err?.action}:${err?.handlerType} - ${err?.error?.message}${EOL}${err?.error?.stack}`
+          `${workflowName}:${err?.action}:${err?.handlerType}${errMsg}${EOL}${err?.error?.stack}`
-          )
         )
+      })
     }
 
     const eventBusService = (

packages/core/workflows-sdk/src/utils/composer/create-workflow.ts

@@ -24,12 +24,25 @@ import {
   CreateWorkflowComposerContext,
   HookHandler,
   ReturnWorkflow,
+  StepExecutionContext,
   StepFunction,
   WorkflowData,
 } from "./type"
 
 global[OrchestrationUtils.SymbolMedusaWorkflowComposerContext] = null
 
+const buildTransactionId = (
+  step: { __step__: string },
+  stepContext: StepExecutionContext
+) => {
+  return (
+    step.__step__ +
+    "-" +
+    (stepContext.transactionId ?? ulid()) +
+    (stepContext.attempt > 1 ? `-attempt-${stepContext.attempt}` : "")
+  )
+}
+
 /**
  * This function creates a workflow with the provided name and a constructor function.
  * The constructor function builds the workflow from steps created by the {@link createStep} function.
@@ -207,8 +220,7 @@ export function createWorkflow<TData, TResult, THooks extends any[]>(
 
         const executionContext = {
           ...(sharedContext?.context ?? {}),
-          transactionId:
+          transactionId: buildTransactionId(step, stepContext),
-            step.__step__ + "-" + (stepContext.transactionId ?? ulid()),
           parentStepIdempotencyKey: stepContext.idempotencyKey,
           preventReleaseEvents: true,
           runId: stepContext.runId,
@@ -218,7 +230,10 @@ export function createWorkflow<TData, TResult, THooks extends any[]>(
         if (workflowEngine && isAsync) {
           transaction = await workflowEngine.run(name, {
             input: stepInput as any,
+            transactionId: executionContext.transactionId,
             context: executionContext,
+            throwOnError: false,
+            logOnError: true,
           })
         } else {
           transaction = await workflow.run({
@@ -235,9 +250,6 @@ export function createWorkflow<TData, TResult, THooks extends any[]>(
       },
       async (transaction, stepContext) => {
         // The step itself has failed, there is nothing to revert
-        if (!transaction) {
-          return
-        }
 
         const { container, ...sharedContext } = stepContext
         const isAsync = stepContext[" stepDefinition"]?.async
@@ -248,27 +260,28 @@ export function createWorkflow<TData, TResult, THooks extends any[]>(
 
         const executionContext = {
           ...(sharedContext?.context ?? {}),
-          transactionId:
+          transactionId: buildTransactionId(step, stepContext),
-            step.__step__ + "-" + (stepContext.transactionId ?? ulid()),
           parentStepIdempotencyKey: stepContext.idempotencyKey,
           preventReleaseEvents: true,
+          cancelingFromParentStep: true,
         }
 
-        const transactionId = step.__step__ + "-" + stepContext.transactionId
-
         if (workflowEngine && isAsync) {
           await workflowEngine.cancel(name, {
-            transactionId: transactionId,
+            transactionId: executionContext.transactionId,
             context: executionContext,
           })
         } else {
           await workflow(container).cancel({
-            transaction: (transaction as WorkflowResult<any>)?.transaction,
+            transaction: ((transaction as WorkflowResult<any>) ?? {})
-            transactionId,
+              ?.transaction,
+            transactionId: executionContext.transactionId,
             container,
             context: executionContext,
           })
         }
+
+        return
       }
     )(input) as ReturnType<StepFunction<TData, TResult>>
 

packages/medusa-test-utils/src/module-test-runner.ts

@@ -8,6 +8,7 @@ import {
   normalizeImportPathWithSource,
   toMikroOrmEntities,
 } from "@medusajs/framework/utils"
+import { logger } from "@medusajs/framework/logger"
 import * as fs from "fs"
 import { getDatabaseURL, getMikroOrmWrapper, TestDatabase } from "./database"
 import { initModules, InitModulesOptions } from "./init-modules"
@@ -23,6 +24,24 @@ export interface SuiteOptions<TService = unknown> {
   }
 }
 
+interface ModuleTestRunnerConfig<TService = any> {
+  moduleName: string
+  moduleModels?: any[]
+  moduleOptions?: Record<string, any>
+  moduleDependencies?: string[]
+  joinerConfig?: any[]
+  schema?: string
+  dbName?: string
+  injectedDependencies?: Record<string, any>
+  resolve?: string
+  debug?: boolean
+  cwd?: string
+  hooks?: {
+    beforeModuleInit?: () => Promise<void>
+    afterModuleInit?: (medusaApp: any, service: TService) => Promise<void>
+  }
+}
+
 function createMikroOrmWrapper(options: {
   moduleModels?: (Function | DmlEntity<any, any>)[]
   resolve?: string
@@ -64,6 +83,220 @@ function createMikroOrmWrapper(options: {
   return { MikroOrmWrapper, models: moduleModels }
 }
 
+class ModuleTestRunner<TService = any> {
+  private moduleName: string
+  private schema: string
+  private dbName: string
+  private dbConfig: {
+    clientUrl: string
+    schema: string
+    debug: boolean
+  }
+  private debug: boolean
+  private resolve?: string
+  private cwd?: string
+  private moduleOptions: Record<string, any>
+  private moduleDependencies?: string[]
+  private joinerConfig: any[]
+  private injectedDependencies: Record<string, any>
+  private hooks: ModuleTestRunnerConfig<TService>["hooks"] = {}
+
+  private connection: any = null
+  private MikroOrmWrapper!: TestDatabase
+  private moduleModels: (Function | DmlEntity<any, any>)[] = []
+  private modulesConfig: any = {}
+  private moduleOptionsConfig!: InitModulesOptions
+
+  private shutdown: () => Promise<void> = async () => void 0
+  private moduleService: any = null
+  private medusaApp: any = {}
+
+  constructor(config: ModuleTestRunnerConfig<TService>) {
+    const tempName = parseInt(process.env.JEST_WORKER_ID || "1")
+    this.moduleName = config.moduleName
+    this.dbName =
+      config.dbName ??
+      `medusa-${config.moduleName.toLowerCase()}-integration-${tempName}`
+    this.schema = config.schema ?? "public"
+    this.debug = config.debug ?? false
+    this.resolve = config.resolve
+    this.cwd = config.cwd
+    this.moduleOptions = config.moduleOptions ?? {}
+    this.moduleDependencies = config.moduleDependencies
+    this.joinerConfig = config.joinerConfig ?? []
+    this.injectedDependencies = config.injectedDependencies ?? {}
+    this.hooks = config.hooks ?? {}
+
+    this.dbConfig = {
+      clientUrl: getDatabaseURL(this.dbName),
+      schema: this.schema,
+      debug: this.debug,
+    }
+
+    this.setupProcessHandlers()
+    this.initializeConfig(config.moduleModels)
+  }
+
+  private setupProcessHandlers(): void {
+    process.on("SIGTERM", async () => {
+      await this.cleanup()
+      process.exit(0)
+    })
+
+    process.on("SIGINT", async () => {
+      await this.cleanup()
+      process.exit(0)
+    })
+  }
+
+  private initializeConfig(moduleModels?: any[]): void {
+    const moduleSdkImports = require("@medusajs/framework/modules-sdk")
+
+    // Use a unique connection for all the entire suite
+    this.connection = ModulesSdkUtils.createPgConnection(this.dbConfig)
+
+    const { MikroOrmWrapper, models } = createMikroOrmWrapper({
+      moduleModels,
+      resolve: this.resolve,
+      dbConfig: this.dbConfig,
+      cwd: this.cwd,
+    })
+
+    this.MikroOrmWrapper = MikroOrmWrapper
+    this.moduleModels = models
+
+    this.modulesConfig = {
+      [this.moduleName]: {
+        definition: moduleSdkImports.ModulesDefinition[this.moduleName],
+        resolve: this.resolve,
+        dependencies: this.moduleDependencies,
+        options: {
+          database: this.dbConfig,
+          ...this.moduleOptions,
+          [isSharedConnectionSymbol]: true,
+        },
+      },
+    }
+
+    this.moduleOptionsConfig = {
+      injectedDependencies: {
+        [ContainerRegistrationKeys.PG_CONNECTION]: this.connection,
+        [Modules.EVENT_BUS]: new MockEventBusService(),
+        [ContainerRegistrationKeys.LOGGER]: console,
+        ...this.injectedDependencies,
+      },
+      modulesConfig: this.modulesConfig,
+      databaseConfig: this.dbConfig,
+      joinerConfig: this.joinerConfig,
+      preventConnectionDestroyWarning: true,
+      cwd: this.cwd,
+    }
+  }
+
+  private createMedusaAppProxy(): any {
+    return new Proxy(
+      {},
+      {
+        get: (target, prop) => {
+          return this.medusaApp?.[prop]
+        },
+      }
+    )
+  }
+
+  private createServiceProxy(): any {
+    return new Proxy(
+      {},
+      {
+        get: (target, prop) => {
+          return this.moduleService?.[prop]
+        },
+      }
+    )
+  }
+
+  public async beforeAll(): Promise<void> {
+    try {
+      this.setupProcessHandlers()
+      process.env.LOG_LEVEL = "error"
+    } catch (error) {
+      await this.cleanup()
+      throw error
+    }
+  }
+
+  public async beforeEach(): Promise<void> {
+    try {
+      if (this.moduleModels.length) {
+        await this.MikroOrmWrapper.setupDatabase()
+      }
+
+      if (this.hooks?.beforeModuleInit) {
+        await this.hooks.beforeModuleInit()
+      }
+
+      const output = await initModules(this.moduleOptionsConfig)
+      this.shutdown = output.shutdown
+      this.medusaApp = output.medusaApp
+      this.moduleService = output.medusaApp.modules[this.moduleName]
+
+      if (this.hooks?.afterModuleInit) {
+        await this.hooks.afterModuleInit(this.medusaApp, this.moduleService)
+      }
+    } catch (error) {
+      logger.error("Error in beforeEach:", error?.message)
+      await this.cleanup()
+      throw error
+    }
+  }
+
+  public async afterEach(): Promise<void> {
+    try {
+      if (this.moduleModels.length) {
+        await this.MikroOrmWrapper.clearDatabase()
+      }
+      await this.shutdown()
+      this.moduleService = {}
+      this.medusaApp = {}
+    } catch (error) {
+      logger.error("Error in afterEach:", error?.message)
+      throw error
+    }
+  }
+
+  public async cleanup(): Promise<void> {
+    try {
+      process.removeAllListeners("SIGTERM")
+      process.removeAllListeners("SIGINT")
+
+      await (this.connection as any)?.context?.destroy()
+      await (this.connection as any)?.destroy()
+
+      this.moduleService = null
+      this.medusaApp = null
+      this.connection = null
+
+      if (global.gc) {
+        global.gc()
+      }
+    } catch (error) {
+      logger.error("Error during cleanup:", error?.message)
+    }
+  }
+
+  public getOptions(): SuiteOptions<TService> {
+    return {
+      MikroOrmWrapper: this.MikroOrmWrapper,
+      medusaApp: this.createMedusaAppProxy(),
+      service: this.createServiceProxy(),
+      dbConfig: {
+        schema: this.schema,
+        clientUrl: this.dbConfig.clientUrl,
+      },
+    }
+  }
+}
+
 export function moduleIntegrationTestRunner<TService = any>({
   moduleName,
   moduleModels,
@@ -76,6 +309,7 @@ export function moduleIntegrationTestRunner<TService = any>({
   resolve,
   injectedDependencies = {},
   cwd,
+  hooks,
 }: {
   moduleName: string
   moduleModels?: any[]
@@ -88,115 +322,68 @@ export function moduleIntegrationTestRunner<TService = any>({
   resolve?: string
   debug?: boolean
   cwd?: string
+  hooks?: ModuleTestRunnerConfig<TService>["hooks"]
   testSuite: (options: SuiteOptions<TService>) => void
 }) {
-  const moduleSdkImports = require("@medusajs/framework/modules-sdk")
+  const runner = new ModuleTestRunner<TService>({
-
+    moduleName,
-  process.env.LOG_LEVEL = "error"
+    moduleModels,
-
+    moduleOptions,
-  const tempName = parseInt(process.env.JEST_WORKER_ID || "1")
+    moduleDependencies,
-  const dbName = `medusa-${moduleName.toLowerCase()}-integration-${tempName}`
+    joinerConfig,
-
-  const dbConfig = {
-    clientUrl: getDatabaseURL(dbName),
     schema,
     debug,
-  }
-
-  // Use a unique connection for all the entire suite
-  const connection = ModulesSdkUtils.createPgConnection(dbConfig)
-
-  const { MikroOrmWrapper, models } = createMikroOrmWrapper({
-    moduleModels,
     resolve,
-    dbConfig,
+    injectedDependencies,
     cwd,
+    hooks,
   })
 
-  moduleModels = models
-
-  const modulesConfig_ = {
-    [moduleName]: {
-      definition: moduleSdkImports.ModulesDefinition[moduleName],
-      resolve,
-      dependencies: moduleDependencies,
-      options: {
-        database: dbConfig,
-        ...moduleOptions,
-        [isSharedConnectionSymbol]: true,
-      },
-    },
-  }
-
-  const moduleOptions_: InitModulesOptions = {
-    injectedDependencies: {
-      [ContainerRegistrationKeys.PG_CONNECTION]: connection,
-      [Modules.EVENT_BUS]: new MockEventBusService(),
-      [ContainerRegistrationKeys.LOGGER]: console,
-      ...injectedDependencies,
-    },
-    modulesConfig: modulesConfig_,
-    databaseConfig: dbConfig,
-    joinerConfig,
-    preventConnectionDestroyWarning: true,
-    cwd,
-  }
-
-  let shutdown: () => Promise<void>
-  let moduleService
-  let medusaApp = {}
-
-  const options = {
-    MikroOrmWrapper,
-    medusaApp: new Proxy(
-      {},
-      {
-        get: (target, prop) => {
-          return medusaApp[prop]
-        },
-      }
-    ),
-    service: new Proxy(
-      {},
-      {
-        get: (target, prop) => {
-          return moduleService[prop]
-        },
-      }
-    ),
-    dbConfig: {
-      schema,
-      clientUrl: dbConfig.clientUrl,
-    },
-  } as SuiteOptions<TService>
-
-  const beforeEach_ = async () => {
-    if (moduleModels.length) {
-      await MikroOrmWrapper.setupDatabase()
-    }
-    const output = await initModules(moduleOptions_)
-    shutdown = output.shutdown
-    medusaApp = output.medusaApp
-    moduleService = output.medusaApp.modules[moduleName]
-  }
-
-  const afterEach_ = async () => {
-    if (moduleModels.length) {
-      await MikroOrmWrapper.clearDatabase()
-    }
-    await shutdown()
-    moduleService = {}
-    medusaApp = {}
-  }
-
   return describe("", () => {
-    beforeEach(beforeEach_)
+    let testOptions: SuiteOptions<TService>
-    afterEach(afterEach_)
+
-    afterAll(async () => {
+    beforeAll(async () => {
-      await (connection as any).context?.destroy()
+      await runner.beforeAll()
-      await (connection as any).destroy()
+      testOptions = runner.getOptions()
     })
 
-    testSuite(options)
+    beforeEach(async () => {
+      await runner.beforeEach()
+    })
+
+    afterEach(async () => {
+      await runner.afterEach()
+    })
+
+    afterAll(async () => {
+      // Run main cleanup
+      await runner.cleanup()
+
+      // Clean references to the test options
+      for (const key in testOptions) {
+        if (typeof testOptions[key] === "function") {
+          testOptions[key] = null
+        } else if (
+          typeof testOptions[key] === "object" &&
+          testOptions[key] !== null
+        ) {
+          Object.keys(testOptions[key]).forEach((k) => {
+            testOptions[key][k] = null
+          })
+          testOptions[key] = null
+        }
+      }
+
+      // Encourage garbage collection
+      // @ts-ignore
+      testOptions = null
+
+      if (global.gc) {
+        global.gc()
+      }
+    })
+
+    // Run test suite with options
+    testSuite(runner.getOptions())
   })
 }

packages/modules/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_async.ts

@@ -30,6 +30,17 @@ const nestedWorkflow = createWorkflow(
   }
 )
 
+const nestedWorkflow2 = createWorkflow(
+  {
+    name: "nested_sub_flow_async_2",
+  },
+  function (input) {
+    const resp = step_1_background(input)
+
+    return resp
+  }
+)
+
 createWorkflow(
   {
     name: "workflow_async_background",
@@ -41,7 +52,7 @@ createWorkflow(
           input,
         })
         .config({ name: "step_sub_flow_1" }),
-      nestedWorkflow
+      nestedWorkflow2
         .runAsStep({
           input,
         })

packages/modules/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_parallel_async.ts

@@ -14,9 +14,25 @@ const step_2 = createStep(
   async (_, { container }) => {
     const we = container.resolve(Modules.WORKFLOW_ENGINE)
 
+    const onFinishPromise = new Promise<void>((resolve, reject) => {
+      void we.subscribe({
+        workflowId: "workflow_sub_workflow",
+        subscriber: (event) => {
+          if (event.eventType === "onFinish") {
+            if (event.errors.length > 0) {
+              reject(event.errors[0])
+            } else {
+              resolve()
+            }
+          }
+        },
+      })
+    })
+
     await we.run("workflow_sub_workflow", {
       throwOnError: true,
     })
+    await onFinishPromise
   }
 )
 
@@ -34,7 +50,6 @@ const step_2_sub = createStep(
 const subFlow = createWorkflow(
   {
     name: "workflow_sub_workflow",
-    retentionTime: 1000,
   },
   function (input) {
     step_2_sub()
@@ -68,7 +83,7 @@ const step_3 = createStep(
 createWorkflow(
   {
     name: "workflow_parallel_async",
-    retentionTime: 1000,
+    retentionTime: 5,
   },
   function (input) {
     parallelize(step_1(), step_2(), step_3())

packages/modules/workflow-engine-inmemory/integration-tests/__tests__/index.spec.ts

@@ -1,4 +1,5 @@
 import { MedusaContainer } from "@medusajs/framework"
+import { asFunction } from "@medusajs/framework/awilix"
 import {
   DistributedTransactionType,
   TransactionState,
@@ -25,7 +26,6 @@ import {
 } from "@medusajs/framework/workflows-sdk"
 import { moduleIntegrationTestRunner } from "@medusajs/test-utils"
 import { WorkflowsModuleService } from "@services"
-import { asFunction } from "@medusajs/framework/awilix"
 import { setTimeout as setTimeoutSync } from "timers"
 import { setTimeout as setTimeoutPromise } from "timers/promises"
 import { ulid } from "ulid"
@@ -39,40 +39,30 @@ import {
   workflowNotIdempotentWithRetentionStep3Invoke,
 } from "../__fixtures__"
 import {
-  eventGroupWorkflowId,
-  workflowEventGroupIdStep1Mock,
-  workflowEventGroupIdStep2Mock,
-} from "../__fixtures__/workflow_event_group_id"
-import {
-  step1InvokeMock as step1InvokeMockAutoRetries,
-  step2InvokeMock as step2InvokeMockAutoRetries,
   step1CompensateMock as step1CompensateMockAutoRetries,
+  step1InvokeMock as step1InvokeMockAutoRetries,
   step2CompensateMock as step2CompensateMockAutoRetries,
+  step2InvokeMock as step2InvokeMockAutoRetries,
 } from "../__fixtures__/workflow_1_auto_retries"
 import {
-  step1InvokeMock as step1InvokeMockAutoRetriesFalse,
-  step2InvokeMock as step2InvokeMockAutoRetriesFalse,
   step1CompensateMock as step1CompensateMockAutoRetriesFalse,
+  step1InvokeMock as step1InvokeMockAutoRetriesFalse,
   step2CompensateMock as step2CompensateMockAutoRetriesFalse,
+  step2InvokeMock as step2InvokeMockAutoRetriesFalse,
 } from "../__fixtures__/workflow_1_auto_retries_false"
 import {
   step1InvokeMock as step1InvokeMockManualRetry,
   step2InvokeMock as step2InvokeMockManualRetry,
 } from "../__fixtures__/workflow_1_manual_retry_step"
+import {
+  eventGroupWorkflowId,
+  workflowEventGroupIdStep1Mock,
+  workflowEventGroupIdStep2Mock,
+} from "../__fixtures__/workflow_event_group_id"
 import { createScheduled } from "../__fixtures__/workflow_scheduled"
 
 jest.setTimeout(60000)
 
-const failTrap = (done, name, timeout = 5000) => {
-  return setTimeoutSync(() => {
-    // REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
-    console.warn(
-      `Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
-    )
-    done()
-  }, timeout)
-}
-
 function times(num) {
   let resolver
   let counter = 0
@@ -159,14 +149,14 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
       })
 
       describe("Cancel transaction", function () {
-        it("should cancel an ongoing execution with async unfinished yet step", (done) => {
+        it("should cancel an ongoing execution with async unfinished yet step", async () => {
           const transactionId = "transaction-to-cancel-id" + ulid()
           const step1 = createStep("step1", async () => {
             return new StepResponse("step1")
           })
 
           const step2 = createStep("step2", async () => {
-            await setTimeoutPromise(500)
+            await setTimeoutPromise(200)
             return new StepResponse("step2")
           })
 
@@ -184,43 +174,39 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
             return new WorkflowResponse("finished")
           })
 
-          workflowOrcModule
+          const onFinish = new Promise<void>((resolve) => {
-            .run(workflowId, {
+            workflowOrcModule.subscribe({
-              input: {},
+              workflowId,
               transactionId,
-            })
+              subscriber: async (event) => {
-            .then(async () => {
+                if (event.eventType === "onFinish") {
-              await setTimeoutPromise(100)
+                  resolve()
-
+                }
-              await workflowOrcModule.cancel(workflowId, {
+              },
-                transactionId,
-              })
-
-              workflowOrcModule.subscribe({
-                workflowId,
-                transactionId,
-                subscriber: async (event) => {
-                  if (event.eventType === "onFinish") {
-                    const execution =
-                      await workflowOrcModule.listWorkflowExecutions({
-                        transaction_id: transactionId,
-                      })
-
-                    expect(execution.length).toEqual(1)
-                    expect(execution[0].state).toEqual(
-                      TransactionState.REVERTED
-                    )
-                    done()
-                    clearTimeout(timeout)
-                  }
-                },
-              })
             })
 
-          const timeout = failTrap(
+            workflowOrcModule
-            done,
+              .run(workflowId, {
-            "should cancel an ongoing execution with async unfinished yet step"
+                input: {},
-          )
+                transactionId,
+              })
+              .then(async () => {
+                await setTimeoutPromise(100)
+
+                await workflowOrcModule.cancel(workflowId, {
+                  transactionId,
+                })
+              })
+          })
+
+          await onFinish
+
+          const execution = await workflowOrcModule.listWorkflowExecutions({
+            transaction_id: transactionId,
+          })
+
+          expect(execution.length).toEqual(1)
+          expect(execution[0].state).toEqual(TransactionState.REVERTED)
         })
 
         it("should cancel a complete execution with a sync workflow running as async", async () => {
@@ -375,11 +361,11 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
         })
       })
 
-      it("should manually retry a step that is taking too long to finish", (done) => {
+      it("should manually retry a step that is taking too long to finish", async () => {
         const transactionId = "transaction-manual-retry" + ulid()
         const workflowId = "workflow_1_manual_retry_step"
 
-        void workflowOrcModule
+        await workflowOrcModule
           .run(workflowId, {
             input: {},
             transactionId,
@@ -388,6 +374,18 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
             expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
             expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(1)
 
+            const onFinishPromise = new Promise<void>((resolve, reject) => {
+              workflowOrcModule.subscribe({
+                workflowId,
+                transactionId,
+                subscriber: async (event) => {
+                  if (event.eventType === "onFinish") {
+                    resolve()
+                  }
+                },
+              })
+            })
+
             void workflowOrcModule.retryStep({
               idempotencyKey: {
                 workflowId,
@@ -396,68 +394,54 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
                 action: "invoke",
               },
             })
+
+            return onFinishPromise
           })
 
-        workflowOrcModule.subscribe({
+        expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
-          workflowId,
+        expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(2)
-          transactionId,
-          subscriber: async (event) => {
-            if (event.eventType === "onFinish") {
-              expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
-              expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(2)
-              done()
-              clearTimeout(timeout)
-            }
-          },
-        })
-
-        const timeout = failTrap(
-          done,
-          "should manually retry a step that is taking too long to finish"
-        )
       })
 
-      it("should retry steps X times automatically when maxRetries is set", (done) => {
+      it("should retry steps X times automatically when maxRetries is set", async () => {
         const transactionId = "transaction-auto-retries" + ulid()
         const workflowId = "workflow_1_auto_retries"
 
+        const onFinishPromise = new Promise<void>((resolve, reject) => {
+          workflowOrcModule.subscribe({
+            workflowId,
+            transactionId,
+            subscriber: async (event) => {
+              if (event.eventType === "onFinish") {
+                resolve()
+              }
+            },
+          })
+        })
+
         void workflowOrcModule.run(workflowId, {
           input: {},
           transactionId,
         })
 
-        workflowOrcModule.subscribe({
+        await onFinishPromise
-          workflowId,
-          transactionId,
-          subscriber: async (event) => {
-            if (event.eventType === "onFinish") {
-              expect(step1InvokeMockAutoRetries).toHaveBeenCalledTimes(1)
-              expect(step2InvokeMockAutoRetries).toHaveBeenCalledTimes(3)
-              expect(step1CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
-              expect(step2CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
-              done()
-              clearTimeout(timeout)
-            }
-          },
-        })
 
-        const timeout = failTrap(
+        expect(step1InvokeMockAutoRetries).toHaveBeenCalledTimes(1)
-          done,
+        expect(step2InvokeMockAutoRetries).toHaveBeenCalledTimes(3)
-          "should retry steps X times automatically when maxRetries is set"
+        expect(step1CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
-        )
+        expect(step2CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
       })
 
-      it("should not retry steps X times automatically when maxRetries is set and autoRetry is false", (done) => {
+      it("should not retry steps X times automatically when maxRetries is set and autoRetry is false", async () => {
-        ;(async () => {
+        const transactionId = "transaction-auto-retries" + ulid()
-          const transactionId = "transaction-auto-retries" + ulid()
+        const workflowId = "workflow_1_auto_retries_false"
-          const workflowId = "workflow_1_auto_retries_false"
 
-          await workflowOrcModule.run(workflowId, {
+        await workflowOrcModule.run(workflowId, {
-            input: {},
+          input: {},
-            transactionId,
+          transactionId,
-            throwOnError: false,
+          throwOnError: false,
-          })
+        })
 
+        const onFinishPromise = new Promise<void>((resolve, reject) => {
           workflowOrcModule.subscribe({
             workflowId,
             transactionId,
@@ -471,37 +455,39 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
                 expect(
                   step2CompensateMockAutoRetriesFalse
                 ).toHaveBeenCalledTimes(1)
-                done()
+                resolve()
               }
             },
           })
+        })
 
-          expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
+        expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
-          expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
+        expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
-          expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
+        expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
-          expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
+        expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
 
-          await setTimeoutPromise(2000)
+        await setTimeoutPromise(2000)
 
-          await workflowOrcModule.run(workflowId, {
+        await workflowOrcModule.run(workflowId, {
-            input: {},
+          input: {},
-            transactionId,
+          transactionId,
-            throwOnError: false,
+          throwOnError: false,
-          })
+        })
 
-          await setTimeoutPromise(2000)
+        await setTimeoutPromise(2000)
 
-          expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
+        expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
-          expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(2)
+        expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(2)
-          expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
+        expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
-          expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
+        expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
 
-          await workflowOrcModule.run(workflowId, {
+        await workflowOrcModule.run(workflowId, {
-            input: {},
+          input: {},
-            transactionId,
+          transactionId,
-            throwOnError: false,
+          throwOnError: false,
-          })
+        })
-        })()
+
+        await onFinishPromise
       })
 
       it("should prevent executing twice the same workflow in perfect concurrency with the same transactionId and non idempotent and not async but retention time is set", async () => {
@@ -611,7 +597,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
         )
       })
 
-      it("should compose nested workflows w/ async steps", (done) => {
+      it("should compose nested workflows w/ async steps", async () => {
         const asyncResults: any[] = []
         const mockStep1Fn = jest.fn().mockImplementation(() => {
           const res = { obj: "return from 1" }
@@ -662,35 +648,31 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
         })
 
         asyncResults.push("begin workflow")
-        workflowOrcModule
+        await workflowOrcModule.run(workflowId, {
-          .run(workflowId, {
+          input: {},
-            input: {},
+        })
-          })
-          .then(() => {
-            asyncResults.push("returned workflow")
 
-            void workflowOrcModule.subscribe({
+        const onFinishPromise = new Promise<void>((resolve) => {
-              workflowId,
+          void workflowOrcModule.subscribe({
-              subscriber: (event) => {
+            workflowId,
-                if (event.eventType === "onFinish") {
+            subscriber: (event) => {
-                  expect(asyncResults).toEqual([
+              if (event.eventType === "onFinish") {
-                    "begin workflow",
+                expect(asyncResults).toEqual([
-                    { obj: "return from 1" },
+                  "begin workflow",
-                    "returned workflow",
+                  { obj: "return from 1" },
-                    { obj: "return from 2" },
+                  "returned workflow",
-                    { obj: "return from 3" },
+                  { obj: "return from 2" },
-                  ])
+                  { obj: "return from 3" },
-                  done()
+                ])
-                  clearTimeout(timeout)
+                resolve()
-                }
+              }
-              },
+            },
-            })
           })
+        })
 
-        const timeout = failTrap(
+        asyncResults.push("returned workflow")
-          done,
+
-          "should subscribe to a async workflow and receive the response when it finishes"
+        await onFinishPromise
-        )
       })
 
       describe("Testing basic workflow", function () {
@@ -871,22 +853,19 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
           expect(transaction.getFlow().state).toEqual("reverted")
         })
 
-        it("should subscribe to a async workflow and receive the response when it finishes", (done) => {
+        it("should subscribe to a async workflow and receive the response when it finishes", async () => {
           const transactionId = "trx_123" + ulid()
 
-          const onFinish = jest.fn(() => {
+          const onFinishPromise = new Promise<void>((resolve) => {
-            done()
+            void workflowOrcModule.subscribe({
-            clearTimeout(timeout)
+              workflowId: "workflow_async_background",
-          })
+              transactionId,
-
+              subscriber: (event) => {
-          void workflowOrcModule.subscribe({
+                if (event.eventType === "onFinish") {
-            workflowId: "workflow_async_background",
+                  resolve()
-            transactionId,
+                }
-            subscriber: (event) => {
+              },
-              if (event.eventType === "onFinish") {
+            })
-                onFinish()
-              }
-            },
           })
 
           void workflowOrcModule.run("workflow_async_background", {
@@ -897,11 +876,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
             throwOnError: false,
           })
 
-          expect(onFinish).toHaveBeenCalledTimes(0)
+          await onFinishPromise
-          const timeout = failTrap(
-            done,
-            "should subscribe to a async workflow and receive the response when it finishes"
-          )
         })
 
         it("should cancel and revert a completed workflow", async () => {
@@ -955,43 +930,43 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
           expect(executions[0].state).toEqual(TransactionState.REVERTED)
         })
 
-        it("should run conditional steps if condition is true", (done) => {
+        it("should run conditional steps if condition is true", async () => {
-          void workflowOrcModule.subscribe({
+          let timeout: NodeJS.Timeout
-            workflowId: "workflow_conditional_step",
+
-            subscriber: (event) => {
+          const onFinishPromise = new Promise<void>((resolve, reject) => {
-              if (event.eventType === "onFinish") {
+            void workflowOrcModule.subscribe({
-                expect(conditionalStep2Invoke).toHaveBeenCalledTimes(2)
+              workflowId: "workflow_conditional_step",
-                expect(conditionalStep3Invoke).toHaveBeenCalledTimes(1)
+              subscriber: (event) => {
-                done()
+                if (event.eventType === "onFinish") {
-                clearTimeout(timeout)
+                  resolve()
-              }
+                }
-            },
+              },
+            })
           })
 
-          workflowOrcModule.run("workflow_conditional_step", {
+          void workflowOrcModule.run("workflow_conditional_step", {
             input: {
               runNewStepName: true,
             },
             throwOnError: true,
           })
 
-          const timeout = failTrap(
+          await onFinishPromise
-            done,
+
-            "should not run conditional steps if condition is false"
+          expect(conditionalStep2Invoke).toHaveBeenCalledTimes(2)
-          )
+          expect(conditionalStep3Invoke).toHaveBeenCalledTimes(1)
         })
 
-        it("should not run conditional steps if condition is false", (done) => {
+        it("should not run conditional steps if condition is false", async () => {
-          void workflowOrcModule.subscribe({
+          const onFinishPromise = new Promise<void>((resolve, reject) => {
-            workflowId: "workflow_conditional_step",
+            void workflowOrcModule.subscribe({
-            subscriber: (event) => {
+              workflowId: "workflow_conditional_step",
-              if (event.eventType === "onFinish") {
+              subscriber: (event) => {
-                expect(conditionalStep2Invoke).toHaveBeenCalledTimes(1)
+                if (event.eventType === "onFinish") {
-                expect(conditionalStep3Invoke).toHaveBeenCalledTimes(0)
+                  resolve()
-                done()
+                }
-                clearTimeout(timeout)
+              },
-              }
+            })
-            },
           })
 
           workflowOrcModule.run("workflow_conditional_step", {
@@ -1001,10 +976,10 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
             throwOnError: true,
           })
 
-          const timeout = failTrap(
+          await onFinishPromise
-            done,
+
-            "should not run conditional steps if condition is false"
+          expect(conditionalStep2Invoke).toHaveBeenCalledTimes(1)
-          )
+          expect(conditionalStep3Invoke).toHaveBeenCalledTimes(0)
         })
       })
 
@@ -1127,36 +1102,36 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
           expect(executionsListAfter).toHaveLength(1)
         })
 
-        it("should display error when multple async steps are running in parallel", (done) => {
+        it("should display error when multple async steps are running in parallel", async () => {
+          let errors: Error[] = []
+          const onFinishPromise = new Promise<void>((resolve) => {
+            void workflowOrcModule.subscribe({
+              workflowId: "workflow_parallel_async",
+              subscriber: (event) => {
+                if (event.eventType === "onFinish") {
+                  errors = event.errors
+                  resolve()
+                }
+              },
+            })
+          })
+
           void workflowOrcModule.run("workflow_parallel_async", {
             input: {},
             throwOnError: false,
           })
 
-          void workflowOrcModule.subscribe({
+          await onFinishPromise
-            workflowId: "workflow_parallel_async",
-            subscriber: (event) => {
-              if (event.eventType === "onFinish") {
-                expect(event.errors).toEqual(
-                  expect.arrayContaining([
-                    expect.objectContaining({
-                      action: "step_2",
-                      handlerType: "invoke",
-                      error: expect.objectContaining({
-                        message: "Error in parallel step",
-                      }),
-                    }),
-                  ])
-                )
-                done()
-                clearTimeout(timeout)
-              }
-            },
-          })
 
-          const timeout = failTrap(
+          const errMessage = errors[0]?.error.message
-            done,
+          expect(errMessage).toContain("Error in parallel step")
-            "should display error when multple async steps are running in parallel"
+          expect(errors).toEqual(
+            expect.arrayContaining([
+              expect.objectContaining({
+                action: "step_2",
+                handlerType: "invoke",
+              }),
+            ])
           )
         })
       })

packages/modules/workflow-engine-inmemory/integration-tests/__tests__/race.spec.ts

@@ -1,36 +1,187 @@
 import { IWorkflowEngineService } from "@medusajs/framework/types"
-import { Modules } from "@medusajs/framework/utils"
+import { Modules, TransactionHandlerType } from "@medusajs/framework/utils"
 import {
   createStep,
   createWorkflow,
+  parallelize,
   StepResponse,
   transform,
   WorkflowResponse,
 } from "@medusajs/framework/workflows-sdk"
 import { moduleIntegrationTestRunner } from "@medusajs/test-utils"
-import { setTimeout as setTimeoutSync } from "timers"
 import { setTimeout } from "timers/promises"
+import { ulid } from "ulid"
 import "../__fixtures__"
 
-jest.setTimeout(300000)
+jest.setTimeout(30000)
-
-const failTrap = (done, name, timeout = 5000) => {
-  return setTimeoutSync(() => {
-    // REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
-    console.warn(
-      `Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
-    )
-    done()
-  }, timeout)
-}
 
 moduleIntegrationTestRunner<IWorkflowEngineService>({
   moduleName: Modules.WORKFLOW_ENGINE,
   resolve: __dirname + "/../..",
-  testSuite: ({ service: workflowOrcModule, medusaApp }) => {
+  testSuite: ({ service: workflowOrcModule }) => {
     // TODO: Debug the issue with this test https://github.com/medusajs/medusa/actions/runs/13900190144/job/38897122803#step:5:5616
-    describe.skip("Testing race condition of the workflow during retry", () => {
+    describe("Testing race condition of the workflow during retry", () => {
-      it("should prevent race continuation of the workflow during retryIntervalAwaiting in background execution", (done) => {
+      it("should manage saving multiple async steps in concurrency", async () => {
+        const step0 = createStep(
+          { name: "step0", async: true, backgroundExecution: true },
+          async () => {
+            return new StepResponse("result from step 0")
+          }
+        )
+
+        const step1 = createStep(
+          { name: "step1", async: true, backgroundExecution: true },
+          async () => {
+            return new StepResponse("result from step 1")
+          }
+        )
+
+        const step2 = createStep(
+          { name: "step2", async: true, backgroundExecution: true },
+          async () => {
+            return new StepResponse("result from step 2")
+          }
+        )
+        const step3 = createStep(
+          { name: "step3", async: true, backgroundExecution: true },
+          async () => {
+            return new StepResponse("result from step 3")
+          }
+        )
+
+        const step4 = createStep(
+          { name: "step4", async: true, backgroundExecution: true },
+          async () => {
+            return new StepResponse("result from step 4")
+          }
+        )
+        const step5 = createStep({ name: "step5" }, async (all: string[]) => {
+          const ret = [...all, "result from step 5"]
+          return new StepResponse(ret)
+        })
+
+        const workflowId = "workflow-1" + ulid()
+        createWorkflow(
+          {
+            name: workflowId,
+            idempotent: true,
+            retentionTime: 5,
+          },
+          function () {
+            const all = parallelize(step0(), step1(), step2(), step3(), step4())
+            const res = step5(all)
+            return new WorkflowResponse(res)
+          }
+        )
+
+        const transactionId = ulid()
+        const done = new Promise<void>((resolve, reject) => {
+          void workflowOrcModule.subscribe({
+            workflowId: workflowId,
+            transactionId,
+            subscriber: async (event) => {
+              if (event.eventType === "onFinish") {
+                resolve(event.result)
+              }
+            },
+          })
+        })
+
+        await workflowOrcModule.run(workflowId, {
+          throwOnError: false,
+          logOnError: true,
+          transactionId,
+        })
+
+        const result = await done
+
+        expect(result).toEqual([
+          "result from step 0",
+          "result from step 1",
+          "result from step 2",
+          "result from step 3",
+          "result from step 4",
+          "result from step 5",
+        ])
+      })
+
+      it("should manage saving multiple async steps in concurrency without background execution while setting steps as success manually concurrently", async () => {
+        const step0 = createStep({ name: "step0", async: true }, async () => {})
+
+        const step1 = createStep({ name: "step1", async: true }, async () => {})
+
+        const step2 = createStep({ name: "step2", async: true }, async () => {})
+        const step3 = createStep({ name: "step3", async: true }, async () => {})
+
+        const step4 = createStep({ name: "step4", async: true }, async () => {})
+        const step5 = createStep({ name: "step5" }, async (all: any[]) => {
+          const ret = [...all, "result from step 5"]
+          return new StepResponse(ret)
+        })
+
+        const workflowId = "workflow-1" + ulid()
+        createWorkflow(
+          {
+            name: workflowId,
+            idempotent: true,
+            retentionTime: 1,
+          },
+          function () {
+            const all = parallelize(step0(), step1(), step2(), step3(), step4())
+            const res = step5(all)
+            return new WorkflowResponse(res)
+          }
+        )
+
+        const transactionId = ulid()
+        const done = new Promise<void>((resolve, reject) => {
+          void workflowOrcModule.subscribe({
+            workflowId: workflowId,
+            transactionId,
+            subscriber: async (event) => {
+              if (event.eventType === "onFinish") {
+                resolve(event.result)
+              }
+            },
+          })
+        })
+
+        await workflowOrcModule.run(workflowId, {
+          throwOnError: false,
+          logOnError: true,
+          transactionId,
+        })
+
+        await setTimeout(100) // Just to wait a bit before firering everything
+
+        for (let i = 0; i <= 4; i++) {
+          void workflowOrcModule.setStepSuccess({
+            idempotencyKey: {
+              workflowId: workflowId,
+              transactionId: transactionId,
+              stepId: `step${i}`,
+              action: TransactionHandlerType.INVOKE,
+            },
+            stepResponse: new StepResponse("result from step " + i),
+          })
+        }
+
+        const res = await done
+
+        expect(res).toEqual([
+          "result from step 0",
+          "result from step 1",
+          "result from step 2",
+          "result from step 3",
+          "result from step 4",
+          "result from step 5",
+        ])
+      })
+
+      it("should prevent race continuation of the workflow during retryIntervalAwaiting in background execution", async () => {
+        const transactionId = "transaction_id" + ulid()
+        const workflowId = "RACE_workflow-1" + ulid()
+
         const step0InvokeMock = jest.fn()
         const step1InvokeMock = jest.fn()
         const step2InvokeMock = jest.fn()
@@ -43,7 +194,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
 
         const step1 = createStep("step1", async (_) => {
           step1InvokeMock()
-          await setTimeout(2000)
+          await setTimeout(200)
           return new StepResponse({ isSuccess: true })
         })
 
@@ -57,57 +208,67 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
           return new WorkflowResponse(status)
         })
 
-        createWorkflow("workflow-1", function () {
+        createWorkflow(
-          const build = step0()
+          {
-
+            name: workflowId,
-          const status = subWorkflow.runAsStep({} as any).config({
+            idempotent: true,
-            async: true,
+            retentionTime: 5,
-            compensateAsync: true,
-            backgroundExecution: true,
-            retryIntervalAwaiting: 1,
-          })
-
-          const transformedResult = transform({ status }, (data) => {
-            transformMock()
-            return {
-              status: data.status,
-            }
-          })
-
-          step2(transformedResult)
-          return new WorkflowResponse(build)
-        })
-
-        void workflowOrcModule.subscribe({
-          workflowId: "workflow-1",
-
-          subscriber: (event) => {
-            if (event.eventType === "onFinish") {
-              expect(step0InvokeMock).toHaveBeenCalledTimes(1)
-              expect(step1InvokeMock.mock.calls.length).toBeGreaterThan(1)
-              expect(step2InvokeMock).toHaveBeenCalledTimes(1)
-              expect(transformMock).toHaveBeenCalledTimes(1)
-              done()
-              clearTimeout(timeout)
-            }
           },
+          function () {
+            const build = step0()
+
+            const status = subWorkflow.runAsStep({} as any).config({
+              async: true,
+              compensateAsync: true,
+              backgroundExecution: true,
+              retryIntervalAwaiting: 0.1,
+            })
+
+            const transformedResult = transform({ status }, (data) => {
+              transformMock()
+              return {
+                status: data.status,
+              }
+            })
+
+            step2(transformedResult)
+            return new WorkflowResponse(build)
+          }
+        )
+
+        const onFinish = new Promise<void>((resolve) => {
+          void workflowOrcModule.subscribe({
+            workflowId,
+            transactionId,
+            subscriber: (event) => {
+              if (event.eventType === "onFinish") {
+                resolve()
+              }
+            },
+          })
         })
 
         workflowOrcModule
-          .run("workflow-1", { throwOnError: false })
+          .run(workflowId, {
+            transactionId,
+            throwOnError: false,
+            logOnError: true,
+          })
           .then(({ result }) => {
             expect(result).toBe("result from step 0")
           })
-          .catch((e) => e)
 
-        const timeout = failTrap(
+        await onFinish
-          done,
+
-          "should prevent race continuation of the workflow during retryIntervalAwaiting in background execution"
+        expect(step0InvokeMock).toHaveBeenCalledTimes(1)
-        )
+        expect(step1InvokeMock.mock.calls.length).toBeGreaterThan(1)
+        expect(step2InvokeMock).toHaveBeenCalledTimes(1)
+        expect(transformMock).toHaveBeenCalledTimes(1)
       })
 
-      it("should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution", (done) => {
+      it("should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution", async () => {
-        const workflowId = "RACE_workflow-1"
+        const transactionId = "transaction_id" + ulid()
+        const workflowId = "RACE_workflow-1" + ulid()
 
         const step0InvokeMock = jest.fn()
         const step0CompensateMock = jest.fn()
@@ -131,7 +292,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
           "RACE_step1",
           async (_) => {
             step1InvokeMock()
-            await setTimeout(300)
+            await setTimeout(1000)
             throw new Error("error from step 1")
           },
           () => {
@@ -149,56 +310,63 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
           return new WorkflowResponse(status)
         })
 
-        createWorkflow(workflowId, function () {
+        createWorkflow(
-          const build = step0()
+          {
-
+            name: workflowId,
-          const status = subWorkflow.runAsStep({} as any).config({
-            async: true,
-            compensateAsync: true,
-            backgroundExecution: true,
-            retryIntervalAwaiting: 0.1,
-          })
-
-          const transformedResult = transform({ status }, (data) => {
-            transformMock()
-            return {
-              status: data.status,
-            }
-          })
-
-          step2(transformedResult)
-          return new WorkflowResponse(build)
-        })
-
-        void workflowOrcModule.subscribe({
-          workflowId: workflowId,
-          subscriber: async (event) => {
-            if (event.eventType === "onFinish") {
-              expect(step0InvokeMock).toHaveBeenCalledTimes(1)
-              expect(step0CompensateMock).toHaveBeenCalledTimes(1)
-              expect(step1InvokeMock.mock.calls.length).toBeGreaterThan(2)
-              expect(step1CompensateMock).toHaveBeenCalledTimes(1)
-              expect(step2InvokeMock).toHaveBeenCalledTimes(0)
-              expect(transformMock).toHaveBeenCalledTimes(0)
-              done()
-              clearTimeout(timeout)
-            }
           },
+          function () {
+            const build = step0()
+
+            const status = subWorkflow.runAsStep({} as any).config({
+              async: true,
+              compensateAsync: true,
+              backgroundExecution: true,
+              retryIntervalAwaiting: 0.1,
+              maxAwaitingRetries: 3,
+            })
+
+            const transformedResult = transform({ status }, (data) => {
+              transformMock()
+              return {
+                status: data.status,
+              }
+            })
+
+            step2(transformedResult)
+            return new WorkflowResponse(build)
+          }
+        )
+
+        const onFinish = new Promise<void>((resolve) => {
+          void workflowOrcModule.subscribe({
+            workflowId,
+            transactionId,
+            subscriber: async (event) => {
+              if (event.eventType === "onFinish") {
+                resolve()
+              }
+            },
+          })
         })
 
-        workflowOrcModule
+        await workflowOrcModule
           .run(workflowId, {
+            transactionId,
             throwOnError: false,
+            logOnError: true,
           })
           .then(({ result }) => {
             expect(result).toBe("result from step 0")
           })
-          .catch((e) => e)
 
-        const timeout = failTrap(
+        await onFinish
-          done,
+
-          "should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution"
+        expect(step0InvokeMock).toHaveBeenCalledTimes(1)
-        )
+        expect(step0CompensateMock).toHaveBeenCalledTimes(1)
+        expect(step1InvokeMock).toHaveBeenCalledTimes(3)
+        expect(step1CompensateMock.mock.calls.length).toBeGreaterThan(0)
+        expect(step2InvokeMock).toHaveBeenCalledTimes(0)
+        expect(transformMock).toHaveBeenCalledTimes(0)
       })
     })
   },

packages/modules/workflow-engine-inmemory/integration-tests/__tests__/subscribe.spec.ts

@@ -0,0 +1,185 @@
+import { IWorkflowEngineService } from "@medusajs/framework/types"
+import { Modules } from "@medusajs/framework/utils"
+import {
+  createStep,
+  createWorkflow,
+  StepResponse,
+  WorkflowResponse,
+} from "@medusajs/framework/workflows-sdk"
+import { moduleIntegrationTestRunner } from "@medusajs/test-utils"
+import { setTimeout as setTimeoutSync } from "timers"
+import { setTimeout as setTimeoutPromise } from "timers/promises"
+import { ulid } from "ulid"
+import "../__fixtures__"
+
+jest.setTimeout(60000)
+
+moduleIntegrationTestRunner<IWorkflowEngineService>({
+  moduleName: Modules.WORKFLOW_ENGINE,
+  resolve: __dirname + "/../..",
+  testSuite: ({ service: workflowOrcModule }) => {
+    describe("Workflow Orchestrator module subscribe", function () {
+      it("should subscribe to a workflow and receive the response when it finishes", async () => {
+        const step1 = createStep({ name: "step1" }, async () => {
+          return new StepResponse("step1")
+        })
+        const step2 = createStep({ name: "step2" }, async () => {
+          await setTimeoutPromise(1000)
+          return new StepResponse("step2")
+        })
+
+        const workflowId = "workflow" + ulid()
+        createWorkflow(workflowId, function (input) {
+          step1()
+          step2().config({
+            async: true,
+          })
+          return new WorkflowResponse("workflow")
+        })
+
+        const step1_1 = createStep({ name: "step1_1" }, async () => {
+          return new StepResponse("step1_1")
+        })
+        const step2_1 = createStep({ name: "step2_1" }, async () => {
+          await setTimeoutPromise(1000)
+          return new StepResponse("step2_1")
+        })
+
+        const workflow2Id = "workflow_2" + ulid()
+        createWorkflow(workflow2Id, function (input) {
+          step1_1()
+          step2_1().config({
+            async: true,
+          })
+          return new WorkflowResponse("workflow_2")
+        })
+
+        const transactionId = "trx_123" + ulid()
+        const transactionId2 = "trx_124" + ulid()
+
+        const onWorkflowFinishSpy = jest.fn()
+
+        const onWorkflowFinishPromise = new Promise<void>((resolve) => {
+          void workflowOrcModule.subscribe({
+            workflowId: workflowId,
+            transactionId,
+            subscriber: (event) => {
+              if (event.eventType === "onFinish") {
+                onWorkflowFinishSpy()
+                workflowOrcModule.run(workflow2Id, {
+                  transactionId: transactionId2,
+                })
+                resolve()
+              }
+            },
+          })
+        })
+
+        const onWorkflow2FinishSpy = jest.fn()
+
+        const workflow2FinishPromise = new Promise<void>((resolve) => {
+          void workflowOrcModule.subscribe({
+            workflowId: workflow2Id,
+            subscriber: (event) => {
+              if (event.eventType === "onFinish") {
+                onWorkflow2FinishSpy()
+                resolve()
+              }
+            },
+          })
+        })
+
+        workflowOrcModule.run(workflowId, {
+          transactionId,
+        })
+
+        await onWorkflowFinishPromise
+        await workflow2FinishPromise
+
+        expect(onWorkflowFinishSpy).toHaveBeenCalledTimes(1)
+        expect(onWorkflow2FinishSpy).toHaveBeenCalledTimes(1)
+      })
+
+      it("should subscribe to a workflow and receive the response when it finishes (2)", async () => {
+        const step1 = createStep({ name: "step1" }, async () => {
+          return new StepResponse("step1")
+        })
+        const step2 = createStep({ name: "step2" }, async () => {
+          await setTimeoutPromise(1000)
+          return new StepResponse("step2")
+        })
+
+        const workflowId = "workflow" + ulid()
+        createWorkflow(workflowId, function (input) {
+          step1()
+          step2().config({
+            async: true,
+          })
+          return new WorkflowResponse("workflow")
+        })
+
+        const step1_1 = createStep({ name: "step1_1" }, async () => {
+          return new StepResponse("step1_1")
+        })
+        const step2_1 = createStep({ name: "step2_1" }, async () => {
+          await setTimeoutPromise(1000)
+          return new StepResponse("step2_1")
+        })
+
+        const workflow2Id = "workflow_2" + ulid()
+        createWorkflow(workflow2Id, function (input) {
+          step1_1()
+          step2_1().config({
+            async: true,
+          })
+          return new WorkflowResponse("workflow_2")
+        })
+
+        const transactionId = "trx_123" + ulid()
+        const transactionId2 = "trx_124" + ulid()
+
+        const onWorkflowFinishSpy = jest.fn()
+
+        const onWorkflowFinishPromise = new Promise<void>((resolve) => {
+          void workflowOrcModule.subscribe({
+            workflowId: workflowId,
+            transactionId,
+            subscriber: (event) => {
+              if (event.eventType === "onFinish") {
+                onWorkflowFinishSpy()
+                workflowOrcModule.run(workflow2Id, {
+                  transactionId: transactionId2,
+                })
+                resolve()
+              }
+            },
+          })
+        })
+
+        const onWorkflow2FinishSpy = jest.fn()
+
+        const workflow2FinishPromise = new Promise<void>((resolve) => {
+          void workflowOrcModule.subscribe({
+            workflowId: workflow2Id,
+            subscriber: (event) => {
+              if (event.eventType === "onFinish") {
+                onWorkflow2FinishSpy()
+                resolve()
+              }
+            },
+          })
+        })
+
+        workflowOrcModule.run(workflowId, {
+          transactionId,
+        })
+
+        await onWorkflowFinishPromise
+        await workflow2FinishPromise
+
+        expect(onWorkflowFinishSpy).toHaveBeenCalledTimes(1)
+        expect(onWorkflow2FinishSpy).toHaveBeenCalledTimes(1)
+      })
+    })
+  },
+})

packages/modules/workflow-engine-inmemory/package.json

@@ -29,7 +29,7 @@
     "resolve:aliases": "tsc --showConfig -p tsconfig.json > tsconfig.resolved.json && tsc-alias -p tsconfig.resolved.json && rimraf tsconfig.resolved.json",
     "build": "rimraf dist && tsc --build && npm run resolve:aliases",
     "test": "jest --passWithNoTests --bail --forceExit -- src",
-    "test:integration": "jest  --forceExit -- integration-tests/**/__tests__/*.ts",
+    "test:integration": "jest --forceExit -- integration-tests/**/__tests__/**/*.spec.ts",
     "migration:initial": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:create --initial",
     "migration:create": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:create",
     "migration:up": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:up",

packages/modules/workflow-engine-inmemory/src/services/workflow-orchestrator.ts

@@ -100,7 +100,7 @@ type Subscribers = Map<WorkflowId, TransactionSubscribers>
 const AnySubscriber = "any"
 
 export class WorkflowOrchestratorService {
-  private subscribers: Subscribers = new Map()
+  private static subscribers: Subscribers = new Map()
   private container_: MedusaContainer
   private inMemoryDistributedTransactionStorage_: InMemoryDistributedTransactionStorage
   readonly #logger: Logger
@@ -110,7 +110,6 @@ export class WorkflowOrchestratorService {
     sharedContainer,
   }: {
     inMemoryDistributedTransactionStorage: InMemoryDistributedTransactionStorage
-    workflowOrchestratorService: WorkflowOrchestratorService
     sharedContainer: MedusaContainer
   }) {
     this.container_ = sharedContainer
@@ -133,9 +132,18 @@ export class WorkflowOrchestratorService {
     await this.inMemoryDistributedTransactionStorage_.onApplicationShutdown()
   }
 
-  private async triggerParentStep(transaction, result) {
+  private async triggerParentStep(transaction, result, errors) {
     const metadata = transaction.flow.metadata
-    const { parentStepIdempotencyKey } = metadata ?? {}
+    const { parentStepIdempotencyKey, cancelingFromParentStep } = metadata ?? {}
+
+    if (cancelingFromParentStep) {
+      /**
+       * If the sub workflow is cancelling from a parent step, we don't want to trigger the parent
+       * step.
+       */
+      return
+    }
+
     if (parentStepIdempotencyKey) {
       const hasFailed = [
         TransactionState.REVERTED,
@@ -145,12 +153,18 @@ export class WorkflowOrchestratorService {
       if (hasFailed) {
         await this.setStepFailure({
           idempotencyKey: parentStepIdempotencyKey,
-          stepResponse: result,
+          stepResponse: errors,
+          options: {
+            logOnError: true,
+          },
         })
       } else {
         await this.setStepSuccess({
           idempotencyKey: parentStepIdempotencyKey,
           stepResponse: result,
+          options: {
+            logOnError: true,
+          },
         })
       }
     }
@@ -237,7 +251,7 @@ export class WorkflowOrchestratorService {
         errors,
       })
 
-      await this.triggerParentStep(ret.transaction, result)
+      await this.triggerParentStep(ret.transaction, result, errors)
     }
 
     if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -349,7 +363,7 @@ export class WorkflowOrchestratorService {
         errors,
       })
 
-      await this.triggerParentStep(ret.transaction, result)
+      await this.triggerParentStep(ret.transaction, result, errors)
     }
 
     if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -443,7 +457,7 @@ export class WorkflowOrchestratorService {
         errors,
       })
 
-      await this.triggerParentStep(ret.transaction, result)
+      await this.triggerParentStep(ret.transaction, result, errors)
     }
 
     if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -514,7 +528,7 @@ export class WorkflowOrchestratorService {
         errors,
       })
 
-      await this.triggerParentStep(ret.transaction, result)
+      await this.triggerParentStep(ret.transaction, result, errors)
     }
 
     if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -587,7 +601,7 @@ export class WorkflowOrchestratorService {
         errors,
       })
 
-      await this.triggerParentStep(ret.transaction, result)
+      await this.triggerParentStep(ret.transaction, result, errors)
     }
 
     if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -608,7 +622,8 @@ export class WorkflowOrchestratorService {
     subscriberId,
   }: SubscribeOptions) {
     subscriber._id = subscriberId
-    const subscribers = this.subscribers.get(workflowId) ?? new Map()
+    const subscribers =
+      WorkflowOrchestratorService.subscribers.get(workflowId) ?? new Map()
 
     const handlerIndex = (handlers) => {
       return handlers.findIndex(
@@ -625,7 +640,7 @@ export class WorkflowOrchestratorService {
 
       transactionSubscribers.push(subscriber)
       subscribers.set(transactionId, transactionSubscribers)
-      this.subscribers.set(workflowId, subscribers)
+      WorkflowOrchestratorService.subscribers.set(workflowId, subscribers)
       return
     }
 
@@ -637,7 +652,7 @@ export class WorkflowOrchestratorService {
 
     workflowSubscribers.push(subscriber)
     subscribers.set(AnySubscriber, workflowSubscribers)
-    this.subscribers.set(workflowId, subscribers)
+    WorkflowOrchestratorService.subscribers.set(workflowId, subscribers)
   }
 
   unsubscribe({
@@ -645,7 +660,8 @@ export class WorkflowOrchestratorService {
     transactionId,
     subscriberOrId,
   }: UnsubscribeOptions) {
-    const subscribers = this.subscribers.get(workflowId) ?? new Map()
+    const subscribers =
+      WorkflowOrchestratorService.subscribers.get(workflowId) ?? new Map()
 
     const filterSubscribers = (handlers: SubscriberHandler[]) => {
       return handlers.filter((handler) => {
@@ -665,7 +681,7 @@ export class WorkflowOrchestratorService {
       } else {
         subscribers.delete(transactionId)
       }
-      this.subscribers.set(workflowId, subscribers)
+      WorkflowOrchestratorService.subscribers.set(workflowId, subscribers)
       return
     }
 
@@ -676,7 +692,7 @@ export class WorkflowOrchestratorService {
     } else {
       subscribers.delete(AnySubscriber)
     }
-    this.subscribers.set(workflowId, subscribers)
+    WorkflowOrchestratorService.subscribers.set(workflowId, subscribers)
   }
 
   private notify(options: NotifyOptions) {
@@ -687,7 +703,7 @@ export class WorkflowOrchestratorService {
   private async processSubscriberNotifications(options: NotifyOptions) {
     const { workflowId, transactionId, eventType } = options
     const subscribers: TransactionSubscribers =
-      this.subscribers.get(workflowId) ?? new Map()
+      WorkflowOrchestratorService.subscribers.get(workflowId) ?? new Map()
 
     const notifySubscribersAsync = async (handlers: SubscriberHandler[]) => {
       const promises = handlers.map(async (handler) => {

packages/modules/workflow-engine-inmemory/src/utils/workflow-orchestrator-storage.ts

@@ -20,10 +20,10 @@ import {
   ModulesSdkTypes,
 } from "@medusajs/framework/types"
 import {
+  isPresent,
   MedusaError,
   TransactionState,
   TransactionStepState,
-  isPresent,
 } from "@medusajs/framework/utils"
 import { WorkflowOrchestratorService } from "@services"
 import { type CronExpression, parseExpression } from "cron-parser"
@@ -31,6 +31,23 @@ import { WorkflowExecution } from "../models/workflow-execution"
 
 const THIRTY_MINUTES_IN_MS = 1000 * 60 * 30
 
+const doneStates = [
+  TransactionStepState.DONE,
+  TransactionStepState.REVERTED,
+  TransactionStepState.FAILED,
+  TransactionStepState.SKIPPED,
+  TransactionStepState.SKIPPED_FAILURE,
+  TransactionStepState.TIMEOUT,
+]
+
+const finishedStates = [
+  TransactionState.DONE,
+  TransactionState.FAILED,
+  TransactionState.REVERTED,
+]
+
+const failedStates = [TransactionState.FAILED, TransactionState.REVERTED]
+
 function calculateDelayFromExpression(expression: CronExpression): number {
   const nextTime = expression.next().getTime()
   const now = Date.now()
@@ -71,24 +88,6 @@ function parseNextExecution(
   return result
 }
 
-const invokingStatesSet = new Set([
-  TransactionStepState.INVOKING,
-  TransactionStepState.NOT_STARTED,
-])
-
-const compensatingStatesSet = new Set([
-  TransactionStepState.COMPENSATING,
-  TransactionStepState.NOT_STARTED,
-])
-
-function isInvokingState(step: TransactionStep) {
-  return invokingStatesSet.has(step.invoke?.state)
-}
-
-function isCompensatingState(step: TransactionStep) {
-  return compensatingStatesSet.has(step.compensate?.state)
-}
-
 export class InMemoryDistributedTransactionStorage
   implements IDistributedTransactionStorage, IDistributedSchedulerStorage
 {
@@ -96,8 +95,7 @@ export class InMemoryDistributedTransactionStorage
   private logger_: Logger
   private workflowOrchestratorService_: WorkflowOrchestratorService
 
-  private storage: Map<string, Omit<TransactionCheckpoint, "context">> =
+  private storage: Record<string, TransactionCheckpoint> = {}
-    new Map()
   private scheduled: Map<
     string,
     {
@@ -112,6 +110,7 @@ export class InMemoryDistributedTransactionStorage
   private pendingTimers: Set<NodeJS.Timeout> = new Set()
 
   private clearTimeout_: NodeJS.Timeout
+  private isLocked: Map<string, boolean> = new Map()
 
   constructor({
     workflowExecutionService,
@@ -179,29 +178,11 @@ export class InMemoryDistributedTransactionStorage
 
   private async saveToDb(data: TransactionCheckpoint, retentionTime?: number) {
     const isNotStarted = data.flow.state === TransactionState.NOT_STARTED
-    const isFinished = [
+    const asyncVersion = data.flow._v
-      TransactionState.DONE,
+    const isFinished = finishedStates.includes(data.flow.state)
-      TransactionState.FAILED,
-      TransactionState.REVERTED,
-    ].includes(data.flow.state)
     const isWaitingToCompensate =
       data.flow.state === TransactionState.WAITING_TO_COMPENSATE
 
-    /**
-     * Bit of explanation:
-     *
-     * When a workflow run, it run all sync step in memory until it reaches a async step.
-     * In that case, it might handover to another process to continue the execution. Thats why
-     * we need to save the current state of the flow. Then from there, it will run again all
-     * sync steps until the next async step. an so on so forth.
-     *
-     * To summarize, we only trully need to save the data when we are reaching any steps that
-     * trigger a handover to a potential other process.
-     *
-     * This allows us to spare some resources and time by not over communicating with the external
-     * database when it is not really needed
-     */
-
     const isFlowInvoking = data.flow.state === TransactionState.INVOKING
 
     const stepsArray = Object.values(data.flow.steps) as TransactionStep[]
@@ -240,7 +221,8 @@ export class InMemoryDistributedTransactionStorage
 
     if (
       !(isNotStarted || isFinished || isWaitingToCompensate) &&
-      !currentStepsIsAsync
+      !currentStepsIsAsync &&
+      !asyncVersion
     ) {
       return
     }
@@ -295,15 +277,14 @@ export class InMemoryDistributedTransactionStorage
         .catch(() => undefined)
 
     if (trx) {
-      const { flow, errors } = this.storage.get(key) ?? {}
+      const { flow, errors } = this.storage[key]
+        ? JSON.parse(JSON.stringify(this.storage[key]))
+        : {}
       const { idempotent } = options ?? {}
       const execution = trx.execution as TransactionFlow
 
       if (!idempotent) {
-        const isFailedOrReverted = [
+        const isFailedOrReverted = failedStates.includes(execution.state)
-          TransactionState.REVERTED,
-          TransactionState.FAILED,
-        ].includes(execution.state)
 
         const isDone = execution.state === TransactionState.DONE
 
@@ -321,11 +302,11 @@ export class InMemoryDistributedTransactionStorage
         }
       }
 
-      return {
+      return new TransactionCheckpoint(
-        flow: flow ?? (trx.execution as TransactionFlow),
+        flow ?? (trx?.execution as TransactionFlow),
-        context: trx.context?.data as TransactionContext,
+        trx?.context?.data as TransactionContext,
-        errors: errors ?? (trx.context?.errors as TransactionStepError[]),
+        errors ?? (trx?.context?.errors as TransactionStepError[])
-      }
+      )
     }
 
     return
@@ -336,68 +317,94 @@ export class InMemoryDistributedTransactionStorage
     data: TransactionCheckpoint,
     ttl?: number,
     options?: TransactionOptions
-  ): Promise<void> {
+  ): Promise<TransactionCheckpoint> {
-    /**
+    if (this.isLocked.has(key)) {
-     * Store the retention time only if the transaction is done, failed or reverted.
+      throw new Error("Transaction storage is locked")
-     * From that moment, this tuple can be later on archived or deleted after the retention time.
-     */
-    const hasFinished = [
-      TransactionState.DONE,
-      TransactionState.FAILED,
-      TransactionState.REVERTED,
-    ].includes(data.flow.state)
-
-    const { retentionTime } = options ?? {}
-
-    await this.#preventRaceConditionExecutionIfNecessary({
-      data,
-      key,
-      options,
-    })
-
-    // Only store retention time if it's provided
-    if (retentionTime) {
-      Object.assign(data, {
-        retention_time: retentionTime,
-      })
     }
 
-    // Store in memory
+    this.isLocked.set(key, true)
-    const isNotStarted = data.flow.state === TransactionState.NOT_STARTED
-    const isManualTransactionId = !data.flow.transactionId.startsWith("auto-")
 
-    if (isNotStarted && isManualTransactionId) {
+    try {
-      const storedData = this.storage.get(key)
+      /**
-      if (storedData) {
+       * Store the retention time only if the transaction is done, failed or reverted.
-        throw new SkipExecutionError(
+       * From that moment, this tuple can be later on archived or deleted after the retention time.
-          "Transaction already started for transactionId: " +
+       */
-            data.flow.transactionId
+      const { retentionTime } = options ?? {}
-        )
+
+      const hasFinished = finishedStates.includes(data.flow.state)
+
+      let cachedCheckpoint: TransactionCheckpoint | undefined
+      const getCheckpoint = async (options?: TransactionOptions) => {
+        if (!cachedCheckpoint) {
+          cachedCheckpoint = await this.get(key, options)
+        }
+        return cachedCheckpoint
       }
-    }
 
-    const { flow, errors } = data
+      await this.#preventRaceConditionExecutionIfNecessary({
-    this.storage.set(key, {
+        data,
-      flow,
+        key,
-      errors,
+        options,
-    })
+        getCheckpoint,
+      })
 
-    // Optimize DB operations - only perform when necessary
+      // Only store retention time if it's provided
-    if (hasFinished) {
+      if (retentionTime) {
-      if (!retentionTime) {
+        Object.assign(data, {
-        // If the workflow is nested, we cant just remove it because it would break the compensation algorithm. Instead, it will get deleted when the top level parent is deleted.
+          retention_time: retentionTime,
-        if (!flow.metadata?.parentStepIdempotencyKey) {
+        })
-          await this.deleteFromDb(data)
+      }
+
+      // Store in memory
+      const isNotStarted = data.flow.state === TransactionState.NOT_STARTED
+      const isManualTransactionId = !data.flow.transactionId.startsWith("auto-")
+
+      if (isNotStarted && isManualTransactionId) {
+        const storedData = this.storage[key]
+        if (storedData) {
+          throw new SkipExecutionError(
+            "Transaction already started for transactionId: " +
+              data.flow.transactionId
+          )
+        }
+      }
+
+      if (data.flow._v) {
+        const storedData = await this.get(key, {
+          isCancelling: !!data.flow.cancelledAt,
+        } as any)
+
+        TransactionCheckpoint.mergeCheckpoints(data, storedData)
+      }
+
+      const { flow, errors } = data
+
+      this.storage[key] = {
+        flow,
+        context: {} as TransactionContext,
+        errors,
+      } as TransactionCheckpoint
+
+      // Optimize DB operations - only perform when necessary
+      if (hasFinished) {
+        if (!retentionTime) {
+          if (!flow.metadata?.parentStepIdempotencyKey) {
+            await this.deleteFromDb(data)
+          } else {
+            await this.saveToDb(data, retentionTime)
+          }
         } else {
           await this.saveToDb(data, retentionTime)
         }
+
+        delete this.storage[key]
       } else {
         await this.saveToDb(data, retentionTime)
       }
 
-      this.storage.delete(key)
+      return data
-    } else {
+    } finally {
-      await this.saveToDb(data, retentionTime)
+      this.isLocked.delete(key)
     }
   }
 
@@ -405,28 +412,25 @@ export class InMemoryDistributedTransactionStorage
     data,
     key,
     options,
+    getCheckpoint,
   }: {
     data: TransactionCheckpoint
     key: string
     options?: TransactionOptions
+    getCheckpoint: (
+      options: TransactionOptions
+    ) => Promise<TransactionCheckpoint | undefined>
   }) {
-    // TODO: comment, we have been able to try to replace this entire function
-    // with a locking first approach. We might come back to that another time.
-    // This remove the necessity of all the below logic to prevent race conditions
-    // by preventing the exact same execution to run at the same time.
-    // See early commits from: https://github.com/medusajs/medusa/pull/13345/commits
-
     const isInitialCheckpoint = [TransactionState.NOT_STARTED].includes(
       data.flow.state
     )
-
     /**
      * In case many execution can succeed simultaneously, we need to ensure that the latest
      * execution does continue if a previous execution is considered finished
      */
     const currentFlow = data.flow
 
-    const rawData = this.storage.get(key)
+    const rawData = this.storage[key]
     let data_ = {} as TransactionCheckpoint
     if (rawData) {
       data_ = rawData as TransactionCheckpoint
@@ -437,13 +441,37 @@ export class InMemoryDistributedTransactionStorage
       } as Parameters<typeof this.get>[1]
 
       data_ =
-        (await this.get(key, getOptions)) ??
+        (await getCheckpoint(getOptions as TransactionOptions)) ??
         ({ flow: {} } as TransactionCheckpoint)
     }
 
     const { flow: latestUpdatedFlow } = data_
+    if (options?.stepId) {
+      const stepId = options.stepId
+      const currentStep = data.flow.steps[stepId]
+      const latestStep = latestUpdatedFlow.steps?.[stepId]
+      if (latestStep && currentStep) {
+        const isCompensating = data.flow.state === TransactionState.COMPENSATING
 
-    if (!isInitialCheckpoint && !isPresent(latestUpdatedFlow)) {
+        const latestState = isCompensating
+          ? latestStep.compensate?.state
+          : latestStep.invoke?.state
+
+        const shouldSkip = doneStates.includes(latestState)
+
+        if (shouldSkip) {
+          throw new SkipStepAlreadyFinishedError(
+            `Step ${stepId} already finished by another execution`
+          )
+        }
+      }
+    }
+
+    if (
+      !isInitialCheckpoint &&
+      !isPresent(latestUpdatedFlow) &&
+      !data.flow.metadata?.parentStepIdempotencyKey
+    ) {
       /**
        * the initial checkpoint expect no other checkpoint to have been stored.
        * In case it is not the initial one and another checkpoint is trying to
@@ -453,54 +481,7 @@ export class InMemoryDistributedTransactionStorage
       throw new SkipExecutionError("Already finished by another execution")
     }
 
-    let currentFlowLatestExecutedStep: TransactionStep | undefined
+    // Ensure that the latest execution was not cancelled, otherwise we skip the execution
-    const currentFlowSteps = Object.values(currentFlow.steps || {})
-    for (let i = currentFlowSteps.length - 1; i >= 0; i--) {
-      if (currentFlowSteps[i].lastAttempt) {
-        currentFlowLatestExecutedStep = currentFlowSteps[i]
-        break
-      }
-    }
-
-    let latestUpdatedFlowLatestExecutedStep: TransactionStep | undefined
-    const latestUpdatedFlowSteps = Object.values(latestUpdatedFlow.steps || {})
-    for (let i = latestUpdatedFlowSteps.length - 1; i >= 0; i--) {
-      if (latestUpdatedFlowSteps[i].lastAttempt) {
-        latestUpdatedFlowLatestExecutedStep = latestUpdatedFlowSteps[i]
-        break
-      }
-    }
-
-    /**
-     * The current flow and the latest updated flow have the same latest executed step.
-     */
-    const isSameLatestExecutedStep =
-      currentFlowLatestExecutedStep &&
-      latestUpdatedFlowLatestExecutedStep &&
-      currentFlowLatestExecutedStep?.id ===
-        latestUpdatedFlowLatestExecutedStep?.id
-
-    /**
-     * The current flow's latest executed step has a last attempt ahead of the latest updated
-     * flow's latest executed step. Therefor it is fine, otherwise another execution has already
-     * finished the step.
-     */
-    const isCurrentLatestExecutedStepLastAttemptAhead =
-      currentFlowLatestExecutedStep?.lastAttempt &&
-      latestUpdatedFlowLatestExecutedStep?.lastAttempt &&
-      currentFlowLatestExecutedStep.lastAttempt >=
-        latestUpdatedFlowLatestExecutedStep.lastAttempt
-
-    if (
-      isSameLatestExecutedStep &&
-      !isCurrentLatestExecutedStepLastAttemptAhead
-    ) {
-      throw new SkipStepAlreadyFinishedError(
-        "Step already in execution ahead of the current one"
-      )
-    }
-
-    // First ensure that the latest execution was not cancelled, otherwise we skip the execution
     const latestTransactionCancelledAt = latestUpdatedFlow.cancelledAt
     const currentTransactionCancelledAt = currentFlow.cancelledAt
 
@@ -512,86 +493,6 @@ export class InMemoryDistributedTransactionStorage
         "Workflow execution has been cancelled during the execution"
       )
     }
-
-    const currentFlowLastInvokingStepIndex =
-      currentFlowSteps.findIndex(isInvokingState)
-
-    let latestUpdatedFlowLastInvokingStepIndex = !latestUpdatedFlow.steps
-      ? 1 // There is no other execution, so the current execution is the latest
-      : -1
-
-    if (latestUpdatedFlow.steps) {
-      for (let i = 0; i < latestUpdatedFlowSteps.length; i++) {
-        if (isInvokingState(latestUpdatedFlowSteps[i])) {
-          latestUpdatedFlowLastInvokingStepIndex = i
-          break
-        }
-      }
-    }
-
-    let currentFlowLastCompensatingStepIndex = -1
-    for (let i = currentFlowSteps.length - 1; i >= 0; i--) {
-      if (isCompensatingState(currentFlowSteps[i])) {
-        currentFlowLastCompensatingStepIndex = currentFlowSteps.length - 1 - i
-        break
-      }
-    }
-
-    let latestUpdatedFlowLastCompensatingStepIndex = !latestUpdatedFlow.steps
-      ? -1 // There is no other execution, so the current execution is the latest
-      : -1
-
-    if (latestUpdatedFlow.steps) {
-      for (let i = latestUpdatedFlowSteps.length - 1; i >= 0; i--) {
-        if (isCompensatingState(latestUpdatedFlowSteps[i])) {
-          latestUpdatedFlowLastCompensatingStepIndex =
-            latestUpdatedFlowSteps.length - 1 - i
-          break
-        }
-      }
-    }
-
-    const isLatestExecutionFinishedIndex = -1
-    const invokeShouldBeSkipped =
-      (latestUpdatedFlowLastInvokingStepIndex ===
-        isLatestExecutionFinishedIndex ||
-        currentFlowLastInvokingStepIndex <
-          latestUpdatedFlowLastInvokingStepIndex) &&
-      currentFlowLastInvokingStepIndex !== isLatestExecutionFinishedIndex
-
-    const compensateShouldBeSkipped =
-      currentFlowLastCompensatingStepIndex <
-        latestUpdatedFlowLastCompensatingStepIndex &&
-      currentFlowLastCompensatingStepIndex !== isLatestExecutionFinishedIndex &&
-      latestUpdatedFlowLastCompensatingStepIndex !==
-        isLatestExecutionFinishedIndex
-
-    const isCompensatingMismatch =
-      latestUpdatedFlow.state === TransactionState.COMPENSATING &&
-      ![TransactionState.REVERTED, TransactionState.FAILED].includes(
-        currentFlow.state
-      ) &&
-      currentFlow.state !== latestUpdatedFlow.state
-
-    const isRevertedMismatch =
-      latestUpdatedFlow.state === TransactionState.REVERTED &&
-      currentFlow.state !== TransactionState.REVERTED
-
-    const isFailedMismatch =
-      latestUpdatedFlow.state === TransactionState.FAILED &&
-      currentFlow.state !== TransactionState.FAILED
-
-    if (
-      (data.flow.state !== TransactionState.COMPENSATING &&
-        invokeShouldBeSkipped) ||
-      (data.flow.state === TransactionState.COMPENSATING &&
-        compensateShouldBeSkipped) ||
-      isCompensatingMismatch ||
-      isRevertedMismatch ||
-      isFailedMismatch
-    ) {
-      throw new SkipExecutionError("Already finished by another execution")
-    }
   }
 
   async scheduleRetry(
@@ -850,7 +751,7 @@ export class InMemoryDistributedTransactionStorage
       updated_at: {
         $lte: raw(
           (alias) =>
-            `CURRENT_TIMESTAMP - (INTERVAL '1 second' * retention_time)`
+            `CURRENT_TIMESTAMP - (INTERVAL '1 second' * "retention_time")`
         ),
       },
       state: {

packages/modules/workflow-engine-redis/integration-tests/__tests__/index.spec.ts

@@ -57,17 +57,7 @@ import {
 } from "../__fixtures__/workflow_1_manual_retry_step"
 import { TestDatabase } from "../utils"
 
-jest.setTimeout(300000)
+jest.setTimeout(30000)
-
-const failTrap = (done, name, timeout = 5000) => {
-  return setTimeoutSync(() => {
-    // REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
-    console.warn(
-      `Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
-    )
-    done()
-  }, timeout)
-}
 
 function times(num) {
   let resolver
@@ -109,6 +99,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
   testSuite: ({ service: workflowOrcModule, medusaApp }) => {
     describe("Workflow Orchestrator module", function () {
       beforeEach(async () => {
+        await TestDatabase.clearTables()
         jest.clearAllMocks()
 
         query = medusaApp.query
@@ -169,14 +160,14 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
 
       describe("Testing basic workflow", function () {
         describe("Cancel transaction", function () {
-          it("should cancel an ongoing execution with async unfinished yet step", (done) => {
+          it("should cancel an ongoing execution with async unfinished yet step", async () => {
             const transactionId = "transaction-to-cancel-id" + ulid()
             const step1 = createStep("step1", async () => {
               return new StepResponse("step1")
             })
 
             const step2 = createStep("step2", async () => {
-              await setTimeout(500)
+              await setTimeout(2000)
               return new StepResponse("step2")
             })
 
@@ -197,43 +188,37 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
               }
             )
 
-            workflowOrcModule
+            const onFinishPromise = new Promise<void>((resolve) => {
-              .run(workflowId, {
+              workflowOrcModule.subscribe({
-                input: {},
+                workflowId,
                 transactionId,
+                subscriber: async (event) => {
+                  if (event.eventType === "onFinish") {
+                    resolve()
+                  }
+                },
               })
-              .then(async () => {
+            })
-                await setTimeout(100)
 
-                await workflowOrcModule.cancel(workflowId, {
+            await workflowOrcModule.run(workflowId, {
-                  transactionId,
+              input: {},
-                })
+              transactionId,
+            })
 
-                workflowOrcModule.subscribe({
+            await setTimeout(100)
-                  workflowId,
-                  transactionId,
-                  subscriber: async (event) => {
-                    if (event.eventType === "onFinish") {
-                      const execution =
-                        await workflowOrcModule.listWorkflowExecutions({
-                          transaction_id: transactionId,
-                        })
 
-                      expect(execution.length).toEqual(1)
+            await workflowOrcModule.cancel(workflowId, {
-                      expect(execution[0].state).toEqual(
+              transactionId,
-                        TransactionState.REVERTED
+            })
-                      )
-                      done()
-                      clearTimeout(timeout)
-                    }
-                  },
-                })
-              })
 
-            const timeout = failTrap(
+            await onFinishPromise
-              done,
+
-              "should cancel an ongoing execution with async unfinished yet step"
+            const execution = await workflowOrcModule.listWorkflowExecutions({
-            )
+              transaction_id: transactionId,
+            })
+
+            expect(execution.length).toEqual(1)
+            expect(execution[0].state).toEqual(TransactionState.REVERTED)
           })
 
           it("should cancel a complete execution with a sync workflow running as async", async () => {
@@ -270,19 +255,29 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
               }
             )
 
+            const onFinishPromise = new Promise<void>((resolve) => {
+              workflowOrcModule.subscribe({
+                workflowId,
+                transactionId,
+                subscriber: (event) => {
+                  if (event.eventType === "onFinish") {
+                    resolve()
+                  }
+                },
+              })
+            })
+
             await workflowOrcModule.run(workflowId, {
               input: {},
               transactionId,
             })
 
-            await setTimeout(100)
+            await onFinishPromise
 
             await workflowOrcModule.cancel(workflowId, {
               transactionId,
             })
 
-            await setTimeout(500)
-
             const execution = await workflowOrcModule.listWorkflowExecutions({
               transaction_id: transactionId,
             })
@@ -397,116 +392,95 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
           })
         })
 
-        it("should manually retry a step that is taking too long to finish", (done) => {
+        it("should manually retry a step that is taking too long to finish", async () => {
           const transactionId = "transaction-manual-retry" + ulid()
           const workflowId = "workflow_1_manual_retry_step"
 
-          void workflowOrcModule
+          await workflowOrcModule.run(workflowId, {
-            .run(workflowId, {
+            input: {},
-              input: {},
-              transactionId,
-            })
-            .then(() => {
-              expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
-              expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(1)
-
-              void workflowOrcModule.retryStep({
-                idempotencyKey: {
-                  workflowId,
-                  transactionId,
-                  stepId: "step_2",
-                  action: "invoke",
-                },
-              })
-            })
-
-          workflowOrcModule.subscribe({
-            workflowId,
             transactionId,
-            subscriber: async (event) => {
+          })
-              if (event.eventType === "onFinish") {
+
-                expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
+          const onFinishPromise = new Promise<void>((resolve) => {
-                expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(2)
+            workflowOrcModule.subscribe({
-                done()
+              workflowId,
-                clearTimeout(timeout)
+              transactionId,
-              }
+              subscriber: async (event) => {
+                if (event.eventType === "onFinish") {
+                  expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
+                  expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(2)
+                  resolve()
+                }
+              },
+            })
+          })
+
+          expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
+          expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(1)
+
+          await workflowOrcModule.retryStep({
+            idempotencyKey: {
+              workflowId,
+              transactionId,
+              stepId: "step_2",
+              action: "invoke",
             },
           })
 
-          const timeout = failTrap(
+          await onFinishPromise
-            done,
-            "should manually retry a step that is taking too long to finish"
-          )
         })
 
-        it("should retry steps X times automatically when maxRetries is set", (done) => {
+        it("should retry steps X times automatically when maxRetries is set", async () => {
           const transactionId = "transaction-auto-retries" + ulid()
           const workflowId = "workflow_1_auto_retries"
 
+          const onFinishPromise = new Promise<void>((resolve) => {
+            workflowOrcModule.subscribe({
+              workflowId,
+              transactionId,
+              subscriber: async (event) => {
+                if (event.eventType === "onFinish") {
+                  resolve()
+                }
+              },
+            })
+          })
+
           void workflowOrcModule.run(workflowId, {
             input: {},
             transactionId,
           })
 
-          workflowOrcModule.subscribe({
+          await onFinishPromise
-            workflowId,
-            transactionId,
-            subscriber: async (event) => {
-              if (event.eventType === "onFinish") {
-                expect(step1InvokeMockAutoRetries).toHaveBeenCalledTimes(1)
-                expect(step2InvokeMockAutoRetries).toHaveBeenCalledTimes(3)
-                expect(step1CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
-                expect(step2CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
-                done()
-                clearTimeout(timeout)
-              }
-            },
-          })
 
-          const timeout = failTrap(
+          expect(step1InvokeMockAutoRetries).toHaveBeenCalledTimes(1)
-            done,
+          expect(step2InvokeMockAutoRetries).toHaveBeenCalledTimes(3)
-            "should retry steps X times automatically when maxRetries is set"
+          expect(step1CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
-          )
+          expect(step2CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
         })
 
         it("should not retry steps X times automatically when maxRetries is set and autoRetry is false", async () => {
           const transactionId = "transaction-auto-retries" + ulid()
           const workflowId = "workflow_1_auto_retries_false"
 
-          await workflowOrcModule.run(workflowId, {
-            input: {},
-            transactionId,
-            throwOnError: false,
-          })
-
           const onFinishPromise = new Promise<void>((resolve, reject) => {
             workflowOrcModule.subscribe({
               workflowId,
               transactionId,
               subscriber: async (event) => {
                 if (event.eventType === "onFinish") {
-                  try {
+                  resolve()
-                    expect(
-                      step1InvokeMockAutoRetriesFalse
-                    ).toHaveBeenCalledTimes(1)
-                    expect(
-                      step2InvokeMockAutoRetriesFalse
-                    ).toHaveBeenCalledTimes(3)
-                    expect(
-                      step1CompensateMockAutoRetriesFalse
-                    ).toHaveBeenCalledTimes(1)
-                    expect(
-                      step2CompensateMockAutoRetriesFalse
-                    ).toHaveBeenCalledTimes(1)
-                    resolve()
-                  } catch (error) {
-                    reject(error)
-                  }
                 }
               },
             })
           })
 
+          await workflowOrcModule.run(workflowId, {
+            input: {},
+            transactionId,
+            throwOnError: false,
+          })
+
           expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
           expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
           expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
@@ -529,6 +503,11 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
           })
 
           await onFinishPromise
+
+          expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
+          expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(3)
+          expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
+          expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
         })
 
         it("should prevent executing twice the same workflow in perfect concurrency with the same transactionId and non idempotent and not async but retention time is set", async () => {
@@ -585,8 +564,6 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
           })
 
           expect(executionsList).toHaveLength(1)
-
-          console.log(">>>>>>>>> setting step success")
           const { result } = await workflowOrcModule.setStepSuccess({
             idempotencyKey: {
               action: TransactionHandlerType.INVOKE,
@@ -597,7 +574,6 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
             stepResponse: { uhuuuu: "yeaah!" },
           })
 
-          console.log(">>>>>>>>> setting step success done")
           ;({ data: executionsList } = await query.graph({
             entity: "workflow_executions",
             fields: ["id"],
@@ -928,41 +904,52 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
           ).toBe(true)
         })
 
-        it("should complete an async workflow that returns a StepResponse", (done) => {
+        it("should complete an async workflow that returns a StepResponse", async () => {
           const transactionId = "transaction_1" + ulid()
-          workflowOrcModule
+
-            .run("workflow_async_background", {
+          const onFinishPromise = new Promise<void>((resolve) => {
+            void workflowOrcModule.subscribe({
+              workflowId: "workflow_async_background",
+              transactionId,
+              subscriber: (event) => {
+                if (event.eventType === "onFinish") {
+                  resolve()
+                }
+              },
+            })
+          })
+
+          const { transaction, result } = await workflowOrcModule.run(
+            "workflow_async_background",
+            {
               input: {
                 myInput: "123",
               },
               transactionId,
               throwOnError: true,
-            })
+            }
-            .then(({ transaction, result }: any) => {
+          )
-              expect(transaction.flow.state).toEqual(
-                TransactionStepState.INVOKING
-              )
-              expect(result).toEqual(undefined)
-            })
 
-          void workflowOrcModule.subscribe({
+          expect(transaction.flow.state).toEqual(TransactionStepState.INVOKING)
-            workflowId: "workflow_async_background",
+          expect(result).toEqual(undefined)
-            transactionId,
-            subscriber: (event) => {
-              if (event.eventType === "onFinish") {
-                done()
-                clearTimeout(timeout)
-              }
-            },
-          })
 
-          const timeout = failTrap(done, "workflow_async_background")
+          await onFinishPromise
         })
 
-        it("should subscribe to a async workflow and receive the response when it finishes", (done) => {
+        it("should subscribe to a async workflow and receive the response when it finishes", async () => {
           const transactionId = "trx_123" + ulid()
 
-          const onFinish = jest.fn()
+          const onFinishPromise = new Promise<void>((resolve) => {
+            void workflowOrcModule.subscribe({
+              workflowId: "workflow_async_background",
+              transactionId,
+              subscriber: (event) => {
+                if (event.eventType === "onFinish") {
+                  resolve()
+                }
+              },
+            })
+          })
 
           void workflowOrcModule.run("workflow_async_background", {
             input: {
@@ -972,25 +959,24 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
             throwOnError: false,
           })
 
-          void workflowOrcModule.subscribe({
+          await onFinishPromise
-            workflowId: "workflow_async_background",
-            transactionId,
-            subscriber: (event) => {
-              if (event.eventType === "onFinish") {
-                onFinish()
-                done()
-                clearTimeout(timeout)
-              }
-            },
-          })
-
-          expect(onFinish).toHaveBeenCalledTimes(0)
-
-          const timeout = failTrap(done, "workflow_async_background")
         })
 
-        it("should not skip step if condition is true", function (done) {
+        it("should not skip step if condition is true", async () => {
           const transactionId = "trx_123_when" + ulid()
+
+          const onFinishPromise = new Promise<void>((resolve) => {
+            void workflowOrcModule.subscribe({
+              workflowId: "wf-when",
+              transactionId,
+              subscriber: (event) => {
+                if (event.eventType === "onFinish") {
+                  resolve()
+                }
+              },
+            })
+          })
+
           void workflowOrcModule.run("wf-when", {
             input: {
               callSubFlow: true,
@@ -1000,23 +986,30 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
             logOnError: true,
           })
 
-          void workflowOrcModule.subscribe({
+          await onFinishPromise
-            workflowId: "wf-when",
-            transactionId,
-            subscriber: (event) => {
-              if (event.eventType === "onFinish") {
-                done()
-                clearTimeout(timeout)
-              }
-            },
-          })
-
-          const timeout = failTrap(done, "wf-when")
         })
 
-        it("should cancel an async sub workflow when compensating", (done) => {
+        it("should cancel an async sub workflow when compensating", async () => {
           const workflowId = "workflow_async_background_fail"
           const transactionId = "trx_123_compensate_async_sub_workflow" + ulid()
+
+          let onCompensateStepSuccess: { step: TransactionStep } | null = null
+
+          const onFinishPromise = new Promise<void>((resolve) => {
+            void workflowOrcModule.subscribe({
+              workflowId,
+              transactionId,
+              subscriber: (event) => {
+                if (event.eventType === "onCompensateStepSuccess") {
+                  onCompensateStepSuccess = event
+                }
+                if (event.eventType === "onFinish") {
+                  resolve()
+                }
+              },
+            })
+          })
+
           void workflowOrcModule.run(workflowId, {
             input: {
               callSubFlow: true,
@@ -1026,31 +1019,16 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
             logOnError: false,
           })
 
-          let onCompensateStepSuccess: { step: TransactionStep } | null = null
+          await onFinishPromise
 
-          void workflowOrcModule.subscribe({
+          expect(onCompensateStepSuccess).toBeDefined()
-            workflowId,
+          expect(onCompensateStepSuccess!.step.id).toEqual(
-            subscriber: (event) => {
+            "_root.nested_sub_flow_async_fail-as-step" // The workflow as step
-              if (event.eventType === "onCompensateStepSuccess") {
+          )
-                onCompensateStepSuccess = event
+          expect(onCompensateStepSuccess!.step.compensate).toEqual({
-              }
+            state: "reverted",
-              if (event.eventType === "onFinish") {
+            status: "ok",
-                expect(onCompensateStepSuccess).toBeDefined()
-                expect(onCompensateStepSuccess!.step.id).toEqual(
-                  "_root.nested_sub_flow_async_fail-as-step" // The workflow as step
-                )
-                expect(onCompensateStepSuccess!.step.compensate).toEqual({
-                  state: "reverted",
-                  status: "ok",
-                })
-
-                done()
-                clearTimeout(timeout)
-              }
-            },
           })
-
-          const timeout = failTrap(done, "workflow_async_background_fail")
         })
 
         it("should cancel and revert a completed workflow", async () => {

packages/modules/workflow-engine-redis/integration-tests/__tests__/race.spec.ts

@@ -1,32 +1,20 @@
 import { IWorkflowEngineService } from "@medusajs/framework/types"
-import { Modules } from "@medusajs/framework/utils"
+import { Modules, TransactionHandlerType } from "@medusajs/framework/utils"
 import {
   createStep,
   createWorkflow,
+  parallelize,
   StepResponse,
   transform,
   WorkflowResponse,
 } from "@medusajs/framework/workflows-sdk"
 import { moduleIntegrationTestRunner } from "@medusajs/test-utils"
-import { setTimeout as setTimeoutSync } from "timers"
 import { setTimeout } from "timers/promises"
 import { ulid } from "ulid"
 import "../__fixtures__"
-import { TestDatabase } from "../utils"
+import { TestDatabase } from "../utils/database"
 
-jest.setTimeout(300000)
+jest.setTimeout(20000)
-
-const failTrap = (done, name, timeout = 5000) => {
-  return setTimeoutSync(() => {
-    // REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
-    console.warn(
-      `Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
-    )
-    done()
-  }, timeout)
-}
-
-// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
 
 moduleIntegrationTestRunner<IWorkflowEngineService>({
   moduleName: Modules.WORKFLOW_ENGINE,
@@ -38,14 +26,244 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
   },
   testSuite: ({ service: workflowOrcModule, medusaApp }) => {
     describe("Testing race condition of the workflow during retry", () => {
+      beforeEach(async () => {
+        await TestDatabase.clearTables()
+        jest.clearAllMocks()
+      })
+
       afterEach(async () => {
         await TestDatabase.clearTables()
       })
 
-      it("should prevent race continuation of the workflow during retryIntervalAwaiting in background execution", (done) => {
+      it("should manage saving multiple async steps in concurrency", async () => {
-        const transactionId = "transaction_id" + ulid()
+        const step0 = createStep(
+          { name: "step0", async: true, backgroundExecution: true },
+          async () => {
+            return new StepResponse("result from step 0")
+          }
+        )
+
+        const step1 = createStep(
+          { name: "step1", async: true, backgroundExecution: true },
+          async () => {
+            return new StepResponse("result from step 1")
+          }
+        )
+
+        const step2 = createStep(
+          { name: "step2", async: true, backgroundExecution: true },
+          async () => {
+            return new StepResponse("result from step 2")
+          }
+        )
+        const step3 = createStep(
+          { name: "step3", async: true, backgroundExecution: true },
+          async () => {
+            return new StepResponse("result from step 3")
+          }
+        )
+
+        const step4 = createStep(
+          { name: "step4", async: true, backgroundExecution: true },
+          async () => {
+            return new StepResponse("result from step 4")
+          }
+        )
+        const step5 = createStep({ name: "step5" }, async (all: string[]) => {
+          const ret = [...all, "result from step 5"]
+          return new StepResponse(ret)
+        })
+
         const workflowId = "workflow-1" + ulid()
-        const subWorkflowId = "sub-" + workflowId
+        createWorkflow(
+          {
+            name: workflowId,
+            idempotent: true,
+            retentionTime: 5,
+          },
+          function () {
+            const all = parallelize(step0(), step1(), step2(), step3(), step4())
+            const res = step5(all)
+            return new WorkflowResponse(res)
+          }
+        )
+
+        const transactionId = ulid()
+        const done = new Promise<void>((resolve, reject) => {
+          void workflowOrcModule.subscribe({
+            workflowId: workflowId,
+            transactionId,
+            subscriber: async (event) => {
+              if (event.eventType === "onFinish") {
+                resolve(event.result)
+              }
+            },
+          })
+        })
+
+        await workflowOrcModule.run(workflowId, {
+          throwOnError: false,
+          logOnError: true,
+          transactionId,
+        })
+
+        const result = await done
+
+        expect(result).toEqual([
+          "result from step 0",
+          "result from step 1",
+          "result from step 2",
+          "result from step 3",
+          "result from step 4",
+          "result from step 5",
+        ])
+      })
+
+      it("should manage saving multiple sync steps in concurrency", async () => {
+        const step0 = createStep({ name: "step0" }, async () => {
+          return new StepResponse("result from step 0")
+        })
+
+        const step1 = createStep({ name: "step1" }, async () => {
+          return new StepResponse("result from step 1")
+        })
+
+        const step2 = createStep({ name: "step2" }, async () => {
+          return new StepResponse("result from step 2")
+        })
+        const step3 = createStep({ name: "step3" }, async () => {
+          return new StepResponse("result from step 3")
+        })
+
+        const step4 = createStep({ name: "step4" }, async () => {
+          return new StepResponse("result from step 4")
+        })
+        const step5 = createStep({ name: "step5" }, async (all: string[]) => {
+          const ret = [...all, "result from step 5"]
+          return new StepResponse(ret)
+        })
+
+        const workflowId = "workflow-1" + ulid()
+        createWorkflow(
+          {
+            name: workflowId,
+            idempotent: true,
+            retentionTime: 5,
+          },
+          function () {
+            const all = parallelize(step0(), step1(), step2(), step3(), step4())
+            const res = step5(all)
+            return new WorkflowResponse(res)
+          }
+        )
+
+        const transactionId = ulid()
+        const done = new Promise<void>((resolve, reject) => {
+          void workflowOrcModule.subscribe({
+            workflowId: workflowId,
+            transactionId,
+            subscriber: async (event) => {
+              if (event.eventType === "onFinish") {
+                resolve(event.result)
+              }
+            },
+          })
+        })
+
+        await workflowOrcModule.run(workflowId, {
+          throwOnError: false,
+          logOnError: true,
+          transactionId,
+        })
+
+        const result = await done
+
+        expect(result).toEqual([
+          "result from step 0",
+          "result from step 1",
+          "result from step 2",
+          "result from step 3",
+          "result from step 4",
+          "result from step 5",
+        ])
+      })
+
+      it("should manage saving multiple async steps in concurrency without background execution while setting steps as success manually concurrently", async () => {
+        const step0 = createStep({ name: "step0", async: true }, async () => {})
+
+        const step1 = createStep({ name: "step1", async: true }, async () => {})
+
+        const step2 = createStep({ name: "step2", async: true }, async () => {})
+        const step3 = createStep({ name: "step3", async: true }, async () => {})
+
+        const step4 = createStep({ name: "step4", async: true }, async () => {})
+        const step5 = createStep({ name: "step5" }, async (all: any[]) => {
+          const ret = [...all, "result from step 5"]
+          return new StepResponse(ret)
+        })
+
+        const workflowId = "workflow-1" + ulid()
+        createWorkflow(
+          {
+            name: workflowId,
+            idempotent: true,
+            retentionTime: 1,
+          },
+          function () {
+            const all = parallelize(step0(), step1(), step2(), step3(), step4())
+            const res = step5(all)
+            return new WorkflowResponse(res)
+          }
+        )
+
+        const transactionId = ulid()
+        const done = new Promise<void>((resolve, reject) => {
+          void workflowOrcModule.subscribe({
+            workflowId: workflowId,
+            transactionId,
+            subscriber: async (event) => {
+              if (event.eventType === "onFinish") {
+                resolve(event.result)
+              }
+            },
+          })
+        })
+
+        await workflowOrcModule.run(workflowId, {
+          throwOnError: false,
+          logOnError: true,
+          transactionId,
+        })
+
+        await setTimeout(100) // Just to wait a bit before firering everything
+
+        for (let i = 0; i <= 4; i++) {
+          void workflowOrcModule.setStepSuccess({
+            idempotencyKey: {
+              workflowId: workflowId,
+              transactionId: transactionId,
+              stepId: `step${i}`,
+              action: TransactionHandlerType.INVOKE,
+            },
+            stepResponse: new StepResponse("result from step " + i),
+          })
+        }
+
+        const res = await done
+
+        expect(res).toEqual([
+          "result from step 0",
+          "result from step 1",
+          "result from step 2",
+          "result from step 3",
+          "result from step 4",
+          "result from step 5",
+        ])
+      })
+
+      it("should prevent race continuation of the workflow during retryIntervalAwaiting in background execution", async () => {
+        const transactionId = "transaction_id" + ulid()
+        const workflowId = "RACE_workflow-1" + ulid()
 
         const step0InvokeMock = jest.fn()
         const step1InvokeMock = jest.fn()
@@ -59,7 +277,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
 
         const step1 = createStep("step1", async (_) => {
           step1InvokeMock()
-          await setTimeout(2000)
+          await setTimeout(1000)
           return new StepResponse({ isSuccess: true })
         })
 
@@ -68,70 +286,70 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
           return new StepResponse({ result: input })
         })
 
-        const subWorkflow = createWorkflow(subWorkflowId, function () {
+        const subWorkflow = createWorkflow("sub-workflow-1", function () {
           const status = step1()
           return new WorkflowResponse(status)
         })
 
-        createWorkflow(workflowId, function () {
+        createWorkflow(
-          const build = step0()
+          {
-
+            name: workflowId,
-          const status = subWorkflow.runAsStep({} as any).config({
+            idempotent: true,
-            async: true,
+            retentionTime: 5,
-            compensateAsync: true,
-            backgroundExecution: true,
-            retryIntervalAwaiting: 1,
-          })
-
-          const transformedResult = transform({ status }, (data) => {
-            transformMock()
-            return {
-              status: data.status,
-            }
-          })
-
-          step2(transformedResult)
-          return new WorkflowResponse(build)
-        })
-
-        void workflowOrcModule.subscribe({
-          workflowId,
-          transactionId,
-          subscriber: async (event) => {
-            if (event.eventType === "onFinish") {
-              try {
-                expect(step0InvokeMock).toHaveBeenCalledTimes(1)
-                expect(
-                  step1InvokeMock.mock.calls.length
-                ).toBeGreaterThanOrEqual(1)
-                expect(step2InvokeMock).toHaveBeenCalledTimes(1)
-                expect(transformMock).toHaveBeenCalledTimes(1)
-
-                // Prevent killing the test to early
-                await setTimeout(500)
-                done()
-              } catch (e) {
-                return done(e)
-              } finally {
-                clearTimeout(timeout)
-              }
-            }
           },
+          function () {
+            const build = step0()
+
+            const status = subWorkflow.runAsStep({} as any).config({
+              async: true,
+              compensateAsync: true,
+              backgroundExecution: true,
+              retryIntervalAwaiting: 0.1,
+            })
+
+            const transformedResult = transform({ status }, (data) => {
+              transformMock()
+              return {
+                status: data.status,
+              }
+            })
+
+            step2(transformedResult)
+            return new WorkflowResponse(build)
+          }
+        )
+
+        const onFinish = new Promise<void>((resolve) => {
+          void workflowOrcModule.subscribe({
+            workflowId,
+            transactionId,
+            subscriber: (event) => {
+              if (event.eventType === "onFinish") {
+                resolve()
+              }
+            },
+          })
         })
 
         workflowOrcModule
-          .run(workflowId, { transactionId })
+          .run(workflowId, {
+            transactionId,
+            throwOnError: false,
+            logOnError: true,
+          })
           .then(({ result }) => {
             expect(result).toBe("result from step 0")
           })
 
-        const timeout = failTrap(
+        await onFinish
-          done,
+
-          "should prevent race continuation of the workflow during retryIntervalAwaiting in background execution"
+        expect(step0InvokeMock).toHaveBeenCalledTimes(1)
-        )
+        expect(step1InvokeMock.mock.calls.length).toBeGreaterThan(1)
+        expect(step2InvokeMock).toHaveBeenCalledTimes(1)
+        expect(transformMock).toHaveBeenCalledTimes(1)
       })
 
-      it("should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution", (done) => {
+      it("should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution", async () => {
         const transactionId = "transaction_id" + ulid()
         const workflowId = "RACE_workflow-1" + ulid()
 
@@ -157,7 +375,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
           "RACE_step1",
           async (_) => {
             step1InvokeMock()
-            await setTimeout(500)
+            await setTimeout(1000)
             throw new Error("error from step 1")
           },
           () => {
@@ -175,61 +393,63 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
           return new WorkflowResponse(status)
         })
 
-        createWorkflow(workflowId, function () {
+        createWorkflow(
-          const build = step0()
+          {
-
+            name: workflowId,
-          const status = subWorkflow.runAsStep({} as any).config({
-            async: true,
-            compensateAsync: true,
-            backgroundExecution: true,
-            retryIntervalAwaiting: 0.1,
-          })
-
-          const transformedResult = transform({ status }, (data) => {
-            transformMock()
-            return {
-              status: data.status,
-            }
-          })
-
-          step2(transformedResult)
-          return new WorkflowResponse(build)
-        })
-
-        void workflowOrcModule.subscribe({
-          workflowId: workflowId,
-          transactionId,
-          subscriber: (event) => {
-            if (event.eventType === "onFinish") {
-              try {
-                expect(step0InvokeMock).toHaveBeenCalledTimes(1)
-                expect(step0CompensateMock).toHaveBeenCalledTimes(1)
-                expect(
-                  step1InvokeMock.mock.calls.length
-                ).toBeGreaterThanOrEqual(2) // Called every 0.1s at least (it can take more than 0.1sdepending on the event loop congestions)
-                expect(step1CompensateMock).toHaveBeenCalledTimes(1)
-                expect(step2InvokeMock).toHaveBeenCalledTimes(0)
-                expect(transformMock).toHaveBeenCalledTimes(0)
-                done()
-              } catch (e) {
-                return done(e)
-              } finally {
-                clearTimeout(timeout)
-              }
-            }
           },
+          function () {
+            const build = step0()
+
+            const status = subWorkflow.runAsStep({} as any).config({
+              async: true,
+              compensateAsync: true,
+              backgroundExecution: true,
+              retryIntervalAwaiting: 0.1,
+              maxAwaitingRetries: 3,
+            })
+
+            const transformedResult = transform({ status }, (data) => {
+              transformMock()
+              return {
+                status: data.status,
+              }
+            })
+
+            step2(transformedResult)
+            return new WorkflowResponse(build)
+          }
+        )
+
+        const onFinish = new Promise<void>((resolve) => {
+          void workflowOrcModule.subscribe({
+            workflowId,
+            transactionId,
+            subscriber: async (event) => {
+              if (event.eventType === "onFinish") {
+                resolve()
+              }
+            },
+          })
         })
 
-        workflowOrcModule
+        await workflowOrcModule
-          .run(workflowId, { transactionId, throwOnError: false })
+          .run(workflowId, {
+            transactionId,
+            throwOnError: false,
+            logOnError: true,
+          })
           .then(({ result }) => {
             expect(result).toBe("result from step 0")
           })
 
-        const timeout = failTrap(
+        await onFinish
-          done,
+
-          "should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution"
+        expect(step0InvokeMock).toHaveBeenCalledTimes(1)
-        )
+        expect(step0CompensateMock).toHaveBeenCalledTimes(1)
+        expect(step1InvokeMock).toHaveBeenCalledTimes(3)
+        expect(step1CompensateMock.mock.calls.length).toBeGreaterThan(0)
+        expect(step2InvokeMock).toHaveBeenCalledTimes(0)
+        expect(transformMock).toHaveBeenCalledTimes(0)
       })
     })
   },

packages/modules/workflow-engine-redis/integration-tests/utils/database.ts

@@ -30,20 +30,16 @@ async function deleteKeysByPattern(pattern) {
     count: 100,
   })
 
+  const pipeline = redis.pipeline()
   for await (const keys of stream) {
     if (keys.length) {
-      const pipeline = redis.pipeline()
       keys.forEach((key) => pipeline.unlink(key))
-      await pipeline.exec()
     }
   }
+  await pipeline.exec()
 }
 
 async function cleanRedis() {
-  try {
+  await deleteKeysByPattern("bull:*")
-    await deleteKeysByPattern("bull:*")
+  await deleteKeysByPattern("dtrx:*")
-    await deleteKeysByPattern("dtrx:*")
-  } catch (error) {
-    console.error("Error:", error)
-  }
 }

packages/modules/workflow-engine-redis/package.json

@@ -29,7 +29,7 @@
     "resolve:aliases": "tsc --showConfig -p tsconfig.json > tsconfig.resolved.json && tsc-alias -p tsconfig.resolved.json && rimraf tsconfig.resolved.json",
     "build": "rimraf dist && tsc --build && npm run resolve:aliases",
     "test": "jest --passWithNoTests --bail --forceExit -- src",
-    "test:integration": "jest  --forceExit -- integration-tests/**/__tests__/*.ts",
+    "test:integration": "jest  --forceExit --runInBand -- integration-tests/**/__tests__/index.spec.ts && jest --forceExit --runInBand -- integration-tests/**/__tests__/race.spec.ts",
     "migration:initial": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:create --initial",
     "migration:create": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:create",
     "migration:up": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:up",

packages/modules/workflow-engine-redis/src/services/workflow-orchestrator.ts

@@ -14,6 +14,7 @@ import {
 } from "@medusajs/framework/types"
 import {
   isString,
+  MedusaError,
   promiseAll,
   TransactionState,
 } from "@medusajs/framework/utils"
@@ -114,7 +115,7 @@ export class WorkflowOrchestratorService {
   protected redisPublisher: Redis
   protected redisSubscriber: Redis
   protected container_: MedusaContainer
-  private subscribers: Subscribers = new Map()
+  private static subscribers: Subscribers = new Map()
 
   readonly #logger: Logger
 
@@ -153,7 +154,7 @@ export class WorkflowOrchestratorService {
 
     this.redisSubscriber.on("message", async (channel, message) => {
       const workflowId = channel.split(":")[1]
-      if (!this.subscribers.has(workflowId)) return
+      if (!WorkflowOrchestratorService.subscribers.has(workflowId)) return
 
       try {
         const { instanceId, data } = JSON.parse(message)
@@ -177,9 +178,17 @@ export class WorkflowOrchestratorService {
     await this.redisDistributedTransactionStorage_.onApplicationStart()
   }
 
-  private async triggerParentStep(transaction, result) {
+  private async triggerParentStep(transaction, result, errors) {
     const metadata = transaction.flow.metadata
-    const { parentStepIdempotencyKey } = metadata ?? {}
+    const { parentStepIdempotencyKey, cancelingFromParentStep } = metadata ?? {}
+
+    if (cancelingFromParentStep) {
+      /**
+       * If the sub workflow is cancelling from a parent step, we don't want to trigger the parent
+       * step.
+       */
+      return
+    }
 
     if (parentStepIdempotencyKey) {
       const hasFailed = [
@@ -190,7 +199,7 @@ export class WorkflowOrchestratorService {
       if (hasFailed) {
         await this.setStepFailure({
           idempotencyKey: parentStepIdempotencyKey,
-          stepResponse: result,
+          stepResponse: errors,
           options: {
             logOnError: true,
           },
@@ -224,13 +233,16 @@ export class WorkflowOrchestratorService {
 
     throwOnError ??= true
     context ??= {}
-    context.transactionId = transactionId ?? ulid()
+    context.transactionId = transactionId ?? "auto-" + ulid()
     const workflowId = isString(workflowIdOrWorkflow)
       ? workflowIdOrWorkflow
       : workflowIdOrWorkflow.getName()
 
     if (!workflowId) {
-      throw new Error("Workflow ID is required")
+      throw new MedusaError(
+        MedusaError.Types.NOT_FOUND,
+        `Workflow ID is required`
+      )
     }
 
     const events: FlowRunOptions["events"] = this.buildWorkflowEvents({
@@ -241,11 +253,14 @@ export class WorkflowOrchestratorService {
 
     const exportedWorkflow = MedusaWorkflow.getWorkflow(workflowId)
     if (!exportedWorkflow) {
-      throw new Error(`Workflow with id "${workflowId}" not found.`)
+      throw new MedusaError(
+        MedusaError.Types.NOT_FOUND,
+        `Workflow with id "${workflowId}" not found.`
+      )
     }
 
+    const { onFinish, ...restEvents } = events
     const originalOnFinishHandler = events.onFinish!
-    delete events.onFinish
 
     const ret = await exportedWorkflow.run({
       input,
@@ -253,7 +268,7 @@ export class WorkflowOrchestratorService {
       logOnError,
       resultFrom,
       context,
-      events,
+      events: restEvents,
       container: container ?? this.container_,
     })
 
@@ -283,7 +298,7 @@ export class WorkflowOrchestratorService {
         errors,
       })
 
-      await this.triggerParentStep(ret.transaction, result)
+      await this.triggerParentStep(ret.transaction, result, errors)
     }
 
     if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -327,7 +342,10 @@ export class WorkflowOrchestratorService {
 
     const exportedWorkflow = MedusaWorkflow.getWorkflow(workflowId)
     if (!exportedWorkflow) {
-      throw new Error(`Workflow with id "${workflowId}" not found.`)
+      throw new MedusaError(
+        MedusaError.Types.NOT_FOUND,
+        `Workflow with id "${workflowId}" not found.`
+      )
     }
 
     const transaction = await this.getRunningTransaction(
@@ -354,12 +372,15 @@ export class WorkflowOrchestratorService {
       transactionId: transactionId,
     })
 
+    const { onFinish, ...restEvents } = events
+    const originalOnFinishHandler = events.onFinish!
+
     const ret = await exportedWorkflow.cancel({
       transaction,
       throwOnError: false,
       logOnError,
       context,
-      events,
+      events: restEvents,
       container: container ?? this.container_,
     })
 
@@ -382,17 +403,13 @@ export class WorkflowOrchestratorService {
     if (hasFinished) {
       const { result, errors } = ret
 
-      this.notify({
+      await originalOnFinishHandler({
-        isFlowAsync: ret.transaction.getFlow().hasAsyncSteps,
+        transaction: ret.transaction,
-        eventType: "onFinish",
-        workflowId,
-        transactionId: transaction.transactionId,
-        state: transactionState as TransactionState,
         result,
         errors,
       })
 
-      await this.triggerParentStep(ret.transaction, result)
+      await this.triggerParentStep(ret.transaction, result, errors)
     }
 
     if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -464,29 +481,28 @@ export class WorkflowOrchestratorService {
       workflowId,
     })
 
+    const { onFinish, ...restEvents } = events
+    const originalOnFinishHandler = events.onFinish!
+
     const ret = await exportedWorkflow.retryStep({
       idempotencyKey: idempotencyKey_,
       context,
       throwOnError: false,
       logOnError,
-      events,
+      events: restEvents,
       container: container ?? this.container_,
     })
 
     if (ret.transaction.hasFinished()) {
       const { result, errors } = ret
 
-      this.notify({
+      await originalOnFinishHandler({
-        isFlowAsync: ret.transaction.getFlow().hasAsyncSteps,
+        transaction: ret.transaction,
-        eventType: "onFinish",
-        workflowId,
-        transactionId,
-        state: ret.transaction.getFlow().state as TransactionState,
         result,
         errors,
       })
 
-      await this.triggerParentStep(ret.transaction, result)
+      await this.triggerParentStep(ret.transaction, result, errors)
     }
 
     if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -534,13 +550,16 @@ export class WorkflowOrchestratorService {
       workflowId,
     })
 
+    const { onFinish, ...restEvents } = events
+    const originalOnFinishHandler = events.onFinish!
+
     const ret = await exportedWorkflow.registerStepSuccess({
       idempotencyKey: idempotencyKey_,
       context,
       resultFrom,
       throwOnError: false,
       logOnError,
-      events,
+      events: restEvents,
       response: stepResponse,
       container: container ?? this.container_,
     })
@@ -548,17 +567,13 @@ export class WorkflowOrchestratorService {
     if (ret.transaction.hasFinished()) {
       const { result, errors } = ret
 
-      this.notify({
+      await originalOnFinishHandler({
-        isFlowAsync: ret.transaction.getFlow().hasAsyncSteps,
+        transaction: ret.transaction,
-        eventType: "onFinish",
-        workflowId,
-        transactionId,
-        state: ret.transaction.getFlow().state as TransactionState,
         result,
         errors,
       })
 
-      await this.triggerParentStep(ret.transaction, result)
+      await this.triggerParentStep(ret.transaction, result, errors)
     }
 
     if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -607,13 +622,16 @@ export class WorkflowOrchestratorService {
       workflowId,
     })
 
+    const { onFinish, ...restEvents } = events
+    const originalOnFinishHandler = events.onFinish!
+
     const ret = await exportedWorkflow.registerStepFailure({
       idempotencyKey: idempotencyKey_,
       context,
       resultFrom,
       throwOnError: false,
       logOnError,
-      events,
+      events: restEvents,
       response: stepResponse,
       container: container ?? this.container_,
       forcePermanentFailure,
@@ -622,17 +640,13 @@ export class WorkflowOrchestratorService {
     if (ret.transaction.hasFinished()) {
       const { result, errors } = ret
 
-      this.notify({
+      await originalOnFinishHandler({
-        isFlowAsync: ret.transaction.getFlow().hasAsyncSteps,
+        transaction: ret.transaction,
-        eventType: "onFinish",
-        workflowId,
-        transactionId,
-        state: ret.transaction.getFlow().state as TransactionState,
         result,
         errors,
       })
 
-      await this.triggerParentStep(ret.transaction, result)
+      await this.triggerParentStep(ret.transaction, result, errors)
     }
 
     if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -653,10 +667,11 @@ export class WorkflowOrchestratorService {
     subscriberId,
   }: SubscribeOptions) {
     subscriber._id = subscriberId
-    const subscribers = this.subscribers.get(workflowId) ?? new Map()
+    const subscribers =
+      WorkflowOrchestratorService.subscribers.get(workflowId) ?? new Map()
 
     // Subscribe instance to redis
-    if (!this.subscribers.has(workflowId)) {
+    if (!WorkflowOrchestratorService.subscribers.has(workflowId)) {
       void this.redisSubscriber.subscribe(this.getChannelName(workflowId))
     }
 
@@ -675,7 +690,7 @@ export class WorkflowOrchestratorService {
 
       transactionSubscribers.push(subscriber)
       subscribers.set(transactionId, transactionSubscribers)
-      this.subscribers.set(workflowId, subscribers)
+      WorkflowOrchestratorService.subscribers.set(workflowId, subscribers)
       return
     }
 
@@ -687,7 +702,7 @@ export class WorkflowOrchestratorService {
 
     workflowSubscribers.push(subscriber)
     subscribers.set(AnySubscriber, workflowSubscribers)
-    this.subscribers.set(workflowId, subscribers)
+    WorkflowOrchestratorService.subscribers.set(workflowId, subscribers)
   }
 
   unsubscribe({
@@ -695,7 +710,7 @@ export class WorkflowOrchestratorService {
     transactionId,
     subscriberOrId,
   }: UnsubscribeOptions) {
-    const subscribers = this.subscribers.get(workflowId)
+    const subscribers = WorkflowOrchestratorService.subscribers.get(workflowId)
     if (!subscribers) {
       return
     }
@@ -735,7 +750,7 @@ export class WorkflowOrchestratorService {
     }
 
     if (subscribers.size === 0) {
-      this.subscribers.delete(workflowId)
+      WorkflowOrchestratorService.subscribers.delete(workflowId)
       void this.redisSubscriber.unsubscribe(this.getChannelName(workflowId))
     }
   }
@@ -774,7 +789,7 @@ export class WorkflowOrchestratorService {
   private async processSubscriberNotifications(options: NotifyOptions) {
     const { workflowId, transactionId, eventType } = options
     const subscribers: TransactionSubscribers =
-      this.subscribers.get(workflowId) ?? new Map()
+      WorkflowOrchestratorService.subscribers.get(workflowId) ?? new Map()
 
     const notifySubscribersAsync = async (handlers: SubscriberHandler[]) => {
       const promises = handlers.map(async (handler) => {
@@ -886,6 +901,9 @@ export class WorkflowOrchestratorService {
         await notify({
           eventType: "onFinish",
           isFlowAsync: transaction.getFlow().hasAsyncSteps,
+          result,
+          errors,
+          state: transaction.getFlow().state as TransactionState,
         })
       },
 

packages/modules/workflow-engine-redis/src/utils/workflow-orchestrator-storage.ts

@@ -36,24 +36,22 @@ enum JobType {
 const THIRTY_MINUTES_IN_MS = 1000 * 60 * 30
 const REPEATABLE_CLEARER_JOB_ID = "clear-expired-executions"
 
-const invokingStatesSet = new Set([
+const doneStates = [
-  TransactionStepState.INVOKING,
+  TransactionStepState.DONE,
-  TransactionStepState.NOT_STARTED,
+  TransactionStepState.REVERTED,
-])
+  TransactionStepState.FAILED,
+  TransactionStepState.SKIPPED,
+  TransactionStepState.SKIPPED_FAILURE,
+  TransactionStepState.TIMEOUT,
+]
 
-const compensatingStatesSet = new Set([
+const finishedStates = [
-  TransactionStepState.COMPENSATING,
+  TransactionState.DONE,
-  TransactionStepState.NOT_STARTED,
+  TransactionState.FAILED,
-])
+  TransactionState.REVERTED,
-
+]
-function isInvokingState(step: TransactionStep) {
-  return invokingStatesSet.has(step.invoke?.state)
-}
-
-function isCompensatingState(step: TransactionStep) {
-  return compensatingStatesSet.has(step.compensate?.state)
-}
 
+const failedStates = [TransactionState.FAILED, TransactionState.REVERTED]
 export class RedisDistributedTransactionStorage
   implements IDistributedTransactionStorage, IDistributedSchedulerStorage
 {
@@ -152,11 +150,17 @@ export class RedisDistributedTransactionStorage
           } with the following data: ${JSON.stringify(job.data)}`
         )
         if (allowedJobs.includes(job.name as JobType)) {
-          await this.executeTransaction(
+          try {
-            job.data.workflowId,
+            await this.executeTransaction(
-            job.data.transactionId,
+              job.data.workflowId,
-            job.data.transactionMetadata
+              job.data.transactionId,
-          )
+              job.data.transactionMetadata
+            )
+          } catch (error) {
+            if (!SkipExecutionError.isSkipExecutionError(error)) {
+              throw error
+            }
+          }
         }
 
         if (job.name === JobType.SCHEDULE) {
@@ -273,29 +277,12 @@ export class RedisDistributedTransactionStorage
 
   private async saveToDb(data: TransactionCheckpoint, retentionTime?: number) {
     const isNotStarted = data.flow.state === TransactionState.NOT_STARTED
-    const isFinished = [
+    const asyncVersion = data.flow._v
-      TransactionState.DONE,
+
-      TransactionState.FAILED,
+    const isFinished = finishedStates.includes(data.flow.state)
-      TransactionState.REVERTED,
-    ].includes(data.flow.state)
     const isWaitingToCompensate =
       data.flow.state === TransactionState.WAITING_TO_COMPENSATE
 
-    /**
-     * Bit of explanation:
-     *
-     * When a workflow run, it run all sync step in memory until it reaches a async step.
-     * In that case, it might handover to another process to continue the execution. Thats why
-     * we need to save the current state of the flow. Then from there, it will run again all
-     * sync steps until the next async step. an so on so forth.
-     *
-     * To summarize, we only trully need to save the data when we are reaching any steps that
-     * trigger a handover to a potential other process.
-     *
-     * This allows us to spare some resources and time by not over communicating with the external
-     * database when it is not really needed
-     */
-
     const isFlowInvoking = data.flow.state === TransactionState.INVOKING
 
     const stepsArray = Object.values(data.flow.steps) as TransactionStep[]
@@ -334,7 +321,8 @@ export class RedisDistributedTransactionStorage
 
     if (
       !(isNotStarted || isFinished || isWaitingToCompensate) &&
-      !currentStepsIsAsync
+      !currentStepsIsAsync &&
+      !asyncVersion
     ) {
       return
     }
@@ -440,10 +428,7 @@ export class RedisDistributedTransactionStorage
       const execution = trx.execution as TransactionFlow
 
       if (!idempotent) {
-        const isFailedOrReverted = [
+        const isFailedOrReverted = failedStates.includes(execution.state)
-          TransactionState.REVERTED,
-          TransactionState.FAILED,
-        ].includes(execution.state)
 
         const isDone = execution.state === TransactionState.DONE
 
@@ -461,11 +446,11 @@ export class RedisDistributedTransactionStorage
         }
       }
 
-      return {
+      return new TransactionCheckpoint(
-        flow: flow ?? (trx.execution as TransactionFlow),
+        flow ?? (trx.execution as TransactionFlow),
-        context: trx.context?.data as TransactionContext,
+        trx.context?.data as TransactionContext,
-        errors: errors ?? (trx.context?.errors as TransactionStepError[]),
+        errors ?? (trx.context?.errors as TransactionStepError[])
-      }
+      )
     }
 
     return
@@ -476,94 +461,120 @@ export class RedisDistributedTransactionStorage
     data: TransactionCheckpoint,
     ttl?: number,
     options?: TransactionOptions
-  ): Promise<void> {
+  ): Promise<TransactionCheckpoint> {
     /**
      * Store the retention time only if the transaction is done, failed or reverted.
-     * From that moment, this tuple can be later on archived or deleted after the retention time.
      */
-    const hasFinished = [
-      TransactionState.DONE,
-      TransactionState.FAILED,
-      TransactionState.REVERTED,
-    ].includes(data.flow.state)
-
     const { retentionTime } = options ?? {}
 
-    await this.#preventRaceConditionExecutionIfNecessary({
+    let lockAcquired = false
-      data,
-      key,
-      options,
-    })
 
-    if (hasFinished && retentionTime) {
+    if (data.flow._v) {
-      Object.assign(data, {
+      lockAcquired = await this.#acquireLock(key)
-        retention_time: retentionTime,
-      })
-    }
 
-    // Only set if not exists
+      if (!lockAcquired) {
-    const shouldSetNX =
+        throw new Error("Lock not acquired")
-      data.flow.state === TransactionState.NOT_STARTED &&
-      !data.flow.transactionId.startsWith("auto-")
-
-    // Prepare operations to be executed in batch or pipeline
-    const data_ = {
-      errors: data.errors,
-      flow: data.flow,
-    }
-    const stringifiedData = JSON.stringify(data_)
-    const pipeline = this.redisClient.pipeline()
-
-    // Execute Redis operations
-    if (!hasFinished) {
-      if (ttl) {
-        if (shouldSetNX) {
-          pipeline.set(key, stringifiedData, "EX", ttl, "NX")
-        } else {
-          pipeline.set(key, stringifiedData, "EX", ttl)
-        }
-      } else {
-        if (shouldSetNX) {
-          pipeline.set(key, stringifiedData, "NX")
-        } else {
-          pipeline.set(key, stringifiedData)
-        }
       }
-    } else {
+
-      pipeline.unlink(key)
+      const storedData = await this.get(key, {
+        isCancelling: !!data.flow.cancelledAt,
+      } as any)
+
+      TransactionCheckpoint.mergeCheckpoints(data, storedData)
     }
 
-    const execPipeline = () => {
+    try {
-      return pipeline.exec().then((result) => {
+      const hasFinished = finishedStates.includes(data.flow.state)
-        if (!shouldSetNX) {
-          return result
-        }
 
-        const actionResult = result?.pop()
+      let cachedCheckpoint: TransactionCheckpoint | undefined
-        const isOk = !!actionResult?.pop()
+      const getCheckpoint = async (options?: TransactionOptions) => {
-        if (!isOk) {
+        if (!cachedCheckpoint) {
-          throw new SkipExecutionError(
+          cachedCheckpoint = await this.get(key, options)
-            "Transaction already started for transactionId: " +
-              data.flow.transactionId
-          )
         }
+        return cachedCheckpoint
+      }
 
-        return result
+      await this.#preventRaceConditionExecutionIfNecessary({
+        data: data,
+        key,
+        options,
+        getCheckpoint,
       })
-    }
 
-    // Database operations
+      // Only set if not exists
-    if (hasFinished && !retentionTime) {
+      const shouldSetNX =
-      // If the workflow is nested, we cant just remove it because it would break the compensation algorithm. Instead, it will get deleted when the top level parent is deleted.
+        data.flow.state === TransactionState.NOT_STARTED &&
-      if (!data.flow.metadata?.parentStepIdempotencyKey) {
+        !data.flow.transactionId.startsWith("auto-")
-        await promiseAll([execPipeline(), this.deleteFromDb(data)])
+
+      if (retentionTime) {
+        Object.assign(data, {
+          retention_time: retentionTime,
+        })
+      }
+
+      const execPipeline = () => {
+        const lightData_ = {
+          errors: data.errors,
+          flow: data.flow,
+        }
+        const stringifiedData = JSON.stringify(lightData_)
+
+        const pipeline = this.redisClient.pipeline()
+
+        if (!hasFinished) {
+          if (ttl) {
+            if (shouldSetNX) {
+              pipeline.set(key, stringifiedData, "EX", ttl, "NX")
+            } else {
+              pipeline.set(key, stringifiedData, "EX", ttl)
+            }
+          } else {
+            if (shouldSetNX) {
+              pipeline.set(key, stringifiedData, "NX")
+            } else {
+              pipeline.set(key, stringifiedData)
+            }
+          }
+        } else {
+          pipeline.unlink(key)
+        }
+
+        return pipeline.exec().then((result) => {
+          if (!shouldSetNX) {
+            return result
+          }
+
+          const actionResult = result?.pop()
+          const isOk = !!actionResult?.pop()
+          if (!isOk) {
+            throw new SkipExecutionError(
+              "Transaction already started for transactionId: " +
+                data.flow.transactionId
+            )
+          }
+
+          return result
+        })
+      }
+
+      if (hasFinished && !retentionTime) {
+        if (!data.flow.metadata?.parentStepIdempotencyKey) {
+          await this.deleteFromDb(data)
+          await execPipeline()
+        } else {
+          await this.saveToDb(data, retentionTime)
+          await execPipeline()
+        }
       } else {
         await this.saveToDb(data, retentionTime)
         await execPipeline()
       }
-    } else {
+
-      await this.saveToDb(data, retentionTime)
+      return data as TransactionCheckpoint
-      await execPipeline()
+    } finally {
+      if (lockAcquired) {
+        await this.#releaseLock(key)
+      }
     }
   }
 
@@ -750,19 +761,47 @@ export class RedisDistributedTransactionStorage
     )
   }
 
+  /**
+   * Generate a lock key for the given transaction key
+   */
+  #getLockKey(key: string): string {
+    return `${key}:lock`
+  }
+
+  async #acquireLock(key: string, ttlSeconds: number = 2): Promise<boolean> {
+    const lockKey = this.#getLockKey(key)
+
+    const result = await this.redisClient.set(
+      lockKey,
+      1,
+      "EX",
+      ttlSeconds,
+      "NX"
+    )
+    return result === "OK"
+  }
+
+  async #releaseLock(key: string): Promise<void> {
+    const lockKey = this.#getLockKey(key)
+    await this.redisClient.del(lockKey)
+  }
+
   async #preventRaceConditionExecutionIfNecessary({
     data,
     key,
     options,
+    getCheckpoint,
   }: {
     data: TransactionCheckpoint
     key: string
     options?: TransactionOptions
+    getCheckpoint: (
+      options: TransactionOptions
+    ) => Promise<TransactionCheckpoint | undefined>
   }) {
     const isInitialCheckpoint = [TransactionState.NOT_STARTED].includes(
       data.flow.state
     )
-
     /**
      * In case many execution can succeed simultaneously, we need to ensure that the latest
      * execution does continue if a previous execution is considered finished
@@ -780,13 +819,37 @@ export class RedisDistributedTransactionStorage
       } as Parameters<typeof this.get>[1]
 
       data_ =
-        (await this.get(key, getOptions)) ??
+        (await getCheckpoint(getOptions as TransactionOptions)) ??
         ({ flow: {} } as TransactionCheckpoint)
     }
 
     const { flow: latestUpdatedFlow } = data_
+    if (options?.stepId) {
+      const stepId = options.stepId
+      const currentStep = data.flow.steps[stepId]
+      const latestStep = latestUpdatedFlow.steps?.[stepId]
+      if (latestStep && currentStep) {
+        const isCompensating = data.flow.state === TransactionState.COMPENSATING
 
-    if (!isInitialCheckpoint && !isPresent(latestUpdatedFlow)) {
+        const latestState = isCompensating
+          ? latestStep.compensate?.state
+          : latestStep.invoke?.state
+
+        const shouldSkip = doneStates.includes(latestState)
+
+        if (shouldSkip) {
+          throw new SkipStepAlreadyFinishedError(
+            `Step ${stepId} already finished by another execution`
+          )
+        }
+      }
+    }
+
+    if (
+      !isInitialCheckpoint &&
+      !isPresent(latestUpdatedFlow) &&
+      !data.flow.metadata?.parentStepIdempotencyKey
+    ) {
       /**
        * the initial checkpoint expect no other checkpoint to have been stored.
        * In case it is not the initial one and another checkpoint is trying to
@@ -796,54 +859,7 @@ export class RedisDistributedTransactionStorage
       throw new SkipExecutionError("Already finished by another execution")
     }
 
-    let currentFlowLatestExecutedStep: TransactionStep | undefined
+    // Ensure that the latest execution was not cancelled, otherwise we skip the execution
-    const currentFlowSteps = Object.values(currentFlow.steps || {})
-    for (let i = currentFlowSteps.length - 1; i >= 0; i--) {
-      if (currentFlowSteps[i].lastAttempt) {
-        currentFlowLatestExecutedStep = currentFlowSteps[i]
-        break
-      }
-    }
-
-    let latestUpdatedFlowLatestExecutedStep: TransactionStep | undefined
-    const latestUpdatedFlowSteps = Object.values(latestUpdatedFlow.steps || {})
-    for (let i = latestUpdatedFlowSteps.length - 1; i >= 0; i--) {
-      if (latestUpdatedFlowSteps[i].lastAttempt) {
-        latestUpdatedFlowLatestExecutedStep = latestUpdatedFlowSteps[i]
-        break
-      }
-    }
-
-    /**
-     * The current flow and the latest updated flow have the same latest executed step.
-     */
-    const isSameLatestExecutedStep =
-      currentFlowLatestExecutedStep &&
-      latestUpdatedFlowLatestExecutedStep &&
-      currentFlowLatestExecutedStep?.id ===
-        latestUpdatedFlowLatestExecutedStep?.id
-
-    /**
-     * The current flow's latest executed step has a last attempt ahead of the latest updated
-     * flow's latest executed step. Therefor it is fine, otherwise another execution has already
-     * finished the step.
-     */
-    const isCurrentLatestExecutedStepLastAttemptAhead =
-      currentFlowLatestExecutedStep?.lastAttempt &&
-      latestUpdatedFlowLatestExecutedStep?.lastAttempt &&
-      currentFlowLatestExecutedStep.lastAttempt >=
-        latestUpdatedFlowLatestExecutedStep.lastAttempt
-
-    if (
-      isSameLatestExecutedStep &&
-      !isCurrentLatestExecutedStepLastAttemptAhead
-    ) {
-      throw new SkipStepAlreadyFinishedError(
-        "Step already finished by another execution"
-      )
-    }
-
-    // First ensure that the latest execution was not cancelled, otherwise we skip the execution
     const latestTransactionCancelledAt = latestUpdatedFlow.cancelledAt
     const currentTransactionCancelledAt = currentFlow.cancelledAt
 
@@ -855,91 +871,6 @@ export class RedisDistributedTransactionStorage
         "Workflow execution has been cancelled during the execution"
       )
     }
-
-    let currentFlowLastInvokingStepIndex = -1
-    for (let i = 0; i < currentFlowSteps.length; i++) {
-      if (isInvokingState(currentFlowSteps[i])) {
-        currentFlowLastInvokingStepIndex = i
-        break
-      }
-    }
-
-    let latestUpdatedFlowLastInvokingStepIndex = !latestUpdatedFlow.steps
-      ? 1 // There is no other execution, so the current execution is the latest
-      : -1
-
-    if (latestUpdatedFlow.steps) {
-      for (let i = 0; i < latestUpdatedFlowSteps.length; i++) {
-        if (isInvokingState(latestUpdatedFlowSteps[i])) {
-          latestUpdatedFlowLastInvokingStepIndex = i
-          break
-        }
-      }
-    }
-
-    let currentFlowLastCompensatingStepIndex = -1
-    for (let i = currentFlowSteps.length - 1; i >= 0; i--) {
-      if (isCompensatingState(currentFlowSteps[i])) {
-        currentFlowLastCompensatingStepIndex = currentFlowSteps.length - 1 - i
-        break
-      }
-    }
-
-    let latestUpdatedFlowLastCompensatingStepIndex = !latestUpdatedFlow.steps
-      ? -1 // There is no other execution, so the current execution is the latest
-      : -1
-
-    if (latestUpdatedFlow.steps) {
-      for (let i = latestUpdatedFlowSteps.length - 1; i >= 0; i--) {
-        if (isCompensatingState(latestUpdatedFlowSteps[i])) {
-          latestUpdatedFlowLastCompensatingStepIndex =
-            latestUpdatedFlowSteps.length - 1 - i
-          break
-        }
-      }
-    }
-
-    const isLatestExecutionFinishedIndex = -1
-    const invokeShouldBeSkipped =
-      (latestUpdatedFlowLastInvokingStepIndex ===
-        isLatestExecutionFinishedIndex ||
-        currentFlowLastInvokingStepIndex <
-          latestUpdatedFlowLastInvokingStepIndex) &&
-      currentFlowLastInvokingStepIndex !== isLatestExecutionFinishedIndex
-
-    const compensateShouldBeSkipped =
-      currentFlowLastCompensatingStepIndex <
-        latestUpdatedFlowLastCompensatingStepIndex &&
-      currentFlowLastCompensatingStepIndex !== isLatestExecutionFinishedIndex &&
-      latestUpdatedFlowLastCompensatingStepIndex !==
-        isLatestExecutionFinishedIndex
-
-    const isCompensatingMismatch =
-      latestUpdatedFlow.state === TransactionState.COMPENSATING &&
-      ![TransactionState.REVERTED, TransactionState.FAILED].includes(
-        currentFlow.state
-      ) &&
-      currentFlow.state !== latestUpdatedFlow.state
-
-    const isRevertedMismatch =
-      latestUpdatedFlow.state === TransactionState.REVERTED &&
-      currentFlow.state !== TransactionState.REVERTED
-
-    const isFailedMismatch =
-      latestUpdatedFlow.state === TransactionState.FAILED &&
-      currentFlow.state !== TransactionState.FAILED
-
-    if (
-      (data.flow.state !== TransactionState.COMPENSATING &&
-        invokeShouldBeSkipped) ||
-      (data.flow.state === TransactionState.COMPENSATING &&
-        compensateShouldBeSkipped) ||
-      isCompensatingMismatch ||
-      isRevertedMismatch ||
-      isFailedMismatch
-    ) {
-      throw new SkipExecutionError("Already finished by another execution")
-    }
   }
 
   async clearExpiredExecutions() {