fix: workflow async concurrency (#13769)

* executeAsync * || 1 * wip * stepId * stepId * wip * wip * continue versioning management changes * fix and improve concurrency * update in memory engine * remove duplicated test * fix script * Create weak-drinks-confess.md * fixes * fix * fix * continuation * centralize merge checkepoint * centralize merge checkpoint * fix locking * rm only * Continue improvements and fixes * fixes * fixes * hasAwaiting will be recomputed * fix orchestrator engine * bump version on async parallel steps only * mark as delivered fix * changeset * check partitions * avoid saving when having parent step * cart test --------- Co-authored-by: Carlos R. L. Rodrigues <rodrigolr@gmail.com> Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com> Co-authored-by: Oli Juhl <59018053+olivermrbl@users.noreply.github.com>
2025-10-20 15:29:19 +02:00
parent d97a60d3c1
commit 516f5a3896
31 changed files with 2712 additions and 1406 deletions
--- a/packages/core/orchestration/src/transaction/datastore/abstract-storage.ts
+++ b/packages/core/orchestration/src/transaction/datastore/abstract-storage.ts
@@ -26,7 +26,7 @@ export interface IDistributedTransactionStorage {
    data: TransactionCheckpoint,
    ttl?: number,
    options?: TransactionOptions
-  ): Promise<void>
+  ): Promise<TransactionCheckpoint>
  scheduleRetry(
    transaction: DistributedTransactionType,
    step: TransactionStep,
@@ -96,7 +96,7 @@ export abstract class DistributedTransactionStorage
    key: string,
    data: TransactionCheckpoint,
    ttl?: number
-  ): Promise<void> {
+  ): Promise<TransactionCheckpoint> {
    throw new Error("Method 'save' not implemented.")
  }

--- a/packages/core/orchestration/src/transaction/datastore/base-in-memory-storage.ts
+++ b/packages/core/orchestration/src/transaction/datastore/base-in-memory-storage.ts
@@ -28,7 +28,7 @@ export class BaseInMemoryDistributedTransactionStorage extends DistributedTransa
    data: TransactionCheckpoint,
    ttl?: number,
    options?: TransactionOptions
-  ): Promise<void> {
+  ): Promise<TransactionCheckpoint> {
    const hasFinished = [
      TransactionState.DONE,
      TransactionState.REVERTED,
@@ -40,6 +40,8 @@ export class BaseInMemoryDistributedTransactionStorage extends DistributedTransa
    } else {
      this.storage.set(key, data)
    }
+
+    return data
  }

  async clearExpiredExecutions(): Promise<void> {}
--- a/packages/core/orchestration/src/transaction/distributed-transaction.ts
+++ b/packages/core/orchestration/src/transaction/distributed-transaction.ts
@@ -1,16 +1,60 @@
-import { isDefined } from "@medusajs/utils"
+import { isDefined, TransactionStepState } from "@medusajs/utils"
 import { EventEmitter } from "events"
+import { setTimeout as setTimeoutPromise } from "node:timers/promises"
 import { IDistributedTransactionStorage } from "./datastore/abstract-storage"
 import { BaseInMemoryDistributedTransactionStorage } from "./datastore/base-in-memory-storage"
-import { NonSerializableCheckPointError } from "./errors"
+import { NonSerializableCheckPointError, SkipExecutionError } from "./errors"
 import { TransactionOrchestrator } from "./transaction-orchestrator"
 import { TransactionStep, TransactionStepHandler } from "./transaction-step"
 import {
  TransactionFlow,
  TransactionHandlerType,
  TransactionState,
+  TransactionStepStatus,
 } from "./types"

+const flowMergeableProperties = [
+  "state",
+  "hasFailedSteps",
+  "hasSkippedOnFailureSteps",
+  "hasSkippedSteps",
+  "hasRevertedSteps",
+  "cancelledAt",
+  "startedAt",
+  "hasAsyncSteps",
+  "_v",
+  "timedOutAt",
+]
+
+const mergeStep = (
+  currentStep: TransactionStep,
+  storedStep: TransactionStep
+) => {
+  const mergeProperties = [
+    "attempts",
+    "failures",
+    "temporaryFailedAt",
+    "retryRescheduledAt",
+    "hasScheduledRetry",
+    "lastAttempt",
+    "_v",
+    "stepFailed",
+    "startedAt",
+  ]
+
+  for (const prop of mergeProperties) {
+    if (prop === "hasScheduledRetry" || prop === "stepFailed") {
+      currentStep[prop] = storedStep[prop] ?? currentStep[prop]
+      continue
+    }
+
+    currentStep[prop] =
+      storedStep[prop] || currentStep[prop]
+        ? Math.max(storedStep[prop] ?? 0, currentStep[prop] ?? 0)
+        : currentStep[prop] ?? storedStep[prop]
+  }
+}
+
 /**
 * @typedef TransactionMetadata
 * @property model_id - The id of the model_id that created the transaction (modelId).
@@ -51,12 +95,261 @@ export class TransactionStepError {
  ) {}
 }

+const stateFlowOrder = [
+  TransactionState.NOT_STARTED,
+  TransactionState.INVOKING,
+  TransactionState.DONE,
+  TransactionState.WAITING_TO_COMPENSATE,
+  TransactionState.COMPENSATING,
+  TransactionState.REVERTED,
+  TransactionState.FAILED,
+]
+
 export class TransactionCheckpoint {
  constructor(
    public flow: TransactionFlow,
    public context: TransactionContext,
    public errors: TransactionStepError[] = []
  ) {}
+
+  /**
+   * Merge the current checkpoint with incoming data from a concurrent save operation.
+   * This handles race conditions when multiple steps complete simultaneously.
+   *
+   * @param storedData - The checkpoint data being saved
+   * @param savingStepId - Optional step ID if this is a step-specific save
+   */
+  static mergeCheckpoints(
+    currentTransactionData: TransactionCheckpoint,
+    storedData?: TransactionCheckpoint
+  ): TransactionCheckpoint {
+    if (!currentTransactionData || !storedData) {
+      return currentTransactionData
+    }
+
+    TransactionCheckpoint.#mergeFlow(currentTransactionData, storedData)
+    TransactionCheckpoint.#mergeErrors(
+      currentTransactionData.errors ?? [],
+      storedData.errors
+    )
+
+    return currentTransactionData
+  }
+
+  static #mergeFlow(
+    currentTransactionData: TransactionCheckpoint,
+    storedData: TransactionCheckpoint
+  ): void {
+    const currentTransactionContext = currentTransactionData.context
+    const storedContext = storedData.context
+
+    if (currentTransactionData.flow._v >= storedData.flow._v) {
+      for (const prop of flowMergeableProperties) {
+        if (
+          prop === "startedAt" ||
+          prop === "cancelledAt" ||
+          prop === "timedOutAt"
+        ) {
+          currentTransactionData.flow[prop] =
+            storedData.flow[prop] || currentTransactionData.flow[prop]
+              ? Math.max(
+                  storedData.flow[prop] ?? 0,
+                  currentTransactionData.flow[prop] ?? 0
+                )
+              : currentTransactionData.flow[prop] ??
+                storedData.flow[prop] ??
+                (undefined as any)
+        } else if (prop === "_v") {
+          currentTransactionData.flow[prop] = Math.max(
+            storedData.flow[prop] ?? 0,
+            currentTransactionData.flow[prop] ?? 0
+          )
+        } else if (prop === "state") {
+          const curState = stateFlowOrder.findIndex(
+            (state) => state === currentTransactionData.flow.state
+          )
+          const storedState = stateFlowOrder.findIndex(
+            (state) => state === storedData.flow.state
+          )
+
+          if (storedState > curState) {
+            currentTransactionData.flow.state = storedData.flow.state
+          } else if (
+            curState < storedState &&
+            currentTransactionData.flow.state !==
+              TransactionState.WAITING_TO_COMPENSATE
+          ) {
+            throw new SkipExecutionError(
+              `Transaction is behind another execution`
+            )
+          }
+        } else if (
+          storedData.flow[prop] &&
+          !currentTransactionData.flow[prop]
+        ) {
+          currentTransactionData.flow[prop] = storedData.flow[prop]
+        }
+      }
+    }
+
+    const storedSteps = Object.values(storedData.flow.steps)
+
+    for (const storedStep of storedSteps) {
+      if (storedStep.id === "_root") {
+        continue
+      }
+
+      const stepName = storedStep.definition.action!
+      const stepId = storedStep.id
+
+      // Merge context responses
+      if (
+        storedContext.invoke[stepName] &&
+        !currentTransactionContext.invoke[stepName]
+      ) {
+        currentTransactionContext.invoke[stepName] =
+          storedContext.invoke[stepName]
+      }
+
+      if (
+        storedContext.compensate[stepName] &&
+        !currentTransactionContext.compensate[stepName]
+      ) {
+        currentTransactionContext.compensate[stepName] =
+          storedContext.compensate[stepName]
+      }
+
+      const currentStepVersion = currentTransactionData.flow.steps[stepId]._v!
+      const storedStepVersion = storedData.flow.steps[stepId]._v!
+
+      if (storedStepVersion > currentStepVersion) {
+        throw new SkipExecutionError(`Transaction is behind another execution`)
+      }
+
+      // Determine which state is further along in the process
+      const shouldUpdateInvoke = TransactionCheckpoint.#shouldUpdateStepState(
+        currentTransactionData.flow.steps[stepId].invoke,
+        storedStep.invoke
+      )
+
+      const shouldUpdateCompensate =
+        TransactionCheckpoint.#shouldUpdateStepState(
+          currentTransactionData.flow.steps[stepId].compensate,
+          storedStep.compensate
+        )
+
+      if (shouldUpdateInvoke) {
+        currentTransactionData.flow.steps[stepId].invoke = storedStep.invoke
+      }
+
+      if (shouldUpdateCompensate) {
+        currentTransactionData.flow.steps[stepId].compensate =
+          storedStep.compensate
+      }
+
+      mergeStep(currentTransactionData.flow.steps[stepId], storedStep)
+    }
+  }
+
+  /**
+   * Determines if the stored step state should replace the current step state.
+   * This validates both state and status transitions according to TransactionStep rules.
+   */
+  static #shouldUpdateStepState(
+    currentStepState: {
+      state: TransactionStepState
+      status: TransactionStepStatus
+    },
+    storedStepState: {
+      state: TransactionStepState
+      status: TransactionStepStatus
+    }
+  ): boolean {
+    // Define allowed state transitions
+    const allowedStateTransitions = {
+      [TransactionStepState.DORMANT]: [TransactionStepState.NOT_STARTED],
+      [TransactionStepState.NOT_STARTED]: [
+        TransactionStepState.INVOKING,
+        TransactionStepState.COMPENSATING,
+        TransactionStepState.FAILED,
+        TransactionStepState.SKIPPED,
+        TransactionStepState.SKIPPED_FAILURE,
+      ],
+      [TransactionStepState.INVOKING]: [
+        TransactionStepState.FAILED,
+        TransactionStepState.DONE,
+        TransactionStepState.TIMEOUT,
+        TransactionStepState.SKIPPED,
+      ],
+      [TransactionStepState.COMPENSATING]: [
+        TransactionStepState.REVERTED,
+        TransactionStepState.FAILED,
+      ],
+      [TransactionStepState.DONE]: [TransactionStepState.COMPENSATING],
+    }
+
+    // Define allowed status transitions
+    const allowedStatusTransitions = {
+      [TransactionStepStatus.WAITING]: [
+        TransactionStepStatus.OK,
+        TransactionStepStatus.TEMPORARY_FAILURE,
+        TransactionStepStatus.PERMANENT_FAILURE,
+      ],
+      [TransactionStepStatus.TEMPORARY_FAILURE]: [
+        TransactionStepStatus.IDLE,
+        TransactionStepStatus.PERMANENT_FAILURE,
+      ],
+      [TransactionStepStatus.PERMANENT_FAILURE]: [TransactionStepStatus.IDLE],
+    }
+
+    if (
+      currentStepState.state === storedStepState.state &&
+      currentStepState.status === storedStepState.status
+    ) {
+      return false
+    }
+
+    // Check if state transition from stored to current is allowed
+    const allowedStatesFromCurrent =
+      allowedStateTransitions[currentStepState.state] || []
+    const isStateTransitionValid = allowedStatesFromCurrent.includes(
+      storedStepState.state
+    )
+
+    if (currentStepState.state !== storedStepState.state) {
+      return isStateTransitionValid
+    }
+
+    // States are the same, check status transition
+    // Special case: WAITING status can always be transitioned
+    if (currentStepState.status === TransactionStepStatus.WAITING) {
+      return true
+    }
+
+    // Check if status transition from stored to current is allowed
+    const allowedStatusesFromCurrent =
+      allowedStatusTransitions[currentStepState.status] || []
+
+    return allowedStatusesFromCurrent.includes(storedStepState.status)
+  }
+
+  static #mergeErrors(
+    currentErrors: TransactionStepError[],
+    incomingErrors: TransactionStepError[]
+  ): void {
+    const existingErrorSignatures = new Set(
+      currentErrors.map(
+        (err) => `${err.action}:${err.handlerType}:${err.error?.message}`
+      )
+    )
+
+    for (const error of incomingErrors) {
+      const signature = `${error.action}:${error.handlerType}:${error.error?.message}`
+      if (!existingErrorSignatures.has(signature)) {
+        currentErrors.push(error)
+      }
+    }
+  }
 }

 export class TransactionPayload {
@@ -81,8 +374,8 @@ class DistributedTransaction extends EventEmitter {
  public transactionId: string
  public runId: string

-  private readonly errors: TransactionStepError[] = []
-  private readonly context: TransactionContext = new TransactionContext()
+  private errors: TransactionStepError[] = []
+  private context: TransactionContext = new TransactionContext()
  private static keyValueStore: IDistributedTransactionStorage

  /**
@@ -195,28 +488,100 @@ class DistributedTransaction extends EventEmitter {
    return this.getFlow().options?.timeout
  }

-  public async saveCheckpoint(
-    ttl = 0
-  ): Promise<TransactionCheckpoint | undefined> {
-    const options =
-      TransactionOrchestrator.getWorkflowOptions(this.modelId) ??
-      this.getFlow().options
+  public async saveCheckpoint({
+    ttl = 0,
+    parallelSteps = 0,
+    stepId,
+    _v,
+  }: {
+    ttl?: number
+    parallelSteps?: number
+    stepId?: string
+    _v?: number
+  } = {}): Promise<TransactionCheckpoint | undefined> {
+    const options = {
+      ...(TransactionOrchestrator.getWorkflowOptions(this.modelId) ??
+        this.getFlow().options),
+    }

    if (!options?.store) {
      return
    }

+    options.stepId = stepId
+    if (_v) {
+      options.parallelSteps = parallelSteps
+      options._v = _v
+    }
+
    const key = TransactionOrchestrator.getKeyName(
      DistributedTransaction.keyPrefix,
      this.modelId,
      this.transactionId
    )

-    const rawData = this.#serializeCheckpointData()
+    let checkpoint

-    await DistributedTransaction.keyValueStore.save(key, rawData, ttl, options)
+    let retries = 0
+    let backoffMs = 50
+    const maxRetries = (options?.parallelSteps || 1) + 2
+    while (retries < maxRetries) {
+      checkpoint = this.#serializeCheckpointData()

-    return rawData
+      try {
+        const savedCheckpoint = await DistributedTransaction.keyValueStore.save(
+          key,
+          checkpoint,
+          ttl,
+          options
+        )
+
+        return savedCheckpoint
+      } catch (error) {
+        if (TransactionOrchestrator.isExpectedError(error)) {
+          throw error
+        } else if (checkpoint.flow.state === TransactionState.NOT_STARTED) {
+          throw new SkipExecutionError(
+            "Transaction already started for transactionId: " +
+              this.transactionId
+          )
+        }
+
+        retries++
+        // Exponential backoff with jitter
+        const jitter = Math.random() * backoffMs
+
+        await setTimeoutPromise(backoffMs + jitter)
+
+        backoffMs = Math.min(backoffMs * 2, 1000)
+
+        const lastCheckpoint = await DistributedTransaction.loadTransaction(
+          this.modelId,
+          this.transactionId
+        )
+
+        if (!lastCheckpoint) {
+          throw new SkipExecutionError("Transaction already finished")
+        }
+
+        TransactionCheckpoint.mergeCheckpoints(checkpoint, lastCheckpoint)
+
+        const [steps] = TransactionOrchestrator.buildSteps(
+          checkpoint.flow.definition,
+          checkpoint.flow.steps
+        )
+        checkpoint.flow.steps = steps
+        this.flow = checkpoint.flow
+        this.errors = checkpoint.errors
+        this.context = checkpoint.context
+
+        continue
+      }
+    }
+
+    throw new Error(
+      `Max retries (${maxRetries}) exceeded for saving checkpoint due to version conflicts`
+    )
  }

  public static async loadTransaction(
--- a/packages/core/orchestration/src/transaction/errors.ts
+++ b/packages/core/orchestration/src/transaction/errors.ts
@@ -1,3 +1,5 @@
+import { OrchestrationUtils } from "@medusajs/utils"
+
 class BaseStepErrror extends Error {
  #stepResponse: unknown

@@ -116,6 +118,12 @@ export class SkipStepAlreadyFinishedError extends Error {
 }

 export class SkipCancelledExecutionError extends Error {
+  readonly #__type = OrchestrationUtils.SymbolWorkflowStepResponse
+
+  get __type() {
+    return this.#__type
+  }
+
  static isSkipCancelledExecutionError(
    error: Error
  ): error is SkipCancelledExecutionError {
--- a/packages/core/orchestration/src/transaction/transaction-orchestrator.ts
+++ b/packages/core/orchestration/src/transaction/transaction-orchestrator.ts
@@ -18,6 +18,7 @@ import {
  TransactionStepStatus,
 } from "./types"

+import { Context } from "@medusajs/types"
 import {
  isDefined,
  isErrorLike,
@@ -38,7 +39,6 @@ import {
  TransactionStepTimeoutError,
  TransactionTimeoutError,
 } from "./errors"
-import { Context } from "@medusajs/types"

 /**
 * @class TransactionOrchestrator is responsible for managing and executing distributed transactions.
@@ -115,7 +115,7 @@ export class TransactionOrchestrator extends EventEmitter {
    }
  }

-  private static isExpectedError(error: Error): boolean {
+  public static isExpectedError(error: Error): boolean {
    return (
      SkipCancelledExecutionError.isSkipCancelledExecutionError(error) ||
      SkipExecutionError.isSkipExecutionError(error) ||
@@ -137,7 +137,7 @@ export class TransactionOrchestrator extends EventEmitter {
    return params.join(this.SEPARATOR)
  }

-  private getPreviousStep(flow: TransactionFlow, step: TransactionStep) {
+  private static getPreviousStep(flow: TransactionFlow, step: TransactionStep) {
    const id = step.id.split(".")
    id.pop()
    const parentId = id.join(".")
@@ -175,6 +175,14 @@ export class TransactionOrchestrator extends EventEmitter {
    return steps
  }

+  private static countSiblings(
+    flow: TransactionFlow,
+    step: TransactionStep
+  ): number {
+    const previous = TransactionOrchestrator.getPreviousStep(flow, step)
+    return previous.next.length
+  }
+
  private canMoveForward(flow: TransactionFlow, previousStep: TransactionStep) {
    const states = [
      TransactionStepState.DONE,
@@ -184,9 +192,10 @@ export class TransactionOrchestrator extends EventEmitter {
      TransactionStepState.SKIPPED_FAILURE,
    ]

-    const siblings = this.getPreviousStep(flow, previousStep).next.map(
-      (sib) => flow.steps[sib]
-    )
+    const siblings = TransactionOrchestrator.getPreviousStep(
+      flow,
+      previousStep
+    ).next.map((sib) => flow.steps[sib])

    return (
      !!previousStep.definition.noWait ||
@@ -214,7 +223,7 @@ export class TransactionOrchestrator extends EventEmitter {
    if (flow.state == TransactionState.COMPENSATING) {
      return this.canMoveBackward(flow, step)
    } else {
-      const previous = this.getPreviousStep(flow, step)
+      const previous = TransactionOrchestrator.getPreviousStep(flow, step)
      if (previous.id === TransactionOrchestrator.ROOT_STEP) {
        return true
      }
@@ -311,6 +320,7 @@ export class TransactionOrchestrator extends EventEmitter {
    completed: number
  }> {
    const flow = transaction.getFlow()
+
    const result = await this.computeCurrentTransactionState(transaction)

    // Handle state transitions and emit events
@@ -324,7 +334,9 @@ export class TransactionOrchestrator extends EventEmitter {

      this.emit(DistributedTransactionEvent.COMPENSATE_BEGIN, { transaction })

-      return await this.checkAllSteps(transaction)
+      const result = await this.checkAllSteps(transaction)
+
+      return result
    } else if (result.completed === result.total) {
      if (result.hasSkippedOnFailure) {
        flow.hasSkippedOnFailureSteps = true
@@ -407,6 +419,7 @@ export class TransactionOrchestrator extends EventEmitter {
        if (stepDef.hasAwaitingRetry()) {
          if (stepDef.canRetryAwaiting()) {
            stepDef.retryRescheduledAt = null
+
            nextSteps.push(stepDef)
          } else if (!stepDef.retryRescheduledAt) {
            stepDef.hasScheduledRetry = true
@@ -501,6 +514,12 @@ export class TransactionOrchestrator extends EventEmitter {

      const stepDef = flow.steps[step]
      const curState = stepDef.getStates()
+
+      if (stepDef._v) {
+        flow._v = 0
+        stepDef._v = 0
+      }
+
      if (
        [TransactionStepState.DONE, TransactionStepState.TIMEOUT].includes(
          curState.state
@@ -547,7 +566,14 @@ export class TransactionOrchestrator extends EventEmitter {
    let shouldEmit = true
    let transactionIsCancelling = false
    try {
-      await transaction.saveCheckpoint()
+      await transaction.saveCheckpoint({
+        _v: step._v,
+        parallelSteps: TransactionOrchestrator.countSiblings(
+          transaction.getFlow(),
+          step
+        ),
+        stepId: step.id,
+      })
    } catch (error) {
      if (!TransactionOrchestrator.isExpectedError(error)) {
        throw error
@@ -567,9 +593,7 @@ export class TransactionOrchestrator extends EventEmitter {
    }

    if (cleaningUp.length) {
-      setImmediate(async () => {
-        await promiseAll(cleaningUp)
-      })
+      await promiseAll(cleaningUp)
    }

    if (shouldEmit) {
@@ -597,7 +621,14 @@ export class TransactionOrchestrator extends EventEmitter {
    transaction.getFlow().hasWaitingSteps = true

    try {
-      await transaction.saveCheckpoint()
+      await transaction.saveCheckpoint({
+        _v: step._v,
+        parallelSteps: TransactionOrchestrator.countSiblings(
+          transaction.getFlow(),
+          step
+        ),
+        stepId: step.id,
+      })
      await transaction.scheduleRetry(step, 0)
    } catch (error) {
      if (!TransactionOrchestrator.isExpectedError(error)) {
@@ -627,7 +658,14 @@ export class TransactionOrchestrator extends EventEmitter {
    let shouldEmit = true
    let transactionIsCancelling = false
    try {
-      await transaction.saveCheckpoint()
+      await transaction.saveCheckpoint({
+        _v: step._v,
+        parallelSteps: TransactionOrchestrator.countSiblings(
+          transaction.getFlow(),
+          step
+        ),
+        stepId: step.id,
+      })
    } catch (error) {
      if (!TransactionOrchestrator.isExpectedError(error)) {
        throw error
@@ -650,9 +688,7 @@ export class TransactionOrchestrator extends EventEmitter {
    }

    if (cleaningUp.length) {
-      setImmediate(async () => {
-        await promiseAll(cleaningUp)
-      })
+      await promiseAll(cleaningUp)
    }

    if (shouldEmit) {
@@ -837,7 +873,14 @@ export class TransactionOrchestrator extends EventEmitter {
    }

    try {
-      await transaction.saveCheckpoint()
+      await transaction.saveCheckpoint({
+        _v: step._v,
+        parallelSteps: TransactionOrchestrator.countSiblings(
+          transaction.getFlow(),
+          step
+        ),
+        stepId: step.id,
+      })
    } catch (error) {
      if (!TransactionOrchestrator.isExpectedError(error)) {
        throw error
@@ -856,9 +899,7 @@ export class TransactionOrchestrator extends EventEmitter {
    }

    if (cleaningUp.length) {
-      setImmediate(async () => {
-        await promiseAll(cleaningUp)
-      })
+      await promiseAll(cleaningUp)
    }

    if (!result.stopExecution) {
@@ -885,14 +926,21 @@ export class TransactionOrchestrator extends EventEmitter {
      }

      const flow = transaction.getFlow()
-      const nextSteps = await this.checkAllSteps(transaction)

-      if (await this.checkTransactionTimeout(transaction, nextSteps.current)) {
+      let nextSteps = await this.checkAllSteps(transaction)
+
+      const hasTimedOut = await this.checkTransactionTimeout(
+        transaction,
+        nextSteps.current
+      )
+
+      if (hasTimedOut) {
        continue
      }

      if (nextSteps.remaining === 0) {
        await this.finalizeTransaction(transaction)
+
        return
      }

@@ -915,6 +963,7 @@ export class TransactionOrchestrator extends EventEmitter {
      })

      const execution: Promise<void | unknown>[] = []
+      const executionAsync: (() => Promise<void | unknown>)[] = []

      let i = 0
      let hasAsyncSteps = false
@@ -939,37 +988,60 @@ export class TransactionOrchestrator extends EventEmitter {

        // Compute current transaction state
        await this.computeCurrentTransactionState(transaction)
-
        if (!continueExecution) {
          break
        }

        const promise = this.createStepExecutionPromise(transaction, step)

+        const hasMultipleAsyncSteps =
+          nextSteps.next.filter((step) => {
+            const isAsync = step.isCompensating()
+              ? step.definition.compensateAsync
+              : step.definition.async
+
+            return isAsync
+          }).length > 1
+
+        const hasVersionControl =
+          hasMultipleAsyncSteps || step.hasAwaitingRetry()
+
+        if (hasVersionControl && !step._v) {
+          transaction.getFlow()._v += 1
+          step._v = transaction.getFlow()._v
+        }
+
        if (!isAsync) {
          execution.push(
            this.executeSyncStep(promise, transaction, step, nextSteps)
          )
        } else {
          // Execute async step in background as part of the next event loop cycle and continue the execution of the transaction
-          process.nextTick(() =>
+          hasAsyncSteps = true
+          executionAsync.push(() =>
            this.executeAsyncStep(promise, transaction, step, nextSteps)
          )
-          hasAsyncSteps = true
        }
      }

      await promiseAll(execution)

-      if (nextSteps.next.length === 0 || (hasAsyncSteps && !execution.length)) {
+      if (!nextSteps.next.length || (hasAsyncSteps && !execution.length)) {
        continueExecution = false
+      }
+
+      if (hasAsyncSteps) {
        await transaction.saveCheckpoint().catch((error) => {
          if (TransactionOrchestrator.isExpectedError(error)) {
-            return
+            continueExecution = false
          }

          throw error
        })
+
+        for (const exec of executionAsync) {
+          void exec()
+        }
      }
    }
  }
@@ -989,6 +1061,7 @@ export class TransactionOrchestrator extends EventEmitter {
        throw error
      }
    })
+
    this.emit(DistributedTransactionEvent.FINISH, { transaction })
  }

@@ -1136,7 +1209,11 @@ export class TransactionOrchestrator extends EventEmitter {
      .then(async (response: any) => {
        await this.handleStepExpiration(transaction, step, nextSteps)

-        const output = response?.__type ? response.output : response
+        const output =
+          response?.__type || response?.output?.__type
+            ? response.output
+            : response
+
        if (SkipStepResponse.isSkipStepResponse(output)) {
          await TransactionOrchestrator.skipStep({
            transaction,
@@ -1175,7 +1252,10 @@ export class TransactionOrchestrator extends EventEmitter {
  ): Promise<void | unknown> {
    return promiseFn()
      .then(async (response: any) => {
-        const output = response?.__type ? response.output : response
+        const output =
+          response?.__type || response?.output?.__type
+            ? response.output
+            : response

        if (SkipStepResponse.isSkipStepResponse(output)) {
          await TransactionOrchestrator.skipStep({
@@ -1335,9 +1415,9 @@ export class TransactionOrchestrator extends EventEmitter {
        flow.state = TransactionState.INVOKING
        flow.startedAt = Date.now()

-        await transaction.saveCheckpoint(
-          flow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
-        )
+        await transaction.saveCheckpoint({
+          ttl: flow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL,
+        })

        if (transaction.hasTimeout()) {
          await transaction.scheduleTransactionTimeout(
@@ -1476,6 +1556,7 @@ export class TransactionOrchestrator extends EventEmitter {
      state: TransactionState.NOT_STARTED,
      definition: this.definition,
      steps,
+      _v: 0, // Initialize version to 0
    }

    return flow
@@ -1506,7 +1587,7 @@ export class TransactionOrchestrator extends EventEmitter {
    return null
  }

-  private static buildSteps(
+  static buildSteps(
    flow: TransactionStepsDefinition,
    existingSteps?: { [key: string]: TransactionStep }
  ): [{ [key: string]: TransactionStep }, StepFeatures] {
@@ -1588,6 +1669,7 @@ export class TransactionOrchestrator extends EventEmitter {
            failures: 0,
            lastAttempt: null,
            next: [],
+            _v: 0, // Initialize step version to 0
          }
        )
      }
@@ -1650,9 +1732,9 @@ export class TransactionOrchestrator extends EventEmitter {
    )

    if (newTransaction && this.getOptions().store) {
-      await transaction.saveCheckpoint(
-        modelFlow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
-      )
+      await transaction.saveCheckpoint({
+        ttl: modelFlow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL,
+      })
    }

    if (onLoad) {
--- a/packages/core/orchestration/src/transaction/transaction-step.ts
+++ b/packages/core/orchestration/src/transaction/transaction-step.ts
@@ -62,6 +62,7 @@ export class TransactionStep {
  startedAt?: number
  next: string[]
  saveResponse: boolean
+  _v?: number

  public getStates() {
    return this.isCompensating() ? this.compensate : this.invoke
@@ -191,8 +192,12 @@ export class TransactionStep {
      this.lastAttempt &&
      Date.now() - this.lastAttempt >
        this.definition.retryIntervalAwaiting! * 1e3 &&
+      // For compensating steps, ignore maxAwaitingRetries and retry indefinitely
+      // Compensation must complete, so we keep checking until the nested workflow finishes
      (!("maxAwaitingRetries" in this.definition) ||
-        this.attempts < this.definition.maxAwaitingRetries!)
+        (this.isCompensating()
+          ? this.attempts < this.definition.maxAwaitingRetries! * 2
+          : this.attempts < this.definition.maxAwaitingRetries!))
    )
  }

--- a/packages/core/orchestration/src/transaction/types.ts
+++ b/packages/core/orchestration/src/transaction/types.ts
@@ -263,6 +263,9 @@ export type StepFeatures = {
  hasAsyncSteps: boolean
  hasStepTimeouts: boolean
  hasRetriesTimeout: boolean
+  parallelSteps?: number
+  stepId?: string
+  _v?: number
 }

 export type TransactionOptions = TransactionModelOptions & StepFeatures
@@ -276,6 +279,7 @@ export type TransactionFlow = {
  metadata?: {
    eventGroupId?: string
    parentIdempotencyKey?: string
+    cancelingFromParentStep?: boolean
    sourcePath?: string
    preventReleaseEvents?: boolean
    parentStepIdempotencyKey?: string
@@ -295,4 +299,5 @@ export type TransactionFlow = {
  steps: {
    [key: string]: TransactionStep
  }
+  _v: number
 }
--- a/packages/core/orchestration/src/workflow/local-workflow.ts
+++ b/packages/core/orchestration/src/workflow/local-workflow.ts
@@ -417,7 +417,9 @@ export class LocalWorkflow {

    if (this.medusaContext) {
      this.medusaContext.eventGroupId =
-        transaction.getFlow().metadata?.eventGroupId
+        transaction.getFlow().metadata!.eventGroupId
+      transaction.getFlow().metadata!.cancelingFromParentStep ??=
+        this.medusaContext.cancelingFromParentStep
    }

    const { cleanUpEventListeners } = this.registerEventCallbacks({
@@ -626,6 +628,8 @@ export class LocalWorkflow {
      this.medusaContext.parentStepIdempotencyKey =
        metadata.parentStepIdempotencyKey
      this.medusaContext.preventReleaseEvents = metadata?.preventReleaseEvents
+      this.medusaContext.cancelingFromParentStep =
+        metadata?.cancelingFromParentStep
    }
  }
 }