fix(): workflows concurrency (#13645)

packages/modules/workflow-engine-inmemory/src/utils/workflow-orchestrator-storage.ts

@@ -1,3 +1,4 @@
+import { raw } from "@medusajs/framework/mikro-orm/core"
 import {
   DistributedTransactionType,
   IDistributedSchedulerStorage,
@@ -24,7 +25,6 @@ import {
   TransactionStepState,
   isPresent,
 } from "@medusajs/framework/utils"
-import { raw } from "@medusajs/framework/mikro-orm/core"
 import { WorkflowOrchestratorService } from "@services"
 import { type CronExpression, parseExpression } from "cron-parser"
 import { WorkflowExecution } from "../models/workflow-execution"
@@ -162,12 +162,15 @@ export class InMemoryDistributedTransactionStorage
   }
 
   private createManagedTimer(
-    callback: () => void,
+    callback: () => void | Promise<void>,
     delay: number
   ): NodeJS.Timeout {
-    const timer = setTimeout(() => {
+    const timer = setTimeout(async () => {
       this.pendingTimers.delete(timer)
-      callback()
+      const res = callback()
+      if (res instanceof Promise) {
+        await res
+      }
     }, delay)
 
     this.pendingTimers.add(timer)
@@ -341,13 +344,11 @@ export class InMemoryDistributedTransactionStorage
 
     const { retentionTime } = options ?? {}
 
-    if (data.flow.hasAsyncSteps) {
-      await this.#preventRaceConditionExecutionIfNecessary({
-        data,
-        key,
-        options,
-      })
-    }
+    await this.#preventRaceConditionExecutionIfNecessary({
+      data,
+      key,
+      options,
+    })
 
     // Only store retention time if it's provided
     if (retentionTime) {
@@ -363,8 +364,7 @@ export class InMemoryDistributedTransactionStorage
     if (isNotStarted && isManualTransactionId) {
       const storedData = this.storage.get(key)
       if (storedData) {
-        throw new MedusaError(
-          MedusaError.Types.INVALID_ARGUMENT,
+        throw new SkipExecutionError(
           "Transaction already started for transactionId: " +
             data.flow.transactionId
         )

packages/modules/workflow-engine-redis/integration-tests/__tests__/index.spec.ts

@@ -1,3 +1,4 @@
+import { asValue } from "@medusajs/framework/awilix"
 import {
   DistributedTransactionType,
   TransactionState,
@@ -27,7 +28,6 @@ import {
   WorkflowResponse,
 } from "@medusajs/framework/workflows-sdk"
 import { moduleIntegrationTestRunner } from "@medusajs/test-utils"
-import { asValue } from "@medusajs/framework/awilix"
 import { setTimeout as setTimeoutSync } from "timers"
 import { setTimeout } from "timers/promises"
 import { ulid } from "ulid"
@@ -37,28 +37,26 @@ import {
   workflowNotIdempotentWithRetentionStep2Invoke,
   workflowNotIdempotentWithRetentionStep3Invoke,
 } from "../__fixtures__"
-import { createScheduled } from "../__fixtures__/workflow_scheduled"
 import {
-  step1InvokeMock as step1InvokeMockAutoRetries,
-  step2InvokeMock as step2InvokeMockAutoRetries,
   step1CompensateMock as step1CompensateMockAutoRetries,
+  step1InvokeMock as step1InvokeMockAutoRetries,
   step2CompensateMock as step2CompensateMockAutoRetries,
+  step2InvokeMock as step2InvokeMockAutoRetries,
 } from "../__fixtures__/workflow_1_auto_retries"
 import {
-  step1InvokeMock as step1InvokeMockAutoRetriesFalse,
-  step2InvokeMock as step2InvokeMockAutoRetriesFalse,
   step1CompensateMock as step1CompensateMockAutoRetriesFalse,
+  step1InvokeMock as step1InvokeMockAutoRetriesFalse,
   step2CompensateMock as step2CompensateMockAutoRetriesFalse,
+  step2InvokeMock as step2InvokeMockAutoRetriesFalse,
 } from "../__fixtures__/workflow_1_auto_retries_false"
+import { createScheduled } from "../__fixtures__/workflow_scheduled"
 
+import { Redis } from "ioredis"
 import {
   step1InvokeMock as step1InvokeMockManualRetry,
   step2InvokeMock as step2InvokeMockManualRetry,
-  step1CompensateMock as step1CompensateMockManualRetry,
-  step2CompensateMock as step2CompensateMockManualRetry,
 } from "../__fixtures__/workflow_1_manual_retry_step"
 import { TestDatabase } from "../utils"
-import { Redis } from "ioredis"
 
 jest.setTimeout(300000)
 
@@ -523,7 +521,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
             expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
             expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
 
-          await setTimeout(3000)
+            await setTimeout(3000)
 
             await workflowOrcModule.run(workflowId, {
               input: {},
@@ -533,7 +531,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
 
             await setTimeout(2000)
 
-          await setTimeout(3000)
+            await setTimeout(3000)
 
             await workflowOrcModule.run(workflowId, {
               input: {},

packages/modules/workflow-engine-redis/src/utils/workflow-orchestrator-storage.ts

@@ -1,3 +1,4 @@
+import { raw } from "@medusajs/framework/mikro-orm/core"
 import {
   DistributedTransactionType,
   IDistributedSchedulerStorage,
@@ -21,7 +22,6 @@ import {
   TransactionState,
   TransactionStepState,
 } from "@medusajs/framework/utils"
-import { raw } from "@medusajs/framework/mikro-orm/core"
 import { WorkflowOrchestratorService } from "@services"
 import { Queue, RepeatOptions, Worker } from "bullmq"
 import Redis from "ioredis"
@@ -425,13 +425,11 @@ export class RedisDistributedTransactionStorage
 
     const { retentionTime } = options ?? {}
 
-    if (data.flow.hasAsyncSteps) {
-      await this.#preventRaceConditionExecutionIfNecessary({
-        data,
-        key,
-        options,
-      })
-    }
+    await this.#preventRaceConditionExecutionIfNecessary({
+      data,
+      key,
+      options,
+    })
 
     if (hasFinished && retentionTime) {
       Object.assign(data, {
@@ -471,34 +469,37 @@ export class RedisDistributedTransactionStorage
       pipeline.unlink(key)
     }
 
-    const pipelinePromise = pipeline.exec().then((result) => {
-      if (!shouldSetNX) {
+    const execPipeline = () => {
+      return pipeline.exec().then((result) => {
+        if (!shouldSetNX) {
+          return result
+        }
+
+        const actionResult = result?.pop()
+        const isOk = !!actionResult?.pop()
+        if (!isOk) {
+          throw new SkipExecutionError(
+            "Transaction already started for transactionId: " +
+              data.flow.transactionId
+          )
+        }
+
         return result
-      }
-
-      const actionResult = result?.pop()
-      const isOk = !!actionResult?.pop()
-      if (!isOk) {
-        throw new MedusaError(
-          MedusaError.Types.INVALID_ARGUMENT,
-          "Transaction already started for transactionId: " +
-            data.flow.transactionId
-        )
-      }
-
-      return result
-    })
+      })
+    }
 
     // Database operations
     if (hasFinished && !retentionTime) {
       // If the workflow is nested, we cant just remove it because it would break the compensation algorithm. Instead, it will get deleted when the top level parent is deleted.
       if (!data.flow.metadata?.parentStepIdempotencyKey) {
-        await promiseAll([pipelinePromise, this.deleteFromDb(data)])
+        await promiseAll([execPipeline(), this.deleteFromDb(data)])
       } else {
-        await promiseAll([pipelinePromise, this.saveToDb(data, retentionTime)])
+        await this.saveToDb(data, retentionTime)
+        await execPipeline()
       }
     } else {
-      await promiseAll([pipelinePromise, this.saveToDb(data, retentionTime)])
+      await this.saveToDb(data, retentionTime)
+      await execPipeline()
     }
   }