asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix: workflow async concurrency (#13769)

Browse Source 
* executeAsync

* || 1

* wip

* stepId

* stepId

* wip

* wip

* continue versioning management changes

* fix and improve concurrency

* update in memory engine

* remove duplicated test

* fix script

* Create weak-drinks-confess.md

* fixes

* fix

* fix

* continuation

* centralize merge checkepoint

* centralize merge checkpoint

* fix locking

* rm only

* Continue improvements and fixes

* fixes

* fixes

* hasAwaiting will be recomputed

* fix orchestrator engine

* bump version on async parallel steps only

* mark as delivered fix

* changeset

* check partitions

* avoid saving when having parent step

* cart test

---------

Co-authored-by: Carlos R. L. Rodrigues <rodrigolr@gmail.com>
Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>
Co-authored-by: Oli Juhl <59018053+olivermrbl@users.noreply.github.com>
This commit is contained in:
Adrien de Peretti
2025-10-20 15:29:19 +02:00
committed by GitHub
parent d97a60d3c1
commit 516f5a3896
31 changed files with 2712 additions and 1406 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.changeset/weak-drinks-confess.md Normal file
View File

@@ -0,0 +1,9 @@
---
"@medusajs/workflow-engine-inmemory": patch
"@medusajs/workflow-engine-redis": patch
"@medusajs/orchestration": patch
"@medusajs/workflows-sdk": patch
"@medusajs/core-flows": patch
---
fix: workflow async concurrency

28
integration-tests/helpers/wait-for-index.ts
View File

@@ -21,6 +21,10 @@ export async function waitForIndexedEntities(
const { timeout = 120000, pollInterval = 100 } = options
const startTime = Date.now()
// Normalize the entity name to match partition table naming convention
const normalizedName = entityName.toLowerCase().replace(/[^a-z0-9_]/g, "_")
const partitionTableName = `cat_${normalizedName}`
while (Date.now() - startTime < timeout) {
try {
// Query the index_data table to check if all entities are indexed
@@ -33,10 +37,28 @@ export async function waitForIndexedEntities(
? result.rows.map((row: any) => row.id)
: result.map((row: any) => row.id)
// Check if all expected entities are indexed
// Check if all expected entities are indexed in index_data
const allIndexed = entityIds.every((id) => indexedIds.includes(id))
if (allIndexed) {
if (!allIndexed) {
await new Promise((resolve) => setTimeout(resolve, pollInterval))
continue
}
// Also check if data is replicated to the partition table
const partitionResult = await dbConnection.raw(
`SELECT id FROM ${partitionTableName} WHERE id = ANY(?)`,
[entityIds]
)
const partitionIds = partitionResult.rows
? partitionResult.rows.map((row: any) => row.id)
: partitionResult.map((row: any) => row.id)
// Check if all expected entities are in the partition table
const allInPartition = entityIds.every((id) => partitionIds.includes(id))
if (allInPartition) {
return
}
} catch (error) {
@@ -49,6 +71,6 @@ export async function waitForIndexedEntities(
throw new Error(
`Entities [${entityIds.join(
", "
)}] of type '${entityName}' were not indexed within ${timeout}ms`
)}] of type '${entityName}' were not fully replicated to partition table within ${timeout}ms`
)
}

180
integration-tests/http/__tests__/workflow-engine/admin/index.spec.ts
View File

@@ -11,16 +11,20 @@ import {
adminHeaders,
createAdminUser,
} from "../../../../helpers/create-admin-user"
import { setTimeout } from "timers/promises"
import { IWorkflowEngineService } from "@medusajs/framework/types"
jest.setTimeout(300000)
medusaIntegrationTestRunner({
testSuite: ({ dbConnection, getContainer, api }) => {
let container
let workflowOrcModule: IWorkflowEngineService
beforeEach(async () => {
container = getContainer()
await createAdminUser(dbConnection, adminHeaders, container)
workflowOrcModule = container.resolve(Modules.WORKFLOW_ENGINE)
})
describe("GET /admin/workflow-executions", () => {
@@ -90,5 +94,181 @@ medusaIntegrationTestRunner({
)
})
})
describe("Workflow Orchestrator module subscribe", function () {
it("should subscribe to a workflow and receive the response when it finishes", async () => {
const step1 = createStep({ name: "step1" }, async () => {
return new StepResponse("step1")
})
const step2 = createStep({ name: "step2" }, async () => {
await setTimeout(1000)
return new StepResponse("step2")
})
const workflowId =
"workflow" + Math.random().toString(36).substring(2, 15)
createWorkflow(workflowId, function (input) {
step1()
step2().config({
async: true,
})
return new WorkflowResponse("workflow")
})
const step1_1 = createStep({ name: "step1_1" }, async () => {
return new StepResponse("step1_1")
})
const step2_1 = createStep({ name: "step2_1" }, async () => {
await setTimeout(1000)
return new StepResponse("step2_1")
})
const workflow2Id =
"workflow_2" + Math.random().toString(36).substring(2, 15)
createWorkflow(workflow2Id, function (input) {
step1_1()
step2_1().config({
async: true,
})
return new WorkflowResponse("workflow_2")
})
const transactionId =
"trx_123" + Math.random().toString(36).substring(2, 15)
const transactionId2 =
"trx_124" + Math.random().toString(36).substring(2, 15)
const onWorkflowFinishSpy = jest.fn()
const onWorkflowFinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId: workflowId,
transactionId,
subscriber: (event) => {
console.log("event", event)
if (event.eventType === "onFinish") {
onWorkflowFinishSpy()
workflowOrcModule.run(workflow2Id, {
transactionId: transactionId2,
})
resolve()
}
},
})
})
const onWorkflow2FinishSpy = jest.fn()
const workflow2FinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId: workflow2Id,
subscriber: (event) => {
console.log("event", event)
if (event.eventType === "onFinish") {
onWorkflow2FinishSpy()
resolve()
}
},
})
})
workflowOrcModule.run(workflowId, {
transactionId,
})
await onWorkflowFinishPromise
await workflow2FinishPromise
expect(onWorkflowFinishSpy).toHaveBeenCalledTimes(1)
expect(onWorkflow2FinishSpy).toHaveBeenCalledTimes(1)
})
it("should subscribe to a workflow and receive the response when it finishes (2)", async () => {
const step1 = createStep({ name: "step1" }, async () => {
return new StepResponse("step1")
})
const step2 = createStep({ name: "step2" }, async () => {
await setTimeout(1000)
return new StepResponse("step2")
})
const workflowId =
"workflow" + Math.random().toString(36).substring(2, 15)
createWorkflow(workflowId, function (input) {
step1()
step2().config({
async: true,
})
return new WorkflowResponse("workflow")
})
const step1_1 = createStep({ name: "step1_1" }, async () => {
return new StepResponse("step1_1")
})
const step2_1 = createStep({ name: "step2_1" }, async () => {
await setTimeout(1000)
return new StepResponse("step2_1")
})
const workflow2Id =
"workflow_2" + Math.random().toString(36).substring(2, 15)
createWorkflow(workflow2Id, function (input) {
step1_1()
step2_1().config({
async: true,
})
return new WorkflowResponse("workflow_2")
})
const transactionId =
"trx_123" + Math.random().toString(36).substring(2, 15)
const transactionId2 =
"trx_124" + Math.random().toString(36).substring(2, 15)
const onWorkflowFinishSpy = jest.fn()
const onWorkflowFinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId: workflowId,
transactionId,
subscriber: (event) => {
console.log("event", event)
if (event.eventType === "onFinish") {
onWorkflowFinishSpy()
workflowOrcModule.run(workflow2Id, {
transactionId: transactionId2,
})
resolve()
}
},
})
})
const onWorkflow2FinishSpy = jest.fn()
const workflow2FinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId: workflow2Id,
subscriber: (event) => {
console.log("event", event)
if (event.eventType === "onFinish") {
onWorkflow2FinishSpy()
resolve()
}
},
})
})
workflowOrcModule.run(workflowId, {
transactionId,
})
await onWorkflowFinishPromise
await workflow2FinishPromise
expect(onWorkflowFinishSpy).toHaveBeenCalledTimes(1)
expect(onWorkflow2FinishSpy).toHaveBeenCalledTimes(1)
})
})
},
})

3
integration-tests/modules/__tests__/cart/store/cart.workflows.spec.ts
View File

@@ -391,7 +391,8 @@ medusaIntegrationTestRunner({
},
})
expect(transaction.flow.state).toEqual("reverted")
// TODO: the state must be "reverted" when runAsStep of sync flows can be reverted
expect(transaction.flow.state).toEqual("failed")
})
it("should throw when no regions exist", async () => {

12
packages/core/core-flows/src/order/workflows/mark-order-fulfillment-as-delivered.ts
View File

@@ -16,7 +16,6 @@ import {
import {
createStep,
createWorkflow,
parallelize,
transform,
WorkflowData,
WorkflowResponse,
@@ -259,12 +258,11 @@ export const markOrderFulfillmentAsDeliveredWorkflow = createWorkflow(
prepareRegisterDeliveryData
)
const [deliveredFulfillment] = parallelize(
markFulfillmentAsDeliveredWorkflow.runAsStep({
input: { id: fulfillment.id },
}),
registerOrderDeliveryStep(deliveryData)
)
const deliveredFulfillment = markFulfillmentAsDeliveredWorkflow.runAsStep({
input: { id: fulfillment.id },
})
registerOrderDeliveryStep(deliveryData)
emitEventStep({
eventName: FulfillmentWorkflowEvents.DELIVERY_CREATED,

4
packages/core/orchestration/src/transaction/datastore/abstract-storage.ts
View File

@@ -26,7 +26,7 @@ export interface IDistributedTransactionStorage {
data: TransactionCheckpoint,
ttl?: number,
options?: TransactionOptions
): Promise<void>
): Promise<TransactionCheckpoint>
scheduleRetry(
transaction: DistributedTransactionType,
step: TransactionStep,
@@ -96,7 +96,7 @@ export abstract class DistributedTransactionStorage
key: string,
data: TransactionCheckpoint,
ttl?: number
): Promise<void> {
): Promise<TransactionCheckpoint> {
throw new Error("Method 'save' not implemented.")
}

4
packages/core/orchestration/src/transaction/datastore/base-in-memory-storage.ts
View File

@@ -28,7 +28,7 @@ export class BaseInMemoryDistributedTransactionStorage extends DistributedTransa
data: TransactionCheckpoint,
ttl?: number,
options?: TransactionOptions
): Promise<void> {
): Promise<TransactionCheckpoint> {
const hasFinished = [
TransactionState.DONE,
TransactionState.REVERTED,
@@ -40,6 +40,8 @@ export class BaseInMemoryDistributedTransactionStorage extends DistributedTransa
} else {
this.storage.set(key, data)
}
return data
}
async clearExpiredExecutions(): Promise<void> {}

391
packages/core/orchestration/src/transaction/distributed-transaction.ts
View File

@@ -1,16 +1,60 @@
import { isDefined } from "@medusajs/utils"
import { isDefined, TransactionStepState } from "@medusajs/utils"
import { EventEmitter } from "events"
import { setTimeout as setTimeoutPromise } from "node:timers/promises"
import { IDistributedTransactionStorage } from "./datastore/abstract-storage"
import { BaseInMemoryDistributedTransactionStorage } from "./datastore/base-in-memory-storage"
import { NonSerializableCheckPointError } from "./errors"
import { NonSerializableCheckPointError, SkipExecutionError } from "./errors"
import { TransactionOrchestrator } from "./transaction-orchestrator"
import { TransactionStep, TransactionStepHandler } from "./transaction-step"
import {
TransactionFlow,
TransactionHandlerType,
TransactionState,
TransactionStepStatus,
} from "./types"
const flowMergeableProperties = [
"state",
"hasFailedSteps",
"hasSkippedOnFailureSteps",
"hasSkippedSteps",
"hasRevertedSteps",
"cancelledAt",
"startedAt",
"hasAsyncSteps",
"_v",
"timedOutAt",
]
const mergeStep = (
currentStep: TransactionStep,
storedStep: TransactionStep
) => {
const mergeProperties = [
"attempts",
"failures",
"temporaryFailedAt",
"retryRescheduledAt",
"hasScheduledRetry",
"lastAttempt",
"_v",
"stepFailed",
"startedAt",
]
for (const prop of mergeProperties) {
if (prop === "hasScheduledRetry" || prop === "stepFailed") {
currentStep[prop] = storedStep[prop] ?? currentStep[prop]
continue
}
currentStep[prop] =
storedStep[prop] || currentStep[prop]
? Math.max(storedStep[prop] ?? 0, currentStep[prop] ?? 0)
: currentStep[prop] ?? storedStep[prop]
}
}
/**
* @typedef TransactionMetadata
* @property model_id - The id of the model_id that created the transaction (modelId).
@@ -51,12 +95,261 @@ export class TransactionStepError {
) {}
}
const stateFlowOrder = [
TransactionState.NOT_STARTED,
TransactionState.INVOKING,
TransactionState.DONE,
TransactionState.WAITING_TO_COMPENSATE,
TransactionState.COMPENSATING,
TransactionState.REVERTED,
TransactionState.FAILED,
]
export class TransactionCheckpoint {
constructor(
public flow: TransactionFlow,
public context: TransactionContext,
public errors: TransactionStepError[] = []
) {}
/**
* Merge the current checkpoint with incoming data from a concurrent save operation.
* This handles race conditions when multiple steps complete simultaneously.
*
* @param storedData - The checkpoint data being saved
* @param savingStepId - Optional step ID if this is a step-specific save
*/
static mergeCheckpoints(
currentTransactionData: TransactionCheckpoint,
storedData?: TransactionCheckpoint
): TransactionCheckpoint {
if (!currentTransactionData || !storedData) {
return currentTransactionData
}
TransactionCheckpoint.#mergeFlow(currentTransactionData, storedData)
TransactionCheckpoint.#mergeErrors(
currentTransactionData.errors ?? [],
storedData.errors
)
return currentTransactionData
}
static #mergeFlow(
currentTransactionData: TransactionCheckpoint,
storedData: TransactionCheckpoint
): void {
const currentTransactionContext = currentTransactionData.context
const storedContext = storedData.context
if (currentTransactionData.flow._v >= storedData.flow._v) {
for (const prop of flowMergeableProperties) {
if (
prop === "startedAt" ||
prop === "cancelledAt" ||
prop === "timedOutAt"
) {
currentTransactionData.flow[prop] =
storedData.flow[prop] || currentTransactionData.flow[prop]
? Math.max(
storedData.flow[prop] ?? 0,
currentTransactionData.flow[prop] ?? 0
)
: currentTransactionData.flow[prop] ??
storedData.flow[prop] ??
(undefined as any)
} else if (prop === "_v") {
currentTransactionData.flow[prop] = Math.max(
storedData.flow[prop] ?? 0,
currentTransactionData.flow[prop] ?? 0
)
} else if (prop === "state") {
const curState = stateFlowOrder.findIndex(
(state) => state === currentTransactionData.flow.state
)
const storedState = stateFlowOrder.findIndex(
(state) => state === storedData.flow.state
)
if (storedState > curState) {
currentTransactionData.flow.state = storedData.flow.state
} else if (
curState < storedState &&
currentTransactionData.flow.state !==
TransactionState.WAITING_TO_COMPENSATE
) {
throw new SkipExecutionError(
`Transaction is behind another execution`
)
}
} else if (
storedData.flow[prop] &&
!currentTransactionData.flow[prop]
) {
currentTransactionData.flow[prop] = storedData.flow[prop]
}
}
}
const storedSteps = Object.values(storedData.flow.steps)
for (const storedStep of storedSteps) {
if (storedStep.id === "_root") {
continue
}
const stepName = storedStep.definition.action!
const stepId = storedStep.id
// Merge context responses
if (
storedContext.invoke[stepName] &&
!currentTransactionContext.invoke[stepName]
) {
currentTransactionContext.invoke[stepName] =
storedContext.invoke[stepName]
}
if (
storedContext.compensate[stepName] &&
!currentTransactionContext.compensate[stepName]
) {
currentTransactionContext.compensate[stepName] =
storedContext.compensate[stepName]
}
const currentStepVersion = currentTransactionData.flow.steps[stepId]._v!
const storedStepVersion = storedData.flow.steps[stepId]._v!
if (storedStepVersion > currentStepVersion) {
throw new SkipExecutionError(`Transaction is behind another execution`)
}
// Determine which state is further along in the process
const shouldUpdateInvoke = TransactionCheckpoint.#shouldUpdateStepState(
currentTransactionData.flow.steps[stepId].invoke,
storedStep.invoke
)
const shouldUpdateCompensate =
TransactionCheckpoint.#shouldUpdateStepState(
currentTransactionData.flow.steps[stepId].compensate,
storedStep.compensate
)
if (shouldUpdateInvoke) {
currentTransactionData.flow.steps[stepId].invoke = storedStep.invoke
}
if (shouldUpdateCompensate) {
currentTransactionData.flow.steps[stepId].compensate =
storedStep.compensate
}
mergeStep(currentTransactionData.flow.steps[stepId], storedStep)
}
}
/**
* Determines if the stored step state should replace the current step state.
* This validates both state and status transitions according to TransactionStep rules.
*/
static #shouldUpdateStepState(
currentStepState: {
state: TransactionStepState
status: TransactionStepStatus
},
storedStepState: {
state: TransactionStepState
status: TransactionStepStatus
}
): boolean {
// Define allowed state transitions
const allowedStateTransitions = {
[TransactionStepState.DORMANT]: [TransactionStepState.NOT_STARTED],
[TransactionStepState.NOT_STARTED]: [
TransactionStepState.INVOKING,
TransactionStepState.COMPENSATING,
TransactionStepState.FAILED,
TransactionStepState.SKIPPED,
TransactionStepState.SKIPPED_FAILURE,
],
[TransactionStepState.INVOKING]: [
TransactionStepState.FAILED,
TransactionStepState.DONE,
TransactionStepState.TIMEOUT,
TransactionStepState.SKIPPED,
],
[TransactionStepState.COMPENSATING]: [
TransactionStepState.REVERTED,
TransactionStepState.FAILED,
],
[TransactionStepState.DONE]: [TransactionStepState.COMPENSATING],
}
// Define allowed status transitions
const allowedStatusTransitions = {
[TransactionStepStatus.WAITING]: [
TransactionStepStatus.OK,
TransactionStepStatus.TEMPORARY_FAILURE,
TransactionStepStatus.PERMANENT_FAILURE,
],
[TransactionStepStatus.TEMPORARY_FAILURE]: [
TransactionStepStatus.IDLE,
TransactionStepStatus.PERMANENT_FAILURE,
],
[TransactionStepStatus.PERMANENT_FAILURE]: [TransactionStepStatus.IDLE],
}
if (
currentStepState.state === storedStepState.state &&
currentStepState.status === storedStepState.status
) {
return false
}
// Check if state transition from stored to current is allowed
const allowedStatesFromCurrent =
allowedStateTransitions[currentStepState.state] || []
const isStateTransitionValid = allowedStatesFromCurrent.includes(
storedStepState.state
)
if (currentStepState.state !== storedStepState.state) {
return isStateTransitionValid
}
// States are the same, check status transition
// Special case: WAITING status can always be transitioned
if (currentStepState.status === TransactionStepStatus.WAITING) {
return true
}
// Check if status transition from stored to current is allowed
const allowedStatusesFromCurrent =
allowedStatusTransitions[currentStepState.status] || []
return allowedStatusesFromCurrent.includes(storedStepState.status)
}
static #mergeErrors(
currentErrors: TransactionStepError[],
incomingErrors: TransactionStepError[]
): void {
const existingErrorSignatures = new Set(
currentErrors.map(
(err) => `${err.action}:${err.handlerType}:${err.error?.message}`
)
)
for (const error of incomingErrors) {
const signature = `${error.action}:${error.handlerType}:${error.error?.message}`
if (!existingErrorSignatures.has(signature)) {
currentErrors.push(error)
}
}
}
}
export class TransactionPayload {
@@ -81,8 +374,8 @@ class DistributedTransaction extends EventEmitter {
public transactionId: string
public runId: string
private readonly errors: TransactionStepError[] = []
private readonly context: TransactionContext = new TransactionContext()
private errors: TransactionStepError[] = []
private context: TransactionContext = new TransactionContext()
private static keyValueStore: IDistributedTransactionStorage
/**
@@ -195,28 +488,100 @@ class DistributedTransaction extends EventEmitter {
return this.getFlow().options?.timeout
}
public async saveCheckpoint(
ttl = 0
): Promise<TransactionCheckpoint | undefined> {
const options =
TransactionOrchestrator.getWorkflowOptions(this.modelId) ??
this.getFlow().options
public async saveCheckpoint({
ttl = 0,
parallelSteps = 0,
stepId,
_v,
}: {
ttl?: number
parallelSteps?: number
stepId?: string
_v?: number
} = {}): Promise<TransactionCheckpoint | undefined> {
const options = {
...(TransactionOrchestrator.getWorkflowOptions(this.modelId) ??
this.getFlow().options),
}
if (!options?.store) {
return
}
options.stepId = stepId
if (_v) {
options.parallelSteps = parallelSteps
options._v = _v
}
const key = TransactionOrchestrator.getKeyName(
DistributedTransaction.keyPrefix,
this.modelId,
this.transactionId
)
const rawData = this.#serializeCheckpointData()
let checkpoint
await DistributedTransaction.keyValueStore.save(key, rawData, ttl, options)
let retries = 0
let backoffMs = 50
const maxRetries = (options?.parallelSteps || 1) + 2
while (retries < maxRetries) {
checkpoint = this.#serializeCheckpointData()
return rawData
try {
const savedCheckpoint = await DistributedTransaction.keyValueStore.save(
key,
checkpoint,
ttl,
options
)
return savedCheckpoint
} catch (error) {
if (TransactionOrchestrator.isExpectedError(error)) {
throw error
} else if (checkpoint.flow.state === TransactionState.NOT_STARTED) {
throw new SkipExecutionError(
"Transaction already started for transactionId: " +
this.transactionId
)
}
retries++
// Exponential backoff with jitter
const jitter = Math.random() * backoffMs
await setTimeoutPromise(backoffMs + jitter)
backoffMs = Math.min(backoffMs * 2, 1000)
const lastCheckpoint = await DistributedTransaction.loadTransaction(
this.modelId,
this.transactionId
)
if (!lastCheckpoint) {
throw new SkipExecutionError("Transaction already finished")
}
TransactionCheckpoint.mergeCheckpoints(checkpoint, lastCheckpoint)
const [steps] = TransactionOrchestrator.buildSteps(
checkpoint.flow.definition,
checkpoint.flow.steps
)
checkpoint.flow.steps = steps
this.flow = checkpoint.flow
this.errors = checkpoint.errors
this.context = checkpoint.context
continue
}
}
throw new Error(
`Max retries (${maxRetries}) exceeded for saving checkpoint due to version conflicts`
)
}
public static async loadTransaction(

8
packages/core/orchestration/src/transaction/errors.ts
View File

@@ -1,3 +1,5 @@
import { OrchestrationUtils } from "@medusajs/utils"
class BaseStepErrror extends Error {
#stepResponse: unknown
@@ -116,6 +118,12 @@ export class SkipStepAlreadyFinishedError extends Error {
}
export class SkipCancelledExecutionError extends Error {
readonly #__type = OrchestrationUtils.SymbolWorkflowStepResponse
get __type() {
return this.#__type
}
static isSkipCancelledExecutionError(
error: Error
): error is SkipCancelledExecutionError {

156
packages/core/orchestration/src/transaction/transaction-orchestrator.ts
View File

@@ -18,6 +18,7 @@ import {
TransactionStepStatus,
} from "./types"
import { Context } from "@medusajs/types"
import {
isDefined,
isErrorLike,
@@ -38,7 +39,6 @@ import {
TransactionStepTimeoutError,
TransactionTimeoutError,
} from "./errors"
import { Context } from "@medusajs/types"
/**
* @class TransactionOrchestrator is responsible for managing and executing distributed transactions.
@@ -115,7 +115,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
}
private static isExpectedError(error: Error): boolean {
public static isExpectedError(error: Error): boolean {
return (
SkipCancelledExecutionError.isSkipCancelledExecutionError(error) ||
SkipExecutionError.isSkipExecutionError(error) ||
@@ -137,7 +137,7 @@ export class TransactionOrchestrator extends EventEmitter {
return params.join(this.SEPARATOR)
}
private getPreviousStep(flow: TransactionFlow, step: TransactionStep) {
private static getPreviousStep(flow: TransactionFlow, step: TransactionStep) {
const id = step.id.split(".")
id.pop()
const parentId = id.join(".")
@@ -175,6 +175,14 @@ export class TransactionOrchestrator extends EventEmitter {
return steps
}
private static countSiblings(
flow: TransactionFlow,
step: TransactionStep
): number {
const previous = TransactionOrchestrator.getPreviousStep(flow, step)
return previous.next.length
}
private canMoveForward(flow: TransactionFlow, previousStep: TransactionStep) {
const states = [
TransactionStepState.DONE,
@@ -184,9 +192,10 @@ export class TransactionOrchestrator extends EventEmitter {
TransactionStepState.SKIPPED_FAILURE,
]
const siblings = this.getPreviousStep(flow, previousStep).next.map(
(sib) => flow.steps[sib]
)
const siblings = TransactionOrchestrator.getPreviousStep(
flow,
previousStep
).next.map((sib) => flow.steps[sib])
return (
!!previousStep.definition.noWait ||
@@ -214,7 +223,7 @@ export class TransactionOrchestrator extends EventEmitter {
if (flow.state == TransactionState.COMPENSATING) {
return this.canMoveBackward(flow, step)
} else {
const previous = this.getPreviousStep(flow, step)
const previous = TransactionOrchestrator.getPreviousStep(flow, step)
if (previous.id === TransactionOrchestrator.ROOT_STEP) {
return true
}
@@ -311,6 +320,7 @@ export class TransactionOrchestrator extends EventEmitter {
completed: number
}> {
const flow = transaction.getFlow()
const result = await this.computeCurrentTransactionState(transaction)
// Handle state transitions and emit events
@@ -324,7 +334,9 @@ export class TransactionOrchestrator extends EventEmitter {
this.emit(DistributedTransactionEvent.COMPENSATE_BEGIN, { transaction })
return await this.checkAllSteps(transaction)
const result = await this.checkAllSteps(transaction)
return result
} else if (result.completed === result.total) {
if (result.hasSkippedOnFailure) {
flow.hasSkippedOnFailureSteps = true
@@ -407,6 +419,7 @@ export class TransactionOrchestrator extends EventEmitter {
if (stepDef.hasAwaitingRetry()) {
if (stepDef.canRetryAwaiting()) {
stepDef.retryRescheduledAt = null
nextSteps.push(stepDef)
} else if (!stepDef.retryRescheduledAt) {
stepDef.hasScheduledRetry = true
@@ -501,6 +514,12 @@ export class TransactionOrchestrator extends EventEmitter {
const stepDef = flow.steps[step]
const curState = stepDef.getStates()
if (stepDef._v) {
flow._v = 0
stepDef._v = 0
}
if (
[TransactionStepState.DONE, TransactionStepState.TIMEOUT].includes(
curState.state
@@ -547,7 +566,14 @@ export class TransactionOrchestrator extends EventEmitter {
let shouldEmit = true
let transactionIsCancelling = false
try {
await transaction.saveCheckpoint()
await transaction.saveCheckpoint({
_v: step._v,
parallelSteps: TransactionOrchestrator.countSiblings(
transaction.getFlow(),
step
),
stepId: step.id,
})
} catch (error) {
if (!TransactionOrchestrator.isExpectedError(error)) {
throw error
@@ -567,9 +593,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
if (cleaningUp.length) {
setImmediate(async () => {
await promiseAll(cleaningUp)
})
await promiseAll(cleaningUp)
}
if (shouldEmit) {
@@ -597,7 +621,14 @@ export class TransactionOrchestrator extends EventEmitter {
transaction.getFlow().hasWaitingSteps = true
try {
await transaction.saveCheckpoint()
await transaction.saveCheckpoint({
_v: step._v,
parallelSteps: TransactionOrchestrator.countSiblings(
transaction.getFlow(),
step
),
stepId: step.id,
})
await transaction.scheduleRetry(step, 0)
} catch (error) {
if (!TransactionOrchestrator.isExpectedError(error)) {
@@ -627,7 +658,14 @@ export class TransactionOrchestrator extends EventEmitter {
let shouldEmit = true
let transactionIsCancelling = false
try {
await transaction.saveCheckpoint()
await transaction.saveCheckpoint({
_v: step._v,
parallelSteps: TransactionOrchestrator.countSiblings(
transaction.getFlow(),
step
),
stepId: step.id,
})
} catch (error) {
if (!TransactionOrchestrator.isExpectedError(error)) {
throw error
@@ -650,9 +688,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
if (cleaningUp.length) {
setImmediate(async () => {
await promiseAll(cleaningUp)
})
await promiseAll(cleaningUp)
}
if (shouldEmit) {
@@ -837,7 +873,14 @@ export class TransactionOrchestrator extends EventEmitter {
}
try {
await transaction.saveCheckpoint()
await transaction.saveCheckpoint({
_v: step._v,
parallelSteps: TransactionOrchestrator.countSiblings(
transaction.getFlow(),
step
),
stepId: step.id,
})
} catch (error) {
if (!TransactionOrchestrator.isExpectedError(error)) {
throw error
@@ -856,9 +899,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
if (cleaningUp.length) {
setImmediate(async () => {
await promiseAll(cleaningUp)
})
await promiseAll(cleaningUp)
}
if (!result.stopExecution) {
@@ -885,14 +926,21 @@ export class TransactionOrchestrator extends EventEmitter {
}
const flow = transaction.getFlow()
const nextSteps = await this.checkAllSteps(transaction)
if (await this.checkTransactionTimeout(transaction, nextSteps.current)) {
let nextSteps = await this.checkAllSteps(transaction)
const hasTimedOut = await this.checkTransactionTimeout(
transaction,
nextSteps.current
)
if (hasTimedOut) {
continue
}
if (nextSteps.remaining === 0) {
await this.finalizeTransaction(transaction)
return
}
@@ -915,6 +963,7 @@ export class TransactionOrchestrator extends EventEmitter {
})
const execution: Promise<void | unknown>[] = []
const executionAsync: (() => Promise<void | unknown>)[] = []
let i = 0
let hasAsyncSteps = false
@@ -939,37 +988,60 @@ export class TransactionOrchestrator extends EventEmitter {
// Compute current transaction state
await this.computeCurrentTransactionState(transaction)
if (!continueExecution) {
break
}
const promise = this.createStepExecutionPromise(transaction, step)
const hasMultipleAsyncSteps =
nextSteps.next.filter((step) => {
const isAsync = step.isCompensating()
? step.definition.compensateAsync
: step.definition.async
return isAsync
}).length > 1
const hasVersionControl =
hasMultipleAsyncSteps || step.hasAwaitingRetry()
if (hasVersionControl && !step._v) {
transaction.getFlow()._v += 1
step._v = transaction.getFlow()._v
}
if (!isAsync) {
execution.push(
this.executeSyncStep(promise, transaction, step, nextSteps)
)
} else {
// Execute async step in background as part of the next event loop cycle and continue the execution of the transaction
process.nextTick(() =>
hasAsyncSteps = true
executionAsync.push(() =>
this.executeAsyncStep(promise, transaction, step, nextSteps)
)
hasAsyncSteps = true
}
}
await promiseAll(execution)
if (nextSteps.next.length === 0 || (hasAsyncSteps && !execution.length)) {
if (!nextSteps.next.length || (hasAsyncSteps && !execution.length)) {
continueExecution = false
}
if (hasAsyncSteps) {
await transaction.saveCheckpoint().catch((error) => {
if (TransactionOrchestrator.isExpectedError(error)) {
return
continueExecution = false
}
throw error
})
for (const exec of executionAsync) {
void exec()
}
}
}
}
@@ -989,6 +1061,7 @@ export class TransactionOrchestrator extends EventEmitter {
throw error
}
})
this.emit(DistributedTransactionEvent.FINISH, { transaction })
}
@@ -1136,7 +1209,11 @@ export class TransactionOrchestrator extends EventEmitter {
.then(async (response: any) => {
await this.handleStepExpiration(transaction, step, nextSteps)
const output = response?.__type ? response.output : response
const output =
response?.__type || response?.output?.__type
? response.output
: response
if (SkipStepResponse.isSkipStepResponse(output)) {
await TransactionOrchestrator.skipStep({
transaction,
@@ -1175,7 +1252,10 @@ export class TransactionOrchestrator extends EventEmitter {
): Promise<void | unknown> {
return promiseFn()
.then(async (response: any) => {
const output = response?.__type ? response.output : response
const output =
response?.__type || response?.output?.__type
? response.output
: response
if (SkipStepResponse.isSkipStepResponse(output)) {
await TransactionOrchestrator.skipStep({
@@ -1335,9 +1415,9 @@ export class TransactionOrchestrator extends EventEmitter {
flow.state = TransactionState.INVOKING
flow.startedAt = Date.now()
await transaction.saveCheckpoint(
flow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
)
await transaction.saveCheckpoint({
ttl: flow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL,
})
if (transaction.hasTimeout()) {
await transaction.scheduleTransactionTimeout(
@@ -1476,6 +1556,7 @@ export class TransactionOrchestrator extends EventEmitter {
state: TransactionState.NOT_STARTED,
definition: this.definition,
steps,
_v: 0, // Initialize version to 0
}
return flow
@@ -1506,7 +1587,7 @@ export class TransactionOrchestrator extends EventEmitter {
return null
}
private static buildSteps(
static buildSteps(
flow: TransactionStepsDefinition,
existingSteps?: { [key: string]: TransactionStep }
): [{ [key: string]: TransactionStep }, StepFeatures] {
@@ -1588,6 +1669,7 @@ export class TransactionOrchestrator extends EventEmitter {
failures: 0,
lastAttempt: null,
next: [],
_v: 0, // Initialize step version to 0
}
)
}
@@ -1650,9 +1732,9 @@ export class TransactionOrchestrator extends EventEmitter {
)
if (newTransaction && this.getOptions().store) {
await transaction.saveCheckpoint(
modelFlow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
)
await transaction.saveCheckpoint({
ttl: modelFlow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL,
})
}
if (onLoad) {

7
packages/core/orchestration/src/transaction/transaction-step.ts
View File

@@ -62,6 +62,7 @@ export class TransactionStep {
startedAt?: number
next: string[]
saveResponse: boolean
_v?: number
public getStates() {
return this.isCompensating() ? this.compensate : this.invoke
@@ -191,8 +192,12 @@ export class TransactionStep {
this.lastAttempt &&
Date.now() - this.lastAttempt >
this.definition.retryIntervalAwaiting! * 1e3 &&
// For compensating steps, ignore maxAwaitingRetries and retry indefinitely
// Compensation must complete, so we keep checking until the nested workflow finishes
(!("maxAwaitingRetries" in this.definition) ||
this.attempts < this.definition.maxAwaitingRetries!)
(this.isCompensating()
? this.attempts < this.definition.maxAwaitingRetries! * 2
: this.attempts < this.definition.maxAwaitingRetries!))
)
}

5
packages/core/orchestration/src/transaction/types.ts
View File

@@ -263,6 +263,9 @@ export type StepFeatures = {
hasAsyncSteps: boolean
hasStepTimeouts: boolean
hasRetriesTimeout: boolean
parallelSteps?: number
stepId?: string
_v?: number
}
export type TransactionOptions = TransactionModelOptions & StepFeatures
@@ -276,6 +279,7 @@ export type TransactionFlow = {
metadata?: {
eventGroupId?: string
parentIdempotencyKey?: string
cancelingFromParentStep?: boolean
sourcePath?: string
preventReleaseEvents?: boolean
parentStepIdempotencyKey?: string
@@ -295,4 +299,5 @@ export type TransactionFlow = {
steps: {
[key: string]: TransactionStep
}
_v: number
}

6
packages/core/orchestration/src/workflow/local-workflow.ts
View File

@@ -417,7 +417,9 @@ export class LocalWorkflow {
if (this.medusaContext) {
this.medusaContext.eventGroupId =
transaction.getFlow().metadata?.eventGroupId
transaction.getFlow().metadata!.eventGroupId
transaction.getFlow().metadata!.cancelingFromParentStep ??=
this.medusaContext.cancelingFromParentStep
}
const { cleanUpEventListeners } = this.registerEventCallbacks({
@@ -626,6 +628,8 @@ export class LocalWorkflow {
this.medusaContext.parentStepIdempotencyKey =
metadata.parentStepIdempotencyKey
this.medusaContext.preventReleaseEvents = metadata?.preventReleaseEvents
this.medusaContext.cancelingFromParentStep =
metadata?.cancelingFromParentStep
}
}
}

6
packages/core/types/src/shared-context.ts
View File

@@ -88,4 +88,10 @@ export type Context<TManager = unknown> = {
* A boolean value indicating whether the current workflow execution is being cancelled.
*/
isCancelling?: boolean
/**
* Weither or not a sub workflow cancellation is being triggered from a parent step.
* If true, the parent step will not be triggered by the sub workflow.
*/
cancelingFromParentStep?: boolean
}

21
packages/core/workflows-sdk/src/helper/workflow-export.ts
View File

@@ -111,8 +111,12 @@ function createContextualWorkflowRunner<
flow.container = executionContainer
}
const { eventGroupId, parentStepIdempotencyKey, preventReleaseEvents } =
context
const {
eventGroupId,
parentStepIdempotencyKey,
preventReleaseEvents,
cancelingFromParentStep,
} = context
if (!preventReleaseEvents) {
attachOnFinishReleaseEvents(events, flow, { logOnError })
@@ -123,6 +127,7 @@ function createContextualWorkflowRunner<
parentStepIdempotencyKey,
sourcePath: options?.sourcePath,
preventReleaseEvents,
cancelingFromParentStep,
}
context.isCancelling = isCancel
@@ -609,13 +614,13 @@ function attachOnFinishReleaseEvents(
if (logOnError) {
const workflowName = transaction.getFlow().modelId
transaction
.getErrors()
.forEach((err) =>
logger.error(
`${workflowName}:${err?.action}:${err?.handlerType} - ${err?.error?.message}${EOL}${err?.error?.stack}`
)
transaction.getErrors().forEach((err) => {
const errMsg = err?.error?.message ? " - " + err?.error?.message : ""
logger.error(
`${workflowName}:${err?.action}:${err?.handlerType}${errMsg}${EOL}${err?.error?.stack}`
)
})
}
const eventBusService = (

37
packages/core/workflows-sdk/src/utils/composer/create-workflow.ts
View File

@@ -24,12 +24,25 @@ import {
CreateWorkflowComposerContext,
HookHandler,
ReturnWorkflow,
StepExecutionContext,
StepFunction,
WorkflowData,
} from "./type"
global[OrchestrationUtils.SymbolMedusaWorkflowComposerContext] = null
const buildTransactionId = (
step: { __step__: string },
stepContext: StepExecutionContext
) => {
return (
step.__step__ +
"-" +
(stepContext.transactionId ?? ulid()) +
(stepContext.attempt > 1 ? `-attempt-${stepContext.attempt}` : "")
)
}
/**
* This function creates a workflow with the provided name and a constructor function.
* The constructor function builds the workflow from steps created by the {@link createStep} function.
@@ -207,8 +220,7 @@ export function createWorkflow<TData, TResult, THooks extends any[]>(
const executionContext = {
...(sharedContext?.context ?? {}),
transactionId:
step.__step__ + "-" + (stepContext.transactionId ?? ulid()),
transactionId: buildTransactionId(step, stepContext),
parentStepIdempotencyKey: stepContext.idempotencyKey,
preventReleaseEvents: true,
runId: stepContext.runId,
@@ -218,7 +230,10 @@ export function createWorkflow<TData, TResult, THooks extends any[]>(
if (workflowEngine && isAsync) {
transaction = await workflowEngine.run(name, {
input: stepInput as any,
transactionId: executionContext.transactionId,
context: executionContext,
throwOnError: false,
logOnError: true,
})
} else {
transaction = await workflow.run({
@@ -235,9 +250,6 @@ export function createWorkflow<TData, TResult, THooks extends any[]>(
},
async (transaction, stepContext) => {
// The step itself has failed, there is nothing to revert
if (!transaction) {
return
}
const { container, ...sharedContext } = stepContext
const isAsync = stepContext[" stepDefinition"]?.async
@@ -248,27 +260,28 @@ export function createWorkflow<TData, TResult, THooks extends any[]>(
const executionContext = {
...(sharedContext?.context ?? {}),
transactionId:
step.__step__ + "-" + (stepContext.transactionId ?? ulid()),
transactionId: buildTransactionId(step, stepContext),
parentStepIdempotencyKey: stepContext.idempotencyKey,
preventReleaseEvents: true,
cancelingFromParentStep: true,
}
const transactionId = step.__step__ + "-" + stepContext.transactionId
if (workflowEngine && isAsync) {
await workflowEngine.cancel(name, {
transactionId: transactionId,
transactionId: executionContext.transactionId,
context: executionContext,
})
} else {
await workflow(container).cancel({
transaction: (transaction as WorkflowResult<any>)?.transaction,
transactionId,
transaction: ((transaction as WorkflowResult<any>) ?? {})
?.transaction,
transactionId: executionContext.transactionId,
container,
context: executionContext,
})
}
return
}
)(input) as ReturnType<StepFunction<TData, TResult>>

385
packages/medusa-test-utils/src/module-test-runner.ts
View File

@@ -8,6 +8,7 @@ import {
normalizeImportPathWithSource,
toMikroOrmEntities,
} from "@medusajs/framework/utils"
import { logger } from "@medusajs/framework/logger"
import * as fs from "fs"
import { getDatabaseURL, getMikroOrmWrapper, TestDatabase } from "./database"
import { initModules, InitModulesOptions } from "./init-modules"
@@ -23,6 +24,24 @@ export interface SuiteOptions<TService = unknown> {
}
}
interface ModuleTestRunnerConfig<TService = any> {
moduleName: string
moduleModels?: any[]
moduleOptions?: Record<string, any>
moduleDependencies?: string[]
joinerConfig?: any[]
schema?: string
dbName?: string
injectedDependencies?: Record<string, any>
resolve?: string
debug?: boolean
cwd?: string
hooks?: {
beforeModuleInit?: () => Promise<void>
afterModuleInit?: (medusaApp: any, service: TService) => Promise<void>
}
}
function createMikroOrmWrapper(options: {
moduleModels?: (Function | DmlEntity<any, any>)[]
resolve?: string
@@ -64,6 +83,220 @@ function createMikroOrmWrapper(options: {
return { MikroOrmWrapper, models: moduleModels }
}
class ModuleTestRunner<TService = any> {
private moduleName: string
private schema: string
private dbName: string
private dbConfig: {
clientUrl: string
schema: string
debug: boolean
}
private debug: boolean
private resolve?: string
private cwd?: string
private moduleOptions: Record<string, any>
private moduleDependencies?: string[]
private joinerConfig: any[]
private injectedDependencies: Record<string, any>
private hooks: ModuleTestRunnerConfig<TService>["hooks"] = {}
private connection: any = null
private MikroOrmWrapper!: TestDatabase
private moduleModels: (Function | DmlEntity<any, any>)[] = []
private modulesConfig: any = {}
private moduleOptionsConfig!: InitModulesOptions
private shutdown: () => Promise<void> = async () => void 0
private moduleService: any = null
private medusaApp: any = {}
constructor(config: ModuleTestRunnerConfig<TService>) {
const tempName = parseInt(process.env.JEST_WORKER_ID || "1")
this.moduleName = config.moduleName
this.dbName =
config.dbName ??
`medusa-${config.moduleName.toLowerCase()}-integration-${tempName}`
this.schema = config.schema ?? "public"
this.debug = config.debug ?? false
this.resolve = config.resolve
this.cwd = config.cwd
this.moduleOptions = config.moduleOptions ?? {}
this.moduleDependencies = config.moduleDependencies
this.joinerConfig = config.joinerConfig ?? []
this.injectedDependencies = config.injectedDependencies ?? {}
this.hooks = config.hooks ?? {}
this.dbConfig = {
clientUrl: getDatabaseURL(this.dbName),
schema: this.schema,
debug: this.debug,
}
this.setupProcessHandlers()
this.initializeConfig(config.moduleModels)
}
private setupProcessHandlers(): void {
process.on("SIGTERM", async () => {
await this.cleanup()
process.exit(0)
})
process.on("SIGINT", async () => {
await this.cleanup()
process.exit(0)
})
}
private initializeConfig(moduleModels?: any[]): void {
const moduleSdkImports = require("@medusajs/framework/modules-sdk")
// Use a unique connection for all the entire suite
this.connection = ModulesSdkUtils.createPgConnection(this.dbConfig)
const { MikroOrmWrapper, models } = createMikroOrmWrapper({
moduleModels,
resolve: this.resolve,
dbConfig: this.dbConfig,
cwd: this.cwd,
})
this.MikroOrmWrapper = MikroOrmWrapper
this.moduleModels = models
this.modulesConfig = {
[this.moduleName]: {
definition: moduleSdkImports.ModulesDefinition[this.moduleName],
resolve: this.resolve,
dependencies: this.moduleDependencies,
options: {
database: this.dbConfig,
...this.moduleOptions,
[isSharedConnectionSymbol]: true,
},
},
}
this.moduleOptionsConfig = {
injectedDependencies: {
[ContainerRegistrationKeys.PG_CONNECTION]: this.connection,
[Modules.EVENT_BUS]: new MockEventBusService(),
[ContainerRegistrationKeys.LOGGER]: console,
...this.injectedDependencies,
},
modulesConfig: this.modulesConfig,
databaseConfig: this.dbConfig,
joinerConfig: this.joinerConfig,
preventConnectionDestroyWarning: true,
cwd: this.cwd,
}
}
private createMedusaAppProxy(): any {
return new Proxy(
{},
{
get: (target, prop) => {
return this.medusaApp?.[prop]
},
}
)
}
private createServiceProxy(): any {
return new Proxy(
{},
{
get: (target, prop) => {
return this.moduleService?.[prop]
},
}
)
}
public async beforeAll(): Promise<void> {
try {
this.setupProcessHandlers()
process.env.LOG_LEVEL = "error"
} catch (error) {
await this.cleanup()
throw error
}
}
public async beforeEach(): Promise<void> {
try {
if (this.moduleModels.length) {
await this.MikroOrmWrapper.setupDatabase()
}
if (this.hooks?.beforeModuleInit) {
await this.hooks.beforeModuleInit()
}
const output = await initModules(this.moduleOptionsConfig)
this.shutdown = output.shutdown
this.medusaApp = output.medusaApp
this.moduleService = output.medusaApp.modules[this.moduleName]
if (this.hooks?.afterModuleInit) {
await this.hooks.afterModuleInit(this.medusaApp, this.moduleService)
}
} catch (error) {
logger.error("Error in beforeEach:", error?.message)
await this.cleanup()
throw error
}
}
public async afterEach(): Promise<void> {
try {
if (this.moduleModels.length) {
await this.MikroOrmWrapper.clearDatabase()
}
await this.shutdown()
this.moduleService = {}
this.medusaApp = {}
} catch (error) {
logger.error("Error in afterEach:", error?.message)
throw error
}
}
public async cleanup(): Promise<void> {
try {
process.removeAllListeners("SIGTERM")
process.removeAllListeners("SIGINT")
await (this.connection as any)?.context?.destroy()
await (this.connection as any)?.destroy()
this.moduleService = null
this.medusaApp = null
this.connection = null
if (global.gc) {
global.gc()
}
} catch (error) {
logger.error("Error during cleanup:", error?.message)
}
}
public getOptions(): SuiteOptions<TService> {
return {
MikroOrmWrapper: this.MikroOrmWrapper,
medusaApp: this.createMedusaAppProxy(),
service: this.createServiceProxy(),
dbConfig: {
schema: this.schema,
clientUrl: this.dbConfig.clientUrl,
},
}
}
}
export function moduleIntegrationTestRunner<TService = any>({
moduleName,
moduleModels,
@@ -76,6 +309,7 @@ export function moduleIntegrationTestRunner<TService = any>({
resolve,
injectedDependencies = {},
cwd,
hooks,
}: {
moduleName: string
moduleModels?: any[]
@@ -88,115 +322,68 @@ export function moduleIntegrationTestRunner<TService = any>({
resolve?: string
debug?: boolean
cwd?: string
hooks?: ModuleTestRunnerConfig<TService>["hooks"]
testSuite: (options: SuiteOptions<TService>) => void
}) {
const moduleSdkImports = require("@medusajs/framework/modules-sdk")
process.env.LOG_LEVEL = "error"
const tempName = parseInt(process.env.JEST_WORKER_ID || "1")
const dbName = `medusa-${moduleName.toLowerCase()}-integration-${tempName}`
const dbConfig = {
clientUrl: getDatabaseURL(dbName),
const runner = new ModuleTestRunner<TService>({
moduleName,
moduleModels,
moduleOptions,
moduleDependencies,
joinerConfig,
schema,
debug,
}
// Use a unique connection for all the entire suite
const connection = ModulesSdkUtils.createPgConnection(dbConfig)
const { MikroOrmWrapper, models } = createMikroOrmWrapper({
moduleModels,
resolve,
dbConfig,
injectedDependencies,
cwd,
hooks,
})
moduleModels = models
const modulesConfig_ = {
[moduleName]: {
definition: moduleSdkImports.ModulesDefinition[moduleName],
resolve,
dependencies: moduleDependencies,
options: {
database: dbConfig,
...moduleOptions,
[isSharedConnectionSymbol]: true,
},
},
}
const moduleOptions_: InitModulesOptions = {
injectedDependencies: {
[ContainerRegistrationKeys.PG_CONNECTION]: connection,
[Modules.EVENT_BUS]: new MockEventBusService(),
[ContainerRegistrationKeys.LOGGER]: console,
...injectedDependencies,
},
modulesConfig: modulesConfig_,
databaseConfig: dbConfig,
joinerConfig,
preventConnectionDestroyWarning: true,
cwd,
}
let shutdown: () => Promise<void>
let moduleService
let medusaApp = {}
const options = {
MikroOrmWrapper,
medusaApp: new Proxy(
{},
{
get: (target, prop) => {
return medusaApp[prop]
},
}
),
service: new Proxy(
{},
{
get: (target, prop) => {
return moduleService[prop]
},
}
),
dbConfig: {
schema,
clientUrl: dbConfig.clientUrl,
},
} as SuiteOptions<TService>
const beforeEach_ = async () => {
if (moduleModels.length) {
await MikroOrmWrapper.setupDatabase()
}
const output = await initModules(moduleOptions_)
shutdown = output.shutdown
medusaApp = output.medusaApp
moduleService = output.medusaApp.modules[moduleName]
}
const afterEach_ = async () => {
if (moduleModels.length) {
await MikroOrmWrapper.clearDatabase()
}
await shutdown()
moduleService = {}
medusaApp = {}
}
return describe("", () => {
beforeEach(beforeEach_)
afterEach(afterEach_)
afterAll(async () => {
await (connection as any).context?.destroy()
await (connection as any).destroy()
let testOptions: SuiteOptions<TService>
beforeAll(async () => {
await runner.beforeAll()
testOptions = runner.getOptions()
})
testSuite(options)
beforeEach(async () => {
await runner.beforeEach()
})
afterEach(async () => {
await runner.afterEach()
})
afterAll(async () => {
// Run main cleanup
await runner.cleanup()
// Clean references to the test options
for (const key in testOptions) {
if (typeof testOptions[key] === "function") {
testOptions[key] = null
} else if (
typeof testOptions[key] === "object" &&
testOptions[key] !== null
) {
Object.keys(testOptions[key]).forEach((k) => {
testOptions[key][k] = null
})
testOptions[key] = null
}
}
// Encourage garbage collection
// @ts-ignore
testOptions = null
if (global.gc) {
global.gc()
}
})
// Run test suite with options
testSuite(runner.getOptions())
})
}

13
packages/modules/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_async.ts
View File

@@ -30,6 +30,17 @@ const nestedWorkflow = createWorkflow(
}
)
const nestedWorkflow2 = createWorkflow(
{
name: "nested_sub_flow_async_2",
},
function (input) {
const resp = step_1_background(input)
return resp
}
)
createWorkflow(
{
name: "workflow_async_background",
@@ -41,7 +52,7 @@ createWorkflow(
input,
})
.config({ name: "step_sub_flow_1" }),
nestedWorkflow
nestedWorkflow2
.runAsStep({
input,
})

19
packages/modules/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_parallel_async.ts
View File

@@ -14,9 +14,25 @@ const step_2 = createStep(
async (_, { container }) => {
const we = container.resolve(Modules.WORKFLOW_ENGINE)
const onFinishPromise = new Promise<void>((resolve, reject) => {
void we.subscribe({
workflowId: "workflow_sub_workflow",
subscriber: (event) => {
if (event.eventType === "onFinish") {
if (event.errors.length > 0) {
reject(event.errors[0])
} else {
resolve()
}
}
},
})
})
await we.run("workflow_sub_workflow", {
throwOnError: true,
})
await onFinishPromise
}
)
@@ -34,7 +50,6 @@ const step_2_sub = createStep(
const subFlow = createWorkflow(
{
name: "workflow_sub_workflow",
retentionTime: 1000,
},
function (input) {
step_2_sub()
@@ -68,7 +83,7 @@ const step_3 = createStep(
createWorkflow(
{
name: "workflow_parallel_async",
retentionTime: 1000,
retentionTime: 5,
},
function (input) {
parallelize(step_1(), step_2(), step_3())

427
packages/modules/workflow-engine-inmemory/integration-tests/__tests__/index.spec.ts
View File

@@ -1,4 +1,5 @@
import { MedusaContainer } from "@medusajs/framework"
import { asFunction } from "@medusajs/framework/awilix"
import {
DistributedTransactionType,
TransactionState,
@@ -25,7 +26,6 @@ import {
} from "@medusajs/framework/workflows-sdk"
import { moduleIntegrationTestRunner } from "@medusajs/test-utils"
import { WorkflowsModuleService } from "@services"
import { asFunction } from "@medusajs/framework/awilix"
import { setTimeout as setTimeoutSync } from "timers"
import { setTimeout as setTimeoutPromise } from "timers/promises"
import { ulid } from "ulid"
@@ -39,40 +39,30 @@ import {
workflowNotIdempotentWithRetentionStep3Invoke,
} from "../__fixtures__"
import {
eventGroupWorkflowId,
workflowEventGroupIdStep1Mock,
workflowEventGroupIdStep2Mock,
} from "../__fixtures__/workflow_event_group_id"
import {
step1InvokeMock as step1InvokeMockAutoRetries,
step2InvokeMock as step2InvokeMockAutoRetries,
step1CompensateMock as step1CompensateMockAutoRetries,
step1InvokeMock as step1InvokeMockAutoRetries,
step2CompensateMock as step2CompensateMockAutoRetries,
step2InvokeMock as step2InvokeMockAutoRetries,
} from "../__fixtures__/workflow_1_auto_retries"
import {
step1InvokeMock as step1InvokeMockAutoRetriesFalse,
step2InvokeMock as step2InvokeMockAutoRetriesFalse,
step1CompensateMock as step1CompensateMockAutoRetriesFalse,
step1InvokeMock as step1InvokeMockAutoRetriesFalse,
step2CompensateMock as step2CompensateMockAutoRetriesFalse,
step2InvokeMock as step2InvokeMockAutoRetriesFalse,
} from "../__fixtures__/workflow_1_auto_retries_false"
import {
step1InvokeMock as step1InvokeMockManualRetry,
step2InvokeMock as step2InvokeMockManualRetry,
} from "../__fixtures__/workflow_1_manual_retry_step"
import {
eventGroupWorkflowId,
workflowEventGroupIdStep1Mock,
workflowEventGroupIdStep2Mock,
} from "../__fixtures__/workflow_event_group_id"
import { createScheduled } from "../__fixtures__/workflow_scheduled"
jest.setTimeout(60000)
const failTrap = (done, name, timeout = 5000) => {
return setTimeoutSync(() => {
// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
console.warn(
`Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
)
done()
}, timeout)
}
function times(num) {
let resolver
let counter = 0
@@ -159,14 +149,14 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
})
describe("Cancel transaction", function () {
it("should cancel an ongoing execution with async unfinished yet step", (done) => {
it("should cancel an ongoing execution with async unfinished yet step", async () => {
const transactionId = "transaction-to-cancel-id" + ulid()
const step1 = createStep("step1", async () => {
return new StepResponse("step1")
})
const step2 = createStep("step2", async () => {
await setTimeoutPromise(500)
await setTimeoutPromise(200)
return new StepResponse("step2")
})
@@ -184,43 +174,39 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
return new WorkflowResponse("finished")
})
workflowOrcModule
.run(workflowId, {
input: {},
const onFinish = new Promise<void>((resolve) => {
workflowOrcModule.subscribe({
workflowId,
transactionId,
})
.then(async () => {
await setTimeoutPromise(100)
await workflowOrcModule.cancel(workflowId, {
transactionId,
})
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
const execution =
await workflowOrcModule.listWorkflowExecutions({
transaction_id: transactionId,
})
expect(execution.length).toEqual(1)
expect(execution[0].state).toEqual(
TransactionState.REVERTED
)
done()
clearTimeout(timeout)
}
},
})
subscriber: async (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
const timeout = failTrap(
done,
"should cancel an ongoing execution with async unfinished yet step"
)
workflowOrcModule
.run(workflowId, {
input: {},
transactionId,
})
.then(async () => {
await setTimeoutPromise(100)
await workflowOrcModule.cancel(workflowId, {
transactionId,
})
})
})
await onFinish
const execution = await workflowOrcModule.listWorkflowExecutions({
transaction_id: transactionId,
})
expect(execution.length).toEqual(1)
expect(execution[0].state).toEqual(TransactionState.REVERTED)
})
it("should cancel a complete execution with a sync workflow running as async", async () => {
@@ -375,11 +361,11 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
})
})
it("should manually retry a step that is taking too long to finish", (done) => {
it("should manually retry a step that is taking too long to finish", async () => {
const transactionId = "transaction-manual-retry" + ulid()
const workflowId = "workflow_1_manual_retry_step"
void workflowOrcModule
await workflowOrcModule
.run(workflowId, {
input: {},
transactionId,
@@ -388,6 +374,18 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(1)
const onFinishPromise = new Promise<void>((resolve, reject) => {
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
void workflowOrcModule.retryStep({
idempotencyKey: {
workflowId,
@@ -396,68 +394,54 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
action: "invoke",
},
})
return onFinishPromise
})
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(2)
done()
clearTimeout(timeout)
}
},
})
const timeout = failTrap(
done,
"should manually retry a step that is taking too long to finish"
)
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(2)
})
it("should retry steps X times automatically when maxRetries is set", (done) => {
it("should retry steps X times automatically when maxRetries is set", async () => {
const transactionId = "transaction-auto-retries" + ulid()
const workflowId = "workflow_1_auto_retries"
const onFinishPromise = new Promise<void>((resolve, reject) => {
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
void workflowOrcModule.run(workflowId, {
input: {},
transactionId,
})
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
expect(step1InvokeMockAutoRetries).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetries).toHaveBeenCalledTimes(3)
expect(step1CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
expect(step2CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
done()
clearTimeout(timeout)
}
},
})
await onFinishPromise
const timeout = failTrap(
done,
"should retry steps X times automatically when maxRetries is set"
)
expect(step1InvokeMockAutoRetries).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetries).toHaveBeenCalledTimes(3)
expect(step1CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
expect(step2CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
})
it("should not retry steps X times automatically when maxRetries is set and autoRetry is false", (done) => {
;(async () => {
const transactionId = "transaction-auto-retries" + ulid()
const workflowId = "workflow_1_auto_retries_false"
it("should not retry steps X times automatically when maxRetries is set and autoRetry is false", async () => {
const transactionId = "transaction-auto-retries" + ulid()
const workflowId = "workflow_1_auto_retries_false"
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
throwOnError: false,
})
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
throwOnError: false,
})
const onFinishPromise = new Promise<void>((resolve, reject) => {
workflowOrcModule.subscribe({
workflowId,
transactionId,
@@ -471,37 +455,39 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
expect(
step2CompensateMockAutoRetriesFalse
).toHaveBeenCalledTimes(1)
done()
resolve()
}
},
})
})
expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
await setTimeoutPromise(2000)
await setTimeoutPromise(2000)
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
throwOnError: false,
})
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
throwOnError: false,
})
await setTimeoutPromise(2000)
await setTimeoutPromise(2000)
expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(2)
expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(2)
expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
throwOnError: false,
})
})()
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
throwOnError: false,
})
await onFinishPromise
})
it("should prevent executing twice the same workflow in perfect concurrency with the same transactionId and non idempotent and not async but retention time is set", async () => {
@@ -611,7 +597,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
)
})
it("should compose nested workflows w/ async steps", (done) => {
it("should compose nested workflows w/ async steps", async () => {
const asyncResults: any[] = []
const mockStep1Fn = jest.fn().mockImplementation(() => {
const res = { obj: "return from 1" }
@@ -662,35 +648,31 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
})
asyncResults.push("begin workflow")
workflowOrcModule
.run(workflowId, {
input: {},
})
.then(() => {
asyncResults.push("returned workflow")
await workflowOrcModule.run(workflowId, {
input: {},
})
void workflowOrcModule.subscribe({
workflowId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
expect(asyncResults).toEqual([
"begin workflow",
{ obj: "return from 1" },
"returned workflow",
{ obj: "return from 2" },
{ obj: "return from 3" },
])
done()
clearTimeout(timeout)
}
},
})
const onFinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
expect(asyncResults).toEqual([
"begin workflow",
{ obj: "return from 1" },
"returned workflow",
{ obj: "return from 2" },
{ obj: "return from 3" },
])
resolve()
}
},
})
})
const timeout = failTrap(
done,
"should subscribe to a async workflow and receive the response when it finishes"
)
asyncResults.push("returned workflow")
await onFinishPromise
})
describe("Testing basic workflow", function () {
@@ -871,22 +853,19 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
expect(transaction.getFlow().state).toEqual("reverted")
})
it("should subscribe to a async workflow and receive the response when it finishes", (done) => {
it("should subscribe to a async workflow and receive the response when it finishes", async () => {
const transactionId = "trx_123" + ulid()
const onFinish = jest.fn(() => {
done()
clearTimeout(timeout)
})
void workflowOrcModule.subscribe({
workflowId: "workflow_async_background",
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
onFinish()
}
},
const onFinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId: "workflow_async_background",
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
void workflowOrcModule.run("workflow_async_background", {
@@ -897,11 +876,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
throwOnError: false,
})
expect(onFinish).toHaveBeenCalledTimes(0)
const timeout = failTrap(
done,
"should subscribe to a async workflow and receive the response when it finishes"
)
await onFinishPromise
})
it("should cancel and revert a completed workflow", async () => {
@@ -955,43 +930,43 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
expect(executions[0].state).toEqual(TransactionState.REVERTED)
})
it("should run conditional steps if condition is true", (done) => {
void workflowOrcModule.subscribe({
workflowId: "workflow_conditional_step",
subscriber: (event) => {
if (event.eventType === "onFinish") {
expect(conditionalStep2Invoke).toHaveBeenCalledTimes(2)
expect(conditionalStep3Invoke).toHaveBeenCalledTimes(1)
done()
clearTimeout(timeout)
}
},
it("should run conditional steps if condition is true", async () => {
let timeout: NodeJS.Timeout
const onFinishPromise = new Promise<void>((resolve, reject) => {
void workflowOrcModule.subscribe({
workflowId: "workflow_conditional_step",
subscriber: (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
workflowOrcModule.run("workflow_conditional_step", {
void workflowOrcModule.run("workflow_conditional_step", {
input: {
runNewStepName: true,
},
throwOnError: true,
})
const timeout = failTrap(
done,
"should not run conditional steps if condition is false"
)
await onFinishPromise
expect(conditionalStep2Invoke).toHaveBeenCalledTimes(2)
expect(conditionalStep3Invoke).toHaveBeenCalledTimes(1)
})
it("should not run conditional steps if condition is false", (done) => {
void workflowOrcModule.subscribe({
workflowId: "workflow_conditional_step",
subscriber: (event) => {
if (event.eventType === "onFinish") {
expect(conditionalStep2Invoke).toHaveBeenCalledTimes(1)
expect(conditionalStep3Invoke).toHaveBeenCalledTimes(0)
done()
clearTimeout(timeout)
}
},
it("should not run conditional steps if condition is false", async () => {
const onFinishPromise = new Promise<void>((resolve, reject) => {
void workflowOrcModule.subscribe({
workflowId: "workflow_conditional_step",
subscriber: (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
workflowOrcModule.run("workflow_conditional_step", {
@@ -1001,10 +976,10 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
throwOnError: true,
})
const timeout = failTrap(
done,
"should not run conditional steps if condition is false"
)
await onFinishPromise
expect(conditionalStep2Invoke).toHaveBeenCalledTimes(1)
expect(conditionalStep3Invoke).toHaveBeenCalledTimes(0)
})
})
@@ -1127,36 +1102,36 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
expect(executionsListAfter).toHaveLength(1)
})
it("should display error when multple async steps are running in parallel", (done) => {
it("should display error when multple async steps are running in parallel", async () => {
let errors: Error[] = []
const onFinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId: "workflow_parallel_async",
subscriber: (event) => {
if (event.eventType === "onFinish") {
errors = event.errors
resolve()
}
},
})
})
void workflowOrcModule.run("workflow_parallel_async", {
input: {},
throwOnError: false,
})
void workflowOrcModule.subscribe({
workflowId: "workflow_parallel_async",
subscriber: (event) => {
if (event.eventType === "onFinish") {
expect(event.errors).toEqual(
expect.arrayContaining([
expect.objectContaining({
action: "step_2",
handlerType: "invoke",
error: expect.objectContaining({
message: "Error in parallel step",
}),
}),
])
)
done()
clearTimeout(timeout)
}
},
})
await onFinishPromise
const timeout = failTrap(
done,
"should display error when multple async steps are running in parallel"
const errMessage = errors[0]?.error.message
expect(errMessage).toContain("Error in parallel step")
expect(errors).toEqual(
expect.arrayContaining([
expect.objectContaining({
action: "step_2",
handlerType: "invoke",
}),
])
)
})
})

366
packages/modules/workflow-engine-inmemory/integration-tests/__tests__/race.spec.ts
View File

@@ -1,36 +1,187 @@
import { IWorkflowEngineService } from "@medusajs/framework/types"
import { Modules } from "@medusajs/framework/utils"
import { Modules, TransactionHandlerType } from "@medusajs/framework/utils"
import {
createStep,
createWorkflow,
parallelize,
StepResponse,
transform,
WorkflowResponse,
} from "@medusajs/framework/workflows-sdk"
import { moduleIntegrationTestRunner } from "@medusajs/test-utils"
import { setTimeout as setTimeoutSync } from "timers"
import { setTimeout } from "timers/promises"
import { ulid } from "ulid"
import "../__fixtures__"
jest.setTimeout(300000)
const failTrap = (done, name, timeout = 5000) => {
return setTimeoutSync(() => {
// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
console.warn(
`Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
)
done()
}, timeout)
}
jest.setTimeout(30000)
moduleIntegrationTestRunner<IWorkflowEngineService>({
moduleName: Modules.WORKFLOW_ENGINE,
resolve: __dirname + "/../..",
testSuite: ({ service: workflowOrcModule, medusaApp }) => {
testSuite: ({ service: workflowOrcModule }) => {
// TODO: Debug the issue with this test https://github.com/medusajs/medusa/actions/runs/13900190144/job/38897122803#step:5:5616
describe.skip("Testing race condition of the workflow during retry", () => {
it("should prevent race continuation of the workflow during retryIntervalAwaiting in background execution", (done) => {
describe("Testing race condition of the workflow during retry", () => {
it("should manage saving multiple async steps in concurrency", async () => {
const step0 = createStep(
{ name: "step0", async: true, backgroundExecution: true },
async () => {
return new StepResponse("result from step 0")
}
)
const step1 = createStep(
{ name: "step1", async: true, backgroundExecution: true },
async () => {
return new StepResponse("result from step 1")
}
)
const step2 = createStep(
{ name: "step2", async: true, backgroundExecution: true },
async () => {
return new StepResponse("result from step 2")
}
)
const step3 = createStep(
{ name: "step3", async: true, backgroundExecution: true },
async () => {
return new StepResponse("result from step 3")
}
)
const step4 = createStep(
{ name: "step4", async: true, backgroundExecution: true },
async () => {
return new StepResponse("result from step 4")
}
)
const step5 = createStep({ name: "step5" }, async (all: string[]) => {
const ret = [...all, "result from step 5"]
return new StepResponse(ret)
})
const workflowId = "workflow-1" + ulid()
createWorkflow(
{
name: workflowId,
idempotent: true,
retentionTime: 5,
},
function () {
const all = parallelize(step0(), step1(), step2(), step3(), step4())
const res = step5(all)
return new WorkflowResponse(res)
}
)
const transactionId = ulid()
const done = new Promise<void>((resolve, reject) => {
void workflowOrcModule.subscribe({
workflowId: workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
resolve(event.result)
}
},
})
})
await workflowOrcModule.run(workflowId, {
throwOnError: false,
logOnError: true,
transactionId,
})
const result = await done
expect(result).toEqual([
"result from step 0",
"result from step 1",
"result from step 2",
"result from step 3",
"result from step 4",
"result from step 5",
])
})
it("should manage saving multiple async steps in concurrency without background execution while setting steps as success manually concurrently", async () => {
const step0 = createStep({ name: "step0", async: true }, async () => {})
const step1 = createStep({ name: "step1", async: true }, async () => {})
const step2 = createStep({ name: "step2", async: true }, async () => {})
const step3 = createStep({ name: "step3", async: true }, async () => {})
const step4 = createStep({ name: "step4", async: true }, async () => {})
const step5 = createStep({ name: "step5" }, async (all: any[]) => {
const ret = [...all, "result from step 5"]
return new StepResponse(ret)
})
const workflowId = "workflow-1" + ulid()
createWorkflow(
{
name: workflowId,
idempotent: true,
retentionTime: 1,
},
function () {
const all = parallelize(step0(), step1(), step2(), step3(), step4())
const res = step5(all)
return new WorkflowResponse(res)
}
)
const transactionId = ulid()
const done = new Promise<void>((resolve, reject) => {
void workflowOrcModule.subscribe({
workflowId: workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
resolve(event.result)
}
},
})
})
await workflowOrcModule.run(workflowId, {
throwOnError: false,
logOnError: true,
transactionId,
})
await setTimeout(100) // Just to wait a bit before firering everything
for (let i = 0; i <= 4; i++) {
void workflowOrcModule.setStepSuccess({
idempotencyKey: {
workflowId: workflowId,
transactionId: transactionId,
stepId: `step${i}`,
action: TransactionHandlerType.INVOKE,
},
stepResponse: new StepResponse("result from step " + i),
})
}
const res = await done
expect(res).toEqual([
"result from step 0",
"result from step 1",
"result from step 2",
"result from step 3",
"result from step 4",
"result from step 5",
])
})
it("should prevent race continuation of the workflow during retryIntervalAwaiting in background execution", async () => {
const transactionId = "transaction_id" + ulid()
const workflowId = "RACE_workflow-1" + ulid()
const step0InvokeMock = jest.fn()
const step1InvokeMock = jest.fn()
const step2InvokeMock = jest.fn()
@@ -43,7 +194,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
const step1 = createStep("step1", async (_) => {
step1InvokeMock()
await setTimeout(2000)
await setTimeout(200)
return new StepResponse({ isSuccess: true })
})
@@ -57,57 +208,67 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
return new WorkflowResponse(status)
})
createWorkflow("workflow-1", function () {
const build = step0()
const status = subWorkflow.runAsStep({} as any).config({
async: true,
compensateAsync: true,
backgroundExecution: true,
retryIntervalAwaiting: 1,
})
const transformedResult = transform({ status }, (data) => {
transformMock()
return {
status: data.status,
}
})
step2(transformedResult)
return new WorkflowResponse(build)
})
void workflowOrcModule.subscribe({
workflowId: "workflow-1",
subscriber: (event) => {
if (event.eventType === "onFinish") {
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
expect(step1InvokeMock.mock.calls.length).toBeGreaterThan(1)
expect(step2InvokeMock).toHaveBeenCalledTimes(1)
expect(transformMock).toHaveBeenCalledTimes(1)
done()
clearTimeout(timeout)
}
createWorkflow(
{
name: workflowId,
idempotent: true,
retentionTime: 5,
},
function () {
const build = step0()
const status = subWorkflow.runAsStep({} as any).config({
async: true,
compensateAsync: true,
backgroundExecution: true,
retryIntervalAwaiting: 0.1,
})
const transformedResult = transform({ status }, (data) => {
transformMock()
return {
status: data.status,
}
})
step2(transformedResult)
return new WorkflowResponse(build)
}
)
const onFinish = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
workflowOrcModule
.run("workflow-1", { throwOnError: false })
.run(workflowId, {
transactionId,
throwOnError: false,
logOnError: true,
})
.then(({ result }) => {
expect(result).toBe("result from step 0")
})
.catch((e) => e)
const timeout = failTrap(
done,
"should prevent race continuation of the workflow during retryIntervalAwaiting in background execution"
)
await onFinish
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
expect(step1InvokeMock.mock.calls.length).toBeGreaterThan(1)
expect(step2InvokeMock).toHaveBeenCalledTimes(1)
expect(transformMock).toHaveBeenCalledTimes(1)
})
it("should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution", (done) => {
const workflowId = "RACE_workflow-1"
it("should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution", async () => {
const transactionId = "transaction_id" + ulid()
const workflowId = "RACE_workflow-1" + ulid()
const step0InvokeMock = jest.fn()
const step0CompensateMock = jest.fn()
@@ -131,7 +292,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
"RACE_step1",
async (_) => {
step1InvokeMock()
await setTimeout(300)
await setTimeout(1000)
throw new Error("error from step 1")
},
() => {
@@ -149,56 +310,63 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
return new WorkflowResponse(status)
})
createWorkflow(workflowId, function () {
const build = step0()
const status = subWorkflow.runAsStep({} as any).config({
async: true,
compensateAsync: true,
backgroundExecution: true,
retryIntervalAwaiting: 0.1,
})
const transformedResult = transform({ status }, (data) => {
transformMock()
return {
status: data.status,
}
})
step2(transformedResult)
return new WorkflowResponse(build)
})
void workflowOrcModule.subscribe({
workflowId: workflowId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
expect(step0CompensateMock).toHaveBeenCalledTimes(1)
expect(step1InvokeMock.mock.calls.length).toBeGreaterThan(2)
expect(step1CompensateMock).toHaveBeenCalledTimes(1)
expect(step2InvokeMock).toHaveBeenCalledTimes(0)
expect(transformMock).toHaveBeenCalledTimes(0)
done()
clearTimeout(timeout)
}
createWorkflow(
{
name: workflowId,
},
function () {
const build = step0()
const status = subWorkflow.runAsStep({} as any).config({
async: true,
compensateAsync: true,
backgroundExecution: true,
retryIntervalAwaiting: 0.1,
maxAwaitingRetries: 3,
})
const transformedResult = transform({ status }, (data) => {
transformMock()
return {
status: data.status,
}
})
step2(transformedResult)
return new WorkflowResponse(build)
}
)
const onFinish = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
workflowOrcModule
await workflowOrcModule
.run(workflowId, {
transactionId,
throwOnError: false,
logOnError: true,
})
.then(({ result }) => {
expect(result).toBe("result from step 0")
})
.catch((e) => e)
const timeout = failTrap(
done,
"should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution"
)
await onFinish
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
expect(step0CompensateMock).toHaveBeenCalledTimes(1)
expect(step1InvokeMock).toHaveBeenCalledTimes(3)
expect(step1CompensateMock.mock.calls.length).toBeGreaterThan(0)
expect(step2InvokeMock).toHaveBeenCalledTimes(0)
expect(transformMock).toHaveBeenCalledTimes(0)
})
})
},

185
packages/modules/workflow-engine-inmemory/integration-tests/__tests__/subscribe.spec.ts Normal file
View File

@@ -0,0 +1,185 @@
import { IWorkflowEngineService } from "@medusajs/framework/types"
import { Modules } from "@medusajs/framework/utils"
import {
createStep,
createWorkflow,
StepResponse,
WorkflowResponse,
} from "@medusajs/framework/workflows-sdk"
import { moduleIntegrationTestRunner } from "@medusajs/test-utils"
import { setTimeout as setTimeoutSync } from "timers"
import { setTimeout as setTimeoutPromise } from "timers/promises"
import { ulid } from "ulid"
import "../__fixtures__"
jest.setTimeout(60000)
moduleIntegrationTestRunner<IWorkflowEngineService>({
moduleName: Modules.WORKFLOW_ENGINE,
resolve: __dirname + "/../..",
testSuite: ({ service: workflowOrcModule }) => {
describe("Workflow Orchestrator module subscribe", function () {
it("should subscribe to a workflow and receive the response when it finishes", async () => {
const step1 = createStep({ name: "step1" }, async () => {
return new StepResponse("step1")
})
const step2 = createStep({ name: "step2" }, async () => {
await setTimeoutPromise(1000)
return new StepResponse("step2")
})
const workflowId = "workflow" + ulid()
createWorkflow(workflowId, function (input) {
step1()
step2().config({
async: true,
})
return new WorkflowResponse("workflow")
})
const step1_1 = createStep({ name: "step1_1" }, async () => {
return new StepResponse("step1_1")
})
const step2_1 = createStep({ name: "step2_1" }, async () => {
await setTimeoutPromise(1000)
return new StepResponse("step2_1")
})
const workflow2Id = "workflow_2" + ulid()
createWorkflow(workflow2Id, function (input) {
step1_1()
step2_1().config({
async: true,
})
return new WorkflowResponse("workflow_2")
})
const transactionId = "trx_123" + ulid()
const transactionId2 = "trx_124" + ulid()
const onWorkflowFinishSpy = jest.fn()
const onWorkflowFinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId: workflowId,
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
onWorkflowFinishSpy()
workflowOrcModule.run(workflow2Id, {
transactionId: transactionId2,
})
resolve()
}
},
})
})
const onWorkflow2FinishSpy = jest.fn()
const workflow2FinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId: workflow2Id,
subscriber: (event) => {
if (event.eventType === "onFinish") {
onWorkflow2FinishSpy()
resolve()
}
},
})
})
workflowOrcModule.run(workflowId, {
transactionId,
})
await onWorkflowFinishPromise
await workflow2FinishPromise
expect(onWorkflowFinishSpy).toHaveBeenCalledTimes(1)
expect(onWorkflow2FinishSpy).toHaveBeenCalledTimes(1)
})
it("should subscribe to a workflow and receive the response when it finishes (2)", async () => {
const step1 = createStep({ name: "step1" }, async () => {
return new StepResponse("step1")
})
const step2 = createStep({ name: "step2" }, async () => {
await setTimeoutPromise(1000)
return new StepResponse("step2")
})
const workflowId = "workflow" + ulid()
createWorkflow(workflowId, function (input) {
step1()
step2().config({
async: true,
})
return new WorkflowResponse("workflow")
})
const step1_1 = createStep({ name: "step1_1" }, async () => {
return new StepResponse("step1_1")
})
const step2_1 = createStep({ name: "step2_1" }, async () => {
await setTimeoutPromise(1000)
return new StepResponse("step2_1")
})
const workflow2Id = "workflow_2" + ulid()
createWorkflow(workflow2Id, function (input) {
step1_1()
step2_1().config({
async: true,
})
return new WorkflowResponse("workflow_2")
})
const transactionId = "trx_123" + ulid()
const transactionId2 = "trx_124" + ulid()
const onWorkflowFinishSpy = jest.fn()
const onWorkflowFinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId: workflowId,
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
onWorkflowFinishSpy()
workflowOrcModule.run(workflow2Id, {
transactionId: transactionId2,
})
resolve()
}
},
})
})
const onWorkflow2FinishSpy = jest.fn()
const workflow2FinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId: workflow2Id,
subscriber: (event) => {
if (event.eventType === "onFinish") {
onWorkflow2FinishSpy()
resolve()
}
},
})
})
workflowOrcModule.run(workflowId, {
transactionId,
})
await onWorkflowFinishPromise
await workflow2FinishPromise
expect(onWorkflowFinishSpy).toHaveBeenCalledTimes(1)
expect(onWorkflow2FinishSpy).toHaveBeenCalledTimes(1)
})
})
},
})

2
packages/modules/workflow-engine-inmemory/package.json
View File

@@ -29,7 +29,7 @@
"resolve:aliases": "tsc --showConfig -p tsconfig.json > tsconfig.resolved.json && tsc-alias -p tsconfig.resolved.json && rimraf tsconfig.resolved.json",
"build": "rimraf dist && tsc --build && npm run resolve:aliases",
"test": "jest --passWithNoTests --bail --forceExit -- src",
"test:integration": "jest --forceExit -- integration-tests/**/__tests__/*.ts",
"test:integration": "jest --forceExit -- integration-tests/**/__tests__/**/*.spec.ts",
"migration:initial": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:create --initial",
"migration:create": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:create",
"migration:up": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:up",

50
packages/modules/workflow-engine-inmemory/src/services/workflow-orchestrator.ts
View File

@@ -100,7 +100,7 @@ type Subscribers = Map<WorkflowId, TransactionSubscribers>
const AnySubscriber = "any"
export class WorkflowOrchestratorService {
private subscribers: Subscribers = new Map()
private static subscribers: Subscribers = new Map()
private container_: MedusaContainer
private inMemoryDistributedTransactionStorage_: InMemoryDistributedTransactionStorage
readonly #logger: Logger
@@ -110,7 +110,6 @@ export class WorkflowOrchestratorService {
sharedContainer,
}: {
inMemoryDistributedTransactionStorage: InMemoryDistributedTransactionStorage
workflowOrchestratorService: WorkflowOrchestratorService
sharedContainer: MedusaContainer
}) {
this.container_ = sharedContainer
@@ -133,9 +132,18 @@ export class WorkflowOrchestratorService {
await this.inMemoryDistributedTransactionStorage_.onApplicationShutdown()
}
private async triggerParentStep(transaction, result) {
private async triggerParentStep(transaction, result, errors) {
const metadata = transaction.flow.metadata
const { parentStepIdempotencyKey } = metadata ?? {}
const { parentStepIdempotencyKey, cancelingFromParentStep } = metadata ?? {}
if (cancelingFromParentStep) {
/**
* If the sub workflow is cancelling from a parent step, we don't want to trigger the parent
* step.
*/
return
}
if (parentStepIdempotencyKey) {
const hasFailed = [
TransactionState.REVERTED,
@@ -145,12 +153,18 @@ export class WorkflowOrchestratorService {
if (hasFailed) {
await this.setStepFailure({
idempotencyKey: parentStepIdempotencyKey,
stepResponse: result,
stepResponse: errors,
options: {
logOnError: true,
},
})
} else {
await this.setStepSuccess({
idempotencyKey: parentStepIdempotencyKey,
stepResponse: result,
options: {
logOnError: true,
},
})
}
}
@@ -237,7 +251,7 @@ export class WorkflowOrchestratorService {
errors,
})
await this.triggerParentStep(ret.transaction, result)
await this.triggerParentStep(ret.transaction, result, errors)
}
if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -349,7 +363,7 @@ export class WorkflowOrchestratorService {
errors,
})
await this.triggerParentStep(ret.transaction, result)
await this.triggerParentStep(ret.transaction, result, errors)
}
if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -443,7 +457,7 @@ export class WorkflowOrchestratorService {
errors,
})
await this.triggerParentStep(ret.transaction, result)
await this.triggerParentStep(ret.transaction, result, errors)
}
if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -514,7 +528,7 @@ export class WorkflowOrchestratorService {
errors,
})
await this.triggerParentStep(ret.transaction, result)
await this.triggerParentStep(ret.transaction, result, errors)
}
if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -587,7 +601,7 @@ export class WorkflowOrchestratorService {
errors,
})
await this.triggerParentStep(ret.transaction, result)
await this.triggerParentStep(ret.transaction, result, errors)
}
if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -608,7 +622,8 @@ export class WorkflowOrchestratorService {
subscriberId,
}: SubscribeOptions) {
subscriber._id = subscriberId
const subscribers = this.subscribers.get(workflowId) ?? new Map()
const subscribers =
WorkflowOrchestratorService.subscribers.get(workflowId) ?? new Map()
const handlerIndex = (handlers) => {
return handlers.findIndex(
@@ -625,7 +640,7 @@ export class WorkflowOrchestratorService {
transactionSubscribers.push(subscriber)
subscribers.set(transactionId, transactionSubscribers)
this.subscribers.set(workflowId, subscribers)
WorkflowOrchestratorService.subscribers.set(workflowId, subscribers)
return
}
@@ -637,7 +652,7 @@ export class WorkflowOrchestratorService {
workflowSubscribers.push(subscriber)
subscribers.set(AnySubscriber, workflowSubscribers)
this.subscribers.set(workflowId, subscribers)
WorkflowOrchestratorService.subscribers.set(workflowId, subscribers)
}
unsubscribe({
@@ -645,7 +660,8 @@ export class WorkflowOrchestratorService {
transactionId,
subscriberOrId,
}: UnsubscribeOptions) {
const subscribers = this.subscribers.get(workflowId) ?? new Map()
const subscribers =
WorkflowOrchestratorService.subscribers.get(workflowId) ?? new Map()
const filterSubscribers = (handlers: SubscriberHandler[]) => {
return handlers.filter((handler) => {
@@ -665,7 +681,7 @@ export class WorkflowOrchestratorService {
} else {
subscribers.delete(transactionId)
}
this.subscribers.set(workflowId, subscribers)
WorkflowOrchestratorService.subscribers.set(workflowId, subscribers)
return
}
@@ -676,7 +692,7 @@ export class WorkflowOrchestratorService {
} else {
subscribers.delete(AnySubscriber)
}
this.subscribers.set(workflowId, subscribers)
WorkflowOrchestratorService.subscribers.set(workflowId, subscribers)
}
private notify(options: NotifyOptions) {
@@ -687,7 +703,7 @@ export class WorkflowOrchestratorService {
private async processSubscriberNotifications(options: NotifyOptions) {
const { workflowId, transactionId, eventType } = options
const subscribers: TransactionSubscribers =
this.subscribers.get(workflowId) ?? new Map()
WorkflowOrchestratorService.subscribers.get(workflowId) ?? new Map()
const notifySubscribersAsync = async (handlers: SubscriberHandler[]) => {
const promises = handlers.map(async (handler) => {

381
packages/modules/workflow-engine-inmemory/src/utils/workflow-orchestrator-storage.ts
View File

@@ -20,10 +20,10 @@ import {
ModulesSdkTypes,
} from "@medusajs/framework/types"
import {
isPresent,
MedusaError,
TransactionState,
TransactionStepState,
isPresent,
} from "@medusajs/framework/utils"
import { WorkflowOrchestratorService } from "@services"
import { type CronExpression, parseExpression } from "cron-parser"
@@ -31,6 +31,23 @@ import { WorkflowExecution } from "../models/workflow-execution"
const THIRTY_MINUTES_IN_MS = 1000 * 60 * 30
const doneStates = [
TransactionStepState.DONE,
TransactionStepState.REVERTED,
TransactionStepState.FAILED,
TransactionStepState.SKIPPED,
TransactionStepState.SKIPPED_FAILURE,
TransactionStepState.TIMEOUT,
]
const finishedStates = [
TransactionState.DONE,
TransactionState.FAILED,
TransactionState.REVERTED,
]
const failedStates = [TransactionState.FAILED, TransactionState.REVERTED]
function calculateDelayFromExpression(expression: CronExpression): number {
const nextTime = expression.next().getTime()
const now = Date.now()
@@ -71,24 +88,6 @@ function parseNextExecution(
return result
}
const invokingStatesSet = new Set([
TransactionStepState.INVOKING,
TransactionStepState.NOT_STARTED,
])
const compensatingStatesSet = new Set([
TransactionStepState.COMPENSATING,
TransactionStepState.NOT_STARTED,
])
function isInvokingState(step: TransactionStep) {
return invokingStatesSet.has(step.invoke?.state)
}
function isCompensatingState(step: TransactionStep) {
return compensatingStatesSet.has(step.compensate?.state)
}
export class InMemoryDistributedTransactionStorage
implements IDistributedTransactionStorage, IDistributedSchedulerStorage
{
@@ -96,8 +95,7 @@ export class InMemoryDistributedTransactionStorage
private logger_: Logger
private workflowOrchestratorService_: WorkflowOrchestratorService
private storage: Map<string, Omit<TransactionCheckpoint, "context">> =
new Map()
private storage: Record<string, TransactionCheckpoint> = {}
private scheduled: Map<
string,
{
@@ -112,6 +110,7 @@ export class InMemoryDistributedTransactionStorage
private pendingTimers: Set<NodeJS.Timeout> = new Set()
private clearTimeout_: NodeJS.Timeout
private isLocked: Map<string, boolean> = new Map()
constructor({
workflowExecutionService,
@@ -179,29 +178,11 @@ export class InMemoryDistributedTransactionStorage
private async saveToDb(data: TransactionCheckpoint, retentionTime?: number) {
const isNotStarted = data.flow.state === TransactionState.NOT_STARTED
const isFinished = [
TransactionState.DONE,
TransactionState.FAILED,
TransactionState.REVERTED,
].includes(data.flow.state)
const asyncVersion = data.flow._v
const isFinished = finishedStates.includes(data.flow.state)
const isWaitingToCompensate =
data.flow.state === TransactionState.WAITING_TO_COMPENSATE
/**
* Bit of explanation:
*
* When a workflow run, it run all sync step in memory until it reaches a async step.
* In that case, it might handover to another process to continue the execution. Thats why
* we need to save the current state of the flow. Then from there, it will run again all
* sync steps until the next async step. an so on so forth.
*
* To summarize, we only trully need to save the data when we are reaching any steps that
* trigger a handover to a potential other process.
*
* This allows us to spare some resources and time by not over communicating with the external
* database when it is not really needed
*/
const isFlowInvoking = data.flow.state === TransactionState.INVOKING
const stepsArray = Object.values(data.flow.steps) as TransactionStep[]
@@ -240,7 +221,8 @@ export class InMemoryDistributedTransactionStorage
if (
!(isNotStarted || isFinished || isWaitingToCompensate) &&
!currentStepsIsAsync
!currentStepsIsAsync &&
!asyncVersion
) {
return
}
@@ -295,15 +277,14 @@ export class InMemoryDistributedTransactionStorage
.catch(() => undefined)
if (trx) {
const { flow, errors } = this.storage.get(key) ?? {}
const { flow, errors } = this.storage[key]
? JSON.parse(JSON.stringify(this.storage[key]))
: {}
const { idempotent } = options ?? {}
const execution = trx.execution as TransactionFlow
if (!idempotent) {
const isFailedOrReverted = [
TransactionState.REVERTED,
TransactionState.FAILED,
].includes(execution.state)
const isFailedOrReverted = failedStates.includes(execution.state)
const isDone = execution.state === TransactionState.DONE
@@ -321,11 +302,11 @@ export class InMemoryDistributedTransactionStorage
}
}
return {
flow: flow ?? (trx.execution as TransactionFlow),
context: trx.context?.data as TransactionContext,
errors: errors ?? (trx.context?.errors as TransactionStepError[]),
}
return new TransactionCheckpoint(
flow ?? (trx?.execution as TransactionFlow),
trx?.context?.data as TransactionContext,
errors ?? (trx?.context?.errors as TransactionStepError[])
)
}
return
@@ -336,68 +317,94 @@ export class InMemoryDistributedTransactionStorage
data: TransactionCheckpoint,
ttl?: number,
options?: TransactionOptions
): Promise<void> {
/**
* Store the retention time only if the transaction is done, failed or reverted.
* From that moment, this tuple can be later on archived or deleted after the retention time.
*/
const hasFinished = [
TransactionState.DONE,
TransactionState.FAILED,
TransactionState.REVERTED,
].includes(data.flow.state)
const { retentionTime } = options ?? {}
await this.#preventRaceConditionExecutionIfNecessary({
data,
key,
options,
})
// Only store retention time if it's provided
if (retentionTime) {
Object.assign(data, {
retention_time: retentionTime,
})
): Promise<TransactionCheckpoint> {
if (this.isLocked.has(key)) {
throw new Error("Transaction storage is locked")
}
// Store in memory
const isNotStarted = data.flow.state === TransactionState.NOT_STARTED
const isManualTransactionId = !data.flow.transactionId.startsWith("auto-")
this.isLocked.set(key, true)
if (isNotStarted && isManualTransactionId) {
const storedData = this.storage.get(key)
if (storedData) {
throw new SkipExecutionError(
"Transaction already started for transactionId: " +
data.flow.transactionId
)
try {
/**
* Store the retention time only if the transaction is done, failed or reverted.
* From that moment, this tuple can be later on archived or deleted after the retention time.
*/
const { retentionTime } = options ?? {}
const hasFinished = finishedStates.includes(data.flow.state)
let cachedCheckpoint: TransactionCheckpoint | undefined
const getCheckpoint = async (options?: TransactionOptions) => {
if (!cachedCheckpoint) {
cachedCheckpoint = await this.get(key, options)
}
return cachedCheckpoint
}
}
const { flow, errors } = data
this.storage.set(key, {
flow,
errors,
})
await this.#preventRaceConditionExecutionIfNecessary({
data,
key,
options,
getCheckpoint,
})
// Optimize DB operations - only perform when necessary
if (hasFinished) {
if (!retentionTime) {
// If the workflow is nested, we cant just remove it because it would break the compensation algorithm. Instead, it will get deleted when the top level parent is deleted.
if (!flow.metadata?.parentStepIdempotencyKey) {
await this.deleteFromDb(data)
// Only store retention time if it's provided
if (retentionTime) {
Object.assign(data, {
retention_time: retentionTime,
})
}
// Store in memory
const isNotStarted = data.flow.state === TransactionState.NOT_STARTED
const isManualTransactionId = !data.flow.transactionId.startsWith("auto-")
if (isNotStarted && isManualTransactionId) {
const storedData = this.storage[key]
if (storedData) {
throw new SkipExecutionError(
"Transaction already started for transactionId: " +
data.flow.transactionId
)
}
}
if (data.flow._v) {
const storedData = await this.get(key, {
isCancelling: !!data.flow.cancelledAt,
} as any)
TransactionCheckpoint.mergeCheckpoints(data, storedData)
}
const { flow, errors } = data
this.storage[key] = {
flow,
context: {} as TransactionContext,
errors,
} as TransactionCheckpoint
// Optimize DB operations - only perform when necessary
if (hasFinished) {
if (!retentionTime) {
if (!flow.metadata?.parentStepIdempotencyKey) {
await this.deleteFromDb(data)
} else {
await this.saveToDb(data, retentionTime)
}
} else {
await this.saveToDb(data, retentionTime)
}
delete this.storage[key]
} else {
await this.saveToDb(data, retentionTime)
}
this.storage.delete(key)
} else {
await this.saveToDb(data, retentionTime)
return data
} finally {
this.isLocked.delete(key)
}
}
@@ -405,28 +412,25 @@ export class InMemoryDistributedTransactionStorage
data,
key,
options,
getCheckpoint,
}: {
data: TransactionCheckpoint
key: string
options?: TransactionOptions
getCheckpoint: (
options: TransactionOptions
) => Promise<TransactionCheckpoint | undefined>
}) {
// TODO: comment, we have been able to try to replace this entire function
// with a locking first approach. We might come back to that another time.
// This remove the necessity of all the below logic to prevent race conditions
// by preventing the exact same execution to run at the same time.
// See early commits from: https://github.com/medusajs/medusa/pull/13345/commits
const isInitialCheckpoint = [TransactionState.NOT_STARTED].includes(
data.flow.state
)
/**
* In case many execution can succeed simultaneously, we need to ensure that the latest
* execution does continue if a previous execution is considered finished
*/
const currentFlow = data.flow
const rawData = this.storage.get(key)
const rawData = this.storage[key]
let data_ = {} as TransactionCheckpoint
if (rawData) {
data_ = rawData as TransactionCheckpoint
@@ -437,13 +441,37 @@ export class InMemoryDistributedTransactionStorage
} as Parameters<typeof this.get>[1]
data_ =
(await this.get(key, getOptions)) ??
(await getCheckpoint(getOptions as TransactionOptions)) ??
({ flow: {} } as TransactionCheckpoint)
}
const { flow: latestUpdatedFlow } = data_
if (options?.stepId) {
const stepId = options.stepId
const currentStep = data.flow.steps[stepId]
const latestStep = latestUpdatedFlow.steps?.[stepId]
if (latestStep && currentStep) {
const isCompensating = data.flow.state === TransactionState.COMPENSATING
if (!isInitialCheckpoint && !isPresent(latestUpdatedFlow)) {
const latestState = isCompensating
? latestStep.compensate?.state
: latestStep.invoke?.state
const shouldSkip = doneStates.includes(latestState)
if (shouldSkip) {
throw new SkipStepAlreadyFinishedError(
`Step ${stepId} already finished by another execution`
)
}
}
}
if (
!isInitialCheckpoint &&
!isPresent(latestUpdatedFlow) &&
!data.flow.metadata?.parentStepIdempotencyKey
) {
/**
* the initial checkpoint expect no other checkpoint to have been stored.
* In case it is not the initial one and another checkpoint is trying to
@@ -453,54 +481,7 @@ export class InMemoryDistributedTransactionStorage
throw new SkipExecutionError("Already finished by another execution")
}
let currentFlowLatestExecutedStep: TransactionStep | undefined
const currentFlowSteps = Object.values(currentFlow.steps || {})
for (let i = currentFlowSteps.length - 1; i >= 0; i--) {
if (currentFlowSteps[i].lastAttempt) {
currentFlowLatestExecutedStep = currentFlowSteps[i]
break
}
}
let latestUpdatedFlowLatestExecutedStep: TransactionStep | undefined
const latestUpdatedFlowSteps = Object.values(latestUpdatedFlow.steps || {})
for (let i = latestUpdatedFlowSteps.length - 1; i >= 0; i--) {
if (latestUpdatedFlowSteps[i].lastAttempt) {
latestUpdatedFlowLatestExecutedStep = latestUpdatedFlowSteps[i]
break
}
}
/**
* The current flow and the latest updated flow have the same latest executed step.
*/
const isSameLatestExecutedStep =
currentFlowLatestExecutedStep &&
latestUpdatedFlowLatestExecutedStep &&
currentFlowLatestExecutedStep?.id ===
latestUpdatedFlowLatestExecutedStep?.id
/**
* The current flow's latest executed step has a last attempt ahead of the latest updated
* flow's latest executed step. Therefor it is fine, otherwise another execution has already
* finished the step.
*/
const isCurrentLatestExecutedStepLastAttemptAhead =
currentFlowLatestExecutedStep?.lastAttempt &&
latestUpdatedFlowLatestExecutedStep?.lastAttempt &&
currentFlowLatestExecutedStep.lastAttempt >=
latestUpdatedFlowLatestExecutedStep.lastAttempt
if (
isSameLatestExecutedStep &&
!isCurrentLatestExecutedStepLastAttemptAhead
) {
throw new SkipStepAlreadyFinishedError(
"Step already in execution ahead of the current one"
)
}
// First ensure that the latest execution was not cancelled, otherwise we skip the execution
// Ensure that the latest execution was not cancelled, otherwise we skip the execution
const latestTransactionCancelledAt = latestUpdatedFlow.cancelledAt
const currentTransactionCancelledAt = currentFlow.cancelledAt
@@ -512,86 +493,6 @@ export class InMemoryDistributedTransactionStorage
"Workflow execution has been cancelled during the execution"
)
}
const currentFlowLastInvokingStepIndex =
currentFlowSteps.findIndex(isInvokingState)
let latestUpdatedFlowLastInvokingStepIndex = !latestUpdatedFlow.steps
? 1 // There is no other execution, so the current execution is the latest
: -1
if (latestUpdatedFlow.steps) {
for (let i = 0; i < latestUpdatedFlowSteps.length; i++) {
if (isInvokingState(latestUpdatedFlowSteps[i])) {
latestUpdatedFlowLastInvokingStepIndex = i
break
}
}
}
let currentFlowLastCompensatingStepIndex = -1
for (let i = currentFlowSteps.length - 1; i >= 0; i--) {
if (isCompensatingState(currentFlowSteps[i])) {
currentFlowLastCompensatingStepIndex = currentFlowSteps.length - 1 - i
break
}
}
let latestUpdatedFlowLastCompensatingStepIndex = !latestUpdatedFlow.steps
? -1 // There is no other execution, so the current execution is the latest
: -1
if (latestUpdatedFlow.steps) {
for (let i = latestUpdatedFlowSteps.length - 1; i >= 0; i--) {
if (isCompensatingState(latestUpdatedFlowSteps[i])) {
latestUpdatedFlowLastCompensatingStepIndex =
latestUpdatedFlowSteps.length - 1 - i
break
}
}
}
const isLatestExecutionFinishedIndex = -1
const invokeShouldBeSkipped =
(latestUpdatedFlowLastInvokingStepIndex ===
isLatestExecutionFinishedIndex ||
currentFlowLastInvokingStepIndex <
latestUpdatedFlowLastInvokingStepIndex) &&
currentFlowLastInvokingStepIndex !== isLatestExecutionFinishedIndex
const compensateShouldBeSkipped =
currentFlowLastCompensatingStepIndex <
latestUpdatedFlowLastCompensatingStepIndex &&
currentFlowLastCompensatingStepIndex !== isLatestExecutionFinishedIndex &&
latestUpdatedFlowLastCompensatingStepIndex !==
isLatestExecutionFinishedIndex
const isCompensatingMismatch =
latestUpdatedFlow.state === TransactionState.COMPENSATING &&
![TransactionState.REVERTED, TransactionState.FAILED].includes(
currentFlow.state
) &&
currentFlow.state !== latestUpdatedFlow.state
const isRevertedMismatch =
latestUpdatedFlow.state === TransactionState.REVERTED &&
currentFlow.state !== TransactionState.REVERTED
const isFailedMismatch =
latestUpdatedFlow.state === TransactionState.FAILED &&
currentFlow.state !== TransactionState.FAILED
if (
(data.flow.state !== TransactionState.COMPENSATING &&
invokeShouldBeSkipped) ||
(data.flow.state === TransactionState.COMPENSATING &&
compensateShouldBeSkipped) ||
isCompensatingMismatch ||
isRevertedMismatch ||
isFailedMismatch
) {
throw new SkipExecutionError("Already finished by another execution")
}
}
async scheduleRetry(
@@ -850,7 +751,7 @@ export class InMemoryDistributedTransactionStorage
updated_at: {
$lte: raw(
(alias) =>
`CURRENT_TIMESTAMP - (INTERVAL '1 second' * retention_time)`
`CURRENT_TIMESTAMP - (INTERVAL '1 second' * "retention_time")`
),
},
state: {

378
packages/modules/workflow-engine-redis/integration-tests/__tests__/index.spec.ts
View File

@@ -57,17 +57,7 @@ import {
} from "../__fixtures__/workflow_1_manual_retry_step"
import { TestDatabase } from "../utils"
jest.setTimeout(300000)
const failTrap = (done, name, timeout = 5000) => {
return setTimeoutSync(() => {
// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
console.warn(
`Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
)
done()
}, timeout)
}
jest.setTimeout(30000)
function times(num) {
let resolver
@@ -109,6 +99,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
testSuite: ({ service: workflowOrcModule, medusaApp }) => {
describe("Workflow Orchestrator module", function () {
beforeEach(async () => {
await TestDatabase.clearTables()
jest.clearAllMocks()
query = medusaApp.query
@@ -169,14 +160,14 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
describe("Testing basic workflow", function () {
describe("Cancel transaction", function () {
it("should cancel an ongoing execution with async unfinished yet step", (done) => {
it("should cancel an ongoing execution with async unfinished yet step", async () => {
const transactionId = "transaction-to-cancel-id" + ulid()
const step1 = createStep("step1", async () => {
return new StepResponse("step1")
})
const step2 = createStep("step2", async () => {
await setTimeout(500)
await setTimeout(2000)
return new StepResponse("step2")
})
@@ -197,43 +188,37 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
}
)
workflowOrcModule
.run(workflowId, {
input: {},
const onFinishPromise = new Promise<void>((resolve) => {
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
.then(async () => {
await setTimeout(100)
})
await workflowOrcModule.cancel(workflowId, {
transactionId,
})
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
})
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
const execution =
await workflowOrcModule.listWorkflowExecutions({
transaction_id: transactionId,
})
await setTimeout(100)
expect(execution.length).toEqual(1)
expect(execution[0].state).toEqual(
TransactionState.REVERTED
)
done()
clearTimeout(timeout)
}
},
})
})
await workflowOrcModule.cancel(workflowId, {
transactionId,
})
const timeout = failTrap(
done,
"should cancel an ongoing execution with async unfinished yet step"
)
await onFinishPromise
const execution = await workflowOrcModule.listWorkflowExecutions({
transaction_id: transactionId,
})
expect(execution.length).toEqual(1)
expect(execution[0].state).toEqual(TransactionState.REVERTED)
})
it("should cancel a complete execution with a sync workflow running as async", async () => {
@@ -270,19 +255,29 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
}
)
const onFinishPromise = new Promise<void>((resolve) => {
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
})
await setTimeout(100)
await onFinishPromise
await workflowOrcModule.cancel(workflowId, {
transactionId,
})
await setTimeout(500)
const execution = await workflowOrcModule.listWorkflowExecutions({
transaction_id: transactionId,
})
@@ -397,116 +392,95 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
})
})
it("should manually retry a step that is taking too long to finish", (done) => {
it("should manually retry a step that is taking too long to finish", async () => {
const transactionId = "transaction-manual-retry" + ulid()
const workflowId = "workflow_1_manual_retry_step"
void workflowOrcModule
.run(workflowId, {
input: {},
transactionId,
})
.then(() => {
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(1)
void workflowOrcModule.retryStep({
idempotencyKey: {
workflowId,
transactionId,
stepId: "step_2",
action: "invoke",
},
})
})
workflowOrcModule.subscribe({
workflowId,
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(2)
done()
clearTimeout(timeout)
}
})
const onFinishPromise = new Promise<void>((resolve) => {
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(2)
resolve()
}
},
})
})
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(1)
await workflowOrcModule.retryStep({
idempotencyKey: {
workflowId,
transactionId,
stepId: "step_2",
action: "invoke",
},
})
const timeout = failTrap(
done,
"should manually retry a step that is taking too long to finish"
)
await onFinishPromise
})
it("should retry steps X times automatically when maxRetries is set", (done) => {
it("should retry steps X times automatically when maxRetries is set", async () => {
const transactionId = "transaction-auto-retries" + ulid()
const workflowId = "workflow_1_auto_retries"
const onFinishPromise = new Promise<void>((resolve) => {
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
void workflowOrcModule.run(workflowId, {
input: {},
transactionId,
})
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
expect(step1InvokeMockAutoRetries).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetries).toHaveBeenCalledTimes(3)
expect(step1CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
expect(step2CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
done()
clearTimeout(timeout)
}
},
})
await onFinishPromise
const timeout = failTrap(
done,
"should retry steps X times automatically when maxRetries is set"
)
expect(step1InvokeMockAutoRetries).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetries).toHaveBeenCalledTimes(3)
expect(step1CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
expect(step2CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
})
it("should not retry steps X times automatically when maxRetries is set and autoRetry is false", async () => {
const transactionId = "transaction-auto-retries" + ulid()
const workflowId = "workflow_1_auto_retries_false"
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
throwOnError: false,
})
const onFinishPromise = new Promise<void>((resolve, reject) => {
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
try {
expect(
step1InvokeMockAutoRetriesFalse
).toHaveBeenCalledTimes(1)
expect(
step2InvokeMockAutoRetriesFalse
).toHaveBeenCalledTimes(3)
expect(
step1CompensateMockAutoRetriesFalse
).toHaveBeenCalledTimes(1)
expect(
step2CompensateMockAutoRetriesFalse
).toHaveBeenCalledTimes(1)
resolve()
} catch (error) {
reject(error)
}
resolve()
}
},
})
})
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
throwOnError: false,
})
expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
@@ -529,6 +503,11 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
})
await onFinishPromise
expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(3)
expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
})
it("should prevent executing twice the same workflow in perfect concurrency with the same transactionId and non idempotent and not async but retention time is set", async () => {
@@ -585,8 +564,6 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
})
expect(executionsList).toHaveLength(1)
console.log(">>>>>>>>> setting step success")
const { result } = await workflowOrcModule.setStepSuccess({
idempotencyKey: {
action: TransactionHandlerType.INVOKE,
@@ -597,7 +574,6 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
stepResponse: { uhuuuu: "yeaah!" },
})
console.log(">>>>>>>>> setting step success done")
;({ data: executionsList } = await query.graph({
entity: "workflow_executions",
fields: ["id"],
@@ -928,41 +904,52 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
).toBe(true)
})
it("should complete an async workflow that returns a StepResponse", (done) => {
it("should complete an async workflow that returns a StepResponse", async () => {
const transactionId = "transaction_1" + ulid()
workflowOrcModule
.run("workflow_async_background", {
const onFinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId: "workflow_async_background",
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
const { transaction, result } = await workflowOrcModule.run(
"workflow_async_background",
{
input: {
myInput: "123",
},
transactionId,
throwOnError: true,
})
.then(({ transaction, result }: any) => {
expect(transaction.flow.state).toEqual(
TransactionStepState.INVOKING
)
expect(result).toEqual(undefined)
})
}
)
void workflowOrcModule.subscribe({
workflowId: "workflow_async_background",
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
done()
clearTimeout(timeout)
}
},
})
expect(transaction.flow.state).toEqual(TransactionStepState.INVOKING)
expect(result).toEqual(undefined)
const timeout = failTrap(done, "workflow_async_background")
await onFinishPromise
})
it("should subscribe to a async workflow and receive the response when it finishes", (done) => {
it("should subscribe to a async workflow and receive the response when it finishes", async () => {
const transactionId = "trx_123" + ulid()
const onFinish = jest.fn()
const onFinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId: "workflow_async_background",
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
void workflowOrcModule.run("workflow_async_background", {
input: {
@@ -972,25 +959,24 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
throwOnError: false,
})
void workflowOrcModule.subscribe({
workflowId: "workflow_async_background",
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
onFinish()
done()
clearTimeout(timeout)
}
},
})
expect(onFinish).toHaveBeenCalledTimes(0)
const timeout = failTrap(done, "workflow_async_background")
await onFinishPromise
})
it("should not skip step if condition is true", function (done) {
it("should not skip step if condition is true", async () => {
const transactionId = "trx_123_when" + ulid()
const onFinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId: "wf-when",
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
void workflowOrcModule.run("wf-when", {
input: {
callSubFlow: true,
@@ -1000,23 +986,30 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
logOnError: true,
})
void workflowOrcModule.subscribe({
workflowId: "wf-when",
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
done()
clearTimeout(timeout)
}
},
})
const timeout = failTrap(done, "wf-when")
await onFinishPromise
})
it("should cancel an async sub workflow when compensating", (done) => {
it("should cancel an async sub workflow when compensating", async () => {
const workflowId = "workflow_async_background_fail"
const transactionId = "trx_123_compensate_async_sub_workflow" + ulid()
let onCompensateStepSuccess: { step: TransactionStep } | null = null
const onFinishPromise = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: (event) => {
if (event.eventType === "onCompensateStepSuccess") {
onCompensateStepSuccess = event
}
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
void workflowOrcModule.run(workflowId, {
input: {
callSubFlow: true,
@@ -1026,31 +1019,16 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
logOnError: false,
})
let onCompensateStepSuccess: { step: TransactionStep } | null = null
await onFinishPromise
void workflowOrcModule.subscribe({
workflowId,
subscriber: (event) => {
if (event.eventType === "onCompensateStepSuccess") {
onCompensateStepSuccess = event
}
if (event.eventType === "onFinish") {
expect(onCompensateStepSuccess).toBeDefined()
expect(onCompensateStepSuccess!.step.id).toEqual(
"_root.nested_sub_flow_async_fail-as-step" // The workflow as step
)
expect(onCompensateStepSuccess!.step.compensate).toEqual({
state: "reverted",
status: "ok",
})
done()
clearTimeout(timeout)
}
},
expect(onCompensateStepSuccess).toBeDefined()
expect(onCompensateStepSuccess!.step.id).toEqual(
"_root.nested_sub_flow_async_fail-as-step" // The workflow as step
)
expect(onCompensateStepSuccess!.step.compensate).toEqual({
state: "reverted",
status: "ok",
})
const timeout = failTrap(done, "workflow_async_background_fail")
})
it("should cancel and revert a completed workflow", async () => {

458
packages/modules/workflow-engine-redis/integration-tests/__tests__/race.spec.ts
View File

@@ -1,32 +1,20 @@
import { IWorkflowEngineService } from "@medusajs/framework/types"
import { Modules } from "@medusajs/framework/utils"
import { Modules, TransactionHandlerType } from "@medusajs/framework/utils"
import {
createStep,
createWorkflow,
parallelize,
StepResponse,
transform,
WorkflowResponse,
} from "@medusajs/framework/workflows-sdk"
import { moduleIntegrationTestRunner } from "@medusajs/test-utils"
import { setTimeout as setTimeoutSync } from "timers"
import { setTimeout } from "timers/promises"
import { ulid } from "ulid"
import "../__fixtures__"
import { TestDatabase } from "../utils"
import { TestDatabase } from "../utils/database"
jest.setTimeout(300000)
const failTrap = (done, name, timeout = 5000) => {
return setTimeoutSync(() => {
// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
console.warn(
`Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
)
done()
}, timeout)
}
// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
jest.setTimeout(20000)
moduleIntegrationTestRunner<IWorkflowEngineService>({
moduleName: Modules.WORKFLOW_ENGINE,
@@ -38,14 +26,244 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
},
testSuite: ({ service: workflowOrcModule, medusaApp }) => {
describe("Testing race condition of the workflow during retry", () => {
beforeEach(async () => {
await TestDatabase.clearTables()
jest.clearAllMocks()
})
afterEach(async () => {
await TestDatabase.clearTables()
})
it("should prevent race continuation of the workflow during retryIntervalAwaiting in background execution", (done) => {
const transactionId = "transaction_id" + ulid()
it("should manage saving multiple async steps in concurrency", async () => {
const step0 = createStep(
{ name: "step0", async: true, backgroundExecution: true },
async () => {
return new StepResponse("result from step 0")
}
)
const step1 = createStep(
{ name: "step1", async: true, backgroundExecution: true },
async () => {
return new StepResponse("result from step 1")
}
)
const step2 = createStep(
{ name: "step2", async: true, backgroundExecution: true },
async () => {
return new StepResponse("result from step 2")
}
)
const step3 = createStep(
{ name: "step3", async: true, backgroundExecution: true },
async () => {
return new StepResponse("result from step 3")
}
)
const step4 = createStep(
{ name: "step4", async: true, backgroundExecution: true },
async () => {
return new StepResponse("result from step 4")
}
)
const step5 = createStep({ name: "step5" }, async (all: string[]) => {
const ret = [...all, "result from step 5"]
return new StepResponse(ret)
})
const workflowId = "workflow-1" + ulid()
const subWorkflowId = "sub-" + workflowId
createWorkflow(
{
name: workflowId,
idempotent: true,
retentionTime: 5,
},
function () {
const all = parallelize(step0(), step1(), step2(), step3(), step4())
const res = step5(all)
return new WorkflowResponse(res)
}
)
const transactionId = ulid()
const done = new Promise<void>((resolve, reject) => {
void workflowOrcModule.subscribe({
workflowId: workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
resolve(event.result)
}
},
})
})
await workflowOrcModule.run(workflowId, {
throwOnError: false,
logOnError: true,
transactionId,
})
const result = await done
expect(result).toEqual([
"result from step 0",
"result from step 1",
"result from step 2",
"result from step 3",
"result from step 4",
"result from step 5",
])
})
it("should manage saving multiple sync steps in concurrency", async () => {
const step0 = createStep({ name: "step0" }, async () => {
return new StepResponse("result from step 0")
})
const step1 = createStep({ name: "step1" }, async () => {
return new StepResponse("result from step 1")
})
const step2 = createStep({ name: "step2" }, async () => {
return new StepResponse("result from step 2")
})
const step3 = createStep({ name: "step3" }, async () => {
return new StepResponse("result from step 3")
})
const step4 = createStep({ name: "step4" }, async () => {
return new StepResponse("result from step 4")
})
const step5 = createStep({ name: "step5" }, async (all: string[]) => {
const ret = [...all, "result from step 5"]
return new StepResponse(ret)
})
const workflowId = "workflow-1" + ulid()
createWorkflow(
{
name: workflowId,
idempotent: true,
retentionTime: 5,
},
function () {
const all = parallelize(step0(), step1(), step2(), step3(), step4())
const res = step5(all)
return new WorkflowResponse(res)
}
)
const transactionId = ulid()
const done = new Promise<void>((resolve, reject) => {
void workflowOrcModule.subscribe({
workflowId: workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
resolve(event.result)
}
},
})
})
await workflowOrcModule.run(workflowId, {
throwOnError: false,
logOnError: true,
transactionId,
})
const result = await done
expect(result).toEqual([
"result from step 0",
"result from step 1",
"result from step 2",
"result from step 3",
"result from step 4",
"result from step 5",
])
})
it("should manage saving multiple async steps in concurrency without background execution while setting steps as success manually concurrently", async () => {
const step0 = createStep({ name: "step0", async: true }, async () => {})
const step1 = createStep({ name: "step1", async: true }, async () => {})
const step2 = createStep({ name: "step2", async: true }, async () => {})
const step3 = createStep({ name: "step3", async: true }, async () => {})
const step4 = createStep({ name: "step4", async: true }, async () => {})
const step5 = createStep({ name: "step5" }, async (all: any[]) => {
const ret = [...all, "result from step 5"]
return new StepResponse(ret)
})
const workflowId = "workflow-1" + ulid()
createWorkflow(
{
name: workflowId,
idempotent: true,
retentionTime: 1,
},
function () {
const all = parallelize(step0(), step1(), step2(), step3(), step4())
const res = step5(all)
return new WorkflowResponse(res)
}
)
const transactionId = ulid()
const done = new Promise<void>((resolve, reject) => {
void workflowOrcModule.subscribe({
workflowId: workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
resolve(event.result)
}
},
})
})
await workflowOrcModule.run(workflowId, {
throwOnError: false,
logOnError: true,
transactionId,
})
await setTimeout(100) // Just to wait a bit before firering everything
for (let i = 0; i <= 4; i++) {
void workflowOrcModule.setStepSuccess({
idempotencyKey: {
workflowId: workflowId,
transactionId: transactionId,
stepId: `step${i}`,
action: TransactionHandlerType.INVOKE,
},
stepResponse: new StepResponse("result from step " + i),
})
}
const res = await done
expect(res).toEqual([
"result from step 0",
"result from step 1",
"result from step 2",
"result from step 3",
"result from step 4",
"result from step 5",
])
})
it("should prevent race continuation of the workflow during retryIntervalAwaiting in background execution", async () => {
const transactionId = "transaction_id" + ulid()
const workflowId = "RACE_workflow-1" + ulid()
const step0InvokeMock = jest.fn()
const step1InvokeMock = jest.fn()
@@ -59,7 +277,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
const step1 = createStep("step1", async (_) => {
step1InvokeMock()
await setTimeout(2000)
await setTimeout(1000)
return new StepResponse({ isSuccess: true })
})
@@ -68,70 +286,70 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
return new StepResponse({ result: input })
})
const subWorkflow = createWorkflow(subWorkflowId, function () {
const subWorkflow = createWorkflow("sub-workflow-1", function () {
const status = step1()
return new WorkflowResponse(status)
})
createWorkflow(workflowId, function () {
const build = step0()
const status = subWorkflow.runAsStep({} as any).config({
async: true,
compensateAsync: true,
backgroundExecution: true,
retryIntervalAwaiting: 1,
})
const transformedResult = transform({ status }, (data) => {
transformMock()
return {
status: data.status,
}
})
step2(transformedResult)
return new WorkflowResponse(build)
})
void workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
try {
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
expect(
step1InvokeMock.mock.calls.length
).toBeGreaterThanOrEqual(1)
expect(step2InvokeMock).toHaveBeenCalledTimes(1)
expect(transformMock).toHaveBeenCalledTimes(1)
// Prevent killing the test to early
await setTimeout(500)
done()
} catch (e) {
return done(e)
} finally {
clearTimeout(timeout)
}
}
createWorkflow(
{
name: workflowId,
idempotent: true,
retentionTime: 5,
},
function () {
const build = step0()
const status = subWorkflow.runAsStep({} as any).config({
async: true,
compensateAsync: true,
backgroundExecution: true,
retryIntervalAwaiting: 0.1,
})
const transformedResult = transform({ status }, (data) => {
transformMock()
return {
status: data.status,
}
})
step2(transformedResult)
return new WorkflowResponse(build)
}
)
const onFinish = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
workflowOrcModule
.run(workflowId, { transactionId })
.run(workflowId, {
transactionId,
throwOnError: false,
logOnError: true,
})
.then(({ result }) => {
expect(result).toBe("result from step 0")
})
const timeout = failTrap(
done,
"should prevent race continuation of the workflow during retryIntervalAwaiting in background execution"
)
await onFinish
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
expect(step1InvokeMock.mock.calls.length).toBeGreaterThan(1)
expect(step2InvokeMock).toHaveBeenCalledTimes(1)
expect(transformMock).toHaveBeenCalledTimes(1)
})
it("should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution", (done) => {
it("should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution", async () => {
const transactionId = "transaction_id" + ulid()
const workflowId = "RACE_workflow-1" + ulid()
@@ -157,7 +375,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
"RACE_step1",
async (_) => {
step1InvokeMock()
await setTimeout(500)
await setTimeout(1000)
throw new Error("error from step 1")
},
() => {
@@ -175,61 +393,63 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
return new WorkflowResponse(status)
})
createWorkflow(workflowId, function () {
const build = step0()
const status = subWorkflow.runAsStep({} as any).config({
async: true,
compensateAsync: true,
backgroundExecution: true,
retryIntervalAwaiting: 0.1,
})
const transformedResult = transform({ status }, (data) => {
transformMock()
return {
status: data.status,
}
})
step2(transformedResult)
return new WorkflowResponse(build)
})
void workflowOrcModule.subscribe({
workflowId: workflowId,
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
try {
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
expect(step0CompensateMock).toHaveBeenCalledTimes(1)
expect(
step1InvokeMock.mock.calls.length
).toBeGreaterThanOrEqual(2) // Called every 0.1s at least (it can take more than 0.1sdepending on the event loop congestions)
expect(step1CompensateMock).toHaveBeenCalledTimes(1)
expect(step2InvokeMock).toHaveBeenCalledTimes(0)
expect(transformMock).toHaveBeenCalledTimes(0)
done()
} catch (e) {
return done(e)
} finally {
clearTimeout(timeout)
}
}
createWorkflow(
{
name: workflowId,
},
function () {
const build = step0()
const status = subWorkflow.runAsStep({} as any).config({
async: true,
compensateAsync: true,
backgroundExecution: true,
retryIntervalAwaiting: 0.1,
maxAwaitingRetries: 3,
})
const transformedResult = transform({ status }, (data) => {
transformMock()
return {
status: data.status,
}
})
step2(transformedResult)
return new WorkflowResponse(build)
}
)
const onFinish = new Promise<void>((resolve) => {
void workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
resolve()
}
},
})
})
workflowOrcModule
.run(workflowId, { transactionId, throwOnError: false })
await workflowOrcModule
.run(workflowId, {
transactionId,
throwOnError: false,
logOnError: true,
})
.then(({ result }) => {
expect(result).toBe("result from step 0")
})
const timeout = failTrap(
done,
"should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution"
)
await onFinish
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
expect(step0CompensateMock).toHaveBeenCalledTimes(1)
expect(step1InvokeMock).toHaveBeenCalledTimes(3)
expect(step1CompensateMock.mock.calls.length).toBeGreaterThan(0)
expect(step2InvokeMock).toHaveBeenCalledTimes(0)
expect(transformMock).toHaveBeenCalledTimes(0)
})
})
},

12
packages/modules/workflow-engine-redis/integration-tests/utils/database.ts
View File

@@ -30,20 +30,16 @@ async function deleteKeysByPattern(pattern) {
count: 100,
})
const pipeline = redis.pipeline()
for await (const keys of stream) {
if (keys.length) {
const pipeline = redis.pipeline()
keys.forEach((key) => pipeline.unlink(key))
await pipeline.exec()
}
}
await pipeline.exec()
}
async function cleanRedis() {
try {
await deleteKeysByPattern("bull:*")
await deleteKeysByPattern("dtrx:*")
} catch (error) {
console.error("Error:", error)
}
await deleteKeysByPattern("bull:*")
await deleteKeysByPattern("dtrx:*")
}

2
packages/modules/workflow-engine-redis/package.json
View File

@@ -29,7 +29,7 @@
"resolve:aliases": "tsc --showConfig -p tsconfig.json > tsconfig.resolved.json && tsc-alias -p tsconfig.resolved.json && rimraf tsconfig.resolved.json",
"build": "rimraf dist && tsc --build && npm run resolve:aliases",
"test": "jest --passWithNoTests --bail --forceExit -- src",
"test:integration": "jest --forceExit -- integration-tests/**/__tests__/*.ts",
"test:integration": "jest --forceExit --runInBand -- integration-tests/**/__tests__/index.spec.ts && jest --forceExit --runInBand -- integration-tests/**/__tests__/race.spec.ts",
"migration:initial": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:create --initial",
"migration:create": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:create",
"migration:up": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:up",

120
packages/modules/workflow-engine-redis/src/services/workflow-orchestrator.ts
View File

@@ -14,6 +14,7 @@ import {
} from "@medusajs/framework/types"
import {
isString,
MedusaError,
promiseAll,
TransactionState,
} from "@medusajs/framework/utils"
@@ -114,7 +115,7 @@ export class WorkflowOrchestratorService {
protected redisPublisher: Redis
protected redisSubscriber: Redis
protected container_: MedusaContainer
private subscribers: Subscribers = new Map()
private static subscribers: Subscribers = new Map()
readonly #logger: Logger
@@ -153,7 +154,7 @@ export class WorkflowOrchestratorService {
this.redisSubscriber.on("message", async (channel, message) => {
const workflowId = channel.split(":")[1]
if (!this.subscribers.has(workflowId)) return
if (!WorkflowOrchestratorService.subscribers.has(workflowId)) return
try {
const { instanceId, data } = JSON.parse(message)
@@ -177,9 +178,17 @@ export class WorkflowOrchestratorService {
await this.redisDistributedTransactionStorage_.onApplicationStart()
}
private async triggerParentStep(transaction, result) {
private async triggerParentStep(transaction, result, errors) {
const metadata = transaction.flow.metadata
const { parentStepIdempotencyKey } = metadata ?? {}
const { parentStepIdempotencyKey, cancelingFromParentStep } = metadata ?? {}
if (cancelingFromParentStep) {
/**
* If the sub workflow is cancelling from a parent step, we don't want to trigger the parent
* step.
*/
return
}
if (parentStepIdempotencyKey) {
const hasFailed = [
@@ -190,7 +199,7 @@ export class WorkflowOrchestratorService {
if (hasFailed) {
await this.setStepFailure({
idempotencyKey: parentStepIdempotencyKey,
stepResponse: result,
stepResponse: errors,
options: {
logOnError: true,
},
@@ -224,13 +233,16 @@ export class WorkflowOrchestratorService {
throwOnError ??= true
context ??= {}
context.transactionId = transactionId ?? ulid()
context.transactionId = transactionId ?? "auto-" + ulid()
const workflowId = isString(workflowIdOrWorkflow)
? workflowIdOrWorkflow
: workflowIdOrWorkflow.getName()
if (!workflowId) {
throw new Error("Workflow ID is required")
throw new MedusaError(
MedusaError.Types.NOT_FOUND,
`Workflow ID is required`
)
}
const events: FlowRunOptions["events"] = this.buildWorkflowEvents({
@@ -241,11 +253,14 @@ export class WorkflowOrchestratorService {
const exportedWorkflow = MedusaWorkflow.getWorkflow(workflowId)
if (!exportedWorkflow) {
throw new Error(`Workflow with id "${workflowId}" not found.`)
throw new MedusaError(
MedusaError.Types.NOT_FOUND,
`Workflow with id "${workflowId}" not found.`
)
}
const { onFinish, ...restEvents } = events
const originalOnFinishHandler = events.onFinish!
delete events.onFinish
const ret = await exportedWorkflow.run({
input,
@@ -253,7 +268,7 @@ export class WorkflowOrchestratorService {
logOnError,
resultFrom,
context,
events,
events: restEvents,
container: container ?? this.container_,
})
@@ -283,7 +298,7 @@ export class WorkflowOrchestratorService {
errors,
})
await this.triggerParentStep(ret.transaction, result)
await this.triggerParentStep(ret.transaction, result, errors)
}
if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -327,7 +342,10 @@ export class WorkflowOrchestratorService {
const exportedWorkflow = MedusaWorkflow.getWorkflow(workflowId)
if (!exportedWorkflow) {
throw new Error(`Workflow with id "${workflowId}" not found.`)
throw new MedusaError(
MedusaError.Types.NOT_FOUND,
`Workflow with id "${workflowId}" not found.`
)
}
const transaction = await this.getRunningTransaction(
@@ -354,12 +372,15 @@ export class WorkflowOrchestratorService {
transactionId: transactionId,
})
const { onFinish, ...restEvents } = events
const originalOnFinishHandler = events.onFinish!
const ret = await exportedWorkflow.cancel({
transaction,
throwOnError: false,
logOnError,
context,
events,
events: restEvents,
container: container ?? this.container_,
})
@@ -382,17 +403,13 @@ export class WorkflowOrchestratorService {
if (hasFinished) {
const { result, errors } = ret
this.notify({
isFlowAsync: ret.transaction.getFlow().hasAsyncSteps,
eventType: "onFinish",
workflowId,
transactionId: transaction.transactionId,
state: transactionState as TransactionState,
await originalOnFinishHandler({
transaction: ret.transaction,
result,
errors,
})
await this.triggerParentStep(ret.transaction, result)
await this.triggerParentStep(ret.transaction, result, errors)
}
if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -464,29 +481,28 @@ export class WorkflowOrchestratorService {
workflowId,
})
const { onFinish, ...restEvents } = events
const originalOnFinishHandler = events.onFinish!
const ret = await exportedWorkflow.retryStep({
idempotencyKey: idempotencyKey_,
context,
throwOnError: false,
logOnError,
events,
events: restEvents,
container: container ?? this.container_,
})
if (ret.transaction.hasFinished()) {
const { result, errors } = ret
this.notify({
isFlowAsync: ret.transaction.getFlow().hasAsyncSteps,
eventType: "onFinish",
workflowId,
transactionId,
state: ret.transaction.getFlow().state as TransactionState,
await originalOnFinishHandler({
transaction: ret.transaction,
result,
errors,
})
await this.triggerParentStep(ret.transaction, result)
await this.triggerParentStep(ret.transaction, result, errors)
}
if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -534,13 +550,16 @@ export class WorkflowOrchestratorService {
workflowId,
})
const { onFinish, ...restEvents } = events
const originalOnFinishHandler = events.onFinish!
const ret = await exportedWorkflow.registerStepSuccess({
idempotencyKey: idempotencyKey_,
context,
resultFrom,
throwOnError: false,
logOnError,
events,
events: restEvents,
response: stepResponse,
container: container ?? this.container_,
})
@@ -548,17 +567,13 @@ export class WorkflowOrchestratorService {
if (ret.transaction.hasFinished()) {
const { result, errors } = ret
this.notify({
isFlowAsync: ret.transaction.getFlow().hasAsyncSteps,
eventType: "onFinish",
workflowId,
transactionId,
state: ret.transaction.getFlow().state as TransactionState,
await originalOnFinishHandler({
transaction: ret.transaction,
result,
errors,
})
await this.triggerParentStep(ret.transaction, result)
await this.triggerParentStep(ret.transaction, result, errors)
}
if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -607,13 +622,16 @@ export class WorkflowOrchestratorService {
workflowId,
})
const { onFinish, ...restEvents } = events
const originalOnFinishHandler = events.onFinish!
const ret = await exportedWorkflow.registerStepFailure({
idempotencyKey: idempotencyKey_,
context,
resultFrom,
throwOnError: false,
logOnError,
events,
events: restEvents,
response: stepResponse,
container: container ?? this.container_,
forcePermanentFailure,
@@ -622,17 +640,13 @@ export class WorkflowOrchestratorService {
if (ret.transaction.hasFinished()) {
const { result, errors } = ret
this.notify({
isFlowAsync: ret.transaction.getFlow().hasAsyncSteps,
eventType: "onFinish",
workflowId,
transactionId,
state: ret.transaction.getFlow().state as TransactionState,
await originalOnFinishHandler({
transaction: ret.transaction,
result,
errors,
})
await this.triggerParentStep(ret.transaction, result)
await this.triggerParentStep(ret.transaction, result, errors)
}
if (throwOnError && (ret.thrownError || ret.errors?.length)) {
@@ -653,10 +667,11 @@ export class WorkflowOrchestratorService {
subscriberId,
}: SubscribeOptions) {
subscriber._id = subscriberId
const subscribers = this.subscribers.get(workflowId) ?? new Map()
const subscribers =
WorkflowOrchestratorService.subscribers.get(workflowId) ?? new Map()
// Subscribe instance to redis
if (!this.subscribers.has(workflowId)) {
if (!WorkflowOrchestratorService.subscribers.has(workflowId)) {
void this.redisSubscriber.subscribe(this.getChannelName(workflowId))
}
@@ -675,7 +690,7 @@ export class WorkflowOrchestratorService {
transactionSubscribers.push(subscriber)
subscribers.set(transactionId, transactionSubscribers)
this.subscribers.set(workflowId, subscribers)
WorkflowOrchestratorService.subscribers.set(workflowId, subscribers)
return
}
@@ -687,7 +702,7 @@ export class WorkflowOrchestratorService {
workflowSubscribers.push(subscriber)
subscribers.set(AnySubscriber, workflowSubscribers)
this.subscribers.set(workflowId, subscribers)
WorkflowOrchestratorService.subscribers.set(workflowId, subscribers)
}
unsubscribe({
@@ -695,7 +710,7 @@ export class WorkflowOrchestratorService {
transactionId,
subscriberOrId,
}: UnsubscribeOptions) {
const subscribers = this.subscribers.get(workflowId)
const subscribers = WorkflowOrchestratorService.subscribers.get(workflowId)
if (!subscribers) {
return
}
@@ -735,7 +750,7 @@ export class WorkflowOrchestratorService {
}
if (subscribers.size === 0) {
this.subscribers.delete(workflowId)
WorkflowOrchestratorService.subscribers.delete(workflowId)
void this.redisSubscriber.unsubscribe(this.getChannelName(workflowId))
}
}
@@ -774,7 +789,7 @@ export class WorkflowOrchestratorService {
private async processSubscriberNotifications(options: NotifyOptions) {
const { workflowId, transactionId, eventType } = options
const subscribers: TransactionSubscribers =
this.subscribers.get(workflowId) ?? new Map()
WorkflowOrchestratorService.subscribers.get(workflowId) ?? new Map()
const notifySubscribersAsync = async (handlers: SubscriberHandler[]) => {
const promises = handlers.map(async (handler) => {
@@ -886,6 +901,9 @@ export class WorkflowOrchestratorService {
await notify({
eventType: "onFinish",
isFlowAsync: transaction.getFlow().hasAsyncSteps,
result,
errors,
state: transaction.getFlow().state as TransactionState,
})
},

443
packages/modules/workflow-engine-redis/src/utils/workflow-orchestrator-storage.ts
View File

@@ -36,24 +36,22 @@ enum JobType {
const THIRTY_MINUTES_IN_MS = 1000 * 60 * 30
const REPEATABLE_CLEARER_JOB_ID = "clear-expired-executions"
const invokingStatesSet = new Set([
TransactionStepState.INVOKING,
TransactionStepState.NOT_STARTED,
])
const doneStates = [
TransactionStepState.DONE,
TransactionStepState.REVERTED,
TransactionStepState.FAILED,
TransactionStepState.SKIPPED,
TransactionStepState.SKIPPED_FAILURE,
TransactionStepState.TIMEOUT,
]
const compensatingStatesSet = new Set([
TransactionStepState.COMPENSATING,
TransactionStepState.NOT_STARTED,
])
function isInvokingState(step: TransactionStep) {
return invokingStatesSet.has(step.invoke?.state)
}
function isCompensatingState(step: TransactionStep) {
return compensatingStatesSet.has(step.compensate?.state)
}
const finishedStates = [
TransactionState.DONE,
TransactionState.FAILED,
TransactionState.REVERTED,
]
const failedStates = [TransactionState.FAILED, TransactionState.REVERTED]
export class RedisDistributedTransactionStorage
implements IDistributedTransactionStorage, IDistributedSchedulerStorage
{
@@ -152,11 +150,17 @@ export class RedisDistributedTransactionStorage
} with the following data: ${JSON.stringify(job.data)}`
)
if (allowedJobs.includes(job.name as JobType)) {
await this.executeTransaction(
job.data.workflowId,
job.data.transactionId,
job.data.transactionMetadata
)
try {
await this.executeTransaction(
job.data.workflowId,
job.data.transactionId,
job.data.transactionMetadata
)
} catch (error) {
if (!SkipExecutionError.isSkipExecutionError(error)) {
throw error
}
}
}
if (job.name === JobType.SCHEDULE) {
@@ -273,29 +277,12 @@ export class RedisDistributedTransactionStorage
private async saveToDb(data: TransactionCheckpoint, retentionTime?: number) {
const isNotStarted = data.flow.state === TransactionState.NOT_STARTED
const isFinished = [
TransactionState.DONE,
TransactionState.FAILED,
TransactionState.REVERTED,
].includes(data.flow.state)
const asyncVersion = data.flow._v
const isFinished = finishedStates.includes(data.flow.state)
const isWaitingToCompensate =
data.flow.state === TransactionState.WAITING_TO_COMPENSATE
/**
* Bit of explanation:
*
* When a workflow run, it run all sync step in memory until it reaches a async step.
* In that case, it might handover to another process to continue the execution. Thats why
* we need to save the current state of the flow. Then from there, it will run again all
* sync steps until the next async step. an so on so forth.
*
* To summarize, we only trully need to save the data when we are reaching any steps that
* trigger a handover to a potential other process.
*
* This allows us to spare some resources and time by not over communicating with the external
* database when it is not really needed
*/
const isFlowInvoking = data.flow.state === TransactionState.INVOKING
const stepsArray = Object.values(data.flow.steps) as TransactionStep[]
@@ -334,7 +321,8 @@ export class RedisDistributedTransactionStorage
if (
!(isNotStarted || isFinished || isWaitingToCompensate) &&
!currentStepsIsAsync
!currentStepsIsAsync &&
!asyncVersion
) {
return
}
@@ -440,10 +428,7 @@ export class RedisDistributedTransactionStorage
const execution = trx.execution as TransactionFlow
if (!idempotent) {
const isFailedOrReverted = [
TransactionState.REVERTED,
TransactionState.FAILED,
].includes(execution.state)
const isFailedOrReverted = failedStates.includes(execution.state)
const isDone = execution.state === TransactionState.DONE
@@ -461,11 +446,11 @@ export class RedisDistributedTransactionStorage
}
}
return {
flow: flow ?? (trx.execution as TransactionFlow),
context: trx.context?.data as TransactionContext,
errors: errors ?? (trx.context?.errors as TransactionStepError[]),
}
return new TransactionCheckpoint(
flow ?? (trx.execution as TransactionFlow),
trx.context?.data as TransactionContext,
errors ?? (trx.context?.errors as TransactionStepError[])
)
}
return
@@ -476,94 +461,120 @@ export class RedisDistributedTransactionStorage
data: TransactionCheckpoint,
ttl?: number,
options?: TransactionOptions
): Promise<void> {
): Promise<TransactionCheckpoint> {
/**
* Store the retention time only if the transaction is done, failed or reverted.
* From that moment, this tuple can be later on archived or deleted after the retention time.
*/
const hasFinished = [
TransactionState.DONE,
TransactionState.FAILED,
TransactionState.REVERTED,
].includes(data.flow.state)
const { retentionTime } = options ?? {}
await this.#preventRaceConditionExecutionIfNecessary({
data,
key,
options,
})
let lockAcquired = false
if (hasFinished && retentionTime) {
Object.assign(data, {
retention_time: retentionTime,
})
}
if (data.flow._v) {
lockAcquired = await this.#acquireLock(key)
// Only set if not exists
const shouldSetNX =
data.flow.state === TransactionState.NOT_STARTED &&
!data.flow.transactionId.startsWith("auto-")
// Prepare operations to be executed in batch or pipeline
const data_ = {
errors: data.errors,
flow: data.flow,
}
const stringifiedData = JSON.stringify(data_)
const pipeline = this.redisClient.pipeline()
// Execute Redis operations
if (!hasFinished) {
if (ttl) {
if (shouldSetNX) {
pipeline.set(key, stringifiedData, "EX", ttl, "NX")
} else {
pipeline.set(key, stringifiedData, "EX", ttl)
}
} else {
if (shouldSetNX) {
pipeline.set(key, stringifiedData, "NX")
} else {
pipeline.set(key, stringifiedData)
}
if (!lockAcquired) {
throw new Error("Lock not acquired")
}
} else {
pipeline.unlink(key)
const storedData = await this.get(key, {
isCancelling: !!data.flow.cancelledAt,
} as any)
TransactionCheckpoint.mergeCheckpoints(data, storedData)
}
const execPipeline = () => {
return pipeline.exec().then((result) => {
if (!shouldSetNX) {
return result
}
try {
const hasFinished = finishedStates.includes(data.flow.state)
const actionResult = result?.pop()
const isOk = !!actionResult?.pop()
if (!isOk) {
throw new SkipExecutionError(
"Transaction already started for transactionId: " +
data.flow.transactionId
)
let cachedCheckpoint: TransactionCheckpoint | undefined
const getCheckpoint = async (options?: TransactionOptions) => {
if (!cachedCheckpoint) {
cachedCheckpoint = await this.get(key, options)
}
return cachedCheckpoint
}
return result
await this.#preventRaceConditionExecutionIfNecessary({
data: data,
key,
options,
getCheckpoint,
})
}
// Database operations
if (hasFinished && !retentionTime) {
// If the workflow is nested, we cant just remove it because it would break the compensation algorithm. Instead, it will get deleted when the top level parent is deleted.
if (!data.flow.metadata?.parentStepIdempotencyKey) {
await promiseAll([execPipeline(), this.deleteFromDb(data)])
// Only set if not exists
const shouldSetNX =
data.flow.state === TransactionState.NOT_STARTED &&
!data.flow.transactionId.startsWith("auto-")
if (retentionTime) {
Object.assign(data, {
retention_time: retentionTime,
})
}
const execPipeline = () => {
const lightData_ = {
errors: data.errors,
flow: data.flow,
}
const stringifiedData = JSON.stringify(lightData_)
const pipeline = this.redisClient.pipeline()
if (!hasFinished) {
if (ttl) {
if (shouldSetNX) {
pipeline.set(key, stringifiedData, "EX", ttl, "NX")
} else {
pipeline.set(key, stringifiedData, "EX", ttl)
}
} else {
if (shouldSetNX) {
pipeline.set(key, stringifiedData, "NX")
} else {
pipeline.set(key, stringifiedData)
}
}
} else {
pipeline.unlink(key)
}
return pipeline.exec().then((result) => {
if (!shouldSetNX) {
return result
}
const actionResult = result?.pop()
const isOk = !!actionResult?.pop()
if (!isOk) {
throw new SkipExecutionError(
"Transaction already started for transactionId: " +
data.flow.transactionId
)
}
return result
})
}
if (hasFinished && !retentionTime) {
if (!data.flow.metadata?.parentStepIdempotencyKey) {
await this.deleteFromDb(data)
await execPipeline()
} else {
await this.saveToDb(data, retentionTime)
await execPipeline()
}
} else {
await this.saveToDb(data, retentionTime)
await execPipeline()
}
} else {
await this.saveToDb(data, retentionTime)
await execPipeline()
return data as TransactionCheckpoint
} finally {
if (lockAcquired) {
await this.#releaseLock(key)
}
}
}
@@ -750,19 +761,47 @@ export class RedisDistributedTransactionStorage
)
}
/**
* Generate a lock key for the given transaction key
*/
#getLockKey(key: string): string {
return `${key}:lock`
}
async #acquireLock(key: string, ttlSeconds: number = 2): Promise<boolean> {
const lockKey = this.#getLockKey(key)
const result = await this.redisClient.set(
lockKey,
1,
"EX",
ttlSeconds,
"NX"
)
return result === "OK"
}
async #releaseLock(key: string): Promise<void> {
const lockKey = this.#getLockKey(key)
await this.redisClient.del(lockKey)
}
async #preventRaceConditionExecutionIfNecessary({
data,
key,
options,
getCheckpoint,
}: {
data: TransactionCheckpoint
key: string
options?: TransactionOptions
getCheckpoint: (
options: TransactionOptions
) => Promise<TransactionCheckpoint | undefined>
}) {
const isInitialCheckpoint = [TransactionState.NOT_STARTED].includes(
data.flow.state
)
/**
* In case many execution can succeed simultaneously, we need to ensure that the latest
* execution does continue if a previous execution is considered finished
@@ -780,13 +819,37 @@ export class RedisDistributedTransactionStorage
} as Parameters<typeof this.get>[1]
data_ =
(await this.get(key, getOptions)) ??
(await getCheckpoint(getOptions as TransactionOptions)) ??
({ flow: {} } as TransactionCheckpoint)
}
const { flow: latestUpdatedFlow } = data_
if (options?.stepId) {
const stepId = options.stepId
const currentStep = data.flow.steps[stepId]
const latestStep = latestUpdatedFlow.steps?.[stepId]
if (latestStep && currentStep) {
const isCompensating = data.flow.state === TransactionState.COMPENSATING
if (!isInitialCheckpoint && !isPresent(latestUpdatedFlow)) {
const latestState = isCompensating
? latestStep.compensate?.state
: latestStep.invoke?.state
const shouldSkip = doneStates.includes(latestState)
if (shouldSkip) {
throw new SkipStepAlreadyFinishedError(
`Step ${stepId} already finished by another execution`
)
}
}
}
if (
!isInitialCheckpoint &&
!isPresent(latestUpdatedFlow) &&
!data.flow.metadata?.parentStepIdempotencyKey
) {
/**
* the initial checkpoint expect no other checkpoint to have been stored.
* In case it is not the initial one and another checkpoint is trying to
@@ -796,54 +859,7 @@ export class RedisDistributedTransactionStorage
throw new SkipExecutionError("Already finished by another execution")
}
let currentFlowLatestExecutedStep: TransactionStep | undefined
const currentFlowSteps = Object.values(currentFlow.steps || {})
for (let i = currentFlowSteps.length - 1; i >= 0; i--) {
if (currentFlowSteps[i].lastAttempt) {
currentFlowLatestExecutedStep = currentFlowSteps[i]
break
}
}
let latestUpdatedFlowLatestExecutedStep: TransactionStep | undefined
const latestUpdatedFlowSteps = Object.values(latestUpdatedFlow.steps || {})
for (let i = latestUpdatedFlowSteps.length - 1; i >= 0; i--) {
if (latestUpdatedFlowSteps[i].lastAttempt) {
latestUpdatedFlowLatestExecutedStep = latestUpdatedFlowSteps[i]
break
}
}
/**
* The current flow and the latest updated flow have the same latest executed step.
*/
const isSameLatestExecutedStep =
currentFlowLatestExecutedStep &&
latestUpdatedFlowLatestExecutedStep &&
currentFlowLatestExecutedStep?.id ===
latestUpdatedFlowLatestExecutedStep?.id
/**
* The current flow's latest executed step has a last attempt ahead of the latest updated
* flow's latest executed step. Therefor it is fine, otherwise another execution has already
* finished the step.
*/
const isCurrentLatestExecutedStepLastAttemptAhead =
currentFlowLatestExecutedStep?.lastAttempt &&
latestUpdatedFlowLatestExecutedStep?.lastAttempt &&
currentFlowLatestExecutedStep.lastAttempt >=
latestUpdatedFlowLatestExecutedStep.lastAttempt
if (
isSameLatestExecutedStep &&
!isCurrentLatestExecutedStepLastAttemptAhead
) {
throw new SkipStepAlreadyFinishedError(
"Step already finished by another execution"
)
}
// First ensure that the latest execution was not cancelled, otherwise we skip the execution
// Ensure that the latest execution was not cancelled, otherwise we skip the execution
const latestTransactionCancelledAt = latestUpdatedFlow.cancelledAt
const currentTransactionCancelledAt = currentFlow.cancelledAt
@@ -855,91 +871,6 @@ export class RedisDistributedTransactionStorage
"Workflow execution has been cancelled during the execution"
)
}
let currentFlowLastInvokingStepIndex = -1
for (let i = 0; i < currentFlowSteps.length; i++) {
if (isInvokingState(currentFlowSteps[i])) {
currentFlowLastInvokingStepIndex = i
break
}
}
let latestUpdatedFlowLastInvokingStepIndex = !latestUpdatedFlow.steps
? 1 // There is no other execution, so the current execution is the latest
: -1
if (latestUpdatedFlow.steps) {
for (let i = 0; i < latestUpdatedFlowSteps.length; i++) {
if (isInvokingState(latestUpdatedFlowSteps[i])) {
latestUpdatedFlowLastInvokingStepIndex = i
break
}
}
}
let currentFlowLastCompensatingStepIndex = -1
for (let i = currentFlowSteps.length - 1; i >= 0; i--) {
if (isCompensatingState(currentFlowSteps[i])) {
currentFlowLastCompensatingStepIndex = currentFlowSteps.length - 1 - i
break
}
}
let latestUpdatedFlowLastCompensatingStepIndex = !latestUpdatedFlow.steps
? -1 // There is no other execution, so the current execution is the latest
: -1
if (latestUpdatedFlow.steps) {
for (let i = latestUpdatedFlowSteps.length - 1; i >= 0; i--) {
if (isCompensatingState(latestUpdatedFlowSteps[i])) {
latestUpdatedFlowLastCompensatingStepIndex =
latestUpdatedFlowSteps.length - 1 - i
break
}
}
}
const isLatestExecutionFinishedIndex = -1
const invokeShouldBeSkipped =
(latestUpdatedFlowLastInvokingStepIndex ===
isLatestExecutionFinishedIndex ||
currentFlowLastInvokingStepIndex <
latestUpdatedFlowLastInvokingStepIndex) &&
currentFlowLastInvokingStepIndex !== isLatestExecutionFinishedIndex
const compensateShouldBeSkipped =
currentFlowLastCompensatingStepIndex <
latestUpdatedFlowLastCompensatingStepIndex &&
currentFlowLastCompensatingStepIndex !== isLatestExecutionFinishedIndex &&
latestUpdatedFlowLastCompensatingStepIndex !==
isLatestExecutionFinishedIndex
const isCompensatingMismatch =
latestUpdatedFlow.state === TransactionState.COMPENSATING &&
![TransactionState.REVERTED, TransactionState.FAILED].includes(
currentFlow.state
) &&
currentFlow.state !== latestUpdatedFlow.state
const isRevertedMismatch =
latestUpdatedFlow.state === TransactionState.REVERTED &&
currentFlow.state !== TransactionState.REVERTED
const isFailedMismatch =
latestUpdatedFlow.state === TransactionState.FAILED &&
currentFlow.state !== TransactionState.FAILED
if (
(data.flow.state !== TransactionState.COMPENSATING &&
invokeShouldBeSkipped) ||
(data.flow.state === TransactionState.COMPENSATING &&
compensateShouldBeSkipped) ||
isCompensatingMismatch ||
isRevertedMismatch ||
isFailedMismatch
) {
throw new SkipExecutionError("Already finished by another execution")
}
}
async clearExpiredExecutions() {