asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore(orchestration): add support for autoRetry, maxAwaitingRetries, retryStep (#13391)

Browse Source 
RESOLVES CORE-1163
RESOLVES CORE-1164

**What**

### Add support for non auto retryable steps.

When marking a step with `maxRetries`, when it will fail it will be marked as temporary failure and then retry itself automatically. Thats the default behaviour, if you now add `autoRetry: false`, when the step will fail it will be marked as temporary failure but not retry automatically. you can now call the workflow engine run to resume the workflow from the failing step to be retried.

### Add support for `maxAwaitingRetries`

When setting `retyIntervalAwaiting` a step that is awaiting will be retried after the specified interval without maximun retry. Now you can set `maxAwaitingRetries` to force a maximum awaiting retry number

### Add support to manually retry an awaiting step

In some scenario, either a machine dies while a step is executing or a step is taking longer than expected, you can now call `retryStep` on the workflow engine to force a retry of the step that is supposedly stucked
This commit is contained in:
Adrien de Peretti
2025-09-08 14:46:30 +02:00
committed by GitHub
parent ac5e23b96c
commit d7692100e7
28 changed files with 1366 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
packages/modules/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_1_auto_retries.ts Normal file
View File

@@ -0,0 +1,56 @@
import {
createStep,
createWorkflow,
StepResponse,
} from "@medusajs/framework/workflows-sdk"
const step1InvokeMock = jest.fn((input) => {
input.test = "test"
return new StepResponse(input, { compensate: 123 })
})
const step1CompensateMock = jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
return new StepResponse({
reverted: true,
})
})
const step2InvokeMock = jest.fn((input) => {
throw new Error("Temporary failure")
})
const step2CompensateMock = jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
return new StepResponse({
reverted: true,
})
})
export {
step1CompensateMock,
step1InvokeMock,
step2CompensateMock,
step2InvokeMock,
}
const step_1 = createStep("step_1", step1InvokeMock, step1CompensateMock)
const step_2 = createStep("step_2", step2InvokeMock, step2CompensateMock)
createWorkflow("workflow_1_auto_retries", function (input) {
step_1(input)
const ret2 = step_2({ hey: "oh" }).config({
async: true,
maxRetries: 2,
})
return ret2
})

57
packages/modules/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_1_auto_retries_false.ts Normal file
View File

@@ -0,0 +1,57 @@
import {
createStep,
createWorkflow,
StepResponse,
} from "@medusajs/framework/workflows-sdk"
const step1InvokeMock = jest.fn((input) => {
input.test = "test"
return new StepResponse(input, { compensate: 123 })
})
const step1CompensateMock = jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
return new StepResponse({
reverted: true,
})
})
const step2InvokeMock = jest.fn((input) => {
throw new Error("Temporary failure")
})
const step2CompensateMock = jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
return new StepResponse({
reverted: true,
})
})
export {
step1CompensateMock,
step1InvokeMock,
step2CompensateMock,
step2InvokeMock,
}
const step_1 = createStep("step_1", step1InvokeMock, step1CompensateMock)
const step_2 = createStep("step_2", step2InvokeMock, step2CompensateMock)
createWorkflow("workflow_1_auto_retries_false", function (input) {
step_1(input)
const ret2 = step_2({ hey: "oh" }).config({
async: true,
maxRetries: 2,
autoRetry: false,
})
return ret2
})

60
packages/modules/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_1_manual_retry_step.ts Normal file
View File

@@ -0,0 +1,60 @@
import {
createStep,
createWorkflow,
StepResponse,
} from "@medusajs/framework/workflows-sdk"
const step1InvokeMock = jest.fn((input) => {
input.test = "test"
return new StepResponse(input, { compensate: 123 })
})
const step1CompensateMock = jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
return new StepResponse({
reverted: true,
})
})
const step2InvokeMock = jest.fn(async (input, context) => {
if (context.metadata.attempt === 1) {
await new Promise((resolve) => setTimeout(resolve, 100000))
return new StepResponse("success after 100 seconds")
}
return new StepResponse("success")
})
const step2CompensateMock = jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
return new StepResponse({
reverted: true,
})
})
export {
step1CompensateMock,
step1InvokeMock,
step2CompensateMock,
step2InvokeMock,
}
const step_1 = createStep("step_1", step1InvokeMock, step1CompensateMock)
const step_2 = createStep("step_2", step2InvokeMock, step2CompensateMock)
createWorkflow("workflow_1_manual_retry_step", function (input) {
step_1(input)
const ret2 = step_2({ hey: "oh" }).config({
async: true,
})
return ret2
})

177
packages/modules/workflow-engine-inmemory/integration-tests/__tests__/index.spec.ts
View File

@@ -43,18 +43,34 @@ import {
workflowEventGroupIdStep1Mock,
workflowEventGroupIdStep2Mock,
} from "../__fixtures__/workflow_event_group_id"
import {
step1InvokeMock as step1InvokeMockAutoRetries,
step2InvokeMock as step2InvokeMockAutoRetries,
step1CompensateMock as step1CompensateMockAutoRetries,
step2CompensateMock as step2CompensateMockAutoRetries,
} from "../__fixtures__/workflow_1_auto_retries"
import {
step1InvokeMock as step1InvokeMockAutoRetriesFalse,
step2InvokeMock as step2InvokeMockAutoRetriesFalse,
step1CompensateMock as step1CompensateMockAutoRetriesFalse,
step2CompensateMock as step2CompensateMockAutoRetriesFalse,
} from "../__fixtures__/workflow_1_auto_retries_false"
import {
step1InvokeMock as step1InvokeMockManualRetry,
step2InvokeMock as step2InvokeMockManualRetry,
} from "../__fixtures__/workflow_1_manual_retry_step"
import { createScheduled } from "../__fixtures__/workflow_scheduled"
jest.setTimeout(60000)
const failTrap = (done, name) => {
setTimeoutSync(() => {
const failTrap = (done, name, timeout = 5000) => {
return setTimeoutSync(() => {
// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
console.warn(
`Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
)
done()
}, 5000)
}, timeout)
}
function times(num) {
@@ -195,12 +211,13 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
TransactionState.REVERTED
)
done()
clearTimeout(timeout)
}
},
})
})
failTrap(
const timeout = failTrap(
done,
"should cancel an ongoing execution with async unfinished yet step"
)
@@ -358,6 +375,137 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
})
})
it("should manually retry a step that is taking too long to finish", (done) => {
const transactionId = "transaction-manual-retry" + ulid()
const workflowId = "workflow_1_manual_retry_step"
void workflowOrcModule
.run(workflowId, {
input: {},
transactionId,
})
.then(() => {
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(1)
void workflowOrcModule.retryStep({
idempotencyKey: {
workflowId,
transactionId,
stepId: "step_2",
action: "invoke",
},
})
})
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(2)
done()
clearTimeout(timeout)
}
},
})
const timeout = failTrap(
done,
"should manually retry a step that is taking too long to finish"
)
})
it("should retry steps X times automatically when maxRetries is set", (done) => {
const transactionId = "transaction-auto-retries" + ulid()
const workflowId = "workflow_1_auto_retries"
void workflowOrcModule.run(workflowId, {
input: {},
transactionId,
})
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
expect(step1InvokeMockAutoRetries).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetries).toHaveBeenCalledTimes(3)
expect(step1CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
expect(step2CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
done()
clearTimeout(timeout)
}
},
})
const timeout = failTrap(
done,
"should retry steps X times automatically when maxRetries is set"
)
})
it("should not retry steps X times automatically when maxRetries is set and autoRetry is false", async () => {
const transactionId = "transaction-auto-retries" + ulid()
const workflowId = "workflow_1_auto_retries_false"
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
throwOnError: false,
})
let lastExepectHaveBeenCalledTimes = 0
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
lastExepectHaveBeenCalledTimes = 1
expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(3)
expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(
1
)
expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(
1
)
}
},
})
expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
throwOnError: false,
})
await setTimeoutPromise(1000)
expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(2)
expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
throwOnError: false,
})
await setTimeoutPromise(1000)
expect(lastExepectHaveBeenCalledTimes).toEqual(1)
})
it("should prevent executing twice the same workflow in perfect concurrency with the same transactionId and non idempotent and not async but retention time is set", async () => {
const transactionId = "transaction_id" + ulid()
const workflowId = "workflow_id" + ulid()
@@ -535,12 +683,13 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
{ obj: "return from 3" },
])
done()
clearTimeout(timeout)
}
},
})
})
failTrap(
const timeout = failTrap(
done,
"should subscribe to a async workflow and receive the response when it finishes"
)
@@ -729,6 +878,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
const onFinish = jest.fn(() => {
done()
clearTimeout(timeout)
})
void workflowOrcModule.subscribe({
@@ -750,7 +900,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
})
expect(onFinish).toHaveBeenCalledTimes(0)
failTrap(
const timeout = failTrap(
done,
"should subscribe to a async workflow and receive the response when it finishes"
)
@@ -812,9 +962,10 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
workflowId: "workflow_conditional_step",
subscriber: (event) => {
if (event.eventType === "onFinish") {
done()
expect(conditionalStep2Invoke).toHaveBeenCalledTimes(2)
expect(conditionalStep3Invoke).toHaveBeenCalledTimes(1)
done()
clearTimeout(timeout)
}
},
})
@@ -826,7 +977,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
throwOnError: true,
})
failTrap(
const timeout = failTrap(
done,
"should not run conditional steps if condition is false"
)
@@ -837,9 +988,10 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
workflowId: "workflow_conditional_step",
subscriber: (event) => {
if (event.eventType === "onFinish") {
done()
expect(conditionalStep2Invoke).toHaveBeenCalledTimes(1)
expect(conditionalStep3Invoke).toHaveBeenCalledTimes(0)
done()
clearTimeout(timeout)
}
},
})
@@ -851,7 +1003,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
throwOnError: true,
})
failTrap(
const timeout = failTrap(
done,
"should not run conditional steps if condition is false"
)
@@ -987,7 +1139,6 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
workflowId: "workflow_parallel_async",
subscriber: (event) => {
if (event.eventType === "onFinish") {
done()
expect(event.errors).toEqual(
expect.arrayContaining([
expect.objectContaining({
@@ -999,11 +1150,13 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
}),
])
)
done()
clearTimeout(timeout)
}
},
})
failTrap(
const timeout = failTrap(
done,
"should display error when multple async steps are running in parallel"
)

16
packages/modules/workflow-engine-inmemory/integration-tests/__tests__/race.spec.ts
View File

@@ -14,14 +14,14 @@ import "../__fixtures__"
jest.setTimeout(300000)
const failTrap = (done, name) => {
setTimeoutSync(() => {
const failTrap = (done, name, timeout = 5000) => {
return setTimeoutSync(() => {
// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
console.warn(
`Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
)
done()
}, 5000)
}, timeout)
}
moduleIntegrationTestRunner<IWorkflowEngineService>({
@@ -87,7 +87,8 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
expect(step1InvokeMock.mock.calls.length).toBeGreaterThan(1)
expect(step2InvokeMock).toHaveBeenCalledTimes(1)
expect(transformMock).toHaveBeenCalledTimes(1)
setTimeoutSync(done, 500)
done()
clearTimeout(timeout)
}
},
})
@@ -99,7 +100,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
})
.catch((e) => e)
failTrap(
const timeout = failTrap(
done,
"should prevent race continuation of the workflow during retryIntervalAwaiting in background execution"
)
@@ -179,7 +180,8 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
expect(step1CompensateMock).toHaveBeenCalledTimes(1)
expect(step2InvokeMock).toHaveBeenCalledTimes(0)
expect(transformMock).toHaveBeenCalledTimes(0)
setTimeoutSync(done, 500)
done()
clearTimeout(timeout)
}
},
})
@@ -193,7 +195,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
})
.catch((e) => e)
failTrap(
const timeout = failTrap(
done,
"should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution"
)

71
packages/modules/workflow-engine-inmemory/src/services/workflow-orchestrator.ts
View File

@@ -47,6 +47,11 @@ type RegisterStepFailureOptions<T> = Omit<
forcePermanentFailure?: boolean
}
type RetryStepOptions<T> = Omit<
WorkflowOrchestratorRunOptions<T>,
"transactionId" | "input" | "resultFrom"
>
type IdempotencyKeyParts = {
workflowId: string
transactionId: string
@@ -379,6 +384,72 @@ export class WorkflowOrchestratorService {
return transaction
}
async retryStep<T = unknown>({
idempotencyKey,
options,
}: {
idempotencyKey: string | IdempotencyKeyParts
options?: RetryStepOptions<T>
}) {
const {
context,
logOnError,
container,
events: eventHandlers,
} = options ?? {}
let { throwOnError } = options ?? {}
throwOnError ??= true
const [idempotencyKey_, { workflowId, transactionId }] =
this.buildIdempotencyKeyAndParts(idempotencyKey)
const exportedWorkflow: any = MedusaWorkflow.getWorkflow(workflowId)
if (!exportedWorkflow) {
throw new Error(`Workflow with id "${workflowId}" not found.`)
}
const events = this.buildWorkflowEvents({
customEventHandlers: eventHandlers,
transactionId,
workflowId,
})
const ret = await exportedWorkflow.retryStep({
idempotencyKey: idempotencyKey_,
context,
throwOnError: false,
logOnError,
events,
container: container ?? this.container_,
})
if (ret.transaction.hasFinished()) {
const { result, errors } = ret
this.notify({
eventType: "onFinish",
workflowId,
transactionId,
state: ret.transaction.getFlow().state as TransactionState,
result,
errors,
})
await this.triggerParentStep(ret.transaction, result)
}
if (throwOnError && (ret.thrownError || ret.errors?.length)) {
if (ret.thrownError) {
throw ret.thrownError
}
throw ret.errors[0].error
}
return ret
}
async setStepSuccess<T = unknown>({
idempotencyKey,
stepResponse,

24
packages/modules/workflow-engine-inmemory/src/services/workflows-module.ts
View File

@@ -3,6 +3,7 @@ import {
DAL,
FilterableWorkflowExecutionProps,
FindConfig,
IdempotencyKeyParts,
InferEntityType,
InternalModuleDeclaration,
ModulesSdkTypes,
@@ -206,6 +207,29 @@ export class WorkflowsModuleService<
)
}
@InjectSharedContext()
async retryStep(
{
idempotencyKey,
options,
}: {
idempotencyKey: string | IdempotencyKeyParts
options?: Record<string, any>
},
@MedusaContext() context: Context = {}
) {
const options_ = JSON.parse(JSON.stringify(options ?? {}))
const { manager, transactionManager, ...restContext } = context
options_.context ??= restContext
return await this.workflowOrchestratorService_.retryStep({
idempotencyKey,
options: options_,
})
}
@InjectSharedContext()
async setStepSuccess(
{

55
packages/modules/workflow-engine-inmemory/src/utils/workflow-orchestrator-storage.ts
View File

@@ -5,6 +5,7 @@ import {
SchedulerOptions,
SkipCancelledExecutionError,
SkipExecutionError,
SkipStepAlreadyFinishedError,
TransactionCheckpoint,
TransactionContext,
TransactionFlow,
@@ -392,6 +393,53 @@ export class InMemoryDistributedTransactionStorage
throw new SkipExecutionError("Already finished by another execution")
}
let currentFlowLatestExecutedStep: TransactionStep | undefined
const currentFlowSteps = Object.values(currentFlow.steps || {})
for (let i = currentFlowSteps.length - 1; i >= 0; i--) {
if (currentFlowSteps[i].lastAttempt) {
currentFlowLatestExecutedStep = currentFlowSteps[i]
break
}
}
let latestUpdatedFlowLatestExecutedStep: TransactionStep | undefined
const latestUpdatedFlowSteps = Object.values(latestUpdatedFlow.steps || {})
for (let i = latestUpdatedFlowSteps.length - 1; i >= 0; i--) {
if (latestUpdatedFlowSteps[i].lastAttempt) {
latestUpdatedFlowLatestExecutedStep = latestUpdatedFlowSteps[i]
break
}
}
/**
* The current flow and the latest updated flow have the same latest executed step.
*/
const isSameLatestExecutedStep =
currentFlowLatestExecutedStep &&
latestUpdatedFlowLatestExecutedStep &&
currentFlowLatestExecutedStep?.id ===
latestUpdatedFlowLatestExecutedStep?.id
/**
* The current flow's latest executed step has a last attempt ahead of the latest updated
* flow's latest executed step. Therefor it is fine, otherwise another execution has already
* finished the step.
*/
const isCurrentLatestExecutedStepLastAttemptAhead =
currentFlowLatestExecutedStep?.lastAttempt &&
latestUpdatedFlowLatestExecutedStep?.lastAttempt &&
currentFlowLatestExecutedStep.lastAttempt >=
latestUpdatedFlowLatestExecutedStep.lastAttempt
if (
isSameLatestExecutedStep &&
!isCurrentLatestExecutedStepLastAttemptAhead
) {
throw new SkipStepAlreadyFinishedError(
"Step already in execution ahead of the current one"
)
}
// First ensure that the latest execution was not cancelled, otherwise we skip the execution
const latestTransactionCancelledAt = latestUpdatedFlow.cancelledAt
const currentTransactionCancelledAt = currentFlow.cancelledAt
@@ -405,13 +453,6 @@ export class InMemoryDistributedTransactionStorage
)
}
const currentFlowSteps = Object.values(currentFlow.steps || {})
const latestUpdatedFlowSteps = latestUpdatedFlow.steps
? Object.values(
latestUpdatedFlow.steps as Record<string, TransactionStep>
)
: []
// Predefined states for quick lookup
const invokingStates = [
TransactionStepState.INVOKING,

56
packages/modules/workflow-engine-redis/integration-tests/__fixtures__/workflow_1_auto_retries.ts Normal file
View File

@@ -0,0 +1,56 @@
import {
createStep,
createWorkflow,
StepResponse,
} from "@medusajs/framework/workflows-sdk"
const step1InvokeMock = jest.fn((input) => {
input.test = "test"
return new StepResponse(input, { compensate: 123 })
})
const step1CompensateMock = jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
return new StepResponse({
reverted: true,
})
})
const step2InvokeMock = jest.fn((input) => {
throw new Error("Temporary failure")
})
const step2CompensateMock = jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
return new StepResponse({
reverted: true,
})
})
export {
step1CompensateMock,
step1InvokeMock,
step2CompensateMock,
step2InvokeMock,
}
const step_1 = createStep("step_1", step1InvokeMock, step1CompensateMock)
const step_2 = createStep("step_2", step2InvokeMock, step2CompensateMock)
createWorkflow("workflow_1_auto_retries", function (input) {
step_1(input)
const ret2 = step_2({ hey: "oh" }).config({
async: true,
maxRetries: 2,
})
return ret2
})

57
packages/modules/workflow-engine-redis/integration-tests/__fixtures__/workflow_1_auto_retries_false.ts Normal file
View File

@@ -0,0 +1,57 @@
import {
createStep,
createWorkflow,
StepResponse,
} from "@medusajs/framework/workflows-sdk"
const step1InvokeMock = jest.fn((input) => {
input.test = "test"
return new StepResponse(input, { compensate: 123 })
})
const step1CompensateMock = jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
return new StepResponse({
reverted: true,
})
})
const step2InvokeMock = jest.fn((input) => {
throw new Error("Temporary failure")
})
const step2CompensateMock = jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
return new StepResponse({
reverted: true,
})
})
export {
step1CompensateMock,
step1InvokeMock,
step2CompensateMock,
step2InvokeMock,
}
const step_1 = createStep("step_1", step1InvokeMock, step1CompensateMock)
const step_2 = createStep("step_2", step2InvokeMock, step2CompensateMock)
createWorkflow("workflow_1_auto_retries_false", function (input) {
step_1(input)
const ret2 = step_2({ hey: "oh" }).config({
async: true,
maxRetries: 2,
autoRetry: false,
})
return ret2
})

60
packages/modules/workflow-engine-redis/integration-tests/__fixtures__/workflow_1_manual_retry_step.ts Normal file
View File

@@ -0,0 +1,60 @@
import {
createStep,
createWorkflow,
StepResponse,
} from "@medusajs/framework/workflows-sdk"
const step1InvokeMock = jest.fn((input) => {
input.test = "test"
return new StepResponse(input, { compensate: 123 })
})
const step1CompensateMock = jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
return new StepResponse({
reverted: true,
})
})
const step2InvokeMock = jest.fn(async (input, context) => {
if (context.metadata.attempt === 1) {
await new Promise((resolve) => setTimeout(resolve, 100000))
return new StepResponse("success after 100 seconds")
}
return new StepResponse("success")
})
const step2CompensateMock = jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
return new StepResponse({
reverted: true,
})
})
export {
step1CompensateMock,
step1InvokeMock,
step2CompensateMock,
step2InvokeMock,
}
const step_1 = createStep("step_1", step1InvokeMock, step1CompensateMock)
const step_2 = createStep("step_2", step2InvokeMock, step2CompensateMock)
createWorkflow("workflow_1_manual_retry_step", function (input) {
step_1(input)
const ret2 = step_2({ hey: "oh" }).config({
async: true,
})
return ret2
})

171
packages/modules/workflow-engine-redis/integration-tests/__tests__/index.spec.ts
View File

@@ -38,19 +38,38 @@ import {
workflowNotIdempotentWithRetentionStep3Invoke,
} from "../__fixtures__"
import { createScheduled } from "../__fixtures__/workflow_scheduled"
import {
step1InvokeMock as step1InvokeMockAutoRetries,
step2InvokeMock as step2InvokeMockAutoRetries,
step1CompensateMock as step1CompensateMockAutoRetries,
step2CompensateMock as step2CompensateMockAutoRetries,
} from "../__fixtures__/workflow_1_auto_retries"
import {
step1InvokeMock as step1InvokeMockAutoRetriesFalse,
step2InvokeMock as step2InvokeMockAutoRetriesFalse,
step1CompensateMock as step1CompensateMockAutoRetriesFalse,
step2CompensateMock as step2CompensateMockAutoRetriesFalse,
} from "../__fixtures__/workflow_1_auto_retries_false"
import {
step1InvokeMock as step1InvokeMockManualRetry,
step2InvokeMock as step2InvokeMockManualRetry,
step1CompensateMock as step1CompensateMockManualRetry,
step2CompensateMock as step2CompensateMockManualRetry,
} from "../__fixtures__/workflow_1_manual_retry_step"
import { TestDatabase } from "../utils"
import { Redis } from "ioredis"
jest.setTimeout(300000)
const failTrap = (done, name) => {
setTimeoutSync(() => {
const failTrap = (done, name, timeout = 5000) => {
return setTimeoutSync(() => {
// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
console.warn(
`Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
)
done()
}, 5000)
}, timeout)
}
function times(num) {
@@ -221,12 +240,13 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
TransactionState.REVERTED
)
done()
clearTimeout(timeout)
}
},
})
})
failTrap(
const timeout = failTrap(
done,
"should cancel an ongoing execution with async unfinished yet step"
)
@@ -393,6 +413,137 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
})
})
it("should manually retry a step that is taking too long to finish", (done) => {
const transactionId = "transaction-manual-retry" + ulid()
const workflowId = "workflow_1_manual_retry_step"
void workflowOrcModule
.run(workflowId, {
input: {},
transactionId,
})
.then(() => {
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(1)
void workflowOrcModule.retryStep({
idempotencyKey: {
workflowId,
transactionId,
stepId: "step_2",
action: "invoke",
},
})
})
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(2)
done()
clearTimeout(timeout)
}
},
})
const timeout = failTrap(
done,
"should manually retry a step that is taking too long to finish"
)
})
it("should retry steps X times automatically when maxRetries is set", (done) => {
const transactionId = "transaction-auto-retries" + ulid()
const workflowId = "workflow_1_auto_retries"
void workflowOrcModule.run(workflowId, {
input: {},
transactionId,
})
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
expect(step1InvokeMockAutoRetries).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetries).toHaveBeenCalledTimes(3)
expect(step1CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
expect(step2CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
done()
clearTimeout(timeout)
}
},
})
const timeout = failTrap(
done,
"should retry steps X times automatically when maxRetries is set"
)
})
it("should not retry steps X times automatically when maxRetries is set and autoRetry is false", async () => {
const transactionId = "transaction-auto-retries" + ulid()
const workflowId = "workflow_1_auto_retries_false"
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
throwOnError: false,
})
let lastExepectHaveBeenCalledTimes = 0
workflowOrcModule.subscribe({
workflowId,
transactionId,
subscriber: async (event) => {
if (event.eventType === "onFinish") {
lastExepectHaveBeenCalledTimes = 1
expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(3)
expect(
step1CompensateMockAutoRetriesFalse
).toHaveBeenCalledTimes(1)
expect(
step2CompensateMockAutoRetriesFalse
).toHaveBeenCalledTimes(1)
}
},
})
expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
throwOnError: false,
})
await setTimeout(1000)
expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(2)
expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
await workflowOrcModule.run(workflowId, {
input: {},
transactionId,
throwOnError: false,
})
await setTimeout(1000)
expect(lastExepectHaveBeenCalledTimes).toEqual(1)
})
it("should prevent executing twice the same workflow in perfect concurrency with the same transactionId and non idempotent and not async but retention time is set", async () => {
const transactionId = "concurrency_transaction_id" + ulid()
const workflowId = "concurrency_workflow_id" + ulid()
@@ -813,11 +964,12 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
subscriber: (event) => {
if (event.eventType === "onFinish") {
done()
clearTimeout(timeout)
}
},
})
failTrap(done, "workflow_async_background")
const timeout = failTrap(done, "workflow_async_background")
})
it("should subscribe to a async workflow and receive the response when it finishes", (done) => {
@@ -840,13 +992,14 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
if (event.eventType === "onFinish") {
onFinish()
done()
clearTimeout(timeout)
}
},
})
expect(onFinish).toHaveBeenCalledTimes(0)
failTrap(done, "workflow_async_background")
const timeout = failTrap(done, "workflow_async_background")
})
it("should not skip step if condition is true", function (done) {
@@ -866,11 +1019,12 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
subscriber: (event) => {
if (event.eventType === "onFinish") {
done()
clearTimeout(timeout)
}
},
})
failTrap(done, "wf-when")
const timeout = failTrap(done, "wf-when")
})
it("should cancel an async sub workflow when compensating", (done) => {
@@ -904,11 +1058,12 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
})
done()
clearTimeout(timeout)
}
},
})
failTrap(done, "workflow_async_background_fail")
const timeout = failTrap(done, "workflow_async_background_fail")
})
it("should cancel and revert a completed workflow", async () => {

14
packages/modules/workflow-engine-redis/integration-tests/__tests__/race.spec.ts
View File

@@ -15,14 +15,14 @@ import "../__fixtures__"
jest.setTimeout(300000)
const failTrap = (done, name) => {
setTimeoutSync(() => {
const failTrap = (done, name, timeout = 5000) => {
return setTimeoutSync(() => {
// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
console.warn(
`Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
)
done()
}, 5000)
}, timeout)
}
// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
@@ -107,6 +107,8 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
done()
} catch (e) {
return done(e)
} finally {
clearTimeout(timeout)
}
}
},
@@ -118,7 +120,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
expect(result).toBe("result from step 0")
})
failTrap(
const timeout = failTrap(
done,
"should prevent race continuation of the workflow during retryIntervalAwaiting in background execution"
)
@@ -206,6 +208,8 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
done()
} catch (e) {
return done(e)
} finally {
clearTimeout(timeout)
}
}
},
@@ -217,7 +221,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
expect(result).toBe("result from step 0")
})
failTrap(
const timeout = failTrap(
done,
"should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution"
)

72
packages/modules/workflow-engine-redis/src/services/workflow-orchestrator.ts
View File

@@ -55,6 +55,11 @@ type RegisterStepFailureOptions<T> = Omit<
forcePermanentFailure?: boolean
}
type RetryStepOptions<T> = Omit<
WorkflowOrchestratorRunOptions<T>,
"transactionId" | "input" | "resultFrom"
>
type IdempotencyKeyParts = {
workflowId: string
transactionId: string
@@ -424,6 +429,73 @@ export class WorkflowOrchestratorService {
return transaction
}
async retryStep<T = unknown>({
idempotencyKey,
options,
}: {
idempotencyKey: string | IdempotencyKeyParts
options?: RetryStepOptions<T>
}) {
const {
context,
logOnError,
container,
events: eventHandlers,
} = options ?? {}
let { throwOnError } = options ?? {}
throwOnError ??= true
const [idempotencyKey_, { workflowId, transactionId }] =
this.buildIdempotencyKeyAndParts(idempotencyKey)
const exportedWorkflow: any = MedusaWorkflow.getWorkflow(workflowId)
if (!exportedWorkflow) {
throw new Error(`Workflow with id "${workflowId}" not found.`)
}
const events = this.buildWorkflowEvents({
customEventHandlers: eventHandlers,
transactionId,
workflowId,
})
const ret = await exportedWorkflow.retryStep({
idempotencyKey: idempotencyKey_,
context,
throwOnError: false,
logOnError,
events,
container: container ?? this.container_,
})
if (ret.transaction.hasFinished()) {
const { result, errors } = ret
this.notify({
isFlowAsync: ret.transaction.getFlow().hasAsyncSteps,
eventType: "onFinish",
workflowId,
transactionId,
state: ret.transaction.getFlow().state as TransactionState,
result,
errors,
})
await this.triggerParentStep(ret.transaction, result)
}
if (throwOnError && (ret.thrownError || ret.errors?.length)) {
if (ret.thrownError) {
throw ret.thrownError
}
throw ret.errors[0].error
}
return ret
}
async setStepSuccess<T = unknown>({
idempotencyKey,
stepResponse,

24
packages/modules/workflow-engine-redis/src/services/workflows-module.ts
View File

@@ -3,6 +3,7 @@ import {
DAL,
FilterableWorkflowExecutionProps,
FindConfig,
IdempotencyKeyParts,
InferEntityType,
InternalModuleDeclaration,
ModulesSdkTypes,
@@ -269,6 +270,29 @@ export class WorkflowsModuleService<
} as any)
}
@InjectSharedContext()
async retryStep(
{
idempotencyKey,
options,
}: {
idempotencyKey: string | IdempotencyKeyParts
options?: Record<string, any>
},
@MedusaContext() context: Context = {}
) {
const options_ = JSON.parse(JSON.stringify(options ?? {}))
const { manager, transactionManager, ...restContext } = context
options_.context ??= restContext
return await this.workflowOrchestratorService_.retryStep({
idempotencyKey,
options: options_,
})
}
@InjectSharedContext()
async subscribe(
args: {

55
packages/modules/workflow-engine-redis/src/utils/workflow-orchestrator-storage.ts
View File

@@ -5,6 +5,7 @@ import {
SchedulerOptions,
SkipCancelledExecutionError,
SkipExecutionError,
SkipStepAlreadyFinishedError,
TransactionCheckpoint,
TransactionContext,
TransactionFlow,
@@ -719,6 +720,53 @@ export class RedisDistributedTransactionStorage
throw new SkipExecutionError("Already finished by another execution")
}
let currentFlowLatestExecutedStep: TransactionStep | undefined
const currentFlowSteps = Object.values(currentFlow.steps || {})
for (let i = currentFlowSteps.length - 1; i >= 0; i--) {
if (currentFlowSteps[i].lastAttempt) {
currentFlowLatestExecutedStep = currentFlowSteps[i]
break
}
}
let latestUpdatedFlowLatestExecutedStep: TransactionStep | undefined
const latestUpdatedFlowSteps = Object.values(latestUpdatedFlow.steps || {})
for (let i = latestUpdatedFlowSteps.length - 1; i >= 0; i--) {
if (latestUpdatedFlowSteps[i].lastAttempt) {
latestUpdatedFlowLatestExecutedStep = latestUpdatedFlowSteps[i]
break
}
}
/**
* The current flow and the latest updated flow have the same latest executed step.
*/
const isSameLatestExecutedStep =
currentFlowLatestExecutedStep &&
latestUpdatedFlowLatestExecutedStep &&
currentFlowLatestExecutedStep?.id ===
latestUpdatedFlowLatestExecutedStep?.id
/**
* The current flow's latest executed step has a last attempt ahead of the latest updated
* flow's latest executed step. Therefor it is fine, otherwise another execution has already
* finished the step.
*/
const isCurrentLatestExecutedStepLastAttemptAhead =
currentFlowLatestExecutedStep?.lastAttempt &&
latestUpdatedFlowLatestExecutedStep?.lastAttempt &&
currentFlowLatestExecutedStep.lastAttempt >=
latestUpdatedFlowLatestExecutedStep.lastAttempt
if (
isSameLatestExecutedStep &&
!isCurrentLatestExecutedStepLastAttemptAhead
) {
throw new SkipStepAlreadyFinishedError(
"Step already finished by another execution"
)
}
// First ensure that the latest execution was not cancelled, otherwise we skip the execution
const latestTransactionCancelledAt = latestUpdatedFlow.cancelledAt
const currentTransactionCancelledAt = currentFlow.cancelledAt
@@ -732,13 +780,6 @@ export class RedisDistributedTransactionStorage
)
}
const currentFlowSteps = Object.values(currentFlow.steps || {})
const latestUpdatedFlowSteps = latestUpdatedFlow.steps
? Object.values(
latestUpdatedFlow.steps as Record<string, TransactionStep>
)
: []
// Predefined states for quick lookup
const invokingStates = [
TransactionStepState.INVOKING,