fix: workflow async concurrency (#13769)
* executeAsync * || 1 * wip * stepId * stepId * wip * wip * continue versioning management changes * fix and improve concurrency * update in memory engine * remove duplicated test * fix script * Create weak-drinks-confess.md * fixes * fix * fix * continuation * centralize merge checkepoint * centralize merge checkpoint * fix locking * rm only * Continue improvements and fixes * fixes * fixes * hasAwaiting will be recomputed * fix orchestrator engine * bump version on async parallel steps only * mark as delivered fix * changeset * check partitions * avoid saving when having parent step * cart test --------- Co-authored-by: Carlos R. L. Rodrigues <rodrigolr@gmail.com> Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com> Co-authored-by: Oli Juhl <59018053+olivermrbl@users.noreply.github.com>
This commit is contained in:
Adrien de Peretti
committed by
GitHub
GitHub
parent
d97a60d3c1
commit
516f5a3896
@@ -57,17 +57,7 @@ import {
|
||||
} from "../__fixtures__/workflow_1_manual_retry_step"
|
||||
import { TestDatabase } from "../utils"
|
||||
|
||||
jest.setTimeout(300000)
|
||||
|
||||
const failTrap = (done, name, timeout = 5000) => {
|
||||
return setTimeoutSync(() => {
|
||||
// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
|
||||
console.warn(
|
||||
`Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
|
||||
)
|
||||
done()
|
||||
}, timeout)
|
||||
}
|
||||
jest.setTimeout(30000)
|
||||
|
||||
function times(num) {
|
||||
let resolver
|
||||
@@ -109,6 +99,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
testSuite: ({ service: workflowOrcModule, medusaApp }) => {
|
||||
describe("Workflow Orchestrator module", function () {
|
||||
beforeEach(async () => {
|
||||
await TestDatabase.clearTables()
|
||||
jest.clearAllMocks()
|
||||
|
||||
query = medusaApp.query
|
||||
@@ -169,14 +160,14 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
|
||||
describe("Testing basic workflow", function () {
|
||||
describe("Cancel transaction", function () {
|
||||
it("should cancel an ongoing execution with async unfinished yet step", (done) => {
|
||||
it("should cancel an ongoing execution with async unfinished yet step", async () => {
|
||||
const transactionId = "transaction-to-cancel-id" + ulid()
|
||||
const step1 = createStep("step1", async () => {
|
||||
return new StepResponse("step1")
|
||||
})
|
||||
|
||||
const step2 = createStep("step2", async () => {
|
||||
await setTimeout(500)
|
||||
await setTimeout(2000)
|
||||
return new StepResponse("step2")
|
||||
})
|
||||
|
||||
@@ -197,43 +188,37 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
}
|
||||
)
|
||||
|
||||
workflowOrcModule
|
||||
.run(workflowId, {
|
||||
input: {},
|
||||
const onFinishPromise = new Promise<void>((resolve) => {
|
||||
workflowOrcModule.subscribe({
|
||||
workflowId,
|
||||
transactionId,
|
||||
subscriber: async (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
resolve()
|
||||
}
|
||||
},
|
||||
})
|
||||
.then(async () => {
|
||||
await setTimeout(100)
|
||||
})
|
||||
|
||||
await workflowOrcModule.cancel(workflowId, {
|
||||
transactionId,
|
||||
})
|
||||
await workflowOrcModule.run(workflowId, {
|
||||
input: {},
|
||||
transactionId,
|
||||
})
|
||||
|
||||
workflowOrcModule.subscribe({
|
||||
workflowId,
|
||||
transactionId,
|
||||
subscriber: async (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
const execution =
|
||||
await workflowOrcModule.listWorkflowExecutions({
|
||||
transaction_id: transactionId,
|
||||
})
|
||||
await setTimeout(100)
|
||||
|
||||
expect(execution.length).toEqual(1)
|
||||
expect(execution[0].state).toEqual(
|
||||
TransactionState.REVERTED
|
||||
)
|
||||
done()
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
await workflowOrcModule.cancel(workflowId, {
|
||||
transactionId,
|
||||
})
|
||||
|
||||
const timeout = failTrap(
|
||||
done,
|
||||
"should cancel an ongoing execution with async unfinished yet step"
|
||||
)
|
||||
await onFinishPromise
|
||||
|
||||
const execution = await workflowOrcModule.listWorkflowExecutions({
|
||||
transaction_id: transactionId,
|
||||
})
|
||||
|
||||
expect(execution.length).toEqual(1)
|
||||
expect(execution[0].state).toEqual(TransactionState.REVERTED)
|
||||
})
|
||||
|
||||
it("should cancel a complete execution with a sync workflow running as async", async () => {
|
||||
@@ -270,19 +255,29 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
}
|
||||
)
|
||||
|
||||
const onFinishPromise = new Promise<void>((resolve) => {
|
||||
workflowOrcModule.subscribe({
|
||||
workflowId,
|
||||
transactionId,
|
||||
subscriber: (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
resolve()
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
await workflowOrcModule.run(workflowId, {
|
||||
input: {},
|
||||
transactionId,
|
||||
})
|
||||
|
||||
await setTimeout(100)
|
||||
await onFinishPromise
|
||||
|
||||
await workflowOrcModule.cancel(workflowId, {
|
||||
transactionId,
|
||||
})
|
||||
|
||||
await setTimeout(500)
|
||||
|
||||
const execution = await workflowOrcModule.listWorkflowExecutions({
|
||||
transaction_id: transactionId,
|
||||
})
|
||||
@@ -397,116 +392,95 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
})
|
||||
})
|
||||
|
||||
it("should manually retry a step that is taking too long to finish", (done) => {
|
||||
it("should manually retry a step that is taking too long to finish", async () => {
|
||||
const transactionId = "transaction-manual-retry" + ulid()
|
||||
const workflowId = "workflow_1_manual_retry_step"
|
||||
|
||||
void workflowOrcModule
|
||||
.run(workflowId, {
|
||||
input: {},
|
||||
transactionId,
|
||||
})
|
||||
.then(() => {
|
||||
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
|
||||
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(1)
|
||||
|
||||
void workflowOrcModule.retryStep({
|
||||
idempotencyKey: {
|
||||
workflowId,
|
||||
transactionId,
|
||||
stepId: "step_2",
|
||||
action: "invoke",
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
workflowOrcModule.subscribe({
|
||||
workflowId,
|
||||
await workflowOrcModule.run(workflowId, {
|
||||
input: {},
|
||||
transactionId,
|
||||
subscriber: async (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
|
||||
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(2)
|
||||
done()
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
})
|
||||
|
||||
const onFinishPromise = new Promise<void>((resolve) => {
|
||||
workflowOrcModule.subscribe({
|
||||
workflowId,
|
||||
transactionId,
|
||||
subscriber: async (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
|
||||
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(2)
|
||||
resolve()
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
expect(step1InvokeMockManualRetry).toHaveBeenCalledTimes(1)
|
||||
expect(step2InvokeMockManualRetry).toHaveBeenCalledTimes(1)
|
||||
|
||||
await workflowOrcModule.retryStep({
|
||||
idempotencyKey: {
|
||||
workflowId,
|
||||
transactionId,
|
||||
stepId: "step_2",
|
||||
action: "invoke",
|
||||
},
|
||||
})
|
||||
|
||||
const timeout = failTrap(
|
||||
done,
|
||||
"should manually retry a step that is taking too long to finish"
|
||||
)
|
||||
await onFinishPromise
|
||||
})
|
||||
|
||||
it("should retry steps X times automatically when maxRetries is set", (done) => {
|
||||
it("should retry steps X times automatically when maxRetries is set", async () => {
|
||||
const transactionId = "transaction-auto-retries" + ulid()
|
||||
const workflowId = "workflow_1_auto_retries"
|
||||
|
||||
const onFinishPromise = new Promise<void>((resolve) => {
|
||||
workflowOrcModule.subscribe({
|
||||
workflowId,
|
||||
transactionId,
|
||||
subscriber: async (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
resolve()
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
void workflowOrcModule.run(workflowId, {
|
||||
input: {},
|
||||
transactionId,
|
||||
})
|
||||
|
||||
workflowOrcModule.subscribe({
|
||||
workflowId,
|
||||
transactionId,
|
||||
subscriber: async (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
expect(step1InvokeMockAutoRetries).toHaveBeenCalledTimes(1)
|
||||
expect(step2InvokeMockAutoRetries).toHaveBeenCalledTimes(3)
|
||||
expect(step1CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
|
||||
expect(step2CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
|
||||
done()
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
},
|
||||
})
|
||||
await onFinishPromise
|
||||
|
||||
const timeout = failTrap(
|
||||
done,
|
||||
"should retry steps X times automatically when maxRetries is set"
|
||||
)
|
||||
expect(step1InvokeMockAutoRetries).toHaveBeenCalledTimes(1)
|
||||
expect(step2InvokeMockAutoRetries).toHaveBeenCalledTimes(3)
|
||||
expect(step1CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
|
||||
expect(step2CompensateMockAutoRetries).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it("should not retry steps X times automatically when maxRetries is set and autoRetry is false", async () => {
|
||||
const transactionId = "transaction-auto-retries" + ulid()
|
||||
const workflowId = "workflow_1_auto_retries_false"
|
||||
|
||||
await workflowOrcModule.run(workflowId, {
|
||||
input: {},
|
||||
transactionId,
|
||||
throwOnError: false,
|
||||
})
|
||||
|
||||
const onFinishPromise = new Promise<void>((resolve, reject) => {
|
||||
workflowOrcModule.subscribe({
|
||||
workflowId,
|
||||
transactionId,
|
||||
subscriber: async (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
try {
|
||||
expect(
|
||||
step1InvokeMockAutoRetriesFalse
|
||||
).toHaveBeenCalledTimes(1)
|
||||
expect(
|
||||
step2InvokeMockAutoRetriesFalse
|
||||
).toHaveBeenCalledTimes(3)
|
||||
expect(
|
||||
step1CompensateMockAutoRetriesFalse
|
||||
).toHaveBeenCalledTimes(1)
|
||||
expect(
|
||||
step2CompensateMockAutoRetriesFalse
|
||||
).toHaveBeenCalledTimes(1)
|
||||
resolve()
|
||||
} catch (error) {
|
||||
reject(error)
|
||||
}
|
||||
resolve()
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
await workflowOrcModule.run(workflowId, {
|
||||
input: {},
|
||||
transactionId,
|
||||
throwOnError: false,
|
||||
})
|
||||
|
||||
expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
|
||||
expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
|
||||
expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(0)
|
||||
@@ -529,6 +503,11 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
})
|
||||
|
||||
await onFinishPromise
|
||||
|
||||
expect(step1InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
|
||||
expect(step2InvokeMockAutoRetriesFalse).toHaveBeenCalledTimes(3)
|
||||
expect(step1CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
|
||||
expect(step2CompensateMockAutoRetriesFalse).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it("should prevent executing twice the same workflow in perfect concurrency with the same transactionId and non idempotent and not async but retention time is set", async () => {
|
||||
@@ -585,8 +564,6 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
})
|
||||
|
||||
expect(executionsList).toHaveLength(1)
|
||||
|
||||
console.log(">>>>>>>>> setting step success")
|
||||
const { result } = await workflowOrcModule.setStepSuccess({
|
||||
idempotencyKey: {
|
||||
action: TransactionHandlerType.INVOKE,
|
||||
@@ -597,7 +574,6 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
stepResponse: { uhuuuu: "yeaah!" },
|
||||
})
|
||||
|
||||
console.log(">>>>>>>>> setting step success done")
|
||||
;({ data: executionsList } = await query.graph({
|
||||
entity: "workflow_executions",
|
||||
fields: ["id"],
|
||||
@@ -928,41 +904,52 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
).toBe(true)
|
||||
})
|
||||
|
||||
it("should complete an async workflow that returns a StepResponse", (done) => {
|
||||
it("should complete an async workflow that returns a StepResponse", async () => {
|
||||
const transactionId = "transaction_1" + ulid()
|
||||
workflowOrcModule
|
||||
.run("workflow_async_background", {
|
||||
|
||||
const onFinishPromise = new Promise<void>((resolve) => {
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId: "workflow_async_background",
|
||||
transactionId,
|
||||
subscriber: (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
resolve()
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
const { transaction, result } = await workflowOrcModule.run(
|
||||
"workflow_async_background",
|
||||
{
|
||||
input: {
|
||||
myInput: "123",
|
||||
},
|
||||
transactionId,
|
||||
throwOnError: true,
|
||||
})
|
||||
.then(({ transaction, result }: any) => {
|
||||
expect(transaction.flow.state).toEqual(
|
||||
TransactionStepState.INVOKING
|
||||
)
|
||||
expect(result).toEqual(undefined)
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId: "workflow_async_background",
|
||||
transactionId,
|
||||
subscriber: (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
done()
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
},
|
||||
})
|
||||
expect(transaction.flow.state).toEqual(TransactionStepState.INVOKING)
|
||||
expect(result).toEqual(undefined)
|
||||
|
||||
const timeout = failTrap(done, "workflow_async_background")
|
||||
await onFinishPromise
|
||||
})
|
||||
|
||||
it("should subscribe to a async workflow and receive the response when it finishes", (done) => {
|
||||
it("should subscribe to a async workflow and receive the response when it finishes", async () => {
|
||||
const transactionId = "trx_123" + ulid()
|
||||
|
||||
const onFinish = jest.fn()
|
||||
const onFinishPromise = new Promise<void>((resolve) => {
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId: "workflow_async_background",
|
||||
transactionId,
|
||||
subscriber: (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
resolve()
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
void workflowOrcModule.run("workflow_async_background", {
|
||||
input: {
|
||||
@@ -972,25 +959,24 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
throwOnError: false,
|
||||
})
|
||||
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId: "workflow_async_background",
|
||||
transactionId,
|
||||
subscriber: (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
onFinish()
|
||||
done()
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
expect(onFinish).toHaveBeenCalledTimes(0)
|
||||
|
||||
const timeout = failTrap(done, "workflow_async_background")
|
||||
await onFinishPromise
|
||||
})
|
||||
|
||||
it("should not skip step if condition is true", function (done) {
|
||||
it("should not skip step if condition is true", async () => {
|
||||
const transactionId = "trx_123_when" + ulid()
|
||||
|
||||
const onFinishPromise = new Promise<void>((resolve) => {
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId: "wf-when",
|
||||
transactionId,
|
||||
subscriber: (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
resolve()
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
void workflowOrcModule.run("wf-when", {
|
||||
input: {
|
||||
callSubFlow: true,
|
||||
@@ -1000,23 +986,30 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
logOnError: true,
|
||||
})
|
||||
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId: "wf-when",
|
||||
transactionId,
|
||||
subscriber: (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
done()
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
const timeout = failTrap(done, "wf-when")
|
||||
await onFinishPromise
|
||||
})
|
||||
|
||||
it("should cancel an async sub workflow when compensating", (done) => {
|
||||
it("should cancel an async sub workflow when compensating", async () => {
|
||||
const workflowId = "workflow_async_background_fail"
|
||||
const transactionId = "trx_123_compensate_async_sub_workflow" + ulid()
|
||||
|
||||
let onCompensateStepSuccess: { step: TransactionStep } | null = null
|
||||
|
||||
const onFinishPromise = new Promise<void>((resolve) => {
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId,
|
||||
transactionId,
|
||||
subscriber: (event) => {
|
||||
if (event.eventType === "onCompensateStepSuccess") {
|
||||
onCompensateStepSuccess = event
|
||||
}
|
||||
if (event.eventType === "onFinish") {
|
||||
resolve()
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
void workflowOrcModule.run(workflowId, {
|
||||
input: {
|
||||
callSubFlow: true,
|
||||
@@ -1026,31 +1019,16 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
logOnError: false,
|
||||
})
|
||||
|
||||
let onCompensateStepSuccess: { step: TransactionStep } | null = null
|
||||
await onFinishPromise
|
||||
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId,
|
||||
subscriber: (event) => {
|
||||
if (event.eventType === "onCompensateStepSuccess") {
|
||||
onCompensateStepSuccess = event
|
||||
}
|
||||
if (event.eventType === "onFinish") {
|
||||
expect(onCompensateStepSuccess).toBeDefined()
|
||||
expect(onCompensateStepSuccess!.step.id).toEqual(
|
||||
"_root.nested_sub_flow_async_fail-as-step" // The workflow as step
|
||||
)
|
||||
expect(onCompensateStepSuccess!.step.compensate).toEqual({
|
||||
state: "reverted",
|
||||
status: "ok",
|
||||
})
|
||||
|
||||
done()
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
},
|
||||
expect(onCompensateStepSuccess).toBeDefined()
|
||||
expect(onCompensateStepSuccess!.step.id).toEqual(
|
||||
"_root.nested_sub_flow_async_fail-as-step" // The workflow as step
|
||||
)
|
||||
expect(onCompensateStepSuccess!.step.compensate).toEqual({
|
||||
state: "reverted",
|
||||
status: "ok",
|
||||
})
|
||||
|
||||
const timeout = failTrap(done, "workflow_async_background_fail")
|
||||
})
|
||||
|
||||
it("should cancel and revert a completed workflow", async () => {
|
||||
|
||||
@@ -1,32 +1,20 @@
|
||||
import { IWorkflowEngineService } from "@medusajs/framework/types"
|
||||
import { Modules } from "@medusajs/framework/utils"
|
||||
import { Modules, TransactionHandlerType } from "@medusajs/framework/utils"
|
||||
import {
|
||||
createStep,
|
||||
createWorkflow,
|
||||
parallelize,
|
||||
StepResponse,
|
||||
transform,
|
||||
WorkflowResponse,
|
||||
} from "@medusajs/framework/workflows-sdk"
|
||||
import { moduleIntegrationTestRunner } from "@medusajs/test-utils"
|
||||
import { setTimeout as setTimeoutSync } from "timers"
|
||||
import { setTimeout } from "timers/promises"
|
||||
import { ulid } from "ulid"
|
||||
import "../__fixtures__"
|
||||
import { TestDatabase } from "../utils"
|
||||
import { TestDatabase } from "../utils/database"
|
||||
|
||||
jest.setTimeout(300000)
|
||||
|
||||
const failTrap = (done, name, timeout = 5000) => {
|
||||
return setTimeoutSync(() => {
|
||||
// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
|
||||
console.warn(
|
||||
`Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
|
||||
)
|
||||
done()
|
||||
}, timeout)
|
||||
}
|
||||
|
||||
// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
|
||||
jest.setTimeout(20000)
|
||||
|
||||
moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
moduleName: Modules.WORKFLOW_ENGINE,
|
||||
@@ -38,14 +26,244 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
},
|
||||
testSuite: ({ service: workflowOrcModule, medusaApp }) => {
|
||||
describe("Testing race condition of the workflow during retry", () => {
|
||||
beforeEach(async () => {
|
||||
await TestDatabase.clearTables()
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
afterEach(async () => {
|
||||
await TestDatabase.clearTables()
|
||||
})
|
||||
|
||||
it("should prevent race continuation of the workflow during retryIntervalAwaiting in background execution", (done) => {
|
||||
const transactionId = "transaction_id" + ulid()
|
||||
it("should manage saving multiple async steps in concurrency", async () => {
|
||||
const step0 = createStep(
|
||||
{ name: "step0", async: true, backgroundExecution: true },
|
||||
async () => {
|
||||
return new StepResponse("result from step 0")
|
||||
}
|
||||
)
|
||||
|
||||
const step1 = createStep(
|
||||
{ name: "step1", async: true, backgroundExecution: true },
|
||||
async () => {
|
||||
return new StepResponse("result from step 1")
|
||||
}
|
||||
)
|
||||
|
||||
const step2 = createStep(
|
||||
{ name: "step2", async: true, backgroundExecution: true },
|
||||
async () => {
|
||||
return new StepResponse("result from step 2")
|
||||
}
|
||||
)
|
||||
const step3 = createStep(
|
||||
{ name: "step3", async: true, backgroundExecution: true },
|
||||
async () => {
|
||||
return new StepResponse("result from step 3")
|
||||
}
|
||||
)
|
||||
|
||||
const step4 = createStep(
|
||||
{ name: "step4", async: true, backgroundExecution: true },
|
||||
async () => {
|
||||
return new StepResponse("result from step 4")
|
||||
}
|
||||
)
|
||||
const step5 = createStep({ name: "step5" }, async (all: string[]) => {
|
||||
const ret = [...all, "result from step 5"]
|
||||
return new StepResponse(ret)
|
||||
})
|
||||
|
||||
const workflowId = "workflow-1" + ulid()
|
||||
const subWorkflowId = "sub-" + workflowId
|
||||
createWorkflow(
|
||||
{
|
||||
name: workflowId,
|
||||
idempotent: true,
|
||||
retentionTime: 5,
|
||||
},
|
||||
function () {
|
||||
const all = parallelize(step0(), step1(), step2(), step3(), step4())
|
||||
const res = step5(all)
|
||||
return new WorkflowResponse(res)
|
||||
}
|
||||
)
|
||||
|
||||
const transactionId = ulid()
|
||||
const done = new Promise<void>((resolve, reject) => {
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId: workflowId,
|
||||
transactionId,
|
||||
subscriber: async (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
resolve(event.result)
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
await workflowOrcModule.run(workflowId, {
|
||||
throwOnError: false,
|
||||
logOnError: true,
|
||||
transactionId,
|
||||
})
|
||||
|
||||
const result = await done
|
||||
|
||||
expect(result).toEqual([
|
||||
"result from step 0",
|
||||
"result from step 1",
|
||||
"result from step 2",
|
||||
"result from step 3",
|
||||
"result from step 4",
|
||||
"result from step 5",
|
||||
])
|
||||
})
|
||||
|
||||
it("should manage saving multiple sync steps in concurrency", async () => {
|
||||
const step0 = createStep({ name: "step0" }, async () => {
|
||||
return new StepResponse("result from step 0")
|
||||
})
|
||||
|
||||
const step1 = createStep({ name: "step1" }, async () => {
|
||||
return new StepResponse("result from step 1")
|
||||
})
|
||||
|
||||
const step2 = createStep({ name: "step2" }, async () => {
|
||||
return new StepResponse("result from step 2")
|
||||
})
|
||||
const step3 = createStep({ name: "step3" }, async () => {
|
||||
return new StepResponse("result from step 3")
|
||||
})
|
||||
|
||||
const step4 = createStep({ name: "step4" }, async () => {
|
||||
return new StepResponse("result from step 4")
|
||||
})
|
||||
const step5 = createStep({ name: "step5" }, async (all: string[]) => {
|
||||
const ret = [...all, "result from step 5"]
|
||||
return new StepResponse(ret)
|
||||
})
|
||||
|
||||
const workflowId = "workflow-1" + ulid()
|
||||
createWorkflow(
|
||||
{
|
||||
name: workflowId,
|
||||
idempotent: true,
|
||||
retentionTime: 5,
|
||||
},
|
||||
function () {
|
||||
const all = parallelize(step0(), step1(), step2(), step3(), step4())
|
||||
const res = step5(all)
|
||||
return new WorkflowResponse(res)
|
||||
}
|
||||
)
|
||||
|
||||
const transactionId = ulid()
|
||||
const done = new Promise<void>((resolve, reject) => {
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId: workflowId,
|
||||
transactionId,
|
||||
subscriber: async (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
resolve(event.result)
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
await workflowOrcModule.run(workflowId, {
|
||||
throwOnError: false,
|
||||
logOnError: true,
|
||||
transactionId,
|
||||
})
|
||||
|
||||
const result = await done
|
||||
|
||||
expect(result).toEqual([
|
||||
"result from step 0",
|
||||
"result from step 1",
|
||||
"result from step 2",
|
||||
"result from step 3",
|
||||
"result from step 4",
|
||||
"result from step 5",
|
||||
])
|
||||
})
|
||||
|
||||
it("should manage saving multiple async steps in concurrency without background execution while setting steps as success manually concurrently", async () => {
|
||||
const step0 = createStep({ name: "step0", async: true }, async () => {})
|
||||
|
||||
const step1 = createStep({ name: "step1", async: true }, async () => {})
|
||||
|
||||
const step2 = createStep({ name: "step2", async: true }, async () => {})
|
||||
const step3 = createStep({ name: "step3", async: true }, async () => {})
|
||||
|
||||
const step4 = createStep({ name: "step4", async: true }, async () => {})
|
||||
const step5 = createStep({ name: "step5" }, async (all: any[]) => {
|
||||
const ret = [...all, "result from step 5"]
|
||||
return new StepResponse(ret)
|
||||
})
|
||||
|
||||
const workflowId = "workflow-1" + ulid()
|
||||
createWorkflow(
|
||||
{
|
||||
name: workflowId,
|
||||
idempotent: true,
|
||||
retentionTime: 1,
|
||||
},
|
||||
function () {
|
||||
const all = parallelize(step0(), step1(), step2(), step3(), step4())
|
||||
const res = step5(all)
|
||||
return new WorkflowResponse(res)
|
||||
}
|
||||
)
|
||||
|
||||
const transactionId = ulid()
|
||||
const done = new Promise<void>((resolve, reject) => {
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId: workflowId,
|
||||
transactionId,
|
||||
subscriber: async (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
resolve(event.result)
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
await workflowOrcModule.run(workflowId, {
|
||||
throwOnError: false,
|
||||
logOnError: true,
|
||||
transactionId,
|
||||
})
|
||||
|
||||
await setTimeout(100) // Just to wait a bit before firering everything
|
||||
|
||||
for (let i = 0; i <= 4; i++) {
|
||||
void workflowOrcModule.setStepSuccess({
|
||||
idempotencyKey: {
|
||||
workflowId: workflowId,
|
||||
transactionId: transactionId,
|
||||
stepId: `step${i}`,
|
||||
action: TransactionHandlerType.INVOKE,
|
||||
},
|
||||
stepResponse: new StepResponse("result from step " + i),
|
||||
})
|
||||
}
|
||||
|
||||
const res = await done
|
||||
|
||||
expect(res).toEqual([
|
||||
"result from step 0",
|
||||
"result from step 1",
|
||||
"result from step 2",
|
||||
"result from step 3",
|
||||
"result from step 4",
|
||||
"result from step 5",
|
||||
])
|
||||
})
|
||||
|
||||
it("should prevent race continuation of the workflow during retryIntervalAwaiting in background execution", async () => {
|
||||
const transactionId = "transaction_id" + ulid()
|
||||
const workflowId = "RACE_workflow-1" + ulid()
|
||||
|
||||
const step0InvokeMock = jest.fn()
|
||||
const step1InvokeMock = jest.fn()
|
||||
@@ -59,7 +277,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
|
||||
const step1 = createStep("step1", async (_) => {
|
||||
step1InvokeMock()
|
||||
await setTimeout(2000)
|
||||
await setTimeout(1000)
|
||||
return new StepResponse({ isSuccess: true })
|
||||
})
|
||||
|
||||
@@ -68,70 +286,70 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
return new StepResponse({ result: input })
|
||||
})
|
||||
|
||||
const subWorkflow = createWorkflow(subWorkflowId, function () {
|
||||
const subWorkflow = createWorkflow("sub-workflow-1", function () {
|
||||
const status = step1()
|
||||
return new WorkflowResponse(status)
|
||||
})
|
||||
|
||||
createWorkflow(workflowId, function () {
|
||||
const build = step0()
|
||||
|
||||
const status = subWorkflow.runAsStep({} as any).config({
|
||||
async: true,
|
||||
compensateAsync: true,
|
||||
backgroundExecution: true,
|
||||
retryIntervalAwaiting: 1,
|
||||
})
|
||||
|
||||
const transformedResult = transform({ status }, (data) => {
|
||||
transformMock()
|
||||
return {
|
||||
status: data.status,
|
||||
}
|
||||
})
|
||||
|
||||
step2(transformedResult)
|
||||
return new WorkflowResponse(build)
|
||||
})
|
||||
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId,
|
||||
transactionId,
|
||||
subscriber: async (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
try {
|
||||
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
|
||||
expect(
|
||||
step1InvokeMock.mock.calls.length
|
||||
).toBeGreaterThanOrEqual(1)
|
||||
expect(step2InvokeMock).toHaveBeenCalledTimes(1)
|
||||
expect(transformMock).toHaveBeenCalledTimes(1)
|
||||
|
||||
// Prevent killing the test to early
|
||||
await setTimeout(500)
|
||||
done()
|
||||
} catch (e) {
|
||||
return done(e)
|
||||
} finally {
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
}
|
||||
createWorkflow(
|
||||
{
|
||||
name: workflowId,
|
||||
idempotent: true,
|
||||
retentionTime: 5,
|
||||
},
|
||||
function () {
|
||||
const build = step0()
|
||||
|
||||
const status = subWorkflow.runAsStep({} as any).config({
|
||||
async: true,
|
||||
compensateAsync: true,
|
||||
backgroundExecution: true,
|
||||
retryIntervalAwaiting: 0.1,
|
||||
})
|
||||
|
||||
const transformedResult = transform({ status }, (data) => {
|
||||
transformMock()
|
||||
return {
|
||||
status: data.status,
|
||||
}
|
||||
})
|
||||
|
||||
step2(transformedResult)
|
||||
return new WorkflowResponse(build)
|
||||
}
|
||||
)
|
||||
|
||||
const onFinish = new Promise<void>((resolve) => {
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId,
|
||||
transactionId,
|
||||
subscriber: (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
resolve()
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
workflowOrcModule
|
||||
.run(workflowId, { transactionId })
|
||||
.run(workflowId, {
|
||||
transactionId,
|
||||
throwOnError: false,
|
||||
logOnError: true,
|
||||
})
|
||||
.then(({ result }) => {
|
||||
expect(result).toBe("result from step 0")
|
||||
})
|
||||
|
||||
const timeout = failTrap(
|
||||
done,
|
||||
"should prevent race continuation of the workflow during retryIntervalAwaiting in background execution"
|
||||
)
|
||||
await onFinish
|
||||
|
||||
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
|
||||
expect(step1InvokeMock.mock.calls.length).toBeGreaterThan(1)
|
||||
expect(step2InvokeMock).toHaveBeenCalledTimes(1)
|
||||
expect(transformMock).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it("should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution", (done) => {
|
||||
it("should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution", async () => {
|
||||
const transactionId = "transaction_id" + ulid()
|
||||
const workflowId = "RACE_workflow-1" + ulid()
|
||||
|
||||
@@ -157,7 +375,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
"RACE_step1",
|
||||
async (_) => {
|
||||
step1InvokeMock()
|
||||
await setTimeout(500)
|
||||
await setTimeout(1000)
|
||||
throw new Error("error from step 1")
|
||||
},
|
||||
() => {
|
||||
@@ -175,61 +393,63 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
|
||||
return new WorkflowResponse(status)
|
||||
})
|
||||
|
||||
createWorkflow(workflowId, function () {
|
||||
const build = step0()
|
||||
|
||||
const status = subWorkflow.runAsStep({} as any).config({
|
||||
async: true,
|
||||
compensateAsync: true,
|
||||
backgroundExecution: true,
|
||||
retryIntervalAwaiting: 0.1,
|
||||
})
|
||||
|
||||
const transformedResult = transform({ status }, (data) => {
|
||||
transformMock()
|
||||
return {
|
||||
status: data.status,
|
||||
}
|
||||
})
|
||||
|
||||
step2(transformedResult)
|
||||
return new WorkflowResponse(build)
|
||||
})
|
||||
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId: workflowId,
|
||||
transactionId,
|
||||
subscriber: (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
try {
|
||||
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
|
||||
expect(step0CompensateMock).toHaveBeenCalledTimes(1)
|
||||
expect(
|
||||
step1InvokeMock.mock.calls.length
|
||||
).toBeGreaterThanOrEqual(2) // Called every 0.1s at least (it can take more than 0.1sdepending on the event loop congestions)
|
||||
expect(step1CompensateMock).toHaveBeenCalledTimes(1)
|
||||
expect(step2InvokeMock).toHaveBeenCalledTimes(0)
|
||||
expect(transformMock).toHaveBeenCalledTimes(0)
|
||||
done()
|
||||
} catch (e) {
|
||||
return done(e)
|
||||
} finally {
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
}
|
||||
createWorkflow(
|
||||
{
|
||||
name: workflowId,
|
||||
},
|
||||
function () {
|
||||
const build = step0()
|
||||
|
||||
const status = subWorkflow.runAsStep({} as any).config({
|
||||
async: true,
|
||||
compensateAsync: true,
|
||||
backgroundExecution: true,
|
||||
retryIntervalAwaiting: 0.1,
|
||||
maxAwaitingRetries: 3,
|
||||
})
|
||||
|
||||
const transformedResult = transform({ status }, (data) => {
|
||||
transformMock()
|
||||
return {
|
||||
status: data.status,
|
||||
}
|
||||
})
|
||||
|
||||
step2(transformedResult)
|
||||
return new WorkflowResponse(build)
|
||||
}
|
||||
)
|
||||
|
||||
const onFinish = new Promise<void>((resolve) => {
|
||||
void workflowOrcModule.subscribe({
|
||||
workflowId,
|
||||
transactionId,
|
||||
subscriber: async (event) => {
|
||||
if (event.eventType === "onFinish") {
|
||||
resolve()
|
||||
}
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
workflowOrcModule
|
||||
.run(workflowId, { transactionId, throwOnError: false })
|
||||
await workflowOrcModule
|
||||
.run(workflowId, {
|
||||
transactionId,
|
||||
throwOnError: false,
|
||||
logOnError: true,
|
||||
})
|
||||
.then(({ result }) => {
|
||||
expect(result).toBe("result from step 0")
|
||||
})
|
||||
|
||||
const timeout = failTrap(
|
||||
done,
|
||||
"should prevent race continuation of the workflow compensation during retryIntervalAwaiting in background execution"
|
||||
)
|
||||
await onFinish
|
||||
|
||||
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
|
||||
expect(step0CompensateMock).toHaveBeenCalledTimes(1)
|
||||
expect(step1InvokeMock).toHaveBeenCalledTimes(3)
|
||||
expect(step1CompensateMock.mock.calls.length).toBeGreaterThan(0)
|
||||
expect(step2InvokeMock).toHaveBeenCalledTimes(0)
|
||||
expect(transformMock).toHaveBeenCalledTimes(0)
|
||||
})
|
||||
})
|
||||
},
|
||||
|
||||
@@ -30,20 +30,16 @@ async function deleteKeysByPattern(pattern) {
|
||||
count: 100,
|
||||
})
|
||||
|
||||
const pipeline = redis.pipeline()
|
||||
for await (const keys of stream) {
|
||||
if (keys.length) {
|
||||
const pipeline = redis.pipeline()
|
||||
keys.forEach((key) => pipeline.unlink(key))
|
||||
await pipeline.exec()
|
||||
}
|
||||
}
|
||||
await pipeline.exec()
|
||||
}
|
||||
|
||||
async function cleanRedis() {
|
||||
try {
|
||||
await deleteKeysByPattern("bull:*")
|
||||
await deleteKeysByPattern("dtrx:*")
|
||||
} catch (error) {
|
||||
console.error("Error:", error)
|
||||
}
|
||||
await deleteKeysByPattern("bull:*")
|
||||
await deleteKeysByPattern("dtrx:*")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user