asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(workflows-*): Allow to re run non idempotent but stored workflow with the same transaction id if considered done (#12362)

Browse Source
This commit is contained in:
Adrien de Peretti
2025-05-06 17:17:49 +02:00
committed by GitHub
parent 97dd520c64
commit 80007f3afd
31 changed files with 809 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
packages/modules/workflow-engine-redis/integration-tests/__fixtures__/index.ts
View File

@@ -6,3 +6,4 @@ export * from "./workflow_step_timeout"
export * from "./workflow_sync"
export * from "./workflow_transaction_timeout"
export * from "./workflow_when"
export * from "./workflow_not_idempotent_with_retention"

71
packages/modules/workflow-engine-redis/integration-tests/__fixtures__/workflow_not_idempotent_with_retention.ts Normal file
View File

@@ -0,0 +1,71 @@
import {
createStep,
createWorkflow,
StepResponse,
} from "@medusajs/framework/workflows-sdk"
const step_1 = createStep(
"step_1",
jest.fn((input) => {
input.test = "test"
return new StepResponse(input, { compensate: 123 })
}),
jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
return new StepResponse({
reverted: true,
})
})
)
export const workflowNotIdempotentWithRetentionStep2Invoke = jest.fn(
(input, context) => {
if (input) {
return new StepResponse({ notAsyncResponse: input.hey })
}
}
)
const step_2 = createStep(
"step_2",
workflowNotIdempotentWithRetentionStep2Invoke,
jest.fn((_, context) => {
return new StepResponse({
step: context.metadata.action,
idempotency_key: context.metadata.idempotency_key,
reverted: true,
})
})
)
export const workflowNotIdempotentWithRetentionStep3Invoke = jest.fn((res) => {
return new StepResponse({
done: {
inputFromSyncStep: res.notAsyncResponse,
},
})
})
const step_3 = createStep(
"step_3",
workflowNotIdempotentWithRetentionStep3Invoke
)
createWorkflow(
{
name: "workflow_not_idempotent_with_retention",
retentionTime: 60,
},
function (input) {
step_1(input)
step_2({ hey: "oh" })
const ret2 = step_2({ hey: "hello" }).config({
name: "new_step_name",
})
return step_3(ret2)
}
)

131
packages/modules/workflow-engine-redis/integration-tests/__tests__/index.spec.ts
View File

@@ -16,6 +16,7 @@ import {
ContainerRegistrationKeys,
Module,
Modules,
promiseAll,
TransactionHandlerType,
TransactionStepState,
} from "@medusajs/framework/utils"
@@ -33,14 +34,19 @@ import { WorkflowsModuleService } from "../../src/services"
import "../__fixtures__"
import { createScheduled } from "../__fixtures__/workflow_scheduled"
import { TestDatabase } from "../utils"
import {
workflowNotIdempotentWithRetentionStep2Invoke,
workflowNotIdempotentWithRetentionStep3Invoke,
} from "../__fixtures__"
import { ulid } from "ulid"
jest.setTimeout(300000)
const failTrap = (done) => {
const failTrap = (done, name) => {
setTimeoutSync(() => {
// REF:https://stackoverflow.com/questions/78028715/jest-async-test-with-event-emitter-isnt-ending
console.warn(
"Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually."
`Jest is breaking the event emit with its debouncer. This allows to continue the test by managing the timeout of the test manually. ${name}`
)
done()
}, 5000)
@@ -132,11 +138,56 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
primaryKey: "workflow_id",
serviceName: "workflows",
},
run_id: {
entity: "WorkflowExecution",
field: "workflowExecution",
linkable: "workflow_execution_run_id",
primaryKey: "run_id",
serviceName: "workflows",
},
},
})
})
describe("Testing basic workflow", function () {
it("should prevent executing twice the same workflow in perfect concurrency with the same transactionId and non idempotent and not async but retention time is set", async () => {
const transactionId = "transaction_id"
const workflowId = "workflow_id" + ulid()
const step1 = createStep("step1", async () => {
await setTimeout(100)
return new StepResponse("step1")
})
createWorkflow(
{
name: workflowId,
retentionTime: 1000,
},
function () {
return new WorkflowResponse(step1())
}
)
const [result1, result2] = await promiseAll([
workflowOrcModule.run(workflowId, {
input: {},
transactionId,
}),
workflowOrcModule
.run(workflowId, {
input: {},
transactionId,
})
.catch((e) => e),
])
expect(result1.result).toEqual("step1")
expect(result2.message).toEqual(
"Transaction already started for transactionId: " + transactionId
)
})
it("should return a list of workflow executions and remove after completed when there is no retentionTime set", async () => {
await workflowOrcModule.run("workflow_1", {
input: {
@@ -294,6 +345,72 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
expect(done).toBe(true)
})
it("should return a list of workflow executions and keep it saved when there is a retentionTime set but allow for executing the same workflow multiple times with different run_id if the workflow is considered done", async () => {
const transactionId = "transaction_1"
await workflowOrcModule.run(
"workflow_not_idempotent_with_retention",
{
input: {
value: "123",
},
transactionId,
}
)
let { data: executionsList } = await query.graph({
entity: "workflow_executions",
fields: ["id", "run_id", "transaction_id"],
})
expect(executionsList).toHaveLength(1)
expect(
workflowNotIdempotentWithRetentionStep2Invoke
).toHaveBeenCalledTimes(2)
expect(
workflowNotIdempotentWithRetentionStep2Invoke.mock.calls[0][0]
).toEqual({ hey: "oh" })
expect(
workflowNotIdempotentWithRetentionStep2Invoke.mock.calls[1][0]
).toEqual({
hey: "hello",
})
expect(
workflowNotIdempotentWithRetentionStep3Invoke
).toHaveBeenCalledTimes(1)
expect(
workflowNotIdempotentWithRetentionStep3Invoke.mock.calls[0][0]
).toEqual({
notAsyncResponse: "hello",
})
await workflowOrcModule.run(
"workflow_not_idempotent_with_retention",
{
input: {
value: "123",
},
transactionId,
}
)
const { data: executionsList2 } = await query.graph({
entity: "workflow_executions",
filters: {
id: { $nin: executionsList.map((e) => e.id) },
},
fields: ["id", "run_id", "transaction_id"],
})
expect(executionsList2).toHaveLength(1)
expect(executionsList2[0].run_id).not.toEqual(
executionsList[0].run_id
)
expect(executionsList2[0].transaction_id).toEqual(
executionsList[0].transaction_id
)
})
it("should revert the entire transaction when a step timeout expires", async () => {
const { transaction, result, errors } = (await workflowOrcModule.run(
"workflow_step_timeout",
@@ -386,7 +503,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
throwOnError: false,
})
await setTimeout(200)
await setTimeout(500)
const { transaction, result, errors } = (await workflowOrcModule.run(
"workflow_transaction_timeout_async",
@@ -437,7 +554,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
},
})
failTrap(done)
failTrap(done, "workflow_async_background")
})
it("should subscribe to a async workflow and receive the response when it finishes", (done) => {
@@ -466,7 +583,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
expect(onFinish).toHaveBeenCalledTimes(0)
failTrap(done)
failTrap(done, "workflow_async_background")
})
it("should not skip step if condition is true", function (done) {
@@ -488,7 +605,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
},
})
failTrap(done)
failTrap(done, "wf-when")
})
it("should cancel an async sub workflow when compensating", (done) => {
@@ -526,7 +643,7 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
},
})
failTrap(done)
failTrap(done, "workflow_async_background_fail")
})
it("should cancel and revert a completed workflow", async () => {

51
packages/modules/workflow-engine-redis/integration-tests/__tests__/race.spec.ts
View File

@@ -10,6 +10,7 @@ import {
import { moduleIntegrationTestRunner } from "@medusajs/test-utils"
import { setTimeout as setTimeoutSync } from "timers"
import { setTimeout } from "timers/promises"
import { ulid } from "ulid"
import "../__fixtures__"
jest.setTimeout(300000)
@@ -38,6 +39,8 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
describe("Testing race condition of the workflow during retry", () => {
it("should prevent race continuation of the workflow during retryIntervalAwaiting in background execution", (done) => {
const transactionId = "transaction_id"
const workflowId = "workflow-1" + ulid()
const subWorkflowId = "sub-" + workflowId
const step0InvokeMock = jest.fn()
const step1InvokeMock = jest.fn()
@@ -60,12 +63,12 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
return new StepResponse({ result: input })
})
const subWorkflow = createWorkflow("sub-workflow-1", function () {
const subWorkflow = createWorkflow(subWorkflowId, function () {
const status = step1()
return new WorkflowResponse(status)
})
createWorkflow("workflow-1", function () {
createWorkflow(workflowId, function () {
const build = step0()
const status = subWorkflow.runAsStep({} as any).config({
@@ -87,21 +90,28 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
})
void workflowOrcModule.subscribe({
workflowId: "workflow-1",
workflowId,
transactionId,
subscriber: (event) => {
subscriber: async (event) => {
if (event.eventType === "onFinish") {
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
expect(step1InvokeMock.mock.calls.length).toBeGreaterThan(1)
expect(step2InvokeMock).toHaveBeenCalledTimes(1)
expect(transformMock).toHaveBeenCalledTimes(1)
setTimeoutSync(done, 500)
try {
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
expect(step1InvokeMock.mock.calls.length).toBeGreaterThan(1)
expect(step2InvokeMock).toHaveBeenCalledTimes(1)
expect(transformMock).toHaveBeenCalledTimes(1)
// Prevent killing the test to early
await setTimeout(500)
done()
} catch (e) {
return done(e)
}
}
},
})
workflowOrcModule
.run("workflow-1", { transactionId })
.run(workflowId, { transactionId })
.then(({ result }) => {
expect(result).toBe("result from step 0")
})
@@ -179,14 +189,19 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
transactionId,
subscriber: (event) => {
if (event.eventType === "onFinish") {
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
expect(step0CompensateMock).toHaveBeenCalledTimes(2) // TODO: review this.
expect(step1InvokeMock).toHaveBeenCalledTimes(1)
expect(step1CompensateMock).toHaveBeenCalledTimes(1)
expect(step2InvokeMock).toHaveBeenCalledTimes(0)
expect(transformMock).toHaveBeenCalledTimes(0)
done()
try {
expect(step0InvokeMock).toHaveBeenCalledTimes(1)
expect(step0CompensateMock).toHaveBeenCalledTimes(1)
expect(
step1InvokeMock.mock.calls.length
).toBeGreaterThanOrEqual(2) // Called every 0.1s at least (it can take more than 0.1sdepending on the event loop congestions)
expect(step1CompensateMock).toHaveBeenCalledTimes(1)
expect(step2InvokeMock).toHaveBeenCalledTimes(0)
expect(transformMock).toHaveBeenCalledTimes(0)
done()
} catch (e) {
return done(e)
}
}
},
})