asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix(workflow-engine-*): Cleanup expired executions and reduce redis storage usage (#12795)

Browse Source
This commit is contained in:
Adrien de Peretti
2025-06-24 13:32:10 +02:00
committed by GitHub
parent c0807f5496
commit 316a325b63
14 changed files with 620 additions and 392 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
packages/modules/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_scheduled.ts
View File

@@ -5,11 +5,19 @@ import {
StepResponse,
} from "@medusajs/framework/workflows-sdk"
export const createScheduled = (name: string, schedule?: SchedulerOptions) => {
export const createScheduled = (
name: string,
next: () => void,
schedule?: SchedulerOptions
) => {
const workflowScheduledStepInvoke = jest.fn((input, { container }) => {
return new StepResponse({
testValue: container.resolve("test-value"),
})
try {
return new StepResponse({
testValue: container.resolve("test-value", { allowUnregistered: true }),
})
} finally {
next()
}
})
const step = createStep("step_1", workflowScheduledStepInvoke)

218
packages/modules/workflow-engine-inmemory/integration-tests/__tests__/index.spec.ts
View File

@@ -6,9 +6,11 @@ import {
import {
Context,
IWorkflowEngineService,
Logger,
RemoteQueryFunction,
} from "@medusajs/framework/types"
import {
ContainerRegistrationKeys,
Module,
Modules,
promiseAll,
@@ -41,6 +43,7 @@ import {
workflowEventGroupIdStep2Mock,
} from "../__fixtures__/workflow_event_group_id"
import { createScheduled } from "../__fixtures__/workflow_scheduled"
import { container, MedusaContainer } from "@medusajs/framework"
jest.setTimeout(60000)
@@ -54,15 +57,44 @@ const failTrap = (done) => {
}, 5000)
}
function times(num) {
let resolver
let counter = 0
const promise = new Promise((resolve) => {
resolver = resolve
})
return {
next: () => {
counter += 1
if (counter === num) {
resolver()
}
},
// Force resolution after 10 seconds to prevent infinite awaiting
promise: Promise.race([
promise,
new Promise((_, reject) => {
setTimeoutSync(
() => reject("times has not been resolved after 10 seconds."),
10000
)
}),
]),
}
}
moduleIntegrationTestRunner<IWorkflowEngineService>({
moduleName: Modules.WORKFLOW_ENGINE,
resolve: __dirname + "/../..",
testSuite: ({ service: workflowOrcModule, medusaApp }) => {
describe("Workflow Orchestrator module", function () {
let query: RemoteQueryFunction
let sharedContainer_: MedusaContainer
beforeEach(() => {
query = medusaApp.query
sharedContainer_ = medusaApp.sharedContainer
})
it(`should export the appropriate linkable configuration`, () => {
@@ -797,7 +829,6 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
describe("Scheduled workflows", () => {
beforeEach(() => {
jest.clearAllMocks()
jest.useFakeTimers()
// Register test-value in the container for all tests
const sharedContainer =
@@ -809,62 +840,48 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
)
})
afterEach(() => {
jest.useRealTimers()
})
it("should execute a scheduled workflow", async () => {
const spy = createScheduled("standard", {
cron: "0 0 * * * *", // Runs at the start of every hour
})
expect(spy).toHaveBeenCalledTimes(0)
await jest.runOnlyPendingTimersAsync()
expect(spy).toHaveBeenCalledTimes(1)
await jest.runOnlyPendingTimersAsync()
const wait = times(2)
const spy = createScheduled("standard", wait.next)
await wait.promise
expect(spy).toHaveBeenCalledTimes(2)
WorkflowManager.unregister("standard")
})
it("should stop executions after the set number of executions", async () => {
const spy = await createScheduled("num-executions", {
const wait = times(2)
const spy = createScheduled("num-executions", wait.next, {
interval: 1000,
numberOfExecutions: 2,
})
expect(spy).toHaveBeenCalledTimes(0)
await jest.advanceTimersByTimeAsync(1100)
expect(spy).toHaveBeenCalledTimes(1)
await jest.advanceTimersByTimeAsync(1100)
await wait.promise
expect(spy).toHaveBeenCalledTimes(2)
await jest.advanceTimersByTimeAsync(1100)
// Make sure that on the next tick it doesn't execute again
await setTimeoutPromise(1100)
expect(spy).toHaveBeenCalledTimes(2)
WorkflowManager.unregister("num-execution")
})
it("should remove scheduled workflow if workflow no longer exists", async () => {
const spy = await createScheduled("remove-scheduled", {
const wait = times(1)
const logger = sharedContainer_.resolve<Logger>(
ContainerRegistrationKeys.LOGGER
)
const spy = createScheduled("remove-scheduled", wait.next, {
interval: 1000,
})
const logSpy = jest.spyOn(console, "warn")
expect(spy).toHaveBeenCalledTimes(0)
await jest.advanceTimersByTimeAsync(1100)
const logSpy = jest.spyOn(logger, "warn")
await wait.promise
expect(spy).toHaveBeenCalledTimes(1)
WorkflowManager["workflows"].delete("remove-scheduled")
await jest.advanceTimersByTimeAsync(1100)
await setTimeoutPromise(1100)
expect(spy).toHaveBeenCalledTimes(1)
expect(logSpy).toHaveBeenCalledWith(
"Tried to execute a scheduled workflow with ID remove-scheduled that does not exist, removing it from the scheduler."
@@ -872,23 +889,20 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
})
it("the scheduled workflow should have access to the shared container", async () => {
const spy = await createScheduled("shared-container-job", {
const wait = times(1)
const spy = await createScheduled("shared-container-job", wait.next, {
interval: 1000,
numberOfExecutions: 1,
})
await wait.promise
const initialCallCount = spy.mock.calls.length
expect(spy).toHaveBeenCalledTimes(1)
await jest.advanceTimersByTimeAsync(1100)
expect(spy).toHaveBeenCalledTimes(initialCallCount + 1)
console.log(spy.mock.results)
expect(spy).toHaveReturnedWith(
expect.objectContaining({ output: { testValue: "test" } })
)
await jest.advanceTimersByTimeAsync(1100)
expect(spy).toHaveBeenCalledTimes(initialCallCount + 1)
WorkflowManager.unregister("shared-container-job")
})
it("should fetch an idempotent workflow after its completion", async () => {
@@ -931,6 +945,120 @@ moduleIntegrationTestRunner<IWorkflowEngineService>({
expect(executionsListAfter).toHaveLength(1)
})
})
describe("Cleaner job", function () {
it("should remove expired executions of finished workflows and keep the others", async () => {
const doneWorkflowId = "done-workflow-" + ulid()
createWorkflow({ name: doneWorkflowId, retentionTime: 1 }, () => {
return new WorkflowResponse("done")
})
const failingWorkflowId = "failing-workflow-" + ulid()
const failingStep = createStep("failing-step", () => {
throw new Error("I am failing")
})
createWorkflow({ name: failingWorkflowId, retentionTime: 1 }, () => {
failingStep()
})
const revertingStep = createStep(
"reverting-step",
() => {
throw new Error("I am reverting")
},
() => {
return new StepResponse("reverted")
}
)
const revertingWorkflowId = "reverting-workflow-" + ulid()
createWorkflow(
{ name: revertingWorkflowId, retentionTime: 1 },
() => {
revertingStep()
return new WorkflowResponse("reverted")
}
)
const runningWorkflowId = "running-workflow-" + ulid()
const longRunningStep = createStep("long-running-step", async () => {
await setTimeoutPromise(10000)
return new StepResponse("long running finished")
})
createWorkflow({ name: runningWorkflowId, retentionTime: 1 }, () => {
longRunningStep().config({ async: true, backgroundExecution: true })
return new WorkflowResponse("running workflow started")
})
const notExpiredWorkflowId = "not-expired-workflow-" + ulid()
createWorkflow(
{ name: notExpiredWorkflowId, retentionTime: 1000 },
() => {
return new WorkflowResponse("not expired")
}
)
const trx_done = "trx-done-" + ulid()
const trx_failed = "trx-failed-" + ulid()
const trx_reverting = "trx-reverting-" + ulid()
const trx_running = "trx-running-" + ulid()
const trx_not_expired = "trx-not-expired-" + ulid()
// run workflows
await workflowOrcModule.run(doneWorkflowId, {
transactionId: trx_done,
})
await workflowOrcModule.run(failingWorkflowId, {
transactionId: trx_failed,
throwOnError: false,
})
await workflowOrcModule.run(revertingWorkflowId, {
transactionId: trx_reverting,
throwOnError: false,
})
await workflowOrcModule.run(runningWorkflowId, {
transactionId: trx_running,
})
await workflowOrcModule.run(notExpiredWorkflowId, {
transactionId: trx_not_expired,
})
const executions = await workflowOrcModule.listWorkflowExecutions()
expect(executions).toHaveLength(5)
await setTimeoutPromise(2000)
// Manually trigger cleaner
await (workflowOrcModule as any).workflowOrchestratorService_[
"inMemoryDistributedTransactionStorage_"
]["clearExpiredExecutions"]()
let remainingExecutions =
await workflowOrcModule.listWorkflowExecutions()
expect(remainingExecutions).toHaveLength(2)
const remainingTrxIds = remainingExecutions
.map((e) => e.transaction_id)
.sort()
expect(remainingTrxIds).toEqual([trx_not_expired, trx_running].sort())
const notExpiredExec = remainingExecutions.find(
(e) => e.transaction_id === trx_not_expired
)
expect(notExpiredExec?.state).toBe(TransactionState.DONE)
const runningExec = remainingExecutions.find(
(e) => e.transaction_id === trx_running
)
expect(runningExec?.state).toBe(TransactionState.INVOKING)
})
})
})
},
})