asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(workflows-*): Allow to re run non idempotent but stored workflow with the same transaction id if considered done (#12362)

Browse Source
This commit is contained in:
Adrien de Peretti
2025-05-06 17:17:49 +02:00
committed by GitHub
parent 97dd520c64
commit 80007f3afd
31 changed files with 809 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
packages/modules/workflow-engine-inmemory/src/migrations/.snapshot-medusa-workflows.json
View File

@@ -24,6 +24,15 @@
"nullable": false,
"mappedType": "text"
},
"run_id": {
"name": "run_id",
"type": "text",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": false,
"mappedType": "text"
},
"id": {
"name": "id",
"type": "text",
@@ -151,13 +160,13 @@
"expression": "CREATE INDEX IF NOT EXISTS \"IDX_workflow_execution_transaction_id\" ON \"workflow_execution\" (transaction_id) WHERE deleted_at IS NULL"
},
{
"keyName": "IDX_workflow_execution_workflow_id_transaction_id_unique",
"keyName": "IDX_workflow_execution_workflow_id_transaction_id_run_id_unique",
"columnNames": [],
"composite": false,
"constraint": false,
"primary": false,
"unique": false,
"expression": "CREATE UNIQUE INDEX IF NOT EXISTS \"IDX_workflow_execution_workflow_id_transaction_id_unique\" ON \"workflow_execution\" (workflow_id, transaction_id) WHERE deleted_at IS NULL"
"expression": "CREATE UNIQUE INDEX IF NOT EXISTS \"IDX_workflow_execution_workflow_id_transaction_id_run_id_unique\" ON \"workflow_execution\" (workflow_id, transaction_id, run_id) WHERE deleted_at IS NULL"
},
{
"keyName": "IDX_workflow_execution_state",
@@ -172,7 +181,8 @@
"keyName": "workflow_execution_pkey",
"columnNames": [
"workflow_id",
"transaction_id"
"transaction_id",
"run_id"
],
"composite": true,
"constraint": true,

45
packages/modules/workflow-engine-inmemory/src/migrations/Migration20250505092459.ts Normal file
View File

@@ -0,0 +1,45 @@
import { Migration } from "@mikro-orm/migrations"
import { ulid } from "ulid"
export class Migration20250505092459 extends Migration {
override async up(): Promise<void> {
this.addSql(
`alter table if exists "workflow_execution" drop constraint if exists "workflow_execution_workflow_id_transaction_id_run_id_unique";`
)
this.addSql(
`drop index if exists "IDX_workflow_execution_workflow_id_transaction_id_unique";`
)
this.addSql(
`alter table if exists "workflow_execution" drop constraint if exists "PK_workflow_execution_workflow_id_transaction_id";`
)
this.addSql(
`alter table if exists "workflow_execution" add column if not exists "run_id" text not null default '${ulid()}';`
)
this.addSql(
`CREATE UNIQUE INDEX IF NOT EXISTS "IDX_workflow_execution_workflow_id_transaction_id_run_id_unique" ON "workflow_execution" (workflow_id, transaction_id, run_id) WHERE deleted_at IS NULL;`
)
this.addSql(
`alter table if exists "workflow_execution" add constraint "workflow_execution_pkey" primary key ("workflow_id", "transaction_id", "run_id");`
)
}
override async down(): Promise<void> {
this.addSql(
`drop index if exists "IDX_workflow_execution_workflow_id_transaction_id_run_id_unique";`
)
this.addSql(
`alter table if exists "workflow_execution" drop constraint if exists "workflow_execution_pkey";`
)
this.addSql(
`alter table if exists "workflow_execution" drop column if exists "run_id";`
)
this.addSql(
`CREATE UNIQUE INDEX IF NOT EXISTS "IDX_workflow_execution_workflow_id_transaction_id_unique" ON "workflow_execution" (workflow_id, transaction_id) WHERE deleted_at IS NULL;`
)
this.addSql(
`alter table if exists "workflow_execution" add constraint "workflow_execution_pkey" primary key ("workflow_id", "transaction_id");`
)
}
}

3
packages/modules/workflow-engine-inmemory/src/models/workflow-execution.ts
View File

@@ -6,6 +6,7 @@ export const WorkflowExecution = model
id: model.id({ prefix: "wf_exec" }),
workflow_id: model.text().primaryKey(),
transaction_id: model.text().primaryKey(),
run_id: model.text().primaryKey(),
execution: model.json().nullable(),
context: model.json().nullable(),
state: model.enum(TransactionState),
@@ -25,7 +26,7 @@ export const WorkflowExecution = model
where: "deleted_at IS NULL",
},
{
on: ["workflow_id", "transaction_id"],
on: ["workflow_id", "transaction_id", "run_id"],
unique: true,
where: "deleted_at IS NULL",
},

1
packages/modules/workflow-engine-inmemory/src/schema/index.ts
View File

@@ -16,6 +16,7 @@ type WorkflowExecution {
deleted_at: DateTime
workflow_id: string
transaction_id: string
run_id: string
execution: JSON
context: JSON
state: TransactionState

8
packages/modules/workflow-engine-inmemory/src/services/workflow-orchestrator.ts
View File

@@ -31,6 +31,7 @@ export type WorkflowOrchestratorRunOptions<T> = Omit<
"container"
> & {
transactionId?: string
runId?: string
container?: ContainerLike
}
@@ -140,7 +141,7 @@ export class WorkflowOrchestratorService {
let { throwOnError, context } = options ?? {}
throwOnError ??= true
context ??= {}
context.transactionId = transactionId ?? ulid()
context.transactionId = transactionId ?? "auto-" + ulid()
const workflowId = isString(workflowIdOrWorkflow)
? workflowIdOrWorkflow
@@ -259,7 +260,10 @@ export class WorkflowOrchestratorService {
const transaction = await this.getRunningTransaction(
workflowId,
transactionId,
options
{
...options,
isCancelling: true,
}
)
if (!transaction) {

99
packages/modules/workflow-engine-inmemory/src/utils/workflow-orchestrator-storage.ts
View File

@@ -5,11 +5,17 @@ import {
SchedulerOptions,
SkipExecutionError,
TransactionCheckpoint,
TransactionContext,
TransactionFlow,
TransactionOptions,
TransactionStep,
TransactionStepError,
} from "@medusajs/framework/orchestration"
import { Logger, ModulesSdkTypes } from "@medusajs/framework/types"
import {
InferEntityType,
Logger,
ModulesSdkTypes,
} from "@medusajs/framework/types"
import {
MedusaError,
TransactionState,
@@ -19,6 +25,7 @@ import {
} from "@medusajs/framework/utils"
import { WorkflowOrchestratorService } from "@services"
import { type CronExpression, parseExpression } from "cron-parser"
import { WorkflowExecution } from "../models/workflow-execution"
function parseNextExecution(
optionsOrExpression: SchedulerOptions | CronExpression | string | number
@@ -86,6 +93,7 @@ export class InMemoryDistributedTransactionStorage
{
workflow_id: data.flow.modelId,
transaction_id: data.flow.transactionId,
run_id: data.flow.runId,
execution: data.flow,
context: {
data: data.context,
@@ -102,13 +110,16 @@ export class InMemoryDistributedTransactionStorage
{
workflow_id: data.flow.modelId,
transaction_id: data.flow.transactionId,
run_id: data.flow.runId,
},
])
}
async get(
key: string,
options?: TransactionOptions
options?: TransactionOptions & {
isCancelling?: boolean
}
): Promise<TransactionCheckpoint | undefined> {
const data = this.storage.get(key)
@@ -122,23 +133,53 @@ export class InMemoryDistributedTransactionStorage
}
const [_, workflowId, transactionId] = key.split(":")
const trx = await this.workflowExecutionService_
.retrieve(
{
workflow_id: workflowId,
transaction_id: transactionId,
},
{
select: ["execution", "context"],
}
)
.catch(() => undefined)
const trx: InferEntityType<typeof WorkflowExecution> | undefined =
await this.workflowExecutionService_
.list(
{
workflow_id: workflowId,
transaction_id: transactionId,
},
{
select: ["execution", "context"],
order: {
id: "desc",
},
take: 1,
}
)
.then((trx) => trx[0])
.catch(() => undefined)
if (trx) {
const execution = trx.execution as TransactionFlow
if (!idempotent) {
const isFailedOrReverted = [
TransactionState.REVERTED,
TransactionState.FAILED,
].includes(execution.state)
const isDone = execution.state === TransactionState.DONE
const isCancellingAndFailedOrReverted =
options?.isCancelling && isFailedOrReverted
const isNotCancellingAndDoneOrFailedOrReverted =
!options?.isCancelling && (isDone || isFailedOrReverted)
if (
isCancellingAndFailedOrReverted ||
isNotCancellingAndDoneOrFailedOrReverted
) {
return
}
}
return {
flow: trx.execution,
context: trx.context.data,
errors: trx.context.errors,
flow: trx.execution as TransactionFlow,
context: trx.context?.data as TransactionContext,
errors: trx.context?.errors as TransactionStepError[],
}
}
@@ -181,6 +222,20 @@ export class InMemoryDistributedTransactionStorage
}
// Store in memory
const isNotStarted = data.flow.state === TransactionState.NOT_STARTED
const isManualTransactionId = !data.flow.transactionId.startsWith("auto-")
if (isNotStarted && isManualTransactionId) {
const storedData = this.storage.get(key)
if (storedData) {
throw new MedusaError(
MedusaError.Types.INVALID_ARGUMENT,
"Transaction already started for transactionId: " +
data.flow.transactionId
)
}
}
this.storage.set(key, data)
// Optimize DB operations - only perform when necessary
@@ -206,15 +261,23 @@ export class InMemoryDistributedTransactionStorage
key: string
options?: TransactionOptions
}) {
const isInitialCheckpoint = data.flow.state === TransactionState.NOT_STARTED
const isInitialCheckpoint = [TransactionState.NOT_STARTED].includes(
data.flow.state
)
/**
* In case many execution can succeed simultaneously, we need to ensure that the latest
* execution does continue if a previous execution is considered finished
*/
const currentFlow = data.flow
const getOptions = {
...options,
isCancelling: !!data.flow.cancelledAt,
} as Parameters<typeof this.get>[1]
const { flow: latestUpdatedFlow } =
(await this.get(key, options)) ??
(await this.get(key, getOptions)) ??
({ flow: {} } as { flow: TransactionFlow })
if (!isInitialCheckpoint && !isPresent(latestUpdatedFlow)) {