asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix(workflow-engines): race condition when retry interval is used (#11771)

Browse Source
This commit is contained in:
Adrien de Peretti
2025-03-12 09:53:34 -03:00
committed by GitHub
parent c97eaa0e0d
commit 72d2cf9207
24 changed files with 1130 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/orchestration/src/transaction/distributed-transaction.ts
+2 -3
View File
@@ -2,6 +2,7 @@ import { isDefined } from "@medusajs/utils"
import { EventEmitter } from "events"
import { IDistributedTransactionStorage } from "./datastore/abstract-storage"
import { BaseInMemoryDistributedTransactionStorage } from "./datastore/base-in-memory-storage"
import { NonSerializableCheckPointError } from "./errors"
import { TransactionOrchestrator } from "./transaction-orchestrator"
import { TransactionStep, TransactionStepHandler } from "./transaction-step"
import {
@@ -9,7 +10,6 @@ import {
TransactionHandlerType,
TransactionState,
} from "./types"
import { NonSerializableCheckPointError } from "./errors"
/**
* @typedef TransactionMetadata
@@ -229,6 +229,7 @@ class DistributedTransaction extends EventEmitter {
)
const options = TransactionOrchestrator.getWorkflowOptions(modelId)
const loadedData = await DistributedTransaction.keyValueStore.get(
key,
options
@@ -248,7 +249,6 @@ class DistributedTransaction extends EventEmitter {
return
}
await this.saveCheckpoint()
await DistributedTransaction.keyValueStore.scheduleRetry(
this,
step,
@@ -267,7 +267,6 @@ class DistributedTransaction extends EventEmitter {
return
}
await this.saveCheckpoint()
await DistributedTransaction.keyValueStore.scheduleTransactionTimeout(
this,
Date.now(),
packages/core/orchestration/src/transaction/errors.ts
+9
View File
@@ -84,3 +84,12 @@ export class NonSerializableCheckPointError extends Error {
this.name = "NonSerializableCheckPointError"
}
}
export class SkipExecutionError extends Error {
static isSkipExecutionError(error: Error): error is SkipExecutionError {
return (
error instanceof SkipExecutionError ||
error?.name === "SkipExecutionError"
)
}
}
packages/core/orchestration/src/transaction/transaction-orchestrator.ts
+91 -57
View File
@@ -28,6 +28,7 @@ import {
import { EventEmitter } from "events"
import {
PermanentStepFailureError,
SkipExecutionError,
SkipStepResponse,
TransactionStepTimeoutError,
TransactionTimeoutError,
@@ -54,7 +55,7 @@ export class TransactionOrchestrator extends EventEmitter {
} = {}
public static getWorkflowOptions(modelId: string): TransactionOptions {
return this.workflowOptions[modelId]
return TransactionOrchestrator.workflowOptions[modelId]
}
/**
@@ -239,6 +240,7 @@ export class TransactionOrchestrator extends EventEmitter {
) {
const flow = transaction.getFlow()
let hasTimedOut = false
if (!flow.timedOutAt && this.hasExpired({ transaction }, Date.now())) {
flow.timedOutAt = Date.now()
@@ -252,8 +254,6 @@ export class TransactionOrchestrator extends EventEmitter {
)
}
await transaction.saveCheckpoint()
this.emit(DistributedTransactionEvent.TIMEOUT, { transaction })
hasTimedOut = true
@@ -281,8 +281,6 @@ export class TransactionOrchestrator extends EventEmitter {
)
hasTimedOut = true
await transaction.saveCheckpoint()
this.emit(DistributedTransactionEvent.TIMEOUT, { transaction })
}
return hasTimedOut
@@ -457,7 +455,9 @@ export class TransactionOrchestrator extends EventEmitter {
transaction: DistributedTransactionType,
step: TransactionStep,
response: unknown
): Promise<void> {
): Promise<{
stopExecution: boolean
}> {
const hasStepTimedOut =
step.getStates().state === TransactionStepState.TIMEOUT
@@ -471,9 +471,6 @@ export class TransactionOrchestrator extends EventEmitter {
)
}
const flow = transaction.getFlow()
const options = TransactionOrchestrator.getWorkflowOptions(flow.modelId)
if (!hasStepTimedOut) {
step.changeStatus(TransactionStepStatus.OK)
}
@@ -484,8 +481,11 @@ export class TransactionOrchestrator extends EventEmitter {
step.changeState(TransactionStepState.DONE)
}
if (step.definition.async || options?.storeExecution) {
let shouldEmit = true
try {
await transaction.saveCheckpoint()
} catch (error) {
shouldEmit = false
}
const cleaningUp: Promise<unknown>[] = []
@@ -498,29 +498,41 @@ export class TransactionOrchestrator extends EventEmitter {
await promiseAll(cleaningUp)
const eventName = step.isCompensating()
? DistributedTransactionEvent.COMPENSATE_STEP_SUCCESS
: DistributedTransactionEvent.STEP_SUCCESS
transaction.emit(eventName, { step, transaction })
if (shouldEmit) {
const eventName = step.isCompensating()
? DistributedTransactionEvent.COMPENSATE_STEP_SUCCESS
: DistributedTransactionEvent.STEP_SUCCESS
transaction.emit(eventName, { step, transaction })
}
return {
stopExecution: !shouldEmit,
}
}
private static async skipStep(
transaction: DistributedTransactionType,
step: TransactionStep
): Promise<void> {
): Promise<{
stopExecution: boolean
}> {
const hasStepTimedOut =
step.getStates().state === TransactionStepState.TIMEOUT
const flow = transaction.getFlow()
const options = TransactionOrchestrator.getWorkflowOptions(flow.modelId)
if (!hasStepTimedOut) {
step.changeStatus(TransactionStepStatus.OK)
step.changeState(TransactionStepState.SKIPPED)
}
if (step.definition.async || options?.storeExecution) {
let shouldEmit = true
try {
await transaction.saveCheckpoint()
} catch (error) {
if (SkipExecutionError.isSkipExecutionError(error)) {
shouldEmit = false
} else {
throw error
}
}
const cleaningUp: Promise<unknown>[] = []
@@ -533,8 +545,14 @@ export class TransactionOrchestrator extends EventEmitter {
await promiseAll(cleaningUp)
const eventName = DistributedTransactionEvent.STEP_SKIPPED
transaction.emit(eventName, { step, transaction })
if (shouldEmit) {
const eventName = DistributedTransactionEvent.STEP_SKIPPED
transaction.emit(eventName, { step, transaction })
}
return {
stopExecution: !shouldEmit,
}
}
private static async setStepTimeout(
@@ -589,7 +607,15 @@ export class TransactionOrchestrator extends EventEmitter {
maxRetries: number = TransactionOrchestrator.DEFAULT_RETRIES,
isTimeout = false,
timeoutError?: TransactionStepTimeoutError | TransactionTimeoutError
): Promise<void> {
): Promise<{
stopExecution: boolean
}> {
if (SkipExecutionError.isSkipExecutionError(error)) {
return {
stopExecution: false,
}
}
step.failures++
if (isErrorLike(error)) {
@@ -604,7 +630,6 @@ export class TransactionOrchestrator extends EventEmitter {
}
const flow = transaction.getFlow()
const options = TransactionOrchestrator.getWorkflowOptions(flow.modelId)
const cleaningUp: Promise<unknown>[] = []
@@ -653,8 +678,15 @@ export class TransactionOrchestrator extends EventEmitter {
}
}
if (step.definition.async || options?.storeExecution) {
let shouldEmit = true
try {
await transaction.saveCheckpoint()
} catch (error) {
if (SkipExecutionError.isSkipExecutionError(error)) {
shouldEmit = false
} else {
throw error
}
}
if (step.hasRetryScheduled()) {
@@ -663,10 +695,16 @@ export class TransactionOrchestrator extends EventEmitter {
await promiseAll(cleaningUp)
const eventName = step.isCompensating()
? DistributedTransactionEvent.COMPENSATE_STEP_FAILURE
: DistributedTransactionEvent.STEP_FAILURE
transaction.emit(eventName, { step, transaction })
if (shouldEmit) {
const eventName = step.isCompensating()
? DistributedTransactionEvent.COMPENSATE_STEP_FAILURE
: DistributedTransactionEvent.STEP_FAILURE
transaction.emit(eventName, { step, transaction })
}
return {
stopExecution: !shouldEmit,
}
}
private async executeNext(
@@ -680,7 +718,6 @@ export class TransactionOrchestrator extends EventEmitter {
}
const flow = transaction.getFlow()
const options = TransactionOrchestrator.getWorkflowOptions(flow.modelId)
const nextSteps = await this.checkAllSteps(transaction)
const execution: Promise<void | unknown>[] = []
@@ -699,11 +736,9 @@ export class TransactionOrchestrator extends EventEmitter {
}
await transaction.saveCheckpoint()
this.emit(DistributedTransactionEvent.FINISH, { transaction })
}
let hasSyncSteps = false
for (const step of nextSteps.next) {
const curState = step.getStates()
const type = step.isCompensating()
@@ -783,19 +818,18 @@ export class TransactionOrchestrator extends EventEmitter {
)
}
await TransactionOrchestrator.setStepFailure(
const ret = await TransactionOrchestrator.setStepFailure(
transaction,
step,
error,
endRetry ? 0 : step.definition.maxRetries
)
if (isAsync) {
await transaction.scheduleRetry(
step,
step.definition.retryInterval ?? 0
)
if (isAsync && !ret.stopExecution) {
await transaction.scheduleRetry(step, 0)
}
return ret
}
const traceData = {
@@ -821,8 +855,6 @@ export class TransactionOrchestrator extends EventEmitter {
] as Parameters<TransactionStepHandler>
if (!isAsync) {
hasSyncSteps = true
const stepHandler = async () => {
return await transaction.handler(...handlerArgs)
}
@@ -875,10 +907,13 @@ export class TransactionOrchestrator extends EventEmitter {
endRetry: true,
response,
})
return
}
await setStepFailure(error, { response })
await setStepFailure(error, {
response,
})
})
)
} else {
@@ -933,10 +968,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
// check nested flow
await transaction.scheduleRetry(
step,
step.definition.retryInterval ?? 0
)
await transaction.scheduleRetry(step, 0)
})
.catch(async (error) => {
const response = error?.getStepResponse?.()
@@ -948,18 +980,27 @@ export class TransactionOrchestrator extends EventEmitter {
endRetry: true,
response,
})
return
}
await setStepFailure(error, { response })
await setStepFailure(error, {
response,
})
})
})
)
}
}
if (hasSyncSteps && options?.storeExecution) {
try {
await transaction.saveCheckpoint()
} catch (error) {
if (SkipExecutionError.isSkipExecutionError(error)) {
break
} else {
throw error
}
}
await promiseAll(execution)
@@ -993,11 +1034,9 @@ export class TransactionOrchestrator extends EventEmitter {
flow.state = TransactionState.INVOKING
flow.startedAt = Date.now()
if (this.getOptions().store) {
await transaction.saveCheckpoint(
flow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
)
}
await transaction.saveCheckpoint(
flow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
)
if (transaction.hasTimeout()) {
await transaction.scheduleTransactionTimeout(
@@ -1079,7 +1118,6 @@ export class TransactionOrchestrator extends EventEmitter {
isIdempotent
) {
this.options.store = true
this.options.storeExecution = true
}
const parsedOptions = {
@@ -1272,11 +1310,7 @@ export class TransactionOrchestrator extends EventEmitter {
existingTransaction?.context
)
if (
newTransaction &&
this.getOptions().store &&
this.getOptions().storeExecution
) {
if (newTransaction && this.getOptions().store) {
await transaction.saveCheckpoint(
modelFlow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
)
packages/core/orchestration/src/transaction/types.ts
+3 -2
View File
@@ -111,20 +111,21 @@ export type TransactionModelOptions = {
/**
* If true, the state of the transaction will be persisted.
*
*
* Learn more in [this documentation](https://docs.medusajs.com/learn/fundamentals/workflows/store-executions).
*/
store?: boolean
/**
* The number of seconds that the workflow execution should be stored in the database.
*
*
* Learn more in [this documentation](https://docs.medusajs.com/learn/fundamentals/workflows/store-executions).
*/
retentionTime?: number
/**
* If true, the execution details of each step will be stored.
* @deprecated no longer needed.
*/
storeExecution?: boolean
packages/core/utils/src/modules-sdk/medusa-internal-service.ts
+7 -3
View File
@@ -137,9 +137,11 @@ export function MedusaInternalService<
const idOrObject_ = Array.isArray(idOrObject)
? idOrObject
: [idOrObject]
primaryKeysCriteria = idOrObject_.map((primaryKeyValue) => ({
$and: primaryKeys.map((key) => ({ [key]: primaryKeyValue[key] })),
}))
primaryKeysCriteria = {
$or: idOrObject_.map((primaryKeyValue) => ({
$and: primaryKeys.map((key) => ({ [key]: primaryKeyValue[key] })),
})),
}
}
const queryOptions = buildQuery(primaryKeysCriteria, config)
@@ -157,6 +159,8 @@ export function MedusaInternalService<
? idOrObject.map((v) =>
[isString(v) ? v : Object.values(v)].join(", ")
)
: isObject(idOrObject)
? Object.values(idOrObject).join(", ")
: idOrObject
} was not found`
)
packages/core/workflows-sdk/src/helper/type.ts
+6 -1
View File
@@ -31,9 +31,14 @@ export type FlowRegisterStepFailureOptions<TData = unknown> =
response?: TData
}
export type FlowCancelOptions = BaseFlowRunOptions & {
export type FlowCancelOptions = {
transaction?: DistributedTransactionType
transactionId?: string
context?: Context
throwOnError?: boolean
logOnError?: boolean
events?: DistributedTransactionEvents
container?: LoadedModule[] | MedusaContainer
}
/**
packages/core/workflows-sdk/src/utils/composer/create-step.ts
+2 -4
View File
@@ -192,8 +192,6 @@ export function applyStep<
ret.__step__ = newStepName
WorkflowManager.update(this.workflowId, this.flow, this.handlers)
//const confRef = proxify(ret)
if (global[OrchestrationUtils.SymbolMedusaWorkflowComposerCondition]) {
const flagSteps =
global[OrchestrationUtils.SymbolMedusaWorkflowComposerCondition].steps
@@ -334,11 +332,11 @@ function wrapConditionalStep(
* createStep,
* StepResponse
* } from "@medusajs/framework/workflows-sdk"
*
*
* interface CreateProductInput {
* title: string
* }
*
*
* export const createProductStep = createStep(
* "createProductStep",
* async function (
packages/core/workflows-sdk/src/utils/composer/create-workflow.ts
+5 -5
View File
@@ -47,22 +47,22 @@ global[OrchestrationUtils.SymbolMedusaWorkflowComposerContext] = null
* createProductStep,
* getProductStep,
* } from "./steps"
*
*
* interface WorkflowInput {
* title: string
* }
*
*
* const myWorkflow = createWorkflow(
* "my-workflow",
* (input: WorkflowInput) => {
* // Everything here will be executed and resolved later
* // during the execution. Including the data access.
*
*
* const product = createProductStep(input)
* return new WorkflowResponse(getProductStep(product.id))
* }
* )
*
*
* export async function GET(
* req: MedusaRequest,
* res: MedusaResponse
@@ -73,7 +73,7 @@ global[OrchestrationUtils.SymbolMedusaWorkflowComposerContext] = null
* title: "Shirt"
* }
* })
*
*
* res.json({
* product
* })