asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix(workflow-engines): race condition when retry interval is used (#11771)

Browse Source
This commit is contained in:
Adrien de Peretti
2025-03-12 09:53:34 -03:00
committed by GitHub
parent c97eaa0e0d
commit 72d2cf9207
24 changed files with 1130 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/orchestration/src/transaction/distributed-transaction.ts
+2 -3
View File
@@ -2,6 +2,7 @@ import { isDefined } from "@medusajs/utils"
import { EventEmitter } from "events"
import { IDistributedTransactionStorage } from "./datastore/abstract-storage"
import { BaseInMemoryDistributedTransactionStorage } from "./datastore/base-in-memory-storage"
import { NonSerializableCheckPointError } from "./errors"
import { TransactionOrchestrator } from "./transaction-orchestrator"
import { TransactionStep, TransactionStepHandler } from "./transaction-step"
import {
@@ -9,7 +10,6 @@ import {
TransactionHandlerType,
TransactionState,
} from "./types"
import { NonSerializableCheckPointError } from "./errors"
/**
* @typedef TransactionMetadata
@@ -229,6 +229,7 @@ class DistributedTransaction extends EventEmitter {
)
const options = TransactionOrchestrator.getWorkflowOptions(modelId)
const loadedData = await DistributedTransaction.keyValueStore.get(
key,
options
@@ -248,7 +249,6 @@ class DistributedTransaction extends EventEmitter {
return
}
await this.saveCheckpoint()
await DistributedTransaction.keyValueStore.scheduleRetry(
this,
step,
@@ -267,7 +267,6 @@ class DistributedTransaction extends EventEmitter {
return
}
await this.saveCheckpoint()
await DistributedTransaction.keyValueStore.scheduleTransactionTimeout(
this,
Date.now(),
packages/core/orchestration/src/transaction/errors.ts
+9
View File
@@ -84,3 +84,12 @@ export class NonSerializableCheckPointError extends Error {
this.name = "NonSerializableCheckPointError"
}
}
export class SkipExecutionError extends Error {
static isSkipExecutionError(error: Error): error is SkipExecutionError {
return (
error instanceof SkipExecutionError ||
error?.name === "SkipExecutionError"
)
}
}
packages/core/orchestration/src/transaction/transaction-orchestrator.ts
+91 -57
View File
@@ -28,6 +28,7 @@ import {
import { EventEmitter } from "events"
import {
PermanentStepFailureError,
SkipExecutionError,
SkipStepResponse,
TransactionStepTimeoutError,
TransactionTimeoutError,
@@ -54,7 +55,7 @@ export class TransactionOrchestrator extends EventEmitter {
} = {}
public static getWorkflowOptions(modelId: string): TransactionOptions {
return this.workflowOptions[modelId]
return TransactionOrchestrator.workflowOptions[modelId]
}
/**
@@ -239,6 +240,7 @@ export class TransactionOrchestrator extends EventEmitter {
) {
const flow = transaction.getFlow()
let hasTimedOut = false
if (!flow.timedOutAt && this.hasExpired({ transaction }, Date.now())) {
flow.timedOutAt = Date.now()
@@ -252,8 +254,6 @@ export class TransactionOrchestrator extends EventEmitter {
)
}
await transaction.saveCheckpoint()
this.emit(DistributedTransactionEvent.TIMEOUT, { transaction })
hasTimedOut = true
@@ -281,8 +281,6 @@ export class TransactionOrchestrator extends EventEmitter {
)
hasTimedOut = true
await transaction.saveCheckpoint()
this.emit(DistributedTransactionEvent.TIMEOUT, { transaction })
}
return hasTimedOut
@@ -457,7 +455,9 @@ export class TransactionOrchestrator extends EventEmitter {
transaction: DistributedTransactionType,
step: TransactionStep,
response: unknown
): Promise<void> {
): Promise<{
stopExecution: boolean
}> {
const hasStepTimedOut =
step.getStates().state === TransactionStepState.TIMEOUT
@@ -471,9 +471,6 @@ export class TransactionOrchestrator extends EventEmitter {
)
}
const flow = transaction.getFlow()
const options = TransactionOrchestrator.getWorkflowOptions(flow.modelId)
if (!hasStepTimedOut) {
step.changeStatus(TransactionStepStatus.OK)
}
@@ -484,8 +481,11 @@ export class TransactionOrchestrator extends EventEmitter {
step.changeState(TransactionStepState.DONE)
}
if (step.definition.async || options?.storeExecution) {
let shouldEmit = true
try {
await transaction.saveCheckpoint()
} catch (error) {
shouldEmit = false
}
const cleaningUp: Promise<unknown>[] = []
@@ -498,29 +498,41 @@ export class TransactionOrchestrator extends EventEmitter {
await promiseAll(cleaningUp)
const eventName = step.isCompensating()
? DistributedTransactionEvent.COMPENSATE_STEP_SUCCESS
: DistributedTransactionEvent.STEP_SUCCESS
transaction.emit(eventName, { step, transaction })
if (shouldEmit) {
const eventName = step.isCompensating()
? DistributedTransactionEvent.COMPENSATE_STEP_SUCCESS
: DistributedTransactionEvent.STEP_SUCCESS
transaction.emit(eventName, { step, transaction })
}
return {
stopExecution: !shouldEmit,
}
}
private static async skipStep(
transaction: DistributedTransactionType,
step: TransactionStep
): Promise<void> {
): Promise<{
stopExecution: boolean
}> {
const hasStepTimedOut =
step.getStates().state === TransactionStepState.TIMEOUT
const flow = transaction.getFlow()
const options = TransactionOrchestrator.getWorkflowOptions(flow.modelId)
if (!hasStepTimedOut) {
step.changeStatus(TransactionStepStatus.OK)
step.changeState(TransactionStepState.SKIPPED)
}
if (step.definition.async || options?.storeExecution) {
let shouldEmit = true
try {
await transaction.saveCheckpoint()
} catch (error) {
if (SkipExecutionError.isSkipExecutionError(error)) {
shouldEmit = false
} else {
throw error
}
}
const cleaningUp: Promise<unknown>[] = []
@@ -533,8 +545,14 @@ export class TransactionOrchestrator extends EventEmitter {
await promiseAll(cleaningUp)
const eventName = DistributedTransactionEvent.STEP_SKIPPED
transaction.emit(eventName, { step, transaction })
if (shouldEmit) {
const eventName = DistributedTransactionEvent.STEP_SKIPPED
transaction.emit(eventName, { step, transaction })
}
return {
stopExecution: !shouldEmit,
}
}
private static async setStepTimeout(
@@ -589,7 +607,15 @@ export class TransactionOrchestrator extends EventEmitter {
maxRetries: number = TransactionOrchestrator.DEFAULT_RETRIES,
isTimeout = false,
timeoutError?: TransactionStepTimeoutError | TransactionTimeoutError
): Promise<void> {
): Promise<{
stopExecution: boolean
}> {
if (SkipExecutionError.isSkipExecutionError(error)) {
return {
stopExecution: false,
}
}
step.failures++
if (isErrorLike(error)) {
@@ -604,7 +630,6 @@ export class TransactionOrchestrator extends EventEmitter {
}
const flow = transaction.getFlow()
const options = TransactionOrchestrator.getWorkflowOptions(flow.modelId)
const cleaningUp: Promise<unknown>[] = []
@@ -653,8 +678,15 @@ export class TransactionOrchestrator extends EventEmitter {
}
}
if (step.definition.async || options?.storeExecution) {
let shouldEmit = true
try {
await transaction.saveCheckpoint()
} catch (error) {
if (SkipExecutionError.isSkipExecutionError(error)) {
shouldEmit = false
} else {
throw error
}
}
if (step.hasRetryScheduled()) {
@@ -663,10 +695,16 @@ export class TransactionOrchestrator extends EventEmitter {
await promiseAll(cleaningUp)
const eventName = step.isCompensating()
? DistributedTransactionEvent.COMPENSATE_STEP_FAILURE
: DistributedTransactionEvent.STEP_FAILURE
transaction.emit(eventName, { step, transaction })
if (shouldEmit) {
const eventName = step.isCompensating()
? DistributedTransactionEvent.COMPENSATE_STEP_FAILURE
: DistributedTransactionEvent.STEP_FAILURE
transaction.emit(eventName, { step, transaction })
}
return {
stopExecution: !shouldEmit,
}
}
private async executeNext(
@@ -680,7 +718,6 @@ export class TransactionOrchestrator extends EventEmitter {
}
const flow = transaction.getFlow()
const options = TransactionOrchestrator.getWorkflowOptions(flow.modelId)
const nextSteps = await this.checkAllSteps(transaction)
const execution: Promise<void | unknown>[] = []
@@ -699,11 +736,9 @@ export class TransactionOrchestrator extends EventEmitter {
}
await transaction.saveCheckpoint()
this.emit(DistributedTransactionEvent.FINISH, { transaction })
}
let hasSyncSteps = false
for (const step of nextSteps.next) {
const curState = step.getStates()
const type = step.isCompensating()
@@ -783,19 +818,18 @@ export class TransactionOrchestrator extends EventEmitter {
)
}
await TransactionOrchestrator.setStepFailure(
const ret = await TransactionOrchestrator.setStepFailure(
transaction,
step,
error,
endRetry ? 0 : step.definition.maxRetries
)
if (isAsync) {
await transaction.scheduleRetry(
step,
step.definition.retryInterval ?? 0
)
if (isAsync && !ret.stopExecution) {
await transaction.scheduleRetry(step, 0)
}
return ret
}
const traceData = {
@@ -821,8 +855,6 @@ export class TransactionOrchestrator extends EventEmitter {
] as Parameters<TransactionStepHandler>
if (!isAsync) {
hasSyncSteps = true
const stepHandler = async () => {
return await transaction.handler(...handlerArgs)
}
@@ -875,10 +907,13 @@ export class TransactionOrchestrator extends EventEmitter {
endRetry: true,
response,
})
return
}
await setStepFailure(error, { response })
await setStepFailure(error, {
response,
})
})
)
} else {
@@ -933,10 +968,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
// check nested flow
await transaction.scheduleRetry(
step,
step.definition.retryInterval ?? 0
)
await transaction.scheduleRetry(step, 0)
})
.catch(async (error) => {
const response = error?.getStepResponse?.()
@@ -948,18 +980,27 @@ export class TransactionOrchestrator extends EventEmitter {
endRetry: true,
response,
})
return
}
await setStepFailure(error, { response })
await setStepFailure(error, {
response,
})
})
})
)
}
}
if (hasSyncSteps && options?.storeExecution) {
try {
await transaction.saveCheckpoint()
} catch (error) {
if (SkipExecutionError.isSkipExecutionError(error)) {
break
} else {
throw error
}
}
await promiseAll(execution)
@@ -993,11 +1034,9 @@ export class TransactionOrchestrator extends EventEmitter {
flow.state = TransactionState.INVOKING
flow.startedAt = Date.now()
if (this.getOptions().store) {
await transaction.saveCheckpoint(
flow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
)
}
await transaction.saveCheckpoint(
flow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
)
if (transaction.hasTimeout()) {
await transaction.scheduleTransactionTimeout(
@@ -1079,7 +1118,6 @@ export class TransactionOrchestrator extends EventEmitter {
isIdempotent
) {
this.options.store = true
this.options.storeExecution = true
}
const parsedOptions = {
@@ -1272,11 +1310,7 @@ export class TransactionOrchestrator extends EventEmitter {
existingTransaction?.context
)
if (
newTransaction &&
this.getOptions().store &&
this.getOptions().storeExecution
) {
if (newTransaction && this.getOptions().store) {
await transaction.saveCheckpoint(
modelFlow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
)
packages/core/orchestration/src/transaction/types.ts
+3 -2
View File
@@ -111,20 +111,21 @@ export type TransactionModelOptions = {
/**
* If true, the state of the transaction will be persisted.
*
*
* Learn more in [this documentation](https://docs.medusajs.com/learn/fundamentals/workflows/store-executions).
*/
store?: boolean
/**
* The number of seconds that the workflow execution should be stored in the database.
*
*
* Learn more in [this documentation](https://docs.medusajs.com/learn/fundamentals/workflows/store-executions).
*/
retentionTime?: number
/**
* If true, the execution details of each step will be stored.
* @deprecated no longer needed.
*/
storeExecution?: boolean