asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore(workflow-engine-*): cleanup and improvements (#13789)

Browse Source 
**What**
Cleanup recent work on workflows
This commit is contained in:
Adrien de Peretti
2025-10-23 12:50:24 +02:00
committed by GitHub
parent 356dcc94ce
commit d51ae2768b
10 changed files with 344 additions and 310 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/orchestration/src/transaction/distributed-transaction.ts
+101 -114
View File
@@ -55,6 +55,9 @@ const mergeStep = (
}
}
const getErrorSignature = (err: TransactionStepError) =>
`${err.action}:${err.handlerType}:${err.error?.message}`
/**
* @typedef TransactionMetadata
* @property model_id - The id of the model_id that created the transaction (modelId).
@@ -105,7 +108,52 @@ const stateFlowOrder = [
TransactionState.FAILED,
]
const stateFlowOrderMap = new Map<TransactionState, number>(
stateFlowOrder.map((state, index) => [state, index])
)
const finishedStatesSet = new Set([
TransactionState.DONE,
TransactionState.REVERTED,
TransactionState.FAILED,
])
export class TransactionCheckpoint {
static readonly #ALLOWED_STATE_TRANSITIONS = {
[TransactionStepState.DORMANT]: [TransactionStepState.NOT_STARTED],
[TransactionStepState.NOT_STARTED]: [
TransactionStepState.INVOKING,
TransactionStepState.COMPENSATING,
TransactionStepState.FAILED,
TransactionStepState.SKIPPED,
TransactionStepState.SKIPPED_FAILURE,
],
[TransactionStepState.INVOKING]: [
TransactionStepState.FAILED,
TransactionStepState.DONE,
TransactionStepState.TIMEOUT,
TransactionStepState.SKIPPED,
],
[TransactionStepState.COMPENSATING]: [
TransactionStepState.REVERTED,
TransactionStepState.FAILED,
],
[TransactionStepState.DONE]: [TransactionStepState.COMPENSATING],
} as const
static readonly #ALLOWED_STATUS_TRANSITIONS = {
[TransactionStepStatus.WAITING]: [
TransactionStepStatus.OK,
TransactionStepStatus.TEMPORARY_FAILURE,
TransactionStepStatus.PERMANENT_FAILURE,
],
[TransactionStepStatus.TEMPORARY_FAILURE]: [
TransactionStepStatus.IDLE,
TransactionStepStatus.PERMANENT_FAILURE,
],
[TransactionStepStatus.PERMANENT_FAILURE]: [TransactionStepStatus.IDLE],
} as const
constructor(
public flow: TransactionFlow,
public context: TransactionContext,
@@ -165,17 +213,15 @@ export class TransactionCheckpoint {
currentTransactionData.flow[prop] ?? 0
)
} else if (prop === "state") {
const curState = stateFlowOrder.findIndex(
(state) => state === currentTransactionData.flow.state
)
const storedState = stateFlowOrder.findIndex(
(state) => state === storedData.flow.state
)
const currentStateIndex =
stateFlowOrderMap.get(currentTransactionData.flow.state) ?? -1
const storedStateIndex =
stateFlowOrderMap.get(storedData.flow.state) ?? -1
if (storedState > curState) {
if (storedStateIndex > currentStateIndex) {
currentTransactionData.flow.state = storedData.flow.state
} else if (
curState < storedState &&
currentStateIndex < storedStateIndex &&
currentTransactionData.flow.state !==
TransactionState.WAITING_TO_COMPENSATE
) {
@@ -265,43 +311,6 @@ export class TransactionCheckpoint {
status: TransactionStepStatus
}
): boolean {
// Define allowed state transitions
const allowedStateTransitions = {
[TransactionStepState.DORMANT]: [TransactionStepState.NOT_STARTED],
[TransactionStepState.NOT_STARTED]: [
TransactionStepState.INVOKING,
TransactionStepState.COMPENSATING,
TransactionStepState.FAILED,
TransactionStepState.SKIPPED,
TransactionStepState.SKIPPED_FAILURE,
],
[TransactionStepState.INVOKING]: [
TransactionStepState.FAILED,
TransactionStepState.DONE,
TransactionStepState.TIMEOUT,
TransactionStepState.SKIPPED,
],
[TransactionStepState.COMPENSATING]: [
TransactionStepState.REVERTED,
TransactionStepState.FAILED,
],
[TransactionStepState.DONE]: [TransactionStepState.COMPENSATING],
}
// Define allowed status transitions
const allowedStatusTransitions = {
[TransactionStepStatus.WAITING]: [
TransactionStepStatus.OK,
TransactionStepStatus.TEMPORARY_FAILURE,
TransactionStepStatus.PERMANENT_FAILURE,
],
[TransactionStepStatus.TEMPORARY_FAILURE]: [
TransactionStepStatus.IDLE,
TransactionStepStatus.PERMANENT_FAILURE,
],
[TransactionStepStatus.PERMANENT_FAILURE]: [TransactionStepStatus.IDLE],
}
if (
currentStepState.state === storedStepState.state &&
currentStepState.status === storedStepState.status
@@ -311,7 +320,9 @@ export class TransactionCheckpoint {
// Check if state transition from stored to current is allowed
const allowedStatesFromCurrent =
allowedStateTransitions[currentStepState.state] || []
TransactionCheckpoint.#ALLOWED_STATE_TRANSITIONS[
currentStepState.state
] || []
const isStateTransitionValid = allowedStatesFromCurrent.includes(
storedStepState.state
)
@@ -328,7 +339,9 @@ export class TransactionCheckpoint {
// Check if status transition from stored to current is allowed
const allowedStatusesFromCurrent =
allowedStatusTransitions[currentStepState.status] || []
TransactionCheckpoint.#ALLOWED_STATUS_TRANSITIONS[
currentStepState.status
] || []
return allowedStatusesFromCurrent.includes(storedStepState.status)
}
@@ -338,15 +351,13 @@ export class TransactionCheckpoint {
incomingErrors: TransactionStepError[]
): void {
const existingErrorSignatures = new Set(
currentErrors.map(
(err) => `${err.action}:${err.handlerType}:${err.error?.message}`
)
currentErrors.map(getErrorSignature)
)
for (const error of incomingErrors) {
const signature = `${error.action}:${error.handlerType}:${error.error?.message}`
if (!existingErrorSignatures.has(signature)) {
if (!existingErrorSignatures.has(getErrorSignature(error))) {
currentErrors.push(error)
existingErrorSignatures.add(getErrorSignature(error))
}
}
}
@@ -451,11 +462,7 @@ class DistributedTransaction extends EventEmitter {
}
public hasFinished(): boolean {
return [
TransactionState.DONE,
TransactionState.REVERTED,
TransactionState.FAILED,
].includes(this.getState())
return finishedStatesSet.has(this.getState())
}
public getState(): TransactionState {
@@ -520,7 +527,7 @@ class DistributedTransaction extends EventEmitter {
this.transactionId
)
let checkpoint
let checkpoint: TransactionCheckpoint
let retries = 0
let backoffMs = 50
@@ -553,7 +560,7 @@ class DistributedTransaction extends EventEmitter {
await setTimeoutPromise(backoffMs + jitter)
backoffMs = Math.min(backoffMs * 2, 1000)
backoffMs = Math.min(backoffMs * 2, 500)
const lastCheckpoint = await DistributedTransaction.loadTransaction(
this.modelId,
@@ -697,65 +704,45 @@ class DistributedTransaction extends EventEmitter {
* @returns
*/
#serializeCheckpointData() {
const data = new TransactionCheckpoint(
this.getFlow(),
this.getContext(),
this.getErrors()
)
const isSerializable = (obj) => {
try {
JSON.parse(JSON.stringify(obj))
return true
} catch {
return false
}
}
let rawData
try {
rawData = JSON.parse(JSON.stringify(data))
} catch (e) {
if (!isSerializable(this.context)) {
// This is a safe guard, we should never reach this point
// If we do, it means that the context is not serializable
// and we should throw an error
throw new NonSerializableCheckPointError(
"Unable to serialize context object. Please make sure the workflow input and steps response are serializable."
)
}
if (!isSerializable(this.errors)) {
const nonSerializableErrors: TransactionStepError[] = []
for (const error of this.errors) {
if (!isSerializable(error.error)) {
error.error = {
name: error.error.name,
message: error.error.message,
stack: error.error.stack,
}
nonSerializableErrors.push({
...error,
error: e,
})
}
}
if (nonSerializableErrors.length) {
this.errors.push(...nonSerializableErrors)
}
}
const data = new TransactionCheckpoint(
this.getFlow(),
this.getContext(),
this.getErrors()
JSON.stringify(this.context)
} catch {
throw new NonSerializableCheckPointError(
"Unable to serialize context object. Please make sure the workflow input and steps response are serializable."
)
rawData = JSON.parse(JSON.stringify(data))
}
return rawData
let errorsToUse = this.getErrors()
try {
JSON.stringify(errorsToUse)
} catch {
// Sanitize non-serializable errors
const sanitizedErrors: TransactionStepError[] = []
for (const error of this.errors) {
try {
JSON.stringify(error)
sanitizedErrors.push(error)
} catch {
sanitizedErrors.push({
action: error.action,
handlerType: error.handlerType,
error: {
name: error.error?.name || "Error",
message: error.error?.message || String(error.error),
stack: error.error?.stack,
},
})
}
}
errorsToUse = sanitizedErrors
this.errors = sanitizedErrors
}
return new TransactionCheckpoint(
JSON.parse(JSON.stringify(this.getFlow())),
this.getContext(),
[...errorsToUse]
)
}
}
packages/core/orchestration/src/transaction/transaction-orchestrator.ts
+60 -62
View File
@@ -40,6 +40,33 @@ import {
TransactionTimeoutError,
} from "./errors"
const canMoveForwardStates = new Set([
TransactionStepState.DONE,
TransactionStepState.FAILED,
TransactionStepState.TIMEOUT,
TransactionStepState.SKIPPED,
TransactionStepState.SKIPPED_FAILURE,
])
const canMoveBackwardStates = new Set([
TransactionStepState.DONE,
TransactionStepState.REVERTED,
TransactionStepState.FAILED,
TransactionStepState.DORMANT,
TransactionStepState.SKIPPED,
])
const flagStepsToRevertStates = new Set([
TransactionStepState.DONE,
TransactionStepState.TIMEOUT,
])
const setStepTimeoutSkipStates = new Set([
TransactionStepState.TIMEOUT,
TransactionStepState.DONE,
TransactionStepState.REVERTED,
])
/**
* @class TransactionOrchestrator is responsible for managing and executing distributed transactions.
* It is based on a single transaction definition, which is used to execute all the transaction steps
@@ -184,14 +211,6 @@ export class TransactionOrchestrator extends EventEmitter {
}
private canMoveForward(flow: TransactionFlow, previousStep: TransactionStep) {
const states = [
TransactionStepState.DONE,
TransactionStepState.FAILED,
TransactionStepState.TIMEOUT,
TransactionStepState.SKIPPED,
TransactionStepState.SKIPPED_FAILURE,
]
const siblings = TransactionOrchestrator.getPreviousStep(
flow,
previousStep
@@ -199,23 +218,15 @@ export class TransactionOrchestrator extends EventEmitter {
return (
!!previousStep.definition.noWait ||
siblings.every((sib) => states.includes(sib.invoke.state))
siblings.every((sib) => canMoveForwardStates.has(sib.invoke.state))
)
}
private canMoveBackward(flow: TransactionFlow, step: TransactionStep) {
const states = [
TransactionStepState.DONE,
TransactionStepState.REVERTED,
TransactionStepState.FAILED,
TransactionStepState.DORMANT,
TransactionStepState.SKIPPED,
]
const siblings = step.next.map((sib) => flow.steps[sib])
return (
siblings.length === 0 ||
siblings.every((sib) => states.includes(sib.compensate.state))
siblings.every((sib) => canMoveBackwardStates.has(sib.compensate.state))
)
}
@@ -521,9 +532,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
if (
[TransactionStepState.DONE, TransactionStepState.TIMEOUT].includes(
curState.state
) ||
flagStepsToRevertStates.has(curState.state) ||
curState.status === TransactionStepStatus.PERMANENT_FAILURE
) {
stepDef.beginCompensation()
@@ -707,13 +716,7 @@ export class TransactionOrchestrator extends EventEmitter {
step: TransactionStep,
error: TransactionStepTimeoutError | TransactionTimeoutError
): Promise<void> {
if (
[
TransactionStepState.TIMEOUT,
TransactionStepState.DONE,
TransactionStepState.REVERTED,
].includes(step.getStates().state)
) {
if (setStepTimeoutSkipStates.has(step.getStates().state)) {
return
}
@@ -773,13 +776,10 @@ export class TransactionOrchestrator extends EventEmitter {
error = serializeError(error)
} else {
try {
if (error?.message) {
error = JSON.parse(JSON.stringify(error))
} else {
error = {
message: JSON.stringify(error),
}
}
const serialized = JSON.stringify(error)
error = error?.message
? JSON.parse(serialized)
: { message: serialized }
} catch (e) {
error = {
message: "Unknown non-serializable error",
@@ -953,6 +953,18 @@ export class TransactionOrchestrator extends EventEmitter {
return shouldContinueExecution
})
let asyncStepCount = 0
for (const s of nextSteps.next) {
const stepIsAsync = s.isCompensating()
? s.definition.compensateAsync
: s.definition.async
if (stepIsAsync) asyncStepCount++
}
const hasMultipleAsyncSteps = asyncStepCount > 1
const hasAsyncSteps = !!asyncStepCount
// If there is any async step, we don't need to save the checkpoint here as it will be saved
// later down there
await transaction.saveCheckpoint().catch((error) => {
if (TransactionOrchestrator.isExpectedError(error)) {
continueExecution = false
@@ -962,11 +974,15 @@ export class TransactionOrchestrator extends EventEmitter {
throw error
})
if (!continueExecution) {
break
}
const execution: Promise<void | unknown>[] = []
const executionAsync: (() => Promise<void | unknown>)[] = []
let i = 0
let hasAsyncSteps = false
for (const step of nextSteps.next) {
const stepIndex = i++
if (!stepsShouldContinueExecution[stepIndex]) {
@@ -988,21 +1004,9 @@ export class TransactionOrchestrator extends EventEmitter {
// Compute current transaction state
await this.computeCurrentTransactionState(transaction)
if (!continueExecution) {
break
}
const promise = this.createStepExecutionPromise(transaction, step)
const hasMultipleAsyncSteps =
nextSteps.next.filter((step) => {
const isAsync = step.isCompensating()
? step.definition.compensateAsync
: step.definition.async
return isAsync
}).length > 1
const hasVersionControl =
hasMultipleAsyncSteps || step.hasAwaitingRetry()
@@ -1017,7 +1021,6 @@ export class TransactionOrchestrator extends EventEmitter {
)
} else {
// Execute async step in background as part of the next event loop cycle and continue the execution of the transaction
hasAsyncSteps = true
executionAsync.push(() =>
this.executeAsyncStep(promise, transaction, step, nextSteps)
)
@@ -1105,12 +1108,9 @@ export class TransactionOrchestrator extends EventEmitter {
private createStepPayload(
transaction: DistributedTransactionType,
step: TransactionStep,
flow: TransactionFlow
flow: TransactionFlow,
type: TransactionHandlerType
): TransactionPayload {
const type = step.isCompensating()
? TransactionHandlerType.COMPENSATE
: TransactionHandlerType.INVOKE
return new TransactionPayload(
{
model_id: flow.modelId,
@@ -1136,13 +1136,9 @@ export class TransactionOrchestrator extends EventEmitter {
private prepareHandlerArgs(
transaction: DistributedTransactionType,
step: TransactionStep,
flow: TransactionFlow,
payload: TransactionPayload
payload: TransactionPayload,
type: TransactionHandlerType
): Parameters<TransactionStepHandler> {
const type = step.isCompensating()
? TransactionHandlerType.COMPENSATE
: TransactionHandlerType.INVOKE
return [
step.definition.action + "",
type,
@@ -1164,11 +1160,13 @@ export class TransactionOrchestrator extends EventEmitter {
? TransactionHandlerType.COMPENSATE
: TransactionHandlerType.INVOKE
const flow = transaction.getFlow()
const payload = this.createStepPayload(transaction, step, flow, type)
const handlerArgs = this.prepareHandlerArgs(
transaction,
step,
transaction.getFlow(),
this.createStepPayload(transaction, step, transaction.getFlow())
payload,
type
)
const traceData = {