asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore(orchestration): idempotent (#7771)

Browse Source
This commit is contained in:
Carlos R. L. Rodrigues
2024-06-25 10:34:00 -03:00
committed by GitHub
parent 66d17fabde
commit 5600e58b7f
19 changed files with 421 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/core/orchestration/src/__tests__/transaction/transaction-orchestrator.ts
+39 -39
View File
@@ -498,10 +498,10 @@ describe("Transaction Orchestrator", () => {
await strategy.resume(transaction)
expect(transaction.transactionId).toBe("transaction_id_123")
expect(mocks.one).toBeCalledTimes(1)
expect(mocks.two).toBeCalledTimes(4)
expect(mocks.one).toHaveBeenCalledTimes(1)
expect(mocks.two).toHaveBeenCalledTimes(4)
expect(transaction.getState()).toBe(TransactionState.REVERTED)
expect(mocks.compensateOne).toBeCalledTimes(1)
expect(mocks.compensateOne).toHaveBeenCalledTimes(1)
expect(mocks.two).nthCalledWith(
1,
@@ -561,7 +561,7 @@ describe("Transaction Orchestrator", () => {
await strategy.resume(transaction)
expect(mocks.one).toBeCalledTimes(2)
expect(mocks.one).toHaveBeenCalledTimes(2)
expect(transaction.getState()).toBe(TransactionState.FAILED)
})
@@ -617,8 +617,8 @@ describe("Transaction Orchestrator", () => {
await strategy.resume(transaction)
expect(transaction.transactionId).toBe("transaction_id_123")
expect(mocks.one).toBeCalledTimes(1)
expect(mocks.two).toBeCalledTimes(2)
expect(mocks.one).toHaveBeenCalledTimes(1)
expect(mocks.two).toHaveBeenCalledTimes(2)
expect(transaction.getState()).toBe(TransactionState.DONE)
expect(transaction.isPartiallyCompleted).toBe(true)
})
@@ -677,8 +677,8 @@ describe("Transaction Orchestrator", () => {
await strategy.resume(transaction)
expect(mocks.one).toBeCalledTimes(1)
expect(mocks.two).toBeCalledTimes(0)
expect(mocks.one).toHaveBeenCalledTimes(1)
expect(mocks.two).toHaveBeenCalledTimes(0)
expect(transaction.getState()).toBe(TransactionState.INVOKING)
expect(transaction.getFlow().hasWaitingSteps).toBe(true)
@@ -771,9 +771,9 @@ describe("Transaction Orchestrator", () => {
await strategy.resume(transaction)
expect(mocks.one).toBeCalledTimes(1)
expect(mocks.compensateOne).toBeCalledTimes(0)
expect(mocks.two).toBeCalledTimes(0)
expect(mocks.one).toHaveBeenCalledTimes(1)
expect(mocks.compensateOne).toHaveBeenCalledTimes(0)
expect(mocks.two).toHaveBeenCalledTimes(0)
const registerBeforeAllowed = await strategy
.registerStepSuccess(mockSecondStepId, handler)
@@ -791,7 +791,7 @@ describe("Transaction Orchestrator", () => {
)
expect(resumedTransaction.getState()).toBe(TransactionState.COMPENSATING)
expect(mocks.compensateOne).toBeCalledTimes(1)
expect(mocks.compensateOne).toHaveBeenCalledTimes(1)
const mocktransactionIdCompensate = TransactionOrchestrator.getKeyName(
"transaction-name",
@@ -881,11 +881,11 @@ describe("Transaction Orchestrator", () => {
await strategy.resume(transaction)
expect(mocks.one).toBeCalledTimes(1)
expect(mocks.compensateOne).toBeCalledTimes(1)
expect(mocks.two).toBeCalledTimes(1)
expect(mocks.compensateTwo).toBeCalledTimes(1)
expect(mocks.three).toBeCalledTimes(1)
expect(mocks.one).toHaveBeenCalledTimes(1)
expect(mocks.compensateOne).toHaveBeenCalledTimes(1)
expect(mocks.two).toHaveBeenCalledTimes(1)
expect(mocks.compensateTwo).toHaveBeenCalledTimes(1)
expect(mocks.three).toHaveBeenCalledTimes(1)
expect(transaction.getState()).toBe(TransactionState.REVERTED)
})
@@ -951,16 +951,16 @@ describe("Transaction Orchestrator", () => {
await strategy.resume(transaction)
expect(transaction.getState()).toBe(TransactionState.DONE)
expect(mocks.one).toBeCalledTimes(1)
expect(mocks.two).toBeCalledTimes(1)
expect(mocks.one).toHaveBeenCalledTimes(1)
expect(mocks.two).toHaveBeenCalledTimes(1)
await strategy.cancelTransaction(transaction)
expect(transaction.getState()).toBe(TransactionState.REVERTED)
expect(mocks.one).toBeCalledTimes(1)
expect(mocks.two).toBeCalledTimes(1)
expect(mocks.oneCompensate).toBeCalledTimes(1)
expect(mocks.twoCompensate).toBeCalledTimes(1)
expect(mocks.one).toHaveBeenCalledTimes(1)
expect(mocks.two).toHaveBeenCalledTimes(1)
expect(mocks.oneCompensate).toHaveBeenCalledTimes(1)
expect(mocks.twoCompensate).toHaveBeenCalledTimes(1)
})
it("Should receive the current transaction as reference in the handler", async () => {
@@ -1081,10 +1081,10 @@ describe("Transaction Orchestrator", () => {
await strategy.resume(transaction)
expect(transaction.transactionId).toBe("transaction_id_123")
expect(mocks.f1).toBeCalledTimes(2)
expect(mocks.f2).toBeCalledTimes(2)
expect(mocks.f3).toBeCalledTimes(2)
expect(mocks.f4).toBeCalledTimes(0)
expect(mocks.f1).toHaveBeenCalledTimes(2)
expect(mocks.f2).toHaveBeenCalledTimes(2)
expect(mocks.f3).toHaveBeenCalledTimes(2)
expect(mocks.f4).toHaveBeenCalledTimes(0)
expect(transaction.getContext().invoke.action1).toBe("content f1")
expect(transaction.getContext().invoke.action2).toBe("delayed content f2")
expect(transaction.getContext().invoke.action3).toBe("content f3")
@@ -1187,10 +1187,10 @@ describe("Transaction Orchestrator", () => {
await strategy.resume(transaction)
expect(transaction.transactionId).toBe("transaction_id_123")
expect(mocks.f1).toBeCalledTimes(1)
expect(mocks.f2).toBeCalledTimes(1)
expect(mocks.f3).toBeCalledTimes(1)
expect(mocks.f4).toBeCalledTimes(0)
expect(mocks.f1).toHaveBeenCalledTimes(1)
expect(mocks.f2).toHaveBeenCalledTimes(1)
expect(mocks.f3).toHaveBeenCalledTimes(1)
expect(mocks.f4).toHaveBeenCalledTimes(0)
expect(transaction.getContext().invoke.action1).toBe("content f1")
expect(transaction.getContext().invoke.action2).toBe("delayed content f2")
expect(transaction.getContext().invoke.action3).toBe("content f3")
@@ -1304,10 +1304,10 @@ describe("Transaction Orchestrator", () => {
await strategy.resume(transaction)
expect(transaction.transactionId).toBe("transaction_id_123")
expect(mocks.f1).toBeCalledTimes(2)
expect(mocks.f2).toBeCalledTimes(2)
expect(mocks.f3).toBeCalledTimes(2)
expect(mocks.f4).toBeCalledTimes(0)
expect(mocks.f1).toHaveBeenCalledTimes(2)
expect(mocks.f2).toHaveBeenCalledTimes(2)
expect(mocks.f3).toHaveBeenCalledTimes(2)
expect(mocks.f4).toHaveBeenCalledTimes(0)
expect(transaction.getContext().invoke.action1).toBe("content f1")
expect(transaction.getContext().invoke.action2).toBe("delayed content f2")
expect(transaction.getContext().invoke.action3).toBe("content f3")
@@ -1413,10 +1413,10 @@ describe("Transaction Orchestrator", () => {
await strategy.resume(transaction)
expect(transaction.transactionId).toBe("transaction_id_123")
expect(mocks.f1).toBeCalledTimes(2)
expect(mocks.f2).toBeCalledTimes(2)
expect(mocks.f3).toBeCalledTimes(2)
expect(mocks.f4).toBeCalledTimes(0)
expect(mocks.f1).toHaveBeenCalledTimes(2)
expect(mocks.f2).toHaveBeenCalledTimes(2)
expect(mocks.f3).toHaveBeenCalledTimes(2)
expect(mocks.f4).toHaveBeenCalledTimes(0)
expect(transaction.getContext().invoke.action1).toBe("content f1")
expect(transaction.getContext().invoke.action2).toBe("delayed content f2")
expect(transaction.getContext().invoke.action3).toBe("content f3")
packages/core/orchestration/src/transaction/datastore/abstract-storage.ts
+11 -3
View File
@@ -3,7 +3,7 @@ import {
TransactionCheckpoint,
} from "../distributed-transaction"
import { TransactionStep } from "../transaction-step"
import { SchedulerOptions } from "../types"
import { SchedulerOptions, TransactionOptions } from "../types"
export interface IDistributedSchedulerStorage {
schedule(
@@ -17,9 +17,17 @@ export interface IDistributedSchedulerStorage {
}
export interface IDistributedTransactionStorage {
get(key: string): Promise<TransactionCheckpoint | undefined>
get(
key: string,
options?: TransactionOptions
): Promise<TransactionCheckpoint | undefined>
list(): Promise<TransactionCheckpoint[]>
save(key: string, data: TransactionCheckpoint, ttl?: number): Promise<void>
save(
key: string,
data: TransactionCheckpoint,
ttl?: number,
options?: TransactionOptions
): Promise<void>
scheduleRetry(
transaction: DistributedTransaction,
step: TransactionStep,
packages/core/orchestration/src/transaction/datastore/base-in-memory-storage.ts
+7 -2
View File
@@ -1,5 +1,6 @@
import { TransactionState } from "@medusajs/utils"
import { TransactionCheckpoint } from "../distributed-transaction"
import { TransactionOptions } from "../types"
import { DistributedTransactionStorage } from "./abstract-storage"
// eslint-disable-next-line max-len
@@ -11,7 +12,10 @@ export class BaseInMemoryDistributedTransactionStorage extends DistributedTransa
this.storage = new Map()
}
async get(key: string): Promise<TransactionCheckpoint | undefined> {
async get(
key: string,
options?: TransactionOptions
): Promise<TransactionCheckpoint | undefined> {
return this.storage.get(key)
}
@@ -22,7 +26,8 @@ export class BaseInMemoryDistributedTransactionStorage extends DistributedTransa
async save(
key: string,
data: TransactionCheckpoint,
ttl?: number
ttl?: number,
options?: TransactionOptions
): Promise<void> {
const hasFinished = [
TransactionState.DONE,
packages/core/orchestration/src/transaction/distributed-transaction.ts
+12 -9
View File
@@ -7,11 +7,7 @@ import {
TransactionOrchestrator,
} from "./transaction-orchestrator"
import { TransactionStep, TransactionStepHandler } from "./transaction-step"
import {
SchedulerOptions,
TransactionHandlerType,
TransactionState,
} from "./types"
import { TransactionHandlerType, TransactionState } from "./types"
/**
* @typedef TransactionMetadata
@@ -90,7 +86,7 @@ export class DistributedTransaction extends EventEmitter {
this.keyValueStore = storage
}
public static keyPrefix = "dtrans"
public static keyPrefix = "dtrx"
constructor(
private flow: TransactionFlow,
@@ -191,7 +187,10 @@ export class DistributedTransaction extends EventEmitter {
public async saveCheckpoint(
ttl = 0
): Promise<TransactionCheckpoint | undefined> {
const options = this.getFlow().options
const options =
TransactionOrchestrator.getWorkflowOptions(this.modelId) ??
this.getFlow().options
if (!options?.store) {
return
}
@@ -207,7 +206,7 @@ export class DistributedTransaction extends EventEmitter {
this.modelId,
this.transactionId
)
await DistributedTransaction.keyValueStore.save(key, data, ttl)
await DistributedTransaction.keyValueStore.save(key, data, ttl, options)
return data
}
@@ -222,7 +221,11 @@ export class DistributedTransaction extends EventEmitter {
transactionId
)
const loadedData = await DistributedTransaction.keyValueStore.get(key)
const options = TransactionOrchestrator.getWorkflowOptions(modelId)
const loadedData = await DistributedTransaction.keyValueStore.get(
key,
options
)
if (loadedData) {
return loadedData
}
packages/core/orchestration/src/transaction/transaction-orchestrator.ts
+55 -22
View File
@@ -6,8 +6,10 @@ import {
import { TransactionStep, TransactionStepHandler } from "./transaction-step"
import {
DistributedTransactionEvent,
StepFeatures,
TransactionHandlerType,
TransactionModelOptions,
TransactionOptions,
TransactionState,
TransactionStepsDefinition,
TransactionStepStatus,
@@ -56,12 +58,22 @@ export class TransactionOrchestrator extends EventEmitter {
private compensateSteps: string[] = []
public static DEFAULT_RETRIES = 0
private static workflowOptions: {
[modelId: string]: TransactionOptions
} = {}
public static getWorkflowOptions(modelId: string): TransactionOptions {
return this.workflowOptions[modelId]
}
constructor(
public id: string,
private definition: TransactionStepsDefinition,
private options?: TransactionModelOptions
) {
super()
this.parseFlowOptions()
}
private static SEPARATOR = ":"
@@ -409,6 +421,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
const flow = transaction.getFlow()
const options = TransactionOrchestrator.getWorkflowOptions(flow.modelId)
if (!hasStepTimedOut) {
step.changeStatus(TransactionStepStatus.OK)
@@ -420,7 +433,7 @@ export class TransactionOrchestrator extends EventEmitter {
step.changeState(TransactionStepState.DONE)
}
if (step.definition.async || flow.options?.storeExecution) {
if (step.definition.async || options?.storeExecution) {
await transaction.saveCheckpoint()
}
@@ -497,6 +510,8 @@ export class TransactionOrchestrator extends EventEmitter {
}
const flow = transaction.getFlow()
const options = TransactionOrchestrator.getWorkflowOptions(flow.modelId)
const cleaningUp: Promise<unknown>[] = []
const hasTimedOut = step.getStates().state === TransactionStepState.TIMEOUT
@@ -536,7 +551,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
}
if (step.definition.async || flow.options?.storeExecution) {
if (step.definition.async || options?.storeExecution) {
await transaction.saveCheckpoint()
}
@@ -563,6 +578,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
const flow = transaction.getFlow()
const options = TransactionOrchestrator.getWorkflowOptions(flow.modelId)
const nextSteps = await this.checkAllSteps(transaction)
const execution: Promise<void | unknown>[] = []
@@ -764,7 +780,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
}
if (hasSyncSteps && flow.options?.storeExecution) {
if (hasSyncSteps && options?.storeExecution) {
await transaction.saveCheckpoint()
}
@@ -798,7 +814,7 @@ export class TransactionOrchestrator extends EventEmitter {
flow.state = TransactionState.INVOKING
flow.startedAt = Date.now()
if (this.options?.store) {
if (this.getOptions().store) {
await transaction.saveCheckpoint(
flow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
)
@@ -843,7 +859,7 @@ export class TransactionOrchestrator extends EventEmitter {
await this.executeNext(transaction)
}
private createTransactionFlow(transactionId: string, flowMetadata?: TransactionFlow['metadata']): TransactionFlow {
private parseFlowOptions() {
const [steps, features] = TransactionOrchestrator.buildSteps(
this.definition
)
@@ -854,22 +870,47 @@ export class TransactionOrchestrator extends EventEmitter {
const hasStepTimeouts = features.hasStepTimeouts
const hasRetriesTimeout = features.hasRetriesTimeout
const hasTransactionTimeout = !!this.options.timeout
const isIdempotent = !!this.options.idempotent
if (hasAsyncSteps) {
this.options.store = true
}
if (hasStepTimeouts || hasRetriesTimeout || hasTransactionTimeout) {
if (
hasStepTimeouts ||
hasRetriesTimeout ||
hasTransactionTimeout ||
isIdempotent
) {
this.options.store = true
this.options.storeExecution = true
}
const parsedOptions = {
...this.options,
hasAsyncSteps,
hasStepTimeouts,
hasRetriesTimeout,
}
TransactionOrchestrator.workflowOptions[this.id] = parsedOptions
return [steps, features]
}
private createTransactionFlow(
transactionId: string,
flowMetadata?: TransactionFlow["metadata"]
): TransactionFlow {
const [steps, features] = TransactionOrchestrator.buildSteps(
this.definition
)
const flow: TransactionFlow = {
modelId: this.id,
options: this.options,
transactionId: transactionId,
metadata: flowMetadata,
hasAsyncSteps,
hasAsyncSteps: features.hasAsyncSteps,
hasFailedSteps: false,
hasSkippedSteps: false,
hasWaitingSteps: false,
@@ -909,14 +950,7 @@ export class TransactionOrchestrator extends EventEmitter {
private static buildSteps(
flow: TransactionStepsDefinition,
existingSteps?: { [key: string]: TransactionStep }
): [
{ [key: string]: TransactionStep },
{
hasAsyncSteps: boolean
hasStepTimeouts: boolean
hasRetriesTimeout: boolean
}
] {
): [{ [key: string]: TransactionStep }, StepFeatures] {
const states: { [key: string]: TransactionStep } = {
[TransactionOrchestrator.ROOT_STEP]: {
id: TransactionOrchestrator.ROOT_STEP,
@@ -938,12 +972,7 @@ export class TransactionOrchestrator extends EventEmitter {
while (queue.length > 0) {
const { obj, level } = queue.shift()
for (const key in obj) {
// eslint-disable-next-line no-prototype-builtins
if (!obj.hasOwnProperty(key)) {
continue
}
for (const key of Object.keys(obj)) {
if (typeof obj[key] === "object" && obj[key] !== null) {
queue.push({ obj: obj[key], level: [...level] })
} else if (key === "action") {
@@ -1039,7 +1068,11 @@ export class TransactionOrchestrator extends EventEmitter {
existingTransaction?.context
)
if (newTransaction && this.options?.store && this.options?.storeExecution) {
if (
newTransaction &&
this.getOptions().store &&
this.getOptions().storeExecution
) {
await transaction.saveCheckpoint(
modelFlow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
)
packages/core/orchestration/src/transaction/types.ts
+13
View File
@@ -118,6 +118,11 @@ export type TransactionModelOptions = {
*/
storeExecution?: boolean
/**
* If true, the workflow will use the transaction ID as the key to ensure only-once execution
*/
idempotent?: boolean
/**
* Defines the workflow as a scheduled workflow that executes based on the cron configuration passed.
* The value can either by a cron expression string, or an object that also allows to define the concurrency behavior.
@@ -207,3 +212,11 @@ export type DistributedTransactionEvents = {
transaction: DistributedTransaction
}) => void
}
export type StepFeatures = {
hasAsyncSteps: boolean
hasStepTimeouts: boolean
hasRetriesTimeout: boolean
}
export type TransactionOptions = TransactionModelOptions & StepFeatures