feat: Workflow engine modules (#6128)

This commit is contained in:
Carlos R. L. Rodrigues
2024-01-23 10:08:08 -03:00
committed by GitHub
parent d85fee42ee
commit 302323916b
119 changed files with 5339 additions and 263 deletions
+11
View File
@@ -0,0 +1,11 @@
---
"@medusajs/workflow-engine-inmemory": patch
"@medusajs/workflow-engine-redis": patch
"@medusajs/orchestration": patch
"@medusajs/workflows-sdk": patch
"@medusajs/modules-sdk": patch
"@medusajs/types": patch
"@medusajs/utils": patch
---
Modules: Workflows Engine in-memory and Redis
+3
View File
@@ -24,6 +24,9 @@ packages/*
!packages/orchestration
!packages/workflows-sdk
!packages/core-flows
!packages/workflow-engine-redis
!packages/workflow-engine-inmemory
**/models/*
+3 -3
View File
@@ -72,9 +72,7 @@ module.exports = {
node: true,
jest: true,
},
ignorePatterns: [
"packages/admin-next/dashboard/**/dist"
],
ignorePatterns: ["packages/admin-next/dashboard/**/dist"],
overrides: [
{
files: ["*.ts"],
@@ -101,6 +99,8 @@ module.exports = {
"./packages/orchestration/tsconfig.json",
"./packages/workflows-sdk/tsconfig.spec.json",
"./packages/core-flows/tsconfig.spec.json",
"./packages/workflow-engine-redis/tsconfig.spec.json",
"./packages/workflow-engine-inmemory/tsconfig.spec.json",
],
},
rules: {
+6 -6
View File
@@ -65,15 +65,15 @@
"scripts": {
"hooks:install": "husky install",
"hooks:uninstall": "husky uninstall",
"build": "turbo run build --no-daemon",
"build": "turbo run build --concurrency=50% --no-daemon",
"lint": "eslint --ignore-path .eslintignore --ext .js,.ts,.tsx .",
"prettier": "prettier",
"jest": "jest",
"test": "turbo run test --no-daemon",
"test:integration:packages": "turbo run test:integration --no-daemon --filter='./packages/*'",
"test:integration:api": "turbo run test:integration --no-daemon --filter=integration-tests-api",
"test:integration:plugins": "turbo run test:integration --no-daemon --filter=integration-tests-plugins",
"test:integration:repositories": "turbo run test:integration --no-daemon --filter=integration-tests-repositories",
"test": "turbo run test --concurrency=50% --no-daemon",
"test:integration:packages": "turbo run test:integration --concurrency=1 --no-daemon --filter='./packages/*'",
"test:integration:api": "turbo run test:integration --concurrency=50% --no-daemon --filter=integration-tests-api",
"test:integration:plugins": "turbo run test:integration --concurrency=50% --no-daemon --filter=integration-tests-plugins",
"test:integration:repositories": "turbo run test:integration --concurrency=50% --no-daemon --filter=integration-tests-repositories",
"openapi:generate": "yarn ./packages/oas/oas-github-ci run ci --with-full-file",
"medusa-oas": "yarn ./packages/oas/medusa-oas-cli run medusa-oas",
"release:snapshot": "changeset publish --no-git-tags --snapshot --tag snapshot",
@@ -1,5 +1,6 @@
import cors from "cors"
import { Router } from "express"
import { parseCorsOrigins } from "medusa-core-utils"
import middlewares from "../../middlewares"
import analyticsConfigs from "./analytics-configs"
import appRoutes from "./apps"
@@ -18,16 +19,18 @@ import noteRoutes from "./notes"
import notificationRoutes from "./notifications"
import orderEditRoutes from "./order-edits"
import orderRoutes from "./orders"
import paymentCollectionRoutes from "./payment-collections"
import paymentRoutes from "./payments"
import priceListRoutes from "./price-lists"
import productCategoryRoutes from "./product-categories"
import productTagRoutes from "./product-tags"
import productTypesRoutes from "./product-types"
import publishableApiKeyRoutes from "./publishable-api-keys"
import productRoutes from "./products"
import publishableApiKeyRoutes from "./publishable-api-keys"
import regionRoutes from "./regions"
import reservationRoutes from "./reservations"
import returnReasonRoutes from "./return-reasons"
import returnRoutes from "./returns"
import reservationRoutes from "./reservations"
import salesChannelRoutes from "./sales-channels"
import shippingOptionRoutes from "./shipping-options"
import shippingProfileRoutes from "./shipping-profiles"
@@ -38,10 +41,6 @@ import taxRateRoutes from "./tax-rates"
import uploadRoutes from "./uploads"
import userRoutes, { unauthenticatedUserRoutes } from "./users"
import variantRoutes from "./variants"
import paymentCollectionRoutes from "./payment-collections"
import paymentRoutes from "./payments"
import productCategoryRoutes from "./product-categories"
import { parseCorsOrigins } from "medusa-core-utils"
const route = Router()
@@ -3,6 +3,7 @@ import {
ModulesDefinition,
registerMedusaModule,
} from "@medusajs/modules-sdk"
import { ContainerRegistrationKeys } from "@medusajs/utils"
import { asValue, createContainer } from "awilix"
import express from "express"
import jwt from "jsonwebtoken"
@@ -63,6 +64,7 @@ export const createServer = async (rootDir) => {
return this
}.bind(container)
container.register(ContainerRegistrationKeys.PG_CONNECTION, asValue({}))
container.register("featureFlagRouter", asValue(featureFlagRouter))
container.register("configModule", asValue(config))
container.register({
Binary file not shown.
+17
View File
@@ -16,6 +16,7 @@ export enum Modules {
PRICING = "pricingService",
PROMOTION = "promotion",
AUTHENTICATION = "authentication",
WORKFLOW_ENGINE = "workflows",
CART = "cart",
CUSTOMER = "customer",
PAYMENT = "payment",
@@ -30,6 +31,7 @@ export enum ModuleRegistrationName {
PRICING = "pricingModuleService",
PROMOTION = "promotionModuleService",
AUTHENTICATION = "authenticationModuleService",
WORKFLOW_ENGINE = "workflowsModuleService",
CART = "cartModuleService",
CUSTOMER = "customerModuleService",
PAYMENT = "paymentModuleService",
@@ -45,6 +47,7 @@ export const MODULE_PACKAGE_NAMES = {
[Modules.PRICING]: "@medusajs/pricing",
[Modules.PROMOTION]: "@medusajs/promotion",
[Modules.AUTHENTICATION]: "@medusajs/authentication",
[Modules.WORKFLOW_ENGINE]: "@medusajs/workflow-engine-inmemory",
[Modules.CART]: "@medusajs/cart",
[Modules.CUSTOMER]: "@medusajs/customer",
[Modules.PAYMENT]: "@medusajs/payment",
@@ -165,6 +168,20 @@ export const ModulesDefinition: { [key: string | Modules]: ModuleDefinition } =
resources: MODULE_RESOURCE_TYPE.SHARED,
},
},
[Modules.WORKFLOW_ENGINE]: {
key: Modules.WORKFLOW_ENGINE,
registrationName: ModuleRegistrationName.WORKFLOW_ENGINE,
defaultPackage: false,
label: upperCaseFirst(ModuleRegistrationName.WORKFLOW_ENGINE),
isRequired: false,
canOverride: true,
isQueryable: true,
dependencies: ["logger"],
defaultModuleDeclaration: {
scope: MODULE_SCOPE.INTERNAL,
resources: MODULE_RESOURCE_TYPE.SHARED,
},
},
[Modules.CART]: {
key: Modules.CART,
registrationName: ModuleRegistrationName.CART,
+6
View File
@@ -245,6 +245,12 @@ export async function MedusaApp({
registerCustomJoinerConfigs(servicesConfig ?? [])
if (
sharedResourcesConfig?.database?.connection &&
!injectedDependencies[ContainerRegistrationKeys.PG_CONNECTION]
) {
injectedDependencies[ContainerRegistrationKeys.PG_CONNECTION] =
sharedResourcesConfig.database.connection
} else if (
dbData.clientUrl &&
!injectedDependencies[ContainerRegistrationKeys.PG_CONNECTION]
) {
@@ -1,10 +1,14 @@
import { TransactionStepState, TransactionStepStatus } from "@medusajs/utils"
import { setTimeout } from "timers/promises"
import {
DistributedTransaction,
TransactionHandlerType,
TransactionOrchestrator,
TransactionPayload,
TransactionState,
TransactionStepTimeoutError,
TransactionStepsDefinition,
TransactionTimeoutError,
} from "../../transaction"
describe("Transaction Orchestrator", () => {
@@ -986,4 +990,454 @@ describe("Transaction Orchestrator", () => {
expect(transaction).toBe(transactionInHandler)
})
describe("Timeouts - Transaction and Step", () => {
it("should fail the current steps and revert the transaction if the Transaction Timeout is reached", async () => {
const mocks = {
f1: jest.fn(() => {
return "content f1"
}),
f2: jest.fn(async () => {
await setTimeout(200)
return "delayed content f2"
}),
f3: jest.fn(() => {
return "content f3"
}),
f4: jest.fn(() => {
return "content f4"
}),
}
async function handler(
actionId: string,
functionHandlerType: TransactionHandlerType,
payload: TransactionPayload
) {
const command = {
action1: {
[TransactionHandlerType.INVOKE]: () => {
return mocks.f1()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f1()
},
},
action2: {
[TransactionHandlerType.INVOKE]: async () => {
return await mocks.f2()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f2()
},
},
action3: {
[TransactionHandlerType.INVOKE]: () => {
return mocks.f3()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f3()
},
},
action4: {
[TransactionHandlerType.INVOKE]: () => {
return mocks.f4()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f4()
},
},
}
return command[actionId][functionHandlerType]()
}
const flow: TransactionStepsDefinition = {
next: {
action: "action1",
next: [
{
action: "action2",
},
{
action: "action3",
next: {
action: "action4",
},
},
],
},
}
const strategy = new TransactionOrchestrator("transaction-name", flow, {
timeout: 0.1, // 100ms
})
const transaction = await strategy.beginTransaction(
"transaction_id_123",
handler
)
await strategy.resume(transaction)
expect(transaction.transactionId).toBe("transaction_id_123")
expect(mocks.f1).toBeCalledTimes(2)
expect(mocks.f2).toBeCalledTimes(2)
expect(mocks.f3).toBeCalledTimes(2)
expect(mocks.f4).toBeCalledTimes(0)
expect(transaction.getContext().invoke.action1).toBe("content f1")
expect(transaction.getContext().invoke.action2).toBe("delayed content f2")
expect(transaction.getContext().invoke.action3).toBe("content f3")
expect(transaction.getContext().invoke.action4).toBe(undefined)
expect(transaction.getErrors()[0].error).toBeInstanceOf(
TransactionTimeoutError
)
expect(transaction.getErrors()[0].action).toBe("action2")
expect(transaction.getState()).toBe(TransactionState.REVERTED)
})
it("should continue the transaction and skip children steps when the Transaction Step Timeout is reached but the step is set to 'continueOnPermanentFailure'", async () => {
const mocks = {
f1: jest.fn(() => {
return "content f1"
}),
f2: jest.fn(async () => {
await setTimeout(200)
return "delayed content f2"
}),
f3: jest.fn(() => {
return "content f3"
}),
f4: jest.fn(() => {
return "content f4"
}),
}
async function handler(
actionId: string,
functionHandlerType: TransactionHandlerType,
payload: TransactionPayload
) {
const command = {
action1: {
[TransactionHandlerType.INVOKE]: () => {
return mocks.f1()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f1()
},
},
action2: {
[TransactionHandlerType.INVOKE]: async () => {
return await mocks.f2()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f2()
},
},
action3: {
[TransactionHandlerType.INVOKE]: () => {
return mocks.f3()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f3()
},
},
action4: {
[TransactionHandlerType.INVOKE]: () => {
return mocks.f4()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f4()
},
},
}
return command[actionId][functionHandlerType]()
}
const flow: TransactionStepsDefinition = {
next: {
action: "action1",
next: [
{
timeout: 0.1, // 100ms
action: "action2",
continueOnPermanentFailure: true,
next: {
action: "action4",
},
},
{
action: "action3",
},
],
},
}
const strategy = new TransactionOrchestrator("transaction-name", flow)
const transaction = await strategy.beginTransaction(
"transaction_id_123",
handler
)
await strategy.resume(transaction)
expect(transaction.transactionId).toBe("transaction_id_123")
expect(mocks.f1).toBeCalledTimes(1)
expect(mocks.f2).toBeCalledTimes(1)
expect(mocks.f3).toBeCalledTimes(1)
expect(mocks.f4).toBeCalledTimes(0)
expect(transaction.getContext().invoke.action1).toBe("content f1")
expect(transaction.getContext().invoke.action2).toBe("delayed content f2")
expect(transaction.getContext().invoke.action3).toBe("content f3")
expect(transaction.getContext().invoke.action4).toBe(undefined)
expect(
transaction.getFlow().steps["_root.action1.action2"].invoke.state
).toBe(TransactionStepState.TIMEOUT)
expect(
transaction.getFlow().steps["_root.action1.action2"].invoke.status
).toBe(TransactionStepStatus.PERMANENT_FAILURE)
expect(
transaction.getFlow().steps["_root.action1.action2"].compensate.state
).toBe(TransactionStepState.DORMANT)
expect(
transaction.getFlow().steps["_root.action1.action2.action4"].invoke
.state
).toBe(TransactionStepState.SKIPPED)
expect(
transaction.getFlow().steps["_root.action1.action2.action4"].invoke
.status
).toBe(TransactionStepStatus.IDLE)
expect(transaction.getState()).toBe(TransactionState.DONE)
})
it("should fail the current steps and revert the transaction if the Step Timeout is reached", async () => {
const mocks = {
f1: jest.fn(() => {
return "content f1"
}),
f2: jest.fn(async () => {
await setTimeout(200)
return "delayed content f2"
}),
f3: jest.fn(() => {
return "content f3"
}),
f4: jest.fn(() => {
return "content f4"
}),
}
async function handler(
actionId: string,
functionHandlerType: TransactionHandlerType,
payload: TransactionPayload
) {
const command = {
action1: {
[TransactionHandlerType.INVOKE]: () => {
return mocks.f1()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f1()
},
},
action2: {
[TransactionHandlerType.INVOKE]: async () => {
return await mocks.f2()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f2()
},
},
action3: {
[TransactionHandlerType.INVOKE]: () => {
return mocks.f3()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f3()
},
},
action4: {
[TransactionHandlerType.INVOKE]: () => {
return mocks.f4()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f4()
},
},
}
return command[actionId][functionHandlerType]()
}
const flow: TransactionStepsDefinition = {
next: {
action: "action1",
next: [
{
action: "action2",
timeout: 0.1, // 100ms
},
{
action: "action3",
next: {
action: "action4",
},
},
],
},
}
const strategy = new TransactionOrchestrator("transaction-name", flow)
const transaction = await strategy.beginTransaction(
"transaction_id_123",
handler
)
await strategy.resume(transaction)
expect(transaction.transactionId).toBe("transaction_id_123")
expect(mocks.f1).toBeCalledTimes(2)
expect(mocks.f2).toBeCalledTimes(2)
expect(mocks.f3).toBeCalledTimes(2)
expect(mocks.f4).toBeCalledTimes(0)
expect(transaction.getContext().invoke.action1).toBe("content f1")
expect(transaction.getContext().invoke.action2).toBe("delayed content f2")
expect(transaction.getContext().invoke.action3).toBe("content f3")
expect(transaction.getContext().invoke.action4).toBe(undefined)
expect(transaction.getErrors()[0].error).toBeInstanceOf(
TransactionStepTimeoutError
)
expect(transaction.getErrors()[0].action).toBe("action2")
expect(transaction.getState()).toBe(TransactionState.REVERTED)
})
it("should fail the current steps and revert the transaction if the Transaction Timeout is reached event if the step is set as 'continueOnPermanentFailure'", async () => {
const mocks = {
f1: jest.fn(() => {
return "content f1"
}),
f2: jest.fn(async () => {
await setTimeout(200)
return "delayed content f2"
}),
f3: jest.fn(async () => {
await setTimeout(200)
return "content f3"
}),
f4: jest.fn(() => {
return "content f4"
}),
}
async function handler(
actionId: string,
functionHandlerType: TransactionHandlerType,
payload: TransactionPayload
) {
const command = {
action1: {
[TransactionHandlerType.INVOKE]: () => {
return mocks.f1()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f1()
},
},
action2: {
[TransactionHandlerType.INVOKE]: async () => {
return await mocks.f2()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f2()
},
},
action3: {
[TransactionHandlerType.INVOKE]: async () => {
return await mocks.f3()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f3()
},
},
action4: {
[TransactionHandlerType.INVOKE]: () => {
return mocks.f4()
},
[TransactionHandlerType.COMPENSATE]: () => {
return mocks.f4()
},
},
}
return command[actionId][functionHandlerType]()
}
const flow: TransactionStepsDefinition = {
next: {
action: "action1",
next: [
{
action: "action2",
continueOnPermanentFailure: true,
},
{
action: "action3",
continueOnPermanentFailure: true,
next: {
action: "action4",
},
},
],
},
}
const strategy = new TransactionOrchestrator("transaction-name", flow, {
timeout: 0.1, // 100ms
})
const transaction = await strategy.beginTransaction(
"transaction_id_123",
handler
)
await strategy.resume(transaction)
expect(transaction.transactionId).toBe("transaction_id_123")
expect(mocks.f1).toBeCalledTimes(2)
expect(mocks.f2).toBeCalledTimes(2)
expect(mocks.f3).toBeCalledTimes(2)
expect(mocks.f4).toBeCalledTimes(0)
expect(transaction.getContext().invoke.action1).toBe("content f1")
expect(transaction.getContext().invoke.action2).toBe("delayed content f2")
expect(transaction.getContext().invoke.action3).toBe("content f3")
expect(transaction.getContext().invoke.action4).toBe(undefined)
expect(transaction.getErrors()).toHaveLength(2)
expect(
TransactionTimeoutError.isTransactionTimeoutError(
transaction.getErrors()[0].error
)
).toBe(true)
expect(transaction.getErrors()[0].action).toBe("action2")
expect(
TransactionTimeoutError.isTransactionTimeoutError(
transaction.getErrors()[1].error
)
).toBe(true)
expect(transaction.getErrors()[1].action).toBe("action3")
expect(transaction.getState()).toBe(TransactionState.REVERTED)
})
})
})
@@ -3,14 +3,11 @@ import {
TransactionCheckpoint,
} from "../distributed-transaction"
import { TransactionStep } from "../transaction-step"
import { TransactionModelOptions } from "../types"
export interface IDistributedTransactionStorage {
get(key: string): Promise<TransactionCheckpoint | undefined>
list(): Promise<TransactionCheckpoint[]>
save(key: string, data: TransactionCheckpoint, ttl?: number): Promise<void>
delete(key: string): Promise<void>
archive(key: string, options?: TransactionModelOptions): Promise<void>
scheduleRetry(
transaction: DistributedTransaction,
step: TransactionStep,
@@ -62,14 +59,6 @@ export abstract class DistributedTransactionStorage
throw new Error("Method 'save' not implemented.")
}
async delete(key: string): Promise<void> {
throw new Error("Method 'delete' not implemented.")
}
async archive(key: string, options?: TransactionModelOptions): Promise<void> {
throw new Error("Method 'archive' not implemented.")
}
async scheduleRetry(
transaction: DistributedTransaction,
step: TransactionStep,
@@ -1,5 +1,5 @@
import { TransactionState } from "@medusajs/utils"
import { TransactionCheckpoint } from "../distributed-transaction"
import { TransactionModelOptions } from "../types"
import { DistributedTransactionStorage } from "./abstract-storage"
// eslint-disable-next-line max-len
@@ -24,14 +24,16 @@ export class BaseInMemoryDistributedTransactionStorage extends DistributedTransa
data: TransactionCheckpoint,
ttl?: number
): Promise<void> {
this.storage.set(key, data)
}
const hasFinished = [
TransactionState.DONE,
TransactionState.REVERTED,
TransactionState.FAILED,
].includes(data.flow.state)
async delete(key: string): Promise<void> {
this.storage.delete(key)
}
async archive(key: string, options?: TransactionModelOptions): Promise<void> {
this.storage.delete(key)
if (hasFinished) {
this.storage.delete(key)
} else {
this.storage.set(key, data)
}
}
}
@@ -86,7 +86,7 @@ export class DistributedTransaction extends EventEmitter {
this.keyValueStore = storage
}
private static keyPrefix = "dtrans"
public static keyPrefix = "dtrans"
constructor(
private flow: TransactionFlow,
@@ -177,18 +177,18 @@ export class DistributedTransaction extends EventEmitter {
}
public hasTimeout(): boolean {
return !!this.getFlow().definition.timeout
return !!this.getTimeout()
}
public getTimeoutInterval(): number | undefined {
return this.getFlow().definition.timeout
public getTimeout(): number | undefined {
return this.getFlow().options?.timeout
}
public async saveCheckpoint(
ttl = 0
): Promise<TransactionCheckpoint | undefined> {
const options = this.getFlow().options
if (!options?.storeExecution) {
if (!options?.store) {
return
}
@@ -226,31 +226,6 @@ export class DistributedTransaction extends EventEmitter {
return null
}
public async deleteCheckpoint(): Promise<void> {
const options = this.getFlow().options
if (!options?.storeExecution) {
return
}
const key = TransactionOrchestrator.getKeyName(
DistributedTransaction.keyPrefix,
this.modelId,
this.transactionId
)
await DistributedTransaction.keyValueStore.delete(key)
}
public async archiveCheckpoint(): Promise<void> {
const options = this.getFlow().options
const key = TransactionOrchestrator.getKeyName(
DistributedTransaction.keyPrefix,
this.modelId,
this.transactionId
)
await DistributedTransaction.keyValueStore.archive(key, options)
}
public async scheduleRetry(
step: TransactionStep,
interval: number
@@ -269,6 +244,11 @@ export class DistributedTransaction extends EventEmitter {
}
public async scheduleTransactionTimeout(interval: number): Promise<void> {
// schedule transaction timeout only if there are async steps
if (!this.getFlow().hasAsyncSteps) {
return
}
await this.saveCheckpoint()
await DistributedTransaction.keyValueStore.scheduleTransactionTimeout(
this,
@@ -278,6 +258,10 @@ export class DistributedTransaction extends EventEmitter {
}
public async clearTransactionTimeout(): Promise<void> {
if (!this.getFlow().hasAsyncSteps) {
return
}
await DistributedTransaction.keyValueStore.clearTransactionTimeout(this)
}
@@ -285,6 +269,11 @@ export class DistributedTransaction extends EventEmitter {
step: TransactionStep,
interval: number
): Promise<void> {
// schedule step timeout only if the step is async
if (!step.definition.async) {
return
}
await this.saveCheckpoint()
await DistributedTransaction.keyValueStore.scheduleStepTimeout(
this,
@@ -295,6 +284,10 @@ export class DistributedTransaction extends EventEmitter {
}
public async clearStepTimeout(step: TransactionStep): Promise<void> {
if (!step.definition.async || step.isCompensating()) {
return
}
await DistributedTransaction.keyValueStore.clearStepTimeout(this, step)
}
}
@@ -4,7 +4,7 @@ export class PermanentStepFailureError extends Error {
): error is PermanentStepFailureError {
return (
error instanceof PermanentStepFailureError ||
error.name === "PermanentStepFailure"
error?.name === "PermanentStepFailure"
)
}
@@ -14,16 +14,19 @@ export class PermanentStepFailureError extends Error {
}
}
export class StepTimeoutError extends Error {
static isStepTimeoutError(error: Error): error is StepTimeoutError {
export class TransactionStepTimeoutError extends Error {
static isTransactionStepTimeoutError(
error: Error
): error is TransactionStepTimeoutError {
return (
error instanceof StepTimeoutError || error.name === "StepTimeoutError"
error instanceof TransactionStepTimeoutError ||
error?.name === "TransactionStepTimeoutError"
)
}
constructor(message?: string) {
super(message)
this.name = "StepTimeoutError"
this.name = "TransactionStepTimeoutError"
}
}
@@ -33,7 +36,7 @@ export class TransactionTimeoutError extends Error {
): error is TransactionTimeoutError {
return (
error instanceof TransactionTimeoutError ||
error.name === "TransactionTimeoutError"
error?.name === "TransactionTimeoutError"
)
}
@@ -314,7 +314,7 @@ export class OrchestratorBuilder {
action: string,
step: InternalStep = this.steps
): InternalStep | undefined {
if (step.action === action) {
if (step.uuid === action || step.action === action) {
return step
}
@@ -357,7 +357,7 @@ export class OrchestratorBuilder {
if (!nextStep) {
continue
}
if (nextStep.action === action) {
if (nextStep.uuid === action || nextStep.action === action) {
return step
}
const foundStep = this.findParentStepByAction(
@@ -13,11 +13,11 @@ import {
TransactionStepStatus,
} from "./types"
import { MedusaError, promiseAll } from "@medusajs/utils"
import { MedusaError, promiseAll, TransactionStepState } from "@medusajs/utils"
import { EventEmitter } from "events"
import {
PermanentStepFailureError,
StepTimeoutError,
TransactionStepTimeoutError,
TransactionTimeoutError,
} from "./errors"
@@ -30,6 +30,7 @@ export type TransactionFlow = {
hasFailedSteps: boolean
hasWaitingSteps: boolean
hasSkippedSteps: boolean
hasRevertedSteps: boolean
timedOutAt: number | null
startedAt?: number
state: TransactionState
@@ -62,10 +63,6 @@ export class TransactionOrchestrator extends EventEmitter {
return params.join(this.SEPARATOR)
}
public getOptions(): TransactionModelOptions {
return this.options ?? {}
}
private getPreviousStep(flow: TransactionFlow, step: TransactionStep) {
const id = step.id.split(".")
id.pop()
@@ -73,6 +70,10 @@ export class TransactionOrchestrator extends EventEmitter {
return flow.steps[parentId]
}
public getOptions(): TransactionModelOptions {
return this.options ?? {}
}
private getInvokeSteps(flow: TransactionFlow): string[] {
if (this.invokeSteps.length) {
return this.invokeSteps
@@ -102,9 +103,10 @@ export class TransactionOrchestrator extends EventEmitter {
private canMoveForward(flow: TransactionFlow, previousStep: TransactionStep) {
const states = [
TransactionState.DONE,
TransactionState.FAILED,
TransactionState.SKIPPED,
TransactionStepState.DONE,
TransactionStepState.FAILED,
TransactionStepState.TIMEOUT,
TransactionStepState.SKIPPED,
]
const siblings = this.getPreviousStep(flow, previousStep).next.map(
@@ -119,10 +121,10 @@ export class TransactionOrchestrator extends EventEmitter {
private canMoveBackward(flow: TransactionFlow, step: TransactionStep) {
const states = [
TransactionState.DONE,
TransactionState.REVERTED,
TransactionState.FAILED,
TransactionState.DORMANT,
TransactionStepState.DONE,
TransactionStepState.REVERTED,
TransactionStepState.FAILED,
TransactionStepState.DORMANT,
]
const siblings = step.next.map((sib) => flow.steps[sib])
return (
@@ -144,29 +146,89 @@ export class TransactionOrchestrator extends EventEmitter {
}
}
private async checkStepTimeout(transaction, step) {
private hasExpired(
{
transaction,
step,
}: {
transaction?: DistributedTransaction
step?: TransactionStep
},
dateNow: number
): boolean {
const hasStepTimedOut =
step &&
step.hasTimeout() &&
!step.isCompensating() &&
dateNow > step.startedAt! + step.getTimeout()! * 1e3
const hasTransactionTimedOut =
transaction &&
transaction.hasTimeout() &&
transaction.getFlow().state !== TransactionState.COMPENSATING &&
dateNow >
transaction.getFlow().startedAt! + transaction.getTimeout()! * 1e3
return !!hasStepTimedOut || !!hasTransactionTimedOut
}
private async checkTransactionTimeout(
transaction: DistributedTransaction,
currentSteps: TransactionStep[]
) {
const flow = transaction.getFlow()
let hasTimedOut = false
if (!flow.timedOutAt && this.hasExpired({ transaction }, Date.now())) {
flow.timedOutAt = Date.now()
void transaction.clearTransactionTimeout()
for (const step of currentSteps) {
await TransactionOrchestrator.setStepTimeout(
transaction,
step,
new TransactionTimeoutError()
)
}
await transaction.saveCheckpoint()
this.emit(DistributedTransactionEvent.TIMEOUT, { transaction })
hasTimedOut = true
}
return hasTimedOut
}
private async checkStepTimeout(
transaction: DistributedTransaction,
step: TransactionStep
) {
let hasTimedOut = false
if (
step.hasTimeout() &&
!step.timedOutAt &&
step.canCancel() &&
step.startedAt! + step.getTimeoutInterval()! * 1e3 < Date.now()
this.hasExpired({ step }, Date.now())
) {
step.timedOutAt = Date.now()
await transaction.saveCheckpoint()
this.emit(DistributedTransactionEvent.TIMEOUT, { transaction })
await TransactionOrchestrator.setStepFailure(
await TransactionOrchestrator.setStepTimeout(
transaction,
step,
new StepTimeoutError(),
0
new TransactionStepTimeoutError()
)
hasTimedOut = true
await transaction.saveCheckpoint()
this.emit(DistributedTransactionEvent.TIMEOUT, { transaction })
}
return hasTimedOut
}
private async checkAllSteps(transaction: DistributedTransaction): Promise<{
current: TransactionStep[]
next: TransactionStep[]
total: number
remaining: number
@@ -182,6 +244,8 @@ export class TransactionOrchestrator extends EventEmitter {
const flow = transaction.getFlow()
const nextSteps: TransactionStep[] = []
const currentSteps: TransactionStep[] = []
const allSteps =
flow.state === TransactionState.COMPENSATING
? this.getCompensationSteps(flow)
@@ -204,6 +268,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
if (curState.status === TransactionStepStatus.WAITING) {
currentSteps.push(stepDef)
hasWaiting = true
if (stepDef.hasAwaitingRetry()) {
@@ -223,6 +288,8 @@ export class TransactionOrchestrator extends EventEmitter {
continue
} else if (curState.status === TransactionStepStatus.TEMPORARY_FAILURE) {
currentSteps.push(stepDef)
if (!stepDef.canRetry()) {
if (stepDef.hasRetryInterval() && !stepDef.retryRescheduledAt) {
stepDef.hasScheduledRetry = true
@@ -243,11 +310,11 @@ export class TransactionOrchestrator extends EventEmitter {
} else {
completedSteps++
if (curState.state === TransactionState.SKIPPED) {
if (curState.state === TransactionStepState.SKIPPED) {
hasSkipped = true
} else if (curState.state === TransactionState.REVERTED) {
} else if (curState.state === TransactionStepState.REVERTED) {
hasReverted = true
} else if (curState.state === TransactionState.FAILED) {
} else if (curState.state === TransactionStepState.FAILED) {
if (stepDef.definition.continueOnPermanentFailure) {
hasIgnoredFailure = true
} else {
@@ -258,6 +325,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
flow.hasWaitingSteps = hasWaiting
flow.hasRevertedSteps = hasReverted
const totalSteps = allSteps.length - 1
if (
@@ -288,6 +356,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
return {
current: currentSteps,
next: nextSteps,
total: totalSteps,
remaining: totalSteps - completedSteps,
@@ -304,11 +373,13 @@ export class TransactionOrchestrator extends EventEmitter {
const stepDef = flow.steps[step]
const curState = stepDef.getStates()
if (
curState.state === TransactionState.DONE ||
[TransactionStepState.DONE, TransactionStepState.TIMEOUT].includes(
curState.state
) ||
curState.status === TransactionStepStatus.PERMANENT_FAILURE
) {
stepDef.beginCompensation()
stepDef.changeState(TransactionState.NOT_STARTED)
stepDef.changeState(TransactionStepState.NOT_STARTED)
}
}
}
@@ -318,6 +389,9 @@ export class TransactionOrchestrator extends EventEmitter {
step: TransactionStep,
response: unknown
): Promise<void> {
const hasStepTimedOut =
step.getStates().state === TransactionStepState.TIMEOUT
if (step.saveResponse) {
transaction.addResponse(
step.definition.action!,
@@ -328,16 +402,19 @@ export class TransactionOrchestrator extends EventEmitter {
)
}
step.changeStatus(TransactionStepStatus.OK)
const flow = transaction.getFlow()
if (step.isCompensating()) {
step.changeState(TransactionState.REVERTED)
} else {
step.changeState(TransactionState.DONE)
if (!hasStepTimedOut) {
step.changeStatus(TransactionStepStatus.OK)
}
const flow = transaction.getFlow()
if (step.definition.async || flow.options?.strictCheckpoints) {
if (step.isCompensating()) {
step.changeState(TransactionStepState.REVERTED)
} else if (!hasStepTimedOut) {
step.changeState(TransactionStepState.DONE)
}
if (step.definition.async || flow.options?.storeExecution) {
await transaction.saveCheckpoint()
}
@@ -357,35 +434,87 @@ export class TransactionOrchestrator extends EventEmitter {
transaction.emit(eventName, { step, transaction })
}
private static async setStepTimeout(
transaction: DistributedTransaction,
step: TransactionStep,
error: TransactionStepTimeoutError | TransactionTimeoutError
): Promise<void> {
if (
[
TransactionStepState.TIMEOUT,
TransactionStepState.DONE,
TransactionStepState.REVERTED,
].includes(step.getStates().state)
) {
return
}
step.changeState(TransactionStepState.TIMEOUT)
transaction.addError(
step.definition.action!,
TransactionHandlerType.INVOKE,
error
)
await TransactionOrchestrator.setStepFailure(
transaction,
step,
undefined,
0,
true,
error
)
await transaction.clearStepTimeout(step)
}
private static async setStepFailure(
transaction: DistributedTransaction,
step: TransactionStep,
error: Error | any,
maxRetries: number = TransactionOrchestrator.DEFAULT_RETRIES
maxRetries: number = TransactionOrchestrator.DEFAULT_RETRIES,
isTimeout = false,
timeoutError?: TransactionStepTimeoutError | TransactionTimeoutError
): Promise<void> {
step.failures++
step.changeStatus(TransactionStepStatus.TEMPORARY_FAILURE)
if (
!isTimeout &&
step.getStates().status !== TransactionStepStatus.PERMANENT_FAILURE
) {
step.changeStatus(TransactionStepStatus.TEMPORARY_FAILURE)
}
const flow = transaction.getFlow()
const cleaningUp: Promise<unknown>[] = []
if (step.failures > maxRetries) {
step.changeState(TransactionState.FAILED)
const hasTimedOut = step.getStates().state === TransactionStepState.TIMEOUT
if (step.failures > maxRetries || hasTimedOut) {
if (!hasTimedOut) {
step.changeState(TransactionStepState.FAILED)
}
step.changeStatus(TransactionStepStatus.PERMANENT_FAILURE)
transaction.addError(
step.definition.action!,
step.isCompensating()
? TransactionHandlerType.COMPENSATE
: TransactionHandlerType.INVOKE,
error
)
if (!isTimeout) {
transaction.addError(
step.definition.action!,
step.isCompensating()
? TransactionHandlerType.COMPENSATE
: TransactionHandlerType.INVOKE,
error
)
}
if (!step.isCompensating()) {
if (step.definition.continueOnPermanentFailure) {
if (
step.definition.continueOnPermanentFailure &&
!TransactionTimeoutError.isTransactionTimeoutError(timeoutError!)
) {
for (const childStep of step.next) {
const child = flow.steps[childStep]
child.changeState(TransactionState.SKIPPED)
child.changeState(TransactionStepState.SKIPPED)
}
} else {
flow.state = TransactionState.WAITING_TO_COMPENSATE
@@ -397,7 +526,7 @@ export class TransactionOrchestrator extends EventEmitter {
}
}
if (step.definition.async || flow.options?.strictCheckpoints) {
if (step.definition.async || flow.options?.storeExecution) {
await transaction.saveCheckpoint()
}
@@ -413,33 +542,6 @@ export class TransactionOrchestrator extends EventEmitter {
transaction.emit(eventName, { step, transaction })
}
private async checkTransactionTimeout(transaction, currentSteps) {
let hasTimedOut = false
const flow = transaction.getFlow()
if (
transaction.hasTimeout() &&
!flow.timedOutAt &&
flow.startedAt! + transaction.getTimeoutInterval()! * 1e3 < Date.now()
) {
flow.timedOutAt = Date.now()
this.emit(DistributedTransactionEvent.TIMEOUT, { transaction })
for (const step of currentSteps) {
await TransactionOrchestrator.setStepFailure(
transaction,
step,
new TransactionTimeoutError(),
0
)
}
await transaction.saveCheckpoint()
hasTimedOut = true
}
return hasTimedOut
}
private async executeNext(
transaction: DistributedTransaction
): Promise<void> {
@@ -456,22 +558,19 @@ export class TransactionOrchestrator extends EventEmitter {
const hasTimedOut = await this.checkTransactionTimeout(
transaction,
nextSteps.next
nextSteps.current
)
if (hasTimedOut) {
continue
}
if (nextSteps.remaining === 0) {
if (transaction.hasTimeout()) {
await transaction.clearTransactionTimeout()
void transaction.clearTransactionTimeout()
}
if (flow.options?.retentionTime == undefined) {
await transaction.deleteCheckpoint()
} else {
await transaction.saveCheckpoint()
}
await transaction.saveCheckpoint()
this.emit(DistributedTransactionEvent.FINISH, { transaction })
}
@@ -486,20 +585,20 @@ export class TransactionOrchestrator extends EventEmitter {
step.lastAttempt = Date.now()
step.attempts++
if (curState.state === TransactionState.NOT_STARTED) {
if (curState.state === TransactionStepState.NOT_STARTED) {
if (!step.startedAt) {
step.startedAt = Date.now()
}
if (step.isCompensating()) {
step.changeState(TransactionState.COMPENSATING)
step.changeState(TransactionStepState.COMPENSATING)
if (step.definition.noCompensation) {
step.changeState(TransactionState.REVERTED)
step.changeState(TransactionStepState.REVERTED)
continue
}
} else if (flow.state === TransactionState.INVOKING) {
step.changeState(TransactionState.INVOKING)
step.changeState(TransactionStepState.INVOKING)
}
}
@@ -554,6 +653,14 @@ export class TransactionOrchestrator extends EventEmitter {
transaction
.handler(step.definition.action + "", type, payload, transaction)
.then(async (response: any) => {
if (this.hasExpired({ transaction, step }, Date.now())) {
await this.checkStepTimeout(transaction, step)
await this.checkTransactionTimeout(
transaction,
nextSteps.next.includes(step) ? nextSteps.next : [step]
)
}
await TransactionOrchestrator.setStepSuccess(
transaction,
step,
@@ -561,6 +668,14 @@ export class TransactionOrchestrator extends EventEmitter {
)
})
.catch(async (error) => {
if (this.hasExpired({ transaction, step }, Date.now())) {
await this.checkStepTimeout(transaction, step)
await this.checkTransactionTimeout(
transaction,
nextSteps.next.includes(step) ? nextSteps.next : [step]
)
}
if (
PermanentStepFailureError.isPermanentStepFailureError(error)
) {
@@ -573,7 +688,7 @@ export class TransactionOrchestrator extends EventEmitter {
)
} else {
execution.push(
transaction.saveCheckpoint().then(async () =>
transaction.saveCheckpoint().then(() => {
transaction
.handler(
step.definition.action + "",
@@ -591,12 +706,12 @@ export class TransactionOrchestrator extends EventEmitter {
await setStepFailure(error)
})
)
})
)
}
}
if (hasSyncSteps && flow.options?.strictCheckpoints) {
if (hasSyncSteps && flow.options?.storeExecution) {
await transaction.saveCheckpoint()
}
@@ -630,16 +745,14 @@ export class TransactionOrchestrator extends EventEmitter {
flow.state = TransactionState.INVOKING
flow.startedAt = Date.now()
if (this.options?.storeExecution) {
if (this.options?.store) {
await transaction.saveCheckpoint(
flow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
)
}
if (transaction.hasTimeout()) {
await transaction.scheduleTransactionTimeout(
transaction.getTimeoutInterval()!
)
await transaction.scheduleTransactionTimeout(transaction.getTimeout()!)
}
this.emit(DistributedTransactionEvent.BEGIN, { transaction })
@@ -682,12 +795,19 @@ export class TransactionOrchestrator extends EventEmitter {
this.definition
)
this.options ??= {}
const hasAsyncSteps = features.hasAsyncSteps
const hasStepTimeouts = features.hasStepTimeouts
const hasRetriesTimeout = features.hasRetriesTimeout
const hasTransactionTimeout = !!this.options.timeout
this.options ??= {}
if (hasAsyncSteps || hasStepTimeouts || hasRetriesTimeout) {
if (hasAsyncSteps) {
this.options.store = true
}
if (hasStepTimeouts || hasRetriesTimeout || hasTransactionTimeout) {
this.options.store = true
this.options.storeExecution = true
}
@@ -699,6 +819,7 @@ export class TransactionOrchestrator extends EventEmitter {
hasFailedSteps: false,
hasSkippedSteps: false,
hasWaitingSteps: false,
hasRevertedSteps: false,
timedOutAt: null,
state: TransactionState.NOT_STARTED,
definition: this.definition,
@@ -807,15 +928,16 @@ export class TransactionOrchestrator extends EventEmitter {
new TransactionStep(),
existingSteps?.[id] || {
id,
uuid: definitionCopy.uuid,
depth: level.length - 1,
definition: definitionCopy,
saveResponse: definitionCopy.saveResponse ?? true,
invoke: {
state: TransactionState.NOT_STARTED,
state: TransactionStepState.NOT_STARTED,
status: TransactionStepStatus.IDLE,
},
compensate: {
state: TransactionState.DORMANT,
state: TransactionStepState.DORMANT,
status: TransactionStepStatus.IDLE,
},
attempts: 0,
@@ -861,11 +983,7 @@ export class TransactionOrchestrator extends EventEmitter {
existingTransaction?.context
)
if (
newTransaction &&
this.options?.storeExecution &&
this.options?.strictCheckpoints
) {
if (newTransaction && this.options?.store && this.options?.storeExecution) {
await transaction.saveCheckpoint(
modelFlow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL
)
@@ -1,4 +1,4 @@
import { MedusaError } from "@medusajs/utils"
import { MedusaError, TransactionStepState } from "@medusajs/utils"
import {
DistributedTransaction,
TransactionPayload,
@@ -6,8 +6,8 @@ import {
import {
TransactionHandlerType,
TransactionState,
TransactionStepsDefinition,
TransactionStepStatus,
TransactionStepsDefinition,
} from "./types"
export type TransactionStepHandler = (
@@ -38,14 +38,15 @@ export class TransactionStep {
*/
private stepFailed = false
id: string
uuid?: string
depth: number
definition: TransactionStepsDefinition
invoke: {
state: TransactionState
state: TransactionStepState
status: TransactionStepStatus
}
compensate: {
state: TransactionState
state: TransactionStepState
status: TransactionStepStatus
}
attempts: number
@@ -81,24 +82,25 @@ export class TransactionStep {
return !this.stepFailed
}
public changeState(toState: TransactionState) {
public changeState(toState: TransactionStepState) {
const allowed = {
[TransactionState.DORMANT]: [TransactionState.NOT_STARTED],
[TransactionState.NOT_STARTED]: [
TransactionState.INVOKING,
TransactionState.COMPENSATING,
TransactionState.FAILED,
TransactionState.SKIPPED,
[TransactionStepState.DORMANT]: [TransactionStepState.NOT_STARTED],
[TransactionStepState.NOT_STARTED]: [
TransactionStepState.INVOKING,
TransactionStepState.COMPENSATING,
TransactionStepState.FAILED,
TransactionStepState.SKIPPED,
],
[TransactionState.INVOKING]: [
TransactionState.FAILED,
TransactionState.DONE,
[TransactionStepState.INVOKING]: [
TransactionStepState.FAILED,
TransactionStepState.DONE,
TransactionStepState.TIMEOUT,
],
[TransactionState.COMPENSATING]: [
TransactionState.REVERTED,
TransactionState.FAILED,
[TransactionStepState.COMPENSATING]: [
TransactionStepState.REVERTED,
TransactionStepState.FAILED,
],
[TransactionState.DONE]: [TransactionState.COMPENSATING],
[TransactionStepState.DONE]: [TransactionStepState.COMPENSATING],
}
const curState = this.getStates()
@@ -155,10 +157,10 @@ export class TransactionStep {
}
hasTimeout(): boolean {
return !!this.definition.timeout
return !!this.getTimeout()
}
getTimeoutInterval(): number | undefined {
getTimeout(): number | undefined {
return this.definition.timeout
}
@@ -190,7 +192,7 @@ export class TransactionStep {
const { status, state } = this.getStates()
return (
(!this.isCompensating() &&
state === TransactionState.NOT_STARTED &&
state === TransactionStepState.NOT_STARTED &&
flowState === TransactionState.INVOKING) ||
status === TransactionStepStatus.TEMPORARY_FAILURE
)
@@ -199,7 +201,7 @@ export class TransactionStep {
canCompensate(flowState: TransactionState): boolean {
return (
this.isCompensating() &&
this.getStates().state === TransactionState.NOT_STARTED &&
this.getStates().state === TransactionStepState.NOT_STARTED &&
flowState === TransactionState.COMPENSATING
)
}
+94 -27
View File
@@ -1,51 +1,118 @@
import { DistributedTransaction } from "./distributed-transaction"
import { TransactionStep } from "./transaction-step"
export {
TransactionHandlerType,
TransactionState,
TransactionStepStatus,
} from "@medusajs/utils"
export enum TransactionHandlerType {
INVOKE = "invoke",
COMPENSATE = "compensate",
}
/**
* Defines the structure and behavior of a single step within a transaction workflow.
*/
export type TransactionStepsDefinition = {
/**
* A unique identifier for the transaction step.
* This is set automatically when declaring a workflow with "createWorkflow"
*/
uuid?: string
/**
* Specifies the action to be performed in this step.
* "name" is an alias for action when creating a workflow with "createWorkflow".
*/
action?: string
/**
* Indicates whether the workflow should continue even if there is a permanent failure in this step.
* In case it is set to true, the children steps of this step will not be executed and their status will be marked as TransactionStepState.SKIPPED.
*/
continueOnPermanentFailure?: boolean
/**
* If true, no compensation action will be triggered for this step in case of a failure.
*/
noCompensation?: boolean
/**
* The maximum number of times this step should be retried in case of temporary failures.
* The default is 0 (no retries).
*/
maxRetries?: number
/**
* The interval (in seconds) between retry attempts after a temporary failure.
* The default is to retry immediately.
*/
retryInterval?: number
/**
* The interval (in seconds) to retry a step even if its status is "TransactionStepStatus.WAITING".
*/
retryIntervalAwaiting?: number
/**
* The maximum amount of time (in seconds) to wait for this step to complete.
* This is NOT an execution timeout, the step will always be executed and wait for its response.
* If the response is not received within the timeout set, it will be marked as "TransactionStepStatus.TIMEOUT" and the workflow will be reverted as soon as it receives the response.
*/
timeout?: number
/**
* If true, the step is executed asynchronously. This means that the workflow will not wait for the response of this step.
* Async steps require to have their responses set using "setStepSuccess" or "setStepFailure".
* If combined with a timeout, and any response is not set within that interval, the step will be marked as "TransactionStepStatus.TIMEOUT" and the workflow will be reverted immediately.
*/
async?: boolean
/**
* If true, the compensation function for this step is executed asynchronously. Which means, the response has to be set using "setStepSuccess" or "setStepFailure".
*/
compensateAsync?: boolean
/**
* If true, the workflow will not wait for their sibling steps to complete before moving to the next step.
*/
noWait?: boolean
/**
* If true, the response of this step will be stored.
* Default is true.
*/
saveResponse?: boolean
/**
* Defines the next step(s) to execute after this step. Can be a single step or an array of steps.
*/
next?: TransactionStepsDefinition | TransactionStepsDefinition[]
// TODO: add metadata field for customizations
}
export enum TransactionStepStatus {
IDLE = "idle",
OK = "ok",
WAITING = "waiting_response",
TEMPORARY_FAILURE = "temp_failure",
PERMANENT_FAILURE = "permanent_failure",
}
export enum TransactionState {
NOT_STARTED = "not_started",
INVOKING = "invoking",
WAITING_TO_COMPENSATE = "waiting_to_compensate",
COMPENSATING = "compensating",
DONE = "done",
REVERTED = "reverted",
FAILED = "failed",
DORMANT = "dormant",
SKIPPED = "skipped",
}
/**
* Defines the options for a transaction model, which are applicable to the entire workflow.
*/
export type TransactionModelOptions = {
/**
* The global timeout for the entire transaction workflow (in seconds).
*/
timeout?: number
storeExecution?: boolean
/**
* If true, the state of the transaction will be persisted.
*/
store?: boolean
/**
* TBD
*/
retentionTime?: number
strictCheckpoints?: boolean
/**
* If true, the execution details of each step will be stored.
*/
storeExecution?: boolean
// TODO: add metadata field for customizations
}
export type TransactionModel = {
@@ -81,9 +81,16 @@ export class WorkflowManager {
const finalFlow = flow instanceof OrchestratorBuilder ? flow.build() : flow
if (WorkflowManager.workflows.has(workflowId)) {
function excludeStepUuid(key, value) {
return key === "uuid" ? undefined : value
}
const areStepsEqual = finalFlow
? JSON.stringify(finalFlow) ===
JSON.stringify(WorkflowManager.workflows.get(workflowId)!.flow_)
? JSON.stringify(finalFlow, excludeStepUuid) ===
JSON.stringify(
WorkflowManager.workflows.get(workflowId)!.flow_,
excludeStepUuid
)
: true
if (!areStepsEqual) {
@@ -131,14 +138,19 @@ export class WorkflowManager {
}
const finalFlow = flow instanceof OrchestratorBuilder ? flow.build() : flow
const updatedOptions = { ...workflow.options, ...options }
WorkflowManager.workflows.set(workflowId, {
id: workflowId,
flow_: finalFlow,
orchestrator: new TransactionOrchestrator(workflowId, finalFlow, options),
orchestrator: new TransactionOrchestrator(
workflowId,
finalFlow,
updatedOptions
),
handler: WorkflowManager.buildHandlers(workflow.handlers_),
handlers_: workflow.handlers_,
options: { ...workflow.options, ...options },
options: updatedOptions,
requiredModules,
optionalModules,
})
-1
View File
@@ -18,4 +18,3 @@ export * as SearchTypes from "./search"
export * as StockLocationTypes from "./stock-location"
export * as TransactionBaseTypes from "./transaction-base"
export * as WorkflowTypes from "./workflow"
+1 -1
View File
@@ -46,7 +46,7 @@ export interface RepositoryService<
update(data: TDTOs["update"][], context?: Context): Promise<T[]>
delete(ids: string[], context?: Context): Promise<void>
delete(idsOrPKs: string[] | object[], context?: Context): Promise<void>
/**
* Soft delete entities and cascade to related entities if configured.
+1 -1
View File
@@ -14,9 +14,9 @@ export * from "./joiner"
export * from "./link-modules"
export * from "./logger"
export * from "./modules-sdk"
export * from "./payment"
export * from "./pricing"
export * from "./product"
export * from "./payment"
export * from "./product-category"
export * from "./promotion"
export * from "./region"
@@ -18,8 +18,8 @@ export async function mikroOrmCreateConnection(
// It is important that the knex package version is the same as the one used by MikroORM knex package
driverOptions = database.connection
clientUrl =
database.connection.context.client.config.connection.connectionString
schema = database.connection.context.client.config.searchPath
database.connection.context?.client?.config?.connection?.connectionString
schema = database.connection.context?.client?.config?.searchPath
}
const { MikroORM } = await import("@mikro-orm/postgresql")
@@ -1,3 +1,3 @@
export * from "./inject-transaction-manager"
export * from "./inject-manager"
export * from "./inject-shared-context"
export * from "./inject-transaction-manager"
@@ -93,7 +93,7 @@ export function loadDatabaseConfig(
database.connection = options.database!.connection
}
if (!database.clientUrl && !silent) {
if (!database.clientUrl && !silent && !database.connection) {
throw new MedusaError(
MedusaError.Types.INVALID_ARGUMENT,
"No database clientUrl provided. Please provide the clientUrl through the [MODULE]_POSTGRES_URL, MEDUSA_POSTGRES_URL or POSTGRES_URL environment variable or the options object in the initialize function."
@@ -1 +1,2 @@
export * from "./symbol"
export * from "./types"
+34
View File
@@ -0,0 +1,34 @@
export enum TransactionHandlerType {
INVOKE = "invoke",
COMPENSATE = "compensate",
}
export enum TransactionStepStatus {
IDLE = "idle",
OK = "ok",
WAITING = "waiting_response",
TEMPORARY_FAILURE = "temp_failure",
PERMANENT_FAILURE = "permanent_failure",
}
export enum TransactionState {
NOT_STARTED = "not_started",
INVOKING = "invoking",
WAITING_TO_COMPENSATE = "waiting_to_compensate",
COMPENSATING = "compensating",
DONE = "done",
REVERTED = "reverted",
FAILED = "failed",
}
export enum TransactionStepState {
NOT_STARTED = "not_started",
INVOKING = "invoking",
COMPENSATING = "compensating",
DONE = "done",
REVERTED = "reverted",
FAILED = "failed",
DORMANT = "dormant",
SKIPPED = "skipped",
TIMEOUT = "timeout",
}
@@ -0,0 +1,6 @@
/dist
node_modules
.DS_store
.env*
.env
*.sql
@@ -0,0 +1 @@
# Workflow Orchestrator
@@ -0,0 +1,4 @@
export * from "./workflow_1"
export * from "./workflow_2"
export * from "./workflow_step_timeout"
export * from "./workflow_transaction_timeout"
@@ -0,0 +1,65 @@
import {
StepResponse,
createStep,
createWorkflow,
} from "@medusajs/workflows-sdk"
const step_1 = createStep(
"step_1",
jest.fn((input) => {
input.test = "test"
return new StepResponse(input, { compensate: 123 })
}),
jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
console.log("reverted", compensateInput.compensate)
return new StepResponse({
reverted: true,
})
})
)
const step_2 = createStep(
"step_2",
jest.fn((input, context) => {
console.log("triggered async request", context.metadata.idempotency_key)
if (input) {
return new StepResponse({ notAsyncResponse: input.hey })
}
}),
jest.fn((_, context) => {
return new StepResponse({
step: context.metadata.action,
idempotency_key: context.metadata.idempotency_key,
reverted: true,
})
})
)
const step_3 = createStep(
"step_3",
jest.fn((res) => {
return new StepResponse({
done: {
inputFromSyncStep: res.notAsyncResponse,
},
})
})
)
createWorkflow("workflow_1", function (input) {
step_1(input)
const ret2 = step_2({ hey: "oh" })
step_2({ hey: "async hello" }).config({
name: "new_step_name",
async: true,
})
return step_3(ret2)
})
@@ -0,0 +1,71 @@
import {
StepResponse,
createStep,
createWorkflow,
} from "@medusajs/workflows-sdk"
const step_1 = createStep(
"step_1",
jest.fn((input) => {
input.test = "test"
return new StepResponse(input, { compensate: 123 })
}),
jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
console.log("reverted", compensateInput.compensate)
return new StepResponse({
reverted: true,
})
})
)
const step_2 = createStep(
"step_2",
jest.fn((input, context) => {
console.log("triggered async request", context.metadata.idempotency_key)
if (input) {
return new StepResponse({ notAsyncResponse: input.hey })
}
}),
jest.fn((_, context) => {
return new StepResponse({
step: context.metadata.action,
idempotency_key: context.metadata.idempotency_key,
reverted: true,
})
})
)
const step_3 = createStep(
"step_3",
jest.fn((res) => {
return new StepResponse({
done: {
inputFromSyncStep: res.notAsyncResponse,
},
})
})
)
createWorkflow(
{
name: "workflow_2",
retentionTime: 1000,
},
function (input) {
step_1(input)
const ret2 = step_2({ hey: "oh" })
step_2({ hey: "async hello" }).config({
name: "new_step_name",
async: true,
})
return step_3(ret2)
}
)
@@ -0,0 +1,29 @@
import {
StepResponse,
createStep,
createWorkflow,
} from "@medusajs/workflows-sdk"
import { setTimeout } from "timers/promises"
const step_1 = createStep(
"step_1",
jest.fn(async (input) => {
await setTimeout(200)
return new StepResponse(input, { compensate: 123 })
}),
jest.fn(() => {})
)
createWorkflow(
{
name: "workflow_step_timeout",
},
function (input) {
const resp = step_1(input).config({
timeout: 0.1, // 0.1 second
})
return resp
}
)
@@ -0,0 +1,36 @@
import {
StepResponse,
createStep,
createWorkflow,
} from "@medusajs/workflows-sdk"
const step_1 = createStep(
"step_1",
jest.fn((input) => {
input.test = "test"
return new StepResponse(input, { compensate: 123 })
}),
jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
return new StepResponse({
reverted: true,
})
})
)
createWorkflow(
{
name: "workflow_transaction_timeout",
timeout: 0.1, // 0.1 second
},
function (input) {
const resp = step_1(input).config({
async: true,
})
return resp
}
)
@@ -0,0 +1,163 @@
import { MedusaApp } from "@medusajs/modules-sdk"
import { RemoteJoinerQuery } from "@medusajs/types"
import { TransactionHandlerType } from "@medusajs/utils"
import { IWorkflowsModuleService } from "@medusajs/workflows-sdk"
import { knex } from "knex"
import { setTimeout } from "timers/promises"
import "../__fixtures__"
import { DB_URL, TestDatabase } from "../utils"
const sharedPgConnection = knex<any, any>({
client: "pg",
searchPath: process.env.MEDUSA_WORKFLOW_ENGINE_DB_SCHEMA,
connection: {
connectionString: DB_URL,
debug: false,
},
})
const afterEach_ = async () => {
await TestDatabase.clearTables(sharedPgConnection)
}
describe("Workflow Orchestrator module", function () {
describe("Testing basic workflow", function () {
let workflowOrcModule: IWorkflowsModuleService
let query: (
query: string | RemoteJoinerQuery | object,
variables?: Record<string, unknown>
) => Promise<any>
afterEach(afterEach_)
beforeAll(async () => {
const {
runMigrations,
query: remoteQuery,
modules,
} = await MedusaApp({
sharedResourcesConfig: {
database: {
connection: sharedPgConnection,
},
},
modulesConfig: {
workflows: {
resolve: __dirname + "/../..",
},
},
})
query = remoteQuery
await runMigrations()
workflowOrcModule =
modules.workflows as unknown as IWorkflowsModuleService
})
afterEach(afterEach_)
it("should return a list of workflow executions and remove after completed when there is no retentionTime set", async () => {
await workflowOrcModule.run("workflow_1", {
input: {
value: "123",
},
throwOnError: true,
})
let executionsList = await query({
workflow_executions: {
fields: ["workflow_id", "transaction_id", "state"],
},
})
expect(executionsList).toHaveLength(1)
const { result } = await workflowOrcModule.setStepSuccess({
idempotencyKey: {
action: TransactionHandlerType.INVOKE,
stepId: "new_step_name",
workflowId: "workflow_1",
transactionId: executionsList[0].transaction_id,
},
stepResponse: { uhuuuu: "yeaah!" },
})
executionsList = await query({
workflow_executions: {
fields: ["id"],
},
})
expect(executionsList).toHaveLength(0)
expect(result).toEqual({
done: {
inputFromSyncStep: "oh",
},
})
})
it("should return a list of workflow executions and keep it saved when there is a retentionTime set", async () => {
await workflowOrcModule.run("workflow_2", {
input: {
value: "123",
},
throwOnError: true,
transactionId: "transaction_1",
})
let executionsList = await query({
workflow_executions: {
fields: ["id"],
},
})
expect(executionsList).toHaveLength(1)
await workflowOrcModule.setStepSuccess({
idempotencyKey: {
action: TransactionHandlerType.INVOKE,
stepId: "new_step_name",
workflowId: "workflow_2",
transactionId: "transaction_1",
},
stepResponse: { uhuuuu: "yeaah!" },
})
executionsList = await query({
workflow_executions: {
fields: ["id"],
},
})
expect(executionsList).toHaveLength(1)
})
it("should revert the entire transaction when a step timeout expires", async () => {
const { transaction } = await workflowOrcModule.run(
"workflow_step_timeout",
{
input: {},
throwOnError: false,
}
)
expect(transaction.flow.state).toEqual("reverted")
})
it("should revert the entire transaction when the transaction timeout expires", async () => {
const { transaction } = await workflowOrcModule.run(
"workflow_transaction_timeout",
{
input: {},
throwOnError: false,
}
)
await setTimeout(200)
expect(transaction.flow.state).toEqual("reverted")
})
})
})
@@ -0,0 +1,6 @@
if (typeof process.env.DB_TEMP_NAME === "undefined") {
const tempName = parseInt(process.env.JEST_WORKER_ID || "1")
process.env.DB_TEMP_NAME = `medusa-workflow-engine-inmemory-${tempName}`
}
process.env.MEDUSA_WORKFLOW_ENGINE_DB_SCHEMA = "public"
@@ -0,0 +1,3 @@
import { JestUtils } from "medusa-test-utils"
JestUtils.afterAllHookDropDatabase()
@@ -0,0 +1,22 @@
import * as process from "process"
const DB_HOST = process.env.DB_HOST ?? "localhost"
const DB_USERNAME = process.env.DB_USERNAME ?? ""
const DB_PASSWORD = process.env.DB_PASSWORD
const DB_NAME = process.env.DB_TEMP_NAME
export const DB_URL = `postgres://${DB_USERNAME}${
DB_PASSWORD ? `:${DB_PASSWORD}` : ""
}@${DB_HOST}/${DB_NAME}`
interface TestDatabase {
clearTables(knex): Promise<void>
}
export const TestDatabase: TestDatabase = {
clearTables: async (knex) => {
await knex.raw(`
TRUNCATE TABLE workflow_execution CASCADE;
`)
},
}
@@ -0,0 +1 @@
export * from "./database"
@@ -0,0 +1,22 @@
module.exports = {
moduleNameMapper: {
"^@models": "<rootDir>/src/models",
"^@services": "<rootDir>/src/services",
"^@repositories": "<rootDir>/src/repositories",
"^@types": "<rootDir>/src/types",
},
transform: {
"^.+\\.[jt]s?$": [
"ts-jest",
{
tsConfig: "tsconfig.spec.json",
isolatedModules: true,
},
],
},
testEnvironment: `node`,
moduleFileExtensions: [`js`, `ts`],
modulePathIgnorePatterns: ["dist/"],
setupFiles: ["<rootDir>/integration-tests/setup-env.js"],
setupFilesAfterEnv: ["<rootDir>/integration-tests/setup.js"],
}
@@ -0,0 +1,8 @@
import * as entities from "./src/models"
module.exports = {
entities: Object.values(entities),
schema: "public",
clientUrl: "postgres://postgres@localhost/medusa-workflow-engine-inmemory",
type: "postgresql",
}
@@ -0,0 +1,59 @@
{
"name": "@medusajs/workflow-engine-inmemory",
"version": "0.0.1",
"description": "Medusa Workflow Orchestrator module",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"files": [
"dist"
],
"engines": {
"node": ">=16"
},
"repository": {
"type": "git",
"url": "https://github.com/medusajs/medusa",
"directory": "packages/workflow-engine-inmemory"
},
"publishConfig": {
"access": "public"
},
"author": "Medusa",
"license": "MIT",
"scripts": {
"watch": "tsc --build --watch",
"watch:test": "tsc --build tsconfig.spec.json --watch",
"prepublishOnly": "cross-env NODE_ENV=production tsc --build && tsc-alias -p tsconfig.json",
"build": "rimraf dist && tsc --build && tsc-alias -p tsconfig.json",
"test": "jest --passWithNoTests --runInBand --bail --forceExit -- src/**/__tests__/**/*.ts",
"test:integration": "jest --runInBand --forceExit -- integration-tests/**/__tests__/**/*.ts",
"migration:generate": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:generate",
"migration:initial": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:create --initial",
"migration:create": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:create",
"migration:up": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:up",
"orm:cache:clear": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm cache:clear"
},
"devDependencies": {
"@mikro-orm/cli": "5.9.7",
"cross-env": "^5.2.1",
"jest": "^29.6.3",
"medusa-test-utils": "^1.1.40",
"rimraf": "^3.0.2",
"ts-jest": "^29.1.1",
"ts-node": "^10.9.1",
"tsc-alias": "^1.8.6",
"typescript": "^5.1.6"
},
"dependencies": {
"@medusajs/modules-sdk": "^1.12.5",
"@medusajs/types": "^1.11.9",
"@medusajs/utils": "^1.11.2",
"@medusajs/workflows-sdk": "^0.1.0",
"@mikro-orm/core": "5.9.7",
"@mikro-orm/migrations": "5.9.7",
"@mikro-orm/postgresql": "5.9.7",
"awilix": "^8.0.0",
"dotenv": "^16.1.4",
"knex": "2.4.2"
}
}
@@ -0,0 +1,22 @@
import { Modules } from "@medusajs/modules-sdk"
import { ModulesSdkUtils } from "@medusajs/utils"
import * as models from "@models"
import { moduleDefinition } from "./module-definition"
export default moduleDefinition
const migrationScriptOptions = {
moduleName: Modules.WORKFLOW_ENGINE,
models: models,
pathToMigrations: __dirname + "/migrations",
}
export const runMigrations = ModulesSdkUtils.buildMigrationScript(
migrationScriptOptions
)
export const revertMigration = ModulesSdkUtils.buildRevertMigrationScript(
migrationScriptOptions
)
export * from "./initialize"
export * from "./loaders"
@@ -0,0 +1,36 @@
import {
ExternalModuleDeclaration,
InternalModuleDeclaration,
MedusaModule,
MODULE_PACKAGE_NAMES,
Modules,
} from "@medusajs/modules-sdk"
import { ModulesSdkTypes } from "@medusajs/types"
import { WorkflowOrchestratorTypes } from "@medusajs/workflows-sdk"
import { moduleDefinition } from "../module-definition"
import { InitializeModuleInjectableDependencies } from "../types"
export const initialize = async (
options?:
| ModulesSdkTypes.ModuleServiceInitializeOptions
| ModulesSdkTypes.ModuleServiceInitializeCustomDataLayerOptions
| ExternalModuleDeclaration
| InternalModuleDeclaration,
injectedDependencies?: InitializeModuleInjectableDependencies
): Promise<WorkflowOrchestratorTypes.IWorkflowsModuleService> => {
const loaded =
// eslint-disable-next-line max-len
await MedusaModule.bootstrap<WorkflowOrchestratorTypes.IWorkflowsModuleService>(
{
moduleKey: Modules.WORKFLOW_ENGINE,
defaultPath: MODULE_PACKAGE_NAMES[Modules.WORKFLOW_ENGINE],
declaration: options as
| InternalModuleDeclaration
| ExternalModuleDeclaration,
injectedDependencies,
moduleExports: moduleDefinition,
}
)
return loaded[Modules.WORKFLOW_ENGINE]
}
@@ -0,0 +1,34 @@
import { Modules } from "@medusajs/modules-sdk"
import { ModuleJoinerConfig } from "@medusajs/types"
import { MapToConfig } from "@medusajs/utils"
import { WorkflowExecution } from "@models"
import moduleSchema from "./schema"
export const LinkableKeys = {
workflow_execution_id: WorkflowExecution.name,
}
const entityLinkableKeysMap: MapToConfig = {}
Object.entries(LinkableKeys).forEach(([key, value]) => {
entityLinkableKeysMap[value] ??= []
entityLinkableKeysMap[value].push({
mapTo: key,
valueFrom: key.split("_").pop()!,
})
})
export const entityNameToLinkableKeysMap: MapToConfig = entityLinkableKeysMap
export const joinerConfig: ModuleJoinerConfig = {
serviceName: Modules.WORKFLOW_ENGINE,
primaryKeys: ["id"],
schema: moduleSchema,
linkableKeys: LinkableKeys,
alias: {
name: ["workflow_execution", "workflow_executions"],
args: {
entity: WorkflowExecution.name,
methodSuffix: "WorkflowExecution",
},
},
}
@@ -0,0 +1,36 @@
import {
InternalModuleDeclaration,
LoaderOptions,
Modules,
} from "@medusajs/modules-sdk"
import { ModulesSdkTypes } from "@medusajs/types"
import { ModulesSdkUtils } from "@medusajs/utils"
import { EntitySchema } from "@mikro-orm/core"
import * as WorkflowOrchestratorModels from "../models"
export default async (
{
options,
container,
logger,
}: LoaderOptions<
| ModulesSdkTypes.ModuleServiceInitializeOptions
| ModulesSdkTypes.ModuleServiceInitializeCustomDataLayerOptions
>,
moduleDeclaration?: InternalModuleDeclaration
): Promise<void> => {
const entities = Object.values(
WorkflowOrchestratorModels
) as unknown as EntitySchema[]
const pathToMigrations = __dirname + "/../migrations"
await ModulesSdkUtils.mikroOrmConnectionLoader({
moduleName: Modules.WORKFLOW_ENGINE,
entities,
container,
options,
moduleDeclaration,
logger,
pathToMigrations,
})
}
@@ -0,0 +1,9 @@
import { MikroOrmBaseRepository, ModulesSdkUtils } from "@medusajs/utils"
import * as ModuleModels from "@models"
import * as ModuleServices from "@services"
export default ModulesSdkUtils.moduleContainerLoaderFactory({
moduleModels: ModuleModels,
moduleServices: ModuleServices,
moduleRepositories: { BaseRepository: MikroOrmBaseRepository },
})
@@ -0,0 +1,3 @@
export * from "./connection"
export * from "./container"
export * from "./utils"
@@ -0,0 +1,10 @@
import { asClass } from "awilix"
import { InMemoryDistributedTransactionStorage } from "../utils"
export default async ({ container }): Promise<void> => {
container.register({
inMemoryDistributedTransactionStorage: asClass(
InMemoryDistributedTransactionStorage
).singleton(),
})
}
@@ -0,0 +1,41 @@
import { Migration } from "@mikro-orm/migrations"
export class Migration20231221104256 extends Migration {
async up(): Promise<void> {
this.addSql(
`
CREATE TABLE IF NOT EXISTS workflow_execution
(
id character varying NOT NULL,
workflow_id character varying NOT NULL,
transaction_id character varying NOT NULL,
execution jsonb NULL,
context jsonb NULL,
state character varying NOT NULL,
created_at timestamp WITHOUT time zone NOT NULL DEFAULT Now(),
updated_at timestamp WITHOUT time zone NOT NULL DEFAULT Now(),
deleted_at timestamp WITHOUT time zone NULL,
CONSTRAINT "PK_workflow_execution_workflow_id_transaction_id" PRIMARY KEY ("workflow_id", "transaction_id")
);
CREATE UNIQUE INDEX IF NOT EXISTS "IDX_workflow_execution_id" ON "workflow_execution" ("id");
CREATE INDEX IF NOT EXISTS "IDX_workflow_execution_workflow_id" ON "workflow_execution" ("workflow_id") WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS "IDX_workflow_execution_transaction_id" ON "workflow_execution" ("transaction_id") WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS "IDX_workflow_execution_state" ON "workflow_execution" ("state") WHERE deleted_at IS NULL;
`
)
}
async down(): Promise<void> {
this.addSql(
`
DROP INDEX "IDX_workflow_execution_id";
DROP INDEX "IDX_workflow_execution_workflow_id";
DROP INDEX "IDX_workflow_execution_transaction_id";
DROP INDEX "IDX_workflow_execution_state";
DROP TABLE IF EXISTS workflow_execution;
`
)
}
}
@@ -0,0 +1 @@
export { default as WorkflowExecution } from "./workflow-execution"
@@ -0,0 +1,76 @@
import { TransactionState } from "@medusajs/orchestration"
import { DALUtils, generateEntityId } from "@medusajs/utils"
import {
BeforeCreate,
Entity,
Enum,
Filter,
Index,
OnInit,
OptionalProps,
PrimaryKey,
Property,
Unique,
} from "@mikro-orm/core"
type OptionalFields = "deleted_at"
@Entity()
@Unique({
name: "IDX_workflow_execution_workflow_id_transaction_id_unique",
properties: ["workflow_id", "transaction_id"],
})
@Filter(DALUtils.mikroOrmSoftDeletableFilterOptions)
export default class WorkflowExecution {
[OptionalProps]?: OptionalFields
@Property({ columnType: "text", nullable: false })
@Index({ name: "IDX_workflow_execution_id" })
id!: string
@Index({ name: "IDX_workflow_execution_workflow_id" })
@PrimaryKey({ columnType: "text" })
workflow_id: string
@Index({ name: "IDX_workflow_execution_transaction_id" })
@PrimaryKey({ columnType: "text" })
transaction_id: string
@Property({ columnType: "jsonb", nullable: true })
execution: Record<string, unknown> | null = null
@Property({ columnType: "jsonb", nullable: true })
context: Record<string, unknown> | null = null
@Index({ name: "IDX_workflow_execution_state" })
@Enum(() => TransactionState)
state: TransactionState
@Property({
onCreate: () => new Date(),
columnType: "timestamptz",
defaultRaw: "now()",
})
created_at: Date
@Property({
onCreate: () => new Date(),
onUpdate: () => new Date(),
columnType: "timestamptz",
defaultRaw: "now()",
})
updated_at: Date
@Property({ columnType: "timestamptz", nullable: true })
deleted_at: Date | null = null
@BeforeCreate()
onCreate() {
this.id = generateEntityId(this.id, "wf_exec")
}
@OnInit()
onInit() {
this.id = generateEntityId(this.id, "wf_exec")
}
}
@@ -0,0 +1,13 @@
import { ModuleExports } from "@medusajs/types"
import { WorkflowsModuleService } from "@services"
import loadConnection from "./loaders/connection"
import loadContainer from "./loaders/container"
import loadUtils from "./loaders/utils"
const service = WorkflowsModuleService
const loaders = [loadContainer, loadConnection, loadUtils] as any
export const moduleDefinition: ModuleExports = {
service,
loaders,
}
@@ -0,0 +1,2 @@
export { MikroOrmBaseRepository as BaseRepository } from "@medusajs/utils"
export { WorkflowExecutionRepository } from "./workflow-execution"
@@ -0,0 +1,7 @@
import { DALUtils } from "@medusajs/utils"
import { WorkflowExecution } from "@models"
// eslint-disable-next-line max-len
export class WorkflowExecutionRepository extends DALUtils.mikroOrmBaseRepositoryFactory(
WorkflowExecution
) {}
@@ -0,0 +1,26 @@
export default `
scalar DateTime
scalar JSON
enum TransactionState {
NOT_STARTED
INVOKING
WAITING_TO_COMPENSATE
COMPENSATING
DONE
REVERTED
FAILED
}
type WorkflowExecution {
id: ID!
created_at: DateTime!
updated_at: DateTime!
deleted_at: DateTime
workflow_id: string
transaction_id: string
execution: JSON
context: JSON
state: TransactionState
}
`
@@ -0,0 +1,5 @@
describe("Noop test", () => {
it("noop check", async () => {
expect(true).toBe(true)
})
})
@@ -0,0 +1,3 @@
export * from "./workflow-execution"
export * from "./workflow-orchestrator"
export * from "./workflows-module"
@@ -0,0 +1,21 @@
import { DAL } from "@medusajs/types"
import { ModulesSdkUtils } from "@medusajs/utils"
import { WorkflowExecution } from "@models"
type InjectedDependencies = {
workflowExecutionRepository: DAL.RepositoryService
}
export class WorkflowExecutionService<
TEntity extends WorkflowExecution = WorkflowExecution
> extends ModulesSdkUtils.abstractServiceFactory<InjectedDependencies, {}>(
WorkflowExecution
)<TEntity> {
protected workflowExecutionRepository_: DAL.RepositoryService<TEntity>
constructor({ workflowExecutionRepository }: InjectedDependencies) {
// @ts-ignore
super(...arguments)
this.workflowExecutionRepository_ = workflowExecutionRepository
}
}
@@ -0,0 +1,528 @@
import {
DistributedTransaction,
DistributedTransactionEvents,
TransactionHandlerType,
TransactionStep,
} from "@medusajs/orchestration"
import { ContainerLike, Context, MedusaContainer } from "@medusajs/types"
import { InjectSharedContext, isString, MedusaContext } from "@medusajs/utils"
import {
type FlowRunOptions,
MedusaWorkflow,
ReturnWorkflow,
} from "@medusajs/workflows-sdk"
import { ulid } from "ulid"
import { InMemoryDistributedTransactionStorage } from "../utils"
export type WorkflowOrchestratorRunOptions<T> = FlowRunOptions<T> & {
transactionId?: string
container?: ContainerLike
}
type RegisterStepSuccessOptions<T> = Omit<
WorkflowOrchestratorRunOptions<T>,
"transactionId" | "input"
>
type IdempotencyKeyParts = {
workflowId: string
transactionId: string
stepId: string
action: "invoke" | "compensate"
}
type NotifyOptions = {
eventType: keyof DistributedTransactionEvents
workflowId: string
transactionId?: string
step?: TransactionStep
response?: unknown
result?: unknown
errors?: unknown[]
}
type WorkflowId = string
type TransactionId = string
type SubscriberHandler = {
(input: NotifyOptions): void
} & {
_id?: string
}
type SubscribeOptions = {
workflowId: string
transactionId?: string
subscriber: SubscriberHandler
subscriberId?: string
}
type UnsubscribeOptions = {
workflowId: string
transactionId?: string
subscriberOrId: string | SubscriberHandler
}
type TransactionSubscribers = Map<TransactionId, SubscriberHandler[]>
type Subscribers = Map<WorkflowId, TransactionSubscribers>
const AnySubscriber = "any"
export class WorkflowOrchestratorService {
private subscribers: Subscribers = new Map()
constructor({
inMemoryDistributedTransactionStorage,
}: {
inMemoryDistributedTransactionStorage: InMemoryDistributedTransactionStorage
workflowOrchestratorService: WorkflowOrchestratorService
}) {
inMemoryDistributedTransactionStorage.setWorkflowOrchestratorService(this)
DistributedTransaction.setStorage(inMemoryDistributedTransactionStorage)
}
@InjectSharedContext()
async run<T = unknown>(
workflowIdOrWorkflow: string | ReturnWorkflow<any, any, any>,
options?: WorkflowOrchestratorRunOptions<T>,
@MedusaContext() sharedContext: Context = {}
) {
let {
input,
context,
transactionId,
resultFrom,
throwOnError,
events: eventHandlers,
container,
} = options ?? {}
const workflowId = isString(workflowIdOrWorkflow)
? workflowIdOrWorkflow
: workflowIdOrWorkflow.getName()
if (!workflowId) {
throw new Error("Workflow ID is required")
}
context ??= {}
context.transactionId ??= transactionId ?? ulid()
const events: FlowRunOptions["events"] = this.buildWorkflowEvents({
customEventHandlers: eventHandlers,
workflowId,
transactionId: context.transactionId,
})
const exportedWorkflow: any = MedusaWorkflow.getWorkflow(workflowId)
if (!exportedWorkflow) {
throw new Error(`Workflow with id "${workflowId}" not found.`)
}
const flow = exportedWorkflow(container as MedusaContainer)
const ret = await flow.run({
input,
throwOnError,
resultFrom,
context,
events,
})
// TODO: temporary
const acknowledgement = {
transactionId: context.transactionId,
workflowId: workflowId,
}
if (ret.transaction.hasFinished()) {
const { result, errors } = ret
this.notify({
eventType: "onFinish",
workflowId,
transactionId: context.transactionId,
result,
errors,
})
}
return { acknowledgement, ...ret }
}
@InjectSharedContext()
async getRunningTransaction(
workflowId: string,
transactionId: string,
options?: WorkflowOrchestratorRunOptions<undefined>,
@MedusaContext() sharedContext: Context = {}
): Promise<DistributedTransaction> {
let { context, container } = options ?? {}
if (!workflowId) {
throw new Error("Workflow ID is required")
}
if (!transactionId) {
throw new Error("TransactionId ID is required")
}
context ??= {}
context.transactionId ??= transactionId
const exportedWorkflow: any = MedusaWorkflow.getWorkflow(workflowId)
if (!exportedWorkflow) {
throw new Error(`Workflow with id "${workflowId}" not found.`)
}
const flow = exportedWorkflow(container as MedusaContainer)
const transaction = await flow.getRunningTransaction(transactionId, context)
return transaction
}
@InjectSharedContext()
async setStepSuccess<T = unknown>(
{
idempotencyKey,
stepResponse,
options,
}: {
idempotencyKey: string | IdempotencyKeyParts
stepResponse: unknown
options?: RegisterStepSuccessOptions<T>
},
@MedusaContext() sharedContext: Context = {}
) {
const {
context,
throwOnError,
resultFrom,
container,
events: eventHandlers,
} = options ?? {}
const [idempotencyKey_, { workflowId, transactionId }] =
this.buildIdempotencyKeyAndParts(idempotencyKey)
const exportedWorkflow: any = MedusaWorkflow.getWorkflow(workflowId)
if (!exportedWorkflow) {
throw new Error(`Workflow with id "${workflowId}" not found.`)
}
const flow = exportedWorkflow(container as MedusaContainer)
const events = this.buildWorkflowEvents({
customEventHandlers: eventHandlers,
transactionId,
workflowId,
})
const ret = await flow.registerStepSuccess({
idempotencyKey: idempotencyKey_,
context,
resultFrom,
throwOnError,
events,
response: stepResponse,
})
if (ret.transaction.hasFinished()) {
const { result, errors } = ret
this.notify({
eventType: "onFinish",
workflowId,
transactionId,
result,
errors,
})
}
return ret
}
@InjectSharedContext()
async setStepFailure<T = unknown>(
{
idempotencyKey,
stepResponse,
options,
}: {
idempotencyKey: string | IdempotencyKeyParts
stepResponse: unknown
options?: RegisterStepSuccessOptions<T>
},
@MedusaContext() sharedContext: Context = {}
) {
const {
context,
throwOnError,
resultFrom,
container,
events: eventHandlers,
} = options ?? {}
const [idempotencyKey_, { workflowId, transactionId }] =
this.buildIdempotencyKeyAndParts(idempotencyKey)
const exportedWorkflow: any = MedusaWorkflow.getWorkflow(workflowId)
if (!exportedWorkflow) {
throw new Error(`Workflow with id "${workflowId}" not found.`)
}
const flow = exportedWorkflow(container as MedusaContainer)
const events = this.buildWorkflowEvents({
customEventHandlers: eventHandlers,
transactionId,
workflowId,
})
const ret = await flow.registerStepFailure({
idempotencyKey: idempotencyKey_,
context,
resultFrom,
throwOnError,
events,
response: stepResponse,
})
if (ret.transaction.hasFinished()) {
const { result, errors } = ret
this.notify({
eventType: "onFinish",
workflowId,
transactionId,
result,
errors,
})
}
return ret
}
@InjectSharedContext()
subscribe(
{ workflowId, transactionId, subscriber, subscriberId }: SubscribeOptions,
@MedusaContext() sharedContext: Context = {}
) {
subscriber._id = subscriberId
const subscribers = this.subscribers.get(workflowId) ?? new Map()
const handlerIndex = (handlers) => {
return handlers.indexOf((s) => s === subscriber || s._id === subscriberId)
}
if (transactionId) {
const transactionSubscribers = subscribers.get(transactionId) ?? []
const subscriberIndex = handlerIndex(transactionSubscribers)
if (subscriberIndex !== -1) {
transactionSubscribers.slice(subscriberIndex, 1)
}
transactionSubscribers.push(subscriber)
subscribers.set(transactionId, transactionSubscribers)
this.subscribers.set(workflowId, subscribers)
return
}
const workflowSubscribers = subscribers.get(AnySubscriber) ?? []
const subscriberIndex = handlerIndex(workflowSubscribers)
if (subscriberIndex !== -1) {
workflowSubscribers.slice(subscriberIndex, 1)
}
workflowSubscribers.push(subscriber)
subscribers.set(AnySubscriber, workflowSubscribers)
this.subscribers.set(workflowId, subscribers)
}
@InjectSharedContext()
unsubscribe(
{ workflowId, transactionId, subscriberOrId }: UnsubscribeOptions,
@MedusaContext() sharedContext: Context = {}
) {
const subscribers = this.subscribers.get(workflowId) ?? new Map()
const filterSubscribers = (handlers: SubscriberHandler[]) => {
return handlers.filter((handler) => {
return handler._id
? handler._id !== (subscriberOrId as string)
: handler !== (subscriberOrId as SubscriberHandler)
})
}
if (transactionId) {
const transactionSubscribers = subscribers.get(transactionId) ?? []
const newTransactionSubscribers = filterSubscribers(
transactionSubscribers
)
subscribers.set(transactionId, newTransactionSubscribers)
this.subscribers.set(workflowId, subscribers)
return
}
const workflowSubscribers = subscribers.get(AnySubscriber) ?? []
const newWorkflowSubscribers = filterSubscribers(workflowSubscribers)
subscribers.set(AnySubscriber, newWorkflowSubscribers)
this.subscribers.set(workflowId, subscribers)
}
private notify(options: NotifyOptions) {
const {
eventType,
workflowId,
transactionId,
errors,
result,
step,
response,
} = options
const subscribers: TransactionSubscribers =
this.subscribers.get(workflowId) ?? new Map()
const notifySubscribers = (handlers: SubscriberHandler[]) => {
handlers.forEach((handler) => {
handler({
eventType,
workflowId,
transactionId,
step,
response,
result,
errors,
})
})
}
if (transactionId) {
const transactionSubscribers = subscribers.get(transactionId) ?? []
notifySubscribers(transactionSubscribers)
}
const workflowSubscribers = subscribers.get(AnySubscriber) ?? []
notifySubscribers(workflowSubscribers)
}
private buildWorkflowEvents({
customEventHandlers,
workflowId,
transactionId,
}): DistributedTransactionEvents {
const notify = ({
eventType,
step,
result,
response,
errors,
}: {
eventType: keyof DistributedTransactionEvents
step?: TransactionStep
response?: unknown
result?: unknown
errors?: unknown[]
}) => {
this.notify({
workflowId,
transactionId,
eventType,
response,
step,
result,
errors,
})
}
return {
onTimeout: ({ transaction }) => {
customEventHandlers?.onTimeout?.({ transaction })
notify({ eventType: "onTimeout" })
},
onBegin: ({ transaction }) => {
customEventHandlers?.onBegin?.({ transaction })
notify({ eventType: "onBegin" })
},
onResume: ({ transaction }) => {
customEventHandlers?.onResume?.({ transaction })
notify({ eventType: "onResume" })
},
onCompensateBegin: ({ transaction }) => {
customEventHandlers?.onCompensateBegin?.({ transaction })
notify({ eventType: "onCompensateBegin" })
},
onFinish: ({ transaction, result, errors }) => {
// TODO: unsubscribe transaction handlers on finish
customEventHandlers?.onFinish?.({ transaction, result, errors })
},
onStepBegin: ({ step, transaction }) => {
customEventHandlers?.onStepBegin?.({ step, transaction })
notify({ eventType: "onStepBegin", step })
},
onStepSuccess: ({ step, transaction }) => {
const response = transaction.getContext().invoke[step.id]
customEventHandlers?.onStepSuccess?.({ step, transaction, response })
notify({ eventType: "onStepSuccess", step, response })
},
onStepFailure: ({ step, transaction }) => {
const errors = transaction.getErrors(TransactionHandlerType.INVOKE)[
step.id
]
customEventHandlers?.onStepFailure?.({ step, transaction, errors })
notify({ eventType: "onStepFailure", step, errors })
},
onCompensateStepSuccess: ({ step, transaction }) => {
const response = transaction.getContext().compensate[step.id]
customEventHandlers?.onStepSuccess?.({ step, transaction, response })
notify({ eventType: "onCompensateStepSuccess", step, response })
},
onCompensateStepFailure: ({ step, transaction }) => {
const errors = transaction.getErrors(TransactionHandlerType.COMPENSATE)[
step.id
]
customEventHandlers?.onStepFailure?.({ step, transaction, errors })
notify({ eventType: "onCompensateStepFailure", step, errors })
},
}
}
private buildIdempotencyKeyAndParts(
idempotencyKey: string | IdempotencyKeyParts
): [string, IdempotencyKeyParts] {
const parts: IdempotencyKeyParts = {
workflowId: "",
transactionId: "",
stepId: "",
action: "invoke",
}
let idempotencyKey_ = idempotencyKey as string
const setParts = (workflowId, transactionId, stepId, action) => {
parts.workflowId = workflowId
parts.transactionId = transactionId
parts.stepId = stepId
parts.action = action
}
if (!isString(idempotencyKey)) {
const { workflowId, transactionId, stepId, action } =
idempotencyKey as IdempotencyKeyParts
idempotencyKey_ = [workflowId, transactionId, stepId, action].join(":")
setParts(workflowId, transactionId, stepId, action)
} else {
const [workflowId, transactionId, stepId, action] =
idempotencyKey_.split(":")
setParts(workflowId, transactionId, stepId, action)
}
return [idempotencyKey_, parts]
}
}
@@ -0,0 +1,199 @@
import {
Context,
DAL,
FindConfig,
InternalModuleDeclaration,
ModuleJoinerConfig,
} from "@medusajs/types"
import {} from "@medusajs/types/src"
import {
InjectManager,
InjectSharedContext,
MedusaContext,
} from "@medusajs/utils"
import type {
ReturnWorkflow,
UnwrapWorkflowInputDataType,
WorkflowOrchestratorTypes,
} from "@medusajs/workflows-sdk"
import {
WorkflowExecutionService,
WorkflowOrchestratorService,
} from "@services"
import { joinerConfig } from "../joiner-config"
type InjectedDependencies = {
baseRepository: DAL.RepositoryService
workflowExecutionService: WorkflowExecutionService
workflowOrchestratorService: WorkflowOrchestratorService
}
export class WorkflowsModuleService
implements WorkflowOrchestratorTypes.IWorkflowsModuleService
{
protected baseRepository_: DAL.RepositoryService
protected workflowExecutionService_: WorkflowExecutionService
protected workflowOrchestratorService_: WorkflowOrchestratorService
constructor(
{
baseRepository,
workflowExecutionService,
workflowOrchestratorService,
}: InjectedDependencies,
protected readonly moduleDeclaration: InternalModuleDeclaration
) {
this.baseRepository_ = baseRepository
this.workflowExecutionService_ = workflowExecutionService
this.workflowOrchestratorService_ = workflowOrchestratorService
}
__joinerConfig(): ModuleJoinerConfig {
return joinerConfig
}
@InjectManager("baseRepository_")
async listWorkflowExecution(
filters: WorkflowOrchestratorTypes.FilterableWorkflowExecutionProps = {},
config: FindConfig<WorkflowOrchestratorTypes.WorkflowExecutionDTO> = {},
@MedusaContext() sharedContext: Context = {}
): Promise<WorkflowOrchestratorTypes.WorkflowExecutionDTO[]> {
const wfExecutions = await this.workflowExecutionService_.list(
filters,
config,
sharedContext
)
return this.baseRepository_.serialize<
WorkflowOrchestratorTypes.WorkflowExecutionDTO[]
>(wfExecutions, {
populate: true,
})
}
@InjectManager("baseRepository_")
async listAndCountWorkflowExecution(
filters: WorkflowOrchestratorTypes.FilterableWorkflowExecutionProps = {},
config: FindConfig<WorkflowOrchestratorTypes.WorkflowExecutionDTO> = {},
@MedusaContext() sharedContext: Context = {}
): Promise<[WorkflowOrchestratorTypes.WorkflowExecutionDTO[], number]> {
const [wfExecutions, count] =
await this.workflowExecutionService_.listAndCount(
filters,
config,
sharedContext
)
return [
await this.baseRepository_.serialize<
WorkflowOrchestratorTypes.WorkflowExecutionDTO[]
>(wfExecutions, {
populate: true,
}),
count,
]
}
@InjectSharedContext()
async run<TWorkflow extends string | ReturnWorkflow<any, any, any>>(
workflowIdOrWorkflow: TWorkflow,
options: WorkflowOrchestratorTypes.WorkflowOrchestratorRunDTO<
TWorkflow extends ReturnWorkflow<any, any, any>
? UnwrapWorkflowInputDataType<TWorkflow>
: unknown
> = {},
@MedusaContext() context: Context = {}
) {
const ret = await this.workflowOrchestratorService_.run<
TWorkflow extends ReturnWorkflow<any, any, any>
? UnwrapWorkflowInputDataType<TWorkflow>
: unknown
>(workflowIdOrWorkflow, options, context)
return ret as any
}
@InjectSharedContext()
async getRunningTransaction(
workflowId: string,
transactionId: string,
@MedusaContext() context: Context = {}
) {
return this.workflowOrchestratorService_.getRunningTransaction(
workflowId,
transactionId,
context
)
}
@InjectSharedContext()
async setStepSuccess(
{
idempotencyKey,
stepResponse,
options,
}: {
idempotencyKey: string | object
stepResponse: unknown
options?: Record<string, any>
},
@MedusaContext() context: Context = {}
) {
return this.workflowOrchestratorService_.setStepSuccess(
{
idempotencyKey,
stepResponse,
options,
} as any,
context
)
}
@InjectSharedContext()
async setStepFailure(
{
idempotencyKey,
stepResponse,
options,
}: {
idempotencyKey: string | object
stepResponse: unknown
options?: Record<string, any>
},
@MedusaContext() context: Context = {}
) {
return this.workflowOrchestratorService_.setStepFailure(
{
idempotencyKey,
stepResponse,
options,
} as any,
context
)
}
@InjectSharedContext()
async subscribe(
args: {
workflowId: string
transactionId?: string
subscriber: Function
subscriberId?: string
},
@MedusaContext() context: Context = {}
) {
return this.workflowOrchestratorService_.subscribe(args as any, context)
}
@InjectSharedContext()
async unsubscribe(
args: {
workflowId: string
transactionId?: string
subscriberOrId: string | Function
},
@MedusaContext() context: Context = {}
) {
return this.workflowOrchestratorService_.unsubscribe(args as any, context)
}
}
@@ -0,0 +1,5 @@
import { Logger } from "@medusajs/types"
export type InitializeModuleInjectableDependencies = {
logger?: Logger
}
@@ -0,0 +1 @@
export * from "./workflow-orchestrator-storage"
@@ -0,0 +1,201 @@
import {
DistributedTransaction,
DistributedTransactionStorage,
TransactionCheckpoint,
TransactionStep,
} from "@medusajs/orchestration"
import { TransactionState } from "@medusajs/utils"
import {
WorkflowExecutionService,
WorkflowOrchestratorService,
} from "@services"
// eslint-disable-next-line max-len
export class InMemoryDistributedTransactionStorage extends DistributedTransactionStorage {
private workflowExecutionService_: WorkflowExecutionService
private workflowOrchestratorService_: WorkflowOrchestratorService
private storage: Map<string, TransactionCheckpoint> = new Map()
private retries: Map<string, unknown> = new Map()
private timeouts: Map<string, unknown> = new Map()
constructor({
workflowExecutionService,
}: {
workflowExecutionService: WorkflowExecutionService
}) {
super()
this.workflowExecutionService_ = workflowExecutionService
}
setWorkflowOrchestratorService(workflowOrchestratorService) {
this.workflowOrchestratorService_ = workflowOrchestratorService
}
private async saveToDb(data: TransactionCheckpoint) {
await this.workflowExecutionService_.upsert([
{
workflow_id: data.flow.modelId,
transaction_id: data.flow.transactionId,
execution: data.flow,
context: {
data: data.context,
errors: data.errors,
},
state: data.flow.state,
},
])
}
private async deleteFromDb(data: TransactionCheckpoint) {
await this.workflowExecutionService_.delete([
{
workflow_id: data.flow.modelId,
transaction_id: data.flow.transactionId,
},
])
}
async get(key: string): Promise<TransactionCheckpoint | undefined> {
return this.storage.get(key)
}
async list(): Promise<TransactionCheckpoint[]> {
return Array.from(this.storage.values())
}
async save(
key: string,
data: TransactionCheckpoint,
ttl?: number
): Promise<void> {
this.storage.set(key, data)
let retentionTime
/**
* Store the retention time only if the transaction is done, failed or reverted.
* From that moment, this tuple can be later on archived or deleted after the retention time.
*/
const hasFinished = [
TransactionState.DONE,
TransactionState.FAILED,
TransactionState.REVERTED,
].includes(data.flow.state)
if (hasFinished) {
retentionTime = data.flow.options?.retentionTime
Object.assign(data, {
retention_time: retentionTime,
})
}
if (hasFinished && !retentionTime) {
await this.deleteFromDb(data)
} else {
await this.saveToDb(data)
}
if (hasFinished) {
this.storage.delete(key)
}
}
async scheduleRetry(
transaction: DistributedTransaction,
step: TransactionStep,
timestamp: number,
interval: number
): Promise<void> {
const { modelId: workflowId, transactionId } = transaction
const inter = setTimeout(async () => {
await this.workflowOrchestratorService_.run(workflowId, {
transactionId,
throwOnError: false,
})
}, interval * 1e3)
const key = `${workflowId}:${transactionId}:${step.id}`
this.retries.set(key, inter)
}
async clearRetry(
transaction: DistributedTransaction,
step: TransactionStep
): Promise<void> {
const { modelId: workflowId, transactionId } = transaction
const key = `${workflowId}:${transactionId}:${step.id}`
const inter = this.retries.get(key)
if (inter) {
clearTimeout(inter as NodeJS.Timeout)
this.retries.delete(key)
}
}
async scheduleTransactionTimeout(
transaction: DistributedTransaction,
timestamp: number,
interval: number
): Promise<void> {
const { modelId: workflowId, transactionId } = transaction
const inter = setTimeout(async () => {
await this.workflowOrchestratorService_.run(workflowId, {
transactionId,
throwOnError: false,
})
}, interval * 1e3)
const key = `${workflowId}:${transactionId}`
this.timeouts.set(key, inter)
}
async clearTransactionTimeout(
transaction: DistributedTransaction
): Promise<void> {
const { modelId: workflowId, transactionId } = transaction
const key = `${workflowId}:${transactionId}`
const inter = this.timeouts.get(key)
if (inter) {
clearTimeout(inter as NodeJS.Timeout)
this.timeouts.delete(key)
}
}
async scheduleStepTimeout(
transaction: DistributedTransaction,
step: TransactionStep,
timestamp: number,
interval: number
): Promise<void> {
const { modelId: workflowId, transactionId } = transaction
const inter = setTimeout(async () => {
await this.workflowOrchestratorService_.run(workflowId, {
transactionId,
throwOnError: false,
})
}, interval * 1e3)
const key = `${workflowId}:${transactionId}:${step.id}`
this.timeouts.set(key, inter)
}
async clearStepTimeout(
transaction: DistributedTransaction,
step: TransactionStep
): Promise<void> {
const { modelId: workflowId, transactionId } = transaction
const key = `${workflowId}:${transactionId}:${step.id}`
const inter = this.timeouts.get(key)
if (inter) {
clearTimeout(inter as NodeJS.Timeout)
this.timeouts.delete(key)
}
}
}
@@ -0,0 +1,38 @@
{
"compilerOptions": {
"lib": ["es2020"],
"target": "es2020",
"outDir": "./dist",
"esModuleInterop": true,
"declarationMap": true,
"declaration": true,
"module": "commonjs",
"moduleResolution": "node",
"emitDecoratorMetadata": true,
"experimentalDecorators": true,
"sourceMap": false,
"noImplicitReturns": true,
"strictNullChecks": true,
"strictFunctionTypes": true,
"noImplicitThis": true,
"allowJs": true,
"skipLibCheck": true,
"downlevelIteration": true, // to use ES5 specific tooling
"baseUrl": ".",
"resolveJsonModule": true,
"paths": {
"@models": ["./src/models"],
"@services": ["./src/services"],
"@repositories": ["./src/repositories"],
"@types": ["./src/types"]
}
},
"include": ["src"],
"exclude": [
"dist",
"./src/**/__tests__",
"./src/**/__mocks__",
"./src/**/__fixtures__",
"node_modules"
]
}
@@ -0,0 +1,8 @@
{
"extends": "./tsconfig.json",
"include": ["src", "integration-tests"],
"exclude": ["node_modules", "dist"],
"compilerOptions": {
"sourceMap": true
}
}
@@ -0,0 +1,6 @@
/dist
node_modules
.DS_store
.env*
.env
*.sql
+1
View File
@@ -0,0 +1 @@
# Workflow Orchestrator
@@ -0,0 +1,4 @@
export * from "./workflow_1"
export * from "./workflow_2"
export * from "./workflow_step_timeout"
export * from "./workflow_transaction_timeout"
@@ -0,0 +1,65 @@
import {
StepResponse,
createStep,
createWorkflow,
} from "@medusajs/workflows-sdk"
const step_1 = createStep(
"step_1",
jest.fn((input) => {
input.test = "test"
return new StepResponse(input, { compensate: 123 })
}),
jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
console.log("reverted", compensateInput.compensate)
return new StepResponse({
reverted: true,
})
})
)
const step_2 = createStep(
"step_2",
jest.fn((input, context) => {
console.log("triggered async request", context.metadata.idempotency_key)
if (input) {
return new StepResponse({ notAsyncResponse: input.hey })
}
}),
jest.fn((_, context) => {
return new StepResponse({
step: context.metadata.action,
idempotency_key: context.metadata.idempotency_key,
reverted: true,
})
})
)
const step_3 = createStep(
"step_3",
jest.fn((res) => {
return new StepResponse({
done: {
inputFromSyncStep: res.notAsyncResponse,
},
})
})
)
createWorkflow("workflow_1", function (input) {
step_1(input)
const ret2 = step_2({ hey: "oh" })
step_2({ hey: "async hello" }).config({
name: "new_step_name",
async: true,
})
return step_3(ret2)
})
@@ -0,0 +1,71 @@
import {
StepResponse,
createStep,
createWorkflow,
} from "@medusajs/workflows-sdk"
const step_1 = createStep(
"step_1",
jest.fn((input) => {
input.test = "test"
return new StepResponse(input, { compensate: 123 })
}),
jest.fn((compensateInput) => {
if (!compensateInput) {
return
}
console.log("reverted", compensateInput.compensate)
return new StepResponse({
reverted: true,
})
})
)
const step_2 = createStep(
"step_2",
jest.fn((input, context) => {
console.log("triggered async request", context.metadata.idempotency_key)
if (input) {
return new StepResponse({ notAsyncResponse: input.hey })
}
}),
jest.fn((_, context) => {
return new StepResponse({
step: context.metadata.action,
idempotency_key: context.metadata.idempotency_key,
reverted: true,
})
})
)
const step_3 = createStep(
"step_3",
jest.fn((res) => {
return new StepResponse({
done: {
inputFromSyncStep: res.notAsyncResponse,
},
})
})
)
createWorkflow(
{
name: "workflow_2",
retentionTime: 1000,
},
function (input) {
step_1(input)
const ret2 = step_2({ hey: "oh" })
step_2({ hey: "async hello" }).config({
name: "new_step_name",
async: true,
})
return step_3(ret2)
}
)
@@ -0,0 +1,51 @@
import {
StepResponse,
createStep,
createWorkflow,
} from "@medusajs/workflows-sdk"
import { setTimeout } from "timers/promises"
const step_1 = createStep(
"step_1",
jest.fn(async (input) => {
await setTimeout(200)
return new StepResponse(input, { compensate: 123 })
})
)
const step_1_async = createStep(
{
name: "step_1_async",
async: true,
timeout: 0.1, // 0.1 second
},
jest.fn(async (input) => {
return new StepResponse(input, { compensate: 123 })
})
)
createWorkflow(
{
name: "workflow_step_timeout",
},
function (input) {
const resp = step_1(input).config({
timeout: 0.1, // 0.1 second
})
return resp
}
)
createWorkflow(
{
name: "workflow_step_timeout_async",
},
function (input) {
const resp = step_1_async(input)
return resp
}
)
@@ -0,0 +1,44 @@
import {
StepResponse,
createStep,
createWorkflow,
} from "@medusajs/workflows-sdk"
import { setTimeout } from "timers/promises"
const step_1 = createStep(
"step_1",
jest.fn(async (input) => {
await setTimeout(200)
return new StepResponse({
executed: true,
})
}),
jest.fn()
)
createWorkflow(
{
name: "workflow_transaction_timeout",
timeout: 0.1, // 0.1 second
},
function (input) {
const resp = step_1(input)
return resp
}
)
createWorkflow(
{
name: "workflow_transaction_timeout_async",
timeout: 0.1, // 0.1 second
},
function (input) {
const resp = step_1(input).config({
async: true,
})
return resp
}
)
@@ -0,0 +1,245 @@
import { MedusaApp } from "@medusajs/modules-sdk"
import {
TransactionStepTimeoutError,
TransactionTimeoutError,
} from "@medusajs/orchestration"
import { RemoteJoinerQuery } from "@medusajs/types"
import { TransactionHandlerType } from "@medusajs/utils"
import { IWorkflowsModuleService } from "@medusajs/workflows-sdk"
import { knex } from "knex"
import { setTimeout } from "timers/promises"
import "../__fixtures__"
import { DB_URL, TestDatabase } from "../utils"
const sharedPgConnection = knex<any, any>({
client: "pg",
searchPath: process.env.MEDUSA_WORKFLOW_ENGINE_DB_SCHEMA,
connection: {
connectionString: DB_URL,
debug: false,
},
})
const afterEach_ = async () => {
await TestDatabase.clearTables(sharedPgConnection)
}
describe("Workflow Orchestrator module", function () {
describe("Testing basic workflow", function () {
let workflowOrcModule: IWorkflowsModuleService
let query: (
query: string | RemoteJoinerQuery | object,
variables?: Record<string, unknown>
) => Promise<any>
afterEach(afterEach_)
beforeAll(async () => {
const {
runMigrations,
query: remoteQuery,
modules,
} = await MedusaApp({
sharedResourcesConfig: {
database: {
connection: sharedPgConnection,
},
},
modulesConfig: {
workflows: {
resolve: __dirname + "/../..",
options: {
redis: {
url: "localhost:6379",
},
},
},
},
})
query = remoteQuery
await runMigrations()
workflowOrcModule =
modules.workflows as unknown as IWorkflowsModuleService
})
afterEach(afterEach_)
it("should return a list of workflow executions and remove after completed when there is no retentionTime set", async () => {
await workflowOrcModule.run("workflow_1", {
input: {
value: "123",
},
throwOnError: true,
})
let executionsList = await query({
workflow_executions: {
fields: ["workflow_id", "transaction_id", "state"],
},
})
expect(executionsList).toHaveLength(1)
const { result } = await workflowOrcModule.setStepSuccess({
idempotencyKey: {
action: TransactionHandlerType.INVOKE,
stepId: "new_step_name",
workflowId: "workflow_1",
transactionId: executionsList[0].transaction_id,
},
stepResponse: { uhuuuu: "yeaah!" },
})
executionsList = await query({
workflow_executions: {
fields: ["id"],
},
})
expect(executionsList).toHaveLength(0)
expect(result).toEqual({
done: {
inputFromSyncStep: "oh",
},
})
})
it("should return a list of workflow executions and keep it saved when there is a retentionTime set", async () => {
await workflowOrcModule.run("workflow_2", {
input: {
value: "123",
},
throwOnError: true,
transactionId: "transaction_1",
})
let executionsList = await query({
workflow_executions: {
fields: ["id"],
},
})
expect(executionsList).toHaveLength(1)
await workflowOrcModule.setStepSuccess({
idempotencyKey: {
action: TransactionHandlerType.INVOKE,
stepId: "new_step_name",
workflowId: "workflow_2",
transactionId: "transaction_1",
},
stepResponse: { uhuuuu: "yeaah!" },
})
executionsList = await query({
workflow_executions: {
fields: ["id"],
},
})
expect(executionsList).toHaveLength(1)
})
it("should revert the entire transaction when a step timeout expires", async () => {
const { transaction, result, errors } = await workflowOrcModule.run(
"workflow_step_timeout",
{
input: {
myInput: "123",
},
throwOnError: false,
}
)
expect(transaction.flow.state).toEqual("reverted")
expect(result).toEqual({
myInput: "123",
})
expect(errors).toHaveLength(1)
expect(errors[0].action).toEqual("step_1")
expect(errors[0].error).toBeInstanceOf(TransactionStepTimeoutError)
})
it("should revert the entire transaction when the transaction timeout expires", async () => {
const { transaction, result, errors } = await workflowOrcModule.run(
"workflow_transaction_timeout",
{
input: {},
transactionId: "trx",
throwOnError: false,
}
)
expect(transaction.flow.state).toEqual("reverted")
expect(result).toEqual({ executed: true })
expect(errors).toHaveLength(1)
expect(errors[0].action).toEqual("step_1")
expect(
TransactionTimeoutError.isTransactionTimeoutError(errors[0].error)
).toBe(true)
})
it("should revert the entire transaction when a step timeout expires in a async step", async () => {
await workflowOrcModule.run("workflow_step_timeout_async", {
input: {
myInput: "123",
},
transactionId: "transaction_1",
throwOnError: false,
})
await setTimeout(200)
const { transaction, result, errors } = await workflowOrcModule.run(
"workflow_step_timeout_async",
{
input: {
myInput: "123",
},
transactionId: "transaction_1",
throwOnError: false,
}
)
expect(transaction.flow.state).toEqual("reverted")
expect(result).toEqual(undefined)
expect(errors).toHaveLength(1)
expect(errors[0].action).toEqual("step_1_async")
expect(
TransactionStepTimeoutError.isTransactionStepTimeoutError(
errors[0].error
)
).toBe(true)
})
it("should revert the entire transaction when the transaction timeout expires in a transaction containing an async step", async () => {
await workflowOrcModule.run("workflow_transaction_timeout_async", {
input: {},
transactionId: "transaction_1",
throwOnError: false,
})
await setTimeout(200)
const { transaction, result, errors } = await workflowOrcModule.run(
"workflow_transaction_timeout_async",
{
input: {},
transactionId: "transaction_1",
throwOnError: false,
}
)
expect(transaction.flow.state).toEqual("reverted")
expect(result).toEqual(undefined)
expect(errors).toHaveLength(1)
expect(errors[0].action).toEqual("step_1")
expect(
TransactionTimeoutError.isTransactionTimeoutError(errors[0].error)
).toBe(true)
})
})
})
@@ -0,0 +1,6 @@
if (typeof process.env.DB_TEMP_NAME === "undefined") {
const tempName = parseInt(process.env.JEST_WORKER_ID || "1")
process.env.DB_TEMP_NAME = `medusa-workflow-engine-redis-${tempName}`
}
process.env.MEDUSA_WORKFLOW_ENGINE_DB_SCHEMA = "public"
@@ -0,0 +1,3 @@
import { JestUtils } from "medusa-test-utils"
JestUtils.afterAllHookDropDatabase()
@@ -0,0 +1,53 @@
import * as process from "process"
const DB_HOST = process.env.DB_HOST ?? "localhost"
const DB_USERNAME = process.env.DB_USERNAME ?? ""
const DB_PASSWORD = process.env.DB_PASSWORD
const DB_NAME = process.env.DB_TEMP_NAME
export const DB_URL = `postgres://${DB_USERNAME}${
DB_PASSWORD ? `:${DB_PASSWORD}` : ""
}@${DB_HOST}/${DB_NAME}`
const Redis = require("ioredis")
const redisUrl = process.env.REDIS_URL || "redis://localhost:6379"
const redis = new Redis(redisUrl)
interface TestDatabase {
clearTables(knex): Promise<void>
}
export const TestDatabase: TestDatabase = {
clearTables: async (knex) => {
await knex.raw(`
TRUNCATE TABLE workflow_execution CASCADE;
`)
await cleanRedis()
},
}
async function deleteKeysByPattern(pattern) {
const stream = redis.scanStream({
match: pattern,
count: 100,
})
for await (const keys of stream) {
if (keys.length) {
const pipeline = redis.pipeline()
keys.forEach((key) => pipeline.del(key))
await pipeline.exec()
}
}
}
async function cleanRedis() {
try {
await deleteKeysByPattern("bull:*")
await deleteKeysByPattern("dtrans:*")
} catch (error) {
console.error("Error:", error)
}
}
@@ -0,0 +1 @@
export * from "./database"
@@ -0,0 +1,21 @@
module.exports = {
moduleNameMapper: {
"^@models": "<rootDir>/src/models",
"^@services": "<rootDir>/src/services",
"^@repositories": "<rootDir>/src/repositories",
},
transform: {
"^.+\\.[jt]s?$": [
"ts-jest",
{
tsConfig: "tsconfig.spec.json",
isolatedModules: true,
},
],
},
testEnvironment: `node`,
moduleFileExtensions: [`js`, `ts`],
modulePathIgnorePatterns: ["dist/"],
setupFiles: ["<rootDir>/integration-tests/setup-env.js"],
setupFilesAfterEnv: ["<rootDir>/integration-tests/setup.js"],
}
@@ -0,0 +1,8 @@
import * as entities from "./src/models"
module.exports = {
entities: Object.values(entities),
schema: "public",
clientUrl: "postgres://postgres@localhost/medusa-workflow-engine-redis",
type: "postgresql",
}
@@ -0,0 +1,61 @@
{
"name": "@medusajs/workflow-engine-redis",
"version": "0.0.1",
"description": "Medusa Workflow Orchestrator module using Redis to track workflows executions",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"files": [
"dist"
],
"engines": {
"node": ">=16"
},
"repository": {
"type": "git",
"url": "https://github.com/medusajs/medusa",
"directory": "packages/workflow-engine-redis"
},
"publishConfig": {
"access": "public"
},
"author": "Medusa",
"license": "MIT",
"scripts": {
"watch": "tsc --build --watch",
"watch:test": "tsc --build tsconfig.spec.json --watch",
"prepublishOnly": "cross-env NODE_ENV=production tsc --build && tsc-alias -p tsconfig.json",
"build": "rimraf dist && tsc --build && tsc-alias -p tsconfig.json",
"test": "jest --passWithNoTests --runInBand --bail --forceExit -- src/**/__tests__/**/*.ts",
"test:integration": "jest --runInBand --forceExit -- integration-tests/**/__tests__/**/*.ts",
"migration:generate": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:generate",
"migration:initial": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:create --initial",
"migration:create": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:create",
"migration:up": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:up",
"orm:cache:clear": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm cache:clear"
},
"devDependencies": {
"@mikro-orm/cli": "5.9.7",
"cross-env": "^5.2.1",
"jest": "^29.6.3",
"medusa-test-utils": "^1.1.40",
"rimraf": "^3.0.2",
"ts-jest": "^29.1.1",
"ts-node": "^10.9.1",
"tsc-alias": "^1.8.6",
"typescript": "^5.1.6"
},
"dependencies": {
"@medusajs/modules-sdk": "^1.12.5",
"@medusajs/types": "^1.11.9",
"@medusajs/utils": "^1.11.2",
"@medusajs/workflows-sdk": "^0.1.0",
"@mikro-orm/core": "5.9.7",
"@mikro-orm/migrations": "5.9.7",
"@mikro-orm/postgresql": "5.9.7",
"awilix": "^8.0.0",
"bullmq": "^5.1.3",
"dotenv": "^16.1.4",
"ioredis": "^5.3.2",
"knex": "2.4.2"
}
}
@@ -0,0 +1,22 @@
import { Modules } from "@medusajs/modules-sdk"
import { ModulesSdkUtils } from "@medusajs/utils"
import * as models from "@models"
import { moduleDefinition } from "./module-definition"
export default moduleDefinition
const migrationScriptOptions = {
moduleName: Modules.WORKFLOW_ENGINE,
models: models,
pathToMigrations: __dirname + "/migrations",
}
export const runMigrations = ModulesSdkUtils.buildMigrationScript(
migrationScriptOptions
)
export const revertMigration = ModulesSdkUtils.buildRevertMigrationScript(
migrationScriptOptions
)
export * from "./initialize"
export * from "./loaders"
@@ -0,0 +1,36 @@
import {
ExternalModuleDeclaration,
InternalModuleDeclaration,
MedusaModule,
MODULE_PACKAGE_NAMES,
Modules,
} from "@medusajs/modules-sdk"
import { ModulesSdkTypes } from "@medusajs/types"
import { WorkflowOrchestratorTypes } from "@medusajs/workflows-sdk"
import { moduleDefinition } from "../module-definition"
import { InitializeModuleInjectableDependencies } from "../types"
export const initialize = async (
options?:
| ModulesSdkTypes.ModuleServiceInitializeOptions
| ModulesSdkTypes.ModuleServiceInitializeCustomDataLayerOptions
| ExternalModuleDeclaration
| InternalModuleDeclaration,
injectedDependencies?: InitializeModuleInjectableDependencies
): Promise<WorkflowOrchestratorTypes.IWorkflowsModuleService> => {
const loaded =
// eslint-disable-next-line max-len
await MedusaModule.bootstrap<WorkflowOrchestratorTypes.IWorkflowsModuleService>(
{
moduleKey: Modules.WORKFLOW_ENGINE,
defaultPath: MODULE_PACKAGE_NAMES[Modules.WORKFLOW_ENGINE],
declaration: options as
| InternalModuleDeclaration
| ExternalModuleDeclaration,
injectedDependencies,
moduleExports: moduleDefinition,
}
)
return loaded[Modules.WORKFLOW_ENGINE]
}
@@ -0,0 +1,34 @@
import { Modules } from "@medusajs/modules-sdk"
import { ModuleJoinerConfig } from "@medusajs/types"
import { MapToConfig } from "@medusajs/utils"
import { WorkflowExecution } from "@models"
import moduleSchema from "./schema"
export const LinkableKeys = {
workflow_execution_id: WorkflowExecution.name,
}
const entityLinkableKeysMap: MapToConfig = {}
Object.entries(LinkableKeys).forEach(([key, value]) => {
entityLinkableKeysMap[value] ??= []
entityLinkableKeysMap[value].push({
mapTo: key,
valueFrom: key.split("_").pop()!,
})
})
export const entityNameToLinkableKeysMap: MapToConfig = entityLinkableKeysMap
export const joinerConfig: ModuleJoinerConfig = {
serviceName: Modules.WORKFLOW_ENGINE,
primaryKeys: ["id"],
schema: moduleSchema,
linkableKeys: LinkableKeys,
alias: {
name: ["workflow_execution", "workflow_executions"],
args: {
entity: WorkflowExecution.name,
methodSuffix: "WorkflowExecution",
},
},
}
@@ -0,0 +1,36 @@
import {
InternalModuleDeclaration,
LoaderOptions,
Modules,
} from "@medusajs/modules-sdk"
import { ModulesSdkTypes } from "@medusajs/types"
import { ModulesSdkUtils } from "@medusajs/utils"
import { EntitySchema } from "@mikro-orm/core"
import * as WorkflowOrchestratorModels from "../models"
export default async (
{
options,
container,
logger,
}: LoaderOptions<
| ModulesSdkTypes.ModuleServiceInitializeOptions
| ModulesSdkTypes.ModuleServiceInitializeCustomDataLayerOptions
>,
moduleDeclaration?: InternalModuleDeclaration
): Promise<void> => {
const entities = Object.values(
WorkflowOrchestratorModels
) as unknown as EntitySchema[]
const pathToMigrations = __dirname + "/../migrations"
await ModulesSdkUtils.mikroOrmConnectionLoader({
moduleName: Modules.WORKFLOW_ENGINE,
entities,
container,
options,
moduleDeclaration,
logger,
pathToMigrations,
})
}
@@ -0,0 +1,9 @@
import { MikroOrmBaseRepository, ModulesSdkUtils } from "@medusajs/utils"
import * as ModuleModels from "@models"
import * as ModuleServices from "@services"
export default ModulesSdkUtils.moduleContainerLoaderFactory({
moduleModels: ModuleModels,
moduleServices: ModuleServices,
moduleRepositories: { BaseRepository: MikroOrmBaseRepository },
})
@@ -0,0 +1,4 @@
export * from "./connection"
export * from "./container"
export * from "./redis"
export * from "./utils"
@@ -0,0 +1,78 @@
import { LoaderOptions } from "@medusajs/modules-sdk"
import { asValue } from "awilix"
import Redis from "ioredis"
import { RedisWorkflowsOptions } from "../types"
export default async ({
container,
logger,
options,
}: LoaderOptions): Promise<void> => {
const {
url,
options: redisOptions,
pubsub,
} = options?.redis as RedisWorkflowsOptions
// TODO: get default from ENV VAR
if (!url) {
throw Error(
"No `redis.url` provided in `workflowOrchestrator` module options. It is required for the Workflow Orchestrator Redis."
)
}
const cnnPubSub = pubsub ?? { url, options: redisOptions }
const queueName = options?.queueName ?? "medusa-workflows"
let connection
let redisPublisher
let redisSubscriber
let workerConnection
try {
connection = await getConnection(url, redisOptions)
workerConnection = await getConnection(url, {
...(redisOptions ?? {}),
maxRetriesPerRequest: null,
})
logger?.info(
`Connection to Redis in module 'workflow-engine-redis' established`
)
} catch (err) {
logger?.error(
`An error occurred while connecting to Redis in module 'workflow-engine-redis': ${err}`
)
}
try {
redisPublisher = await getConnection(cnnPubSub.url, cnnPubSub.options)
redisSubscriber = await getConnection(cnnPubSub.url, cnnPubSub.options)
logger?.info(
`Connection to Redis PubSub in module 'workflow-engine-redis' established`
)
} catch (err) {
logger?.error(
`An error occurred while connecting to Redis PubSub in module 'workflow-engine-redis': ${err}`
)
}
container.register({
redisConnection: asValue(connection),
redisWorkerConnection: asValue(workerConnection),
redisPublisher: asValue(redisPublisher),
redisSubscriber: asValue(redisSubscriber),
redisQueueName: asValue(queueName),
})
}
async function getConnection(url, redisOptions) {
const connection = new Redis(url, {
lazyConnect: true,
...(redisOptions ?? {}),
})
await connection.connect()
return connection
}
@@ -0,0 +1,10 @@
import { asClass } from "awilix"
import { RedisDistributedTransactionStorage } from "../utils"
export default async ({ container }): Promise<void> => {
container.register({
redisDistributedTransactionStorage: asClass(
RedisDistributedTransactionStorage
).singleton(),
})
}
@@ -0,0 +1,41 @@
import { Migration } from "@mikro-orm/migrations"
export class Migration20231221104256 extends Migration {
async up(): Promise<void> {
this.addSql(
`
CREATE TABLE IF NOT EXISTS workflow_execution
(
id character varying NOT NULL,
workflow_id character varying NOT NULL,
transaction_id character varying NOT NULL,
execution jsonb NULL,
context jsonb NULL,
state character varying NOT NULL,
created_at timestamp WITHOUT time zone NOT NULL DEFAULT Now(),
updated_at timestamp WITHOUT time zone NOT NULL DEFAULT Now(),
deleted_at timestamp WITHOUT time zone NULL,
CONSTRAINT "PK_workflow_execution_workflow_id_transaction_id" PRIMARY KEY ("workflow_id", "transaction_id")
);
CREATE UNIQUE INDEX IF NOT EXISTS "IDX_workflow_execution_id" ON "workflow_execution" ("id");
CREATE INDEX IF NOT EXISTS "IDX_workflow_execution_workflow_id" ON "workflow_execution" ("workflow_id") WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS "IDX_workflow_execution_transaction_id" ON "workflow_execution" ("transaction_id") WHERE deleted_at IS NULL;
CREATE INDEX IF NOT EXISTS "IDX_workflow_execution_state" ON "workflow_execution" ("state") WHERE deleted_at IS NULL;
`
)
}
async down(): Promise<void> {
this.addSql(
`
DROP INDEX "IDX_workflow_execution_id";
DROP INDEX "IDX_workflow_execution_workflow_id";
DROP INDEX "IDX_workflow_execution_transaction_id";
DROP INDEX "IDX_workflow_execution_state";
DROP TABLE IF EXISTS workflow_execution;
`
)
}
}
@@ -0,0 +1 @@
export { default as WorkflowExecution } from "./workflow-execution"
@@ -0,0 +1,76 @@
import { TransactionState } from "@medusajs/orchestration"
import { DALUtils, generateEntityId } from "@medusajs/utils"
import {
BeforeCreate,
Entity,
Enum,
Filter,
Index,
OnInit,
OptionalProps,
PrimaryKey,
Property,
Unique,
} from "@mikro-orm/core"
type OptionalFields = "deleted_at"
@Entity()
@Unique({
name: "IDX_workflow_execution_workflow_id_transaction_id_unique",
properties: ["workflow_id", "transaction_id"],
})
@Filter(DALUtils.mikroOrmSoftDeletableFilterOptions)
export default class WorkflowExecution {
[OptionalProps]?: OptionalFields
@Property({ columnType: "text", nullable: false })
@Index({ name: "IDX_workflow_execution_id" })
id!: string
@Index({ name: "IDX_workflow_execution_workflow_id" })
@PrimaryKey({ columnType: "text" })
workflow_id: string
@Index({ name: "IDX_workflow_execution_transaction_id" })
@PrimaryKey({ columnType: "text" })
transaction_id: string
@Property({ columnType: "jsonb", nullable: true })
execution: Record<string, unknown> | null = null
@Property({ columnType: "jsonb", nullable: true })
context: Record<string, unknown> | null = null
@Index({ name: "IDX_workflow_execution_state" })
@Enum(() => TransactionState)
state: TransactionState
@Property({
onCreate: () => new Date(),
columnType: "timestamptz",
defaultRaw: "now()",
})
created_at: Date
@Property({
onCreate: () => new Date(),
onUpdate: () => new Date(),
columnType: "timestamptz",
defaultRaw: "now()",
})
updated_at: Date
@Property({ columnType: "timestamptz", nullable: true })
deleted_at: Date | null = null
@BeforeCreate()
onCreate() {
this.id = generateEntityId(this.id, "wf_exec")
}
@OnInit()
onInit() {
this.id = generateEntityId(this.id, "wf_exec")
}
}
@@ -0,0 +1,19 @@
import { ModuleExports } from "@medusajs/types"
import { WorkflowsModuleService } from "@services"
import loadConnection from "./loaders/connection"
import loadContainer from "./loaders/container"
import redisConnection from "./loaders/redis"
import loadUtils from "./loaders/utils"
const service = WorkflowsModuleService
const loaders = [
loadContainer,
loadConnection,
loadUtils,
redisConnection,
] as any
export const moduleDefinition: ModuleExports = {
service,
loaders,
}
@@ -0,0 +1,2 @@
export { MikroOrmBaseRepository as BaseRepository } from "@medusajs/utils"
export { WorkflowExecutionRepository } from "./workflow-execution"
@@ -0,0 +1,7 @@
import { DALUtils } from "@medusajs/utils"
import { WorkflowExecution } from "@models"
// eslint-disable-next-line max-len
export class WorkflowExecutionRepository extends DALUtils.mikroOrmBaseRepositoryFactory(
WorkflowExecution
) {}
@@ -0,0 +1,26 @@
export default `
scalar DateTime
scalar JSON
enum TransactionState {
NOT_STARTED
INVOKING
WAITING_TO_COMPENSATE
COMPENSATING
DONE
REVERTED
FAILED
}
type WorkflowExecution {
id: ID!
created_at: DateTime!
updated_at: DateTime!
deleted_at: DateTime
workflow_id: string
transaction_id: string
execution: JSON
context: JSON
state: TransactionState
}
`
@@ -0,0 +1,5 @@
describe("Noop test", () => {
it("noop check", async () => {
expect(true).toBe(true)
})
})
@@ -0,0 +1,3 @@
export * from "./workflow-execution"
export * from "./workflow-orchestrator"
export * from "./workflows-module"

Some files were not shown because too many files have changed in this diff Show More