diff --git a/.changeset/pink-balloons-search.md b/.changeset/pink-balloons-search.md new file mode 100644 index 0000000000..cab0d69c78 --- /dev/null +++ b/.changeset/pink-balloons-search.md @@ -0,0 +1,11 @@ +--- +"@medusajs/workflow-engine-inmemory": patch +"@medusajs/workflow-engine-redis": patch +"@medusajs/orchestration": patch +"@medusajs/workflows-sdk": patch +"@medusajs/modules-sdk": patch +"@medusajs/types": patch +"@medusajs/utils": patch +--- + +Modules: Workflows Engine in-memory and Redis diff --git a/.eslintignore b/.eslintignore index d05cda3f13..2af245f62d 100644 --- a/.eslintignore +++ b/.eslintignore @@ -24,6 +24,9 @@ packages/* !packages/orchestration !packages/workflows-sdk !packages/core-flows +!packages/workflow-engine-redis +!packages/workflow-engine-inmemory + **/models/* diff --git a/.eslintrc.js b/.eslintrc.js index 22a44eb628..caafd0d9f6 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -72,9 +72,7 @@ module.exports = { node: true, jest: true, }, - ignorePatterns: [ - "packages/admin-next/dashboard/**/dist" - ], + ignorePatterns: ["packages/admin-next/dashboard/**/dist"], overrides: [ { files: ["*.ts"], @@ -101,6 +99,8 @@ module.exports = { "./packages/orchestration/tsconfig.json", "./packages/workflows-sdk/tsconfig.spec.json", "./packages/core-flows/tsconfig.spec.json", + "./packages/workflow-engine-redis/tsconfig.spec.json", + "./packages/workflow-engine-inmemory/tsconfig.spec.json", ], }, rules: { diff --git a/package.json b/package.json index 336ea9e9a3..b4a476e036 100644 --- a/package.json +++ b/package.json @@ -65,15 +65,15 @@ "scripts": { "hooks:install": "husky install", "hooks:uninstall": "husky uninstall", - "build": "turbo run build --no-daemon", + "build": "turbo run build --concurrency=50% --no-daemon", "lint": "eslint --ignore-path .eslintignore --ext .js,.ts,.tsx .", "prettier": "prettier", "jest": "jest", - "test": "turbo run test --no-daemon", - "test:integration:packages": "turbo run test:integration --no-daemon --filter='./packages/*'", - "test:integration:api": "turbo run test:integration --no-daemon --filter=integration-tests-api", - "test:integration:plugins": "turbo run test:integration --no-daemon --filter=integration-tests-plugins", - "test:integration:repositories": "turbo run test:integration --no-daemon --filter=integration-tests-repositories", + "test": "turbo run test --concurrency=50% --no-daemon", + "test:integration:packages": "turbo run test:integration --concurrency=1 --no-daemon --filter='./packages/*'", + "test:integration:api": "turbo run test:integration --concurrency=50% --no-daemon --filter=integration-tests-api", + "test:integration:plugins": "turbo run test:integration --concurrency=50% --no-daemon --filter=integration-tests-plugins", + "test:integration:repositories": "turbo run test:integration --concurrency=50% --no-daemon --filter=integration-tests-repositories", "openapi:generate": "yarn ./packages/oas/oas-github-ci run ci --with-full-file", "medusa-oas": "yarn ./packages/oas/medusa-oas-cli run medusa-oas", "release:snapshot": "changeset publish --no-git-tags --snapshot --tag snapshot", diff --git a/packages/medusa/src/api/routes/admin/index.js b/packages/medusa/src/api/routes/admin/index.js index 855e5ed084..da882ef408 100644 --- a/packages/medusa/src/api/routes/admin/index.js +++ b/packages/medusa/src/api/routes/admin/index.js @@ -1,5 +1,6 @@ import cors from "cors" import { Router } from "express" +import { parseCorsOrigins } from "medusa-core-utils" import middlewares from "../../middlewares" import analyticsConfigs from "./analytics-configs" import appRoutes from "./apps" @@ -18,16 +19,18 @@ import noteRoutes from "./notes" import notificationRoutes from "./notifications" import orderEditRoutes from "./order-edits" import orderRoutes from "./orders" +import paymentCollectionRoutes from "./payment-collections" +import paymentRoutes from "./payments" import priceListRoutes from "./price-lists" +import productCategoryRoutes from "./product-categories" import productTagRoutes from "./product-tags" import productTypesRoutes from "./product-types" -import publishableApiKeyRoutes from "./publishable-api-keys" import productRoutes from "./products" +import publishableApiKeyRoutes from "./publishable-api-keys" import regionRoutes from "./regions" import reservationRoutes from "./reservations" import returnReasonRoutes from "./return-reasons" import returnRoutes from "./returns" -import reservationRoutes from "./reservations" import salesChannelRoutes from "./sales-channels" import shippingOptionRoutes from "./shipping-options" import shippingProfileRoutes from "./shipping-profiles" @@ -38,10 +41,6 @@ import taxRateRoutes from "./tax-rates" import uploadRoutes from "./uploads" import userRoutes, { unauthenticatedUserRoutes } from "./users" import variantRoutes from "./variants" -import paymentCollectionRoutes from "./payment-collections" -import paymentRoutes from "./payments" -import productCategoryRoutes from "./product-categories" -import { parseCorsOrigins } from "medusa-core-utils" const route = Router() diff --git a/packages/medusa/src/loaders/helpers/routing/__fixtures__/server/index.js b/packages/medusa/src/loaders/helpers/routing/__fixtures__/server/index.js index 11f5934140..6f13e301cb 100644 --- a/packages/medusa/src/loaders/helpers/routing/__fixtures__/server/index.js +++ b/packages/medusa/src/loaders/helpers/routing/__fixtures__/server/index.js @@ -3,6 +3,7 @@ import { ModulesDefinition, registerMedusaModule, } from "@medusajs/modules-sdk" +import { ContainerRegistrationKeys } from "@medusajs/utils" import { asValue, createContainer } from "awilix" import express from "express" import jwt from "jsonwebtoken" @@ -63,6 +64,7 @@ export const createServer = async (rootDir) => { return this }.bind(container) + container.register(ContainerRegistrationKeys.PG_CONNECTION, asValue({})) container.register("featureFlagRouter", asValue(featureFlagRouter)) container.register("configModule", asValue(config)) container.register({ diff --git a/packages/modules-sdk/medusajs-modules-sdk-1.12.6.tgz b/packages/modules-sdk/medusajs-modules-sdk-1.12.6.tgz new file mode 100644 index 0000000000..968452f203 Binary files /dev/null and b/packages/modules-sdk/medusajs-modules-sdk-1.12.6.tgz differ diff --git a/packages/modules-sdk/src/definitions.ts b/packages/modules-sdk/src/definitions.ts index 59e4c71fed..4be77eb5e3 100644 --- a/packages/modules-sdk/src/definitions.ts +++ b/packages/modules-sdk/src/definitions.ts @@ -16,6 +16,7 @@ export enum Modules { PRICING = "pricingService", PROMOTION = "promotion", AUTHENTICATION = "authentication", + WORKFLOW_ENGINE = "workflows", CART = "cart", CUSTOMER = "customer", PAYMENT = "payment", @@ -30,6 +31,7 @@ export enum ModuleRegistrationName { PRICING = "pricingModuleService", PROMOTION = "promotionModuleService", AUTHENTICATION = "authenticationModuleService", + WORKFLOW_ENGINE = "workflowsModuleService", CART = "cartModuleService", CUSTOMER = "customerModuleService", PAYMENT = "paymentModuleService", @@ -45,6 +47,7 @@ export const MODULE_PACKAGE_NAMES = { [Modules.PRICING]: "@medusajs/pricing", [Modules.PROMOTION]: "@medusajs/promotion", [Modules.AUTHENTICATION]: "@medusajs/authentication", + [Modules.WORKFLOW_ENGINE]: "@medusajs/workflow-engine-inmemory", [Modules.CART]: "@medusajs/cart", [Modules.CUSTOMER]: "@medusajs/customer", [Modules.PAYMENT]: "@medusajs/payment", @@ -165,6 +168,20 @@ export const ModulesDefinition: { [key: string | Modules]: ModuleDefinition } = resources: MODULE_RESOURCE_TYPE.SHARED, }, }, + [Modules.WORKFLOW_ENGINE]: { + key: Modules.WORKFLOW_ENGINE, + registrationName: ModuleRegistrationName.WORKFLOW_ENGINE, + defaultPackage: false, + label: upperCaseFirst(ModuleRegistrationName.WORKFLOW_ENGINE), + isRequired: false, + canOverride: true, + isQueryable: true, + dependencies: ["logger"], + defaultModuleDeclaration: { + scope: MODULE_SCOPE.INTERNAL, + resources: MODULE_RESOURCE_TYPE.SHARED, + }, + }, [Modules.CART]: { key: Modules.CART, registrationName: ModuleRegistrationName.CART, diff --git a/packages/modules-sdk/src/medusa-app.ts b/packages/modules-sdk/src/medusa-app.ts index 4359cd5b53..fff2bf0fec 100644 --- a/packages/modules-sdk/src/medusa-app.ts +++ b/packages/modules-sdk/src/medusa-app.ts @@ -245,6 +245,12 @@ export async function MedusaApp({ registerCustomJoinerConfigs(servicesConfig ?? []) if ( + sharedResourcesConfig?.database?.connection && + !injectedDependencies[ContainerRegistrationKeys.PG_CONNECTION] + ) { + injectedDependencies[ContainerRegistrationKeys.PG_CONNECTION] = + sharedResourcesConfig.database.connection + } else if ( dbData.clientUrl && !injectedDependencies[ContainerRegistrationKeys.PG_CONNECTION] ) { diff --git a/packages/orchestration/src/__tests__/transaction/transaction-orchestrator.ts b/packages/orchestration/src/__tests__/transaction/transaction-orchestrator.ts index 19dcbdcfbb..cd5addce28 100644 --- a/packages/orchestration/src/__tests__/transaction/transaction-orchestrator.ts +++ b/packages/orchestration/src/__tests__/transaction/transaction-orchestrator.ts @@ -1,10 +1,14 @@ +import { TransactionStepState, TransactionStepStatus } from "@medusajs/utils" +import { setTimeout } from "timers/promises" import { DistributedTransaction, TransactionHandlerType, TransactionOrchestrator, TransactionPayload, TransactionState, + TransactionStepTimeoutError, TransactionStepsDefinition, + TransactionTimeoutError, } from "../../transaction" describe("Transaction Orchestrator", () => { @@ -986,4 +990,454 @@ describe("Transaction Orchestrator", () => { expect(transaction).toBe(transactionInHandler) }) + + describe("Timeouts - Transaction and Step", () => { + it("should fail the current steps and revert the transaction if the Transaction Timeout is reached", async () => { + const mocks = { + f1: jest.fn(() => { + return "content f1" + }), + f2: jest.fn(async () => { + await setTimeout(200) + return "delayed content f2" + }), + f3: jest.fn(() => { + return "content f3" + }), + f4: jest.fn(() => { + return "content f4" + }), + } + + async function handler( + actionId: string, + functionHandlerType: TransactionHandlerType, + payload: TransactionPayload + ) { + const command = { + action1: { + [TransactionHandlerType.INVOKE]: () => { + return mocks.f1() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f1() + }, + }, + action2: { + [TransactionHandlerType.INVOKE]: async () => { + return await mocks.f2() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f2() + }, + }, + action3: { + [TransactionHandlerType.INVOKE]: () => { + return mocks.f3() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f3() + }, + }, + action4: { + [TransactionHandlerType.INVOKE]: () => { + return mocks.f4() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f4() + }, + }, + } + + return command[actionId][functionHandlerType]() + } + + const flow: TransactionStepsDefinition = { + next: { + action: "action1", + next: [ + { + action: "action2", + }, + { + action: "action3", + next: { + action: "action4", + }, + }, + ], + }, + } + + const strategy = new TransactionOrchestrator("transaction-name", flow, { + timeout: 0.1, // 100ms + }) + + const transaction = await strategy.beginTransaction( + "transaction_id_123", + handler + ) + + await strategy.resume(transaction) + + expect(transaction.transactionId).toBe("transaction_id_123") + expect(mocks.f1).toBeCalledTimes(2) + expect(mocks.f2).toBeCalledTimes(2) + expect(mocks.f3).toBeCalledTimes(2) + expect(mocks.f4).toBeCalledTimes(0) + expect(transaction.getContext().invoke.action1).toBe("content f1") + expect(transaction.getContext().invoke.action2).toBe("delayed content f2") + expect(transaction.getContext().invoke.action3).toBe("content f3") + expect(transaction.getContext().invoke.action4).toBe(undefined) + + expect(transaction.getErrors()[0].error).toBeInstanceOf( + TransactionTimeoutError + ) + expect(transaction.getErrors()[0].action).toBe("action2") + + expect(transaction.getState()).toBe(TransactionState.REVERTED) + }) + + it("should continue the transaction and skip children steps when the Transaction Step Timeout is reached but the step is set to 'continueOnPermanentFailure'", async () => { + const mocks = { + f1: jest.fn(() => { + return "content f1" + }), + f2: jest.fn(async () => { + await setTimeout(200) + return "delayed content f2" + }), + f3: jest.fn(() => { + return "content f3" + }), + f4: jest.fn(() => { + return "content f4" + }), + } + + async function handler( + actionId: string, + functionHandlerType: TransactionHandlerType, + payload: TransactionPayload + ) { + const command = { + action1: { + [TransactionHandlerType.INVOKE]: () => { + return mocks.f1() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f1() + }, + }, + action2: { + [TransactionHandlerType.INVOKE]: async () => { + return await mocks.f2() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f2() + }, + }, + action3: { + [TransactionHandlerType.INVOKE]: () => { + return mocks.f3() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f3() + }, + }, + action4: { + [TransactionHandlerType.INVOKE]: () => { + return mocks.f4() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f4() + }, + }, + } + + return command[actionId][functionHandlerType]() + } + + const flow: TransactionStepsDefinition = { + next: { + action: "action1", + next: [ + { + timeout: 0.1, // 100ms + action: "action2", + continueOnPermanentFailure: true, + next: { + action: "action4", + }, + }, + { + action: "action3", + }, + ], + }, + } + + const strategy = new TransactionOrchestrator("transaction-name", flow) + + const transaction = await strategy.beginTransaction( + "transaction_id_123", + handler + ) + + await strategy.resume(transaction) + + expect(transaction.transactionId).toBe("transaction_id_123") + expect(mocks.f1).toBeCalledTimes(1) + expect(mocks.f2).toBeCalledTimes(1) + expect(mocks.f3).toBeCalledTimes(1) + expect(mocks.f4).toBeCalledTimes(0) + expect(transaction.getContext().invoke.action1).toBe("content f1") + expect(transaction.getContext().invoke.action2).toBe("delayed content f2") + expect(transaction.getContext().invoke.action3).toBe("content f3") + expect(transaction.getContext().invoke.action4).toBe(undefined) + expect( + transaction.getFlow().steps["_root.action1.action2"].invoke.state + ).toBe(TransactionStepState.TIMEOUT) + expect( + transaction.getFlow().steps["_root.action1.action2"].invoke.status + ).toBe(TransactionStepStatus.PERMANENT_FAILURE) + expect( + transaction.getFlow().steps["_root.action1.action2"].compensate.state + ).toBe(TransactionStepState.DORMANT) + expect( + transaction.getFlow().steps["_root.action1.action2.action4"].invoke + .state + ).toBe(TransactionStepState.SKIPPED) + expect( + transaction.getFlow().steps["_root.action1.action2.action4"].invoke + .status + ).toBe(TransactionStepStatus.IDLE) + + expect(transaction.getState()).toBe(TransactionState.DONE) + }) + + it("should fail the current steps and revert the transaction if the Step Timeout is reached", async () => { + const mocks = { + f1: jest.fn(() => { + return "content f1" + }), + f2: jest.fn(async () => { + await setTimeout(200) + return "delayed content f2" + }), + f3: jest.fn(() => { + return "content f3" + }), + f4: jest.fn(() => { + return "content f4" + }), + } + + async function handler( + actionId: string, + functionHandlerType: TransactionHandlerType, + payload: TransactionPayload + ) { + const command = { + action1: { + [TransactionHandlerType.INVOKE]: () => { + return mocks.f1() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f1() + }, + }, + action2: { + [TransactionHandlerType.INVOKE]: async () => { + return await mocks.f2() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f2() + }, + }, + action3: { + [TransactionHandlerType.INVOKE]: () => { + return mocks.f3() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f3() + }, + }, + action4: { + [TransactionHandlerType.INVOKE]: () => { + return mocks.f4() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f4() + }, + }, + } + + return command[actionId][functionHandlerType]() + } + + const flow: TransactionStepsDefinition = { + next: { + action: "action1", + next: [ + { + action: "action2", + timeout: 0.1, // 100ms + }, + { + action: "action3", + next: { + action: "action4", + }, + }, + ], + }, + } + + const strategy = new TransactionOrchestrator("transaction-name", flow) + + const transaction = await strategy.beginTransaction( + "transaction_id_123", + handler + ) + + await strategy.resume(transaction) + + expect(transaction.transactionId).toBe("transaction_id_123") + expect(mocks.f1).toBeCalledTimes(2) + expect(mocks.f2).toBeCalledTimes(2) + expect(mocks.f3).toBeCalledTimes(2) + expect(mocks.f4).toBeCalledTimes(0) + expect(transaction.getContext().invoke.action1).toBe("content f1") + expect(transaction.getContext().invoke.action2).toBe("delayed content f2") + expect(transaction.getContext().invoke.action3).toBe("content f3") + expect(transaction.getContext().invoke.action4).toBe(undefined) + + expect(transaction.getErrors()[0].error).toBeInstanceOf( + TransactionStepTimeoutError + ) + expect(transaction.getErrors()[0].action).toBe("action2") + + expect(transaction.getState()).toBe(TransactionState.REVERTED) + }) + + it("should fail the current steps and revert the transaction if the Transaction Timeout is reached event if the step is set as 'continueOnPermanentFailure'", async () => { + const mocks = { + f1: jest.fn(() => { + return "content f1" + }), + f2: jest.fn(async () => { + await setTimeout(200) + return "delayed content f2" + }), + f3: jest.fn(async () => { + await setTimeout(200) + return "content f3" + }), + f4: jest.fn(() => { + return "content f4" + }), + } + + async function handler( + actionId: string, + functionHandlerType: TransactionHandlerType, + payload: TransactionPayload + ) { + const command = { + action1: { + [TransactionHandlerType.INVOKE]: () => { + return mocks.f1() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f1() + }, + }, + action2: { + [TransactionHandlerType.INVOKE]: async () => { + return await mocks.f2() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f2() + }, + }, + action3: { + [TransactionHandlerType.INVOKE]: async () => { + return await mocks.f3() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f3() + }, + }, + action4: { + [TransactionHandlerType.INVOKE]: () => { + return mocks.f4() + }, + [TransactionHandlerType.COMPENSATE]: () => { + return mocks.f4() + }, + }, + } + + return command[actionId][functionHandlerType]() + } + + const flow: TransactionStepsDefinition = { + next: { + action: "action1", + next: [ + { + action: "action2", + continueOnPermanentFailure: true, + }, + { + action: "action3", + continueOnPermanentFailure: true, + next: { + action: "action4", + }, + }, + ], + }, + } + + const strategy = new TransactionOrchestrator("transaction-name", flow, { + timeout: 0.1, // 100ms + }) + + const transaction = await strategy.beginTransaction( + "transaction_id_123", + handler + ) + + await strategy.resume(transaction) + + expect(transaction.transactionId).toBe("transaction_id_123") + expect(mocks.f1).toBeCalledTimes(2) + expect(mocks.f2).toBeCalledTimes(2) + expect(mocks.f3).toBeCalledTimes(2) + expect(mocks.f4).toBeCalledTimes(0) + expect(transaction.getContext().invoke.action1).toBe("content f1") + expect(transaction.getContext().invoke.action2).toBe("delayed content f2") + expect(transaction.getContext().invoke.action3).toBe("content f3") + expect(transaction.getContext().invoke.action4).toBe(undefined) + + expect(transaction.getErrors()).toHaveLength(2) + expect( + TransactionTimeoutError.isTransactionTimeoutError( + transaction.getErrors()[0].error + ) + ).toBe(true) + expect(transaction.getErrors()[0].action).toBe("action2") + + expect( + TransactionTimeoutError.isTransactionTimeoutError( + transaction.getErrors()[1].error + ) + ).toBe(true) + expect(transaction.getErrors()[1].action).toBe("action3") + + expect(transaction.getState()).toBe(TransactionState.REVERTED) + }) + }) }) diff --git a/packages/orchestration/src/transaction/datastore/abstract-storage.ts b/packages/orchestration/src/transaction/datastore/abstract-storage.ts index aab8c5e677..defc91d6cd 100644 --- a/packages/orchestration/src/transaction/datastore/abstract-storage.ts +++ b/packages/orchestration/src/transaction/datastore/abstract-storage.ts @@ -3,14 +3,11 @@ import { TransactionCheckpoint, } from "../distributed-transaction" import { TransactionStep } from "../transaction-step" -import { TransactionModelOptions } from "../types" export interface IDistributedTransactionStorage { get(key: string): Promise list(): Promise save(key: string, data: TransactionCheckpoint, ttl?: number): Promise - delete(key: string): Promise - archive(key: string, options?: TransactionModelOptions): Promise scheduleRetry( transaction: DistributedTransaction, step: TransactionStep, @@ -62,14 +59,6 @@ export abstract class DistributedTransactionStorage throw new Error("Method 'save' not implemented.") } - async delete(key: string): Promise { - throw new Error("Method 'delete' not implemented.") - } - - async archive(key: string, options?: TransactionModelOptions): Promise { - throw new Error("Method 'archive' not implemented.") - } - async scheduleRetry( transaction: DistributedTransaction, step: TransactionStep, diff --git a/packages/orchestration/src/transaction/datastore/base-in-memory-storage.ts b/packages/orchestration/src/transaction/datastore/base-in-memory-storage.ts index 69ab557a03..23ac1438cb 100644 --- a/packages/orchestration/src/transaction/datastore/base-in-memory-storage.ts +++ b/packages/orchestration/src/transaction/datastore/base-in-memory-storage.ts @@ -1,5 +1,5 @@ +import { TransactionState } from "@medusajs/utils" import { TransactionCheckpoint } from "../distributed-transaction" -import { TransactionModelOptions } from "../types" import { DistributedTransactionStorage } from "./abstract-storage" // eslint-disable-next-line max-len @@ -24,14 +24,16 @@ export class BaseInMemoryDistributedTransactionStorage extends DistributedTransa data: TransactionCheckpoint, ttl?: number ): Promise { - this.storage.set(key, data) - } + const hasFinished = [ + TransactionState.DONE, + TransactionState.REVERTED, + TransactionState.FAILED, + ].includes(data.flow.state) - async delete(key: string): Promise { - this.storage.delete(key) - } - - async archive(key: string, options?: TransactionModelOptions): Promise { - this.storage.delete(key) + if (hasFinished) { + this.storage.delete(key) + } else { + this.storage.set(key, data) + } } } diff --git a/packages/orchestration/src/transaction/distributed-transaction.ts b/packages/orchestration/src/transaction/distributed-transaction.ts index 16d05c2f62..0a9e14e787 100644 --- a/packages/orchestration/src/transaction/distributed-transaction.ts +++ b/packages/orchestration/src/transaction/distributed-transaction.ts @@ -86,7 +86,7 @@ export class DistributedTransaction extends EventEmitter { this.keyValueStore = storage } - private static keyPrefix = "dtrans" + public static keyPrefix = "dtrans" constructor( private flow: TransactionFlow, @@ -177,18 +177,18 @@ export class DistributedTransaction extends EventEmitter { } public hasTimeout(): boolean { - return !!this.getFlow().definition.timeout + return !!this.getTimeout() } - public getTimeoutInterval(): number | undefined { - return this.getFlow().definition.timeout + public getTimeout(): number | undefined { + return this.getFlow().options?.timeout } public async saveCheckpoint( ttl = 0 ): Promise { const options = this.getFlow().options - if (!options?.storeExecution) { + if (!options?.store) { return } @@ -226,31 +226,6 @@ export class DistributedTransaction extends EventEmitter { return null } - public async deleteCheckpoint(): Promise { - const options = this.getFlow().options - if (!options?.storeExecution) { - return - } - - const key = TransactionOrchestrator.getKeyName( - DistributedTransaction.keyPrefix, - this.modelId, - this.transactionId - ) - await DistributedTransaction.keyValueStore.delete(key) - } - - public async archiveCheckpoint(): Promise { - const options = this.getFlow().options - - const key = TransactionOrchestrator.getKeyName( - DistributedTransaction.keyPrefix, - this.modelId, - this.transactionId - ) - await DistributedTransaction.keyValueStore.archive(key, options) - } - public async scheduleRetry( step: TransactionStep, interval: number @@ -269,6 +244,11 @@ export class DistributedTransaction extends EventEmitter { } public async scheduleTransactionTimeout(interval: number): Promise { + // schedule transaction timeout only if there are async steps + if (!this.getFlow().hasAsyncSteps) { + return + } + await this.saveCheckpoint() await DistributedTransaction.keyValueStore.scheduleTransactionTimeout( this, @@ -278,6 +258,10 @@ export class DistributedTransaction extends EventEmitter { } public async clearTransactionTimeout(): Promise { + if (!this.getFlow().hasAsyncSteps) { + return + } + await DistributedTransaction.keyValueStore.clearTransactionTimeout(this) } @@ -285,6 +269,11 @@ export class DistributedTransaction extends EventEmitter { step: TransactionStep, interval: number ): Promise { + // schedule step timeout only if the step is async + if (!step.definition.async) { + return + } + await this.saveCheckpoint() await DistributedTransaction.keyValueStore.scheduleStepTimeout( this, @@ -295,6 +284,10 @@ export class DistributedTransaction extends EventEmitter { } public async clearStepTimeout(step: TransactionStep): Promise { + if (!step.definition.async || step.isCompensating()) { + return + } + await DistributedTransaction.keyValueStore.clearStepTimeout(this, step) } } diff --git a/packages/orchestration/src/transaction/errors.ts b/packages/orchestration/src/transaction/errors.ts index 331784e80c..2bbda62dc2 100644 --- a/packages/orchestration/src/transaction/errors.ts +++ b/packages/orchestration/src/transaction/errors.ts @@ -4,7 +4,7 @@ export class PermanentStepFailureError extends Error { ): error is PermanentStepFailureError { return ( error instanceof PermanentStepFailureError || - error.name === "PermanentStepFailure" + error?.name === "PermanentStepFailure" ) } @@ -14,16 +14,19 @@ export class PermanentStepFailureError extends Error { } } -export class StepTimeoutError extends Error { - static isStepTimeoutError(error: Error): error is StepTimeoutError { +export class TransactionStepTimeoutError extends Error { + static isTransactionStepTimeoutError( + error: Error + ): error is TransactionStepTimeoutError { return ( - error instanceof StepTimeoutError || error.name === "StepTimeoutError" + error instanceof TransactionStepTimeoutError || + error?.name === "TransactionStepTimeoutError" ) } constructor(message?: string) { super(message) - this.name = "StepTimeoutError" + this.name = "TransactionStepTimeoutError" } } @@ -33,7 +36,7 @@ export class TransactionTimeoutError extends Error { ): error is TransactionTimeoutError { return ( error instanceof TransactionTimeoutError || - error.name === "TransactionTimeoutError" + error?.name === "TransactionTimeoutError" ) } diff --git a/packages/orchestration/src/transaction/orchestrator-builder.ts b/packages/orchestration/src/transaction/orchestrator-builder.ts index 711902c5d6..a645cf72fd 100644 --- a/packages/orchestration/src/transaction/orchestrator-builder.ts +++ b/packages/orchestration/src/transaction/orchestrator-builder.ts @@ -314,7 +314,7 @@ export class OrchestratorBuilder { action: string, step: InternalStep = this.steps ): InternalStep | undefined { - if (step.action === action) { + if (step.uuid === action || step.action === action) { return step } @@ -357,7 +357,7 @@ export class OrchestratorBuilder { if (!nextStep) { continue } - if (nextStep.action === action) { + if (nextStep.uuid === action || nextStep.action === action) { return step } const foundStep = this.findParentStepByAction( diff --git a/packages/orchestration/src/transaction/transaction-orchestrator.ts b/packages/orchestration/src/transaction/transaction-orchestrator.ts index 1a6de79a67..7a7fa54c2a 100644 --- a/packages/orchestration/src/transaction/transaction-orchestrator.ts +++ b/packages/orchestration/src/transaction/transaction-orchestrator.ts @@ -13,11 +13,11 @@ import { TransactionStepStatus, } from "./types" -import { MedusaError, promiseAll } from "@medusajs/utils" +import { MedusaError, promiseAll, TransactionStepState } from "@medusajs/utils" import { EventEmitter } from "events" import { PermanentStepFailureError, - StepTimeoutError, + TransactionStepTimeoutError, TransactionTimeoutError, } from "./errors" @@ -30,6 +30,7 @@ export type TransactionFlow = { hasFailedSteps: boolean hasWaitingSteps: boolean hasSkippedSteps: boolean + hasRevertedSteps: boolean timedOutAt: number | null startedAt?: number state: TransactionState @@ -62,10 +63,6 @@ export class TransactionOrchestrator extends EventEmitter { return params.join(this.SEPARATOR) } - public getOptions(): TransactionModelOptions { - return this.options ?? {} - } - private getPreviousStep(flow: TransactionFlow, step: TransactionStep) { const id = step.id.split(".") id.pop() @@ -73,6 +70,10 @@ export class TransactionOrchestrator extends EventEmitter { return flow.steps[parentId] } + public getOptions(): TransactionModelOptions { + return this.options ?? {} + } + private getInvokeSteps(flow: TransactionFlow): string[] { if (this.invokeSteps.length) { return this.invokeSteps @@ -102,9 +103,10 @@ export class TransactionOrchestrator extends EventEmitter { private canMoveForward(flow: TransactionFlow, previousStep: TransactionStep) { const states = [ - TransactionState.DONE, - TransactionState.FAILED, - TransactionState.SKIPPED, + TransactionStepState.DONE, + TransactionStepState.FAILED, + TransactionStepState.TIMEOUT, + TransactionStepState.SKIPPED, ] const siblings = this.getPreviousStep(flow, previousStep).next.map( @@ -119,10 +121,10 @@ export class TransactionOrchestrator extends EventEmitter { private canMoveBackward(flow: TransactionFlow, step: TransactionStep) { const states = [ - TransactionState.DONE, - TransactionState.REVERTED, - TransactionState.FAILED, - TransactionState.DORMANT, + TransactionStepState.DONE, + TransactionStepState.REVERTED, + TransactionStepState.FAILED, + TransactionStepState.DORMANT, ] const siblings = step.next.map((sib) => flow.steps[sib]) return ( @@ -144,29 +146,89 @@ export class TransactionOrchestrator extends EventEmitter { } } - private async checkStepTimeout(transaction, step) { + private hasExpired( + { + transaction, + step, + }: { + transaction?: DistributedTransaction + step?: TransactionStep + }, + dateNow: number + ): boolean { + const hasStepTimedOut = + step && + step.hasTimeout() && + !step.isCompensating() && + dateNow > step.startedAt! + step.getTimeout()! * 1e3 + + const hasTransactionTimedOut = + transaction && + transaction.hasTimeout() && + transaction.getFlow().state !== TransactionState.COMPENSATING && + dateNow > + transaction.getFlow().startedAt! + transaction.getTimeout()! * 1e3 + + return !!hasStepTimedOut || !!hasTransactionTimedOut + } + + private async checkTransactionTimeout( + transaction: DistributedTransaction, + currentSteps: TransactionStep[] + ) { + const flow = transaction.getFlow() + let hasTimedOut = false + if (!flow.timedOutAt && this.hasExpired({ transaction }, Date.now())) { + flow.timedOutAt = Date.now() + + void transaction.clearTransactionTimeout() + + for (const step of currentSteps) { + await TransactionOrchestrator.setStepTimeout( + transaction, + step, + new TransactionTimeoutError() + ) + } + + await transaction.saveCheckpoint() + + this.emit(DistributedTransactionEvent.TIMEOUT, { transaction }) + + hasTimedOut = true + } + + return hasTimedOut + } + + private async checkStepTimeout( + transaction: DistributedTransaction, + step: TransactionStep + ) { let hasTimedOut = false if ( - step.hasTimeout() && !step.timedOutAt && step.canCancel() && - step.startedAt! + step.getTimeoutInterval()! * 1e3 < Date.now() + this.hasExpired({ step }, Date.now()) ) { step.timedOutAt = Date.now() - await transaction.saveCheckpoint() - this.emit(DistributedTransactionEvent.TIMEOUT, { transaction }) - await TransactionOrchestrator.setStepFailure( + + await TransactionOrchestrator.setStepTimeout( transaction, step, - new StepTimeoutError(), - 0 + new TransactionStepTimeoutError() ) hasTimedOut = true + + await transaction.saveCheckpoint() + + this.emit(DistributedTransactionEvent.TIMEOUT, { transaction }) } return hasTimedOut } private async checkAllSteps(transaction: DistributedTransaction): Promise<{ + current: TransactionStep[] next: TransactionStep[] total: number remaining: number @@ -182,6 +244,8 @@ export class TransactionOrchestrator extends EventEmitter { const flow = transaction.getFlow() const nextSteps: TransactionStep[] = [] + const currentSteps: TransactionStep[] = [] + const allSteps = flow.state === TransactionState.COMPENSATING ? this.getCompensationSteps(flow) @@ -204,6 +268,7 @@ export class TransactionOrchestrator extends EventEmitter { } if (curState.status === TransactionStepStatus.WAITING) { + currentSteps.push(stepDef) hasWaiting = true if (stepDef.hasAwaitingRetry()) { @@ -223,6 +288,8 @@ export class TransactionOrchestrator extends EventEmitter { continue } else if (curState.status === TransactionStepStatus.TEMPORARY_FAILURE) { + currentSteps.push(stepDef) + if (!stepDef.canRetry()) { if (stepDef.hasRetryInterval() && !stepDef.retryRescheduledAt) { stepDef.hasScheduledRetry = true @@ -243,11 +310,11 @@ export class TransactionOrchestrator extends EventEmitter { } else { completedSteps++ - if (curState.state === TransactionState.SKIPPED) { + if (curState.state === TransactionStepState.SKIPPED) { hasSkipped = true - } else if (curState.state === TransactionState.REVERTED) { + } else if (curState.state === TransactionStepState.REVERTED) { hasReverted = true - } else if (curState.state === TransactionState.FAILED) { + } else if (curState.state === TransactionStepState.FAILED) { if (stepDef.definition.continueOnPermanentFailure) { hasIgnoredFailure = true } else { @@ -258,6 +325,7 @@ export class TransactionOrchestrator extends EventEmitter { } flow.hasWaitingSteps = hasWaiting + flow.hasRevertedSteps = hasReverted const totalSteps = allSteps.length - 1 if ( @@ -288,6 +356,7 @@ export class TransactionOrchestrator extends EventEmitter { } return { + current: currentSteps, next: nextSteps, total: totalSteps, remaining: totalSteps - completedSteps, @@ -304,11 +373,13 @@ export class TransactionOrchestrator extends EventEmitter { const stepDef = flow.steps[step] const curState = stepDef.getStates() if ( - curState.state === TransactionState.DONE || + [TransactionStepState.DONE, TransactionStepState.TIMEOUT].includes( + curState.state + ) || curState.status === TransactionStepStatus.PERMANENT_FAILURE ) { stepDef.beginCompensation() - stepDef.changeState(TransactionState.NOT_STARTED) + stepDef.changeState(TransactionStepState.NOT_STARTED) } } } @@ -318,6 +389,9 @@ export class TransactionOrchestrator extends EventEmitter { step: TransactionStep, response: unknown ): Promise { + const hasStepTimedOut = + step.getStates().state === TransactionStepState.TIMEOUT + if (step.saveResponse) { transaction.addResponse( step.definition.action!, @@ -328,16 +402,19 @@ export class TransactionOrchestrator extends EventEmitter { ) } - step.changeStatus(TransactionStepStatus.OK) + const flow = transaction.getFlow() - if (step.isCompensating()) { - step.changeState(TransactionState.REVERTED) - } else { - step.changeState(TransactionState.DONE) + if (!hasStepTimedOut) { + step.changeStatus(TransactionStepStatus.OK) } - const flow = transaction.getFlow() - if (step.definition.async || flow.options?.strictCheckpoints) { + if (step.isCompensating()) { + step.changeState(TransactionStepState.REVERTED) + } else if (!hasStepTimedOut) { + step.changeState(TransactionStepState.DONE) + } + + if (step.definition.async || flow.options?.storeExecution) { await transaction.saveCheckpoint() } @@ -357,35 +434,87 @@ export class TransactionOrchestrator extends EventEmitter { transaction.emit(eventName, { step, transaction }) } + private static async setStepTimeout( + transaction: DistributedTransaction, + step: TransactionStep, + error: TransactionStepTimeoutError | TransactionTimeoutError + ): Promise { + if ( + [ + TransactionStepState.TIMEOUT, + TransactionStepState.DONE, + TransactionStepState.REVERTED, + ].includes(step.getStates().state) + ) { + return + } + + step.changeState(TransactionStepState.TIMEOUT) + + transaction.addError( + step.definition.action!, + TransactionHandlerType.INVOKE, + error + ) + + await TransactionOrchestrator.setStepFailure( + transaction, + step, + undefined, + 0, + true, + error + ) + + await transaction.clearStepTimeout(step) + } + private static async setStepFailure( transaction: DistributedTransaction, step: TransactionStep, error: Error | any, - maxRetries: number = TransactionOrchestrator.DEFAULT_RETRIES + maxRetries: number = TransactionOrchestrator.DEFAULT_RETRIES, + isTimeout = false, + timeoutError?: TransactionStepTimeoutError | TransactionTimeoutError ): Promise { step.failures++ - step.changeStatus(TransactionStepStatus.TEMPORARY_FAILURE) + if ( + !isTimeout && + step.getStates().status !== TransactionStepStatus.PERMANENT_FAILURE + ) { + step.changeStatus(TransactionStepStatus.TEMPORARY_FAILURE) + } const flow = transaction.getFlow() const cleaningUp: Promise[] = [] - if (step.failures > maxRetries) { - step.changeState(TransactionState.FAILED) + + const hasTimedOut = step.getStates().state === TransactionStepState.TIMEOUT + if (step.failures > maxRetries || hasTimedOut) { + if (!hasTimedOut) { + step.changeState(TransactionStepState.FAILED) + } + step.changeStatus(TransactionStepStatus.PERMANENT_FAILURE) - transaction.addError( - step.definition.action!, - step.isCompensating() - ? TransactionHandlerType.COMPENSATE - : TransactionHandlerType.INVOKE, - error - ) + if (!isTimeout) { + transaction.addError( + step.definition.action!, + step.isCompensating() + ? TransactionHandlerType.COMPENSATE + : TransactionHandlerType.INVOKE, + error + ) + } if (!step.isCompensating()) { - if (step.definition.continueOnPermanentFailure) { + if ( + step.definition.continueOnPermanentFailure && + !TransactionTimeoutError.isTransactionTimeoutError(timeoutError!) + ) { for (const childStep of step.next) { const child = flow.steps[childStep] - child.changeState(TransactionState.SKIPPED) + child.changeState(TransactionStepState.SKIPPED) } } else { flow.state = TransactionState.WAITING_TO_COMPENSATE @@ -397,7 +526,7 @@ export class TransactionOrchestrator extends EventEmitter { } } - if (step.definition.async || flow.options?.strictCheckpoints) { + if (step.definition.async || flow.options?.storeExecution) { await transaction.saveCheckpoint() } @@ -413,33 +542,6 @@ export class TransactionOrchestrator extends EventEmitter { transaction.emit(eventName, { step, transaction }) } - private async checkTransactionTimeout(transaction, currentSteps) { - let hasTimedOut = false - const flow = transaction.getFlow() - if ( - transaction.hasTimeout() && - !flow.timedOutAt && - flow.startedAt! + transaction.getTimeoutInterval()! * 1e3 < Date.now() - ) { - flow.timedOutAt = Date.now() - this.emit(DistributedTransactionEvent.TIMEOUT, { transaction }) - - for (const step of currentSteps) { - await TransactionOrchestrator.setStepFailure( - transaction, - step, - new TransactionTimeoutError(), - 0 - ) - } - - await transaction.saveCheckpoint() - - hasTimedOut = true - } - return hasTimedOut - } - private async executeNext( transaction: DistributedTransaction ): Promise { @@ -456,22 +558,19 @@ export class TransactionOrchestrator extends EventEmitter { const hasTimedOut = await this.checkTransactionTimeout( transaction, - nextSteps.next + nextSteps.current ) + if (hasTimedOut) { continue } if (nextSteps.remaining === 0) { if (transaction.hasTimeout()) { - await transaction.clearTransactionTimeout() + void transaction.clearTransactionTimeout() } - if (flow.options?.retentionTime == undefined) { - await transaction.deleteCheckpoint() - } else { - await transaction.saveCheckpoint() - } + await transaction.saveCheckpoint() this.emit(DistributedTransactionEvent.FINISH, { transaction }) } @@ -486,20 +585,20 @@ export class TransactionOrchestrator extends EventEmitter { step.lastAttempt = Date.now() step.attempts++ - if (curState.state === TransactionState.NOT_STARTED) { + if (curState.state === TransactionStepState.NOT_STARTED) { if (!step.startedAt) { step.startedAt = Date.now() } if (step.isCompensating()) { - step.changeState(TransactionState.COMPENSATING) + step.changeState(TransactionStepState.COMPENSATING) if (step.definition.noCompensation) { - step.changeState(TransactionState.REVERTED) + step.changeState(TransactionStepState.REVERTED) continue } } else if (flow.state === TransactionState.INVOKING) { - step.changeState(TransactionState.INVOKING) + step.changeState(TransactionStepState.INVOKING) } } @@ -554,6 +653,14 @@ export class TransactionOrchestrator extends EventEmitter { transaction .handler(step.definition.action + "", type, payload, transaction) .then(async (response: any) => { + if (this.hasExpired({ transaction, step }, Date.now())) { + await this.checkStepTimeout(transaction, step) + await this.checkTransactionTimeout( + transaction, + nextSteps.next.includes(step) ? nextSteps.next : [step] + ) + } + await TransactionOrchestrator.setStepSuccess( transaction, step, @@ -561,6 +668,14 @@ export class TransactionOrchestrator extends EventEmitter { ) }) .catch(async (error) => { + if (this.hasExpired({ transaction, step }, Date.now())) { + await this.checkStepTimeout(transaction, step) + await this.checkTransactionTimeout( + transaction, + nextSteps.next.includes(step) ? nextSteps.next : [step] + ) + } + if ( PermanentStepFailureError.isPermanentStepFailureError(error) ) { @@ -573,7 +688,7 @@ export class TransactionOrchestrator extends EventEmitter { ) } else { execution.push( - transaction.saveCheckpoint().then(async () => + transaction.saveCheckpoint().then(() => { transaction .handler( step.definition.action + "", @@ -591,12 +706,12 @@ export class TransactionOrchestrator extends EventEmitter { await setStepFailure(error) }) - ) + }) ) } } - if (hasSyncSteps && flow.options?.strictCheckpoints) { + if (hasSyncSteps && flow.options?.storeExecution) { await transaction.saveCheckpoint() } @@ -630,16 +745,14 @@ export class TransactionOrchestrator extends EventEmitter { flow.state = TransactionState.INVOKING flow.startedAt = Date.now() - if (this.options?.storeExecution) { + if (this.options?.store) { await transaction.saveCheckpoint( flow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL ) } if (transaction.hasTimeout()) { - await transaction.scheduleTransactionTimeout( - transaction.getTimeoutInterval()! - ) + await transaction.scheduleTransactionTimeout(transaction.getTimeout()!) } this.emit(DistributedTransactionEvent.BEGIN, { transaction }) @@ -682,12 +795,19 @@ export class TransactionOrchestrator extends EventEmitter { this.definition ) + this.options ??= {} + const hasAsyncSteps = features.hasAsyncSteps const hasStepTimeouts = features.hasStepTimeouts const hasRetriesTimeout = features.hasRetriesTimeout + const hasTransactionTimeout = !!this.options.timeout - this.options ??= {} - if (hasAsyncSteps || hasStepTimeouts || hasRetriesTimeout) { + if (hasAsyncSteps) { + this.options.store = true + } + + if (hasStepTimeouts || hasRetriesTimeout || hasTransactionTimeout) { + this.options.store = true this.options.storeExecution = true } @@ -699,6 +819,7 @@ export class TransactionOrchestrator extends EventEmitter { hasFailedSteps: false, hasSkippedSteps: false, hasWaitingSteps: false, + hasRevertedSteps: false, timedOutAt: null, state: TransactionState.NOT_STARTED, definition: this.definition, @@ -807,15 +928,16 @@ export class TransactionOrchestrator extends EventEmitter { new TransactionStep(), existingSteps?.[id] || { id, + uuid: definitionCopy.uuid, depth: level.length - 1, definition: definitionCopy, saveResponse: definitionCopy.saveResponse ?? true, invoke: { - state: TransactionState.NOT_STARTED, + state: TransactionStepState.NOT_STARTED, status: TransactionStepStatus.IDLE, }, compensate: { - state: TransactionState.DORMANT, + state: TransactionStepState.DORMANT, status: TransactionStepStatus.IDLE, }, attempts: 0, @@ -861,11 +983,7 @@ export class TransactionOrchestrator extends EventEmitter { existingTransaction?.context ) - if ( - newTransaction && - this.options?.storeExecution && - this.options?.strictCheckpoints - ) { + if (newTransaction && this.options?.store && this.options?.storeExecution) { await transaction.saveCheckpoint( modelFlow.hasAsyncSteps ? 0 : TransactionOrchestrator.DEFAULT_TTL ) diff --git a/packages/orchestration/src/transaction/transaction-step.ts b/packages/orchestration/src/transaction/transaction-step.ts index 57b06acf31..bf20b5635a 100644 --- a/packages/orchestration/src/transaction/transaction-step.ts +++ b/packages/orchestration/src/transaction/transaction-step.ts @@ -1,4 +1,4 @@ -import { MedusaError } from "@medusajs/utils" +import { MedusaError, TransactionStepState } from "@medusajs/utils" import { DistributedTransaction, TransactionPayload, @@ -6,8 +6,8 @@ import { import { TransactionHandlerType, TransactionState, - TransactionStepsDefinition, TransactionStepStatus, + TransactionStepsDefinition, } from "./types" export type TransactionStepHandler = ( @@ -38,14 +38,15 @@ export class TransactionStep { */ private stepFailed = false id: string + uuid?: string depth: number definition: TransactionStepsDefinition invoke: { - state: TransactionState + state: TransactionStepState status: TransactionStepStatus } compensate: { - state: TransactionState + state: TransactionStepState status: TransactionStepStatus } attempts: number @@ -81,24 +82,25 @@ export class TransactionStep { return !this.stepFailed } - public changeState(toState: TransactionState) { + public changeState(toState: TransactionStepState) { const allowed = { - [TransactionState.DORMANT]: [TransactionState.NOT_STARTED], - [TransactionState.NOT_STARTED]: [ - TransactionState.INVOKING, - TransactionState.COMPENSATING, - TransactionState.FAILED, - TransactionState.SKIPPED, + [TransactionStepState.DORMANT]: [TransactionStepState.NOT_STARTED], + [TransactionStepState.NOT_STARTED]: [ + TransactionStepState.INVOKING, + TransactionStepState.COMPENSATING, + TransactionStepState.FAILED, + TransactionStepState.SKIPPED, ], - [TransactionState.INVOKING]: [ - TransactionState.FAILED, - TransactionState.DONE, + [TransactionStepState.INVOKING]: [ + TransactionStepState.FAILED, + TransactionStepState.DONE, + TransactionStepState.TIMEOUT, ], - [TransactionState.COMPENSATING]: [ - TransactionState.REVERTED, - TransactionState.FAILED, + [TransactionStepState.COMPENSATING]: [ + TransactionStepState.REVERTED, + TransactionStepState.FAILED, ], - [TransactionState.DONE]: [TransactionState.COMPENSATING], + [TransactionStepState.DONE]: [TransactionStepState.COMPENSATING], } const curState = this.getStates() @@ -155,10 +157,10 @@ export class TransactionStep { } hasTimeout(): boolean { - return !!this.definition.timeout + return !!this.getTimeout() } - getTimeoutInterval(): number | undefined { + getTimeout(): number | undefined { return this.definition.timeout } @@ -190,7 +192,7 @@ export class TransactionStep { const { status, state } = this.getStates() return ( (!this.isCompensating() && - state === TransactionState.NOT_STARTED && + state === TransactionStepState.NOT_STARTED && flowState === TransactionState.INVOKING) || status === TransactionStepStatus.TEMPORARY_FAILURE ) @@ -199,7 +201,7 @@ export class TransactionStep { canCompensate(flowState: TransactionState): boolean { return ( this.isCompensating() && - this.getStates().state === TransactionState.NOT_STARTED && + this.getStates().state === TransactionStepState.NOT_STARTED && flowState === TransactionState.COMPENSATING ) } diff --git a/packages/orchestration/src/transaction/types.ts b/packages/orchestration/src/transaction/types.ts index 16f27985f5..bd5abab5ca 100644 --- a/packages/orchestration/src/transaction/types.ts +++ b/packages/orchestration/src/transaction/types.ts @@ -1,51 +1,118 @@ import { DistributedTransaction } from "./distributed-transaction" import { TransactionStep } from "./transaction-step" +export { + TransactionHandlerType, + TransactionState, + TransactionStepStatus, +} from "@medusajs/utils" -export enum TransactionHandlerType { - INVOKE = "invoke", - COMPENSATE = "compensate", -} - +/** + * Defines the structure and behavior of a single step within a transaction workflow. + */ export type TransactionStepsDefinition = { + /** + * A unique identifier for the transaction step. + * This is set automatically when declaring a workflow with "createWorkflow" + */ + uuid?: string + + /** + * Specifies the action to be performed in this step. + * "name" is an alias for action when creating a workflow with "createWorkflow". + */ action?: string + + /** + * Indicates whether the workflow should continue even if there is a permanent failure in this step. + * In case it is set to true, the children steps of this step will not be executed and their status will be marked as TransactionStepState.SKIPPED. + */ continueOnPermanentFailure?: boolean + + /** + * If true, no compensation action will be triggered for this step in case of a failure. + */ noCompensation?: boolean + + /** + * The maximum number of times this step should be retried in case of temporary failures. + * The default is 0 (no retries). + */ maxRetries?: number + + /** + * The interval (in seconds) between retry attempts after a temporary failure. + * The default is to retry immediately. + */ retryInterval?: number + + /** + * The interval (in seconds) to retry a step even if its status is "TransactionStepStatus.WAITING". + */ retryIntervalAwaiting?: number + + /** + * The maximum amount of time (in seconds) to wait for this step to complete. + * This is NOT an execution timeout, the step will always be executed and wait for its response. + * If the response is not received within the timeout set, it will be marked as "TransactionStepStatus.TIMEOUT" and the workflow will be reverted as soon as it receives the response. + */ timeout?: number + + /** + * If true, the step is executed asynchronously. This means that the workflow will not wait for the response of this step. + * Async steps require to have their responses set using "setStepSuccess" or "setStepFailure". + * If combined with a timeout, and any response is not set within that interval, the step will be marked as "TransactionStepStatus.TIMEOUT" and the workflow will be reverted immediately. + */ async?: boolean + + /** + * If true, the compensation function for this step is executed asynchronously. Which means, the response has to be set using "setStepSuccess" or "setStepFailure". + */ compensateAsync?: boolean + + /** + * If true, the workflow will not wait for their sibling steps to complete before moving to the next step. + */ noWait?: boolean + + /** + * If true, the response of this step will be stored. + * Default is true. + */ saveResponse?: boolean + + /** + * Defines the next step(s) to execute after this step. Can be a single step or an array of steps. + */ next?: TransactionStepsDefinition | TransactionStepsDefinition[] + + // TODO: add metadata field for customizations } -export enum TransactionStepStatus { - IDLE = "idle", - OK = "ok", - WAITING = "waiting_response", - TEMPORARY_FAILURE = "temp_failure", - PERMANENT_FAILURE = "permanent_failure", -} - -export enum TransactionState { - NOT_STARTED = "not_started", - INVOKING = "invoking", - WAITING_TO_COMPENSATE = "waiting_to_compensate", - COMPENSATING = "compensating", - DONE = "done", - REVERTED = "reverted", - FAILED = "failed", - DORMANT = "dormant", - SKIPPED = "skipped", -} - +/** + * Defines the options for a transaction model, which are applicable to the entire workflow. + */ export type TransactionModelOptions = { + /** + * The global timeout for the entire transaction workflow (in seconds). + */ timeout?: number - storeExecution?: boolean + + /** + * If true, the state of the transaction will be persisted. + */ + store?: boolean + + /** + * TBD + */ retentionTime?: number - strictCheckpoints?: boolean + + /** + * If true, the execution details of each step will be stored. + */ + storeExecution?: boolean + + // TODO: add metadata field for customizations } export type TransactionModel = { diff --git a/packages/orchestration/src/workflow/workflow-manager.ts b/packages/orchestration/src/workflow/workflow-manager.ts index f0f0c06dcf..62c648b523 100644 --- a/packages/orchestration/src/workflow/workflow-manager.ts +++ b/packages/orchestration/src/workflow/workflow-manager.ts @@ -81,9 +81,16 @@ export class WorkflowManager { const finalFlow = flow instanceof OrchestratorBuilder ? flow.build() : flow if (WorkflowManager.workflows.has(workflowId)) { + function excludeStepUuid(key, value) { + return key === "uuid" ? undefined : value + } + const areStepsEqual = finalFlow - ? JSON.stringify(finalFlow) === - JSON.stringify(WorkflowManager.workflows.get(workflowId)!.flow_) + ? JSON.stringify(finalFlow, excludeStepUuid) === + JSON.stringify( + WorkflowManager.workflows.get(workflowId)!.flow_, + excludeStepUuid + ) : true if (!areStepsEqual) { @@ -131,14 +138,19 @@ export class WorkflowManager { } const finalFlow = flow instanceof OrchestratorBuilder ? flow.build() : flow + const updatedOptions = { ...workflow.options, ...options } WorkflowManager.workflows.set(workflowId, { id: workflowId, flow_: finalFlow, - orchestrator: new TransactionOrchestrator(workflowId, finalFlow, options), + orchestrator: new TransactionOrchestrator( + workflowId, + finalFlow, + updatedOptions + ), handler: WorkflowManager.buildHandlers(workflow.handlers_), handlers_: workflow.handlers_, - options: { ...workflow.options, ...options }, + options: updatedOptions, requiredModules, optionalModules, }) diff --git a/packages/types/src/bundles.ts b/packages/types/src/bundles.ts index da5d2a0950..ff2518bdf5 100644 --- a/packages/types/src/bundles.ts +++ b/packages/types/src/bundles.ts @@ -18,4 +18,3 @@ export * as SearchTypes from "./search" export * as StockLocationTypes from "./stock-location" export * as TransactionBaseTypes from "./transaction-base" export * as WorkflowTypes from "./workflow" - diff --git a/packages/types/src/dal/repository-service.ts b/packages/types/src/dal/repository-service.ts index 652a3a9b02..cc18daff1a 100644 --- a/packages/types/src/dal/repository-service.ts +++ b/packages/types/src/dal/repository-service.ts @@ -46,7 +46,7 @@ export interface RepositoryService< update(data: TDTOs["update"][], context?: Context): Promise - delete(ids: string[], context?: Context): Promise + delete(idsOrPKs: string[] | object[], context?: Context): Promise /** * Soft delete entities and cascade to related entities if configured. diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 823b2eada2..c7cf134dd6 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -14,9 +14,9 @@ export * from "./joiner" export * from "./link-modules" export * from "./logger" export * from "./modules-sdk" +export * from "./payment" export * from "./pricing" export * from "./product" -export * from "./payment" export * from "./product-category" export * from "./promotion" export * from "./region" diff --git a/packages/utils/src/dal/mikro-orm/mikro-orm-create-connection.ts b/packages/utils/src/dal/mikro-orm/mikro-orm-create-connection.ts index ecfedbc5cc..7b35fa2f9f 100644 --- a/packages/utils/src/dal/mikro-orm/mikro-orm-create-connection.ts +++ b/packages/utils/src/dal/mikro-orm/mikro-orm-create-connection.ts @@ -18,8 +18,8 @@ export async function mikroOrmCreateConnection( // It is important that the knex package version is the same as the one used by MikroORM knex package driverOptions = database.connection clientUrl = - database.connection.context.client.config.connection.connectionString - schema = database.connection.context.client.config.searchPath + database.connection.context?.client?.config?.connection?.connectionString + schema = database.connection.context?.client?.config?.searchPath } const { MikroORM } = await import("@mikro-orm/postgresql") diff --git a/packages/utils/src/modules-sdk/decorators/index.ts b/packages/utils/src/modules-sdk/decorators/index.ts index ef9137051a..fbea009476 100644 --- a/packages/utils/src/modules-sdk/decorators/index.ts +++ b/packages/utils/src/modules-sdk/decorators/index.ts @@ -1,3 +1,3 @@ -export * from "./inject-transaction-manager" export * from "./inject-manager" export * from "./inject-shared-context" +export * from "./inject-transaction-manager" diff --git a/packages/utils/src/modules-sdk/load-module-database-config.ts b/packages/utils/src/modules-sdk/load-module-database-config.ts index 2f9a2ec924..b92defb173 100644 --- a/packages/utils/src/modules-sdk/load-module-database-config.ts +++ b/packages/utils/src/modules-sdk/load-module-database-config.ts @@ -93,7 +93,7 @@ export function loadDatabaseConfig( database.connection = options.database!.connection } - if (!database.clientUrl && !silent) { + if (!database.clientUrl && !silent && !database.connection) { throw new MedusaError( MedusaError.Types.INVALID_ARGUMENT, "No database clientUrl provided. Please provide the clientUrl through the [MODULE]_POSTGRES_URL, MEDUSA_POSTGRES_URL or POSTGRES_URL environment variable or the options object in the initialize function." diff --git a/packages/utils/src/orchestration/index.ts b/packages/utils/src/orchestration/index.ts index e6355e4311..98bcdbbef7 100644 --- a/packages/utils/src/orchestration/index.ts +++ b/packages/utils/src/orchestration/index.ts @@ -1 +1,2 @@ export * from "./symbol" +export * from "./types" diff --git a/packages/utils/src/orchestration/types.ts b/packages/utils/src/orchestration/types.ts new file mode 100644 index 0000000000..26cba445d6 --- /dev/null +++ b/packages/utils/src/orchestration/types.ts @@ -0,0 +1,34 @@ +export enum TransactionHandlerType { + INVOKE = "invoke", + COMPENSATE = "compensate", +} + +export enum TransactionStepStatus { + IDLE = "idle", + OK = "ok", + WAITING = "waiting_response", + TEMPORARY_FAILURE = "temp_failure", + PERMANENT_FAILURE = "permanent_failure", +} + +export enum TransactionState { + NOT_STARTED = "not_started", + INVOKING = "invoking", + WAITING_TO_COMPENSATE = "waiting_to_compensate", + COMPENSATING = "compensating", + DONE = "done", + REVERTED = "reverted", + FAILED = "failed", +} + +export enum TransactionStepState { + NOT_STARTED = "not_started", + INVOKING = "invoking", + COMPENSATING = "compensating", + DONE = "done", + REVERTED = "reverted", + FAILED = "failed", + DORMANT = "dormant", + SKIPPED = "skipped", + TIMEOUT = "timeout", +} diff --git a/packages/workflow-engine-inmemory/.gitignore b/packages/workflow-engine-inmemory/.gitignore new file mode 100644 index 0000000000..874c6c69d3 --- /dev/null +++ b/packages/workflow-engine-inmemory/.gitignore @@ -0,0 +1,6 @@ +/dist +node_modules +.DS_store +.env* +.env +*.sql diff --git a/packages/workflow-engine-inmemory/CHANGELOG.md b/packages/workflow-engine-inmemory/CHANGELOG.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/workflow-engine-inmemory/README.md b/packages/workflow-engine-inmemory/README.md new file mode 100644 index 0000000000..b34e46ea20 --- /dev/null +++ b/packages/workflow-engine-inmemory/README.md @@ -0,0 +1 @@ +# Workflow Orchestrator diff --git a/packages/workflow-engine-inmemory/integration-tests/__fixtures__/index.ts b/packages/workflow-engine-inmemory/integration-tests/__fixtures__/index.ts new file mode 100644 index 0000000000..987a8a99bd --- /dev/null +++ b/packages/workflow-engine-inmemory/integration-tests/__fixtures__/index.ts @@ -0,0 +1,4 @@ +export * from "./workflow_1" +export * from "./workflow_2" +export * from "./workflow_step_timeout" +export * from "./workflow_transaction_timeout" diff --git a/packages/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_1.ts b/packages/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_1.ts new file mode 100644 index 0000000000..cb0056466e --- /dev/null +++ b/packages/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_1.ts @@ -0,0 +1,65 @@ +import { + StepResponse, + createStep, + createWorkflow, +} from "@medusajs/workflows-sdk" + +const step_1 = createStep( + "step_1", + jest.fn((input) => { + input.test = "test" + return new StepResponse(input, { compensate: 123 }) + }), + jest.fn((compensateInput) => { + if (!compensateInput) { + return + } + + console.log("reverted", compensateInput.compensate) + return new StepResponse({ + reverted: true, + }) + }) +) + +const step_2 = createStep( + "step_2", + jest.fn((input, context) => { + console.log("triggered async request", context.metadata.idempotency_key) + + if (input) { + return new StepResponse({ notAsyncResponse: input.hey }) + } + }), + jest.fn((_, context) => { + return new StepResponse({ + step: context.metadata.action, + idempotency_key: context.metadata.idempotency_key, + reverted: true, + }) + }) +) + +const step_3 = createStep( + "step_3", + jest.fn((res) => { + return new StepResponse({ + done: { + inputFromSyncStep: res.notAsyncResponse, + }, + }) + }) +) + +createWorkflow("workflow_1", function (input) { + step_1(input) + + const ret2 = step_2({ hey: "oh" }) + + step_2({ hey: "async hello" }).config({ + name: "new_step_name", + async: true, + }) + + return step_3(ret2) +}) diff --git a/packages/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_2.ts b/packages/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_2.ts new file mode 100644 index 0000000000..f15d51889f --- /dev/null +++ b/packages/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_2.ts @@ -0,0 +1,71 @@ +import { + StepResponse, + createStep, + createWorkflow, +} from "@medusajs/workflows-sdk" + +const step_1 = createStep( + "step_1", + jest.fn((input) => { + input.test = "test" + return new StepResponse(input, { compensate: 123 }) + }), + jest.fn((compensateInput) => { + if (!compensateInput) { + return + } + + console.log("reverted", compensateInput.compensate) + return new StepResponse({ + reverted: true, + }) + }) +) + +const step_2 = createStep( + "step_2", + jest.fn((input, context) => { + console.log("triggered async request", context.metadata.idempotency_key) + + if (input) { + return new StepResponse({ notAsyncResponse: input.hey }) + } + }), + jest.fn((_, context) => { + return new StepResponse({ + step: context.metadata.action, + idempotency_key: context.metadata.idempotency_key, + reverted: true, + }) + }) +) + +const step_3 = createStep( + "step_3", + jest.fn((res) => { + return new StepResponse({ + done: { + inputFromSyncStep: res.notAsyncResponse, + }, + }) + }) +) + +createWorkflow( + { + name: "workflow_2", + retentionTime: 1000, + }, + function (input) { + step_1(input) + + const ret2 = step_2({ hey: "oh" }) + + step_2({ hey: "async hello" }).config({ + name: "new_step_name", + async: true, + }) + + return step_3(ret2) + } +) diff --git a/packages/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_step_timeout.ts b/packages/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_step_timeout.ts new file mode 100644 index 0000000000..a97112ffc1 --- /dev/null +++ b/packages/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_step_timeout.ts @@ -0,0 +1,29 @@ +import { + StepResponse, + createStep, + createWorkflow, +} from "@medusajs/workflows-sdk" +import { setTimeout } from "timers/promises" + +const step_1 = createStep( + "step_1", + jest.fn(async (input) => { + await setTimeout(200) + + return new StepResponse(input, { compensate: 123 }) + }), + jest.fn(() => {}) +) + +createWorkflow( + { + name: "workflow_step_timeout", + }, + function (input) { + const resp = step_1(input).config({ + timeout: 0.1, // 0.1 second + }) + + return resp + } +) diff --git a/packages/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_transaction_timeout.ts b/packages/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_transaction_timeout.ts new file mode 100644 index 0000000000..154da2b5d4 --- /dev/null +++ b/packages/workflow-engine-inmemory/integration-tests/__fixtures__/workflow_transaction_timeout.ts @@ -0,0 +1,36 @@ +import { + StepResponse, + createStep, + createWorkflow, +} from "@medusajs/workflows-sdk" + +const step_1 = createStep( + "step_1", + jest.fn((input) => { + input.test = "test" + return new StepResponse(input, { compensate: 123 }) + }), + jest.fn((compensateInput) => { + if (!compensateInput) { + return + } + + return new StepResponse({ + reverted: true, + }) + }) +) + +createWorkflow( + { + name: "workflow_transaction_timeout", + timeout: 0.1, // 0.1 second + }, + function (input) { + const resp = step_1(input).config({ + async: true, + }) + + return resp + } +) diff --git a/packages/workflow-engine-inmemory/integration-tests/__tests__/index.spec.ts b/packages/workflow-engine-inmemory/integration-tests/__tests__/index.spec.ts new file mode 100644 index 0000000000..11b92ab0cb --- /dev/null +++ b/packages/workflow-engine-inmemory/integration-tests/__tests__/index.spec.ts @@ -0,0 +1,163 @@ +import { MedusaApp } from "@medusajs/modules-sdk" +import { RemoteJoinerQuery } from "@medusajs/types" +import { TransactionHandlerType } from "@medusajs/utils" +import { IWorkflowsModuleService } from "@medusajs/workflows-sdk" +import { knex } from "knex" +import { setTimeout } from "timers/promises" +import "../__fixtures__" +import { DB_URL, TestDatabase } from "../utils" + +const sharedPgConnection = knex({ + client: "pg", + searchPath: process.env.MEDUSA_WORKFLOW_ENGINE_DB_SCHEMA, + connection: { + connectionString: DB_URL, + debug: false, + }, +}) + +const afterEach_ = async () => { + await TestDatabase.clearTables(sharedPgConnection) +} + +describe("Workflow Orchestrator module", function () { + describe("Testing basic workflow", function () { + let workflowOrcModule: IWorkflowsModuleService + let query: ( + query: string | RemoteJoinerQuery | object, + variables?: Record + ) => Promise + + afterEach(afterEach_) + + beforeAll(async () => { + const { + runMigrations, + query: remoteQuery, + modules, + } = await MedusaApp({ + sharedResourcesConfig: { + database: { + connection: sharedPgConnection, + }, + }, + modulesConfig: { + workflows: { + resolve: __dirname + "/../..", + }, + }, + }) + + query = remoteQuery + + await runMigrations() + + workflowOrcModule = + modules.workflows as unknown as IWorkflowsModuleService + }) + + afterEach(afterEach_) + + it("should return a list of workflow executions and remove after completed when there is no retentionTime set", async () => { + await workflowOrcModule.run("workflow_1", { + input: { + value: "123", + }, + throwOnError: true, + }) + + let executionsList = await query({ + workflow_executions: { + fields: ["workflow_id", "transaction_id", "state"], + }, + }) + + expect(executionsList).toHaveLength(1) + + const { result } = await workflowOrcModule.setStepSuccess({ + idempotencyKey: { + action: TransactionHandlerType.INVOKE, + stepId: "new_step_name", + workflowId: "workflow_1", + transactionId: executionsList[0].transaction_id, + }, + stepResponse: { uhuuuu: "yeaah!" }, + }) + + executionsList = await query({ + workflow_executions: { + fields: ["id"], + }, + }) + + expect(executionsList).toHaveLength(0) + expect(result).toEqual({ + done: { + inputFromSyncStep: "oh", + }, + }) + }) + + it("should return a list of workflow executions and keep it saved when there is a retentionTime set", async () => { + await workflowOrcModule.run("workflow_2", { + input: { + value: "123", + }, + throwOnError: true, + transactionId: "transaction_1", + }) + + let executionsList = await query({ + workflow_executions: { + fields: ["id"], + }, + }) + + expect(executionsList).toHaveLength(1) + + await workflowOrcModule.setStepSuccess({ + idempotencyKey: { + action: TransactionHandlerType.INVOKE, + stepId: "new_step_name", + workflowId: "workflow_2", + transactionId: "transaction_1", + }, + stepResponse: { uhuuuu: "yeaah!" }, + }) + + executionsList = await query({ + workflow_executions: { + fields: ["id"], + }, + }) + + expect(executionsList).toHaveLength(1) + }) + + it("should revert the entire transaction when a step timeout expires", async () => { + const { transaction } = await workflowOrcModule.run( + "workflow_step_timeout", + { + input: {}, + throwOnError: false, + } + ) + + expect(transaction.flow.state).toEqual("reverted") + }) + + it("should revert the entire transaction when the transaction timeout expires", async () => { + const { transaction } = await workflowOrcModule.run( + "workflow_transaction_timeout", + { + input: {}, + throwOnError: false, + } + ) + + await setTimeout(200) + + expect(transaction.flow.state).toEqual("reverted") + }) + }) +}) diff --git a/packages/workflow-engine-inmemory/integration-tests/setup-env.js b/packages/workflow-engine-inmemory/integration-tests/setup-env.js new file mode 100644 index 0000000000..7de2d9de24 --- /dev/null +++ b/packages/workflow-engine-inmemory/integration-tests/setup-env.js @@ -0,0 +1,6 @@ +if (typeof process.env.DB_TEMP_NAME === "undefined") { + const tempName = parseInt(process.env.JEST_WORKER_ID || "1") + process.env.DB_TEMP_NAME = `medusa-workflow-engine-inmemory-${tempName}` +} + +process.env.MEDUSA_WORKFLOW_ENGINE_DB_SCHEMA = "public" diff --git a/packages/workflow-engine-inmemory/integration-tests/setup.js b/packages/workflow-engine-inmemory/integration-tests/setup.js new file mode 100644 index 0000000000..43f99aab4a --- /dev/null +++ b/packages/workflow-engine-inmemory/integration-tests/setup.js @@ -0,0 +1,3 @@ +import { JestUtils } from "medusa-test-utils" + +JestUtils.afterAllHookDropDatabase() diff --git a/packages/workflow-engine-inmemory/integration-tests/utils/database.ts b/packages/workflow-engine-inmemory/integration-tests/utils/database.ts new file mode 100644 index 0000000000..ed61b5e489 --- /dev/null +++ b/packages/workflow-engine-inmemory/integration-tests/utils/database.ts @@ -0,0 +1,22 @@ +import * as process from "process" + +const DB_HOST = process.env.DB_HOST ?? "localhost" +const DB_USERNAME = process.env.DB_USERNAME ?? "" +const DB_PASSWORD = process.env.DB_PASSWORD +const DB_NAME = process.env.DB_TEMP_NAME + +export const DB_URL = `postgres://${DB_USERNAME}${ + DB_PASSWORD ? `:${DB_PASSWORD}` : "" +}@${DB_HOST}/${DB_NAME}` + +interface TestDatabase { + clearTables(knex): Promise +} + +export const TestDatabase: TestDatabase = { + clearTables: async (knex) => { + await knex.raw(` + TRUNCATE TABLE workflow_execution CASCADE; + `) + }, +} diff --git a/packages/workflow-engine-inmemory/integration-tests/utils/index.ts b/packages/workflow-engine-inmemory/integration-tests/utils/index.ts new file mode 100644 index 0000000000..6b917ed30e --- /dev/null +++ b/packages/workflow-engine-inmemory/integration-tests/utils/index.ts @@ -0,0 +1 @@ +export * from "./database" diff --git a/packages/workflow-engine-inmemory/jest.config.js b/packages/workflow-engine-inmemory/jest.config.js new file mode 100644 index 0000000000..456054fe8a --- /dev/null +++ b/packages/workflow-engine-inmemory/jest.config.js @@ -0,0 +1,22 @@ +module.exports = { + moduleNameMapper: { + "^@models": "/src/models", + "^@services": "/src/services", + "^@repositories": "/src/repositories", + "^@types": "/src/types", + }, + transform: { + "^.+\\.[jt]s?$": [ + "ts-jest", + { + tsConfig: "tsconfig.spec.json", + isolatedModules: true, + }, + ], + }, + testEnvironment: `node`, + moduleFileExtensions: [`js`, `ts`], + modulePathIgnorePatterns: ["dist/"], + setupFiles: ["/integration-tests/setup-env.js"], + setupFilesAfterEnv: ["/integration-tests/setup.js"], +} diff --git a/packages/workflow-engine-inmemory/mikro-orm.config.dev.ts b/packages/workflow-engine-inmemory/mikro-orm.config.dev.ts new file mode 100644 index 0000000000..81651a7600 --- /dev/null +++ b/packages/workflow-engine-inmemory/mikro-orm.config.dev.ts @@ -0,0 +1,8 @@ +import * as entities from "./src/models" + +module.exports = { + entities: Object.values(entities), + schema: "public", + clientUrl: "postgres://postgres@localhost/medusa-workflow-engine-inmemory", + type: "postgresql", +} diff --git a/packages/workflow-engine-inmemory/package.json b/packages/workflow-engine-inmemory/package.json new file mode 100644 index 0000000000..d82f33d8b7 --- /dev/null +++ b/packages/workflow-engine-inmemory/package.json @@ -0,0 +1,59 @@ +{ + "name": "@medusajs/workflow-engine-inmemory", + "version": "0.0.1", + "description": "Medusa Workflow Orchestrator module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "dist" + ], + "engines": { + "node": ">=16" + }, + "repository": { + "type": "git", + "url": "https://github.com/medusajs/medusa", + "directory": "packages/workflow-engine-inmemory" + }, + "publishConfig": { + "access": "public" + }, + "author": "Medusa", + "license": "MIT", + "scripts": { + "watch": "tsc --build --watch", + "watch:test": "tsc --build tsconfig.spec.json --watch", + "prepublishOnly": "cross-env NODE_ENV=production tsc --build && tsc-alias -p tsconfig.json", + "build": "rimraf dist && tsc --build && tsc-alias -p tsconfig.json", + "test": "jest --passWithNoTests --runInBand --bail --forceExit -- src/**/__tests__/**/*.ts", + "test:integration": "jest --runInBand --forceExit -- integration-tests/**/__tests__/**/*.ts", + "migration:generate": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:generate", + "migration:initial": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:create --initial", + "migration:create": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:create", + "migration:up": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:up", + "orm:cache:clear": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm cache:clear" + }, + "devDependencies": { + "@mikro-orm/cli": "5.9.7", + "cross-env": "^5.2.1", + "jest": "^29.6.3", + "medusa-test-utils": "^1.1.40", + "rimraf": "^3.0.2", + "ts-jest": "^29.1.1", + "ts-node": "^10.9.1", + "tsc-alias": "^1.8.6", + "typescript": "^5.1.6" + }, + "dependencies": { + "@medusajs/modules-sdk": "^1.12.5", + "@medusajs/types": "^1.11.9", + "@medusajs/utils": "^1.11.2", + "@medusajs/workflows-sdk": "^0.1.0", + "@mikro-orm/core": "5.9.7", + "@mikro-orm/migrations": "5.9.7", + "@mikro-orm/postgresql": "5.9.7", + "awilix": "^8.0.0", + "dotenv": "^16.1.4", + "knex": "2.4.2" + } +} diff --git a/packages/workflow-engine-inmemory/src/index.ts b/packages/workflow-engine-inmemory/src/index.ts new file mode 100644 index 0000000000..7804040565 --- /dev/null +++ b/packages/workflow-engine-inmemory/src/index.ts @@ -0,0 +1,22 @@ +import { Modules } from "@medusajs/modules-sdk" +import { ModulesSdkUtils } from "@medusajs/utils" +import * as models from "@models" +import { moduleDefinition } from "./module-definition" + +export default moduleDefinition + +const migrationScriptOptions = { + moduleName: Modules.WORKFLOW_ENGINE, + models: models, + pathToMigrations: __dirname + "/migrations", +} + +export const runMigrations = ModulesSdkUtils.buildMigrationScript( + migrationScriptOptions +) +export const revertMigration = ModulesSdkUtils.buildRevertMigrationScript( + migrationScriptOptions +) + +export * from "./initialize" +export * from "./loaders" diff --git a/packages/workflow-engine-inmemory/src/initialize/index.ts b/packages/workflow-engine-inmemory/src/initialize/index.ts new file mode 100644 index 0000000000..20f4f49231 --- /dev/null +++ b/packages/workflow-engine-inmemory/src/initialize/index.ts @@ -0,0 +1,36 @@ +import { + ExternalModuleDeclaration, + InternalModuleDeclaration, + MedusaModule, + MODULE_PACKAGE_NAMES, + Modules, +} from "@medusajs/modules-sdk" +import { ModulesSdkTypes } from "@medusajs/types" +import { WorkflowOrchestratorTypes } from "@medusajs/workflows-sdk" +import { moduleDefinition } from "../module-definition" +import { InitializeModuleInjectableDependencies } from "../types" + +export const initialize = async ( + options?: + | ModulesSdkTypes.ModuleServiceInitializeOptions + | ModulesSdkTypes.ModuleServiceInitializeCustomDataLayerOptions + | ExternalModuleDeclaration + | InternalModuleDeclaration, + injectedDependencies?: InitializeModuleInjectableDependencies +): Promise => { + const loaded = + // eslint-disable-next-line max-len + await MedusaModule.bootstrap( + { + moduleKey: Modules.WORKFLOW_ENGINE, + defaultPath: MODULE_PACKAGE_NAMES[Modules.WORKFLOW_ENGINE], + declaration: options as + | InternalModuleDeclaration + | ExternalModuleDeclaration, + injectedDependencies, + moduleExports: moduleDefinition, + } + ) + + return loaded[Modules.WORKFLOW_ENGINE] +} diff --git a/packages/workflow-engine-inmemory/src/joiner-config.ts b/packages/workflow-engine-inmemory/src/joiner-config.ts new file mode 100644 index 0000000000..7999e9c3ab --- /dev/null +++ b/packages/workflow-engine-inmemory/src/joiner-config.ts @@ -0,0 +1,34 @@ +import { Modules } from "@medusajs/modules-sdk" +import { ModuleJoinerConfig } from "@medusajs/types" +import { MapToConfig } from "@medusajs/utils" +import { WorkflowExecution } from "@models" +import moduleSchema from "./schema" + +export const LinkableKeys = { + workflow_execution_id: WorkflowExecution.name, +} + +const entityLinkableKeysMap: MapToConfig = {} +Object.entries(LinkableKeys).forEach(([key, value]) => { + entityLinkableKeysMap[value] ??= [] + entityLinkableKeysMap[value].push({ + mapTo: key, + valueFrom: key.split("_").pop()!, + }) +}) + +export const entityNameToLinkableKeysMap: MapToConfig = entityLinkableKeysMap + +export const joinerConfig: ModuleJoinerConfig = { + serviceName: Modules.WORKFLOW_ENGINE, + primaryKeys: ["id"], + schema: moduleSchema, + linkableKeys: LinkableKeys, + alias: { + name: ["workflow_execution", "workflow_executions"], + args: { + entity: WorkflowExecution.name, + methodSuffix: "WorkflowExecution", + }, + }, +} diff --git a/packages/workflow-engine-inmemory/src/loaders/connection.ts b/packages/workflow-engine-inmemory/src/loaders/connection.ts new file mode 100644 index 0000000000..580e05e95c --- /dev/null +++ b/packages/workflow-engine-inmemory/src/loaders/connection.ts @@ -0,0 +1,36 @@ +import { + InternalModuleDeclaration, + LoaderOptions, + Modules, +} from "@medusajs/modules-sdk" +import { ModulesSdkTypes } from "@medusajs/types" +import { ModulesSdkUtils } from "@medusajs/utils" +import { EntitySchema } from "@mikro-orm/core" +import * as WorkflowOrchestratorModels from "../models" + +export default async ( + { + options, + container, + logger, + }: LoaderOptions< + | ModulesSdkTypes.ModuleServiceInitializeOptions + | ModulesSdkTypes.ModuleServiceInitializeCustomDataLayerOptions + >, + moduleDeclaration?: InternalModuleDeclaration +): Promise => { + const entities = Object.values( + WorkflowOrchestratorModels + ) as unknown as EntitySchema[] + const pathToMigrations = __dirname + "/../migrations" + + await ModulesSdkUtils.mikroOrmConnectionLoader({ + moduleName: Modules.WORKFLOW_ENGINE, + entities, + container, + options, + moduleDeclaration, + logger, + pathToMigrations, + }) +} diff --git a/packages/workflow-engine-inmemory/src/loaders/container.ts b/packages/workflow-engine-inmemory/src/loaders/container.ts new file mode 100644 index 0000000000..9a0c5553b4 --- /dev/null +++ b/packages/workflow-engine-inmemory/src/loaders/container.ts @@ -0,0 +1,9 @@ +import { MikroOrmBaseRepository, ModulesSdkUtils } from "@medusajs/utils" +import * as ModuleModels from "@models" +import * as ModuleServices from "@services" + +export default ModulesSdkUtils.moduleContainerLoaderFactory({ + moduleModels: ModuleModels, + moduleServices: ModuleServices, + moduleRepositories: { BaseRepository: MikroOrmBaseRepository }, +}) diff --git a/packages/workflow-engine-inmemory/src/loaders/index.ts b/packages/workflow-engine-inmemory/src/loaders/index.ts new file mode 100644 index 0000000000..5445bc7412 --- /dev/null +++ b/packages/workflow-engine-inmemory/src/loaders/index.ts @@ -0,0 +1,3 @@ +export * from "./connection" +export * from "./container" +export * from "./utils" diff --git a/packages/workflow-engine-inmemory/src/loaders/utils.ts b/packages/workflow-engine-inmemory/src/loaders/utils.ts new file mode 100644 index 0000000000..3131eb8f92 --- /dev/null +++ b/packages/workflow-engine-inmemory/src/loaders/utils.ts @@ -0,0 +1,10 @@ +import { asClass } from "awilix" +import { InMemoryDistributedTransactionStorage } from "../utils" + +export default async ({ container }): Promise => { + container.register({ + inMemoryDistributedTransactionStorage: asClass( + InMemoryDistributedTransactionStorage + ).singleton(), + }) +} diff --git a/packages/workflow-engine-inmemory/src/migrations/Migration20231228143900.ts b/packages/workflow-engine-inmemory/src/migrations/Migration20231228143900.ts new file mode 100644 index 0000000000..af9958e80a --- /dev/null +++ b/packages/workflow-engine-inmemory/src/migrations/Migration20231228143900.ts @@ -0,0 +1,41 @@ +import { Migration } from "@mikro-orm/migrations" + +export class Migration20231221104256 extends Migration { + async up(): Promise { + this.addSql( + ` + CREATE TABLE IF NOT EXISTS workflow_execution + ( + id character varying NOT NULL, + workflow_id character varying NOT NULL, + transaction_id character varying NOT NULL, + execution jsonb NULL, + context jsonb NULL, + state character varying NOT NULL, + created_at timestamp WITHOUT time zone NOT NULL DEFAULT Now(), + updated_at timestamp WITHOUT time zone NOT NULL DEFAULT Now(), + deleted_at timestamp WITHOUT time zone NULL, + CONSTRAINT "PK_workflow_execution_workflow_id_transaction_id" PRIMARY KEY ("workflow_id", "transaction_id") + ); + + CREATE UNIQUE INDEX IF NOT EXISTS "IDX_workflow_execution_id" ON "workflow_execution" ("id"); + CREATE INDEX IF NOT EXISTS "IDX_workflow_execution_workflow_id" ON "workflow_execution" ("workflow_id") WHERE deleted_at IS NULL; + CREATE INDEX IF NOT EXISTS "IDX_workflow_execution_transaction_id" ON "workflow_execution" ("transaction_id") WHERE deleted_at IS NULL; + CREATE INDEX IF NOT EXISTS "IDX_workflow_execution_state" ON "workflow_execution" ("state") WHERE deleted_at IS NULL; + ` + ) + } + + async down(): Promise { + this.addSql( + ` + DROP INDEX "IDX_workflow_execution_id"; + DROP INDEX "IDX_workflow_execution_workflow_id"; + DROP INDEX "IDX_workflow_execution_transaction_id"; + DROP INDEX "IDX_workflow_execution_state"; + + DROP TABLE IF EXISTS workflow_execution; + ` + ) + } +} diff --git a/packages/workflow-engine-inmemory/src/models/index.ts b/packages/workflow-engine-inmemory/src/models/index.ts new file mode 100644 index 0000000000..78fcbfa921 --- /dev/null +++ b/packages/workflow-engine-inmemory/src/models/index.ts @@ -0,0 +1 @@ +export { default as WorkflowExecution } from "./workflow-execution" diff --git a/packages/workflow-engine-inmemory/src/models/workflow-execution.ts b/packages/workflow-engine-inmemory/src/models/workflow-execution.ts new file mode 100644 index 0000000000..753d9e62db --- /dev/null +++ b/packages/workflow-engine-inmemory/src/models/workflow-execution.ts @@ -0,0 +1,76 @@ +import { TransactionState } from "@medusajs/orchestration" +import { DALUtils, generateEntityId } from "@medusajs/utils" +import { + BeforeCreate, + Entity, + Enum, + Filter, + Index, + OnInit, + OptionalProps, + PrimaryKey, + Property, + Unique, +} from "@mikro-orm/core" + +type OptionalFields = "deleted_at" + +@Entity() +@Unique({ + name: "IDX_workflow_execution_workflow_id_transaction_id_unique", + properties: ["workflow_id", "transaction_id"], +}) +@Filter(DALUtils.mikroOrmSoftDeletableFilterOptions) +export default class WorkflowExecution { + [OptionalProps]?: OptionalFields + + @Property({ columnType: "text", nullable: false }) + @Index({ name: "IDX_workflow_execution_id" }) + id!: string + + @Index({ name: "IDX_workflow_execution_workflow_id" }) + @PrimaryKey({ columnType: "text" }) + workflow_id: string + + @Index({ name: "IDX_workflow_execution_transaction_id" }) + @PrimaryKey({ columnType: "text" }) + transaction_id: string + + @Property({ columnType: "jsonb", nullable: true }) + execution: Record | null = null + + @Property({ columnType: "jsonb", nullable: true }) + context: Record | null = null + + @Index({ name: "IDX_workflow_execution_state" }) + @Enum(() => TransactionState) + state: TransactionState + + @Property({ + onCreate: () => new Date(), + columnType: "timestamptz", + defaultRaw: "now()", + }) + created_at: Date + + @Property({ + onCreate: () => new Date(), + onUpdate: () => new Date(), + columnType: "timestamptz", + defaultRaw: "now()", + }) + updated_at: Date + + @Property({ columnType: "timestamptz", nullable: true }) + deleted_at: Date | null = null + + @BeforeCreate() + onCreate() { + this.id = generateEntityId(this.id, "wf_exec") + } + + @OnInit() + onInit() { + this.id = generateEntityId(this.id, "wf_exec") + } +} diff --git a/packages/workflow-engine-inmemory/src/module-definition.ts b/packages/workflow-engine-inmemory/src/module-definition.ts new file mode 100644 index 0000000000..b86c23807b --- /dev/null +++ b/packages/workflow-engine-inmemory/src/module-definition.ts @@ -0,0 +1,13 @@ +import { ModuleExports } from "@medusajs/types" +import { WorkflowsModuleService } from "@services" +import loadConnection from "./loaders/connection" +import loadContainer from "./loaders/container" +import loadUtils from "./loaders/utils" + +const service = WorkflowsModuleService +const loaders = [loadContainer, loadConnection, loadUtils] as any + +export const moduleDefinition: ModuleExports = { + service, + loaders, +} diff --git a/packages/workflow-engine-inmemory/src/repositories/index.ts b/packages/workflow-engine-inmemory/src/repositories/index.ts new file mode 100644 index 0000000000..8def202608 --- /dev/null +++ b/packages/workflow-engine-inmemory/src/repositories/index.ts @@ -0,0 +1,2 @@ +export { MikroOrmBaseRepository as BaseRepository } from "@medusajs/utils" +export { WorkflowExecutionRepository } from "./workflow-execution" diff --git a/packages/workflow-engine-inmemory/src/repositories/workflow-execution.ts b/packages/workflow-engine-inmemory/src/repositories/workflow-execution.ts new file mode 100644 index 0000000000..9e6553ec74 --- /dev/null +++ b/packages/workflow-engine-inmemory/src/repositories/workflow-execution.ts @@ -0,0 +1,7 @@ +import { DALUtils } from "@medusajs/utils" +import { WorkflowExecution } from "@models" + +// eslint-disable-next-line max-len +export class WorkflowExecutionRepository extends DALUtils.mikroOrmBaseRepositoryFactory( + WorkflowExecution +) {} diff --git a/packages/workflow-engine-inmemory/src/schema/index.ts b/packages/workflow-engine-inmemory/src/schema/index.ts new file mode 100644 index 0000000000..3d7d91edea --- /dev/null +++ b/packages/workflow-engine-inmemory/src/schema/index.ts @@ -0,0 +1,26 @@ +export default ` +scalar DateTime +scalar JSON + +enum TransactionState { + NOT_STARTED + INVOKING + WAITING_TO_COMPENSATE + COMPENSATING + DONE + REVERTED + FAILED +} + +type WorkflowExecution { + id: ID! + created_at: DateTime! + updated_at: DateTime! + deleted_at: DateTime + workflow_id: string + transaction_id: string + execution: JSON + context: JSON + state: TransactionState +} +` diff --git a/packages/workflow-engine-inmemory/src/services/__tests__/index.spec.ts b/packages/workflow-engine-inmemory/src/services/__tests__/index.spec.ts new file mode 100644 index 0000000000..728f6245c6 --- /dev/null +++ b/packages/workflow-engine-inmemory/src/services/__tests__/index.spec.ts @@ -0,0 +1,5 @@ +describe("Noop test", () => { + it("noop check", async () => { + expect(true).toBe(true) + }) +}) diff --git a/packages/workflow-engine-inmemory/src/services/index.ts b/packages/workflow-engine-inmemory/src/services/index.ts new file mode 100644 index 0000000000..5a6d313d86 --- /dev/null +++ b/packages/workflow-engine-inmemory/src/services/index.ts @@ -0,0 +1,3 @@ +export * from "./workflow-execution" +export * from "./workflow-orchestrator" +export * from "./workflows-module" diff --git a/packages/workflow-engine-inmemory/src/services/workflow-execution.ts b/packages/workflow-engine-inmemory/src/services/workflow-execution.ts new file mode 100644 index 0000000000..158557ec0b --- /dev/null +++ b/packages/workflow-engine-inmemory/src/services/workflow-execution.ts @@ -0,0 +1,21 @@ +import { DAL } from "@medusajs/types" +import { ModulesSdkUtils } from "@medusajs/utils" +import { WorkflowExecution } from "@models" + +type InjectedDependencies = { + workflowExecutionRepository: DAL.RepositoryService +} + +export class WorkflowExecutionService< + TEntity extends WorkflowExecution = WorkflowExecution +> extends ModulesSdkUtils.abstractServiceFactory( + WorkflowExecution +) { + protected workflowExecutionRepository_: DAL.RepositoryService + + constructor({ workflowExecutionRepository }: InjectedDependencies) { + // @ts-ignore + super(...arguments) + this.workflowExecutionRepository_ = workflowExecutionRepository + } +} diff --git a/packages/workflow-engine-inmemory/src/services/workflow-orchestrator.ts b/packages/workflow-engine-inmemory/src/services/workflow-orchestrator.ts new file mode 100644 index 0000000000..55b2f33f15 --- /dev/null +++ b/packages/workflow-engine-inmemory/src/services/workflow-orchestrator.ts @@ -0,0 +1,528 @@ +import { + DistributedTransaction, + DistributedTransactionEvents, + TransactionHandlerType, + TransactionStep, +} from "@medusajs/orchestration" +import { ContainerLike, Context, MedusaContainer } from "@medusajs/types" +import { InjectSharedContext, isString, MedusaContext } from "@medusajs/utils" +import { + type FlowRunOptions, + MedusaWorkflow, + ReturnWorkflow, +} from "@medusajs/workflows-sdk" +import { ulid } from "ulid" +import { InMemoryDistributedTransactionStorage } from "../utils" + +export type WorkflowOrchestratorRunOptions = FlowRunOptions & { + transactionId?: string + container?: ContainerLike +} + +type RegisterStepSuccessOptions = Omit< + WorkflowOrchestratorRunOptions, + "transactionId" | "input" +> + +type IdempotencyKeyParts = { + workflowId: string + transactionId: string + stepId: string + action: "invoke" | "compensate" +} + +type NotifyOptions = { + eventType: keyof DistributedTransactionEvents + workflowId: string + transactionId?: string + step?: TransactionStep + response?: unknown + result?: unknown + errors?: unknown[] +} + +type WorkflowId = string +type TransactionId = string + +type SubscriberHandler = { + (input: NotifyOptions): void +} & { + _id?: string +} + +type SubscribeOptions = { + workflowId: string + transactionId?: string + subscriber: SubscriberHandler + subscriberId?: string +} + +type UnsubscribeOptions = { + workflowId: string + transactionId?: string + subscriberOrId: string | SubscriberHandler +} + +type TransactionSubscribers = Map +type Subscribers = Map + +const AnySubscriber = "any" + +export class WorkflowOrchestratorService { + private subscribers: Subscribers = new Map() + + constructor({ + inMemoryDistributedTransactionStorage, + }: { + inMemoryDistributedTransactionStorage: InMemoryDistributedTransactionStorage + workflowOrchestratorService: WorkflowOrchestratorService + }) { + inMemoryDistributedTransactionStorage.setWorkflowOrchestratorService(this) + DistributedTransaction.setStorage(inMemoryDistributedTransactionStorage) + } + + @InjectSharedContext() + async run( + workflowIdOrWorkflow: string | ReturnWorkflow, + options?: WorkflowOrchestratorRunOptions, + @MedusaContext() sharedContext: Context = {} + ) { + let { + input, + context, + transactionId, + resultFrom, + throwOnError, + events: eventHandlers, + container, + } = options ?? {} + + const workflowId = isString(workflowIdOrWorkflow) + ? workflowIdOrWorkflow + : workflowIdOrWorkflow.getName() + + if (!workflowId) { + throw new Error("Workflow ID is required") + } + + context ??= {} + context.transactionId ??= transactionId ?? ulid() + + const events: FlowRunOptions["events"] = this.buildWorkflowEvents({ + customEventHandlers: eventHandlers, + workflowId, + transactionId: context.transactionId, + }) + + const exportedWorkflow: any = MedusaWorkflow.getWorkflow(workflowId) + if (!exportedWorkflow) { + throw new Error(`Workflow with id "${workflowId}" not found.`) + } + + const flow = exportedWorkflow(container as MedusaContainer) + + const ret = await flow.run({ + input, + throwOnError, + resultFrom, + context, + events, + }) + + // TODO: temporary + const acknowledgement = { + transactionId: context.transactionId, + workflowId: workflowId, + } + + if (ret.transaction.hasFinished()) { + const { result, errors } = ret + this.notify({ + eventType: "onFinish", + workflowId, + transactionId: context.transactionId, + result, + errors, + }) + } + + return { acknowledgement, ...ret } + } + + @InjectSharedContext() + async getRunningTransaction( + workflowId: string, + transactionId: string, + options?: WorkflowOrchestratorRunOptions, + @MedusaContext() sharedContext: Context = {} + ): Promise { + let { context, container } = options ?? {} + + if (!workflowId) { + throw new Error("Workflow ID is required") + } + + if (!transactionId) { + throw new Error("TransactionId ID is required") + } + + context ??= {} + context.transactionId ??= transactionId + + const exportedWorkflow: any = MedusaWorkflow.getWorkflow(workflowId) + if (!exportedWorkflow) { + throw new Error(`Workflow with id "${workflowId}" not found.`) + } + + const flow = exportedWorkflow(container as MedusaContainer) + + const transaction = await flow.getRunningTransaction(transactionId, context) + + return transaction + } + + @InjectSharedContext() + async setStepSuccess( + { + idempotencyKey, + stepResponse, + options, + }: { + idempotencyKey: string | IdempotencyKeyParts + stepResponse: unknown + options?: RegisterStepSuccessOptions + }, + @MedusaContext() sharedContext: Context = {} + ) { + const { + context, + throwOnError, + resultFrom, + container, + events: eventHandlers, + } = options ?? {} + + const [idempotencyKey_, { workflowId, transactionId }] = + this.buildIdempotencyKeyAndParts(idempotencyKey) + + const exportedWorkflow: any = MedusaWorkflow.getWorkflow(workflowId) + if (!exportedWorkflow) { + throw new Error(`Workflow with id "${workflowId}" not found.`) + } + + const flow = exportedWorkflow(container as MedusaContainer) + + const events = this.buildWorkflowEvents({ + customEventHandlers: eventHandlers, + transactionId, + workflowId, + }) + + const ret = await flow.registerStepSuccess({ + idempotencyKey: idempotencyKey_, + context, + resultFrom, + throwOnError, + events, + response: stepResponse, + }) + + if (ret.transaction.hasFinished()) { + const { result, errors } = ret + this.notify({ + eventType: "onFinish", + workflowId, + transactionId, + result, + errors, + }) + } + + return ret + } + + @InjectSharedContext() + async setStepFailure( + { + idempotencyKey, + stepResponse, + options, + }: { + idempotencyKey: string | IdempotencyKeyParts + stepResponse: unknown + options?: RegisterStepSuccessOptions + }, + @MedusaContext() sharedContext: Context = {} + ) { + const { + context, + throwOnError, + resultFrom, + container, + events: eventHandlers, + } = options ?? {} + + const [idempotencyKey_, { workflowId, transactionId }] = + this.buildIdempotencyKeyAndParts(idempotencyKey) + + const exportedWorkflow: any = MedusaWorkflow.getWorkflow(workflowId) + if (!exportedWorkflow) { + throw new Error(`Workflow with id "${workflowId}" not found.`) + } + + const flow = exportedWorkflow(container as MedusaContainer) + + const events = this.buildWorkflowEvents({ + customEventHandlers: eventHandlers, + transactionId, + workflowId, + }) + + const ret = await flow.registerStepFailure({ + idempotencyKey: idempotencyKey_, + context, + resultFrom, + throwOnError, + events, + response: stepResponse, + }) + + if (ret.transaction.hasFinished()) { + const { result, errors } = ret + this.notify({ + eventType: "onFinish", + workflowId, + transactionId, + result, + errors, + }) + } + + return ret + } + + @InjectSharedContext() + subscribe( + { workflowId, transactionId, subscriber, subscriberId }: SubscribeOptions, + @MedusaContext() sharedContext: Context = {} + ) { + subscriber._id = subscriberId + const subscribers = this.subscribers.get(workflowId) ?? new Map() + + const handlerIndex = (handlers) => { + return handlers.indexOf((s) => s === subscriber || s._id === subscriberId) + } + + if (transactionId) { + const transactionSubscribers = subscribers.get(transactionId) ?? [] + const subscriberIndex = handlerIndex(transactionSubscribers) + if (subscriberIndex !== -1) { + transactionSubscribers.slice(subscriberIndex, 1) + } + + transactionSubscribers.push(subscriber) + subscribers.set(transactionId, transactionSubscribers) + this.subscribers.set(workflowId, subscribers) + return + } + + const workflowSubscribers = subscribers.get(AnySubscriber) ?? [] + const subscriberIndex = handlerIndex(workflowSubscribers) + if (subscriberIndex !== -1) { + workflowSubscribers.slice(subscriberIndex, 1) + } + + workflowSubscribers.push(subscriber) + subscribers.set(AnySubscriber, workflowSubscribers) + this.subscribers.set(workflowId, subscribers) + } + + @InjectSharedContext() + unsubscribe( + { workflowId, transactionId, subscriberOrId }: UnsubscribeOptions, + @MedusaContext() sharedContext: Context = {} + ) { + const subscribers = this.subscribers.get(workflowId) ?? new Map() + + const filterSubscribers = (handlers: SubscriberHandler[]) => { + return handlers.filter((handler) => { + return handler._id + ? handler._id !== (subscriberOrId as string) + : handler !== (subscriberOrId as SubscriberHandler) + }) + } + + if (transactionId) { + const transactionSubscribers = subscribers.get(transactionId) ?? [] + const newTransactionSubscribers = filterSubscribers( + transactionSubscribers + ) + subscribers.set(transactionId, newTransactionSubscribers) + this.subscribers.set(workflowId, subscribers) + return + } + + const workflowSubscribers = subscribers.get(AnySubscriber) ?? [] + const newWorkflowSubscribers = filterSubscribers(workflowSubscribers) + subscribers.set(AnySubscriber, newWorkflowSubscribers) + this.subscribers.set(workflowId, subscribers) + } + + private notify(options: NotifyOptions) { + const { + eventType, + workflowId, + transactionId, + errors, + result, + step, + response, + } = options + + const subscribers: TransactionSubscribers = + this.subscribers.get(workflowId) ?? new Map() + + const notifySubscribers = (handlers: SubscriberHandler[]) => { + handlers.forEach((handler) => { + handler({ + eventType, + workflowId, + transactionId, + step, + response, + result, + errors, + }) + }) + } + + if (transactionId) { + const transactionSubscribers = subscribers.get(transactionId) ?? [] + notifySubscribers(transactionSubscribers) + } + + const workflowSubscribers = subscribers.get(AnySubscriber) ?? [] + notifySubscribers(workflowSubscribers) + } + + private buildWorkflowEvents({ + customEventHandlers, + workflowId, + transactionId, + }): DistributedTransactionEvents { + const notify = ({ + eventType, + step, + result, + response, + errors, + }: { + eventType: keyof DistributedTransactionEvents + step?: TransactionStep + response?: unknown + result?: unknown + errors?: unknown[] + }) => { + this.notify({ + workflowId, + transactionId, + eventType, + response, + step, + result, + errors, + }) + } + + return { + onTimeout: ({ transaction }) => { + customEventHandlers?.onTimeout?.({ transaction }) + notify({ eventType: "onTimeout" }) + }, + + onBegin: ({ transaction }) => { + customEventHandlers?.onBegin?.({ transaction }) + notify({ eventType: "onBegin" }) + }, + onResume: ({ transaction }) => { + customEventHandlers?.onResume?.({ transaction }) + notify({ eventType: "onResume" }) + }, + onCompensateBegin: ({ transaction }) => { + customEventHandlers?.onCompensateBegin?.({ transaction }) + notify({ eventType: "onCompensateBegin" }) + }, + onFinish: ({ transaction, result, errors }) => { + // TODO: unsubscribe transaction handlers on finish + customEventHandlers?.onFinish?.({ transaction, result, errors }) + }, + + onStepBegin: ({ step, transaction }) => { + customEventHandlers?.onStepBegin?.({ step, transaction }) + + notify({ eventType: "onStepBegin", step }) + }, + onStepSuccess: ({ step, transaction }) => { + const response = transaction.getContext().invoke[step.id] + customEventHandlers?.onStepSuccess?.({ step, transaction, response }) + + notify({ eventType: "onStepSuccess", step, response }) + }, + onStepFailure: ({ step, transaction }) => { + const errors = transaction.getErrors(TransactionHandlerType.INVOKE)[ + step.id + ] + customEventHandlers?.onStepFailure?.({ step, transaction, errors }) + + notify({ eventType: "onStepFailure", step, errors }) + }, + + onCompensateStepSuccess: ({ step, transaction }) => { + const response = transaction.getContext().compensate[step.id] + customEventHandlers?.onStepSuccess?.({ step, transaction, response }) + + notify({ eventType: "onCompensateStepSuccess", step, response }) + }, + onCompensateStepFailure: ({ step, transaction }) => { + const errors = transaction.getErrors(TransactionHandlerType.COMPENSATE)[ + step.id + ] + customEventHandlers?.onStepFailure?.({ step, transaction, errors }) + + notify({ eventType: "onCompensateStepFailure", step, errors }) + }, + } + } + + private buildIdempotencyKeyAndParts( + idempotencyKey: string | IdempotencyKeyParts + ): [string, IdempotencyKeyParts] { + const parts: IdempotencyKeyParts = { + workflowId: "", + transactionId: "", + stepId: "", + action: "invoke", + } + let idempotencyKey_ = idempotencyKey as string + + const setParts = (workflowId, transactionId, stepId, action) => { + parts.workflowId = workflowId + parts.transactionId = transactionId + parts.stepId = stepId + parts.action = action + } + + if (!isString(idempotencyKey)) { + const { workflowId, transactionId, stepId, action } = + idempotencyKey as IdempotencyKeyParts + idempotencyKey_ = [workflowId, transactionId, stepId, action].join(":") + setParts(workflowId, transactionId, stepId, action) + } else { + const [workflowId, transactionId, stepId, action] = + idempotencyKey_.split(":") + setParts(workflowId, transactionId, stepId, action) + } + + return [idempotencyKey_, parts] + } +} diff --git a/packages/workflow-engine-inmemory/src/services/workflows-module.ts b/packages/workflow-engine-inmemory/src/services/workflows-module.ts new file mode 100644 index 0000000000..31be5674d5 --- /dev/null +++ b/packages/workflow-engine-inmemory/src/services/workflows-module.ts @@ -0,0 +1,199 @@ +import { + Context, + DAL, + FindConfig, + InternalModuleDeclaration, + ModuleJoinerConfig, +} from "@medusajs/types" +import {} from "@medusajs/types/src" +import { + InjectManager, + InjectSharedContext, + MedusaContext, +} from "@medusajs/utils" +import type { + ReturnWorkflow, + UnwrapWorkflowInputDataType, + WorkflowOrchestratorTypes, +} from "@medusajs/workflows-sdk" +import { + WorkflowExecutionService, + WorkflowOrchestratorService, +} from "@services" +import { joinerConfig } from "../joiner-config" + +type InjectedDependencies = { + baseRepository: DAL.RepositoryService + workflowExecutionService: WorkflowExecutionService + workflowOrchestratorService: WorkflowOrchestratorService +} + +export class WorkflowsModuleService + implements WorkflowOrchestratorTypes.IWorkflowsModuleService +{ + protected baseRepository_: DAL.RepositoryService + protected workflowExecutionService_: WorkflowExecutionService + protected workflowOrchestratorService_: WorkflowOrchestratorService + + constructor( + { + baseRepository, + workflowExecutionService, + workflowOrchestratorService, + }: InjectedDependencies, + protected readonly moduleDeclaration: InternalModuleDeclaration + ) { + this.baseRepository_ = baseRepository + this.workflowExecutionService_ = workflowExecutionService + this.workflowOrchestratorService_ = workflowOrchestratorService + } + + __joinerConfig(): ModuleJoinerConfig { + return joinerConfig + } + + @InjectManager("baseRepository_") + async listWorkflowExecution( + filters: WorkflowOrchestratorTypes.FilterableWorkflowExecutionProps = {}, + config: FindConfig = {}, + @MedusaContext() sharedContext: Context = {} + ): Promise { + const wfExecutions = await this.workflowExecutionService_.list( + filters, + config, + sharedContext + ) + + return this.baseRepository_.serialize< + WorkflowOrchestratorTypes.WorkflowExecutionDTO[] + >(wfExecutions, { + populate: true, + }) + } + + @InjectManager("baseRepository_") + async listAndCountWorkflowExecution( + filters: WorkflowOrchestratorTypes.FilterableWorkflowExecutionProps = {}, + config: FindConfig = {}, + @MedusaContext() sharedContext: Context = {} + ): Promise<[WorkflowOrchestratorTypes.WorkflowExecutionDTO[], number]> { + const [wfExecutions, count] = + await this.workflowExecutionService_.listAndCount( + filters, + config, + sharedContext + ) + + return [ + await this.baseRepository_.serialize< + WorkflowOrchestratorTypes.WorkflowExecutionDTO[] + >(wfExecutions, { + populate: true, + }), + count, + ] + } + + @InjectSharedContext() + async run>( + workflowIdOrWorkflow: TWorkflow, + options: WorkflowOrchestratorTypes.WorkflowOrchestratorRunDTO< + TWorkflow extends ReturnWorkflow + ? UnwrapWorkflowInputDataType + : unknown + > = {}, + @MedusaContext() context: Context = {} + ) { + const ret = await this.workflowOrchestratorService_.run< + TWorkflow extends ReturnWorkflow + ? UnwrapWorkflowInputDataType + : unknown + >(workflowIdOrWorkflow, options, context) + + return ret as any + } + + @InjectSharedContext() + async getRunningTransaction( + workflowId: string, + transactionId: string, + @MedusaContext() context: Context = {} + ) { + return this.workflowOrchestratorService_.getRunningTransaction( + workflowId, + transactionId, + context + ) + } + + @InjectSharedContext() + async setStepSuccess( + { + idempotencyKey, + stepResponse, + options, + }: { + idempotencyKey: string | object + stepResponse: unknown + options?: Record + }, + @MedusaContext() context: Context = {} + ) { + return this.workflowOrchestratorService_.setStepSuccess( + { + idempotencyKey, + stepResponse, + options, + } as any, + context + ) + } + + @InjectSharedContext() + async setStepFailure( + { + idempotencyKey, + stepResponse, + options, + }: { + idempotencyKey: string | object + stepResponse: unknown + options?: Record + }, + @MedusaContext() context: Context = {} + ) { + return this.workflowOrchestratorService_.setStepFailure( + { + idempotencyKey, + stepResponse, + options, + } as any, + context + ) + } + + @InjectSharedContext() + async subscribe( + args: { + workflowId: string + transactionId?: string + subscriber: Function + subscriberId?: string + }, + @MedusaContext() context: Context = {} + ) { + return this.workflowOrchestratorService_.subscribe(args as any, context) + } + + @InjectSharedContext() + async unsubscribe( + args: { + workflowId: string + transactionId?: string + subscriberOrId: string | Function + }, + @MedusaContext() context: Context = {} + ) { + return this.workflowOrchestratorService_.unsubscribe(args as any, context) + } +} diff --git a/packages/workflow-engine-inmemory/src/types/index.ts b/packages/workflow-engine-inmemory/src/types/index.ts new file mode 100644 index 0000000000..0f252977b0 --- /dev/null +++ b/packages/workflow-engine-inmemory/src/types/index.ts @@ -0,0 +1,5 @@ +import { Logger } from "@medusajs/types" + +export type InitializeModuleInjectableDependencies = { + logger?: Logger +} diff --git a/packages/workflow-engine-inmemory/src/utils/index.ts b/packages/workflow-engine-inmemory/src/utils/index.ts new file mode 100644 index 0000000000..01bae8b302 --- /dev/null +++ b/packages/workflow-engine-inmemory/src/utils/index.ts @@ -0,0 +1 @@ +export * from "./workflow-orchestrator-storage" diff --git a/packages/workflow-engine-inmemory/src/utils/workflow-orchestrator-storage.ts b/packages/workflow-engine-inmemory/src/utils/workflow-orchestrator-storage.ts new file mode 100644 index 0000000000..7254f3b90d --- /dev/null +++ b/packages/workflow-engine-inmemory/src/utils/workflow-orchestrator-storage.ts @@ -0,0 +1,201 @@ +import { + DistributedTransaction, + DistributedTransactionStorage, + TransactionCheckpoint, + TransactionStep, +} from "@medusajs/orchestration" +import { TransactionState } from "@medusajs/utils" +import { + WorkflowExecutionService, + WorkflowOrchestratorService, +} from "@services" + +// eslint-disable-next-line max-len +export class InMemoryDistributedTransactionStorage extends DistributedTransactionStorage { + private workflowExecutionService_: WorkflowExecutionService + private workflowOrchestratorService_: WorkflowOrchestratorService + + private storage: Map = new Map() + private retries: Map = new Map() + private timeouts: Map = new Map() + + constructor({ + workflowExecutionService, + }: { + workflowExecutionService: WorkflowExecutionService + }) { + super() + + this.workflowExecutionService_ = workflowExecutionService + } + + setWorkflowOrchestratorService(workflowOrchestratorService) { + this.workflowOrchestratorService_ = workflowOrchestratorService + } + + private async saveToDb(data: TransactionCheckpoint) { + await this.workflowExecutionService_.upsert([ + { + workflow_id: data.flow.modelId, + transaction_id: data.flow.transactionId, + execution: data.flow, + context: { + data: data.context, + errors: data.errors, + }, + state: data.flow.state, + }, + ]) + } + + private async deleteFromDb(data: TransactionCheckpoint) { + await this.workflowExecutionService_.delete([ + { + workflow_id: data.flow.modelId, + transaction_id: data.flow.transactionId, + }, + ]) + } + + async get(key: string): Promise { + return this.storage.get(key) + } + + async list(): Promise { + return Array.from(this.storage.values()) + } + + async save( + key: string, + data: TransactionCheckpoint, + ttl?: number + ): Promise { + this.storage.set(key, data) + + let retentionTime + + /** + * Store the retention time only if the transaction is done, failed or reverted. + * From that moment, this tuple can be later on archived or deleted after the retention time. + */ + const hasFinished = [ + TransactionState.DONE, + TransactionState.FAILED, + TransactionState.REVERTED, + ].includes(data.flow.state) + + if (hasFinished) { + retentionTime = data.flow.options?.retentionTime + Object.assign(data, { + retention_time: retentionTime, + }) + } + + if (hasFinished && !retentionTime) { + await this.deleteFromDb(data) + } else { + await this.saveToDb(data) + } + + if (hasFinished) { + this.storage.delete(key) + } + } + + async scheduleRetry( + transaction: DistributedTransaction, + step: TransactionStep, + timestamp: number, + interval: number + ): Promise { + const { modelId: workflowId, transactionId } = transaction + + const inter = setTimeout(async () => { + await this.workflowOrchestratorService_.run(workflowId, { + transactionId, + throwOnError: false, + }) + }, interval * 1e3) + + const key = `${workflowId}:${transactionId}:${step.id}` + this.retries.set(key, inter) + } + + async clearRetry( + transaction: DistributedTransaction, + step: TransactionStep + ): Promise { + const { modelId: workflowId, transactionId } = transaction + + const key = `${workflowId}:${transactionId}:${step.id}` + const inter = this.retries.get(key) + if (inter) { + clearTimeout(inter as NodeJS.Timeout) + this.retries.delete(key) + } + } + + async scheduleTransactionTimeout( + transaction: DistributedTransaction, + timestamp: number, + interval: number + ): Promise { + const { modelId: workflowId, transactionId } = transaction + + const inter = setTimeout(async () => { + await this.workflowOrchestratorService_.run(workflowId, { + transactionId, + throwOnError: false, + }) + }, interval * 1e3) + + const key = `${workflowId}:${transactionId}` + this.timeouts.set(key, inter) + } + + async clearTransactionTimeout( + transaction: DistributedTransaction + ): Promise { + const { modelId: workflowId, transactionId } = transaction + + const key = `${workflowId}:${transactionId}` + const inter = this.timeouts.get(key) + if (inter) { + clearTimeout(inter as NodeJS.Timeout) + this.timeouts.delete(key) + } + } + + async scheduleStepTimeout( + transaction: DistributedTransaction, + step: TransactionStep, + timestamp: number, + interval: number + ): Promise { + const { modelId: workflowId, transactionId } = transaction + + const inter = setTimeout(async () => { + await this.workflowOrchestratorService_.run(workflowId, { + transactionId, + throwOnError: false, + }) + }, interval * 1e3) + + const key = `${workflowId}:${transactionId}:${step.id}` + this.timeouts.set(key, inter) + } + + async clearStepTimeout( + transaction: DistributedTransaction, + step: TransactionStep + ): Promise { + const { modelId: workflowId, transactionId } = transaction + + const key = `${workflowId}:${transactionId}:${step.id}` + const inter = this.timeouts.get(key) + if (inter) { + clearTimeout(inter as NodeJS.Timeout) + this.timeouts.delete(key) + } + } +} diff --git a/packages/workflow-engine-inmemory/tsconfig.json b/packages/workflow-engine-inmemory/tsconfig.json new file mode 100644 index 0000000000..d4e5080094 --- /dev/null +++ b/packages/workflow-engine-inmemory/tsconfig.json @@ -0,0 +1,38 @@ +{ + "compilerOptions": { + "lib": ["es2020"], + "target": "es2020", + "outDir": "./dist", + "esModuleInterop": true, + "declarationMap": true, + "declaration": true, + "module": "commonjs", + "moduleResolution": "node", + "emitDecoratorMetadata": true, + "experimentalDecorators": true, + "sourceMap": false, + "noImplicitReturns": true, + "strictNullChecks": true, + "strictFunctionTypes": true, + "noImplicitThis": true, + "allowJs": true, + "skipLibCheck": true, + "downlevelIteration": true, // to use ES5 specific tooling + "baseUrl": ".", + "resolveJsonModule": true, + "paths": { + "@models": ["./src/models"], + "@services": ["./src/services"], + "@repositories": ["./src/repositories"], + "@types": ["./src/types"] + } + }, + "include": ["src"], + "exclude": [ + "dist", + "./src/**/__tests__", + "./src/**/__mocks__", + "./src/**/__fixtures__", + "node_modules" + ] +} diff --git a/packages/workflow-engine-inmemory/tsconfig.spec.json b/packages/workflow-engine-inmemory/tsconfig.spec.json new file mode 100644 index 0000000000..48e47e8cbb --- /dev/null +++ b/packages/workflow-engine-inmemory/tsconfig.spec.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "include": ["src", "integration-tests"], + "exclude": ["node_modules", "dist"], + "compilerOptions": { + "sourceMap": true + } +} diff --git a/packages/workflow-engine-redis/.gitignore b/packages/workflow-engine-redis/.gitignore new file mode 100644 index 0000000000..874c6c69d3 --- /dev/null +++ b/packages/workflow-engine-redis/.gitignore @@ -0,0 +1,6 @@ +/dist +node_modules +.DS_store +.env* +.env +*.sql diff --git a/packages/workflow-engine-redis/CHANGELOG.md b/packages/workflow-engine-redis/CHANGELOG.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/workflow-engine-redis/README.md b/packages/workflow-engine-redis/README.md new file mode 100644 index 0000000000..b34e46ea20 --- /dev/null +++ b/packages/workflow-engine-redis/README.md @@ -0,0 +1 @@ +# Workflow Orchestrator diff --git a/packages/workflow-engine-redis/integration-tests/__fixtures__/index.ts b/packages/workflow-engine-redis/integration-tests/__fixtures__/index.ts new file mode 100644 index 0000000000..987a8a99bd --- /dev/null +++ b/packages/workflow-engine-redis/integration-tests/__fixtures__/index.ts @@ -0,0 +1,4 @@ +export * from "./workflow_1" +export * from "./workflow_2" +export * from "./workflow_step_timeout" +export * from "./workflow_transaction_timeout" diff --git a/packages/workflow-engine-redis/integration-tests/__fixtures__/workflow_1.ts b/packages/workflow-engine-redis/integration-tests/__fixtures__/workflow_1.ts new file mode 100644 index 0000000000..cb0056466e --- /dev/null +++ b/packages/workflow-engine-redis/integration-tests/__fixtures__/workflow_1.ts @@ -0,0 +1,65 @@ +import { + StepResponse, + createStep, + createWorkflow, +} from "@medusajs/workflows-sdk" + +const step_1 = createStep( + "step_1", + jest.fn((input) => { + input.test = "test" + return new StepResponse(input, { compensate: 123 }) + }), + jest.fn((compensateInput) => { + if (!compensateInput) { + return + } + + console.log("reverted", compensateInput.compensate) + return new StepResponse({ + reverted: true, + }) + }) +) + +const step_2 = createStep( + "step_2", + jest.fn((input, context) => { + console.log("triggered async request", context.metadata.idempotency_key) + + if (input) { + return new StepResponse({ notAsyncResponse: input.hey }) + } + }), + jest.fn((_, context) => { + return new StepResponse({ + step: context.metadata.action, + idempotency_key: context.metadata.idempotency_key, + reverted: true, + }) + }) +) + +const step_3 = createStep( + "step_3", + jest.fn((res) => { + return new StepResponse({ + done: { + inputFromSyncStep: res.notAsyncResponse, + }, + }) + }) +) + +createWorkflow("workflow_1", function (input) { + step_1(input) + + const ret2 = step_2({ hey: "oh" }) + + step_2({ hey: "async hello" }).config({ + name: "new_step_name", + async: true, + }) + + return step_3(ret2) +}) diff --git a/packages/workflow-engine-redis/integration-tests/__fixtures__/workflow_2.ts b/packages/workflow-engine-redis/integration-tests/__fixtures__/workflow_2.ts new file mode 100644 index 0000000000..f15d51889f --- /dev/null +++ b/packages/workflow-engine-redis/integration-tests/__fixtures__/workflow_2.ts @@ -0,0 +1,71 @@ +import { + StepResponse, + createStep, + createWorkflow, +} from "@medusajs/workflows-sdk" + +const step_1 = createStep( + "step_1", + jest.fn((input) => { + input.test = "test" + return new StepResponse(input, { compensate: 123 }) + }), + jest.fn((compensateInput) => { + if (!compensateInput) { + return + } + + console.log("reverted", compensateInput.compensate) + return new StepResponse({ + reverted: true, + }) + }) +) + +const step_2 = createStep( + "step_2", + jest.fn((input, context) => { + console.log("triggered async request", context.metadata.idempotency_key) + + if (input) { + return new StepResponse({ notAsyncResponse: input.hey }) + } + }), + jest.fn((_, context) => { + return new StepResponse({ + step: context.metadata.action, + idempotency_key: context.metadata.idempotency_key, + reverted: true, + }) + }) +) + +const step_3 = createStep( + "step_3", + jest.fn((res) => { + return new StepResponse({ + done: { + inputFromSyncStep: res.notAsyncResponse, + }, + }) + }) +) + +createWorkflow( + { + name: "workflow_2", + retentionTime: 1000, + }, + function (input) { + step_1(input) + + const ret2 = step_2({ hey: "oh" }) + + step_2({ hey: "async hello" }).config({ + name: "new_step_name", + async: true, + }) + + return step_3(ret2) + } +) diff --git a/packages/workflow-engine-redis/integration-tests/__fixtures__/workflow_step_timeout.ts b/packages/workflow-engine-redis/integration-tests/__fixtures__/workflow_step_timeout.ts new file mode 100644 index 0000000000..0bdbf9fd9c --- /dev/null +++ b/packages/workflow-engine-redis/integration-tests/__fixtures__/workflow_step_timeout.ts @@ -0,0 +1,51 @@ +import { + StepResponse, + createStep, + createWorkflow, +} from "@medusajs/workflows-sdk" +import { setTimeout } from "timers/promises" + +const step_1 = createStep( + "step_1", + jest.fn(async (input) => { + await setTimeout(200) + + return new StepResponse(input, { compensate: 123 }) + }) +) + +const step_1_async = createStep( + { + name: "step_1_async", + async: true, + timeout: 0.1, // 0.1 second + }, + + jest.fn(async (input) => { + return new StepResponse(input, { compensate: 123 }) + }) +) + +createWorkflow( + { + name: "workflow_step_timeout", + }, + function (input) { + const resp = step_1(input).config({ + timeout: 0.1, // 0.1 second + }) + + return resp + } +) + +createWorkflow( + { + name: "workflow_step_timeout_async", + }, + function (input) { + const resp = step_1_async(input) + + return resp + } +) diff --git a/packages/workflow-engine-redis/integration-tests/__fixtures__/workflow_transaction_timeout.ts b/packages/workflow-engine-redis/integration-tests/__fixtures__/workflow_transaction_timeout.ts new file mode 100644 index 0000000000..6e1c2852f2 --- /dev/null +++ b/packages/workflow-engine-redis/integration-tests/__fixtures__/workflow_transaction_timeout.ts @@ -0,0 +1,44 @@ +import { + StepResponse, + createStep, + createWorkflow, +} from "@medusajs/workflows-sdk" +import { setTimeout } from "timers/promises" + +const step_1 = createStep( + "step_1", + jest.fn(async (input) => { + await setTimeout(200) + + return new StepResponse({ + executed: true, + }) + }), + jest.fn() +) + +createWorkflow( + { + name: "workflow_transaction_timeout", + timeout: 0.1, // 0.1 second + }, + function (input) { + const resp = step_1(input) + + return resp + } +) + +createWorkflow( + { + name: "workflow_transaction_timeout_async", + timeout: 0.1, // 0.1 second + }, + function (input) { + const resp = step_1(input).config({ + async: true, + }) + + return resp + } +) diff --git a/packages/workflow-engine-redis/integration-tests/__tests__/index.spec.ts b/packages/workflow-engine-redis/integration-tests/__tests__/index.spec.ts new file mode 100644 index 0000000000..802fff3418 --- /dev/null +++ b/packages/workflow-engine-redis/integration-tests/__tests__/index.spec.ts @@ -0,0 +1,245 @@ +import { MedusaApp } from "@medusajs/modules-sdk" +import { + TransactionStepTimeoutError, + TransactionTimeoutError, +} from "@medusajs/orchestration" +import { RemoteJoinerQuery } from "@medusajs/types" +import { TransactionHandlerType } from "@medusajs/utils" +import { IWorkflowsModuleService } from "@medusajs/workflows-sdk" +import { knex } from "knex" +import { setTimeout } from "timers/promises" +import "../__fixtures__" +import { DB_URL, TestDatabase } from "../utils" + +const sharedPgConnection = knex({ + client: "pg", + searchPath: process.env.MEDUSA_WORKFLOW_ENGINE_DB_SCHEMA, + connection: { + connectionString: DB_URL, + debug: false, + }, +}) + +const afterEach_ = async () => { + await TestDatabase.clearTables(sharedPgConnection) +} + +describe("Workflow Orchestrator module", function () { + describe("Testing basic workflow", function () { + let workflowOrcModule: IWorkflowsModuleService + let query: ( + query: string | RemoteJoinerQuery | object, + variables?: Record + ) => Promise + + afterEach(afterEach_) + + beforeAll(async () => { + const { + runMigrations, + query: remoteQuery, + modules, + } = await MedusaApp({ + sharedResourcesConfig: { + database: { + connection: sharedPgConnection, + }, + }, + modulesConfig: { + workflows: { + resolve: __dirname + "/../..", + options: { + redis: { + url: "localhost:6379", + }, + }, + }, + }, + }) + + query = remoteQuery + + await runMigrations() + + workflowOrcModule = + modules.workflows as unknown as IWorkflowsModuleService + }) + + afterEach(afterEach_) + + it("should return a list of workflow executions and remove after completed when there is no retentionTime set", async () => { + await workflowOrcModule.run("workflow_1", { + input: { + value: "123", + }, + throwOnError: true, + }) + + let executionsList = await query({ + workflow_executions: { + fields: ["workflow_id", "transaction_id", "state"], + }, + }) + + expect(executionsList).toHaveLength(1) + + const { result } = await workflowOrcModule.setStepSuccess({ + idempotencyKey: { + action: TransactionHandlerType.INVOKE, + stepId: "new_step_name", + workflowId: "workflow_1", + transactionId: executionsList[0].transaction_id, + }, + stepResponse: { uhuuuu: "yeaah!" }, + }) + + executionsList = await query({ + workflow_executions: { + fields: ["id"], + }, + }) + + expect(executionsList).toHaveLength(0) + expect(result).toEqual({ + done: { + inputFromSyncStep: "oh", + }, + }) + }) + + it("should return a list of workflow executions and keep it saved when there is a retentionTime set", async () => { + await workflowOrcModule.run("workflow_2", { + input: { + value: "123", + }, + throwOnError: true, + transactionId: "transaction_1", + }) + + let executionsList = await query({ + workflow_executions: { + fields: ["id"], + }, + }) + + expect(executionsList).toHaveLength(1) + + await workflowOrcModule.setStepSuccess({ + idempotencyKey: { + action: TransactionHandlerType.INVOKE, + stepId: "new_step_name", + workflowId: "workflow_2", + transactionId: "transaction_1", + }, + stepResponse: { uhuuuu: "yeaah!" }, + }) + + executionsList = await query({ + workflow_executions: { + fields: ["id"], + }, + }) + + expect(executionsList).toHaveLength(1) + }) + + it("should revert the entire transaction when a step timeout expires", async () => { + const { transaction, result, errors } = await workflowOrcModule.run( + "workflow_step_timeout", + { + input: { + myInput: "123", + }, + throwOnError: false, + } + ) + + expect(transaction.flow.state).toEqual("reverted") + expect(result).toEqual({ + myInput: "123", + }) + expect(errors).toHaveLength(1) + expect(errors[0].action).toEqual("step_1") + expect(errors[0].error).toBeInstanceOf(TransactionStepTimeoutError) + }) + + it("should revert the entire transaction when the transaction timeout expires", async () => { + const { transaction, result, errors } = await workflowOrcModule.run( + "workflow_transaction_timeout", + { + input: {}, + transactionId: "trx", + throwOnError: false, + } + ) + + expect(transaction.flow.state).toEqual("reverted") + expect(result).toEqual({ executed: true }) + expect(errors).toHaveLength(1) + expect(errors[0].action).toEqual("step_1") + expect( + TransactionTimeoutError.isTransactionTimeoutError(errors[0].error) + ).toBe(true) + }) + + it("should revert the entire transaction when a step timeout expires in a async step", async () => { + await workflowOrcModule.run("workflow_step_timeout_async", { + input: { + myInput: "123", + }, + transactionId: "transaction_1", + throwOnError: false, + }) + + await setTimeout(200) + + const { transaction, result, errors } = await workflowOrcModule.run( + "workflow_step_timeout_async", + { + input: { + myInput: "123", + }, + transactionId: "transaction_1", + throwOnError: false, + } + ) + + expect(transaction.flow.state).toEqual("reverted") + expect(result).toEqual(undefined) + expect(errors).toHaveLength(1) + expect(errors[0].action).toEqual("step_1_async") + expect( + TransactionStepTimeoutError.isTransactionStepTimeoutError( + errors[0].error + ) + ).toBe(true) + }) + + it("should revert the entire transaction when the transaction timeout expires in a transaction containing an async step", async () => { + await workflowOrcModule.run("workflow_transaction_timeout_async", { + input: {}, + transactionId: "transaction_1", + throwOnError: false, + }) + + await setTimeout(200) + + const { transaction, result, errors } = await workflowOrcModule.run( + "workflow_transaction_timeout_async", + { + input: {}, + transactionId: "transaction_1", + throwOnError: false, + } + ) + + expect(transaction.flow.state).toEqual("reverted") + expect(result).toEqual(undefined) + expect(errors).toHaveLength(1) + expect(errors[0].action).toEqual("step_1") + expect( + TransactionTimeoutError.isTransactionTimeoutError(errors[0].error) + ).toBe(true) + }) + }) +}) diff --git a/packages/workflow-engine-redis/integration-tests/setup-env.js b/packages/workflow-engine-redis/integration-tests/setup-env.js new file mode 100644 index 0000000000..18f30b372c --- /dev/null +++ b/packages/workflow-engine-redis/integration-tests/setup-env.js @@ -0,0 +1,6 @@ +if (typeof process.env.DB_TEMP_NAME === "undefined") { + const tempName = parseInt(process.env.JEST_WORKER_ID || "1") + process.env.DB_TEMP_NAME = `medusa-workflow-engine-redis-${tempName}` +} + +process.env.MEDUSA_WORKFLOW_ENGINE_DB_SCHEMA = "public" diff --git a/packages/workflow-engine-redis/integration-tests/setup.js b/packages/workflow-engine-redis/integration-tests/setup.js new file mode 100644 index 0000000000..43f99aab4a --- /dev/null +++ b/packages/workflow-engine-redis/integration-tests/setup.js @@ -0,0 +1,3 @@ +import { JestUtils } from "medusa-test-utils" + +JestUtils.afterAllHookDropDatabase() diff --git a/packages/workflow-engine-redis/integration-tests/utils/database.ts b/packages/workflow-engine-redis/integration-tests/utils/database.ts new file mode 100644 index 0000000000..582baee15c --- /dev/null +++ b/packages/workflow-engine-redis/integration-tests/utils/database.ts @@ -0,0 +1,53 @@ +import * as process from "process" + +const DB_HOST = process.env.DB_HOST ?? "localhost" +const DB_USERNAME = process.env.DB_USERNAME ?? "" +const DB_PASSWORD = process.env.DB_PASSWORD +const DB_NAME = process.env.DB_TEMP_NAME + +export const DB_URL = `postgres://${DB_USERNAME}${ + DB_PASSWORD ? `:${DB_PASSWORD}` : "" +}@${DB_HOST}/${DB_NAME}` + +const Redis = require("ioredis") + +const redisUrl = process.env.REDIS_URL || "redis://localhost:6379" +const redis = new Redis(redisUrl) + +interface TestDatabase { + clearTables(knex): Promise +} + +export const TestDatabase: TestDatabase = { + clearTables: async (knex) => { + await knex.raw(` + TRUNCATE TABLE workflow_execution CASCADE; + `) + + await cleanRedis() + }, +} + +async function deleteKeysByPattern(pattern) { + const stream = redis.scanStream({ + match: pattern, + count: 100, + }) + + for await (const keys of stream) { + if (keys.length) { + const pipeline = redis.pipeline() + keys.forEach((key) => pipeline.del(key)) + await pipeline.exec() + } + } +} + +async function cleanRedis() { + try { + await deleteKeysByPattern("bull:*") + await deleteKeysByPattern("dtrans:*") + } catch (error) { + console.error("Error:", error) + } +} diff --git a/packages/workflow-engine-redis/integration-tests/utils/index.ts b/packages/workflow-engine-redis/integration-tests/utils/index.ts new file mode 100644 index 0000000000..6b917ed30e --- /dev/null +++ b/packages/workflow-engine-redis/integration-tests/utils/index.ts @@ -0,0 +1 @@ +export * from "./database" diff --git a/packages/workflow-engine-redis/jest.config.js b/packages/workflow-engine-redis/jest.config.js new file mode 100644 index 0000000000..860ba90a49 --- /dev/null +++ b/packages/workflow-engine-redis/jest.config.js @@ -0,0 +1,21 @@ +module.exports = { + moduleNameMapper: { + "^@models": "/src/models", + "^@services": "/src/services", + "^@repositories": "/src/repositories", + }, + transform: { + "^.+\\.[jt]s?$": [ + "ts-jest", + { + tsConfig: "tsconfig.spec.json", + isolatedModules: true, + }, + ], + }, + testEnvironment: `node`, + moduleFileExtensions: [`js`, `ts`], + modulePathIgnorePatterns: ["dist/"], + setupFiles: ["/integration-tests/setup-env.js"], + setupFilesAfterEnv: ["/integration-tests/setup.js"], +} diff --git a/packages/workflow-engine-redis/mikro-orm.config.dev.ts b/packages/workflow-engine-redis/mikro-orm.config.dev.ts new file mode 100644 index 0000000000..5468c7a41d --- /dev/null +++ b/packages/workflow-engine-redis/mikro-orm.config.dev.ts @@ -0,0 +1,8 @@ +import * as entities from "./src/models" + +module.exports = { + entities: Object.values(entities), + schema: "public", + clientUrl: "postgres://postgres@localhost/medusa-workflow-engine-redis", + type: "postgresql", +} diff --git a/packages/workflow-engine-redis/package.json b/packages/workflow-engine-redis/package.json new file mode 100644 index 0000000000..2e8631f9c3 --- /dev/null +++ b/packages/workflow-engine-redis/package.json @@ -0,0 +1,61 @@ +{ + "name": "@medusajs/workflow-engine-redis", + "version": "0.0.1", + "description": "Medusa Workflow Orchestrator module using Redis to track workflows executions", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "files": [ + "dist" + ], + "engines": { + "node": ">=16" + }, + "repository": { + "type": "git", + "url": "https://github.com/medusajs/medusa", + "directory": "packages/workflow-engine-redis" + }, + "publishConfig": { + "access": "public" + }, + "author": "Medusa", + "license": "MIT", + "scripts": { + "watch": "tsc --build --watch", + "watch:test": "tsc --build tsconfig.spec.json --watch", + "prepublishOnly": "cross-env NODE_ENV=production tsc --build && tsc-alias -p tsconfig.json", + "build": "rimraf dist && tsc --build && tsc-alias -p tsconfig.json", + "test": "jest --passWithNoTests --runInBand --bail --forceExit -- src/**/__tests__/**/*.ts", + "test:integration": "jest --runInBand --forceExit -- integration-tests/**/__tests__/**/*.ts", + "migration:generate": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:generate", + "migration:initial": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:create --initial", + "migration:create": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:create", + "migration:up": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm migration:up", + "orm:cache:clear": " MIKRO_ORM_CLI=./mikro-orm.config.dev.ts mikro-orm cache:clear" + }, + "devDependencies": { + "@mikro-orm/cli": "5.9.7", + "cross-env": "^5.2.1", + "jest": "^29.6.3", + "medusa-test-utils": "^1.1.40", + "rimraf": "^3.0.2", + "ts-jest": "^29.1.1", + "ts-node": "^10.9.1", + "tsc-alias": "^1.8.6", + "typescript": "^5.1.6" + }, + "dependencies": { + "@medusajs/modules-sdk": "^1.12.5", + "@medusajs/types": "^1.11.9", + "@medusajs/utils": "^1.11.2", + "@medusajs/workflows-sdk": "^0.1.0", + "@mikro-orm/core": "5.9.7", + "@mikro-orm/migrations": "5.9.7", + "@mikro-orm/postgresql": "5.9.7", + "awilix": "^8.0.0", + "bullmq": "^5.1.3", + "dotenv": "^16.1.4", + "ioredis": "^5.3.2", + "knex": "2.4.2" + } +} diff --git a/packages/workflow-engine-redis/src/index.ts b/packages/workflow-engine-redis/src/index.ts new file mode 100644 index 0000000000..7804040565 --- /dev/null +++ b/packages/workflow-engine-redis/src/index.ts @@ -0,0 +1,22 @@ +import { Modules } from "@medusajs/modules-sdk" +import { ModulesSdkUtils } from "@medusajs/utils" +import * as models from "@models" +import { moduleDefinition } from "./module-definition" + +export default moduleDefinition + +const migrationScriptOptions = { + moduleName: Modules.WORKFLOW_ENGINE, + models: models, + pathToMigrations: __dirname + "/migrations", +} + +export const runMigrations = ModulesSdkUtils.buildMigrationScript( + migrationScriptOptions +) +export const revertMigration = ModulesSdkUtils.buildRevertMigrationScript( + migrationScriptOptions +) + +export * from "./initialize" +export * from "./loaders" diff --git a/packages/workflow-engine-redis/src/initialize/index.ts b/packages/workflow-engine-redis/src/initialize/index.ts new file mode 100644 index 0000000000..20f4f49231 --- /dev/null +++ b/packages/workflow-engine-redis/src/initialize/index.ts @@ -0,0 +1,36 @@ +import { + ExternalModuleDeclaration, + InternalModuleDeclaration, + MedusaModule, + MODULE_PACKAGE_NAMES, + Modules, +} from "@medusajs/modules-sdk" +import { ModulesSdkTypes } from "@medusajs/types" +import { WorkflowOrchestratorTypes } from "@medusajs/workflows-sdk" +import { moduleDefinition } from "../module-definition" +import { InitializeModuleInjectableDependencies } from "../types" + +export const initialize = async ( + options?: + | ModulesSdkTypes.ModuleServiceInitializeOptions + | ModulesSdkTypes.ModuleServiceInitializeCustomDataLayerOptions + | ExternalModuleDeclaration + | InternalModuleDeclaration, + injectedDependencies?: InitializeModuleInjectableDependencies +): Promise => { + const loaded = + // eslint-disable-next-line max-len + await MedusaModule.bootstrap( + { + moduleKey: Modules.WORKFLOW_ENGINE, + defaultPath: MODULE_PACKAGE_NAMES[Modules.WORKFLOW_ENGINE], + declaration: options as + | InternalModuleDeclaration + | ExternalModuleDeclaration, + injectedDependencies, + moduleExports: moduleDefinition, + } + ) + + return loaded[Modules.WORKFLOW_ENGINE] +} diff --git a/packages/workflow-engine-redis/src/joiner-config.ts b/packages/workflow-engine-redis/src/joiner-config.ts new file mode 100644 index 0000000000..7999e9c3ab --- /dev/null +++ b/packages/workflow-engine-redis/src/joiner-config.ts @@ -0,0 +1,34 @@ +import { Modules } from "@medusajs/modules-sdk" +import { ModuleJoinerConfig } from "@medusajs/types" +import { MapToConfig } from "@medusajs/utils" +import { WorkflowExecution } from "@models" +import moduleSchema from "./schema" + +export const LinkableKeys = { + workflow_execution_id: WorkflowExecution.name, +} + +const entityLinkableKeysMap: MapToConfig = {} +Object.entries(LinkableKeys).forEach(([key, value]) => { + entityLinkableKeysMap[value] ??= [] + entityLinkableKeysMap[value].push({ + mapTo: key, + valueFrom: key.split("_").pop()!, + }) +}) + +export const entityNameToLinkableKeysMap: MapToConfig = entityLinkableKeysMap + +export const joinerConfig: ModuleJoinerConfig = { + serviceName: Modules.WORKFLOW_ENGINE, + primaryKeys: ["id"], + schema: moduleSchema, + linkableKeys: LinkableKeys, + alias: { + name: ["workflow_execution", "workflow_executions"], + args: { + entity: WorkflowExecution.name, + methodSuffix: "WorkflowExecution", + }, + }, +} diff --git a/packages/workflow-engine-redis/src/loaders/connection.ts b/packages/workflow-engine-redis/src/loaders/connection.ts new file mode 100644 index 0000000000..580e05e95c --- /dev/null +++ b/packages/workflow-engine-redis/src/loaders/connection.ts @@ -0,0 +1,36 @@ +import { + InternalModuleDeclaration, + LoaderOptions, + Modules, +} from "@medusajs/modules-sdk" +import { ModulesSdkTypes } from "@medusajs/types" +import { ModulesSdkUtils } from "@medusajs/utils" +import { EntitySchema } from "@mikro-orm/core" +import * as WorkflowOrchestratorModels from "../models" + +export default async ( + { + options, + container, + logger, + }: LoaderOptions< + | ModulesSdkTypes.ModuleServiceInitializeOptions + | ModulesSdkTypes.ModuleServiceInitializeCustomDataLayerOptions + >, + moduleDeclaration?: InternalModuleDeclaration +): Promise => { + const entities = Object.values( + WorkflowOrchestratorModels + ) as unknown as EntitySchema[] + const pathToMigrations = __dirname + "/../migrations" + + await ModulesSdkUtils.mikroOrmConnectionLoader({ + moduleName: Modules.WORKFLOW_ENGINE, + entities, + container, + options, + moduleDeclaration, + logger, + pathToMigrations, + }) +} diff --git a/packages/workflow-engine-redis/src/loaders/container.ts b/packages/workflow-engine-redis/src/loaders/container.ts new file mode 100644 index 0000000000..9a0c5553b4 --- /dev/null +++ b/packages/workflow-engine-redis/src/loaders/container.ts @@ -0,0 +1,9 @@ +import { MikroOrmBaseRepository, ModulesSdkUtils } from "@medusajs/utils" +import * as ModuleModels from "@models" +import * as ModuleServices from "@services" + +export default ModulesSdkUtils.moduleContainerLoaderFactory({ + moduleModels: ModuleModels, + moduleServices: ModuleServices, + moduleRepositories: { BaseRepository: MikroOrmBaseRepository }, +}) diff --git a/packages/workflow-engine-redis/src/loaders/index.ts b/packages/workflow-engine-redis/src/loaders/index.ts new file mode 100644 index 0000000000..8b66bc0be4 --- /dev/null +++ b/packages/workflow-engine-redis/src/loaders/index.ts @@ -0,0 +1,4 @@ +export * from "./connection" +export * from "./container" +export * from "./redis" +export * from "./utils" diff --git a/packages/workflow-engine-redis/src/loaders/redis.ts b/packages/workflow-engine-redis/src/loaders/redis.ts new file mode 100644 index 0000000000..8321a6d147 --- /dev/null +++ b/packages/workflow-engine-redis/src/loaders/redis.ts @@ -0,0 +1,78 @@ +import { LoaderOptions } from "@medusajs/modules-sdk" +import { asValue } from "awilix" +import Redis from "ioredis" +import { RedisWorkflowsOptions } from "../types" + +export default async ({ + container, + logger, + options, +}: LoaderOptions): Promise => { + const { + url, + options: redisOptions, + pubsub, + } = options?.redis as RedisWorkflowsOptions + + // TODO: get default from ENV VAR + if (!url) { + throw Error( + "No `redis.url` provided in `workflowOrchestrator` module options. It is required for the Workflow Orchestrator Redis." + ) + } + + const cnnPubSub = pubsub ?? { url, options: redisOptions } + + const queueName = options?.queueName ?? "medusa-workflows" + + let connection + let redisPublisher + let redisSubscriber + let workerConnection + + try { + connection = await getConnection(url, redisOptions) + workerConnection = await getConnection(url, { + ...(redisOptions ?? {}), + maxRetriesPerRequest: null, + }) + logger?.info( + `Connection to Redis in module 'workflow-engine-redis' established` + ) + } catch (err) { + logger?.error( + `An error occurred while connecting to Redis in module 'workflow-engine-redis': ${err}` + ) + } + + try { + redisPublisher = await getConnection(cnnPubSub.url, cnnPubSub.options) + redisSubscriber = await getConnection(cnnPubSub.url, cnnPubSub.options) + logger?.info( + `Connection to Redis PubSub in module 'workflow-engine-redis' established` + ) + } catch (err) { + logger?.error( + `An error occurred while connecting to Redis PubSub in module 'workflow-engine-redis': ${err}` + ) + } + + container.register({ + redisConnection: asValue(connection), + redisWorkerConnection: asValue(workerConnection), + redisPublisher: asValue(redisPublisher), + redisSubscriber: asValue(redisSubscriber), + redisQueueName: asValue(queueName), + }) +} + +async function getConnection(url, redisOptions) { + const connection = new Redis(url, { + lazyConnect: true, + ...(redisOptions ?? {}), + }) + + await connection.connect() + + return connection +} diff --git a/packages/workflow-engine-redis/src/loaders/utils.ts b/packages/workflow-engine-redis/src/loaders/utils.ts new file mode 100644 index 0000000000..f662dc1e17 --- /dev/null +++ b/packages/workflow-engine-redis/src/loaders/utils.ts @@ -0,0 +1,10 @@ +import { asClass } from "awilix" +import { RedisDistributedTransactionStorage } from "../utils" + +export default async ({ container }): Promise => { + container.register({ + redisDistributedTransactionStorage: asClass( + RedisDistributedTransactionStorage + ).singleton(), + }) +} diff --git a/packages/workflow-engine-redis/src/migrations/Migration20231228143900.ts b/packages/workflow-engine-redis/src/migrations/Migration20231228143900.ts new file mode 100644 index 0000000000..af9958e80a --- /dev/null +++ b/packages/workflow-engine-redis/src/migrations/Migration20231228143900.ts @@ -0,0 +1,41 @@ +import { Migration } from "@mikro-orm/migrations" + +export class Migration20231221104256 extends Migration { + async up(): Promise { + this.addSql( + ` + CREATE TABLE IF NOT EXISTS workflow_execution + ( + id character varying NOT NULL, + workflow_id character varying NOT NULL, + transaction_id character varying NOT NULL, + execution jsonb NULL, + context jsonb NULL, + state character varying NOT NULL, + created_at timestamp WITHOUT time zone NOT NULL DEFAULT Now(), + updated_at timestamp WITHOUT time zone NOT NULL DEFAULT Now(), + deleted_at timestamp WITHOUT time zone NULL, + CONSTRAINT "PK_workflow_execution_workflow_id_transaction_id" PRIMARY KEY ("workflow_id", "transaction_id") + ); + + CREATE UNIQUE INDEX IF NOT EXISTS "IDX_workflow_execution_id" ON "workflow_execution" ("id"); + CREATE INDEX IF NOT EXISTS "IDX_workflow_execution_workflow_id" ON "workflow_execution" ("workflow_id") WHERE deleted_at IS NULL; + CREATE INDEX IF NOT EXISTS "IDX_workflow_execution_transaction_id" ON "workflow_execution" ("transaction_id") WHERE deleted_at IS NULL; + CREATE INDEX IF NOT EXISTS "IDX_workflow_execution_state" ON "workflow_execution" ("state") WHERE deleted_at IS NULL; + ` + ) + } + + async down(): Promise { + this.addSql( + ` + DROP INDEX "IDX_workflow_execution_id"; + DROP INDEX "IDX_workflow_execution_workflow_id"; + DROP INDEX "IDX_workflow_execution_transaction_id"; + DROP INDEX "IDX_workflow_execution_state"; + + DROP TABLE IF EXISTS workflow_execution; + ` + ) + } +} diff --git a/packages/workflow-engine-redis/src/models/index.ts b/packages/workflow-engine-redis/src/models/index.ts new file mode 100644 index 0000000000..78fcbfa921 --- /dev/null +++ b/packages/workflow-engine-redis/src/models/index.ts @@ -0,0 +1 @@ +export { default as WorkflowExecution } from "./workflow-execution" diff --git a/packages/workflow-engine-redis/src/models/workflow-execution.ts b/packages/workflow-engine-redis/src/models/workflow-execution.ts new file mode 100644 index 0000000000..753d9e62db --- /dev/null +++ b/packages/workflow-engine-redis/src/models/workflow-execution.ts @@ -0,0 +1,76 @@ +import { TransactionState } from "@medusajs/orchestration" +import { DALUtils, generateEntityId } from "@medusajs/utils" +import { + BeforeCreate, + Entity, + Enum, + Filter, + Index, + OnInit, + OptionalProps, + PrimaryKey, + Property, + Unique, +} from "@mikro-orm/core" + +type OptionalFields = "deleted_at" + +@Entity() +@Unique({ + name: "IDX_workflow_execution_workflow_id_transaction_id_unique", + properties: ["workflow_id", "transaction_id"], +}) +@Filter(DALUtils.mikroOrmSoftDeletableFilterOptions) +export default class WorkflowExecution { + [OptionalProps]?: OptionalFields + + @Property({ columnType: "text", nullable: false }) + @Index({ name: "IDX_workflow_execution_id" }) + id!: string + + @Index({ name: "IDX_workflow_execution_workflow_id" }) + @PrimaryKey({ columnType: "text" }) + workflow_id: string + + @Index({ name: "IDX_workflow_execution_transaction_id" }) + @PrimaryKey({ columnType: "text" }) + transaction_id: string + + @Property({ columnType: "jsonb", nullable: true }) + execution: Record | null = null + + @Property({ columnType: "jsonb", nullable: true }) + context: Record | null = null + + @Index({ name: "IDX_workflow_execution_state" }) + @Enum(() => TransactionState) + state: TransactionState + + @Property({ + onCreate: () => new Date(), + columnType: "timestamptz", + defaultRaw: "now()", + }) + created_at: Date + + @Property({ + onCreate: () => new Date(), + onUpdate: () => new Date(), + columnType: "timestamptz", + defaultRaw: "now()", + }) + updated_at: Date + + @Property({ columnType: "timestamptz", nullable: true }) + deleted_at: Date | null = null + + @BeforeCreate() + onCreate() { + this.id = generateEntityId(this.id, "wf_exec") + } + + @OnInit() + onInit() { + this.id = generateEntityId(this.id, "wf_exec") + } +} diff --git a/packages/workflow-engine-redis/src/module-definition.ts b/packages/workflow-engine-redis/src/module-definition.ts new file mode 100644 index 0000000000..0a3d33f580 --- /dev/null +++ b/packages/workflow-engine-redis/src/module-definition.ts @@ -0,0 +1,19 @@ +import { ModuleExports } from "@medusajs/types" +import { WorkflowsModuleService } from "@services" +import loadConnection from "./loaders/connection" +import loadContainer from "./loaders/container" +import redisConnection from "./loaders/redis" +import loadUtils from "./loaders/utils" + +const service = WorkflowsModuleService +const loaders = [ + loadContainer, + loadConnection, + loadUtils, + redisConnection, +] as any + +export const moduleDefinition: ModuleExports = { + service, + loaders, +} diff --git a/packages/workflow-engine-redis/src/repositories/index.ts b/packages/workflow-engine-redis/src/repositories/index.ts new file mode 100644 index 0000000000..8def202608 --- /dev/null +++ b/packages/workflow-engine-redis/src/repositories/index.ts @@ -0,0 +1,2 @@ +export { MikroOrmBaseRepository as BaseRepository } from "@medusajs/utils" +export { WorkflowExecutionRepository } from "./workflow-execution" diff --git a/packages/workflow-engine-redis/src/repositories/workflow-execution.ts b/packages/workflow-engine-redis/src/repositories/workflow-execution.ts new file mode 100644 index 0000000000..9e6553ec74 --- /dev/null +++ b/packages/workflow-engine-redis/src/repositories/workflow-execution.ts @@ -0,0 +1,7 @@ +import { DALUtils } from "@medusajs/utils" +import { WorkflowExecution } from "@models" + +// eslint-disable-next-line max-len +export class WorkflowExecutionRepository extends DALUtils.mikroOrmBaseRepositoryFactory( + WorkflowExecution +) {} diff --git a/packages/workflow-engine-redis/src/schema/index.ts b/packages/workflow-engine-redis/src/schema/index.ts new file mode 100644 index 0000000000..3d7d91edea --- /dev/null +++ b/packages/workflow-engine-redis/src/schema/index.ts @@ -0,0 +1,26 @@ +export default ` +scalar DateTime +scalar JSON + +enum TransactionState { + NOT_STARTED + INVOKING + WAITING_TO_COMPENSATE + COMPENSATING + DONE + REVERTED + FAILED +} + +type WorkflowExecution { + id: ID! + created_at: DateTime! + updated_at: DateTime! + deleted_at: DateTime + workflow_id: string + transaction_id: string + execution: JSON + context: JSON + state: TransactionState +} +` diff --git a/packages/workflow-engine-redis/src/services/__tests__/index.spec.ts b/packages/workflow-engine-redis/src/services/__tests__/index.spec.ts new file mode 100644 index 0000000000..728f6245c6 --- /dev/null +++ b/packages/workflow-engine-redis/src/services/__tests__/index.spec.ts @@ -0,0 +1,5 @@ +describe("Noop test", () => { + it("noop check", async () => { + expect(true).toBe(true) + }) +}) diff --git a/packages/workflow-engine-redis/src/services/index.ts b/packages/workflow-engine-redis/src/services/index.ts new file mode 100644 index 0000000000..5a6d313d86 --- /dev/null +++ b/packages/workflow-engine-redis/src/services/index.ts @@ -0,0 +1,3 @@ +export * from "./workflow-execution" +export * from "./workflow-orchestrator" +export * from "./workflows-module" diff --git a/packages/workflow-engine-redis/src/services/workflow-execution.ts b/packages/workflow-engine-redis/src/services/workflow-execution.ts new file mode 100644 index 0000000000..158557ec0b --- /dev/null +++ b/packages/workflow-engine-redis/src/services/workflow-execution.ts @@ -0,0 +1,21 @@ +import { DAL } from "@medusajs/types" +import { ModulesSdkUtils } from "@medusajs/utils" +import { WorkflowExecution } from "@models" + +type InjectedDependencies = { + workflowExecutionRepository: DAL.RepositoryService +} + +export class WorkflowExecutionService< + TEntity extends WorkflowExecution = WorkflowExecution +> extends ModulesSdkUtils.abstractServiceFactory( + WorkflowExecution +) { + protected workflowExecutionRepository_: DAL.RepositoryService + + constructor({ workflowExecutionRepository }: InjectedDependencies) { + // @ts-ignore + super(...arguments) + this.workflowExecutionRepository_ = workflowExecutionRepository + } +} diff --git a/packages/workflow-engine-redis/src/services/workflow-orchestrator.ts b/packages/workflow-engine-redis/src/services/workflow-orchestrator.ts new file mode 100644 index 0000000000..77770a5c74 --- /dev/null +++ b/packages/workflow-engine-redis/src/services/workflow-orchestrator.ts @@ -0,0 +1,577 @@ +import { + DistributedTransaction, + DistributedTransactionEvents, + TransactionHandlerType, + TransactionStep, +} from "@medusajs/orchestration" +import { ContainerLike, Context, MedusaContainer } from "@medusajs/types" +import { InjectSharedContext, MedusaContext, isString } from "@medusajs/utils" +import { + FlowRunOptions, + MedusaWorkflow, + ReturnWorkflow, +} from "@medusajs/workflows-sdk" +import Redis from "ioredis" +import { ulid } from "ulid" +import type { RedisDistributedTransactionStorage } from "../utils" + +export type WorkflowOrchestratorRunOptions = FlowRunOptions & { + transactionId?: string + container?: ContainerLike +} + +type RegisterStepSuccessOptions = Omit< + WorkflowOrchestratorRunOptions, + "transactionId" | "input" +> + +type IdempotencyKeyParts = { + workflowId: string + transactionId: string + stepId: string + action: "invoke" | "compensate" +} + +type NotifyOptions = { + eventType: keyof DistributedTransactionEvents + workflowId: string + transactionId?: string + step?: TransactionStep + response?: unknown + result?: unknown + errors?: unknown[] +} + +type WorkflowId = string +type TransactionId = string + +type SubscriberHandler = { + (input: NotifyOptions): void +} & { + _id?: string +} + +type SubscribeOptions = { + workflowId: string + transactionId?: string + subscriber: SubscriberHandler + subscriberId?: string +} + +type UnsubscribeOptions = { + workflowId: string + transactionId?: string + subscriberOrId: string | SubscriberHandler +} + +type TransactionSubscribers = Map +type Subscribers = Map + +const AnySubscriber = "any" + +export class WorkflowOrchestratorService { + private instanceId = ulid() + protected redisPublisher: Redis + protected redisSubscriber: Redis + private subscribers: Subscribers = new Map() + + constructor({ + redisDistributedTransactionStorage, + redisPublisher, + redisSubscriber, + }: { + redisDistributedTransactionStorage: RedisDistributedTransactionStorage + workflowOrchestratorService: WorkflowOrchestratorService + redisPublisher: Redis + redisSubscriber: Redis + }) { + this.redisPublisher = redisPublisher + this.redisSubscriber = redisSubscriber + + redisDistributedTransactionStorage.setWorkflowOrchestratorService(this) + DistributedTransaction.setStorage(redisDistributedTransactionStorage) + + this.redisSubscriber.on("message", async (_, message) => { + const { instanceId, data } = JSON.parse(message) + + await this.notify(data, false, instanceId) + }) + } + + @InjectSharedContext() + async run( + workflowIdOrWorkflow: string | ReturnWorkflow, + options?: WorkflowOrchestratorRunOptions, + @MedusaContext() sharedContext: Context = {} + ) { + let { + input, + context, + transactionId, + resultFrom, + throwOnError, + events: eventHandlers, + container, + } = options ?? {} + + const workflowId = isString(workflowIdOrWorkflow) + ? workflowIdOrWorkflow + : workflowIdOrWorkflow.getName() + + if (!workflowId) { + throw new Error("Workflow ID is required") + } + + context ??= {} + context.transactionId ??= transactionId ?? ulid() + + const events: FlowRunOptions["events"] = this.buildWorkflowEvents({ + customEventHandlers: eventHandlers, + workflowId, + transactionId: context.transactionId, + }) + + const exportedWorkflow: any = MedusaWorkflow.getWorkflow(workflowId) + if (!exportedWorkflow) { + throw new Error(`Workflow with id "${workflowId}" not found.`) + } + + const flow = exportedWorkflow(container as MedusaContainer) + + const ret = await flow.run({ + input, + throwOnError, + resultFrom, + context, + events, + }) + + // TODO: temporary + const acknowledgement = { + transactionId: context.transactionId, + workflowId: workflowId, + } + + if (ret.transaction.hasFinished()) { + const { result, errors } = ret + await this.notify({ + eventType: "onFinish", + workflowId, + transactionId: context.transactionId, + result, + errors, + }) + } + + return { acknowledgement, ...ret } + } + + @InjectSharedContext() + async getRunningTransaction( + workflowId: string, + transactionId: string, + options?: WorkflowOrchestratorRunOptions, + @MedusaContext() sharedContext: Context = {} + ): Promise { + let { context, container } = options ?? {} + + if (!workflowId) { + throw new Error("Workflow ID is required") + } + + if (!transactionId) { + throw new Error("TransactionId ID is required") + } + + context ??= {} + context.transactionId ??= transactionId + + const exportedWorkflow: any = MedusaWorkflow.getWorkflow(workflowId) + if (!exportedWorkflow) { + throw new Error(`Workflow with id "${workflowId}" not found.`) + } + + const flow = exportedWorkflow(container as MedusaContainer) + + const transaction = await flow.getRunningTransaction(transactionId, context) + + return transaction + } + + @InjectSharedContext() + async setStepSuccess( + { + idempotencyKey, + stepResponse, + options, + }: { + idempotencyKey: string | IdempotencyKeyParts + stepResponse: unknown + options?: RegisterStepSuccessOptions + }, + @MedusaContext() sharedContext: Context = {} + ) { + const { + context, + throwOnError, + resultFrom, + container, + events: eventHandlers, + } = options ?? {} + + const [idempotencyKey_, { workflowId, transactionId }] = + this.buildIdempotencyKeyAndParts(idempotencyKey) + + const exportedWorkflow: any = MedusaWorkflow.getWorkflow(workflowId) + if (!exportedWorkflow) { + throw new Error(`Workflow with id "${workflowId}" not found.`) + } + + const flow = exportedWorkflow(container as MedusaContainer) + + const events = this.buildWorkflowEvents({ + customEventHandlers: eventHandlers, + transactionId, + workflowId, + }) + + const ret = await flow.registerStepSuccess({ + idempotencyKey: idempotencyKey_, + context, + resultFrom, + throwOnError, + events, + response: stepResponse, + }) + + if (ret.transaction.hasFinished()) { + const { result, errors } = ret + await this.notify({ + eventType: "onFinish", + workflowId, + transactionId, + result, + errors, + }) + } + + return ret + } + + @InjectSharedContext() + async setStepFailure( + { + idempotencyKey, + stepResponse, + options, + }: { + idempotencyKey: string | IdempotencyKeyParts + stepResponse: unknown + options?: RegisterStepSuccessOptions + }, + @MedusaContext() sharedContext: Context = {} + ) { + const { + context, + throwOnError, + resultFrom, + container, + events: eventHandlers, + } = options ?? {} + + const [idempotencyKey_, { workflowId, transactionId }] = + this.buildIdempotencyKeyAndParts(idempotencyKey) + + const exportedWorkflow: any = MedusaWorkflow.getWorkflow(workflowId) + if (!exportedWorkflow) { + throw new Error(`Workflow with id "${workflowId}" not found.`) + } + + const flow = exportedWorkflow(container as MedusaContainer) + + const events = this.buildWorkflowEvents({ + customEventHandlers: eventHandlers, + transactionId, + workflowId, + }) + + const ret = await flow.registerStepFailure({ + idempotencyKey: idempotencyKey_, + context, + resultFrom, + throwOnError, + events, + response: stepResponse, + }) + + if (ret.transaction.hasFinished()) { + const { result, errors } = ret + await this.notify({ + eventType: "onFinish", + workflowId, + transactionId, + result, + errors, + }) + } + + return ret + } + + @InjectSharedContext() + subscribe( + { workflowId, transactionId, subscriber, subscriberId }: SubscribeOptions, + @MedusaContext() sharedContext: Context = {} + ) { + subscriber._id = subscriberId + const subscribers = this.subscribers.get(workflowId) ?? new Map() + + // Subscribe instance to redis + if (!this.subscribers.has(workflowId)) { + void this.redisSubscriber.subscribe(this.getChannelName(workflowId)) + } + + const handlerIndex = (handlers) => { + return handlers.indexOf((s) => s === subscriber || s._id === subscriberId) + } + + if (transactionId) { + const transactionSubscribers = subscribers.get(transactionId) ?? [] + const subscriberIndex = handlerIndex(transactionSubscribers) + if (subscriberIndex !== -1) { + transactionSubscribers.slice(subscriberIndex, 1) + } + + transactionSubscribers.push(subscriber) + subscribers.set(transactionId, transactionSubscribers) + this.subscribers.set(workflowId, subscribers) + return + } + + const workflowSubscribers = subscribers.get(AnySubscriber) ?? [] + const subscriberIndex = handlerIndex(workflowSubscribers) + if (subscriberIndex !== -1) { + workflowSubscribers.slice(subscriberIndex, 1) + } + + workflowSubscribers.push(subscriber) + subscribers.set(AnySubscriber, workflowSubscribers) + this.subscribers.set(workflowId, subscribers) + } + + @InjectSharedContext() + unsubscribe( + { workflowId, transactionId, subscriberOrId }: UnsubscribeOptions, + @MedusaContext() sharedContext: Context = {} + ) { + const subscribers = this.subscribers.get(workflowId) ?? new Map() + + const filterSubscribers = (handlers: SubscriberHandler[]) => { + return handlers.filter((handler) => { + return handler._id + ? handler._id !== (subscriberOrId as string) + : handler !== (subscriberOrId as SubscriberHandler) + }) + } + + // Unsubscribe instance + if (!this.subscribers.has(workflowId)) { + void this.redisSubscriber.unsubscribe(this.getChannelName(workflowId)) + } + + if (transactionId) { + const transactionSubscribers = subscribers.get(transactionId) ?? [] + const newTransactionSubscribers = filterSubscribers( + transactionSubscribers + ) + subscribers.set(transactionId, newTransactionSubscribers) + this.subscribers.set(workflowId, subscribers) + return + } + + const workflowSubscribers = subscribers.get(AnySubscriber) ?? [] + const newWorkflowSubscribers = filterSubscribers(workflowSubscribers) + subscribers.set(AnySubscriber, newWorkflowSubscribers) + this.subscribers.set(workflowId, subscribers) + } + + private async notify( + options: NotifyOptions, + publish = true, + instanceId = this.instanceId + ) { + if (!publish && instanceId === this.instanceId) { + return + } + + if (publish) { + const channel = this.getChannelName(options.workflowId) + + const message = JSON.stringify({ + instanceId: this.instanceId, + data: options, + }) + await this.redisPublisher.publish(channel, message) + } + + const { + eventType, + workflowId, + transactionId, + errors, + result, + step, + response, + } = options + + const subscribers: TransactionSubscribers = + this.subscribers.get(workflowId) ?? new Map() + + const notifySubscribers = (handlers: SubscriberHandler[]) => { + handlers.forEach((handler) => { + handler({ + eventType, + workflowId, + transactionId, + step, + response, + result, + errors, + }) + }) + } + + if (transactionId) { + const transactionSubscribers = subscribers.get(transactionId) ?? [] + notifySubscribers(transactionSubscribers) + } + + const workflowSubscribers = subscribers.get(AnySubscriber) ?? [] + notifySubscribers(workflowSubscribers) + } + + private getChannelName(workflowId: string): string { + return `orchestrator:${workflowId}` + } + + private buildWorkflowEvents({ + customEventHandlers, + workflowId, + transactionId, + }): DistributedTransactionEvents { + const notify = async ({ + eventType, + step, + result, + response, + errors, + }: { + eventType: keyof DistributedTransactionEvents + step?: TransactionStep + response?: unknown + result?: unknown + errors?: unknown[] + }) => { + await this.notify({ + workflowId, + transactionId, + eventType, + response, + step, + result, + errors, + }) + } + + return { + onTimeout: async ({ transaction }) => { + customEventHandlers?.onTimeout?.({ transaction }) + await notify({ eventType: "onTimeout" }) + }, + + onBegin: async ({ transaction }) => { + customEventHandlers?.onBegin?.({ transaction }) + await notify({ eventType: "onBegin" }) + }, + onResume: async ({ transaction }) => { + customEventHandlers?.onResume?.({ transaction }) + await notify({ eventType: "onResume" }) + }, + onCompensateBegin: async ({ transaction }) => { + customEventHandlers?.onCompensateBegin?.({ transaction }) + await notify({ eventType: "onCompensateBegin" }) + }, + onFinish: async ({ transaction, result, errors }) => { + // TODO: unsubscribe transaction handlers on finish + customEventHandlers?.onFinish?.({ transaction, result, errors }) + }, + + onStepBegin: async ({ step, transaction }) => { + customEventHandlers?.onStepBegin?.({ step, transaction }) + + await notify({ eventType: "onStepBegin", step }) + }, + onStepSuccess: async ({ step, transaction }) => { + const response = transaction.getContext().invoke[step.id] + customEventHandlers?.onStepSuccess?.({ step, transaction, response }) + + await notify({ eventType: "onStepSuccess", step, response }) + }, + onStepFailure: async ({ step, transaction }) => { + const errors = transaction.getErrors(TransactionHandlerType.INVOKE)[ + step.id + ] + customEventHandlers?.onStepFailure?.({ step, transaction, errors }) + + await notify({ eventType: "onStepFailure", step, errors }) + }, + + onCompensateStepSuccess: async ({ step, transaction }) => { + const response = transaction.getContext().compensate[step.id] + customEventHandlers?.onStepSuccess?.({ step, transaction, response }) + + await notify({ eventType: "onCompensateStepSuccess", step, response }) + }, + onCompensateStepFailure: async ({ step, transaction }) => { + const errors = transaction.getErrors(TransactionHandlerType.COMPENSATE)[ + step.id + ] + customEventHandlers?.onStepFailure?.({ step, transaction, errors }) + + await notify({ eventType: "onCompensateStepFailure", step, errors }) + }, + } + } + + private buildIdempotencyKeyAndParts( + idempotencyKey: string | IdempotencyKeyParts + ): [string, IdempotencyKeyParts] { + const parts: IdempotencyKeyParts = { + workflowId: "", + transactionId: "", + stepId: "", + action: "invoke", + } + let idempotencyKey_ = idempotencyKey as string + + const setParts = (workflowId, transactionId, stepId, action) => { + parts.workflowId = workflowId + parts.transactionId = transactionId + parts.stepId = stepId + parts.action = action + } + + if (!isString(idempotencyKey)) { + const { workflowId, transactionId, stepId, action } = + idempotencyKey as IdempotencyKeyParts + idempotencyKey_ = [workflowId, transactionId, stepId, action].join(":") + setParts(workflowId, transactionId, stepId, action) + } else { + const [workflowId, transactionId, stepId, action] = + idempotencyKey_.split(":") + setParts(workflowId, transactionId, stepId, action) + } + + return [idempotencyKey_, parts] + } +} diff --git a/packages/workflow-engine-redis/src/services/workflows-module.ts b/packages/workflow-engine-redis/src/services/workflows-module.ts new file mode 100644 index 0000000000..31be5674d5 --- /dev/null +++ b/packages/workflow-engine-redis/src/services/workflows-module.ts @@ -0,0 +1,199 @@ +import { + Context, + DAL, + FindConfig, + InternalModuleDeclaration, + ModuleJoinerConfig, +} from "@medusajs/types" +import {} from "@medusajs/types/src" +import { + InjectManager, + InjectSharedContext, + MedusaContext, +} from "@medusajs/utils" +import type { + ReturnWorkflow, + UnwrapWorkflowInputDataType, + WorkflowOrchestratorTypes, +} from "@medusajs/workflows-sdk" +import { + WorkflowExecutionService, + WorkflowOrchestratorService, +} from "@services" +import { joinerConfig } from "../joiner-config" + +type InjectedDependencies = { + baseRepository: DAL.RepositoryService + workflowExecutionService: WorkflowExecutionService + workflowOrchestratorService: WorkflowOrchestratorService +} + +export class WorkflowsModuleService + implements WorkflowOrchestratorTypes.IWorkflowsModuleService +{ + protected baseRepository_: DAL.RepositoryService + protected workflowExecutionService_: WorkflowExecutionService + protected workflowOrchestratorService_: WorkflowOrchestratorService + + constructor( + { + baseRepository, + workflowExecutionService, + workflowOrchestratorService, + }: InjectedDependencies, + protected readonly moduleDeclaration: InternalModuleDeclaration + ) { + this.baseRepository_ = baseRepository + this.workflowExecutionService_ = workflowExecutionService + this.workflowOrchestratorService_ = workflowOrchestratorService + } + + __joinerConfig(): ModuleJoinerConfig { + return joinerConfig + } + + @InjectManager("baseRepository_") + async listWorkflowExecution( + filters: WorkflowOrchestratorTypes.FilterableWorkflowExecutionProps = {}, + config: FindConfig = {}, + @MedusaContext() sharedContext: Context = {} + ): Promise { + const wfExecutions = await this.workflowExecutionService_.list( + filters, + config, + sharedContext + ) + + return this.baseRepository_.serialize< + WorkflowOrchestratorTypes.WorkflowExecutionDTO[] + >(wfExecutions, { + populate: true, + }) + } + + @InjectManager("baseRepository_") + async listAndCountWorkflowExecution( + filters: WorkflowOrchestratorTypes.FilterableWorkflowExecutionProps = {}, + config: FindConfig = {}, + @MedusaContext() sharedContext: Context = {} + ): Promise<[WorkflowOrchestratorTypes.WorkflowExecutionDTO[], number]> { + const [wfExecutions, count] = + await this.workflowExecutionService_.listAndCount( + filters, + config, + sharedContext + ) + + return [ + await this.baseRepository_.serialize< + WorkflowOrchestratorTypes.WorkflowExecutionDTO[] + >(wfExecutions, { + populate: true, + }), + count, + ] + } + + @InjectSharedContext() + async run>( + workflowIdOrWorkflow: TWorkflow, + options: WorkflowOrchestratorTypes.WorkflowOrchestratorRunDTO< + TWorkflow extends ReturnWorkflow + ? UnwrapWorkflowInputDataType + : unknown + > = {}, + @MedusaContext() context: Context = {} + ) { + const ret = await this.workflowOrchestratorService_.run< + TWorkflow extends ReturnWorkflow + ? UnwrapWorkflowInputDataType + : unknown + >(workflowIdOrWorkflow, options, context) + + return ret as any + } + + @InjectSharedContext() + async getRunningTransaction( + workflowId: string, + transactionId: string, + @MedusaContext() context: Context = {} + ) { + return this.workflowOrchestratorService_.getRunningTransaction( + workflowId, + transactionId, + context + ) + } + + @InjectSharedContext() + async setStepSuccess( + { + idempotencyKey, + stepResponse, + options, + }: { + idempotencyKey: string | object + stepResponse: unknown + options?: Record + }, + @MedusaContext() context: Context = {} + ) { + return this.workflowOrchestratorService_.setStepSuccess( + { + idempotencyKey, + stepResponse, + options, + } as any, + context + ) + } + + @InjectSharedContext() + async setStepFailure( + { + idempotencyKey, + stepResponse, + options, + }: { + idempotencyKey: string | object + stepResponse: unknown + options?: Record + }, + @MedusaContext() context: Context = {} + ) { + return this.workflowOrchestratorService_.setStepFailure( + { + idempotencyKey, + stepResponse, + options, + } as any, + context + ) + } + + @InjectSharedContext() + async subscribe( + args: { + workflowId: string + transactionId?: string + subscriber: Function + subscriberId?: string + }, + @MedusaContext() context: Context = {} + ) { + return this.workflowOrchestratorService_.subscribe(args as any, context) + } + + @InjectSharedContext() + async unsubscribe( + args: { + workflowId: string + transactionId?: string + subscriberOrId: string | Function + }, + @MedusaContext() context: Context = {} + ) { + return this.workflowOrchestratorService_.unsubscribe(args as any, context) + } +} diff --git a/packages/workflow-engine-redis/src/types/index.ts b/packages/workflow-engine-redis/src/types/index.ts new file mode 100644 index 0000000000..1b066ce1d8 --- /dev/null +++ b/packages/workflow-engine-redis/src/types/index.ts @@ -0,0 +1,34 @@ +import { Logger } from "@medusajs/types" +import { RedisOptions } from "ioredis" + +export type InitializeModuleInjectableDependencies = { + logger?: Logger +} + +/** + * Module config type + */ +export type RedisWorkflowsOptions = { + /** + * Redis connection string + */ + url?: string + + /** + * Queue name used for retries and timeouts + */ + queueName?: string + + /** + * Redis client options + */ + options?: RedisOptions + + /** + * Optiona connection string and options to pub/sub + */ + pubsub?: { + url: string + options?: RedisOptions + } +} diff --git a/packages/workflow-engine-redis/src/utils/index.ts b/packages/workflow-engine-redis/src/utils/index.ts new file mode 100644 index 0000000000..01bae8b302 --- /dev/null +++ b/packages/workflow-engine-redis/src/utils/index.ts @@ -0,0 +1 @@ +export * from "./workflow-orchestrator-storage" diff --git a/packages/workflow-engine-redis/src/utils/workflow-orchestrator-storage.ts b/packages/workflow-engine-redis/src/utils/workflow-orchestrator-storage.ts new file mode 100644 index 0000000000..533181cf7f --- /dev/null +++ b/packages/workflow-engine-redis/src/utils/workflow-orchestrator-storage.ts @@ -0,0 +1,304 @@ +import { + DistributedTransaction, + DistributedTransactionStorage, + TransactionCheckpoint, + TransactionStep, +} from "@medusajs/orchestration" +import { TransactionState } from "@medusajs/utils" +import { + WorkflowExecutionService, + WorkflowOrchestratorService, +} from "@services" +import { Queue, Worker } from "bullmq" +import Redis from "ioredis" + +enum JobType { + RETRY = "retry", + STEP_TIMEOUT = "step_timeout", + TRANSACTION_TIMEOUT = "transaction_timeout", +} + +// eslint-disable-next-line max-len +export class RedisDistributedTransactionStorage extends DistributedTransactionStorage { + private static TTL_AFTER_COMPLETED = 60 * 15 // 15 minutes + private workflowExecutionService_: WorkflowExecutionService + private workflowOrchestratorService_: WorkflowOrchestratorService + + private redisClient: Redis + private queue: Queue + private worker: Worker + + constructor({ + workflowExecutionService, + redisConnection, + redisWorkerConnection, + redisQueueName, + }: { + workflowExecutionService: WorkflowExecutionService + redisConnection: Redis + redisWorkerConnection: Redis + redisQueueName: string + }) { + super() + + this.workflowExecutionService_ = workflowExecutionService + + this.redisClient = redisConnection + + this.queue = new Queue(redisQueueName, { connection: this.redisClient }) + this.worker = new Worker( + redisQueueName, + async (job) => { + const allJobs = [ + JobType.RETRY, + JobType.STEP_TIMEOUT, + JobType.TRANSACTION_TIMEOUT, + ] + + if (allJobs.includes(job.name as JobType)) { + await this.executeTransaction( + job.data.workflowId, + job.data.transactionId + ) + } + }, + { connection: redisWorkerConnection } + ) + } + + setWorkflowOrchestratorService(workflowOrchestratorService) { + this.workflowOrchestratorService_ = workflowOrchestratorService + } + + private async saveToDb(data: TransactionCheckpoint) { + await this.workflowExecutionService_.upsert([ + { + workflow_id: data.flow.modelId, + transaction_id: data.flow.transactionId, + execution: data.flow, + context: { + data: data.context, + errors: data.errors, + }, + state: data.flow.state, + }, + ]) + } + + private async deleteFromDb(data: TransactionCheckpoint) { + await this.workflowExecutionService_.delete([ + { + workflow_id: data.flow.modelId, + transaction_id: data.flow.transactionId, + }, + ]) + } + + private async executeTransaction(workflowId: string, transactionId: string) { + return await this.workflowOrchestratorService_.run(workflowId, { + transactionId, + throwOnError: false, + }) + } + + private stringifyWithSymbol(key, value) { + if (key === "__type" && typeof value === "symbol") { + return Symbol.keyFor(value) + } + + return value + } + + private jsonWithSymbol(key, value) { + if (key === "__type" && typeof value === "string") { + return Symbol.for(value) + } + + return value + } + + async get(key: string): Promise { + const data = await this.redisClient.get(key) + + return data ? JSON.parse(data, this.jsonWithSymbol) : undefined + } + + async list(): Promise { + const keys = await this.redisClient.keys( + DistributedTransaction.keyPrefix + ":*" + ) + const transactions: any[] = [] + for (const key of keys) { + const data = await this.redisClient.get(key) + if (data) { + transactions.push(JSON.parse(data, this.jsonWithSymbol)) + } + } + return transactions + } + + async save( + key: string, + data: TransactionCheckpoint, + ttl?: number + ): Promise { + let retentionTime + + /** + * Store the retention time only if the transaction is done, failed or reverted. + * From that moment, this tuple can be later on archived or deleted after the retention time. + */ + const hasFinished = [ + TransactionState.DONE, + TransactionState.FAILED, + TransactionState.REVERTED, + ].includes(data.flow.state) + + if (hasFinished) { + retentionTime = data.flow.options?.retentionTime + Object.assign(data, { + retention_time: retentionTime, + }) + } + + if (!hasFinished) { + if (ttl) { + await this.redisClient.set( + key, + JSON.stringify(data, this.stringifyWithSymbol), + "EX", + ttl + ) + } else { + await this.redisClient.set( + key, + JSON.stringify(data, this.stringifyWithSymbol) + ) + } + } + + if (hasFinished && !retentionTime) { + await this.deleteFromDb(data) + } else { + await this.saveToDb(data) + } + + if (hasFinished) { + // await this.redisClient.del(key) + await this.redisClient.set( + key, + JSON.stringify(data, this.stringifyWithSymbol), + "EX", + RedisDistributedTransactionStorage.TTL_AFTER_COMPLETED + ) + } + } + + async scheduleRetry( + transaction: DistributedTransaction, + step: TransactionStep, + timestamp: number, + interval: number + ): Promise { + await this.queue.add( + JobType.RETRY, + { + workflowId: transaction.modelId, + transactionId: transaction.transactionId, + stepId: step.id, + }, + { + delay: interval * 1000, + jobId: this.getJobId(JobType.RETRY, transaction, step), + removeOnComplete: true, + } + ) + } + + async clearRetry( + transaction: DistributedTransaction, + step: TransactionStep + ): Promise { + await this.removeJob(JobType.RETRY, transaction, step) + } + + async scheduleTransactionTimeout( + transaction: DistributedTransaction, + timestamp: number, + interval: number + ): Promise { + await this.queue.add( + JobType.TRANSACTION_TIMEOUT, + { + workflowId: transaction.modelId, + transactionId: transaction.transactionId, + }, + { + delay: interval * 1000, + jobId: this.getJobId(JobType.TRANSACTION_TIMEOUT, transaction), + removeOnComplete: true, + } + ) + } + + async clearTransactionTimeout( + transaction: DistributedTransaction + ): Promise { + await this.removeJob(JobType.TRANSACTION_TIMEOUT, transaction) + } + + async scheduleStepTimeout( + transaction: DistributedTransaction, + step: TransactionStep, + timestamp: number, + interval: number + ): Promise { + await this.queue.add( + JobType.STEP_TIMEOUT, + { + workflowId: transaction.modelId, + transactionId: transaction.transactionId, + stepId: step.id, + }, + { + delay: interval * 1000, + jobId: this.getJobId(JobType.STEP_TIMEOUT, transaction, step), + removeOnComplete: true, + } + ) + } + + async clearStepTimeout( + transaction: DistributedTransaction, + step: TransactionStep + ): Promise { + await this.removeJob(JobType.STEP_TIMEOUT, transaction, step) + } + + private getJobId( + type: JobType, + transaction: DistributedTransaction, + step?: TransactionStep + ) { + const key = [type, transaction.modelId, transaction.transactionId] + + if (step) { + key.push(step.id) + } + + return key.join(":") + } + + private async removeJob( + type: JobType, + transaction: DistributedTransaction, + step?: TransactionStep + ) { + const jobId = this.getJobId(type, transaction, step) + const job = await this.queue.getJob(jobId) + + if (job && job.attemptsStarted === 0) { + await job.remove() + } + } +} diff --git a/packages/workflow-engine-redis/tsconfig.json b/packages/workflow-engine-redis/tsconfig.json new file mode 100644 index 0000000000..d4e5080094 --- /dev/null +++ b/packages/workflow-engine-redis/tsconfig.json @@ -0,0 +1,38 @@ +{ + "compilerOptions": { + "lib": ["es2020"], + "target": "es2020", + "outDir": "./dist", + "esModuleInterop": true, + "declarationMap": true, + "declaration": true, + "module": "commonjs", + "moduleResolution": "node", + "emitDecoratorMetadata": true, + "experimentalDecorators": true, + "sourceMap": false, + "noImplicitReturns": true, + "strictNullChecks": true, + "strictFunctionTypes": true, + "noImplicitThis": true, + "allowJs": true, + "skipLibCheck": true, + "downlevelIteration": true, // to use ES5 specific tooling + "baseUrl": ".", + "resolveJsonModule": true, + "paths": { + "@models": ["./src/models"], + "@services": ["./src/services"], + "@repositories": ["./src/repositories"], + "@types": ["./src/types"] + } + }, + "include": ["src"], + "exclude": [ + "dist", + "./src/**/__tests__", + "./src/**/__mocks__", + "./src/**/__fixtures__", + "node_modules" + ] +} diff --git a/packages/workflow-engine-redis/tsconfig.spec.json b/packages/workflow-engine-redis/tsconfig.spec.json new file mode 100644 index 0000000000..48e47e8cbb --- /dev/null +++ b/packages/workflow-engine-redis/tsconfig.spec.json @@ -0,0 +1,8 @@ +{ + "extends": "./tsconfig.json", + "include": ["src", "integration-tests"], + "exclude": ["node_modules", "dist"], + "compilerOptions": { + "sourceMap": true + } +} diff --git a/integration-tests/plugins/__tests__/workflows/utils/composer/compose.ts b/packages/workflows-sdk/src/helper/__tests__/compose.ts similarity index 99% rename from integration-tests/plugins/__tests__/workflows/utils/composer/compose.ts rename to packages/workflows-sdk/src/helper/__tests__/compose.ts index 9fdedc22c2..b0c594596b 100644 --- a/integration-tests/plugins/__tests__/workflows/utils/composer/compose.ts +++ b/packages/workflows-sdk/src/helper/__tests__/compose.ts @@ -6,7 +6,7 @@ import { parallelize, StepResponse, transform, -} from "@medusajs/workflows-sdk" +} from "../.." jest.setTimeout(30000) diff --git a/packages/workflows-sdk/src/index.ts b/packages/workflows-sdk/src/index.ts index 9c27d4e26a..cec02f1e03 100644 --- a/packages/workflows-sdk/src/index.ts +++ b/packages/workflows-sdk/src/index.ts @@ -1,4 +1,5 @@ export * from "./helper" export * from "./medusa-workflow" +export * as WorkflowOrchestratorTypes from "./types" export * from "./utils/composer" export * as Composer from "./utils/composer" diff --git a/packages/workflows-sdk/src/types/common.ts b/packages/workflows-sdk/src/types/common.ts new file mode 100644 index 0000000000..f3a81e7271 --- /dev/null +++ b/packages/workflows-sdk/src/types/common.ts @@ -0,0 +1,21 @@ +import { BaseFilterable } from "@medusajs/types" + +export interface WorkflowExecutionDTO { + id: string + workflow_id: string + transaction_id: string + execution: string + context: string + state: any + created_at: Date + updated_at: Date + deleted_at: Date +} + +export interface FilterableWorkflowExecutionProps + extends BaseFilterable { + id?: string[] + workflow_id?: string[] + transaction_id?: string[] + state?: any[] +} diff --git a/packages/workflows-sdk/src/types/index.ts b/packages/workflows-sdk/src/types/index.ts new file mode 100644 index 0000000000..0c73656566 --- /dev/null +++ b/packages/workflows-sdk/src/types/index.ts @@ -0,0 +1,3 @@ +export * from "./common" +export * from "./mutations" +export * from "./service" diff --git a/packages/workflows-sdk/src/types/mutations.ts b/packages/workflows-sdk/src/types/mutations.ts new file mode 100644 index 0000000000..ef3234143e --- /dev/null +++ b/packages/workflows-sdk/src/types/mutations.ts @@ -0,0 +1,7 @@ +export interface UpsertWorkflowExecutionDTO { + workflow_id: string + transaction_id: string + execution: Record + context: Record + state: any +} diff --git a/packages/workflows-sdk/src/types/service.ts b/packages/workflows-sdk/src/types/service.ts new file mode 100644 index 0000000000..ed055e39e6 --- /dev/null +++ b/packages/workflows-sdk/src/types/service.ts @@ -0,0 +1,116 @@ +import { + ContainerLike, + Context, + FindConfig, + IModuleService, +} from "@medusajs/types" +import { ReturnWorkflow, UnwrapWorkflowInputDataType } from "../utils/composer" +import { + FilterableWorkflowExecutionProps, + WorkflowExecutionDTO, +} from "./common" + +type FlowRunOptions = { + input?: TData + context?: Context + resultFrom?: string | string[] | Symbol + throwOnError?: boolean + events?: Record +} + +export interface WorkflowOrchestratorRunDTO + extends FlowRunOptions { + transactionId?: string + container?: ContainerLike +} + +export type IdempotencyKeyParts = { + workflowId: string + transactionId: string + stepId: string + action: "invoke" | "compensate" +} + +export interface IWorkflowsModuleService extends IModuleService { + listWorkflowExecution( + filters?: FilterableWorkflowExecutionProps, + config?: FindConfig, + sharedContext?: Context + ): Promise + + listAndCountWorkflowExecution( + filters?: FilterableWorkflowExecutionProps, + config?: FindConfig, + sharedContext?: Context + ): Promise<[WorkflowExecutionDTO[], number]> + + run< + TWorkflow extends ReturnWorkflow = ReturnWorkflow< + any, + any, + any + >, + TData = UnwrapWorkflowInputDataType + >( + workflowId: string, + options?: WorkflowOrchestratorRunDTO, + sharedContext?: Context + ): Promise<{ + errors: Error[] + transaction: object + result: any + acknowledgement: object + }> + + getRunningTransaction( + workflowId: string, + transactionId: string, + options?: Record, + sharedContext?: Context + ): Promise + + setStepSuccess( + { + idempotencyKey, + stepResponse, + options, + }: { + idempotencyKey: string | IdempotencyKeyParts + stepResponse: unknown + options?: Record + }, + sharedContext?: Context + ) + + setStepFailure( + { + idempotencyKey, + stepResponse, + options, + }: { + idempotencyKey: string | object + stepResponse: unknown + options?: Record + }, + sharedContext?: Context + ) + + subscribe( + args: { + workflowId: string + transactionId?: string + subscriber: Function + subscriberId?: string + }, + sharedContext?: Context + ): Promise + + unsubscribe( + args: { + workflowId: string + transactionId?: string + subscriberOrId: string | Function + }, + sharedContext?: Context + ) +} diff --git a/packages/workflows-sdk/src/utils/_playground.ts b/packages/workflows-sdk/src/utils/_playground.ts index 6bc33ea235..8224af5046 100644 --- a/packages/workflows-sdk/src/utils/_playground.ts +++ b/packages/workflows-sdk/src/utils/_playground.ts @@ -26,3 +26,16 @@ workflow() .then((res) => { console.log(res.result) // result: { step2: { test: "test", test2: "step1" } } }) + +/*type type0 = typeof workflow extends ReturnWorkflow + ? T + : never + +function run< + TWorkflow extends ReturnWorkflow, + TData = TWorkflow extends ReturnWorkflow + ? T + : never +>(name: string, options: FlowRunOptions) {} + +const test = run("workflow", { input: "string" })*/ diff --git a/packages/workflows-sdk/src/utils/composer/create-step.ts b/packages/workflows-sdk/src/utils/composer/create-step.ts index 9a7edabd6f..0cd3142352 100644 --- a/packages/workflows-sdk/src/utils/composer/create-step.ts +++ b/packages/workflows-sdk/src/utils/composer/create-step.ts @@ -1,4 +1,11 @@ -import { resolveValue, StepResponse } from "./helpers" +import { + TransactionStepsDefinition, + WorkflowManager, +} from "@medusajs/orchestration" +import { OrchestrationUtils, isString } from "@medusajs/utils" +import { ulid } from "ulid" +import { StepResponse, resolveValue } from "./helpers" +import { proxify } from "./helpers/proxy" import { CreateWorkflowComposerContext, StepExecutionContext, @@ -6,9 +13,6 @@ import { StepFunctionResult, WorkflowData, } from "./type" -import { proxify } from "./helpers/proxy" -import { TransactionStepsDefinition } from "@medusajs/orchestration" -import { isString, OrchestrationUtils } from "@medusajs/utils" /** * The type of invocation function passed to a step. @@ -166,19 +170,37 @@ function applyStep< : undefined, } - stepConfig!.noCompensation = !compensateFn + stepConfig.uuid = ulid() + stepConfig.noCompensation = !compensateFn this.flow.addAction(stepName, stepConfig) - this.handlers.set(stepName, handler) + + if (!this.handlers.has(stepName)) { + this.handlers.set(stepName, handler) + } const ret = { __type: OrchestrationUtils.SymbolWorkflowStep, __step__: stepName, - config: (config: Pick) => { - this.flow.replaceAction(stepName, stepName, { + config: ( + localConfig: { name?: string } & Omit< + TransactionStepsDefinition, + "next" | "uuid" | "action" + > + ) => { + const newStepName = localConfig.name ?? stepName + + delete localConfig.name + + this.handlers.set(newStepName, handler) + + this.flow.replaceAction(stepConfig.uuid!, newStepName, { ...stepConfig, - ...config, + ...localConfig, }) + + WorkflowManager.update(this.workflowId, this.flow, this.handlers) + return proxify(ret) }, } @@ -241,11 +263,14 @@ export function createStep< TInvokeResultCompensateInput >( /** - * The name of the step or its configuration (currently support maxRetries). + * The name of the step or its configuration. */ nameOrConfig: | string - | ({ name: string } & Pick), + | ({ name: string } & Omit< + TransactionStepsDefinition, + "next" | "uuid" | "action" + >), /** * An invocation function that will be executed when the workflow is executed. The function must return an instance of {@link StepResponse}. The constructor of {@link StepResponse} * accepts the output of the step as a first argument, and optionally as a second argument the data to be passed to the compensation function as a parameter. diff --git a/packages/workflows-sdk/src/utils/composer/create-workflow.ts b/packages/workflows-sdk/src/utils/composer/create-workflow.ts index 945dd5cf33..c9f6b66964 100644 --- a/packages/workflows-sdk/src/utils/composer/create-workflow.ts +++ b/packages/workflows-sdk/src/utils/composer/create-workflow.ts @@ -5,7 +5,7 @@ import { WorkflowManager, } from "@medusajs/orchestration" import { LoadedModule, MedusaContainer } from "@medusajs/types" -import { OrchestrationUtils } from "@medusajs/utils" +import { isString, OrchestrationUtils } from "@medusajs/utils" import { ExportedWorkflow, exportWorkflow } from "../../helper" import { proxify } from "./helpers/proxy" import { @@ -63,7 +63,11 @@ global[OrchestrationUtils.SymbolMedusaWorkflowComposerContext] = null * } * ``` */ -type ReturnWorkflow> = { +export type ReturnWorkflow< + TData, + TResult, + THooks extends Record +> = { ( container?: LoadedModule[] | MedusaContainer ): Omit< @@ -73,8 +77,20 @@ type ReturnWorkflow> = { ExportedWorkflow } & THooks & { getName: () => string + } & { + config: (config: TransactionModelOptions) => void } +/** + * Extract the raw type of the expected input data of a workflow. + * + * @example + * type WorkflowInputData = UnwrapWorkflowInputDataType + */ +export type UnwrapWorkflowInputDataType< + T extends ReturnWorkflow +> = T extends ReturnWorkflow ? TData : never + /** * This function creates a workflow with the provided name and a constructor function. * The constructor function builds the workflow from steps created by the {@link createStep} function. @@ -136,9 +152,9 @@ export function createWorkflow< THooks extends Record = Record >( /** - * The name of the workflow. + * The name of the workflow or its configuration. */ - name: string, + nameOrConfig: string | ({ name: string } & TransactionModelOptions), /** * The constructor function that is executed when the `run` method in {@link ReturnWorkflow} is used. * The function can't be an arrow function or an asynchronus function. It also can't directly manipulate data. @@ -151,9 +167,11 @@ export function createWorkflow< [K in keyof TResult]: | WorkflowData | WorkflowDataProperties - }, - options?: TransactionModelOptions + } ): ReturnWorkflow { + const name = isString(nameOrConfig) ? nameOrConfig : nameOrConfig.name + const options = isString(nameOrConfig) ? {} : nameOrConfig + const handlers: WorkflowHandler = new Map() if (WorkflowManager.getWorkflow(name)) { @@ -185,13 +203,17 @@ export function createWorkflow< const inputPlaceHolder = proxify({ __type: OrchestrationUtils.SymbolInputReference, __step__: "", + config: () => { + // TODO: config default value? + throw new Error("Config is not available for the input object.") + }, }) const returnedStep = composer.apply(context, [inputPlaceHolder]) delete global[OrchestrationUtils.SymbolMedusaWorkflowComposerContext] - WorkflowManager.update(name, context.flow, handlers) + WorkflowManager.update(name, context.flow, handlers, options) const workflow = exportWorkflow( name, @@ -206,8 +228,12 @@ export function createWorkflow< container?: LoadedModule[] | MedusaContainer ) => { const workflow_ = workflow(container) + const expandedFlow: any = workflow_ + expandedFlow.config = (config) => { + workflow_.setOptions(config) + } - return workflow_ + return expandedFlow } let shouldRegisterHookHandler = true diff --git a/packages/workflows-sdk/src/utils/composer/type.ts b/packages/workflows-sdk/src/utils/composer/type.ts index 3e05390c50..2ef18de247 100644 --- a/packages/workflows-sdk/src/utils/composer/type.ts +++ b/packages/workflows-sdk/src/utils/composer/type.ts @@ -37,13 +37,8 @@ export type StepFunction = (keyof TInput extends [] }) & WorkflowDataProperties<{ [K in keyof TOutput]: TOutput[K] - }> & { - config( - config: Pick - ): WorkflowData<{ - [K in keyof TOutput]: TOutput[K] - }> - } & WorkflowDataProperties<{ + }> & + WorkflowDataProperties<{ [K in keyof TOutput]: TOutput[K] }> @@ -62,7 +57,22 @@ export type WorkflowData = (T extends object [Key in keyof T]: WorkflowData } : WorkflowDataProperties) & - WorkflowDataProperties + WorkflowDataProperties & { + config( + config: { name?: string } & Omit< + TransactionStepsDefinition, + "next" | "uuid" | "action" + > + ): T extends object + ? WorkflowData< + T extends object + ? { + [K in keyof T]: T[K] + } + : T + > + : T + } export type CreateWorkflowComposerContext = { hooks_: string[] diff --git a/yarn.lock b/yarn.lock index dcfec06735..d32f10e6b4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8699,7 +8699,61 @@ __metadata: languageName: unknown linkType: soft -"@medusajs/workflows-sdk@^0.1.1, @medusajs/workflows-sdk@workspace:packages/workflows-sdk": +"@medusajs/workflow-engine-inmemory@workspace:packages/workflow-engine-inmemory": + version: 0.0.0-use.local + resolution: "@medusajs/workflow-engine-inmemory@workspace:packages/workflow-engine-inmemory" + dependencies: + "@medusajs/modules-sdk": ^1.12.5 + "@medusajs/types": ^1.11.9 + "@medusajs/utils": ^1.11.2 + "@medusajs/workflows-sdk": ^0.1.0 + "@mikro-orm/cli": 5.9.7 + "@mikro-orm/core": 5.9.7 + "@mikro-orm/migrations": 5.9.7 + "@mikro-orm/postgresql": 5.9.7 + awilix: ^8.0.0 + cross-env: ^5.2.1 + dotenv: ^16.1.4 + jest: ^29.6.3 + knex: 2.4.2 + medusa-test-utils: ^1.1.40 + rimraf: ^3.0.2 + ts-jest: ^29.1.1 + ts-node: ^10.9.1 + tsc-alias: ^1.8.6 + typescript: ^5.1.6 + languageName: unknown + linkType: soft + +"@medusajs/workflow-engine-redis@workspace:packages/workflow-engine-redis": + version: 0.0.0-use.local + resolution: "@medusajs/workflow-engine-redis@workspace:packages/workflow-engine-redis" + dependencies: + "@medusajs/modules-sdk": ^1.12.5 + "@medusajs/types": ^1.11.9 + "@medusajs/utils": ^1.11.2 + "@medusajs/workflows-sdk": ^0.1.0 + "@mikro-orm/cli": 5.9.7 + "@mikro-orm/core": 5.9.7 + "@mikro-orm/migrations": 5.9.7 + "@mikro-orm/postgresql": 5.9.7 + awilix: ^8.0.0 + bullmq: ^5.1.3 + cross-env: ^5.2.1 + dotenv: ^16.1.4 + ioredis: ^5.3.2 + jest: ^29.6.3 + knex: 2.4.2 + medusa-test-utils: ^1.1.40 + rimraf: ^3.0.2 + ts-jest: ^29.1.1 + ts-node: ^10.9.1 + tsc-alias: ^1.8.6 + typescript: ^5.1.6 + languageName: unknown + linkType: soft + +"@medusajs/workflows-sdk@^0.1.0, @medusajs/workflows-sdk@^0.1.1, @medusajs/workflows-sdk@workspace:packages/workflows-sdk": version: 0.0.0-use.local resolution: "@medusajs/workflows-sdk@workspace:packages/workflows-sdk" dependencies: @@ -21778,6 +21832,23 @@ __metadata: languageName: node linkType: hard +"bullmq@npm:^5.1.3": + version: 5.1.3 + resolution: "bullmq@npm:5.1.3" + dependencies: + cron-parser: ^4.6.0 + glob: ^8.0.3 + ioredis: ^5.3.2 + lodash: ^4.17.21 + msgpackr: ^1.10.1 + node-abort-controller: ^3.1.1 + semver: ^7.5.4 + tslib: ^2.0.0 + uuid: ^9.0.0 + checksum: dc2177dfd736b2d008ccab1ba9f77f80cc730ce6197c9ffa0f37327e1cf34bd8b97d83ee9f9008253ef0c0854bbd04f8c925889a3370a0899e8f5c7a34fd3ab3 + languageName: node + linkType: hard + "bundle-name@npm:^3.0.0": version: 3.0.0 resolution: "bundle-name@npm:3.0.0" @@ -38529,6 +38600,18 @@ __metadata: languageName: node linkType: hard +"msgpackr@npm:^1.10.1": + version: 1.10.1 + resolution: "msgpackr@npm:1.10.1" + dependencies: + msgpackr-extract: ^3.0.2 + dependenciesMeta: + msgpackr-extract: + optional: true + checksum: 2e6ed91af89ec15d1e5595c5b837a4adcbb185b0fbd4773d728ced89ab4abbdd3401f6777b193d487d9807e1cb0cf3da1ba9a0bd2d5a553e22355cea84a36bab + languageName: node + linkType: hard + "msgpackr@npm:^1.5.4, msgpackr@npm:^1.6.2": version: 1.9.5 resolution: "msgpackr@npm:1.9.5" @@ -38826,6 +38909,13 @@ __metadata: languageName: node linkType: hard +"node-abort-controller@npm:^3.1.1": + version: 3.1.1 + resolution: "node-abort-controller@npm:3.1.1" + checksum: f7ad0e7a8e33809d4f3a0d1d65036a711c39e9d23e0319d80ebe076b9a3b4432b4d6b86a7fab65521de3f6872ffed36fc35d1327487c48eb88c517803403eda3 + languageName: node + linkType: hard + "node-addon-api@npm:^4.3.0": version: 4.3.0 resolution: "node-addon-api@npm:4.3.0"