asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore(orchestration): idempotent (#7771)

Browse Source
This commit is contained in:
Carlos R. L. Rodrigues
2024-06-25 10:34:00 -03:00
committed by GitHub
parent 66d17fabde
commit 5600e58b7f
19 changed files with 421 additions and 403 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/modules/workflow-engine-redis/integration-tests/__tests__/index.spec.ts
+50 -31
View File
@@ -4,8 +4,18 @@ import {
TransactionTimeoutError,
WorkflowManager,
} from "@medusajs/orchestration"
import { IWorkflowEngineService, RemoteQueryFunction } from "@medusajs/types"
import { TransactionHandlerType, TransactionStepState } from "@medusajs/utils"
import {
IWorkflowEngineService,
MedusaContainer,
RemoteQueryFunction,
} from "@medusajs/types"
import {
ContainerRegistrationKeys,
TransactionHandlerType,
TransactionStepState,
createMedusaContainer,
} from "@medusajs/utils"
import { asValue } from "awilix"
import { knex } from "knex"
import { setTimeout } from "timers/promises"
import "../__fixtures__"
@@ -28,42 +38,47 @@ const afterEach_ = async () => {
}
describe("Workflow Orchestrator module", function () {
describe("Testing basic workflow", function () {
let workflowOrcModule: IWorkflowEngineService
let query: RemoteQueryFunction
let workflowOrcModule: IWorkflowEngineService
let query: RemoteQueryFunction
let sharedContainer_: MedusaContainer
afterEach(afterEach_)
beforeAll(async () => {
const container = createMedusaContainer()
container.register(ContainerRegistrationKeys.LOGGER, asValue(console))
beforeAll(async () => {
const {
runMigrations,
query: remoteQuery,
modules,
} = await MedusaApp({
sharedResourcesConfig: {
database: {
connection: sharedPgConnection,
},
const {
runMigrations,
query: remoteQuery,
modules,
sharedContainer,
} = await MedusaApp({
sharedContainer: container,
sharedResourcesConfig: {
database: {
connection: sharedPgConnection,
},
modulesConfig: {
workflows: {
resolve: __dirname + "/../..",
options: {
redis: {
url: "localhost:6379",
},
},
modulesConfig: {
workflows: {
resolve: __dirname + "/../..",
options: {
redis: {
url: "localhost:6379",
},
},
},
})
query = remoteQuery
await runMigrations()
workflowOrcModule = modules.workflows as unknown as IWorkflowEngineService
},
})
query = remoteQuery
sharedContainer_ = sharedContainer!
await runMigrations()
workflowOrcModule = modules.workflows as unknown as IWorkflowEngineService
})
describe("Testing basic workflow", function () {
afterEach(afterEach_)
it("should return a list of workflow executions and remove after completed when there is no retentionTime set", async () => {
@@ -320,10 +335,14 @@ describe("Workflow Orchestrator module", function () {
})
it("should remove scheduled workflow if workflow no longer exists", async () => {
const logger = sharedContainer_.resolve<Logger>(
ContainerRegistrationKeys.LOGGER
)
const spy = await createScheduled("remove-scheduled", {
cron: "* * * * * *",
})
const logSpy = jest.spyOn(console, "warn")
const logSpy = jest.spyOn(logger, "warn")
await setTimeout(1100)
expect(spy).toHaveBeenCalledTimes(1)
packages/modules/workflow-engine-redis/integration-tests/utils/database.ts
+1 -1
View File
@@ -46,7 +46,7 @@ async function deleteKeysByPattern(pattern) {
async function cleanRedis() {
try {
await deleteKeysByPattern("bull:*")
await deleteKeysByPattern("dtrans:*")
await deleteKeysByPattern("dtrx:*")
} catch (error) {
console.error("Error:", error)
}
packages/modules/workflow-engine-redis/src/index.ts
+1
View File
@@ -3,3 +3,4 @@ import { moduleDefinition } from "./module-definition"
export default moduleDefinition
export * from "./loaders"
export * from "./models"
packages/modules/workflow-engine-redis/src/services/workflows-module.ts
+12 -124
View File
@@ -1,26 +1,23 @@
import {
Context,
DAL,
FindConfig,
InternalModuleDeclaration,
IWorkflowEngineService,
ModuleJoinerConfig,
ModulesSdkTypes,
WorkflowsSdkTypes,
} from "@medusajs/types"
import {
InjectManager,
InjectSharedContext,
isString,
MedusaContext,
MedusaError,
ModulesSdkUtils,
} from "@medusajs/utils"
import type {
ReturnWorkflow,
UnwrapWorkflowInputDataType,
} from "@medusajs/workflows-sdk"
import { WorkflowExecution } from "@models"
import { WorkflowOrchestratorService } from "@services"
import { joinerConfig } from "../joiner-config"
import { entityNameToLinkableKeysMap, joinerConfig } from "../joiner-config"
type InjectedDependencies = {
baseRepository: DAL.RepositoryService
@@ -29,9 +26,13 @@ type InjectedDependencies = {
redisDisconnectHandler: () => Promise<void>
}
export class WorkflowsModuleService implements IWorkflowEngineService {
export class WorkflowsModuleService<
TWorkflowExecution extends WorkflowExecution = WorkflowExecution
> extends ModulesSdkUtils.MedusaService<{
WorkflowExecution: { dto: WorkflowExecution }
}>({ WorkflowExecution }, entityNameToLinkableKeysMap) {
protected baseRepository_: DAL.RepositoryService
protected workflowExecutionService_: ModulesSdkTypes.IMedusaInternalService<any>
protected workflowExecutionService_: ModulesSdkTypes.IMedusaInternalService<TWorkflowExecution>
protected workflowOrchestratorService_: WorkflowOrchestratorService
protected redisDisconnectHandler_: () => Promise<void>
@@ -44,6 +45,9 @@ export class WorkflowsModuleService implements IWorkflowEngineService {
}: InjectedDependencies,
protected readonly moduleDeclaration: InternalModuleDeclaration
) {
// @ts-ignore
super(...arguments)
this.baseRepository_ = baseRepository
this.workflowExecutionService_ = workflowExecutionService
this.workflowOrchestratorService_ = workflowOrchestratorService
@@ -64,122 +68,6 @@ export class WorkflowsModuleService implements IWorkflowEngineService {
},
}
@InjectManager("baseRepository_")
async retrieveWorkflowExecution(
idOrObject:
| string
| {
workflow_id: string
transaction_id: string
},
config: FindConfig<WorkflowsSdkTypes.WorkflowExecutionDTO> = {},
@MedusaContext() sharedContext: Context = {}
): Promise<WorkflowsSdkTypes.WorkflowExecutionDTO> {
const objValue = isString(idOrObject)
? { id: idOrObject }
: {
workflow_id: idOrObject.workflow_id,
transaction_id: idOrObject.transaction_id,
}
const wfExecution = await this.workflowExecutionService_.list(
objValue,
config,
sharedContext
)
if (wfExecution.length === 0) {
throw new MedusaError(
MedusaError.Types.NOT_FOUND,
`WorkflowExecution with ${Object.keys(objValue).join(
", "
)}: ${Object.values(objValue).join(", ")} was not found`
)
}
// eslint-disable-next-line max-len
return await this.baseRepository_.serialize<WorkflowsSdkTypes.WorkflowExecutionDTO>(
wfExecution[0],
{
populate: true,
}
)
}
@InjectManager("baseRepository_")
async listWorkflowExecutions(
filters: WorkflowsSdkTypes.FilterableWorkflowExecutionProps = {},
config: FindConfig<WorkflowsSdkTypes.WorkflowExecutionDTO> = {},
@MedusaContext() sharedContext: Context = {}
): Promise<WorkflowsSdkTypes.WorkflowExecutionDTO[]> {
if (filters.transaction_id) {
if (Array.isArray(filters.transaction_id)) {
filters.transaction_id = {
$in: filters.transaction_id,
}
}
}
if (filters.workflow_id) {
if (Array.isArray(filters.workflow_id)) {
filters.workflow_id = {
$in: filters.workflow_id,
}
}
}
const wfExecutions = await this.workflowExecutionService_.list(
filters,
config,
sharedContext
)
return await this.baseRepository_.serialize<
WorkflowsSdkTypes.WorkflowExecutionDTO[]
>(wfExecutions, {
populate: true,
})
}
@InjectManager("baseRepository_")
async listAndCountWorkflowExecutions(
filters: WorkflowsSdkTypes.FilterableWorkflowExecutionProps = {},
config: FindConfig<WorkflowsSdkTypes.WorkflowExecutionDTO> = {},
@MedusaContext() sharedContext: Context = {}
): Promise<[WorkflowsSdkTypes.WorkflowExecutionDTO[], number]> {
if (filters.transaction_id) {
if (Array.isArray(filters.transaction_id)) {
filters.transaction_id = {
$in: filters.transaction_id,
}
}
}
if (filters.workflow_id) {
if (Array.isArray(filters.workflow_id)) {
filters.workflow_id = {
$in: filters.workflow_id,
}
}
}
const [wfExecutions, count] =
await this.workflowExecutionService_.listAndCount(
filters,
config,
sharedContext
)
return [
await this.baseRepository_.serialize<
WorkflowsSdkTypes.WorkflowExecutionDTO[]
>(wfExecutions, {
populate: true,
}),
count,
]
}
@InjectSharedContext()
async run<TWorkflow extends string | ReturnWorkflow<any, any, any>>(
workflowIdOrWorkflow: TWorkflow,
packages/modules/workflow-engine-redis/src/utils/workflow-orchestrator-storage.ts
+47 -12
View File
@@ -4,9 +4,10 @@ import {
IDistributedTransactionStorage,
SchedulerOptions,
TransactionCheckpoint,
TransactionOptions,
TransactionStep,
} from "@medusajs/orchestration"
import { ModulesSdkTypes } from "@medusajs/types"
import { Logger, ModulesSdkTypes } from "@medusajs/types"
import { MedusaError, TransactionState, promiseAll } from "@medusajs/utils"
import { WorkflowOrchestratorService } from "@services"
import { Queue, Worker } from "bullmq"
@@ -19,12 +20,12 @@ enum JobType {
TRANSACTION_TIMEOUT = "transaction_timeout",
}
// eslint-disable-next-line max-len
export class RedisDistributedTransactionStorage
implements IDistributedTransactionStorage, IDistributedSchedulerStorage
{
private static TTL_AFTER_COMPLETED = 60 * 15 // 15 minutes
private static TTL_AFTER_COMPLETED = 60 * 2 // 2 minutes
private workflowExecutionService_: ModulesSdkTypes.IMedusaInternalService<any>
private logger_: Logger
private workflowOrchestratorService_: WorkflowOrchestratorService
private redisClient: Redis
@@ -36,13 +37,16 @@ export class RedisDistributedTransactionStorage
redisConnection,
redisWorkerConnection,
redisQueueName,
logger,
}: {
workflowExecutionService: ModulesSdkTypes.IMedusaInternalService<any>
redisConnection: Redis
redisWorkerConnection: Redis
redisQueueName: string
logger: Logger
}) {
this.workflowExecutionService_ = workflowExecutionService
this.logger_ = logger
this.redisClient = redisConnection
@@ -131,7 +135,7 @@ export class RedisDistributedTransactionStorage
})
} catch (e) {
if (e instanceof MedusaError && e.type === MedusaError.Types.NOT_FOUND) {
console.warn(
this.logger_?.warn(
`Tried to execute a scheduled workflow with ID ${jobId} that does not exist, removing it from the scheduler.`
)
@@ -143,10 +147,42 @@ export class RedisDistributedTransactionStorage
}
}
async get(key: string): Promise<TransactionCheckpoint | undefined> {
async get(
key: string,
options?: TransactionOptions
): Promise<TransactionCheckpoint | undefined> {
const data = await this.redisClient.get(key)
return data ? JSON.parse(data) : undefined
if (data) {
return JSON.parse(data)
}
const { idempotent } = options ?? {}
if (!idempotent) {
return
}
const [_, workflowId, transactionId] = key.split(":")
const trx = await this.workflowExecutionService_
.retrieve(
{
workflow_id: workflowId,
transaction_id: transactionId,
},
{
select: ["execution", "context"],
}
)
.catch(() => undefined)
if (trx) {
return {
flow: trx.execution,
context: trx.context.data,
errors: trx.context.errors,
}
}
return
}
async list(): Promise<TransactionCheckpoint[]> {
@@ -166,10 +202,9 @@ export class RedisDistributedTransactionStorage
async save(
key: string,
data: TransactionCheckpoint,
ttl?: number
ttl?: number,
options?: TransactionOptions
): Promise<void> {
let retentionTime
/**
* Store the retention time only if the transaction is done, failed or reverted.
* From that moment, this tuple can be later on archived or deleted after the retention time.
@@ -180,8 +215,9 @@ export class RedisDistributedTransactionStorage
TransactionState.REVERTED,
].includes(data.flow.state)
const { retentionTime, idempotent } = options ?? {}
if (hasFinished) {
retentionTime = data.flow.options?.retentionTime
Object.assign(data, {
retention_time: retentionTime,
})
@@ -198,14 +234,13 @@ export class RedisDistributedTransactionStorage
}
}
if (hasFinished && !retentionTime) {
if (hasFinished && !retentionTime && !idempotent) {
await this.deleteFromDb(parsedData)
} else {
await this.saveToDb(parsedData)
}
if (hasFinished) {
// await this.redisClient.del(key)
await this.redisClient.set(
key,
stringifiedData,