asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore(event-bus, workflow-engine): Enable more granualar queues configuration (#14201)

Browse Source 
Summary

  This PR adds BullMQ queue and worker configuration options to the workflow-engine-redis module, bringing feature parity with the event-bus-redis module. It also introduces per-queue
  configuration options for fine-grained control over the three internal queues (main, job, and cleaner).

  Key changes:
  - Added per-queue BullMQ configuration options (mainQueueOptions, jobQueueOptions, cleanerQueueOptions and their worker counterparts) with shared defaults
  - Unified Redis option naming across modules: deprecated url → redisUrl, options → redisOptions (with backward compatibility)
  - Moved configuration resolution to the loader and registered options in the DI container
  - Added comprehensive JSDoc documentation for all configuration options
  - Added unit tests for option merging and queue/worker configuration

  Configuration Example

```ts
  // Simple configuration - same options for all queues
  {
    redisUrl: "redis://localhost:6379",
    queueOptions: { defaultJobOptions: { removeOnComplete: 1000 } },
    workerOptions: { concurrency: 10 }
  }
```

```ts
  // Advanced configuration - per-queue overrides
  {
    redisUrl: "redis://localhost:6379",
    workerOptions: { concurrency: 10 },        // shared default
    jobWorkerOptions: { concurrency: 5 },      // override for scheduled workflows
    cleanerWorkerOptions: { concurrency: 1 }   // override for cleanup (low priority)
  }
```
This commit is contained in:
Adrien de Peretti
2025-12-05 13:03:12 +01:00
committed by GitHub
parent 3e3e6c37bd
commit 144f0f4e2e
10 changed files with 1013 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/modules/workflow-engine-redis/src/utils/__tests__/workflow-orchestrator-storage.spec.ts
+321
View File
@@ -0,0 +1,321 @@
import { Logger, ModulesSdkTypes } from "@medusajs/framework/types"
import { Queue, Worker } from "bullmq"
import Redis from "ioredis"
import { RedisDistributedTransactionStorage } from "../workflow-orchestrator-storage"
jest.mock("bullmq")
jest.mock("ioredis")
const loggerMock = {
info: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
debug: jest.fn(),
} as unknown as Logger
const redisMock = {
status: "ready",
disconnect: jest.fn(),
get: jest.fn(),
set: jest.fn(),
del: jest.fn(),
pipeline: jest.fn(() => ({
exec: jest.fn(),
})),
} as unknown as Redis
const workflowExecutionServiceMock = {
list: jest.fn(),
upsert: jest.fn(),
delete: jest.fn(),
} as unknown as ModulesSdkTypes.IMedusaInternalService<any>
const baseModuleDeps = {
workflowExecutionService: workflowExecutionServiceMock,
redisConnection: redisMock,
redisWorkerConnection: redisMock,
redisQueueName: "medusa-workflows",
redisJobQueueName: "medusa-workflows-jobs",
logger: loggerMock,
isWorkerMode: true,
}
describe("RedisDistributedTransactionStorage", () => {
beforeEach(() => {
jest.clearAllMocks()
})
describe("constructor - Queue configuration", () => {
it("should create queues with default empty options when no options provided", () => {
new RedisDistributedTransactionStorage({
...baseModuleDeps,
redisMainQueueOptions: {},
redisMainWorkerOptions: {},
redisJobQueueOptions: {},
redisJobWorkerOptions: {},
redisCleanerQueueOptions: {},
redisCleanerWorkerOptions: {},
})
expect(Queue).toHaveBeenCalledWith("medusa-workflows", {
connection: redisMock,
})
expect(Queue).toHaveBeenCalledWith("medusa-workflows-jobs", {
connection: redisMock,
})
expect(Queue).toHaveBeenCalledWith("workflows-cleaner", {
connection: redisMock,
})
expect(Queue).toHaveBeenCalledTimes(3)
})
it("should create main queue with custom options", () => {
const mainQueueOptions = {
defaultJobOptions: {
removeOnComplete: 1000,
removeOnFail: 5000,
},
}
new RedisDistributedTransactionStorage({
...baseModuleDeps,
redisMainQueueOptions: mainQueueOptions,
redisMainWorkerOptions: {},
redisJobQueueOptions: {},
redisJobWorkerOptions: {},
redisCleanerQueueOptions: {},
redisCleanerWorkerOptions: {},
})
expect(Queue).toHaveBeenCalledWith("medusa-workflows", {
...mainQueueOptions,
connection: redisMock,
})
})
it("should create job queue with custom options", () => {
const jobQueueOptions = {
defaultJobOptions: {
attempts: 5,
backoff: { type: "exponential", delay: 1000 },
},
}
new RedisDistributedTransactionStorage({
...baseModuleDeps,
redisMainQueueOptions: {},
redisMainWorkerOptions: {},
redisJobQueueOptions: jobQueueOptions,
redisJobWorkerOptions: {},
redisCleanerQueueOptions: {},
redisCleanerWorkerOptions: {},
})
expect(Queue).toHaveBeenCalledWith("medusa-workflows-jobs", {
...jobQueueOptions,
connection: redisMock,
})
})
it("should create cleaner queue with custom options", () => {
const cleanerQueueOptions = {
defaultJobOptions: {
removeOnComplete: true,
removeOnFail: true,
},
}
new RedisDistributedTransactionStorage({
...baseModuleDeps,
redisMainQueueOptions: {},
redisMainWorkerOptions: {},
redisJobQueueOptions: {},
redisJobWorkerOptions: {},
redisCleanerQueueOptions: cleanerQueueOptions,
redisCleanerWorkerOptions: {},
})
expect(Queue).toHaveBeenCalledWith("workflows-cleaner", {
...cleanerQueueOptions,
connection: redisMock,
})
})
it("should create each queue with different options", () => {
const mainQueueOptions = { defaultJobOptions: { removeOnComplete: 100 } }
const jobQueueOptions = { defaultJobOptions: { removeOnComplete: 200 } }
const cleanerQueueOptions = {
defaultJobOptions: { removeOnComplete: 300 },
}
new RedisDistributedTransactionStorage({
...baseModuleDeps,
redisMainQueueOptions: mainQueueOptions,
redisMainWorkerOptions: {},
redisJobQueueOptions: jobQueueOptions,
redisJobWorkerOptions: {},
redisCleanerQueueOptions: cleanerQueueOptions,
redisCleanerWorkerOptions: {},
})
expect(Queue).toHaveBeenNthCalledWith(1, "medusa-workflows", {
...mainQueueOptions,
connection: redisMock,
})
expect(Queue).toHaveBeenNthCalledWith(2, "medusa-workflows-jobs", {
...jobQueueOptions,
connection: redisMock,
})
expect(Queue).toHaveBeenNthCalledWith(3, "workflows-cleaner", {
...cleanerQueueOptions,
connection: redisMock,
})
})
it("should not create job and cleaner queues when not in worker mode", () => {
new RedisDistributedTransactionStorage({
...baseModuleDeps,
isWorkerMode: false,
redisMainQueueOptions: {},
redisMainWorkerOptions: {},
redisJobQueueOptions: {},
redisJobWorkerOptions: {},
redisCleanerQueueOptions: {},
redisCleanerWorkerOptions: {},
})
expect(Queue).toHaveBeenCalledTimes(1)
expect(Queue).toHaveBeenCalledWith("medusa-workflows", {
connection: redisMock,
})
})
})
describe("onApplicationStart - Worker configuration", () => {
it("should create workers with custom options", async () => {
const mainWorkerOptions = {
concurrency: 10,
limiter: { max: 100, duration: 1000 },
}
const jobWorkerOptions = { concurrency: 5 }
const cleanerWorkerOptions = { concurrency: 1 }
const storage = new RedisDistributedTransactionStorage({
...baseModuleDeps,
redisMainQueueOptions: {},
redisMainWorkerOptions: mainWorkerOptions,
redisJobQueueOptions: {},
redisJobWorkerOptions: jobWorkerOptions,
redisCleanerQueueOptions: {},
redisCleanerWorkerOptions: cleanerWorkerOptions,
})
const mockQueue = {
getRepeatableJobs: jest.fn().mockResolvedValue([]),
add: jest.fn().mockResolvedValue({}),
}
;(storage as any).queue = mockQueue
;(storage as any).cleanerQueue_ = mockQueue
await storage.onApplicationStart()
expect(Worker).toHaveBeenCalledWith(
"medusa-workflows",
expect.any(Function),
{
...mainWorkerOptions,
connection: redisMock,
}
)
expect(Worker).toHaveBeenCalledWith(
"medusa-workflows-jobs",
expect.any(Function),
{
...jobWorkerOptions,
connection: redisMock,
}
)
expect(Worker).toHaveBeenCalledWith(
"workflows-cleaner",
expect.any(Function),
{
...cleanerWorkerOptions,
connection: redisMock,
}
)
expect(Worker).toHaveBeenCalledTimes(3)
})
it("should create each worker with different concurrency settings", async () => {
const storage = new RedisDistributedTransactionStorage({
...baseModuleDeps,
redisMainQueueOptions: {},
redisMainWorkerOptions: { concurrency: 20 },
redisJobQueueOptions: {},
redisJobWorkerOptions: { concurrency: 10 },
redisCleanerQueueOptions: {},
redisCleanerWorkerOptions: { concurrency: 1 },
})
const mockQueue = {
getRepeatableJobs: jest.fn().mockResolvedValue([]),
add: jest.fn().mockResolvedValue({}),
}
;(storage as any).queue = mockQueue
;(storage as any).cleanerQueue_ = mockQueue
await storage.onApplicationStart()
expect(Worker).toHaveBeenNthCalledWith(
1,
"medusa-workflows",
expect.any(Function),
expect.objectContaining({ concurrency: 20 })
)
expect(Worker).toHaveBeenNthCalledWith(
2,
"medusa-workflows-jobs",
expect.any(Function),
expect.objectContaining({ concurrency: 10 })
)
expect(Worker).toHaveBeenNthCalledWith(
3,
"workflows-cleaner",
expect.any(Function),
expect.objectContaining({ concurrency: 1 })
)
})
it("should not create workers when not in worker mode", async () => {
const storage = new RedisDistributedTransactionStorage({
...baseModuleDeps,
isWorkerMode: false,
redisMainQueueOptions: {},
redisMainWorkerOptions: {},
redisJobQueueOptions: {},
redisJobWorkerOptions: {},
redisCleanerQueueOptions: {},
redisCleanerWorkerOptions: {},
})
const mockQueue = {
getRepeatableJobs: jest.fn().mockResolvedValue([]),
}
;(storage as any).queue = mockQueue
await storage.onApplicationStart()
expect(Worker).not.toHaveBeenCalled()
})
})
})
packages/modules/workflow-engine-redis/src/utils/workflow-orchestrator-storage.ts
+51 -6
View File
@@ -24,7 +24,7 @@ import {
TransactionStepState,
} from "@medusajs/framework/utils"
import { WorkflowOrchestratorService } from "@services"
import { Queue, RepeatOptions, Worker } from "bullmq"
import { Queue, QueueOptions, RepeatOptions, Worker, WorkerOptions } from "bullmq"
import Redis from "ioredis"
enum JobType {
@@ -75,6 +75,14 @@ export class RedisDistributedTransactionStorage
private cleanerWorker_: Worker
private cleanerQueue_?: Queue
// Per-queue options
private mainQueueOptions_: Omit<QueueOptions, "connection">
private mainWorkerOptions_: Omit<WorkerOptions, "connection">
private jobQueueOptions_: Omit<QueueOptions, "connection">
private jobWorkerOptions_: Omit<WorkerOptions, "connection">
private cleanerQueueOptions_: Omit<QueueOptions, "connection">
private cleanerWorkerOptions_: Omit<WorkerOptions, "connection">
#isWorkerMode: boolean = false
constructor({
@@ -83,6 +91,12 @@ export class RedisDistributedTransactionStorage
redisWorkerConnection,
redisQueueName,
redisJobQueueName,
redisMainQueueOptions,
redisMainWorkerOptions,
redisJobQueueOptions,
redisJobWorkerOptions,
redisCleanerQueueOptions,
redisCleanerWorkerOptions,
logger,
isWorkerMode,
}: {
@@ -91,6 +105,12 @@ export class RedisDistributedTransactionStorage
redisWorkerConnection: Redis
redisQueueName: string
redisJobQueueName: string
redisMainQueueOptions: Omit<QueueOptions, "connection">
redisMainWorkerOptions: Omit<WorkerOptions, "connection">
redisJobQueueOptions: Omit<QueueOptions, "connection">
redisJobWorkerOptions: Omit<WorkerOptions, "connection">
redisCleanerQueueOptions: Omit<QueueOptions, "connection">
redisCleanerWorkerOptions: Omit<WorkerOptions, "connection">
logger: Logger
isWorkerMode: boolean
}) {
@@ -101,14 +121,29 @@ export class RedisDistributedTransactionStorage
this.cleanerQueueName = "workflows-cleaner"
this.queueName = redisQueueName
this.jobQueueName = redisJobQueueName
this.queue = new Queue(redisQueueName, { connection: this.redisClient })
// Store per-queue options
this.mainQueueOptions_ = redisMainQueueOptions ?? {}
this.mainWorkerOptions_ = redisMainWorkerOptions ?? {}
this.jobQueueOptions_ = redisJobQueueOptions ?? {}
this.jobWorkerOptions_ = redisJobWorkerOptions ?? {}
this.cleanerQueueOptions_ = redisCleanerQueueOptions ?? {}
this.cleanerWorkerOptions_ = redisCleanerWorkerOptions ?? {}
// Create queues with their respective options
this.queue = new Queue(redisQueueName, {
...this.mainQueueOptions_,
connection: this.redisClient,
})
this.jobQueue = isWorkerMode
? new Queue(redisJobQueueName, {
...this.jobQueueOptions_,
connection: this.redisClient,
})
: undefined
this.cleanerQueue_ = isWorkerMode
? new Queue(this.cleanerQueueName, {
...this.cleanerQueueOptions_,
connection: this.redisClient,
})
: undefined
@@ -137,7 +172,17 @@ export class RedisDistributedTransactionStorage
JobType.TRANSACTION_TIMEOUT,
]
const workerOptions = {
// Per-worker options with their respective configurations
const mainWorkerOptions: WorkerOptions = {
...this.mainWorkerOptions_,
connection: this.redisWorkerConnection,
}
const jobWorkerOptions: WorkerOptions = {
...this.jobWorkerOptions_,
connection: this.redisWorkerConnection,
}
const cleanerWorkerOptions: WorkerOptions = {
...this.cleanerWorkerOptions_,
connection: this.redisWorkerConnection,
}
@@ -173,7 +218,7 @@ export class RedisDistributedTransactionStorage
await this.remove(job.data.jobId)
}
},
workerOptions
mainWorkerOptions
)
this.jobWorker = new Worker(
@@ -191,7 +236,7 @@ export class RedisDistributedTransactionStorage
job.data.schedulerOptions
)
},
workerOptions
jobWorkerOptions
)
this.cleanerWorker_ = new Worker(
@@ -199,7 +244,7 @@ export class RedisDistributedTransactionStorage
async () => {
await this.clearExpiredExecutions()
},
workerOptions
cleanerWorkerOptions
)
await this.cleanerQueue_?.add(