asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

chore(event-bus, workflow-engine): Enable more granualar queues configuration (#14201)

Browse Source 
Summary

  This PR adds BullMQ queue and worker configuration options to the workflow-engine-redis module, bringing feature parity with the event-bus-redis module. It also introduces per-queue
  configuration options for fine-grained control over the three internal queues (main, job, and cleaner).

  Key changes:
  - Added per-queue BullMQ configuration options (mainQueueOptions, jobQueueOptions, cleanerQueueOptions and their worker counterparts) with shared defaults
  - Unified Redis option naming across modules: deprecated url → redisUrl, options → redisOptions (with backward compatibility)
  - Moved configuration resolution to the loader and registered options in the DI container
  - Added comprehensive JSDoc documentation for all configuration options
  - Added unit tests for option merging and queue/worker configuration

  Configuration Example

```ts
  // Simple configuration - same options for all queues
  {
    redisUrl: "redis://localhost:6379",
    queueOptions: { defaultJobOptions: { removeOnComplete: 1000 } },
    workerOptions: { concurrency: 10 }
  }
```

```ts
  // Advanced configuration - per-queue overrides
  {
    redisUrl: "redis://localhost:6379",
    workerOptions: { concurrency: 10 },        // shared default
    jobWorkerOptions: { concurrency: 5 },      // override for scheduled workflows
    cleanerWorkerOptions: { concurrency: 1 }   // override for cleanup (low priority)
  }
```
This commit is contained in:
Adrien de Peretti
2025-12-05 13:03:12 +01:00
committed by GitHub
parent 3e3e6c37bd
commit 144f0f4e2e
10 changed files with 1013 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files

323
packages/modules/workflow-engine-redis/src/loaders/__tests__/redis.spec.ts Normal file
View File

@@ -0,0 +1,323 @@
import { Logger } from "@medusajs/framework/types"
import redisLoader from "../redis"
jest.mock("ioredis", () => {
return jest.fn().mockImplementation(() => ({
connect: jest.fn((callback) => {
if (callback) callback()
return Promise.resolve()
}),
disconnect: jest.fn(),
}))
})
const loggerMock = {
info: jest.fn(),
warn: jest.fn(),
error: jest.fn(),
debug: jest.fn(),
} as unknown as Logger
describe("Redis Loader", () => {
let containerMock: { register: jest.Mock }
beforeEach(() => {
jest.clearAllMocks()
containerMock = {
register: jest.fn(),
}
})
describe("Option merging", () => {
it("should use shared queueOptions as default for all queues", async () => {
const sharedQueueOptions = {
defaultJobOptions: { removeOnComplete: 1000 },
}
await redisLoader(
{
container: containerMock as any,
logger: loggerMock,
options: {
redis: {
redisUrl: "redis://localhost:6379",
queueOptions: sharedQueueOptions,
},
},
} as any,
{} as any
)
const registerCall = containerMock.register.mock.calls[0][0]
expect(registerCall.redisMainQueueOptions.resolve()).toEqual(
sharedQueueOptions
)
expect(registerCall.redisJobQueueOptions.resolve()).toEqual(
sharedQueueOptions
)
expect(registerCall.redisCleanerQueueOptions.resolve()).toEqual(
sharedQueueOptions
)
})
it("should use shared workerOptions as default for all workers", async () => {
const sharedWorkerOptions = { concurrency: 10 }
await redisLoader(
{
container: containerMock as any,
logger: loggerMock,
options: {
redis: {
redisUrl: "redis://localhost:6379",
workerOptions: sharedWorkerOptions,
},
},
} as any,
{} as any
)
const registerCall = containerMock.register.mock.calls[0][0]
expect(registerCall.redisMainWorkerOptions.resolve()).toEqual(
sharedWorkerOptions
)
expect(registerCall.redisJobWorkerOptions.resolve()).toEqual(
sharedWorkerOptions
)
expect(registerCall.redisCleanerWorkerOptions.resolve()).toEqual(
sharedWorkerOptions
)
})
it("should override shared options with per-queue options", async () => {
const sharedQueueOptions = {
defaultJobOptions: { removeOnComplete: 1000 },
}
const mainQueueOptions = {
defaultJobOptions: { removeOnComplete: 500, attempts: 3 },
}
await redisLoader(
{
container: containerMock as any,
logger: loggerMock,
options: {
redis: {
redisUrl: "redis://localhost:6379",
queueOptions: sharedQueueOptions,
mainQueueOptions: mainQueueOptions,
},
},
} as any,
{} as any
)
const registerCall = containerMock.register.mock.calls[0][0]
expect(registerCall.redisMainQueueOptions.resolve()).toEqual({
defaultJobOptions: { removeOnComplete: 500, attempts: 3 },
})
expect(registerCall.redisJobQueueOptions.resolve()).toEqual(
sharedQueueOptions
)
expect(registerCall.redisCleanerQueueOptions.resolve()).toEqual(
sharedQueueOptions
)
})
it("should override shared worker options with per-worker options", async () => {
const sharedWorkerOptions = { concurrency: 10 }
const jobWorkerOptions = { concurrency: 5 }
const cleanerWorkerOptions = { concurrency: 1 }
await redisLoader(
{
container: containerMock as any,
logger: loggerMock,
options: {
redis: {
redisUrl: "redis://localhost:6379",
workerOptions: sharedWorkerOptions,
jobWorkerOptions: jobWorkerOptions,
cleanerWorkerOptions: cleanerWorkerOptions,
},
},
} as any,
{} as any
)
const registerCall = containerMock.register.mock.calls[0][0]
expect(registerCall.redisMainWorkerOptions.resolve()).toEqual(
sharedWorkerOptions
)
expect(registerCall.redisJobWorkerOptions.resolve()).toEqual(
jobWorkerOptions
)
expect(registerCall.redisCleanerWorkerOptions.resolve()).toEqual(
cleanerWorkerOptions
)
})
it("should merge nested options correctly", async () => {
const sharedWorkerOptions = {
concurrency: 10,
limiter: { max: 100, duration: 1000 },
}
const mainWorkerOptions = {
concurrency: 20,
}
await redisLoader(
{
container: containerMock as any,
logger: loggerMock,
options: {
redis: {
redisUrl: "redis://localhost:6379",
workerOptions: sharedWorkerOptions,
mainWorkerOptions: mainWorkerOptions,
},
},
} as any,
{} as any
)
const registerCall = containerMock.register.mock.calls[0][0]
expect(registerCall.redisMainWorkerOptions.resolve()).toEqual({
concurrency: 20,
limiter: { max: 100, duration: 1000 },
})
})
})
describe("Deprecation warnings", () => {
it("should log warning when using deprecated 'url' option", async () => {
await redisLoader(
{
container: containerMock as any,
logger: loggerMock,
options: {
redis: {
url: "redis://localhost:6379",
},
},
} as any,
{} as any
)
expect(loggerMock.warn).toHaveBeenCalledWith(
"[Workflow-engine-redis] The `url` option is deprecated. Please use `redisUrl` instead for consistency with other modules."
)
})
it("should log warning when using deprecated 'options' option", async () => {
await redisLoader(
{
container: containerMock as any,
logger: loggerMock,
options: {
redis: {
redisUrl: "redis://localhost:6379",
options: { maxRetriesPerRequest: 3 },
},
},
} as any,
{} as any
)
expect(loggerMock.warn).toHaveBeenCalledWith(
"[Workflow-engine-redis] The `options` option is deprecated. Please use `redisOptions` instead for consistency with other modules."
)
})
it("should not log warning when using new option names", async () => {
await redisLoader(
{
container: containerMock as any,
logger: loggerMock,
options: {
redis: {
redisUrl: "redis://localhost:6379",
redisOptions: { maxRetriesPerRequest: 3 },
},
},
} as any,
{} as any
)
expect(loggerMock.warn).not.toHaveBeenCalled()
})
})
describe("Queue names", () => {
it("should use default queue names when not provided", async () => {
await redisLoader(
{
container: containerMock as any,
logger: loggerMock,
options: {
redis: {
redisUrl: "redis://localhost:6379",
},
},
} as any,
{} as any
)
const registerCall = containerMock.register.mock.calls[0][0]
expect(registerCall.redisQueueName.resolve()).toEqual("medusa-workflows")
expect(registerCall.redisJobQueueName.resolve()).toEqual(
"medusa-workflows-jobs"
)
})
it("should use custom queue names when provided", async () => {
await redisLoader(
{
container: containerMock as any,
logger: loggerMock,
options: {
redis: {
redisUrl: "redis://localhost:6379",
queueName: "custom-workflows",
jobQueueName: "custom-jobs",
},
},
} as any,
{} as any
)
const registerCall = containerMock.register.mock.calls[0][0]
expect(registerCall.redisQueueName.resolve()).toEqual("custom-workflows")
expect(registerCall.redisJobQueueName.resolve()).toEqual("custom-jobs")
})
})
describe("Error handling", () => {
it("should throw error when redisUrl is not provided", async () => {
await expect(
redisLoader(
{
container: containerMock as any,
logger: loggerMock,
options: {
redis: {},
},
} as any,
{} as any
)
).rejects.toThrow(
"No `redis.redisUrl` (or deprecated `redis.url`) provided"
)
})
})
})

77
packages/modules/workflow-engine-redis/src/loaders/redis.ts
View File

@@ -12,32 +12,86 @@ export default async (
): Promise<void> => {
const {
url,
options: redisOptions,
redisUrl,
options: deprecatedRedisOptions,
redisOptions: newRedisOptions,
jobQueueName,
queueName,
// Shared options
queueOptions,
workerOptions,
// Per-queue options
mainQueueOptions,
mainWorkerOptions,
jobQueueOptions,
jobWorkerOptions,
cleanerQueueOptions,
cleanerWorkerOptions,
pubsub,
} = options?.redis as RedisWorkflowsOptions
// TODO: get default from ENV VAR
if (!url) {
throw Error(
"No `redis.url` provided in `workflowOrchestrator` module options. It is required for the Workflow Orchestrator Redis."
// Handle backward compatibility for deprecated options
const resolvedUrl = redisUrl ?? url
const redisOptions = newRedisOptions ?? deprecatedRedisOptions
// Log deprecation warnings
if (url && !redisUrl) {
logger?.warn(
"[Workflow-engine-redis] The `url` option is deprecated. Please use `redisUrl` instead for consistency with other modules."
)
}
if (deprecatedRedisOptions && !newRedisOptions) {
logger?.warn(
"[Workflow-engine-redis] The `options` option is deprecated. Please use `redisOptions` instead for consistency with other modules."
)
}
const cnnPubSub = pubsub ?? { url, options: redisOptions }
// TODO: get default from ENV VAR
if (!resolvedUrl) {
throw Error(
"No `redis.redisUrl` (or deprecated `redis.url`) provided in `workflowOrchestrator` module options. It is required for the Workflow Orchestrator Redis."
)
}
const cnnPubSub = pubsub ?? { url: resolvedUrl, options: redisOptions }
const queueName_ = queueName ?? "medusa-workflows"
const jobQueueName_ = jobQueueName ?? "medusa-workflows-jobs"
// Resolve per-queue options by merging shared defaults with per-queue overrides
const resolvedMainQueueOptions = {
...(queueOptions ?? {}),
...(mainQueueOptions ?? {}),
}
const resolvedMainWorkerOptions = {
...(workerOptions ?? {}),
...(mainWorkerOptions ?? {}),
}
const resolvedJobQueueOptions = {
...(queueOptions ?? {}),
...(jobQueueOptions ?? {}),
}
const resolvedJobWorkerOptions = {
...(workerOptions ?? {}),
...(jobWorkerOptions ?? {}),
}
const resolvedCleanerQueueOptions = {
...(queueOptions ?? {}),
...(cleanerQueueOptions ?? {}),
}
const resolvedCleanerWorkerOptions = {
...(workerOptions ?? {}),
...(cleanerWorkerOptions ?? {}),
}
let connection
let redisPublisher
let redisSubscriber
let workerConnection
try {
connection = await getConnection(url, redisOptions)
workerConnection = await getConnection(url, {
connection = await getConnection(resolvedUrl, redisOptions)
workerConnection = await getConnection(resolvedUrl, {
...(redisOptions ?? {}),
maxRetriesPerRequest: null,
})
@@ -71,6 +125,13 @@ export default async (
redisSubscriber: asValue(redisSubscriber),
redisQueueName: asValue(queueName_),
redisJobQueueName: asValue(jobQueueName_),
// Per-queue resolved options
redisMainQueueOptions: asValue(resolvedMainQueueOptions),
redisMainWorkerOptions: asValue(resolvedMainWorkerOptions),
redisJobQueueOptions: asValue(resolvedJobQueueOptions),
redisJobWorkerOptions: asValue(resolvedJobWorkerOptions),
redisCleanerQueueOptions: asValue(resolvedCleanerQueueOptions),
redisCleanerWorkerOptions: asValue(resolvedCleanerWorkerOptions),
redisDisconnectHandler: asValue(async () => {
connection.disconnect()
workerConnection.disconnect()