feat(medusa,event-bus-local,event-bus-redis): Event Bus modules (#2599)
This commit is contained in:
Oliver Windall Juhl authored and 
GitHub committed
1 parent
7408111d11
commit
ef5ef9f5a2
114 files changed
+2423
-1669
No files matched your search
@@ -0,0 +1,6 @@
|
||||
/dist
|
||||
node_modules
|
||||
.DS_store
|
||||
.env*
|
||||
.env
|
||||
*.sql
|
||||
@@ -0,0 +1,79 @@
|
||||
<p align="center">
|
||||
<a href="https://www.medusajs.com">
|
||||
<img alt="Medusa" src="https://user-images.githubusercontent.com/7554214/153162406-bf8fd16f-aa98-4604-b87b-e13ab4baf604.png" width="100" />
|
||||
</a>
|
||||
</p>
|
||||
<h1 align="center">
|
||||
@medusajs/event-bus-redis
|
||||
</h1>
|
||||
|
||||
<h4 align="center">
|
||||
<a href="https://docs.medusajs.com">Documentation</a> |
|
||||
<a href="https://www.medusajs.com">Website</a>
|
||||
</h4>
|
||||
|
||||
<p align="center">
|
||||
An open source composable commerce engine built for developers.
|
||||
</p>
|
||||
<p align="center">
|
||||
<a href="https://github.com/medusajs/medusa/blob/master/LICENSE">
|
||||
<img src="https://img.shields.io/badge/license-MIT-blue.svg" alt="Medusa is released under the MIT license." />
|
||||
</a>
|
||||
<a href="https://circleci.com/gh/medusajs/medusa">
|
||||
<img src="https://circleci.com/gh/medusajs/medusa.svg?style=shield" alt="Current CircleCI build status." />
|
||||
</a>
|
||||
<a href="https://github.com/medusajs/medusa/blob/master/CONTRIBUTING.md">
|
||||
<img src="https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat" alt="PRs welcome!" />
|
||||
</a>
|
||||
<a href="https://www.producthunt.com/posts/medusa"><img src="https://img.shields.io/badge/Product%20Hunt-%231%20Product%20of%20the%20Day-%23DA552E" alt="Product Hunt"></a>
|
||||
<a href="https://discord.gg/xpCwq3Kfn8">
|
||||
<img src="https://img.shields.io/badge/chat-on%20discord-7289DA.svg" alt="Discord Chat" />
|
||||
</a>
|
||||
<a href="https://twitter.com/intent/follow?screen_name=medusajs">
|
||||
<img src="https://img.shields.io/twitter/follow/medusajs.svg?label=Follow%20@medusajs" alt="Follow @medusajs" />
|
||||
</a>
|
||||
</p>
|
||||
|
||||
## Overview
|
||||
|
||||
Redis Event Bus module for Medusa. When installed, the events system of Medusa is powered by BullMQ and `io-redis`. BullMQ is responsible for the message queue and worker. `io-redis` is the underlying Redis client, that BullMQ connects to for events storage.
|
||||
|
||||
## Getting started
|
||||
|
||||
Install the module:
|
||||
|
||||
```bash
|
||||
yarn add @medusajs/event-bus-redis
|
||||
```
|
||||
|
||||
Add the module to your `medusa-config.js`:
|
||||
|
||||
```js
|
||||
module.exports = {
|
||||
// ...
|
||||
modules: [
|
||||
{
|
||||
resolve: "@medusajs/event-bus-redis",
|
||||
options: {
|
||||
redisUrl: "redis:.."
|
||||
},
|
||||
},
|
||||
],
|
||||
// ...
|
||||
}
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
The module can be configured with the following options:
|
||||
|
||||
| Option | Type | Description | Default |
|
||||
| --------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- |
|
||||
| `redisUrl` | `string` | URL of the Redis instance to connect to. | `events-worker` |
|
||||
| `queueName` | `string?` | Name of the BullMQ queue. | `events-queue` |
|
||||
| `queueOptions` | `object?` | Options for the BullMQ queue. See BullMQ's [documentation](https://api.docs.bullmq.io/interfaces/QueueOptions.html). | `{}` |
|
||||
| `redisOptions` | `object?` | Options for the Redis instance. See `io-redis`'s [documentation](https://luin.github.io/ioredis/index.html#RedisOptions) | `{}` |
|
||||
|
||||
**Info**: See how the options are applied in the [RedisEventBusService](https://github.com/medusajs/medusa/blob/0c1d1d590463fa30b083c4312293348bdf6596be/packages/event-bus-redis/src/services/event-bus-redis.ts#L52) and [loader](https://github.com/medusajs/medusa/blob/0c1d1d590463fa30b083c4312293348bdf6596be/packages/event-bus-redis/src/loaders/index.ts).
|
||||
|
||||
If you do not provide a `redisUrl` in the module options, the server will fail to start.
|
||||
@@ -0,0 +1,13 @@
|
||||
module.exports = {
|
||||
globals: {
|
||||
"ts-jest": {
|
||||
tsConfig: "tsconfig.spec.json",
|
||||
isolatedModules: false,
|
||||
},
|
||||
},
|
||||
transform: {
|
||||
"^.+\\.[jt]s?$": "ts-jest",
|
||||
},
|
||||
testEnvironment: `node`,
|
||||
moduleFileExtensions: [`js`, `jsx`, `ts`, `tsx`, `json`],
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
{
|
||||
"name": "@medusajs/event-bus-redis",
|
||||
"version": "0.1.0",
|
||||
"description": "Redis Event Bus Module for Medusa",
|
||||
"main": "dist/index.js",
|
||||
"files": [
|
||||
"dist"
|
||||
],
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://github.com/medusajs/medusa",
|
||||
"directory": "packages/event-bus-redis"
|
||||
},
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"author": "Medusa",
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
"@medusajs/types": "^0.0.1",
|
||||
"cross-env": "^5.2.1",
|
||||
"jest": "^25.5.2",
|
||||
"medusa-test-utils": "^1.1.39",
|
||||
"ts-jest": "^25.5.1",
|
||||
"typescript": "^4.4.4"
|
||||
},
|
||||
"scripts": {
|
||||
"watch": "tsc --build --watch",
|
||||
"prepare": "cross-env NODE_ENV=production yarn run build",
|
||||
"build": "tsc --build",
|
||||
"test": "jest --passWithNoTests",
|
||||
"test:unit": "jest --passWithNoTests"
|
||||
},
|
||||
"dependencies": {
|
||||
"@medusajs/modules-sdk": "*",
|
||||
"@medusajs/utils": "^0.0.1",
|
||||
"bullmq": "^3.5.6",
|
||||
"ioredis": "^5.2.5"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
import { ModuleExports } from "@medusajs/modules-sdk"
|
||||
import Loader from "./loaders"
|
||||
import RedisEventBusService from "./services/event-bus-redis"
|
||||
|
||||
const service = RedisEventBusService
|
||||
const loaders = [Loader]
|
||||
|
||||
const moduleDefinition: ModuleExports = {
|
||||
service,
|
||||
loaders,
|
||||
}
|
||||
|
||||
export default moduleDefinition
|
||||
@@ -0,0 +1,41 @@
|
||||
import { LoaderOptions } from "@medusajs/modules-sdk"
|
||||
import { asValue } from "awilix"
|
||||
import Redis from "ioredis"
|
||||
import { EOL } from "os"
|
||||
import { EventBusRedisModuleOptions } from "../types"
|
||||
|
||||
export default async ({
|
||||
container,
|
||||
logger,
|
||||
options,
|
||||
}: LoaderOptions): Promise<void> => {
|
||||
const { redisUrl, redisOptions } = options as EventBusRedisModuleOptions
|
||||
|
||||
if (!redisUrl) {
|
||||
throw Error(
|
||||
"No `redis_url` provided in project config. It is required for the Redis Event Bus."
|
||||
)
|
||||
}
|
||||
|
||||
const connection = new Redis(redisUrl, {
|
||||
// Required config. See: https://github.com/OptimalBits/bull/blob/develop/CHANGELOG.md#breaking-changes
|
||||
maxRetriesPerRequest: null,
|
||||
enableReadyCheck: false,
|
||||
// Lazy connect to properly handle connection errors
|
||||
lazyConnect: true,
|
||||
...(redisOptions ?? {}),
|
||||
})
|
||||
|
||||
try {
|
||||
await connection.connect()
|
||||
logger?.info(`Connection to Redis in module 'event-bus-redis' established`)
|
||||
} catch (err) {
|
||||
logger?.error(
|
||||
`An error occurred while connecting to Redis in module 'event-bus-redis':${EOL} ${err}`
|
||||
)
|
||||
}
|
||||
|
||||
container.register({
|
||||
eventBusRedisConnection: asValue(connection),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,319 @@
|
||||
import { Queue, Worker } from "bullmq"
|
||||
import { MockManager } from "medusa-test-utils"
|
||||
import RedisEventBusService from "../event-bus-redis"
|
||||
|
||||
jest.genMockFromModule("bullmq")
|
||||
jest.genMockFromModule("ioredis")
|
||||
jest.mock("bullmq")
|
||||
jest.mock("ioredis")
|
||||
|
||||
const loggerMock = {
|
||||
info: jest.fn().mockReturnValue(console.log),
|
||||
warn: jest.fn().mockReturnValue(console.log),
|
||||
error: jest.fn().mockReturnValue(console.log),
|
||||
}
|
||||
|
||||
const simpleModuleOptions = { redisUrl: "test-url" }
|
||||
const moduleDeps = {
|
||||
manager: MockManager,
|
||||
logger: loggerMock,
|
||||
eventBusRedisConnection: {},
|
||||
}
|
||||
|
||||
describe("RedisEventBusService", () => {
|
||||
let eventBus
|
||||
|
||||
describe("constructor", () => {
|
||||
beforeAll(() => {
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it("Creates a queue + worker", () => {
|
||||
eventBus = new RedisEventBusService(moduleDeps, simpleModuleOptions, {
|
||||
resources: "shared",
|
||||
})
|
||||
|
||||
expect(Queue).toHaveBeenCalledTimes(1)
|
||||
expect(Queue).toHaveBeenCalledWith("events-queue", {
|
||||
connection: expect.any(Object),
|
||||
prefix: "RedisEventBusService",
|
||||
})
|
||||
|
||||
expect(Worker).toHaveBeenCalledTimes(1)
|
||||
expect(Worker).toHaveBeenCalledWith(
|
||||
"events-queue",
|
||||
expect.any(Function),
|
||||
{
|
||||
connection: expect.any(Object),
|
||||
prefix: "RedisEventBusService",
|
||||
}
|
||||
)
|
||||
})
|
||||
|
||||
it("Throws on isolated module declaration", () => {
|
||||
try {
|
||||
eventBus = new RedisEventBusService(moduleDeps, simpleModuleOptions, {
|
||||
resources: "isolated",
|
||||
})
|
||||
} catch (error) {
|
||||
expect(error.message).toEqual(
|
||||
"At the moment this module can only be used with shared resources"
|
||||
)
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
describe("emit", () => {
|
||||
describe("Successfully emits events", () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks()
|
||||
})
|
||||
|
||||
it("Adds job to queue with default options", () => {
|
||||
eventBus = new RedisEventBusService(moduleDeps, simpleModuleOptions, {
|
||||
resources: "shared",
|
||||
})
|
||||
|
||||
eventBus.queue_.addBulk.mockImplementationOnce(() => "hi")
|
||||
eventBus.emit("eventName", { hi: "1234" })
|
||||
|
||||
expect(eventBus.queue_.addBulk).toHaveBeenCalledTimes(1)
|
||||
expect(eventBus.queue_.addBulk).toHaveBeenCalledWith([
|
||||
{
|
||||
name: "eventName",
|
||||
data: { eventName: "eventName", data: { hi: "1234" } },
|
||||
opts: {
|
||||
attempts: 1,
|
||||
removeOnComplete: true,
|
||||
},
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it("Adds job to queue with custom options passed directly upon emitting", () => {
|
||||
eventBus = new RedisEventBusService(moduleDeps, simpleModuleOptions, {
|
||||
resources: "shared",
|
||||
})
|
||||
|
||||
eventBus.queue_.addBulk.mockImplementationOnce(() => "hi")
|
||||
eventBus.emit(
|
||||
"eventName",
|
||||
{ hi: "1234" },
|
||||
{ attempts: 3, backoff: 5000, delay: 1000 }
|
||||
)
|
||||
|
||||
expect(eventBus.queue_.addBulk).toHaveBeenCalledTimes(1)
|
||||
expect(eventBus.queue_.addBulk).toHaveBeenCalledWith([
|
||||
{
|
||||
name: "eventName",
|
||||
data: { eventName: "eventName", data: { hi: "1234" } },
|
||||
opts: {
|
||||
attempts: 3,
|
||||
backoff: 5000,
|
||||
delay: 1000,
|
||||
removeOnComplete: true,
|
||||
},
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it("Adds job to queue with module job options", () => {
|
||||
eventBus = new RedisEventBusService(
|
||||
moduleDeps,
|
||||
{
|
||||
...simpleModuleOptions,
|
||||
jobOptions: {
|
||||
removeOnComplete: {
|
||||
age: 5,
|
||||
},
|
||||
attempts: 7,
|
||||
},
|
||||
},
|
||||
{
|
||||
resources: "shared",
|
||||
}
|
||||
)
|
||||
|
||||
eventBus.queue_.addBulk.mockImplementationOnce(() => "hi")
|
||||
eventBus.emit("eventName", { hi: "1234" })
|
||||
|
||||
expect(eventBus.queue_.addBulk).toHaveBeenCalledTimes(1)
|
||||
expect(eventBus.queue_.addBulk).toHaveBeenCalledWith([
|
||||
{
|
||||
name: "eventName",
|
||||
data: { eventName: "eventName", data: { hi: "1234" } },
|
||||
opts: {
|
||||
attempts: 7,
|
||||
removeOnComplete: {
|
||||
age: 5,
|
||||
},
|
||||
},
|
||||
},
|
||||
])
|
||||
})
|
||||
|
||||
it("Adds job to queue with default, local, and global options merged", () => {
|
||||
eventBus = new RedisEventBusService(
|
||||
moduleDeps,
|
||||
{
|
||||
...simpleModuleOptions,
|
||||
jobOptions: {
|
||||
removeOnComplete: 5,
|
||||
},
|
||||
},
|
||||
{
|
||||
resources: "shared",
|
||||
}
|
||||
)
|
||||
|
||||
eventBus.queue_.addBulk.mockImplementationOnce(() => "hi")
|
||||
eventBus.emit("eventName", { hi: "1234" }, { delay: 1000 })
|
||||
|
||||
expect(eventBus.queue_.addBulk).toHaveBeenCalledTimes(1)
|
||||
expect(eventBus.queue_.addBulk).toHaveBeenCalledWith([
|
||||
{
|
||||
name: "eventName",
|
||||
data: { eventName: "eventName", data: { hi: "1234" } },
|
||||
opts: {
|
||||
attempts: 1,
|
||||
removeOnComplete: 5,
|
||||
delay: 1000,
|
||||
},
|
||||
},
|
||||
])
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
describe("worker_", () => {
|
||||
let result
|
||||
|
||||
describe("Successfully processes the jobs", () => {
|
||||
beforeEach(async () => {
|
||||
jest.clearAllMocks()
|
||||
|
||||
eventBus = new RedisEventBusService(moduleDeps, simpleModuleOptions, {
|
||||
resources: "shared",
|
||||
})
|
||||
})
|
||||
|
||||
it("Processes a simple event with no options", async () => {
|
||||
eventBus.subscribe("eventName", () => Promise.resolve("hi"))
|
||||
|
||||
result = await eventBus.worker_({
|
||||
data: { eventName: "eventName", data: {} },
|
||||
opts: { attempts: 1 },
|
||||
})
|
||||
|
||||
expect(loggerMock.info).toHaveBeenCalledTimes(1)
|
||||
expect(loggerMock.info).toHaveBeenCalledWith(
|
||||
"Processing eventName which has 1 subscribers"
|
||||
)
|
||||
|
||||
expect(result).toEqual(["hi"])
|
||||
})
|
||||
|
||||
it("Processes event with failing subscribers", async () => {
|
||||
eventBus.subscribe("eventName", () => Promise.resolve("hi"))
|
||||
eventBus.subscribe("eventName", () => Promise.reject("fail1"))
|
||||
eventBus.subscribe("eventName", () => Promise.resolve("hi2"))
|
||||
eventBus.subscribe("eventName", () => Promise.reject("fail2"))
|
||||
|
||||
result = await eventBus.worker_({
|
||||
data: { eventName: "eventName", data: {} },
|
||||
update: (data) => data,
|
||||
opts: { attempts: 1 },
|
||||
})
|
||||
|
||||
expect(loggerMock.info).toHaveBeenCalledTimes(1)
|
||||
expect(loggerMock.info).toHaveBeenCalledWith(
|
||||
"Processing eventName which has 4 subscribers"
|
||||
)
|
||||
|
||||
expect(loggerMock.warn).toHaveBeenCalledTimes(3)
|
||||
expect(loggerMock.warn).toHaveBeenCalledWith(
|
||||
"An error occurred while processing eventName: fail1"
|
||||
)
|
||||
expect(loggerMock.warn).toHaveBeenCalledWith(
|
||||
"An error occurred while processing eventName: fail2"
|
||||
)
|
||||
|
||||
expect(loggerMock.warn).toHaveBeenCalledWith(
|
||||
"One or more subscribers of eventName failed. Retrying is not configured. Use 'attempts' option when emitting events."
|
||||
)
|
||||
|
||||
expect(result).toEqual(["hi", "fail1", "hi2", "fail2"])
|
||||
})
|
||||
|
||||
it("Retries processing when subcribers fail, if configured - final attempt", async () => {
|
||||
eventBus.subscribe("eventName", async () => Promise.resolve("hi"), {
|
||||
subscriberId: "1",
|
||||
})
|
||||
eventBus.subscribe("eventName", async () => Promise.reject("fail1"), {
|
||||
subscriberId: "2",
|
||||
})
|
||||
|
||||
result = await eventBus
|
||||
.worker_({
|
||||
data: {
|
||||
eventName: "eventName",
|
||||
data: {},
|
||||
completedSubscriberIds: ["1"],
|
||||
},
|
||||
attemptsMade: 2,
|
||||
update: (data) => data,
|
||||
opts: { attempts: 2 },
|
||||
})
|
||||
.catch((error) => void 0)
|
||||
|
||||
expect(loggerMock.warn).toHaveBeenCalledTimes(1)
|
||||
expect(loggerMock.warn).toHaveBeenCalledWith(
|
||||
"An error occurred while processing eventName: fail1"
|
||||
)
|
||||
|
||||
expect(loggerMock.info).toHaveBeenCalledTimes(2)
|
||||
expect(loggerMock.info).toHaveBeenCalledWith(
|
||||
"Final retry attempt for eventName"
|
||||
)
|
||||
expect(loggerMock.info).toHaveBeenCalledWith(
|
||||
"Retrying eventName which has 2 subscribers (1 of them failed)"
|
||||
)
|
||||
})
|
||||
|
||||
it("Retries processing when subcribers fail, if configured", async () => {
|
||||
eventBus.subscribe("eventName", async () => Promise.resolve("hi"), {
|
||||
subscriberId: "1",
|
||||
})
|
||||
eventBus.subscribe("eventName", async () => Promise.reject("fail1"), {
|
||||
subscriberId: "2",
|
||||
})
|
||||
|
||||
result = await eventBus
|
||||
.worker_({
|
||||
data: {
|
||||
eventName: "eventName",
|
||||
data: {},
|
||||
completedSubscriberIds: ["1"],
|
||||
},
|
||||
attemptsMade: 2,
|
||||
update: (data) => data,
|
||||
opts: { attempts: 3 },
|
||||
})
|
||||
.catch((err) => void 0)
|
||||
|
||||
expect(loggerMock.warn).toHaveBeenCalledTimes(2)
|
||||
expect(loggerMock.warn).toHaveBeenCalledWith(
|
||||
"An error occurred while processing eventName: fail1"
|
||||
)
|
||||
expect(loggerMock.warn).toHaveBeenCalledWith(
|
||||
"One or more subscribers of eventName failed. Retrying..."
|
||||
)
|
||||
|
||||
expect(loggerMock.info).toHaveBeenCalledTimes(1)
|
||||
expect(loggerMock.info).toHaveBeenCalledWith(
|
||||
"Retrying eventName which has 2 subscribers (1 of them failed)"
|
||||
)
|
||||
})
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,209 @@
|
||||
import { InternalModuleDeclaration, Logger } from "@medusajs/modules-sdk"
|
||||
import { ConfigModule, EmitData } from "@medusajs/types"
|
||||
import { AbstractEventBusModuleService } from "@medusajs/utils"
|
||||
import { BulkJobOptions, JobsOptions, Queue, Worker } from "bullmq"
|
||||
import { Redis } from "ioredis"
|
||||
import { BullJob, EmitOptions, EventBusRedisModuleOptions } from "../types"
|
||||
|
||||
type InjectedDependencies = {
|
||||
logger: Logger
|
||||
configModule: ConfigModule
|
||||
eventBusRedisConnection: Redis
|
||||
}
|
||||
|
||||
/**
|
||||
* Can keep track of multiple subscribers to different events and run the
|
||||
* subscribers when events happen. Events will run asynchronously.
|
||||
*/
|
||||
export default class RedisEventBusService extends AbstractEventBusModuleService {
|
||||
protected readonly config_: ConfigModule
|
||||
protected readonly logger_: Logger
|
||||
protected readonly moduleOptions_: EventBusRedisModuleOptions
|
||||
protected readonly moduleDeclaration_: InternalModuleDeclaration
|
||||
|
||||
protected queue_: Queue
|
||||
|
||||
constructor(
|
||||
{ configModule, logger, eventBusRedisConnection }: InjectedDependencies,
|
||||
moduleOptions: EventBusRedisModuleOptions = {},
|
||||
moduleDeclaration: InternalModuleDeclaration
|
||||
) {
|
||||
// @ts-ignore
|
||||
super(...arguments)
|
||||
|
||||
this.moduleOptions_ = moduleOptions
|
||||
this.config_ = configModule
|
||||
this.logger_ = logger
|
||||
|
||||
this.queue_ = new Queue(moduleOptions.queueName ?? `events-queue`, {
|
||||
prefix: `${this.constructor.name}`,
|
||||
...(moduleOptions.queueOptions ?? {}),
|
||||
connection: eventBusRedisConnection,
|
||||
})
|
||||
|
||||
// Register our worker to handle emit calls
|
||||
new Worker(moduleOptions.queueName ?? "events-queue", this.worker_, {
|
||||
prefix: `${this.constructor.name}`,
|
||||
...(moduleOptions.workerOptions ?? {}),
|
||||
connection: eventBusRedisConnection,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* Emit a single event
|
||||
* @param {string} eventName - the name of the event to be process.
|
||||
* @param data - the data to send to the subscriber.
|
||||
* @param options - options to add the job with
|
||||
*/
|
||||
async emit<T>(
|
||||
eventName: string,
|
||||
data: T,
|
||||
options: Record<string, unknown>
|
||||
): Promise<void>
|
||||
|
||||
/**
|
||||
* Emit a number of events
|
||||
* @param {EmitData} data - the data to send to the subscriber.
|
||||
*/
|
||||
async emit<T>(data: EmitData<T>[]): Promise<void>
|
||||
|
||||
async emit<T, TInput extends string | EmitData<T>[] = string>(
|
||||
eventNameOrData: TInput,
|
||||
data?: T,
|
||||
options: BulkJobOptions | JobsOptions = {}
|
||||
): Promise<void> {
|
||||
const globalJobOptions = this.moduleOptions_.jobOptions ?? {}
|
||||
|
||||
const isBulkEmit = Array.isArray(eventNameOrData)
|
||||
|
||||
const opts = {
|
||||
// default options
|
||||
removeOnComplete: true,
|
||||
attempts: 1,
|
||||
// global options
|
||||
...globalJobOptions,
|
||||
} as EmitOptions
|
||||
|
||||
const events = isBulkEmit
|
||||
? eventNameOrData.map((event) => ({
|
||||
name: event.eventName,
|
||||
data: { eventName: event.eventName, data: event.data },
|
||||
opts: {
|
||||
...opts,
|
||||
// local options
|
||||
...event.options,
|
||||
},
|
||||
}))
|
||||
: [
|
||||
{
|
||||
name: eventNameOrData as string,
|
||||
data: { eventName: eventNameOrData, data },
|
||||
opts: {
|
||||
...opts,
|
||||
// local options
|
||||
...options,
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
await this.queue_.addBulk(events)
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles incoming jobs.
|
||||
* @param job The job object
|
||||
* @return resolves to the results of the subscriber calls.
|
||||
*/
|
||||
worker_ = async <T>(job: BullJob<T>): Promise<unknown> => {
|
||||
const { eventName, data } = job.data
|
||||
const eventSubscribers = this.eventToSubscribersMap.get(eventName) || []
|
||||
const wildcardSubscribers = this.eventToSubscribersMap.get("*") || []
|
||||
|
||||
const allSubscribers = eventSubscribers.concat(wildcardSubscribers)
|
||||
|
||||
// Pull already completed subscribers from the job data
|
||||
const completedSubscribers = job.data.completedSubscriberIds || []
|
||||
|
||||
// Filter out already completed subscribers from the all subscribers
|
||||
const subscribersInCurrentAttempt = allSubscribers.filter(
|
||||
(subscriber) =>
|
||||
subscriber.id && !completedSubscribers.includes(subscriber.id)
|
||||
)
|
||||
|
||||
const currentAttempt = job.attemptsMade
|
||||
const isRetry = currentAttempt > 1
|
||||
const configuredAttempts = job.opts.attempts
|
||||
|
||||
const isFinalAttempt = currentAttempt === configuredAttempts
|
||||
|
||||
if (isRetry) {
|
||||
if (isFinalAttempt) {
|
||||
this.logger_.info(`Final retry attempt for ${eventName}`)
|
||||
}
|
||||
|
||||
this.logger_.info(
|
||||
`Retrying ${eventName} which has ${eventSubscribers.length} subscribers (${subscribersInCurrentAttempt.length} of them failed)`
|
||||
)
|
||||
} else {
|
||||
this.logger_.info(
|
||||
`Processing ${eventName} which has ${eventSubscribers.length} subscribers`
|
||||
)
|
||||
}
|
||||
|
||||
const completedSubscribersInCurrentAttempt: string[] = []
|
||||
|
||||
const subscribersResult = await Promise.all(
|
||||
subscribersInCurrentAttempt.map(async ({ id, subscriber }) => {
|
||||
return await subscriber(data, eventName)
|
||||
.then(async (data) => {
|
||||
// For every subscriber that completes successfully, add their id to the list of completed subscribers
|
||||
completedSubscribersInCurrentAttempt.push(id)
|
||||
return data
|
||||
})
|
||||
.catch((err) => {
|
||||
this.logger_.warn(
|
||||
`An error occurred while processing ${eventName}: ${err}`
|
||||
)
|
||||
return err
|
||||
})
|
||||
})
|
||||
)
|
||||
|
||||
// If the number of completed subscribers is different from the number of subcribers to process in current attempt, some of them failed
|
||||
const didSubscribersFail =
|
||||
completedSubscribersInCurrentAttempt.length !==
|
||||
subscribersInCurrentAttempt.length
|
||||
|
||||
const isRetriesConfigured = configuredAttempts! > 1
|
||||
|
||||
// Therefore, if retrying is configured, we try again
|
||||
const shouldRetry =
|
||||
didSubscribersFail && isRetriesConfigured && !isFinalAttempt
|
||||
|
||||
if (shouldRetry) {
|
||||
const updatedCompletedSubscribers = [
|
||||
...completedSubscribers,
|
||||
...completedSubscribersInCurrentAttempt,
|
||||
]
|
||||
|
||||
job.data.completedSubscriberIds = updatedCompletedSubscribers
|
||||
|
||||
await job.update(job.data)
|
||||
|
||||
const errorMessage = `One or more subscribers of ${eventName} failed. Retrying...`
|
||||
|
||||
this.logger_.warn(errorMessage)
|
||||
|
||||
return Promise.reject(Error(errorMessage))
|
||||
}
|
||||
|
||||
if (didSubscribersFail && !isFinalAttempt) {
|
||||
// If retrying is not configured, we log a warning to allow server admins to recover manually
|
||||
this.logger_.warn(
|
||||
`One or more subscribers of ${eventName} failed. Retrying is not configured. Use 'attempts' option when emitting events.`
|
||||
)
|
||||
}
|
||||
|
||||
return Promise.resolve(subscribersResult)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
import { Job, JobsOptions, QueueOptions, WorkerOptions } from "bullmq"
|
||||
import { RedisOptions } from "ioredis"
|
||||
|
||||
export type JobData<T> = {
|
||||
eventName: string
|
||||
data: T
|
||||
completedSubscriberIds?: string[] | undefined
|
||||
}
|
||||
|
||||
export type BullJob<T> = {
|
||||
data: JobData<T>
|
||||
} & Job
|
||||
|
||||
export type EmitOptions = JobsOptions
|
||||
|
||||
export type EventBusRedisModuleOptions = {
|
||||
queueName?: string
|
||||
queueOptions?: QueueOptions
|
||||
|
||||
workerOptions?: WorkerOptions
|
||||
|
||||
redisUrl?: string
|
||||
redisOptions?: RedisOptions
|
||||
|
||||
/**
|
||||
* Global options passed to all `EventBusService.emit` in the core as well as your own emitters. The options are forwarded to Bull's `Queue.add` method.
|
||||
*
|
||||
* The global options can be overridden by passing options to `EventBusService.emit` directly.
|
||||
*
|
||||
* Example
|
||||
* ```js
|
||||
* {
|
||||
* removeOnComplete: { age: 10 },
|
||||
* }
|
||||
* ```
|
||||
*
|
||||
* @see https://api.docs.bullmq.io/interfaces/BaseJobOptions.html
|
||||
*/
|
||||
jobOptions?: EmitOptions
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"lib": ["es2020"],
|
||||
"target": "es2020",
|
||||
"outDir": "./dist",
|
||||
"esModuleInterop": true,
|
||||
"declaration": true,
|
||||
"module": "commonjs",
|
||||
"moduleResolution": "node",
|
||||
"emitDecoratorMetadata": true,
|
||||
"experimentalDecorators": true,
|
||||
"sourceMap": true,
|
||||
"noImplicitReturns": true,
|
||||
"strictNullChecks": true,
|
||||
"strictFunctionTypes": true,
|
||||
"noImplicitThis": true,
|
||||
"allowJs": true,
|
||||
"skipLibCheck": true,
|
||||
"downlevelIteration": true // to use ES5 specific tooling
|
||||
},
|
||||
"include": ["src"],
|
||||
"exclude": [
|
||||
"dist",
|
||||
"./src/**/__tests__",
|
||||
"./src/**/__mocks__",
|
||||
"./src/**/__fixtures__",
|
||||
"node_modules"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
{
|
||||
"extends": "./tsconfig.json",
|
||||
"include": ["src"],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
Reference in new issue
Block a user