chore(): Reorganize modules (#7210)

**What**
Move all modules to the modules directory

packages/modules/event-bus-redis/src/index.ts

@@ -0,0 +1,15 @@
+import { ModuleExports } from "@medusajs/modules-sdk"
+import Loader from "./loaders"
+import RedisEventBusService from "./services/event-bus-redis"
+
+const service = RedisEventBusService
+const loaders = [Loader]
+
+const moduleDefinition: ModuleExports = {
+  service,
+  loaders,
+}
+
+export default moduleDefinition
+export * from "./initialize"
+export * from "./types"

packages/modules/event-bus-redis/src/initialize/index.ts

@@ -0,0 +1,23 @@
+import {
+  ExternalModuleDeclaration,
+  InternalModuleDeclaration,
+  MedusaModule,
+  Modules,
+} from "@medusajs/modules-sdk"
+import { IEventBusService } from "@medusajs/types"
+import { EventBusRedisModuleOptions } from "../types"
+
+export const initialize = async (
+  options?: EventBusRedisModuleOptions | ExternalModuleDeclaration
+): Promise<IEventBusService> => {
+  const serviceKey = Modules.EVENT_BUS
+  const loaded = await MedusaModule.bootstrap<IEventBusService>({
+    moduleKey: serviceKey,
+    defaultPath: "@medusajs/event-bus-redis",
+    declaration: options as
+      | InternalModuleDeclaration
+      | ExternalModuleDeclaration,
+  })
+
+  return loaded[serviceKey]
+}

packages/modules/event-bus-redis/src/loaders/index.ts

@@ -0,0 +1,43 @@
+import { LoaderOptions } from "@medusajs/modules-sdk"
+import { asValue } from "awilix"
+import Redis from "ioredis"
+import { EOL } from "os"
+import { EventBusRedisModuleOptions } from "../types"
+
+export default async ({
+  container,
+  logger,
+  options,
+}: LoaderOptions): Promise<void> => {
+  const { redisUrl, redisOptions } = options as EventBusRedisModuleOptions
+
+  if (!redisUrl) {
+    throw Error(
+      "No `redis_url` provided in project config. It is required for the Redis Event Bus."
+    )
+  }
+
+  const connection = new Redis(redisUrl, {
+    // Required config. See: https://github.com/OptimalBits/bull/blob/develop/CHANGELOG.md#breaking-changes
+    maxRetriesPerRequest: null,
+    enableReadyCheck: false,
+    // Lazy connect to properly handle connection errors
+    lazyConnect: true,
+    ...(redisOptions ?? {}),
+  })
+
+  try {
+    await new Promise(async resolve => {
+      await connection.connect(resolve)
+    })
+    logger?.info(`Connection to Redis in module 'event-bus-redis' established`)
+  } catch (err) {
+    logger?.error(
+      `An error occurred while connecting to Redis in module 'event-bus-redis':${EOL} ${err}`
+    )
+  }
+
+  container.register({
+    eventBusRedisConnection: asValue(connection),
+  })
+}

packages/modules/event-bus-redis/src/services/__tests__/event-bus.js

@@ -0,0 +1,319 @@
+import { Queue, Worker } from "bullmq"
+import { MockManager } from "medusa-test-utils"
+import RedisEventBusService from "../event-bus-redis"
+
+jest.genMockFromModule("bullmq")
+jest.genMockFromModule("ioredis")
+jest.mock("bullmq")
+jest.mock("ioredis")
+
+const loggerMock = {
+  info: jest.fn().mockReturnValue(console.log),
+  warn: jest.fn().mockReturnValue(console.log),
+  error: jest.fn().mockReturnValue(console.log),
+}
+
+const simpleModuleOptions = { redisUrl: "test-url" }
+const moduleDeps = {
+  manager: MockManager,
+  logger: loggerMock,
+  eventBusRedisConnection: {},
+}
+
+describe("RedisEventBusService", () => {
+  let eventBus
+
+  describe("constructor", () => {
+    beforeAll(() => {
+      jest.clearAllMocks()
+    })
+
+    it("Creates a queue + worker", () => {
+      eventBus = new RedisEventBusService(moduleDeps, simpleModuleOptions, {
+        resources: "shared",
+      })
+
+      expect(Queue).toHaveBeenCalledTimes(1)
+      expect(Queue).toHaveBeenCalledWith("events-queue", {
+        connection: expect.any(Object),
+        prefix: "RedisEventBusService",
+      })
+
+      expect(Worker).toHaveBeenCalledTimes(1)
+      expect(Worker).toHaveBeenCalledWith(
+        "events-queue",
+        expect.any(Function),
+        {
+          connection: expect.any(Object),
+          prefix: "RedisEventBusService",
+        }
+      )
+    })
+
+    it("Throws on isolated module declaration", () => {
+      try {
+        eventBus = new RedisEventBusService(moduleDeps, simpleModuleOptions, {
+          resources: "isolated",
+        })
+      } catch (error) {
+        expect(error.message).toEqual(
+          "At the moment this module can only be used with shared resources"
+        )
+      }
+    })
+  })
+
+  describe("emit", () => {
+    describe("Successfully emits events", () => {
+      beforeEach(() => {
+        jest.clearAllMocks()
+      })
+
+      it("Adds job to queue with default options", () => {
+        eventBus = new RedisEventBusService(moduleDeps, simpleModuleOptions, {
+          resources: "shared",
+        })
+
+        eventBus.queue_.addBulk.mockImplementationOnce(() => "hi")
+        eventBus.emit("eventName", { hi: "1234" })
+
+        expect(eventBus.queue_.addBulk).toHaveBeenCalledTimes(1)
+        expect(eventBus.queue_.addBulk).toHaveBeenCalledWith([
+          {
+            name: "eventName",
+            data: { eventName: "eventName", data: { hi: "1234" } },
+            opts: {
+              attempts: 1,
+              removeOnComplete: true,
+            },
+          },
+        ])
+      })
+
+      it("Adds job to queue with custom options passed directly upon emitting", () => {
+        eventBus = new RedisEventBusService(moduleDeps, simpleModuleOptions, {
+          resources: "shared",
+        })
+
+        eventBus.queue_.addBulk.mockImplementationOnce(() => "hi")
+        eventBus.emit(
+          "eventName",
+          { hi: "1234" },
+          { attempts: 3, backoff: 5000, delay: 1000 }
+        )
+
+        expect(eventBus.queue_.addBulk).toHaveBeenCalledTimes(1)
+        expect(eventBus.queue_.addBulk).toHaveBeenCalledWith([
+          {
+            name: "eventName",
+            data: { eventName: "eventName", data: { hi: "1234" } },
+            opts: {
+              attempts: 3,
+              backoff: 5000,
+              delay: 1000,
+              removeOnComplete: true,
+            },
+          },
+        ])
+      })
+
+      it("Adds job to queue with module job options", () => {
+        eventBus = new RedisEventBusService(
+          moduleDeps,
+          {
+            ...simpleModuleOptions,
+            jobOptions: {
+              removeOnComplete: {
+                age: 5,
+              },
+              attempts: 7,
+            },
+          },
+          {
+            resources: "shared",
+          }
+        )
+
+        eventBus.queue_.addBulk.mockImplementationOnce(() => "hi")
+        eventBus.emit("eventName", { hi: "1234" })
+
+        expect(eventBus.queue_.addBulk).toHaveBeenCalledTimes(1)
+        expect(eventBus.queue_.addBulk).toHaveBeenCalledWith([
+          {
+            name: "eventName",
+            data: { eventName: "eventName", data: { hi: "1234" } },
+            opts: {
+              attempts: 7,
+              removeOnComplete: {
+                age: 5,
+              },
+            },
+          },
+        ])
+      })
+
+      it("Adds job to queue with default, local, and global options merged", () => {
+        eventBus = new RedisEventBusService(
+          moduleDeps,
+          {
+            ...simpleModuleOptions,
+            jobOptions: {
+              removeOnComplete: 5,
+            },
+          },
+          {
+            resources: "shared",
+          }
+        )
+
+        eventBus.queue_.addBulk.mockImplementationOnce(() => "hi")
+        eventBus.emit("eventName", { hi: "1234" }, { delay: 1000 })
+
+        expect(eventBus.queue_.addBulk).toHaveBeenCalledTimes(1)
+        expect(eventBus.queue_.addBulk).toHaveBeenCalledWith([
+          {
+            name: "eventName",
+            data: { eventName: "eventName", data: { hi: "1234" } },
+            opts: {
+              attempts: 1,
+              removeOnComplete: 5,
+              delay: 1000,
+            },
+          },
+        ])
+      })
+    })
+  })
+
+  describe("worker_", () => {
+    let result
+
+    describe("Successfully processes the jobs", () => {
+      beforeEach(async () => {
+        jest.clearAllMocks()
+
+        eventBus = new RedisEventBusService(moduleDeps, simpleModuleOptions, {
+          resources: "shared",
+        })
+      })
+
+      it("Processes a simple event with no options", async () => {
+        eventBus.subscribe("eventName", () => Promise.resolve("hi"))
+
+        result = await eventBus.worker_({
+          data: { eventName: "eventName", data: {} },
+          opts: { attempts: 1 },
+        })
+
+        expect(loggerMock.info).toHaveBeenCalledTimes(1)
+        expect(loggerMock.info).toHaveBeenCalledWith(
+          "Processing eventName which has 1 subscribers"
+        )
+
+        expect(result).toEqual(["hi"])
+      })
+
+      it("Processes event with failing subscribers", async () => {
+        eventBus.subscribe("eventName", () => Promise.resolve("hi"))
+        eventBus.subscribe("eventName", () => Promise.reject("fail1"))
+        eventBus.subscribe("eventName", () => Promise.resolve("hi2"))
+        eventBus.subscribe("eventName", () => Promise.reject("fail2"))
+
+        result = await eventBus.worker_({
+          data: { eventName: "eventName", data: {} },
+          update: (data) => data,
+          opts: { attempts: 1 },
+        })
+
+        expect(loggerMock.info).toHaveBeenCalledTimes(1)
+        expect(loggerMock.info).toHaveBeenCalledWith(
+          "Processing eventName which has 4 subscribers"
+        )
+
+        expect(loggerMock.warn).toHaveBeenCalledTimes(3)
+        expect(loggerMock.warn).toHaveBeenCalledWith(
+          "An error occurred while processing eventName: fail1"
+        )
+        expect(loggerMock.warn).toHaveBeenCalledWith(
+          "An error occurred while processing eventName: fail2"
+        )
+
+        expect(loggerMock.warn).toHaveBeenCalledWith(
+          "One or more subscribers of eventName failed. Retrying is not configured. Use 'attempts' option when emitting events."
+        )
+
+        expect(result).toEqual(["hi", "fail1", "hi2", "fail2"])
+      })
+
+      it("Retries processing when subcribers fail, if configured - final attempt", async () => {
+        eventBus.subscribe("eventName", async () => Promise.resolve("hi"), {
+          subscriberId: "1",
+        })
+        eventBus.subscribe("eventName", async () => Promise.reject("fail1"), {
+          subscriberId: "2",
+        })
+
+        result = await eventBus
+          .worker_({
+            data: {
+              eventName: "eventName",
+              data: {},
+              completedSubscriberIds: ["1"],
+            },
+            attemptsMade: 2,
+            update: (data) => data,
+            opts: { attempts: 2 },
+          })
+          .catch((error) => void 0)
+
+        expect(loggerMock.warn).toHaveBeenCalledTimes(1)
+        expect(loggerMock.warn).toHaveBeenCalledWith(
+          "An error occurred while processing eventName: fail1"
+        )
+
+        expect(loggerMock.info).toHaveBeenCalledTimes(2)
+        expect(loggerMock.info).toHaveBeenCalledWith(
+          "Final retry attempt for eventName"
+        )
+        expect(loggerMock.info).toHaveBeenCalledWith(
+          "Retrying eventName which has 2 subscribers (1 of them failed)"
+        )
+      })
+
+      it("Retries processing when subcribers fail, if configured", async () => {
+        eventBus.subscribe("eventName", async () => Promise.resolve("hi"), {
+          subscriberId: "1",
+        })
+        eventBus.subscribe("eventName", async () => Promise.reject("fail1"), {
+          subscriberId: "2",
+        })
+
+        result = await eventBus
+          .worker_({
+            data: {
+              eventName: "eventName",
+              data: {},
+              completedSubscriberIds: ["1"],
+            },
+            attemptsMade: 2,
+            updateData: (data) => data,
+            opts: { attempts: 3 },
+          })
+          .catch((err) => void 0)
+
+        expect(loggerMock.warn).toHaveBeenCalledTimes(2)
+        expect(loggerMock.warn).toHaveBeenCalledWith(
+          "An error occurred while processing eventName: fail1"
+        )
+        expect(loggerMock.warn).toHaveBeenCalledWith(
+          "One or more subscribers of eventName failed. Retrying..."
+        )
+
+        expect(loggerMock.info).toHaveBeenCalledTimes(1)
+        expect(loggerMock.info).toHaveBeenCalledWith(
+          "Retrying eventName which has 2 subscribers (1 of them failed)"
+        )
+      })
+    })
+  })
+})

packages/modules/event-bus-redis/src/services/event-bus-redis.ts

@@ -0,0 +1,240 @@
+import { InternalModuleDeclaration } from "@medusajs/modules-sdk"
+import { EmitData, Logger, Message } from "@medusajs/types"
+import { AbstractEventBusModuleService, isString } from "@medusajs/utils"
+import { BulkJobOptions, JobsOptions, Queue, Worker } from "bullmq"
+import { Redis } from "ioredis"
+import { BullJob, EmitOptions, EventBusRedisModuleOptions } from "../types"
+
+type InjectedDependencies = {
+  logger: Logger
+  eventBusRedisConnection: Redis
+}
+
+/**
+ * Can keep track of multiple subscribers to different events and run the
+ * subscribers when events happen. Events will run asynchronously.
+ */
+// eslint-disable-next-line max-len
+export default class RedisEventBusService extends AbstractEventBusModuleService {
+  protected readonly logger_: Logger
+  protected readonly moduleOptions_: EventBusRedisModuleOptions
+  // eslint-disable-next-line max-len
+  protected readonly moduleDeclaration_: InternalModuleDeclaration
+  protected readonly eventBusRedisConnection_: Redis
+
+  protected queue_: Queue
+  protected bullWorker_: Worker
+
+  constructor(
+    { logger, eventBusRedisConnection }: InjectedDependencies,
+    moduleOptions: EventBusRedisModuleOptions = {},
+    moduleDeclaration: InternalModuleDeclaration
+  ) {
+    // @ts-ignore
+    // eslint-disable-next-line prefer-rest-params
+    super(...arguments)
+
+    this.eventBusRedisConnection_ = eventBusRedisConnection
+
+    this.moduleOptions_ = moduleOptions
+    this.logger_ = logger
+
+    this.queue_ = new Queue(moduleOptions.queueName ?? `events-queue`, {
+      prefix: `${this.constructor.name}`,
+      ...(moduleOptions.queueOptions ?? {}),
+      connection: eventBusRedisConnection,
+    })
+
+    // Register our worker to handle emit calls
+    const shouldStartWorker = moduleDeclaration.worker_mode !== "server"
+    if (shouldStartWorker) {
+      this.bullWorker_ = new Worker(
+        moduleOptions.queueName ?? "events-queue",
+        this.worker_,
+        {
+          prefix: `${this.constructor.name}`,
+          ...(moduleOptions.workerOptions ?? {}),
+          connection: eventBusRedisConnection,
+        }
+      )
+    }
+  }
+
+  __hooks = {
+    onApplicationShutdown: async () => {
+      await this.queue_.close()
+      // eslint-disable-next-line max-len
+      this.eventBusRedisConnection_.disconnect()
+    },
+    onApplicationPrepareShutdown: async () => {
+      await this.bullWorker_?.close()
+    },
+  }
+
+  /**
+   * Emit a single event
+   * @param {string} eventName - the name of the event to be process.
+   * @param data - the data to send to the subscriber.
+   * @param options - options to add the job with
+   */
+  async emit<T>(
+    eventName: string,
+    data: T,
+    options: Record<string, unknown>
+  ): Promise<void>
+
+  /**
+   * Emit a number of events
+   * @param {EmitData} data - the data to send to the subscriber.
+   */
+  async emit<T>(data: EmitData<T>[]): Promise<void>
+
+  async emit<T>(data: Message<T>[]): Promise<void>
+
+  async emit<T, TInput extends string | EmitData<T>[] | Message<T>[] = string>(
+    eventNameOrData: TInput,
+    data?: T,
+    options: BulkJobOptions | JobsOptions = {}
+  ): Promise<void> {
+    const globalJobOptions = this.moduleOptions_.jobOptions ?? {}
+
+    const isBulkEmit = Array.isArray(eventNameOrData)
+
+    const opts = {
+      // default options
+      removeOnComplete: true,
+      attempts: 1,
+      // global options
+      ...globalJobOptions,
+    } as EmitOptions
+
+    const dataBody = isString(eventNameOrData)
+      ? data ?? (data as Message<T>).body
+      : undefined
+
+    const events = isBulkEmit
+      ? eventNameOrData.map((event) => ({
+          name: event.eventName,
+          data: {
+            eventName: event.eventName,
+            data: (event as EmitData).data ?? (event as Message<T>).body,
+          },
+          opts: {
+            ...opts,
+            // local options
+            ...event.options,
+          },
+        }))
+      : [
+          {
+            name: eventNameOrData as string,
+            data: { eventName: eventNameOrData, data: dataBody },
+            opts: {
+              ...opts,
+              // local options
+              ...options,
+            },
+          },
+        ]
+
+    await this.queue_.addBulk(events)
+  }
+
+  /**
+   * Handles incoming jobs.
+   * @param job The job object
+   * @return resolves to the results of the subscriber calls.
+   */
+  worker_ = async <T>(job: BullJob<T>): Promise<unknown> => {
+    const { eventName, data } = job.data
+    const eventSubscribers = this.eventToSubscribersMap.get(eventName) || []
+    const wildcardSubscribers = this.eventToSubscribersMap.get("*") || []
+
+    const allSubscribers = eventSubscribers.concat(wildcardSubscribers)
+
+    // Pull already completed subscribers from the job data
+    const completedSubscribers = job.data.completedSubscriberIds || []
+
+    // Filter out already completed subscribers from the all subscribers
+    const subscribersInCurrentAttempt = allSubscribers.filter(
+      (subscriber) =>
+        subscriber.id && !completedSubscribers.includes(subscriber.id)
+    )
+
+    const currentAttempt = job.attemptsMade
+    const isRetry = currentAttempt > 1
+    const configuredAttempts = job.opts.attempts
+
+    const isFinalAttempt = currentAttempt === configuredAttempts
+
+    if (isRetry) {
+      if (isFinalAttempt) {
+        this.logger_.info(`Final retry attempt for ${eventName}`)
+      }
+
+      this.logger_.info(
+        `Retrying ${eventName} which has ${eventSubscribers.length} subscribers (${subscribersInCurrentAttempt.length} of them failed)`
+      )
+    } else {
+      this.logger_.info(
+        `Processing ${eventName} which has ${eventSubscribers.length} subscribers`
+      )
+    }
+
+    const completedSubscribersInCurrentAttempt: string[] = []
+
+    const subscribersResult = await Promise.all(
+      subscribersInCurrentAttempt.map(async ({ id, subscriber }) => {
+        return await subscriber(data, eventName)
+          .then(async (data) => {
+            // For every subscriber that completes successfully, add their id to the list of completed subscribers
+            completedSubscribersInCurrentAttempt.push(id)
+            return data
+          })
+          .catch((err) => {
+            this.logger_.warn(
+              `An error occurred while processing ${eventName}: ${err}`
+            )
+            return err
+          })
+      })
+    )
+
+    // If the number of completed subscribers is different from the number of subcribers to process in current attempt, some of them failed
+    const didSubscribersFail =
+      completedSubscribersInCurrentAttempt.length !==
+      subscribersInCurrentAttempt.length
+
+    const isRetriesConfigured = configuredAttempts! > 1
+
+    // Therefore, if retrying is configured, we try again
+    const shouldRetry =
+      didSubscribersFail && isRetriesConfigured && !isFinalAttempt
+
+    if (shouldRetry) {
+      const updatedCompletedSubscribers = [
+        ...completedSubscribers,
+        ...completedSubscribersInCurrentAttempt,
+      ]
+
+      job.data.completedSubscriberIds = updatedCompletedSubscribers
+
+      await job.updateData(job.data)
+
+      const errorMessage = `One or more subscribers of ${eventName} failed. Retrying...`
+
+      this.logger_.warn(errorMessage)
+
+      return Promise.reject(Error(errorMessage))
+    }
+
+    if (didSubscribersFail && !isFinalAttempt) {
+      // If retrying is not configured, we log a warning to allow server admins to recover manually
+      this.logger_.warn(
+        `One or more subscribers of ${eventName} failed. Retrying is not configured. Use 'attempts' option when emitting events.`
+      )
+    }
+
+    return Promise.resolve(subscribersResult)
+  }
+}

packages/modules/event-bus-redis/src/types/index.ts

@@ -0,0 +1,40 @@
+import { Job, JobsOptions, QueueOptions, WorkerOptions } from "bullmq"
+import { RedisOptions } from "ioredis"
+
+export type JobData<T> = {
+  eventName: string
+  data: T
+  completedSubscriberIds?: string[] | undefined
+}
+
+export type BullJob<T> = {
+  data: JobData<T>
+} & Job
+
+export type EmitOptions = JobsOptions
+
+export type EventBusRedisModuleOptions = {
+  queueName?: string
+  queueOptions?: QueueOptions
+
+  workerOptions?: WorkerOptions
+
+  redisUrl?: string
+  redisOptions?: RedisOptions
+
+  /**
+   * Global options passed to all `EventBusService.emit` in the core as well as your own emitters. The options are forwarded to Bull's `Queue.add` method.
+   *
+   * The global options can be overridden by passing options to `EventBusService.emit` directly.
+   *
+   * Example
+   * ```js
+   * {
+   *    removeOnComplete: { age: 10 },
+   * }
+   * ```
+   *
+   * @see https://api.docs.bullmq.io/interfaces/BaseJobOptions.html
+   */
+  jobOptions?: EmitOptions
+}