Chore/integration tests modules utils (#6581)

new wrapper for medusa integration tests.
for now it is only applied to the modules directory, but it could be used in the api integration tests or any other integrations that requires a db and a server up and running. 

It is not perfect, but I wanted to have something working and centralised before improving it, also avoiding too many conflicts with other prs

packages/medusa-test-utils/package.json

@@ -24,18 +24,46 @@
   "author": "Sebastian Rindom",
   "license": "MIT",
   "devDependencies": {
+    "@medusajs/types": "^1.1.41",
     "cross-env": "^5.2.1",
     "jest": "^25.5.4",
     "rimraf": "^3.0.2",
     "typescript": "^5.1.6"
   },
+  "peerDependencies": {
+    "@medusajs/medusa": ">1.19",
+    "axios": "^0.28.0",
+    "express": "^4.18.3",
+    "get-port": "^5.1.0",
+    "pg-god": "^1.0.12",
+    "typeorm": "^0.2.43"
+  },
+  "peerDependenciesMeta": {
+    "@medusajs/medusa": {
+      "optional": true
+    },
+    "axios": {
+      "optional": true
+    },
+    "express": {
+      "optional": true
+    },
+    "get-port": {
+      "optional": true
+    },
+    "pg-god": {
+      "optional": true
+    },
+    "typeorm": {
+      "optional": true
+    }
+  },
   "dependencies": {
     "@medusajs/modules-sdk": "^1.12.6",
     "@medusajs/utils": "^1.1.41",
     "@mikro-orm/migrations": "5.9.7",
     "@mikro-orm/postgresql": "5.9.7",
     "medusa-core-utils": "^1.2.1",
-    "pg-god": "^1.0.12",
     "randomatic": "^3.1.1"
   },
   "gitHead": "81a7ff73d012fda722f6e9ef0bd9ba0232d37808"

packages/medusa-test-utils/src/index.ts

@@ -6,3 +6,4 @@ export { default as MockRepository } from "./mock-repository"
 export * from "./init-modules"
 export { default as MockEventBusService } from "./mock-event-bus-service"
 export * from "./module-test-runner"
+export * from "./medusa-test-runner"

packages/medusa-test-utils/src/medusa-test-runner-utils/bootstrap-app.js

@@ -0,0 +1,79 @@
+const path = require("path")
+const express = require("express")
+const getPort = require("get-port")
+const { isObject } = require("@medusajs/utils")
+
+async function bootstrapApp({ cwd, env = {} } = {}) {
+  const app = express()
+
+  if (isObject(env)) {
+    Object.entries(env).forEach(([k, v]) => (process.env[k] = v))
+  }
+
+  const loaders = require("@medusajs/medusa/dist/loaders").default
+
+  const { container, dbConnection, pgConnection, disposeResources } =
+    await loaders({
+      directory: path.resolve(cwd || process.cwd()),
+      expressApp: app,
+      isTest: false,
+    })
+
+  const PORT = await getPort()
+
+  return {
+    disposeResources,
+    container,
+    db: dbConnection,
+    pgConnection,
+    app,
+    port: PORT,
+  }
+}
+
+module.exports = {
+  startBootstrapApp: async ({
+    cwd,
+    env = {},
+    skipExpressListen = false,
+  } = {}) => {
+    const { app, port, container, db, pgConnection } = await bootstrapApp({
+      cwd,
+      env,
+    })
+    let expressServer
+
+    if (skipExpressListen) {
+      return
+    }
+
+    const shutdown = async () => {
+      await Promise.all([
+        container.dispose(),
+        expressServer.close(),
+        db?.destroy(),
+        pgConnection?.context?.destroy(),
+        container.dispose(),
+      ])
+
+      if (typeof global !== "undefined" && global?.gc) {
+        global.gc()
+      }
+    }
+
+    return await new Promise((resolve, reject) => {
+      expressServer = app.listen(port, async (err) => {
+        if (err) {
+          await shutdown()
+          return reject(err)
+        }
+        process.send(port)
+        resolve({
+          shutdown,
+          container,
+          port,
+        })
+      })
+    })
+  },
+}

packages/medusa-test-utils/src/medusa-test-runner-utils/use-db.js

@@ -0,0 +1,113 @@
+const path = require("path")
+
+const { getConfigFile } = require("medusa-core-utils")
+const { asValue } = require("awilix")
+const {
+  isObject,
+  createMedusaContainer,
+  MedusaV2Flag,
+} = require("@medusajs/utils")
+const { DataSource } = require("typeorm")
+const { ContainerRegistrationKeys } = require("@medusajs/utils")
+const { migrateMedusaApp } = require("@medusajs/medusa/dist/loaders/medusa-app")
+const { logger } = require("@medusajs/medusa-cli/dist/reporter")
+
+module.exports = {
+  initDb: async function ({
+    cwd,
+    database_extra,
+    env,
+    force_modules_migration,
+    dbUrl = DB_URL,
+    dbSchema = "public",
+  }) {
+    if (isObject(env)) {
+      Object.entries(env).forEach(([k, v]) => (process.env[k] = v))
+    }
+
+    const { configModule } = getConfigFile(cwd, `medusa-config`)
+
+    const featureFlagsLoader =
+      require("@medusajs/medusa/dist/loaders/feature-flags").default
+
+    const featureFlagRouter = featureFlagsLoader(configModule)
+    const modelsLoader = require("@medusajs/medusa/dist/loaders/models").default
+    const entities = modelsLoader({}, { register: false })
+
+    // get migrations with enabled featureflags
+    const migrationDir = path.resolve(
+      path.join(
+        cwd,
+        `../../`,
+        `node_modules`,
+        `@medusajs`,
+        `medusa`,
+        `dist`,
+        `migrations`,
+        `*.js`
+      )
+    )
+
+    const {
+      getEnabledMigrations,
+      getModuleSharedResources,
+    } = require("@medusajs/medusa/dist/commands/utils/get-migrations")
+
+    const { migrations: moduleMigrations, models: moduleModels } =
+      getModuleSharedResources(configModule, featureFlagRouter)
+
+    const enabledMigrations = getEnabledMigrations([migrationDir], (flag) =>
+      featureFlagRouter.isFeatureEnabled(flag)
+    )
+
+    const enabledEntities = entities.filter(
+      (e) => typeof e.isFeatureEnabled === "undefined" || e.isFeatureEnabled()
+    )
+
+    const dbDataSource = new DataSource({
+      type: "postgres",
+      url: dbUrl,
+      entities: enabledEntities.concat(moduleModels),
+      migrations: enabledMigrations.concat(moduleMigrations),
+      extra: database_extra ?? {},
+      //name: "integration-tests",
+      schema: dbSchema,
+    })
+
+    await dbDataSource.initialize()
+
+    await dbDataSource.runMigrations()
+
+    if (
+      force_modules_migration ||
+      featureFlagRouter.isFeatureEnabled(MedusaV2Flag.key)
+    ) {
+      const pgConnectionLoader =
+        require("@medusajs/medusa/dist/loaders/pg-connection").default
+
+      const featureFlagLoader =
+        require("@medusajs/medusa/dist/loaders/feature-flags").default
+
+      const container = createMedusaContainer()
+
+      const featureFlagRouter = await featureFlagLoader(configModule)
+
+      const pgConnection = await pgConnectionLoader({ configModule, container })
+
+      container.register({
+        [ContainerRegistrationKeys.CONFIG_MODULE]: asValue(configModule),
+        [ContainerRegistrationKeys.LOGGER]: asValue(logger),
+        [ContainerRegistrationKeys.MANAGER]: asValue(dbDataSource.manager),
+        [ContainerRegistrationKeys.PG_CONNECTION]: asValue(pgConnection),
+        featureFlagRouter: asValue(featureFlagRouter),
+      })
+
+      await migrateMedusaApp(
+        { configModule, container },
+        { registerInContainer: false }
+      )
+    }
+
+    return { dbDataSource, pgConnection }
+  },
+}

packages/medusa-test-utils/src/medusa-test-runner.ts

@@ -0,0 +1,238 @@
+import { getDatabaseURL } from "./database"
+import { initDb } from "./medusa-test-runner-utils/use-db"
+import { startBootstrapApp } from "./medusa-test-runner-utils/bootstrap-app"
+import { createDatabase, dropDatabase } from "pg-god"
+import { ContainerLike } from "@medusajs/types"
+
+const axios = require("axios").default
+
+const keepTables = [
+  "store",
+  "staged_job",
+  "shipping_profile",
+  "fulfillment_provider",
+  "payment_provider",
+  "country",
+  "region_country",
+  "currency",
+  "migrations",
+  "mikro_orm_migrations",
+]
+
+const DB_HOST = process.env.DB_HOST
+const DB_USERNAME = process.env.DB_USERNAME
+const DB_PASSWORD = process.env.DB_PASSWORD
+
+const pgGodCredentials = {
+  user: DB_USERNAME,
+  password: DB_PASSWORD,
+  host: DB_HOST,
+}
+
+const dbTestUtilFactory = (): any => ({
+  db_: null,
+  pgConnection_: null,
+
+  clear: async function () {
+    this.db_?.synchronize(true)
+  },
+
+  create: async function (dbName: string) {
+    await createDatabase({ databaseName: dbName }, pgGodCredentials)
+  },
+
+  teardown: async function ({
+    forceDelete,
+    schema,
+  }: { forceDelete?: string[]; schema?: string } = {}) {
+    forceDelete = forceDelete || []
+    const manager = this.db_.manager
+
+    await manager.query(`SET session_replication_role = 'replica';`)
+    const tableNames = await manager.query(`SELECT table_name
+                                            FROM information_schema.tables
+                                            WHERE table_schema = '${
+                                              schema ?? "public"
+                                            }';`)
+
+    for (const { table_name } of tableNames) {
+      if (
+        keepTables.includes(table_name) &&
+        !forceDelete.includes(table_name)
+      ) {
+        continue
+      }
+
+      await manager.query(`DELETE
+                           FROM "${table_name}";`)
+    }
+
+    await manager.query(`SET session_replication_role = 'origin';`)
+  },
+
+  shutdown: async function (dbName: string) {
+    await this.db_?.destroy()
+    await this.pgConnection_?.context?.destroy()
+
+    return await dropDatabase(
+      { databaseName: dbName, errorIfNonExist: false },
+      pgGodCredentials
+    )
+  },
+})
+
+export interface MedusaSuiteOptions<TService = unknown> {
+  dbUtils: any
+  dbConnection: any // Legacy typeorm connection
+  getContainer: () => ContainerLike
+  api: any
+  dbConfig: {
+    dbName: string
+    schema: string
+    clientUrl: string
+  }
+}
+
+export function medusaIntegrationTestRunner({
+  moduleName,
+  dbName,
+  schema = "public",
+  env = {},
+  force_modules_migration = false,
+  debug = false,
+  testSuite,
+}: {
+  moduleName?: string
+  env?: Record<string, string>
+  dbName?: string
+  schema?: string
+  debug?: boolean
+  force_modules_migration?: boolean
+  testSuite: <TService = unknown>(
+    options: MedusaSuiteOptions<TService>
+  ) => () => void
+}) {
+  const tempName = parseInt(process.env.JEST_WORKER_ID || "1")
+  moduleName = moduleName ?? Math.random().toString(36).substring(7)
+  dbName ??= `medusa-${moduleName.toLowerCase()}-integration-${tempName}`
+
+  let dbConfig = {
+    dbName,
+    clientUrl: getDatabaseURL(dbName),
+    schema,
+    debug,
+  }
+
+  // Intercept call to this utils to apply the unique client url for the current suite
+  const originalCreatePgConnection =
+    require("@medusajs/utils/dist/modules-sdk/create-pg-connection").createPgConnection
+  require("@medusajs/utils/dist/modules-sdk/create-pg-connection").createPgConnection =
+    (options: any) => {
+      return originalCreatePgConnection({
+        ...options,
+        clientUrl: dbConfig.clientUrl,
+      })
+    }
+
+  const originalDatabaseLoader =
+    require("@medusajs/medusa/dist/loaders/database").default
+  require("@medusajs/medusa/dist/loaders/database").default = (
+    options: any
+  ) => {
+    options.configModule.projectConfig.database_url
+    return originalDatabaseLoader({
+      ...options,
+      configModule: {
+        ...options.configModule,
+        projectConfig: {
+          database_logging: debug, // Will be used for the debug flag of the database options
+          ...options.configModule.projectConfig,
+          database_url: dbConfig.clientUrl,
+        },
+      },
+    })
+  }
+
+  const cwd = process.cwd()
+
+  let shutdown: () => Promise<void>
+  let dbUtils = dbTestUtilFactory()
+  let container: ContainerLike
+  let apiUtils: any
+
+  let options = {
+    dbUtils,
+    api: new Proxy(
+      {},
+      {
+        get: (target, prop) => {
+          return apiUtils[prop]
+        },
+      }
+    ),
+    dbConnection: new Proxy(
+      {},
+      {
+        get: (target, prop) => {
+          return dbUtils.db_[prop]
+        },
+      }
+    ),
+    getContainer: () => container,
+  } as MedusaSuiteOptions
+
+  const beforeAll_ = async () => {
+    try {
+      await dbUtils.create(dbName)
+      const { dbDataSource, pgConnection } = await initDb({
+        cwd,
+        env,
+        force_modules_migration,
+        database_extra: {},
+        dbUrl: dbConfig.clientUrl,
+        dbSchema: dbConfig.schema,
+      })
+      dbUtils.db_ = dbDataSource
+      dbUtils.pgConnection_ = pgConnection
+
+      const {
+        shutdown: shutdown_,
+        container: container_,
+        port,
+      } = await startBootstrapApp({
+        cwd,
+        env,
+      })
+
+      apiUtils = axios.create({ baseURL: `http://localhost:${port}` })
+
+      container = container_
+      shutdown = shutdown_
+    } catch (error) {
+      console.error("Error setting up integration environment:", error)
+    }
+  }
+
+  const afterEach_ = async () => {
+    try {
+      await dbUtils.teardown({ forceDelete: [], schema })
+    } catch (error) {
+      console.error("Error tearing down database:", error)
+    }
+  }
+
+  return describe("", () => {
+    beforeAll(beforeAll_)
+    afterEach(afterEach_)
+    afterAll(async () => {
+      try {
+        await dbUtils.shutdown(dbName)
+        await shutdown()
+      } catch (error) {
+        console.error("Error shutting down integration environment:", error)
+      }
+    })
+
+    testSuite(options!)
+  })
+}

packages/medusa-test-utils/src/module-test-runner.ts

@@ -39,6 +39,8 @@ export function moduleIntegrationTestRunner({
   debug?: boolean
   testSuite: <TService = unknown>(options: SuiteOptions<TService>) => () => void
 }) {
+  process.env.LOG_LEVEL = "error"
+
   moduleModels = Object.values(require(`${process.cwd()}/src/models`))
   // migrationPath ??= process.cwd() + "/src/migrations/!(*.d).{js,ts,cjs}"
 

packages/medusa/src/loaders/medusa-app.ts

@@ -66,8 +66,12 @@ export async function migrateMedusaApp(
 
   const sharedResourcesConfig = {
     database: {
-      clientUrl: configModule.projectConfig.database_url,
+      clientUrl:
+        injectedDependencies[ContainerRegistrationKeys.PG_CONNECTION]?.client
+          ?.config?.connection?.connectionString ??
+        configModule.projectConfig.database_url,
       driverOptions: configModule.projectConfig.database_extra,
+      debug: !!(configModule.projectConfig.database_logging ?? false),
     },
   }
 
@@ -85,7 +89,7 @@ export async function migrateMedusaApp(
       ).options ??= {
         database: {
           type: "postgres",
-          url: configModule.projectConfig.database_url,
+          url: sharedResourcesConfig.database.clientUrl,
           extra: configModule.projectConfig.database_extra,
           schema: configModule.projectConfig.database_schema,
           logging: configModule.projectConfig.database_logging,
@@ -132,6 +136,7 @@ export const loadMedusaApp = async (
     database: {
       clientUrl: configModule.projectConfig.database_url,
       driverOptions: configModule.projectConfig.database_extra,
+      debug: !!(configModule.projectConfig.database_logging ?? false),
     },
   }
 

packages/modules-sdk/src/medusa-app.ts

@@ -337,7 +337,7 @@ async function MedusaApp_({
       )
     }
 
-    const linkModuleOpt = { ...linkModuleOptions }
+    const linkModuleOpt = { ...(linkModuleOptions ?? {}) }
     linkModuleOpt.database ??= {
       ...(sharedResourcesConfig?.database ?? {}),
     }

packages/modules-sdk/src/medusa-module.ts

@@ -285,11 +285,14 @@ export class MedusaModule {
       moduleDefinition
     )
 
+    const logger_ =
+      container.resolve("logger", { allowUnregistered: true }) ?? logger
+
     try {
       await moduleLoader({
         container,
         moduleResolutions,
-        logger,
+        logger: logger_,
         migrationOnly,
       })
     } catch (err) {
@@ -314,7 +317,7 @@ export class MedusaModule {
         ].__joinerConfig()
 
         if (!joinerConfig.primaryKeys) {
-          logger.warn(
+          logger_.warn(
             `Primary keys are not defined by the module ${keyName}. Setting default primary key to 'id'${EOL}`
           )
 
@@ -407,11 +410,14 @@ export class MedusaModule {
       moduleExports
     )
 
+    const logger_ =
+      container.resolve("logger", { allowUnregistered: true }) ?? logger
+
     try {
       await moduleLoader({
         container,
         moduleResolutions,
-        logger,
+        logger: logger_,
       })
     } catch (err) {
       errorLoading(err)