From f776ed234fcfccf23041ffebecbae6c9a8b7e922 Mon Sep 17 00:00:00 2001 From: "Carlos R. L. Rodrigues" <37986729+carlos-r-l-rodrigues@users.noreply.github.com> Date: Wed, 1 Feb 2023 08:10:59 -0300 Subject: [PATCH] feat(medusa): Run shared module migrations (#3109) --- .changeset/sweet-windows-float.md | 5 + integration-tests/development/server.js | 5 +- .../development/use-db-development.js | 50 ++++---- integration-tests/helpers/use-db.js | 19 +-- integration-tests/helpers/use-template-db.js | 15 ++- package.json | 4 +- packages/medusa/src/commands/migrate.js | 78 ++++++++----- packages/medusa/src/commands/seed.ts | 23 +++- .../src/commands/utils/get-migrations.js | 109 +++++++++++++++++- 9 files changed, 230 insertions(+), 78 deletions(-) create mode 100644 .changeset/sweet-windows-float.md diff --git a/.changeset/sweet-windows-float.md b/.changeset/sweet-windows-float.md new file mode 100644 index 0000000000..153c70e24d --- /dev/null +++ b/.changeset/sweet-windows-float.md @@ -0,0 +1,5 @@ +--- +"@medusajs/medusa": patch +--- + +feat(medusa): Run shared module migrations diff --git a/integration-tests/development/server.js b/integration-tests/development/server.js index 3cd7be8a21..a4dc7e494e 100644 --- a/integration-tests/development/server.js +++ b/integration-tests/development/server.js @@ -3,9 +3,10 @@ const express = require("express") const importFrom = require("import-from") const chokidar = require("chokidar") -require("dotenv").config({ path: path.join(__dirname, ".env.development") }) - process.env.DEV_MODE = !!process[Symbol.for("ts-node.register.instance")] +process.env.NODE_ENV = process.env.DEV_MODE && "development" + +require("dotenv").config({ path: path.join(__dirname, ".env.development") }) require("./dev-require") diff --git a/integration-tests/development/use-db-development.js b/integration-tests/development/use-db-development.js index c1efd5d809..1ae65fac42 100644 --- a/integration-tests/development/use-db-development.js +++ b/integration-tests/development/use-db-development.js @@ -1,6 +1,7 @@ const path = require("path") const { createConnection } = require("typeorm") +const { getConfigFile } = require("medusa-core-utils") const DB_HOST = process.env.DB_HOST const DB_USERNAME = process.env.DB_USERNAME @@ -8,6 +9,8 @@ const DB_PASSWORD = process.env.DB_PASSWORD const DB_NAME = process.env.DB_NAME const DB_URL = `postgres://${DB_USERNAME}:${DB_PASSWORD}@${DB_HOST}/${DB_NAME}` +process.env.NODE_ENV = "development" + require("./dev-require") async function createDB() { @@ -25,26 +28,26 @@ module.exports = { initDb: async function () { const cwd = path.resolve(path.join(__dirname, "../..")) - const configPath = path.resolve( - path.join(__dirname, "../api/medusa-config.js") + const { configModule } = getConfigFile( + path.join(__dirname), + `medusa-config` ) - const { featureFlags } = require(configPath) + + const { featureFlags } = configModule const basePath = path.join(cwd, "packages/medusa/src") - const featureFlagsLoader = require(path.join( - basePath, - `loaders`, - `feature-flags` - )).default + const featureFlagsLoader = + require("@medusajs/medusa/dist/loaders/feature-flags").default const featureFlagsRouter = featureFlagsLoader({ featureFlags }) - const modelsLoader = require(path.join( - basePath, - `loaders`, - `models` - )).default + const modelsLoader = require("@medusajs/medusa/dist/loaders/models").default + + const { + getEnabledMigrations, + getModuleSharedResources, + } = require("@medusajs/medusa/dist/commands/utils/get-migrations") const entities = modelsLoader({}, { register: false }) @@ -53,16 +56,14 @@ module.exports = { path.join(basePath, `migrations`, `*.{j,t}s`) ) - const { getEnabledMigrations } = require(path.join( - basePath, - `commands`, - `utils`, - `get-migrations` - )) + const isFlagEnabled = (flag) => featureFlagsRouter.isFeatureEnabled(flag) - const enabledMigrations = await getEnabledMigrations( + const { migrations: moduleMigrations, models: moduleModels } = + getModuleSharedResources(configModule, featureFlagsRouter) + + const enabledMigrations = getEnabledMigrations( [migrationDir], - (flag) => featureFlagsRouter.isFeatureEnabled(flag) + isFlagEnabled ) const enabledEntities = entities.filter( @@ -70,12 +71,13 @@ module.exports = { ) await createDB() + const dbConnection = await createConnection({ type: "postgres", url: DB_URL, - entities: enabledEntities, - migrations: enabledMigrations, - //logging: true, + entities: enabledEntities.concat(moduleModels), + migrations: enabledMigrations.concat(moduleMigrations), + // logging: true, }) await dbConnection.runMigrations() diff --git a/integration-tests/helpers/use-db.js b/integration-tests/helpers/use-db.js index b8f685f998..e64bab03f4 100644 --- a/integration-tests/helpers/use-db.js +++ b/integration-tests/helpers/use-db.js @@ -1,5 +1,6 @@ const path = require("path") +const { getConfigFile } = require("medusa-core-utils") const { dropDatabase } = require("pg-god") const { createConnection } = require("typeorm") const dbFactory = require("./use-template-db") @@ -43,6 +44,7 @@ const DbTestUtil = { forceDelete = forceDelete || [] const entities = this.db_.entityMetadatas + const manager = this.db_.manager if (connectionType === "sqlite") { @@ -79,8 +81,8 @@ const instance = DbTestUtil module.exports = { initDb: async function ({ cwd, database_extra }) { - const configPath = path.resolve(path.join(cwd, `medusa-config.js`)) - const { projectConfig, featureFlags } = require(configPath) + const { configModule } = getConfigFile(cwd, `medusa-config`) + const { projectConfig, featureFlags } = configModule const featureFlagsLoader = require("@medusajs/medusa/dist/loaders/feature-flags").default @@ -122,11 +124,14 @@ module.exports = { const { getEnabledMigrations, + getModuleSharedResources, } = require("@medusajs/medusa/dist/commands/utils/get-migrations") - const enabledMigrations = await getEnabledMigrations( - [migrationDir], - (flag) => featureFlagsRouter.isFeatureEnabled(flag) + const { migrations: moduleMigrations, models: moduleModels } = + getModuleSharedResources(configModule, featureFlagsRouter) + + const enabledMigrations = getEnabledMigrations([migrationDir], (flag) => + featureFlagsRouter.isFeatureEnabled(flag) ) const enabledEntities = entities.filter( @@ -136,8 +141,8 @@ module.exports = { const dbConnection = await createConnection({ type: "postgres", url: DB_URL, - entities: enabledEntities, - migrations: enabledMigrations, + entities: enabledEntities.concat(moduleModels), + migrations: enabledMigrations.concat(moduleMigrations), extra: database_extra ?? {}, name: "integration-tests", }) diff --git a/integration-tests/helpers/use-template-db.js b/integration-tests/helpers/use-template-db.js index 9055e92b34..2946c15d46 100644 --- a/integration-tests/helpers/use-template-db.js +++ b/integration-tests/helpers/use-template-db.js @@ -2,6 +2,7 @@ const path = require("path") require("dotenv").config({ path: path.join(__dirname, "../.env.test") }) +const { getConfigFile } = require("medusa-core-utils") const { createDatabase, dropDatabase } = require("pg-god") const { createConnection, getConnection } = require("typeorm") @@ -24,8 +25,10 @@ class DatabaseFactory { } async createTemplateDb_({ cwd }) { - // const cwd = path.resolve(path.join(__dirname, "..")) + const { configModule } = getConfigFile(cwd, `medusa-config`) + const connection = await this.getMasterConnection() + const migrationDir = path.resolve( path.join( __dirname, @@ -41,14 +44,18 @@ class DatabaseFactory { const { getEnabledMigrations, + getModuleSharedResources, } = require("@medusajs/medusa/dist/commands/utils/get-migrations") // filter migrations to only include those that don't have feature flags - const enabledMigrations = await getEnabledMigrations( + const enabledMigrations = getEnabledMigrations( [migrationDir], (flag) => false ) + const { migrations: moduleMigrations } = + getModuleSharedResources(configModule) + await dropDatabase( { databaseName: this.templateDbName, @@ -65,7 +72,7 @@ class DatabaseFactory { type: "postgres", name: "templateConnection", url: `${DB_URL}/${this.templateDbName}`, - migrations: enabledMigrations, + migrations: enabledMigrations.concat(moduleMigrations), }) await templateDbConnection.runMigrations() @@ -76,7 +83,7 @@ class DatabaseFactory { async getMasterConnection() { try { - return await getConnection(this.masterConnectionName) + return getConnection(this.masterConnectionName) } catch (err) { return await this.createMasterConnection() } diff --git a/package.json b/package.json index 40eee59e34..57506352ba 100644 --- a/package.json +++ b/package.json @@ -77,8 +77,8 @@ "generate:entities": "typedoc --options typedoc.entities.js", "release:snapshot": "changeset publish --no-git-tags --snapshot --tag snapshot", "generate:announcement": "node ./scripts/doc-change-release.js", - "develop": "NODE_ENV=development ts-node --transpile-only ./integration-tests/development/server.js", - "develop:create:db": "NODE_ENV=development ts-node --transpile-only ./integration-tests/development/create-database.js", + "develop": "ts-node --transpile-only ./integration-tests/development/server.js", + "develop:create:db": "ts-node --transpile-only ./integration-tests/development/create-database.js", "release:next": "changeset publish --no-git-tags --snapshot --tag next", "version:next": "changeset version --snapshot next", "check:freshness": "node ./scripts/freshness-check.js" diff --git a/packages/medusa/src/commands/migrate.js b/packages/medusa/src/commands/migrate.js index 7bb4fc5387..48d7d6e015 100644 --- a/packages/medusa/src/commands/migrate.js +++ b/packages/medusa/src/commands/migrate.js @@ -1,51 +1,73 @@ import { createConnection } from "typeorm" -import { getConfigFile } from "medusa-core-utils" import featureFlagLoader from "../loaders/feature-flags" -import { handleConfigError } from "../loaders/config" +import configModuleLoader from "../loaders/config" import Logger from "../loaders/logger" -import getMigrations from "./utils/get-migrations" +import getMigrations, { getModuleSharedResources } from "./utils/get-migrations" -const t = async function ({ directory }) { +const getDataSource = async (directory) => { + const configModule = configModuleLoader(directory) + + const featureFlagRouter = featureFlagLoader(configModule) + + const { coreMigrations } = getMigrations(directory, featureFlagRouter) + + const { migrations: moduleMigrations } = getModuleSharedResources( + configModule, + featureFlagRouter + ) + + return await createConnection({ + type: configModule.projectConfig.database_type, + url: configModule.projectConfig.database_url, + extra: configModule.projectConfig.database_extra || {}, + schema: configModule.projectConfig.database_schema, + migrations: coreMigrations.concat(moduleMigrations), + logging: true, + }) +} + +const main = async function ({ directory }) { const args = process.argv args.shift() args.shift() args.shift() - const { configModule, error } = getConfigFile(directory, `medusa-config`) - - if (error) { - handleConfigError(error) - } - - const featureFlagRouter = featureFlagLoader(configModule) - - const enabledMigrations = await getMigrations(directory, featureFlagRouter) - - const connection = await createConnection({ - type: configModule.projectConfig.database_type, - url: configModule.projectConfig.database_url, - schema: configModule.projectConfig.database_schema, - extra: configModule.projectConfig.database_extra || {}, - migrations: enabledMigrations, - logging: true, - }) - if (args[0] === "run") { - await connection.runMigrations() - await connection.close() + const dataSource = await getDataSource(directory) + + await dataSource.runMigrations() + await dataSource.close() Logger.info("Migrations completed.") process.exit() } else if (args[0] === "revert") { - await connection.undoLastMigration({ transaction: "all" }) - await connection.close() + const dataSource = await getDataSource(directory) + + await dataSource.undoLastMigration({ transaction: "all" }) + await dataSource.close() Logger.info("Migrations reverted.") + process.exit() } else if (args[0] === "show") { + const configModule = configModuleLoader(directory) + + const featureFlagRouter = featureFlagLoader(configModule) + + const { coreMigrations } = getMigrations(directory, featureFlagRouter) + + const connection = await createConnection({ + type: configModule.projectConfig.database_type, + url: configModule.projectConfig.database_url, + extra: configModule.projectConfig.database_extra || {}, + schema: configModule.projectConfig.database_schema, + migrations: coreMigrations, + logging: true, + }) + const unapplied = await connection.showMigrations() await connection.close() process.exit(unapplied ? 1 : 0) } } -export default t +export default main diff --git a/packages/medusa/src/commands/seed.ts b/packages/medusa/src/commands/seed.ts index f1152d60cf..db6d9754ec 100644 --- a/packages/medusa/src/commands/seed.ts +++ b/packages/medusa/src/commands/seed.ts @@ -23,7 +23,7 @@ import { } from "../services" import { ConfigModule } from "../types/global" import { CreateProductInput } from "../types/product" -import getMigrations from "./utils/get-migrations" +import getMigrations, { getModuleSharedResources } from "./utils/get-migrations" type SeedOptions = { directory: string @@ -58,7 +58,12 @@ const seed = async function ({ directory, migrate, seedFile }: SeedOptions) { const dbType = configModule.projectConfig.database_type if (migrate && dbType !== "sqlite") { - const migrationDirs = await getMigrations(directory, featureFlagRouter) + const { coreMigrations } = getMigrations(directory, featureFlagRouter) + + const { migrations: moduleMigrations } = getModuleSharedResources( + configModule, + featureFlagRouter + ) const connectionOptions = { type: configModule.projectConfig.database_type, @@ -66,7 +71,7 @@ const seed = async function ({ directory, migrate, seedFile }: SeedOptions) { schema: configModule.projectConfig.database_schema, url: configModule.projectConfig.database_url, extra: configModule.projectConfig.database_extra || {}, - migrations: migrationDirs, + migrations: coreMigrations.concat(moduleMigrations), logging: true, } as ConnectionOptions @@ -91,9 +96,15 @@ const seed = async function ({ directory, migrate, seedFile }: SeedOptions) { const regionService: RegionService = container.resolve("regionService") const productService: ProductService = container.resolve("productService") /* eslint-disable */ - const productVariantService: ProductVariantService = container.resolve("productVariantService") - const shippingOptionService: ShippingOptionService = container.resolve("shippingOptionService") - const shippingProfileService: ShippingProfileService = container.resolve("shippingProfileService") + const productVariantService: ProductVariantService = container.resolve( + "productVariantService" + ) + const shippingOptionService: ShippingOptionService = container.resolve( + "shippingOptionService" + ) + const shippingProfileService: ShippingProfileService = container.resolve( + "shippingProfileService" + ) /* eslint-enable */ await manager.transaction(async (tx) => { diff --git a/packages/medusa/src/commands/utils/get-migrations.js b/packages/medusa/src/commands/utils/get-migrations.js index d0148f6d30..d6449f7bbf 100644 --- a/packages/medusa/src/commands/utils/get-migrations.js +++ b/packages/medusa/src/commands/utils/get-migrations.js @@ -5,7 +5,7 @@ import { isString } from "lodash" import { sync as existsSync } from "fs-exists-cached" import { getConfigFile, createRequireFromPath } from "medusa-core-utils" import { handleConfigError } from "../../loaders/config" -import logger from "../../loaders/logger" +import registerModuleDefinitions from "../../loaders/module-definitions" function createFileContentHash(path, files) { return path + files @@ -89,7 +89,37 @@ function resolvePlugin(pluginName) { } } -export default async (directory, featureFlagRouter) => { +export function getInternalModules(configModule) { + const modules = [] + + const moduleResolutions = registerModuleDefinitions(configModule) + + for (const moduleResolution of Object.values(moduleResolutions)) { + if ( + !moduleResolution.resolutionPath || + moduleResolution.moduleDeclaration.scope !== "internal" + ) { + continue + } + + let loadedModule = null + try { + loadedModule = require(moduleResolution.moduleDeclaration.resolve).default + } catch (error) { + console.log("Error loading Module", error) + continue + } + + modules.push({ + moduleDeclaration: moduleResolution.moduleDeclaration, + loadedModule, + }) + } + + return modules +} + +export default (directory, featureFlagRouter) => { const { configModule, error } = getConfigFile(directory, `medusa-config`) if (error) { @@ -118,11 +148,11 @@ export default async (directory, featureFlagRouter) => { }) const migrationDirs = [] - const coreMigrations = path.resolve( + const corePackageMigrations = path.resolve( path.join(__dirname, "..", "..", "migrations") ) - migrationDirs.push(path.join(coreMigrations, "*.js")) + migrationDirs.push(path.join(corePackageMigrations, "*.js")) for (const p of resolved) { const exists = existsSync(`${p.resolve}/migrations`) @@ -131,9 +161,15 @@ export default async (directory, featureFlagRouter) => { } } - return getEnabledMigrations(migrationDirs, (flag) => + const isFeatureFlagEnabled = (flag) => featureFlagRouter.isFeatureEnabled(flag) + + const coreMigrations = getEnabledMigrations( + migrationDirs, + isFeatureFlagEnabled ) + + return { coreMigrations } } export const getEnabledMigrations = (migrationDirs, isFlagEnabled) => { @@ -154,3 +190,66 @@ export const getEnabledMigrations = (migrationDirs, isFlagEnabled) => { }) .filter(Boolean) } + +export const getModuleMigrations = (configModule, isFlagEnabled) => { + const loadedModules = getInternalModules(configModule) + + const allModules = [] + + for (const loadedModule of loadedModules) { + const mod = loadedModule.loadedModule + + const isolatedMigrations = {} + const moduleMigrations = (mod.migrations ?? []) + .map((migrations) => { + const all = [] + for (const migration of Object.values(migrations)) { + // TODO: revisit how Modules export their migration entrypoints up/down + if (["up", "down"].includes(migration.name)) { + isolatedMigrations[migration.name] = migration + } else if ( + typeof migration.featureFlag === "undefined" || + isFlagEnabled(migration.featureFlag) + ) { + all.push(migration) + } + } + return all + }) + .flat() + + allModules.push({ + moduleDeclaration: loadedModule.moduleDeclaration, + models: mod.models ?? [], + migrations: moduleMigrations, + externalMigrations: isolatedMigrations, + }) + } + + return allModules +} + +export const getModuleSharedResources = (configModule, featureFlagsRouter) => { + const isFlagEnabled = (flag) => + featureFlagsRouter && featureFlagsRouter.isFeatureEnabled(flag) + + const loadedModules = getModuleMigrations(configModule, isFlagEnabled) + + let migrations = [] + let models = [] + + for (const mod of loadedModules) { + if (mod.moduleDeclaration.resources !== "shared") { + continue + } + + migrations = migrations.concat(mod.migrations) + + models = models.concat(mod.models ?? []) + } + + return { + models, + migrations, + } +}