chore: parallelize integration tests and use template dbs (#907)

.github/workflows/action.yml

@@ -86,7 +86,6 @@ jobs:
         working-directory: integration-tests/api
 
       - name: Run integration tests
-        run: yarn test
+        run: yarn test:integration --maxWorkers=50% --silent=false
-        working-directory: integration-tests/api
         env:
           DB_PASSWORD: postgres

integration-tests/api/medusa-config.js

@@ -1,11 +1,12 @@
 const DB_USERNAME = process.env.DB_USERNAME || "postgres"
 const DB_PASSWORD = process.env.DB_PASSWORD || ""
+const workerId = parseInt(process.env.JEST_WORKER_ID || "1")
 
 module.exports = {
   plugins: [],
   projectConfig: {
     // redis_url: REDIS_URL,
-    database_url: `postgres://${DB_USERNAME}:${DB_PASSWORD}@localhost/medusa-integration`,
+    database_url: `postgres://${DB_USERNAME}:${DB_PASSWORD}@localhost/medusa-integration-${workerId}`,
     database_type: "postgres",
   },
 }

integration-tests/api/package.json

@@ -4,19 +4,19 @@
   "main": "index.js",
   "license": "MIT",
   "scripts": {
-    "test": "jest --runInBand --silent=false",
+    "test": "jest --maxWorkers=50% --silent=false",
     "build": "babel src -d dist --extensions \".ts,.js\""
   },
   "dependencies": {
-    "@medusajs/medusa": "1.1.57-dev-1638874121913",
+    "@medusajs/medusa": "1.1.59-dev-1639132284143",
-    "medusa-interfaces": "1.1.31-dev-1638874121913",
+    "medusa-interfaces": "1.1.32-dev-1639132284143",
     "typeorm": "^0.2.31"
   },
   "devDependencies": {
     "@babel/cli": "^7.12.10",
     "@babel/core": "^7.12.10",
     "@babel/node": "^7.12.10",
-    "babel-preset-medusa-package": "1.1.18-dev-1638874121913",
+    "babel-preset-medusa-package": "1.1.19-dev-1639132284143",
     "jest": "^26.6.3"
   }
 }

integration-tests/api/yarn.lock

@@ -1256,10 +1256,10 @@
     "@types/yargs" "^15.0.0"
     chalk "^4.0.0"
 
-"@medusajs/medusa-cli@1.1.23-dev-1638874121913":
+"@medusajs/medusa-cli@1.1.24-dev-1639132284143":
-  version "1.1.23-dev-1638874121913"
+  version "1.1.24-dev-1639132284143"
-  resolved "http://localhost:4873/@medusajs%2fmedusa-cli/-/medusa-cli-1.1.23-dev-1638874121913.tgz#6f12e4f54c08f4846ce3c213fb3407833b1febec"
+  resolved "http://localhost:4873/@medusajs%2fmedusa-cli/-/medusa-cli-1.1.24-dev-1639132284143.tgz#80eeb1f732db447797d8c1c141fc2590a00f89e1"
-  integrity sha512-ZhO3UhIdyjKOoqjfWn09wmV7Vk2A52qOE3JwLvJkyiPVt3tO23/2H7DHF93faYtrKjBCo32xSLBuZYqsRm6p+w==
+  integrity sha512-IEEq9iJpTHeXAHm8KbGepYVO8xxntlrviCuGfh3OCMBzTAgnQJ/CDvRhAgb4fRGxJZ0uwLJLtQG/7ePCPAotOw==
   dependencies:
     "@babel/polyfill" "^7.8.7"
     "@babel/runtime" "^7.9.6"
@@ -1277,8 +1277,8 @@
     is-valid-path "^0.1.1"
     joi-objectid "^3.0.1"
     meant "^1.0.1"
-    medusa-core-utils "1.1.30-dev-1638874121913"
+    medusa-core-utils "1.1.31-dev-1639132284143"
-    medusa-telemetry "0.0.10-dev-1638874121913"
+    medusa-telemetry "0.0.11-dev-1639132284143"
     netrc-parser "^3.1.6"
     open "^8.0.6"
     ora "^5.4.1"
@@ -1292,13 +1292,13 @@
     winston "^3.3.3"
     yargs "^15.3.1"
 
-"@medusajs/medusa@1.1.57-dev-1638874121913":
+"@medusajs/medusa@1.1.59-dev-1639132284143":
-  version "1.1.57-dev-1638874121913"
+  version "1.1.59-dev-1639132284143"
-  resolved "http://localhost:4873/@medusajs%2fmedusa/-/medusa-1.1.57-dev-1638874121913.tgz#c4a40d588a2567a6f80a84b8017dd64382e224bc"
+  resolved "http://localhost:4873/@medusajs%2fmedusa/-/medusa-1.1.59-dev-1639132284143.tgz#9cb56910e415e7f8654e94aea0468b960c2ffb12"
-  integrity sha512-WSTL4U3TnB7u8YjaqaEnfjQQCmQMpZIqAwRA2+1SuaRrW3A3SYbxoRkL/3bgH13KGv/PCBAvkV1e7l9xcziAnQ==
+  integrity sha512-uxKIJLKMwOR/UbuLFW9IvdReODvHace5DVWbTMIK1g031PBX3/ghto0yIbcZVaNdMLETS5SiEwPHkx0QAw2k2A==
   dependencies:
     "@hapi/joi" "^16.1.8"
-    "@medusajs/medusa-cli" "1.1.23-dev-1638874121913"
+    "@medusajs/medusa-cli" "1.1.24-dev-1639132284143"
     "@types/lodash" "^4.14.168"
     awilix "^4.2.3"
     body-parser "^1.19.0"
@@ -1322,8 +1322,8 @@
     joi "^17.3.0"
     joi-objectid "^3.0.1"
     jsonwebtoken "^8.5.1"
-    medusa-core-utils "1.1.30-dev-1638874121913"
+    medusa-core-utils "1.1.31-dev-1639132284143"
-    medusa-test-utils "1.1.33-dev-1638874121913"
+    medusa-test-utils "1.1.34-dev-1639132284143"
     morgan "^1.9.1"
     multer "^1.4.2"
     passport "^0.4.0"
@@ -1947,10 +1947,10 @@ babel-preset-jest@^26.6.2:
     babel-plugin-jest-hoist "^26.6.2"
     babel-preset-current-node-syntax "^1.0.0"
 
-babel-preset-medusa-package@1.1.18-dev-1638874121913:
+babel-preset-medusa-package@1.1.19-dev-1639132284143:
-  version "1.1.18-dev-1638874121913"
+  version "1.1.19-dev-1639132284143"
-  resolved "http://localhost:4873/babel-preset-medusa-package/-/babel-preset-medusa-package-1.1.18-dev-1638874121913.tgz#d50b513bbbfe691318c9b9463c7ec0c0c1ca8711"
+  resolved "http://localhost:4873/babel-preset-medusa-package/-/babel-preset-medusa-package-1.1.19-dev-1639132284143.tgz#a98da041ddda18db99244ae79bf415c1c674f302"
-  integrity sha512-FDJyHCp+FyncJDlapHIwxHRUkM/RCxKy7+XIMmlJRGsiF6a7KGjPwQF6KUhVN3vkCl7LMBrO4qPa40uE9N5zVQ==
+  integrity sha512-o5za+lj+zXs4j7gCA3s3fYvvyiPqHiOOlsahEuXqi7FXwu3CUm1+SInxroG1U1kKeu3GbRaBzS+BFQ+xjg/K4w==
   dependencies:
     "@babel/plugin-proposal-class-properties" "^7.12.1"
     "@babel/plugin-proposal-decorators" "^7.12.1"
@@ -5135,25 +5135,25 @@ media-typer@0.3.0:
   resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748"
   integrity sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g=
 
-medusa-core-utils@1.1.30-dev-1638874121913:
+medusa-core-utils@1.1.31-dev-1639132284143:
-  version "1.1.30-dev-1638874121913"
+  version "1.1.31-dev-1639132284143"
-  resolved "http://localhost:4873/medusa-core-utils/-/medusa-core-utils-1.1.30-dev-1638874121913.tgz#72e5b0694fecbc83a61c58af212d993bb7b2104a"
+  resolved "http://localhost:4873/medusa-core-utils/-/medusa-core-utils-1.1.31-dev-1639132284143.tgz#c0085cf2ec65ca9cc7500d02b1f61f8e86e95bec"
-  integrity sha512-ldQ7WXJTdaGzUMVsF8LJqr5uLCJT0ZYUUTYUGazCgj30kULSUV3/VZm4+B7a5D7UoqJMexBqG+qmRDzCZ74Drg==
+  integrity sha512-l5+3ozll9BMwmPDpQlv4zqdlXnQbiy6iW5XQkykRoNiF6pGeq12ZVCoBop4TE9GKYck6UmRBbVF/kawq6nptoQ==
   dependencies:
     joi "^17.3.0"
     joi-objectid "^3.0.1"
 
-medusa-interfaces@1.1.31-dev-1638874121913:
+medusa-interfaces@1.1.32-dev-1639132284143:
-  version "1.1.31-dev-1638874121913"
+  version "1.1.32-dev-1639132284143"
-  resolved "http://localhost:4873/medusa-interfaces/-/medusa-interfaces-1.1.31-dev-1638874121913.tgz#0ee6c3efe637eff3364b1bd69419001a8db14422"
+  resolved "http://localhost:4873/medusa-interfaces/-/medusa-interfaces-1.1.32-dev-1639132284143.tgz#a220cba78f2f9a25f711c968893a53c4ad1da0ae"
-  integrity sha512-NUzBuml4/mxSpx5BBgkQVDNqItdrYlXqqNjwhq5GjHgqkCDVrQRJA5GGcQJ/17NbRSr6KKVEJuKkWlFhOgadkQ==
+  integrity sha512-/AMaqvbTVaTlAFZ48lSpBREG2s3mjFzPb3B4pr2R3x1fRfEYxMuTNXoUbFX8ubrIlKMwpB61Efon9tXouE5Cog==
   dependencies:
-    medusa-core-utils "1.1.30-dev-1638874121913"
+    medusa-core-utils "1.1.31-dev-1639132284143"
 
-medusa-telemetry@0.0.10-dev-1638874121913:
+medusa-telemetry@0.0.11-dev-1639132284143:
-  version "0.0.10-dev-1638874121913"
+  version "0.0.11-dev-1639132284143"
-  resolved "http://localhost:4873/medusa-telemetry/-/medusa-telemetry-0.0.10-dev-1638874121913.tgz#b8e58daa306dfeffbf28bdadca12ac852beb7912"
+  resolved "http://localhost:4873/medusa-telemetry/-/medusa-telemetry-0.0.11-dev-1639132284143.tgz#39e32d1ef44aa1ec6a7007c27ed8a52d8786193b"
-  integrity sha512-3V/Um1B2NzmztGttGFNSFzT0ZPyGg6vsxZc1M4XTDQfB1YeioQYImQiGw62mIpnKxTyRLWpQfkaiRIK9ILNL+Q==
+  integrity sha512-6grC1gkKskrvtlHXrWBobZKsKnjJBSB94JzY7U3Sv9pD1FG3rY/100u4qixkIve9ileICzYSFLS6KcTXbDG31A==
   dependencies:
     axios "^0.21.1"
     axios-retry "^3.1.9"
@@ -5165,13 +5165,13 @@ medusa-telemetry@0.0.10-dev-1638874121913:
     remove-trailing-slash "^0.1.1"
     uuid "^8.3.2"
 
-medusa-test-utils@1.1.33-dev-1638874121913:
+medusa-test-utils@1.1.34-dev-1639132284143:
-  version "1.1.33-dev-1638874121913"
+  version "1.1.34-dev-1639132284143"
-  resolved "http://localhost:4873/medusa-test-utils/-/medusa-test-utils-1.1.33-dev-1638874121913.tgz#422aa74ceff6e09abf28c070ea941982544afbad"
+  resolved "http://localhost:4873/medusa-test-utils/-/medusa-test-utils-1.1.34-dev-1639132284143.tgz#8495b6e3e86ce9cf23074ce46dea06b357b3d1eb"
-  integrity sha512-I7Z0GCdPQWMwygffjKw9CnGkFeudA/TCmxNTkWna+4WOlK3FFoilAi/T6Ps3q4ZDXkJ1O+vNG9yBG9zAldGPmw==
+  integrity sha512-pstRNkau48cJodUBRSbXoW/gzbOsUXaFW+grBPNzgnSD0ap7KKSM0jBGfPFgFCnBpnu5m7MBtYZBu3Ajp/8Enw==
   dependencies:
     "@babel/plugin-transform-classes" "^7.9.5"
-    medusa-core-utils "1.1.30-dev-1638874121913"
+    medusa-core-utils "1.1.31-dev-1639132284143"
     randomatic "^3.1.1"
 
 merge-descriptors@1.0.1:

integration-tests/globalSetup.js

@@ -0,0 +1,5 @@
+const dbFactory = require("./helpers/use-template-db")
+
+module.exports = async () => {
+  await dbFactory.createTemplateDb_()
+}

integration-tests/globalTeardown.js

@@ -0,0 +1,5 @@
+const dbFactory = require("./helpers/use-template-db")
+
+module.exports = async () => {
+  await dbFactory.destroy()
+}

integration-tests/helpers/setup-server.js

@@ -1,6 +1,5 @@
 const path = require("path")
 const { spawn } = require("child_process")
-
 const { setPort } = require("./use-api")
 
 module.exports = ({ cwd, verbose }) => {

integration-tests/helpers/use-db.js

@@ -1,12 +1,14 @@
 const path = require("path")
 require("dotenv").config({ path: path.join(__dirname, "../.env") })
 
-const { dropDatabase, createDatabase } = require("pg-god")
+const { dropDatabase } = require("pg-god")
 const { createConnection } = require("typeorm")
+const dbFactory = require("./use-template-db")
 
+const workerId = parseInt(process.env.JEST_WORKER_ID || "1")
 const DB_USERNAME = process.env.DB_USERNAME || "postgres"
 const DB_PASSWORD = process.env.DB_PASSWORD || ""
-const DB_URL = `postgres://${DB_USERNAME}:${DB_PASSWORD}@localhost/medusa-integration`
+const DB_URL = `postgres://${DB_USERNAME}:${DB_PASSWORD}@localhost/medusa-integration-${workerId}`
 
 const pgGodCredentials = {
   user: DB_USERNAME,
@@ -61,7 +63,7 @@ const DbTestUtil = {
 
   shutdown: async function () {
     await this.db_.close()
-    const databaseName = "medusa-integration"
+    const databaseName = `medusa-integration-${workerId}`
     return await dropDatabase({ databaseName }, pgGodCredentials)
   },
 }
@@ -96,28 +98,9 @@ module.exports = {
       instance.setDb(dbConnection)
       return dbConnection
     } else {
-      const migrationDir = path.resolve(
+      const databaseName = `medusa-integration-${workerId}`
-        path.join(
-          cwd,
-          `node_modules`,
-          `@medusajs`,
-          `medusa`,
-          `dist`,
-          `migrations`
-        )
-      )
 
-      const databaseName = "medusa-integration"
+      await dbFactory.createFromTemplate(databaseName)
-      await createDatabase({ databaseName }, pgGodCredentials)
-
-      const connection = await createConnection({
-        type: "postgres",
-        url: DB_URL,
-        migrations: [`${migrationDir}/*.js`],
-      })
-
-      await connection.runMigrations()
-      await connection.close()
 
       const dbConnection = await createConnection({
         type: "postgres",

integration-tests/helpers/use-template-db.js

@@ -0,0 +1,104 @@
+const path = require("path")
+require("dotenv").config({ path: path.join(__dirname, "../.env") })
+
+const { createDatabase, dropDatabase } = require("pg-god")
+const { createConnection, getConnection } = require("typeorm")
+
+const DB_USERNAME = process.env.DB_USERNAME || "postgres"
+const DB_PASSWORD = process.env.DB_PASSWORD || ""
+const DB_URL = `postgres://${DB_USERNAME}:${DB_PASSWORD}@localhost`
+
+const pgGodCredentials = {
+  user: DB_USERNAME,
+  password: DB_PASSWORD,
+}
+
+class DatabaseFactory {
+  constructor() {
+    this.connection_ = null
+    this.masterConnectionName = "name"
+    this.templateDbName = "medusa-integration-template"
+  }
+
+  async createTemplateDb_() {
+    try {
+      const connection = await this.getMasterConnection()
+
+      const migrationDir = path.resolve(
+        path.join(
+          process.cwd(),
+          "integration-tests",
+          "api",
+          `node_modules`,
+          `@medusajs`,
+          `medusa`,
+          `dist`,
+          `migrations`
+        )
+      )
+
+      await dropDatabase(
+        {
+          databaseName: this.templateDbName,
+          errorIfNonExist: false,
+        },
+        pgGodCredentials
+      )
+      await createDatabase(
+        { databaseName: this.templateDbName },
+        pgGodCredentials
+      )
+
+      const templateDbConnection = await createConnection({
+        type: "postgres",
+        name: "templateConnection",
+        url: `${DB_URL}/${this.templateDbName}`,
+        migrations: [`${migrationDir}/*.js`],
+      })
+
+      await templateDbConnection.runMigrations()
+      await templateDbConnection.close()
+
+      return connection
+    } catch (err) {
+      console.log("error in createTemplateDb_")
+      console.log(err)
+    }
+  }
+
+  async getMasterConnection() {
+    try {
+      return await getConnection(this.masterConnectionName)
+    } catch (err) {
+      return await this.createMasterConnection()
+    }
+  }
+
+  async createMasterConnection() {
+    const connection = await createConnection({
+      type: "postgres",
+      name: this.masterConnectionName,
+      url: `${DB_URL}`,
+    })
+
+    return connection
+  }
+
+  async createFromTemplate(dbName) {
+    const connection = await this.getMasterConnection()
+
+    await connection.query(`DROP DATABASE IF EXISTS "${dbName}";`)
+    await connection.query(
+      `CREATE DATABASE "${dbName}" TEMPLATE "${this.templateDbName}";`
+    )
+  }
+
+  async destroy() {
+    let connection = await this.getMasterConnection()
+
+    await connection.query(`DROP DATABASE IF EXISTS "${this.templateDbName}";`)
+    await connection.close()
+  }
+}
+
+module.exports = new DatabaseFactory()

integration-tests/jest.config.js

@@ -6,6 +6,8 @@ const pkgs = glob
 
 module.exports = {
   testEnvironment: `node`,
+  globalSetup: "<rootDir>/integration-tests/globalSetup.js",
+  globalTeardown: "<rootDir>/integration-tests/globalTeardown.js",
   rootDir: `../`,
   roots: pkgs,
   testPathIgnorePatterns: [

integration-tests/setup.js

@@ -12,5 +12,9 @@ const pgGodCredentials = {
 }
 
 afterAll(() => {
-  dropDatabase({ databaseName: "medusa-integration" }, pgGodCredentials)
+  const workerId = parseInt(process.env.JEST_WORKER_ID || "1")
+  dropDatabase(
+    { databaseName: `medusa-integration-${workerId}` },
+    pgGodCredentials
+  )
 })

package.json

@@ -51,7 +51,7 @@
     "jest": "jest",
     "test": "jest",
     "prettier": "prettier",
-    "test:integration": "jest --config=integration-tests/jest.config.js --runInBand",
+    "test:integration": "jest --config=integration-tests/jest.config.js",
     "test:fixtures": "jest --config=docs-util/jest.config.js --runInBand"
   },
   "dependencies": {