From 69620e572e686a5be19d42bdbf798c22ccbf4e57 Mon Sep 17 00:00:00 2001 From: Zakaria El Asri <33696020+zakariaelas@users.noreply.github.com> Date: Tue, 14 Dec 2021 21:45:57 +0100 Subject: [PATCH] chore: parallelize integration tests and use template dbs (#907) --- .github/workflows/action.yml | 3 +- integration-tests/api/medusa-config.js | 3 +- integration-tests/api/package.json | 8 +- integration-tests/api/yarn.lock | 70 ++++++------- integration-tests/globalSetup.js | 5 + integration-tests/globalTeardown.js | 5 + integration-tests/helpers/setup-server.js | 1 - integration-tests/helpers/use-db.js | 31 ++---- integration-tests/helpers/use-template-db.js | 104 +++++++++++++++++++ integration-tests/jest.config.js | 2 + integration-tests/setup.js | 6 +- package.json | 2 +- 12 files changed, 171 insertions(+), 69 deletions(-) create mode 100644 integration-tests/globalSetup.js create mode 100644 integration-tests/globalTeardown.js create mode 100644 integration-tests/helpers/use-template-db.js diff --git a/.github/workflows/action.yml b/.github/workflows/action.yml index 988fea3097..1a12797ef7 100644 --- a/.github/workflows/action.yml +++ b/.github/workflows/action.yml @@ -86,7 +86,6 @@ jobs: working-directory: integration-tests/api - name: Run integration tests - run: yarn test - working-directory: integration-tests/api + run: yarn test:integration --maxWorkers=50% --silent=false env: DB_PASSWORD: postgres diff --git a/integration-tests/api/medusa-config.js b/integration-tests/api/medusa-config.js index 9c7ac0424e..f4b027d81b 100644 --- a/integration-tests/api/medusa-config.js +++ b/integration-tests/api/medusa-config.js @@ -1,11 +1,12 @@ const DB_USERNAME = process.env.DB_USERNAME || "postgres" const DB_PASSWORD = process.env.DB_PASSWORD || "" +const workerId = parseInt(process.env.JEST_WORKER_ID || "1") module.exports = { plugins: [], projectConfig: { // redis_url: REDIS_URL, - database_url: `postgres://${DB_USERNAME}:${DB_PASSWORD}@localhost/medusa-integration`, + database_url: `postgres://${DB_USERNAME}:${DB_PASSWORD}@localhost/medusa-integration-${workerId}`, database_type: "postgres", }, } diff --git a/integration-tests/api/package.json b/integration-tests/api/package.json index dfd998c940..74f98107a1 100644 --- a/integration-tests/api/package.json +++ b/integration-tests/api/package.json @@ -4,19 +4,19 @@ "main": "index.js", "license": "MIT", "scripts": { - "test": "jest --runInBand --silent=false", + "test": "jest --maxWorkers=50% --silent=false", "build": "babel src -d dist --extensions \".ts,.js\"" }, "dependencies": { - "@medusajs/medusa": "1.1.57-dev-1638874121913", - "medusa-interfaces": "1.1.31-dev-1638874121913", + "@medusajs/medusa": "1.1.59-dev-1639132284143", + "medusa-interfaces": "1.1.32-dev-1639132284143", "typeorm": "^0.2.31" }, "devDependencies": { "@babel/cli": "^7.12.10", "@babel/core": "^7.12.10", "@babel/node": "^7.12.10", - "babel-preset-medusa-package": "1.1.18-dev-1638874121913", + "babel-preset-medusa-package": "1.1.19-dev-1639132284143", "jest": "^26.6.3" } } diff --git a/integration-tests/api/yarn.lock b/integration-tests/api/yarn.lock index 9a9f50ebab..9420b98d3d 100644 --- a/integration-tests/api/yarn.lock +++ b/integration-tests/api/yarn.lock @@ -1256,10 +1256,10 @@ "@types/yargs" "^15.0.0" chalk "^4.0.0" -"@medusajs/medusa-cli@1.1.23-dev-1638874121913": - version "1.1.23-dev-1638874121913" - resolved "http://localhost:4873/@medusajs%2fmedusa-cli/-/medusa-cli-1.1.23-dev-1638874121913.tgz#6f12e4f54c08f4846ce3c213fb3407833b1febec" - integrity sha512-ZhO3UhIdyjKOoqjfWn09wmV7Vk2A52qOE3JwLvJkyiPVt3tO23/2H7DHF93faYtrKjBCo32xSLBuZYqsRm6p+w== +"@medusajs/medusa-cli@1.1.24-dev-1639132284143": + version "1.1.24-dev-1639132284143" + resolved "http://localhost:4873/@medusajs%2fmedusa-cli/-/medusa-cli-1.1.24-dev-1639132284143.tgz#80eeb1f732db447797d8c1c141fc2590a00f89e1" + integrity sha512-IEEq9iJpTHeXAHm8KbGepYVO8xxntlrviCuGfh3OCMBzTAgnQJ/CDvRhAgb4fRGxJZ0uwLJLtQG/7ePCPAotOw== dependencies: "@babel/polyfill" "^7.8.7" "@babel/runtime" "^7.9.6" @@ -1277,8 +1277,8 @@ is-valid-path "^0.1.1" joi-objectid "^3.0.1" meant "^1.0.1" - medusa-core-utils "1.1.30-dev-1638874121913" - medusa-telemetry "0.0.10-dev-1638874121913" + medusa-core-utils "1.1.31-dev-1639132284143" + medusa-telemetry "0.0.11-dev-1639132284143" netrc-parser "^3.1.6" open "^8.0.6" ora "^5.4.1" @@ -1292,13 +1292,13 @@ winston "^3.3.3" yargs "^15.3.1" -"@medusajs/medusa@1.1.57-dev-1638874121913": - version "1.1.57-dev-1638874121913" - resolved "http://localhost:4873/@medusajs%2fmedusa/-/medusa-1.1.57-dev-1638874121913.tgz#c4a40d588a2567a6f80a84b8017dd64382e224bc" - integrity sha512-WSTL4U3TnB7u8YjaqaEnfjQQCmQMpZIqAwRA2+1SuaRrW3A3SYbxoRkL/3bgH13KGv/PCBAvkV1e7l9xcziAnQ== +"@medusajs/medusa@1.1.59-dev-1639132284143": + version "1.1.59-dev-1639132284143" + resolved "http://localhost:4873/@medusajs%2fmedusa/-/medusa-1.1.59-dev-1639132284143.tgz#9cb56910e415e7f8654e94aea0468b960c2ffb12" + integrity sha512-uxKIJLKMwOR/UbuLFW9IvdReODvHace5DVWbTMIK1g031PBX3/ghto0yIbcZVaNdMLETS5SiEwPHkx0QAw2k2A== dependencies: "@hapi/joi" "^16.1.8" - "@medusajs/medusa-cli" "1.1.23-dev-1638874121913" + "@medusajs/medusa-cli" "1.1.24-dev-1639132284143" "@types/lodash" "^4.14.168" awilix "^4.2.3" body-parser "^1.19.0" @@ -1322,8 +1322,8 @@ joi "^17.3.0" joi-objectid "^3.0.1" jsonwebtoken "^8.5.1" - medusa-core-utils "1.1.30-dev-1638874121913" - medusa-test-utils "1.1.33-dev-1638874121913" + medusa-core-utils "1.1.31-dev-1639132284143" + medusa-test-utils "1.1.34-dev-1639132284143" morgan "^1.9.1" multer "^1.4.2" passport "^0.4.0" @@ -1947,10 +1947,10 @@ babel-preset-jest@^26.6.2: babel-plugin-jest-hoist "^26.6.2" babel-preset-current-node-syntax "^1.0.0" -babel-preset-medusa-package@1.1.18-dev-1638874121913: - version "1.1.18-dev-1638874121913" - resolved "http://localhost:4873/babel-preset-medusa-package/-/babel-preset-medusa-package-1.1.18-dev-1638874121913.tgz#d50b513bbbfe691318c9b9463c7ec0c0c1ca8711" - integrity sha512-FDJyHCp+FyncJDlapHIwxHRUkM/RCxKy7+XIMmlJRGsiF6a7KGjPwQF6KUhVN3vkCl7LMBrO4qPa40uE9N5zVQ== +babel-preset-medusa-package@1.1.19-dev-1639132284143: + version "1.1.19-dev-1639132284143" + resolved "http://localhost:4873/babel-preset-medusa-package/-/babel-preset-medusa-package-1.1.19-dev-1639132284143.tgz#a98da041ddda18db99244ae79bf415c1c674f302" + integrity sha512-o5za+lj+zXs4j7gCA3s3fYvvyiPqHiOOlsahEuXqi7FXwu3CUm1+SInxroG1U1kKeu3GbRaBzS+BFQ+xjg/K4w== dependencies: "@babel/plugin-proposal-class-properties" "^7.12.1" "@babel/plugin-proposal-decorators" "^7.12.1" @@ -5135,25 +5135,25 @@ media-typer@0.3.0: resolved "https://registry.yarnpkg.com/media-typer/-/media-typer-0.3.0.tgz#8710d7af0aa626f8fffa1ce00168545263255748" integrity sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g= -medusa-core-utils@1.1.30-dev-1638874121913: - version "1.1.30-dev-1638874121913" - resolved "http://localhost:4873/medusa-core-utils/-/medusa-core-utils-1.1.30-dev-1638874121913.tgz#72e5b0694fecbc83a61c58af212d993bb7b2104a" - integrity sha512-ldQ7WXJTdaGzUMVsF8LJqr5uLCJT0ZYUUTYUGazCgj30kULSUV3/VZm4+B7a5D7UoqJMexBqG+qmRDzCZ74Drg== +medusa-core-utils@1.1.31-dev-1639132284143: + version "1.1.31-dev-1639132284143" + resolved "http://localhost:4873/medusa-core-utils/-/medusa-core-utils-1.1.31-dev-1639132284143.tgz#c0085cf2ec65ca9cc7500d02b1f61f8e86e95bec" + integrity sha512-l5+3ozll9BMwmPDpQlv4zqdlXnQbiy6iW5XQkykRoNiF6pGeq12ZVCoBop4TE9GKYck6UmRBbVF/kawq6nptoQ== dependencies: joi "^17.3.0" joi-objectid "^3.0.1" -medusa-interfaces@1.1.31-dev-1638874121913: - version "1.1.31-dev-1638874121913" - resolved "http://localhost:4873/medusa-interfaces/-/medusa-interfaces-1.1.31-dev-1638874121913.tgz#0ee6c3efe637eff3364b1bd69419001a8db14422" - integrity sha512-NUzBuml4/mxSpx5BBgkQVDNqItdrYlXqqNjwhq5GjHgqkCDVrQRJA5GGcQJ/17NbRSr6KKVEJuKkWlFhOgadkQ== +medusa-interfaces@1.1.32-dev-1639132284143: + version "1.1.32-dev-1639132284143" + resolved "http://localhost:4873/medusa-interfaces/-/medusa-interfaces-1.1.32-dev-1639132284143.tgz#a220cba78f2f9a25f711c968893a53c4ad1da0ae" + integrity sha512-/AMaqvbTVaTlAFZ48lSpBREG2s3mjFzPb3B4pr2R3x1fRfEYxMuTNXoUbFX8ubrIlKMwpB61Efon9tXouE5Cog== dependencies: - medusa-core-utils "1.1.30-dev-1638874121913" + medusa-core-utils "1.1.31-dev-1639132284143" -medusa-telemetry@0.0.10-dev-1638874121913: - version "0.0.10-dev-1638874121913" - resolved "http://localhost:4873/medusa-telemetry/-/medusa-telemetry-0.0.10-dev-1638874121913.tgz#b8e58daa306dfeffbf28bdadca12ac852beb7912" - integrity sha512-3V/Um1B2NzmztGttGFNSFzT0ZPyGg6vsxZc1M4XTDQfB1YeioQYImQiGw62mIpnKxTyRLWpQfkaiRIK9ILNL+Q== +medusa-telemetry@0.0.11-dev-1639132284143: + version "0.0.11-dev-1639132284143" + resolved "http://localhost:4873/medusa-telemetry/-/medusa-telemetry-0.0.11-dev-1639132284143.tgz#39e32d1ef44aa1ec6a7007c27ed8a52d8786193b" + integrity sha512-6grC1gkKskrvtlHXrWBobZKsKnjJBSB94JzY7U3Sv9pD1FG3rY/100u4qixkIve9ileICzYSFLS6KcTXbDG31A== dependencies: axios "^0.21.1" axios-retry "^3.1.9" @@ -5165,13 +5165,13 @@ medusa-telemetry@0.0.10-dev-1638874121913: remove-trailing-slash "^0.1.1" uuid "^8.3.2" -medusa-test-utils@1.1.33-dev-1638874121913: - version "1.1.33-dev-1638874121913" - resolved "http://localhost:4873/medusa-test-utils/-/medusa-test-utils-1.1.33-dev-1638874121913.tgz#422aa74ceff6e09abf28c070ea941982544afbad" - integrity sha512-I7Z0GCdPQWMwygffjKw9CnGkFeudA/TCmxNTkWna+4WOlK3FFoilAi/T6Ps3q4ZDXkJ1O+vNG9yBG9zAldGPmw== +medusa-test-utils@1.1.34-dev-1639132284143: + version "1.1.34-dev-1639132284143" + resolved "http://localhost:4873/medusa-test-utils/-/medusa-test-utils-1.1.34-dev-1639132284143.tgz#8495b6e3e86ce9cf23074ce46dea06b357b3d1eb" + integrity sha512-pstRNkau48cJodUBRSbXoW/gzbOsUXaFW+grBPNzgnSD0ap7KKSM0jBGfPFgFCnBpnu5m7MBtYZBu3Ajp/8Enw== dependencies: "@babel/plugin-transform-classes" "^7.9.5" - medusa-core-utils "1.1.30-dev-1638874121913" + medusa-core-utils "1.1.31-dev-1639132284143" randomatic "^3.1.1" merge-descriptors@1.0.1: diff --git a/integration-tests/globalSetup.js b/integration-tests/globalSetup.js new file mode 100644 index 0000000000..947a9e9cae --- /dev/null +++ b/integration-tests/globalSetup.js @@ -0,0 +1,5 @@ +const dbFactory = require("./helpers/use-template-db") + +module.exports = async () => { + await dbFactory.createTemplateDb_() +} diff --git a/integration-tests/globalTeardown.js b/integration-tests/globalTeardown.js new file mode 100644 index 0000000000..7245b52f91 --- /dev/null +++ b/integration-tests/globalTeardown.js @@ -0,0 +1,5 @@ +const dbFactory = require("./helpers/use-template-db") + +module.exports = async () => { + await dbFactory.destroy() +} diff --git a/integration-tests/helpers/setup-server.js b/integration-tests/helpers/setup-server.js index f10c772e5c..105b90c13a 100644 --- a/integration-tests/helpers/setup-server.js +++ b/integration-tests/helpers/setup-server.js @@ -1,6 +1,5 @@ const path = require("path") const { spawn } = require("child_process") - const { setPort } = require("./use-api") module.exports = ({ cwd, verbose }) => { diff --git a/integration-tests/helpers/use-db.js b/integration-tests/helpers/use-db.js index 5526034755..10e70c9783 100644 --- a/integration-tests/helpers/use-db.js +++ b/integration-tests/helpers/use-db.js @@ -1,12 +1,14 @@ const path = require("path") require("dotenv").config({ path: path.join(__dirname, "../.env") }) -const { dropDatabase, createDatabase } = require("pg-god") +const { dropDatabase } = require("pg-god") const { createConnection } = require("typeorm") +const dbFactory = require("./use-template-db") +const workerId = parseInt(process.env.JEST_WORKER_ID || "1") const DB_USERNAME = process.env.DB_USERNAME || "postgres" const DB_PASSWORD = process.env.DB_PASSWORD || "" -const DB_URL = `postgres://${DB_USERNAME}:${DB_PASSWORD}@localhost/medusa-integration` +const DB_URL = `postgres://${DB_USERNAME}:${DB_PASSWORD}@localhost/medusa-integration-${workerId}` const pgGodCredentials = { user: DB_USERNAME, @@ -61,7 +63,7 @@ const DbTestUtil = { shutdown: async function () { await this.db_.close() - const databaseName = "medusa-integration" + const databaseName = `medusa-integration-${workerId}` return await dropDatabase({ databaseName }, pgGodCredentials) }, } @@ -96,28 +98,9 @@ module.exports = { instance.setDb(dbConnection) return dbConnection } else { - const migrationDir = path.resolve( - path.join( - cwd, - `node_modules`, - `@medusajs`, - `medusa`, - `dist`, - `migrations` - ) - ) + const databaseName = `medusa-integration-${workerId}` - const databaseName = "medusa-integration" - await createDatabase({ databaseName }, pgGodCredentials) - - const connection = await createConnection({ - type: "postgres", - url: DB_URL, - migrations: [`${migrationDir}/*.js`], - }) - - await connection.runMigrations() - await connection.close() + await dbFactory.createFromTemplate(databaseName) const dbConnection = await createConnection({ type: "postgres", diff --git a/integration-tests/helpers/use-template-db.js b/integration-tests/helpers/use-template-db.js new file mode 100644 index 0000000000..32b2699aa7 --- /dev/null +++ b/integration-tests/helpers/use-template-db.js @@ -0,0 +1,104 @@ +const path = require("path") +require("dotenv").config({ path: path.join(__dirname, "../.env") }) + +const { createDatabase, dropDatabase } = require("pg-god") +const { createConnection, getConnection } = require("typeorm") + +const DB_USERNAME = process.env.DB_USERNAME || "postgres" +const DB_PASSWORD = process.env.DB_PASSWORD || "" +const DB_URL = `postgres://${DB_USERNAME}:${DB_PASSWORD}@localhost` + +const pgGodCredentials = { + user: DB_USERNAME, + password: DB_PASSWORD, +} + +class DatabaseFactory { + constructor() { + this.connection_ = null + this.masterConnectionName = "name" + this.templateDbName = "medusa-integration-template" + } + + async createTemplateDb_() { + try { + const connection = await this.getMasterConnection() + + const migrationDir = path.resolve( + path.join( + process.cwd(), + "integration-tests", + "api", + `node_modules`, + `@medusajs`, + `medusa`, + `dist`, + `migrations` + ) + ) + + await dropDatabase( + { + databaseName: this.templateDbName, + errorIfNonExist: false, + }, + pgGodCredentials + ) + await createDatabase( + { databaseName: this.templateDbName }, + pgGodCredentials + ) + + const templateDbConnection = await createConnection({ + type: "postgres", + name: "templateConnection", + url: `${DB_URL}/${this.templateDbName}`, + migrations: [`${migrationDir}/*.js`], + }) + + await templateDbConnection.runMigrations() + await templateDbConnection.close() + + return connection + } catch (err) { + console.log("error in createTemplateDb_") + console.log(err) + } + } + + async getMasterConnection() { + try { + return await getConnection(this.masterConnectionName) + } catch (err) { + return await this.createMasterConnection() + } + } + + async createMasterConnection() { + const connection = await createConnection({ + type: "postgres", + name: this.masterConnectionName, + url: `${DB_URL}`, + }) + + return connection + } + + async createFromTemplate(dbName) { + const connection = await this.getMasterConnection() + + await connection.query(`DROP DATABASE IF EXISTS "${dbName}";`) + await connection.query( + `CREATE DATABASE "${dbName}" TEMPLATE "${this.templateDbName}";` + ) + } + + async destroy() { + let connection = await this.getMasterConnection() + + await connection.query(`DROP DATABASE IF EXISTS "${this.templateDbName}";`) + await connection.close() + } +} + +module.exports = new DatabaseFactory() diff --git a/integration-tests/jest.config.js b/integration-tests/jest.config.js index 83f803a68e..1063e4faf1 100644 --- a/integration-tests/jest.config.js +++ b/integration-tests/jest.config.js @@ -6,6 +6,8 @@ const pkgs = glob module.exports = { testEnvironment: `node`, + globalSetup: "/integration-tests/globalSetup.js", + globalTeardown: "/integration-tests/globalTeardown.js", rootDir: `../`, roots: pkgs, testPathIgnorePatterns: [ diff --git a/integration-tests/setup.js b/integration-tests/setup.js index 3f689687b9..422d6f58c9 100644 --- a/integration-tests/setup.js +++ b/integration-tests/setup.js @@ -12,5 +12,9 @@ const pgGodCredentials = { } afterAll(() => { - dropDatabase({ databaseName: "medusa-integration" }, pgGodCredentials) + const workerId = parseInt(process.env.JEST_WORKER_ID || "1") + dropDatabase( + { databaseName: `medusa-integration-${workerId}` }, + pgGodCredentials + ) }) diff --git a/package.json b/package.json index 527ba8c00e..0d60086c7d 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "jest": "jest", "test": "jest", "prettier": "prettier", - "test:integration": "jest --config=integration-tests/jest.config.js --runInBand", + "test:integration": "jest --config=integration-tests/jest.config.js", "test:fixtures": "jest --config=docs-util/jest.config.js --runInBand" }, "dependencies": {