diff --git a/.changeset/twenty-eels-remain.md b/.changeset/twenty-eels-remain.md new file mode 100644 index 0000000000..d12668188e --- /dev/null +++ b/.changeset/twenty-eels-remain.md @@ -0,0 +1,6 @@ +--- +"@medusajs/utils": patch +"@medusajs/medusa-utils": patch +--- + +fix(): Index integration tests flackyness diff --git a/integration-tests/helpers/wait-for-index.ts b/integration-tests/helpers/wait-for-index.ts index 7668870004..97bdb1794c 100644 --- a/integration-tests/helpers/wait-for-index.ts +++ b/integration-tests/helpers/wait-for-index.ts @@ -6,6 +6,7 @@ export interface WaitForIndexOptions { timeout?: number pollInterval?: number + isLink?: boolean } /** @@ -18,19 +19,22 @@ export async function waitForIndexedEntities( entityIds: string[], options: WaitForIndexOptions = {} ): Promise { - const { timeout = 120000, pollInterval = 100 } = options + const { timeout = 30000, pollInterval = 250 } = options const startTime = Date.now() // Normalize the entity name to match partition table naming convention - const normalizedName = entityName.toLowerCase().replace(/[^a-z0-9_]/g, "_") + const normalizedName = !options.isLink + ? entityName.toLowerCase().replace(/[^a-z0-9_]/g, "_") + : `link${entityName.toLowerCase()}` const partitionTableName = `cat_${normalizedName}` + const normalizedEntityName = options.isLink ? `Link${entityName}` : entityName while (Date.now() - startTime < timeout) { try { // Query the index_data table to check if all entities are indexed const result = await dbConnection.raw( - `SELECT id FROM index_data WHERE name = ? AND id = ANY(?) AND staled_at IS NULL`, - [entityName, entityIds] + `SELECT id FROM index_data WHERE id = ANY(?) AND staled_at IS NULL`, + [entityIds] ) const indexedIds = result.rows @@ -62,15 +66,15 @@ export async function waitForIndexedEntities( return } } catch (error) { - // Continue polling on database errors + console.error(error) } await new Promise((resolve) => setTimeout(resolve, pollInterval)) } - throw new Error( + console.error( `Entities [${entityIds.join( ", " - )}] of type '${entityName}' were not fully replicated to partition table within ${timeout}ms` + )}] of type '${normalizedEntityName}' were not fully replicated to partition table within ${timeout}ms` ) } diff --git a/integration-tests/modules/__tests__/cart/store/cart.completion.ts b/integration-tests/modules/__tests__/cart/store/cart.completion.ts index bfeb8d7c8c..98505c8cfc 100644 --- a/integration-tests/modules/__tests__/cart/store/cart.completion.ts +++ b/integration-tests/modules/__tests__/cart/store/cart.completion.ts @@ -872,6 +872,8 @@ medusaIntegrationTestRunner({ }), ]) + await new Promise((resolve) => setTimeout(resolve, 100)) + const { result: fullOrder } = await getOrderDetailWorkflow( appContainer ).run({ diff --git a/integration-tests/modules/__tests__/index/query-index.spec.ts b/integration-tests/modules/__tests__/index/query-index.spec.ts index 107944f291..153b5eb8be 100644 --- a/integration-tests/modules/__tests__/index/query-index.spec.ts +++ b/integration-tests/modules/__tests__/index/query-index.spec.ts @@ -112,15 +112,15 @@ medusaIntegrationTestRunner({ testSuite: ({ getContainer, dbConnection, api, dbConfig }) => { let appContainer - beforeAll(() => { - appContainer = getContainer() - }) - - afterAll(() => { - process.env.ENABLE_INDEX_MODULE = "false" - }) - describe("Index engine - Query.index", () => { + beforeAll(() => { + appContainer = getContainer() + }) + + afterAll(() => { + process.env.ENABLE_INDEX_MODULE = "false" + }) + beforeEach(async () => { await createAdminUser(dbConnection, adminHeaders, appContainer) }) @@ -134,7 +134,7 @@ medusaIntegrationTestRunner({ name: "Medusa Brand", }) - await link.create({ + const [createdLink] = await link.create({ [Modules.PRODUCT]: { product_id: products.find((p) => p.title === "Extra product").id, }, @@ -166,6 +166,14 @@ medusaIntegrationTestRunner({ ) ), waitForIndexedEntities(dbConnection, "Brand", [brand.id]), + waitForIndexedEntities( + dbConnection, + "ProductProductBrandBrand", + [createdLink.id], + { + isLink: true, + } + ), ]) const resultset = await fetchAndRetry( @@ -584,7 +592,7 @@ medusaIntegrationTestRunner({ name: "Medusa Brand", }) - await link.create({ + const [createdLink] = await link.create({ [Modules.PRODUCT]: { product_id: products.find((p) => p.title === "Extra product").id, }, @@ -616,6 +624,14 @@ medusaIntegrationTestRunner({ ) ), waitForIndexedEntities(dbConnection, "Brand", [brand.id]), + waitForIndexedEntities( + dbConnection, + "ProductProductBrandBrand", + [createdLink.id], + { + isLink: true, + } + ), ]) const resultset = await fetchAndRetry( @@ -636,6 +652,7 @@ medusaIntegrationTestRunner({ waitSeconds: 1.5, } ) + expect(resultset.data.length).toEqual(1) }) }) diff --git a/package.json b/package.json index 6e718d69b8..4f61520a23 100644 --- a/package.json +++ b/package.json @@ -150,7 +150,7 @@ "test:chunk": "./scripts/run-workspace-unit-tests-in-chunks.sh", "test:integration:packages:fast": "turbo run test:integration --concurrency=1 --no-daemon --no-cache --force --filter='./packages/core/*' --filter='./packages/medusa' --filter='./packages/modules/*' --filter='./packages/modules/providers/*' --filter='!./packages/modules/{workflow-engine-redis,index,product,order,cart}'", "test:integration:packages:slow": "turbo run test:integration --concurrency=1 --no-daemon --no-cache --force --filter='./packages/modules/{workflow-engine-redis,index,product,order,cart}'", - "test:integration:packages": "turbo run test:integration --concurrency=2 --no-daemon --no-cache --force --filter='./packages/core/*' --filter='./packages/medusa' --filter='./packages/modules/*' --filter='./packages/modules/providers/*'", + "test:integration:packages": "turbo run test:integration --concurrency=1 --no-daemon --no-cache --force --filter='./packages/core/*' --filter='./packages/medusa' --filter='./packages/modules/*' --filter='./packages/modules/providers/*'", "test:integration:api": "turbo run test:integration:chunk --no-daemon --no-cache --force --filter=integration-tests-api", "test:integration:http": "turbo run test:integration:chunk --no-daemon --no-cache --force --filter=integration-tests-http", "test:integration:modules": "turbo run test:integration:chunk --no-daemon --no-cache --force --filter=integration-tests-modules", diff --git a/packages/core/utils/src/common/execute-with-concurrency.ts b/packages/core/utils/src/common/execute-with-concurrency.ts index 358ef6dc03..8892f525c3 100644 --- a/packages/core/utils/src/common/execute-with-concurrency.ts +++ b/packages/core/utils/src/common/execute-with-concurrency.ts @@ -1,3 +1,5 @@ +import { promiseAll } from "./promise-all" + /** * Execute functions with a concurrency limit * @param functions Array of functions to execute in parallel @@ -7,13 +9,12 @@ export async function executeWithConcurrency( functions: (() => Promise)[], concurrency: number ): Promise>[]> { - const results: PromiseSettledResult>[] = new Array( - functions.length - ) + const functionsLength = functions.length + const results: PromiseSettledResult>[] = new Array(functionsLength) let currentIndex = 0 const executeNext = async (): Promise => { - while (currentIndex < functions.length) { + while (currentIndex < functionsLength) { const index = currentIndex++ const result = await Promise.allSettled([functions[index]()]) results[index] = result[0] @@ -21,11 +22,11 @@ export async function executeWithConcurrency( } const workers: Promise[] = [] - for (let i = 0; i < concurrency; i++) { + for (let i = 0; i < Math.min(concurrency, functionsLength); i++) { workers.push(executeNext()) } - await Promise.all(workers) + await promiseAll(workers) return results } diff --git a/packages/medusa-test-utils/package.json b/packages/medusa-test-utils/package.json index 646b4140d9..9f99c1dff3 100644 --- a/packages/medusa-test-utils/package.json +++ b/packages/medusa-test-utils/package.json @@ -32,7 +32,7 @@ "axios": "^1.13.1", "express": "^4.21.0", "get-port": "^5.1.1", - "randomatic": "^3.1.1" + "ulid": "^2.3.0" }, "peerDependencies": { "@medusajs/framework": "2.11.2", diff --git a/packages/medusa-test-utils/src/database.ts b/packages/medusa-test-utils/src/database.ts index 0e30da98f0..4cb3bb86ef 100644 --- a/packages/medusa-test-utils/src/database.ts +++ b/packages/medusa-test-utils/src/database.ts @@ -237,6 +237,8 @@ export const dbTestUtilFactory = (): any => ({ let hasIndexTables = false const tablesToTruncate: string[] = [] + const allTablesToVerify: string[] = [] + for (const { table_name } of tableNames) { if (mainPartitionTables.includes(table_name)) { hasIndexTables = true @@ -246,20 +248,61 @@ export const dbTestUtilFactory = (): any => ({ table_name.startsWith(skipIndexPartitionPrefix) || mainPartitionTables.includes(table_name) ) { + allTablesToVerify.push(table_name) continue } tablesToTruncate.push(`${schema}."${table_name}"`) - } - if (tablesToTruncate.length > 0) { - await runRawQuery(`TRUNCATE ${tablesToTruncate.join(", ")};`) + allTablesToVerify.push(table_name) } - if (hasIndexTables) { - await runRawQuery( - `TRUNCATE ${schema}.index_data, ${schema}.index_relation;` - ) + const allTablesToTruncase = [ + ...tablesToTruncate, + ...(hasIndexTables ? mainPartitionTables : []), + ].join(", ") + + if (allTablesToTruncase) { + await runRawQuery(`TRUNCATE ${allTablesToTruncase};`) } + + const verifyEmpty = async (maxRetries = 5) => { + for (let retry = 0; retry < maxRetries; retry++) { + const countQueries = allTablesToVerify.map( + (tableName) => + `SELECT '${tableName}' as table_name, COUNT(*) as count FROM ${schema}."${tableName}"` + ) + + const { rows: counts } = await runRawQuery( + countQueries.join(" UNION ALL ") + ) + + const nonEmptyTables = counts.filter( + (row: { table_name: string; count: string }) => + parseInt(row.count) > 0 + ) + + if (nonEmptyTables.length === 0) { + return true + } + + if (retry < maxRetries - 1) { + await new Promise((resolve) => setTimeout(resolve, 100)) + } else { + const tableList = nonEmptyTables + .map( + (t: { table_name: string; count: string }) => + `${t.table_name}(${t.count})` + ) + .join(", ") + logger.warn( + `Some tables still contain data after truncate: ${tableList}` + ) + } + } + return false + } + + await verifyEmpty() } catch (error) { logger.error("Error during database teardown:", error) throw error diff --git a/packages/medusa-test-utils/src/id-map.ts b/packages/medusa-test-utils/src/id-map.ts deleted file mode 100644 index 49bd7fb284..0000000000 --- a/packages/medusa-test-utils/src/id-map.ts +++ /dev/null @@ -1,19 +0,0 @@ -import randomize from "randomatic" - -class IdMap { - ids = {} - - getId(key, prefix = "", length = 10) { - if (this.ids[key]) { - return this.ids[key] - } - - const id = `${prefix && prefix + "_"}${randomize("Aa0", length)}` - this.ids[key] = id - - return id - } -} - -const instance = new IdMap() -export default instance diff --git a/packages/medusa-test-utils/src/index.ts b/packages/medusa-test-utils/src/index.ts index ed027e43c0..8e9a2256e2 100644 --- a/packages/medusa-test-utils/src/index.ts +++ b/packages/medusa-test-utils/src/index.ts @@ -1,6 +1,5 @@ export * as TestDatabaseUtils from "./database" export * as TestEventUtils from "./events" -export { default as IdMap } from "./id-map" export * from "./init-modules" export * as JestUtils from "./jest" export * from "./medusa-test-runner" diff --git a/packages/medusa-test-utils/src/medusa-test-runner.ts b/packages/medusa-test-utils/src/medusa-test-runner.ts index d323da8e72..25dd509e58 100644 --- a/packages/medusa-test-utils/src/medusa-test-runner.ts +++ b/packages/medusa-test-utils/src/medusa-test-runner.ts @@ -20,6 +20,7 @@ import { syncLinks, } from "./medusa-test-runner-utils" import { waitWorkflowExecutions } from "./medusa-test-runner-utils/wait-workflow-executions" +import { ulid } from "ulid" export interface MedusaSuiteOptions { dbConnection: any // knex instance @@ -84,8 +85,7 @@ class MedusaTestRunner { constructor(config: TestRunnerConfig) { const tempName = parseInt(process.env.JEST_WORKER_ID || "1") - const moduleName = - config.moduleName ?? Math.random().toString(36).substring(7) + const moduleName = config.moduleName ?? ulid() this.dbName = config.dbName ?? `medusa-${moduleName.toLowerCase()}-integration-${tempName}` diff --git a/packages/medusa-test-utils/src/module-test-runner.ts b/packages/medusa-test-utils/src/module-test-runner.ts index 6007e4e1ca..a850bb9bb9 100644 --- a/packages/medusa-test-utils/src/module-test-runner.ts +++ b/packages/medusa-test-utils/src/module-test-runner.ts @@ -13,6 +13,7 @@ import * as fs from "fs" import { getDatabaseURL, getMikroOrmWrapper, TestDatabase } from "./database" import { initModules, InitModulesOptions } from "./init-modules" import { default as MockEventBusService } from "./mock-event-bus-service" +import { ulid } from "ulid" export interface SuiteOptions { MikroOrmWrapper: TestDatabase @@ -114,9 +115,10 @@ class ModuleTestRunner { constructor(config: ModuleTestRunnerConfig) { const tempName = parseInt(process.env.JEST_WORKER_ID || "1") this.moduleName = config.moduleName + const moduleName = this.moduleName ?? ulid() this.dbName = config.dbName ?? - `medusa-${config.moduleName.toLowerCase()}-integration-${tempName}` + `medusa-${moduleName.toLowerCase()}-integration-${tempName}` this.schema = config.schema ?? "public" this.debug = config.debug ?? false this.resolve = config.resolve diff --git a/packages/modules/workflow-engine-inmemory/package.json b/packages/modules/workflow-engine-inmemory/package.json index 8a7b7b4336..e9edb8bb77 100644 --- a/packages/modules/workflow-engine-inmemory/package.json +++ b/packages/modules/workflow-engine-inmemory/package.json @@ -29,7 +29,7 @@ "resolve:aliases": "yarn run -T tsc --showConfig -p tsconfig.json > tsconfig.resolved.json && yarn run -T tsc-alias -p tsconfig.resolved.json && yarn run -T rimraf tsconfig.resolved.json", "build": "yarn run -T rimraf dist && yarn run -T tsc --build && npm run resolve:aliases", "test": "../../../node_modules/.bin/jest --passWithNoTests --bail --forceExit --testPathPattern=src", - "test:integration": "../../../node_modules/.bin/jest --passWithNoTests --forceExit --testPathPattern=\"integration-tests/__tests__/.*\\.spec\\.ts\"", + "test:integration": "../../../node_modules/.bin/jest --passWithNoTests --forceExit --testPathPattern=\"integration-tests/__tests__/index\\.spec\\.ts\" && ../../../node_modules/.bin/jest --passWithNoTests --forceExit --testPathPattern=\"integration-tests/__tests__/race\\.spec\\.ts\" && ../../../node_modules/.bin/jest --passWithNoTests --forceExit --testPathPattern=\"integration-tests/__tests__/subscribe\\.spec\\.ts\"", "migration:initial": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:create --initial", "migration:create": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:create", "migration:up": "MIKRO_ORM_CLI_CONFIG=./mikro-orm.config.dev.ts MIKRO_ORM_ALLOW_GLOBAL_CLI=true medusa-mikro-orm migration:up", diff --git a/yarn.lock b/yarn.lock index c4727c02e9..4e5ec6ef65 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4058,7 +4058,7 @@ __metadata: axios: ^1.13.1 express: ^4.21.0 get-port: ^5.1.1 - randomatic: ^3.1.1 + ulid: ^2.3.0 peerDependencies: "@medusajs/framework": 2.11.2 "@medusajs/medusa": 2.11.2 @@ -18800,13 +18800,6 @@ __metadata: languageName: node linkType: hard -"is-number@npm:^4.0.0": - version: 4.0.0 - resolution: "is-number@npm:4.0.0" - checksum: bb17a331f357eb59a7f8db848086c41886715b2ea1db03f284a99d14001cda094083a5b6a7b343b5bcf410ccef668a70bc626d07bc2032cc4ab46dd264cea244 - languageName: node - linkType: hard - "is-number@npm:^7.0.0": version: 7.0.0 resolution: "is-number@npm:7.0.0" @@ -20057,13 +20050,6 @@ __metadata: languageName: node linkType: hard -"kind-of@npm:^6.0.0": - version: 6.0.3 - resolution: "kind-of@npm:6.0.3" - checksum: 61cdff9623dabf3568b6445e93e31376bee1cdb93f8ba7033d86022c2a9b1791a1d9510e026e6465ebd701a6dd2f7b0808483ad8838341ac52f003f512e0b4c4 - languageName: node - linkType: hard - "kleur@npm:4.1.5": version: 4.1.5 resolution: "kleur@npm:4.1.5" @@ -20729,13 +20715,6 @@ __metadata: languageName: node linkType: hard -"math-random@npm:^1.0.1": - version: 1.0.4 - resolution: "math-random@npm:1.0.4" - checksum: 7b0ddc17f5dfe3b426c1e92505122e6a32f884dd50f5e0bb3898e5ce2da60b4ffb47c9b607809cf0beb5b8bf253b9dcc3b6f7331b20ce59b8bd7e8dbbbb1e347 - languageName: node - linkType: hard - "mdn-data@npm:2.0.28": version: 2.0.28 resolution: "mdn-data@npm:2.0.28" @@ -23546,17 +23525,6 @@ __metadata: languageName: node linkType: hard -"randomatic@npm:^3.1.1": - version: 3.1.1 - resolution: "randomatic@npm:3.1.1" - dependencies: - is-number: ^4.0.0 - kind-of: ^6.0.0 - math-random: ^1.0.1 - checksum: 4b1da4b8e234d3d0bd2294a42541dfa03edbde85ee06fa0722e2b004e845da197d72fa7995723d32ea7d7402823ea62550034118cf22e94638560a509cec5bfc - languageName: node - linkType: hard - "randombytes@npm:^2.1.0": version: 2.1.0 resolution: "randombytes@npm:2.1.0"