asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(index): full sync operations (#11178)

Browse Source 
Closes: FRMW-2892, FRMW-2893

**What**
Wired up the building block that we merged previously in order to manage data synchronization. The flow is as follow
- On application start
  - Build schema object representation from configuration
  - Check configuration changes
    - if new entities configured
      - Data synchronizer initialize orchestrator and start sync
        - for each entity
          - acquire lock
          - mark existing data as staled
          - sync all data by batch
          - marked them not staled anymore
          - acknowledge each processed batch and renew lock
          - update metadata with last synced cursor for entity X
          - release lock
      - remove all remaining staled data
    - if any entities removed from last configuration
      - remove the index data and relations

Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>
This commit is contained in:
Adrien de Peretti
2025-02-05 16:49:18 +00:00
committed by GitHub
co-authored by Carlos R. L. Rodrigues
parent 60f46e07fd
commit a33aebd895
35 changed files with 1677 additions and 727 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/modules/index/integration-tests/__tests__/data-synchronizer.spec.ts
+65 -101
View File
@@ -8,18 +8,18 @@ import { MedusaAppOutput, MedusaModule } from "@medusajs/framework/modules-sdk"
import { IndexTypes, InferEntityType } from "@medusajs/framework/types"
import {
ContainerRegistrationKeys,
ModuleRegistrationName,
Modules,
toMikroORMEntity,
} from "@medusajs/framework/utils"
import { initDb, TestDatabaseUtils } from "@medusajs/test-utils"
import { asValue } from "awilix"
import * as path from "path"
import { DataSynchronizer } from "../../src/utils/sync/data-synchronizer"
import { EventBusServiceMock } from "../__fixtures__"
import { dbName } from "../__fixtures__/medusa-config"
import { EntityManager } from "@mikro-orm/postgresql"
import { IndexData, IndexRelation } from "@models"
import { DataSynchronizer } from "@services"
import { asValue } from "awilix"
import * as path from "path"
import { setTimeout } from "timers/promises"
import { EventBusServiceMock } from "../__fixtures__"
import { dbName } from "../__fixtures__/medusa-config"
const eventBusMock = new EventBusServiceMock()
const queryMock = {
@@ -28,7 +28,7 @@ const queryMock = {
const dbUtils = TestDatabaseUtils.dbTestUtilFactory()
jest.setTimeout(30000)
jest.setTimeout(300000)
const testProductId = "test_prod_1"
const testProductId2 = "test_prod_2"
@@ -80,11 +80,10 @@ const beforeAll_ = async () => {
container.register({
[ContainerRegistrationKeys.LOGGER]: asValue(logger),
[ContainerRegistrationKeys.QUERY]: asValue(null),
[ContainerRegistrationKeys.PG_CONNECTION]: asValue(dbUtils.pgConnection_),
})
medusaAppLoader = new MedusaAppLoader(container as any)
medusaAppLoader = new MedusaAppLoader()
// Migrations
await medusaAppLoader.runModulesMigrations()
@@ -97,14 +96,16 @@ const beforeAll_ = async () => {
// Bootstrap modules
const globalApp = await medusaAppLoader.load()
container.register({
[ContainerRegistrationKeys.QUERY]: asValue(queryMock),
[ContainerRegistrationKeys.REMOTE_QUERY]: asValue(queryMock),
[Modules.EVENT_BUS]: asValue(eventBusMock),
})
index = container.resolve(Modules.INDEX)
// Mock event bus the index module
;(index as any).eventBusModuleService_ = eventBusMock
await globalApp.onApplicationStart()
;(index as any).storageProvider_.query_ = queryMock
await setTimeout(1000)
return globalApp
} catch (error) {
@@ -125,9 +126,6 @@ describe("DataSynchronizer", () => {
medusaApp = await beforeAll_()
onApplicationPrepareShutdown = medusaApp.onApplicationPrepareShutdown
onApplicationShutdown = medusaApp.onApplicationShutdown
manager = (
medusaApp.sharedContainer!.resolve(ModuleRegistrationName.INDEX) as any
).container_.manager as EntityManager
})
afterAll(async () => {
@@ -139,55 +137,9 @@ describe("DataSynchronizer", () => {
beforeEach(async () => {
jest.clearAllMocks()
index = container.resolve(Modules.INDEX)
manager = (index as any).container_.manager as EntityManager
const productSchemaObjectRepresentation: IndexTypes.SchemaObjectEntityRepresentation =
{
fields: ["id", "title", "updated_at"],
alias: "product",
moduleConfig: {
linkableKeys: {
id: "Product",
product_id: "Product",
product_variant_id: "ProductVariant",
},
},
entity: "Product",
parents: [],
listeners: ["product.created"],
}
const productVariantSchemaObjectRepresentation: IndexTypes.SchemaObjectEntityRepresentation =
{
fields: ["id", "title", "product.id", "updated_at"],
alias: "product_variant",
moduleConfig: {
linkableKeys: {
id: "ProductVariant",
product_id: "Product",
product_variant_id: "ProductVariant",
},
},
entity: "ProductVariant",
parents: [
{
ref: productSchemaObjectRepresentation,
inSchemaRef: productSchemaObjectRepresentation,
targetProp: "id",
},
],
listeners: ["product-variant.created"],
}
const mockSchemaRepresentation = {
product: productSchemaObjectRepresentation,
product_variant: productVariantSchemaObjectRepresentation,
}
dataSynchronizer = new DataSynchronizer({
storageProvider: (index as any).storageProvider_,
schemaObjectRepresentation: mockSchemaRepresentation,
query: queryMock as any,
})
dataSynchronizer = (index as any).dataSynchronizer_
})
describe("sync", () => {
@@ -223,8 +175,8 @@ describe("DataSynchronizer", () => {
const ackMock = jest.fn()
const result = await dataSynchronizer.sync({
entityName: "product",
const result = await dataSynchronizer.syncEntity({
entityName: "Product",
ack: ackMock,
})
@@ -237,7 +189,7 @@ describe("DataSynchronizer", () => {
order: {
id: "asc",
},
take: 1000,
take: 100,
},
})
@@ -247,7 +199,7 @@ describe("DataSynchronizer", () => {
filters: {
id: [testProductId],
},
fields: ["id", "title", "updated_at"],
fields: ["id", "title"],
})
// Second loop fetching products
@@ -263,7 +215,7 @@ describe("DataSynchronizer", () => {
order: {
id: "asc",
},
take: 1000,
take: 100,
},
})
@@ -273,7 +225,7 @@ describe("DataSynchronizer", () => {
filters: {
id: [testProductId2],
},
fields: ["id", "title", "updated_at"],
fields: ["id", "title"],
})
expect(ackMock).toHaveBeenNthCalledWith(1, {
@@ -304,8 +256,16 @@ describe("DataSynchronizer", () => {
)
expect(indexData).toHaveLength(2)
expect(indexData[0].id).toEqual(testProductId)
expect(indexData[1].id).toEqual(testProductId2)
expect(indexData).toEqual(
expect.arrayContaining([
expect.objectContaining({
id: testProductId,
}),
expect.objectContaining({
id: testProductId2,
}),
])
)
expect(indexRelationData).toHaveLength(0)
})
@@ -369,15 +329,15 @@ describe("DataSynchronizer", () => {
const ackMock = jest.fn()
await dataSynchronizer.sync({
entityName: "product",
await dataSynchronizer.syncEntity({
entityName: "Product",
ack: ackMock,
})
jest.clearAllMocks()
const result = await dataSynchronizer.sync({
entityName: "product_variant",
const result = await dataSynchronizer.syncEntity({
entityName: "ProductVariant",
ack: ackMock,
})
@@ -390,7 +350,7 @@ describe("DataSynchronizer", () => {
order: {
id: "asc",
},
take: 1000,
take: 100,
},
})
@@ -400,7 +360,7 @@ describe("DataSynchronizer", () => {
filters: {
id: [testVariantId],
},
fields: ["id", "title", "product.id", "updated_at"],
fields: ["id", "product.id", "product_id", "sku"],
})
// Second loop fetching product variants
@@ -416,7 +376,7 @@ describe("DataSynchronizer", () => {
order: {
id: "asc",
},
take: 1000,
take: 100,
},
})
@@ -426,7 +386,7 @@ describe("DataSynchronizer", () => {
filters: {
id: [testVariantId2],
},
fields: ["id", "title", "product.id", "updated_at"],
fields: ["id", "product.id", "product_id", "sku"],
})
expect(ackMock).toHaveBeenNthCalledWith(1, {
@@ -456,29 +416,33 @@ describe("DataSynchronizer", () => {
>(toMikroORMEntity(IndexRelation), {})
expect(indexData).toHaveLength(4)
expect(indexData[0].id).toEqual(testProductId)
expect(indexData[1].id).toEqual(testProductId2)
expect(indexData[2].id).toEqual(testVariantId)
expect(indexData[3].id).toEqual(testVariantId2)
expect(indexData).toEqual(
expect.arrayContaining([
expect.objectContaining({ id: testProductId }),
expect.objectContaining({ id: testProductId2 }),
expect.objectContaining({ id: testVariantId }),
expect.objectContaining({ id: testVariantId2 }),
])
)
expect(indexRelationData).toHaveLength(2)
expect(indexRelationData[0]).toEqual(
expect.objectContaining({
parent_id: testProductId,
child_id: testVariantId,
parent_name: "Product",
child_name: "ProductVariant",
pivot: "Product-ProductVariant",
})
)
expect(indexRelationData[1]).toEqual(
expect.objectContaining({
parent_id: testProductId2,
child_id: testVariantId2,
parent_name: "Product",
child_name: "ProductVariant",
pivot: "Product-ProductVariant",
})
expect(indexRelationData).toEqual(
expect.arrayContaining([
expect.objectContaining({
parent_id: testProductId,
child_id: testVariantId,
parent_name: "Product",
child_name: "ProductVariant",
pivot: "Product-ProductVariant",
}),
expect.objectContaining({
parent_id: testProductId2,
child_id: testVariantId2,
parent_name: "Product",
child_name: "ProductVariant",
pivot: "Product-ProductVariant",
}),
])
)
})