feat(index): full sync operations (#11178)

Closes: FRMW-2892, FRMW-2893

**What**
Wired up the building block that we merged previously in order to manage data synchronization. The flow is as follow
- On application start
  - Build schema object representation from configuration
  - Check configuration changes
    - if new entities configured
      - Data synchronizer initialize orchestrator and start sync
        - for each entity
          - acquire lock
          - mark existing data as staled
          - sync all data by batch
          - marked them not staled anymore
          - acknowledge each processed batch and renew lock
          - update metadata with last synced cursor for entity X
          - release lock
      - remove all remaining staled data
    - if any entities removed from last configuration
      - remove the index data and relations

Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>

packages/modules/index/integration-tests/__tests__/config-sync.spec.ts

@@ -5,13 +5,10 @@ import {
   MedusaAppLoader,
 } from "@medusajs/framework"
 import { MedusaAppOutput, MedusaModule } from "@medusajs/framework/modules-sdk"
-import {
-  ContainerRegistrationKeys,
-  ModuleRegistrationName,
-  Modules,
-} from "@medusajs/framework/utils"
+import { ContainerRegistrationKeys, Modules } from "@medusajs/framework/utils"
 import { initDb, TestDatabaseUtils } from "@medusajs/test-utils"
-import { EntityManager } from "@mikro-orm/postgresql"
+import { IndexTypes, ModulesSdkTypes } from "@medusajs/types"
+import { Configuration } from "@utils"
 import { asValue } from "awilix"
 import path from "path"
 import { setTimeout } from "timers/promises"
@@ -21,9 +18,9 @@ import { updateRemovedSchema } from "../__fixtures__/update-removed-schema"
 import { updatedSchema } from "../__fixtures__/updated-schema"
 
 const eventBusMock = new EventBusServiceMock()
-const queryMock = jest.fn().mockReturnValue({
-  graph: jest.fn(),
-})
+const queryMock = {
+  graph: jest.fn().mockImplementation(async () => ({ data: [] })),
+}
 
 const dbUtils = TestDatabaseUtils.dbTestUtilFactory()
 
@@ -31,6 +28,7 @@ jest.setTimeout(300000)
 
 let isFirstTime = true
 let medusaAppLoader!: MedusaAppLoader
+let index: IndexTypes.IIndexService
 
 const beforeAll_ = async () => {
   try {
@@ -45,11 +43,10 @@ const beforeAll_ = async () => {
 
     container.register({
       [ContainerRegistrationKeys.LOGGER]: asValue(logger),
-      [ContainerRegistrationKeys.QUERY]: asValue(null),
       [ContainerRegistrationKeys.PG_CONNECTION]: asValue(dbUtils.pgConnection_),
     })
 
-    medusaAppLoader = new MedusaAppLoader(container as any)
+    medusaAppLoader = new MedusaAppLoader()
 
     // Migrations
     await medusaAppLoader.runModulesMigrations()
@@ -62,14 +59,16 @@ const beforeAll_ = async () => {
 
     // Bootstrap modules
     const globalApp = await medusaAppLoader.load()
+    container.register({
+      [ContainerRegistrationKeys.QUERY]: asValue(queryMock),
+      [ContainerRegistrationKeys.REMOTE_QUERY]: asValue(queryMock),
+      [Modules.EVENT_BUS]: asValue(eventBusMock),
+    })
 
-    const index = container.resolve(Modules.INDEX)
-
-    // Mock event bus  the index module
-    ;(index as any).eventBusModuleService_ = eventBusMock
+    index = container.resolve(Modules.INDEX)
 
     await globalApp.onApplicationStart()
-    ;(index as any).storageProvider_.query_ = queryMock
+    await setTimeout(1000)
 
     return globalApp
   } catch (error) {
@@ -105,8 +104,9 @@ const afterEach_ = async () => {
 
 describe("IndexModuleService syncIndexConfig", function () {
   let medusaApp: MedusaAppOutput
-  let module: any
-  let manager: EntityManager
+  let indexMetadataService: ModulesSdkTypes.IMedusaInternalService<any>
+  let indexSyncService: ModulesSdkTypes.IMedusaInternalService<any>
+  let dataSynchronizer: ModulesSdkTypes.IMedusaInternalService<any>
   let onApplicationPrepareShutdown!: () => Promise<void>
   let onApplicationShutdown!: () => Promise<void>
 
@@ -125,8 +125,10 @@ describe("IndexModuleService syncIndexConfig", function () {
   beforeEach(async () => {
     await beforeEach_()
 
-    module = medusaApp.sharedContainer!.resolve(ModuleRegistrationName.INDEX)
-    manager = module.container_.manager as EntityManager
+    index = container.resolve(Modules.INDEX)
+    indexMetadataService = (index as any).indexMetadataService_
+    indexSyncService = (index as any).indexSyncService_
+    dataSynchronizer = (index as any).dataSynchronizer_
   })
 
   afterEach(afterEach_)
@@ -134,7 +136,7 @@ describe("IndexModuleService syncIndexConfig", function () {
   it("should full sync all entities when the config has changed", async () => {
     await setTimeout(1000)
 
-    const currentMetadata = await module.listIndexMetadata()
+    const currentMetadata = await indexMetadataService.list()
 
     expect(currentMetadata).toHaveLength(7)
     expect(currentMetadata).toEqual(
@@ -177,13 +179,25 @@ describe("IndexModuleService syncIndexConfig", function () {
       ])
     )
 
-    // update config schema
-    module.schemaObjectRepresentation_ = null
-    module.moduleOptions_ ??= {}
-    module.moduleOptions_.schema = updatedSchema
-    module.buildSchemaObjectRepresentation_()
+    let indexSync = await indexSyncService.list({
+      last_key: null,
+    })
+    expect(indexSync).toHaveLength(7)
 
-    const syncRequired = await module.syncIndexConfig()
+    // update config schema
+    ;(index as any).schemaObjectRepresentation_ = null
+    ;(index as any).moduleOptions_ ??= {}
+    ;(index as any).moduleOptions_.schema = updatedSchema
+    ;(index as any).buildSchemaObjectRepresentation_()
+
+    let configurationChecker = new Configuration({
+      schemaObjectRepresentation: (index as any).schemaObjectRepresentation_,
+      indexMetadataService,
+      indexSyncService,
+      dataSynchronizer,
+    })
+
+    const syncRequired = await configurationChecker.checkChanges()
 
     expect(syncRequired).toHaveLength(2)
     expect(syncRequired).toEqual(
@@ -201,7 +215,12 @@ describe("IndexModuleService syncIndexConfig", function () {
       ])
     )
 
-    const updatedMetadata = await module.listIndexMetadata()
+    indexSync = await indexSyncService.list({
+      last_key: null,
+    })
+    expect(indexSync).toHaveLength(7)
+
+    const updatedMetadata = await indexMetadataService.list()
 
     expect(updatedMetadata).toHaveLength(7)
     expect(updatedMetadata).toEqual(
@@ -243,15 +262,28 @@ describe("IndexModuleService syncIndexConfig", function () {
         }),
       ])
     )
-    await module.syncEntities(syncRequired)
+
+    await (index as any).dataSynchronizer_.syncEntities(syncRequired)
 
     // Sync again removing entities not linked
-    module.schemaObjectRepresentation_ = null
-    module.moduleOptions_ ??= {}
-    module.moduleOptions_.schema = updateRemovedSchema
-    module.buildSchemaObjectRepresentation_()
+    ;(index as any).schemaObjectRepresentation_ = null
+    ;(index as any).moduleOptions_ ??= {}
+    ;(index as any).moduleOptions_.schema = updateRemovedSchema
+    ;(index as any).buildSchemaObjectRepresentation_()
 
-    const syncRequired2 = await module.syncIndexConfig()
+    const spyDataSynchronizer_ = jest.spyOn(
+      (index as any).dataSynchronizer_,
+      "removeEntities"
+    )
+
+    configurationChecker = new Configuration({
+      schemaObjectRepresentation: (index as any).schemaObjectRepresentation_,
+      indexMetadataService,
+      indexSyncService,
+      dataSynchronizer,
+    })
+
+    const syncRequired2 = await configurationChecker.checkChanges()
     expect(syncRequired2).toHaveLength(1)
     expect(syncRequired2).toEqual(
       expect.arrayContaining([
@@ -263,8 +295,8 @@ describe("IndexModuleService syncIndexConfig", function () {
       ])
     )
 
-    const updatedMetadata2 = await module.listIndexMetadata()
-    expect(updatedMetadata2).toHaveLength(5)
+    const updatedMetadata2 = await indexMetadataService.list()
+    expect(updatedMetadata2).toHaveLength(2)
     expect(updatedMetadata2).toEqual(
       expect.arrayContaining([
         expect.objectContaining({
@@ -272,27 +304,14 @@ describe("IndexModuleService syncIndexConfig", function () {
           fields: "handle,id,title",
           status: "done",
         }),
-        expect.objectContaining({
-          entity: "PriceSet",
-          fields: "id",
-          status: "done",
-        }),
-        expect.objectContaining({
-          entity: "Price",
-          fields: "amount,currency_code,price_set.id",
-          status: "done",
-        }),
         expect.objectContaining({
           entity: "ProductVariant",
           fields: "description,id,product.id,product_id,sku",
           status: "pending",
         }),
-        expect.objectContaining({
-          entity: "LinkProductVariantPriceSet",
-          fields: "id,price_set_id,variant_id",
-          status: "done",
-        }),
       ])
     )
+
+    expect(spyDataSynchronizer_).toHaveBeenCalledTimes(1)
   })
 })