feat(index): full sync operations (#11178)

Closes: FRMW-2892, FRMW-2893

**What**
Wired up the building block that we merged previously in order to manage data synchronization. The flow is as follow
- On application start
  - Build schema object representation from configuration
  - Check configuration changes
    - if new entities configured
      - Data synchronizer initialize orchestrator and start sync
        - for each entity
          - acquire lock
          - mark existing data as staled
          - sync all data by batch
          - marked them not staled anymore
          - acknowledge each processed batch and renew lock
          - update metadata with last synced cursor for entity X
          - release lock
      - remove all remaining staled data
    - if any entities removed from last configuration
      - remove the index data and relations

Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>
This commit is contained in:
Adrien de Peretti
2025-02-05 16:49:18 +00:00
committed by GitHub
co-authored by Carlos R. L. Rodrigues
parent 60f46e07fd
commit a33aebd895
35 changed files with 1677 additions and 727 deletions
@@ -8,7 +8,6 @@ import { MedusaAppOutput, MedusaModule } from "@medusajs/framework/modules-sdk"
import { EventBusTypes, IndexTypes } from "@medusajs/framework/types"
import {
ContainerRegistrationKeys,
ModuleRegistrationName,
Modules,
toMikroORMEntity,
} from "@medusajs/framework/utils"
@@ -17,6 +16,7 @@ import { EntityManager } from "@mikro-orm/postgresql"
import { IndexData, IndexRelation } from "@models"
import { asValue } from "awilix"
import * as path from "path"
import { setTimeout } from "timers/promises"
import { EventBusServiceMock } from "../__fixtures__"
import { dbName } from "../__fixtures__/medusa-config"
@@ -151,6 +151,7 @@ const beforeEach_ = async (eventDataToEmit) => {
if (isFirstTime) {
isFirstTime = false
await sendEvents(eventDataToEmit)
return
}
@@ -241,11 +242,11 @@ describe("IndexModuleService", function () {
]
beforeEach(async () => {
await setTimeout(1000)
await beforeEach_(eventDataToEmit)
manager = (
medusaApp.sharedContainer!.resolve(ModuleRegistrationName.INDEX) as any
).container_.manager as EntityManager
manager = (medusaApp.sharedContainer!.resolve(Modules.INDEX) as any)
.container_.manager as EntityManager
})
afterEach(afterEach_)
@@ -404,9 +405,8 @@ describe("IndexModuleService", function () {
beforeEach(async () => {
await beforeEach_(eventDataToEmit)
manager = (
medusaApp.sharedContainer!.resolve(ModuleRegistrationName.INDEX) as any
).container_.manager as EntityManager
manager = (medusaApp.sharedContainer!.resolve(Modules.INDEX) as any)
.container_.manager as EntityManager
})
afterEach(afterEach_)
@@ -565,9 +565,8 @@ describe("IndexModuleService", function () {
beforeEach(async () => {
await beforeEach_(eventDataToEmit)
manager = (
medusaApp.sharedContainer!.resolve(ModuleRegistrationName.INDEX) as any
).container_.manager as EntityManager
manager = (medusaApp.sharedContainer!.resolve(Modules.INDEX) as any)
.container_.manager as EntityManager
await updateData(manager)
@@ -686,9 +685,8 @@ describe("IndexModuleService", function () {
beforeEach(async () => {
await beforeEach_(eventDataToEmit)
manager = (
medusaApp.sharedContainer!.resolve(ModuleRegistrationName.INDEX) as any
).container_.manager as EntityManager
manager = (medusaApp.sharedContainer!.resolve(Modules.INDEX) as any)
.container_.manager as EntityManager
queryMock.graph = jest.fn().mockImplementation((query) => {
const entity = query.entity