feat(index): full sync operations (#11178)
Closes: FRMW-2892, FRMW-2893
**What**
Wired up the building block that we merged previously in order to manage data synchronization. The flow is as follow
- On application start
- Build schema object representation from configuration
- Check configuration changes
- if new entities configured
- Data synchronizer initialize orchestrator and start sync
- for each entity
- acquire lock
- mark existing data as staled
- sync all data by batch
- marked them not staled anymore
- acknowledge each processed batch and renew lock
- update metadata with last synced cursor for entity X
- release lock
- remove all remaining staled data
- if any entities removed from last configuration
- remove the index data and relations
Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>
This commit is contained in:
co-authored by
Carlos R. L. Rodrigues
parent
60f46e07fd
commit
a33aebd895
@@ -8,7 +8,6 @@ import { MedusaAppOutput, MedusaModule } from "@medusajs/framework/modules-sdk"
|
||||
import { EventBusTypes, IndexTypes } from "@medusajs/framework/types"
|
||||
import {
|
||||
ContainerRegistrationKeys,
|
||||
ModuleRegistrationName,
|
||||
Modules,
|
||||
toMikroORMEntity,
|
||||
} from "@medusajs/framework/utils"
|
||||
@@ -17,6 +16,7 @@ import { EntityManager } from "@mikro-orm/postgresql"
|
||||
import { IndexData, IndexRelation } from "@models"
|
||||
import { asValue } from "awilix"
|
||||
import * as path from "path"
|
||||
import { setTimeout } from "timers/promises"
|
||||
import { EventBusServiceMock } from "../__fixtures__"
|
||||
import { dbName } from "../__fixtures__/medusa-config"
|
||||
|
||||
@@ -151,6 +151,7 @@ const beforeEach_ = async (eventDataToEmit) => {
|
||||
if (isFirstTime) {
|
||||
isFirstTime = false
|
||||
await sendEvents(eventDataToEmit)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
@@ -241,11 +242,11 @@ describe("IndexModuleService", function () {
|
||||
]
|
||||
|
||||
beforeEach(async () => {
|
||||
await setTimeout(1000)
|
||||
await beforeEach_(eventDataToEmit)
|
||||
|
||||
manager = (
|
||||
medusaApp.sharedContainer!.resolve(ModuleRegistrationName.INDEX) as any
|
||||
).container_.manager as EntityManager
|
||||
manager = (medusaApp.sharedContainer!.resolve(Modules.INDEX) as any)
|
||||
.container_.manager as EntityManager
|
||||
})
|
||||
|
||||
afterEach(afterEach_)
|
||||
@@ -404,9 +405,8 @@ describe("IndexModuleService", function () {
|
||||
beforeEach(async () => {
|
||||
await beforeEach_(eventDataToEmit)
|
||||
|
||||
manager = (
|
||||
medusaApp.sharedContainer!.resolve(ModuleRegistrationName.INDEX) as any
|
||||
).container_.manager as EntityManager
|
||||
manager = (medusaApp.sharedContainer!.resolve(Modules.INDEX) as any)
|
||||
.container_.manager as EntityManager
|
||||
})
|
||||
|
||||
afterEach(afterEach_)
|
||||
@@ -565,9 +565,8 @@ describe("IndexModuleService", function () {
|
||||
beforeEach(async () => {
|
||||
await beforeEach_(eventDataToEmit)
|
||||
|
||||
manager = (
|
||||
medusaApp.sharedContainer!.resolve(ModuleRegistrationName.INDEX) as any
|
||||
).container_.manager as EntityManager
|
||||
manager = (medusaApp.sharedContainer!.resolve(Modules.INDEX) as any)
|
||||
.container_.manager as EntityManager
|
||||
|
||||
await updateData(manager)
|
||||
|
||||
@@ -686,9 +685,8 @@ describe("IndexModuleService", function () {
|
||||
beforeEach(async () => {
|
||||
await beforeEach_(eventDataToEmit)
|
||||
|
||||
manager = (
|
||||
medusaApp.sharedContainer!.resolve(ModuleRegistrationName.INDEX) as any
|
||||
).container_.manager as EntityManager
|
||||
manager = (medusaApp.sharedContainer!.resolve(Modules.INDEX) as any)
|
||||
.container_.manager as EntityManager
|
||||
|
||||
queryMock.graph = jest.fn().mockImplementation((query) => {
|
||||
const entity = query.entity
|
||||
|
||||
Reference in New Issue
Block a user