feat(index): full sync operations (#11178)

Closes: FRMW-2892, FRMW-2893

**What**
Wired up the building block that we merged previously in order to manage data synchronization. The flow is as follow
- On application start
  - Build schema object representation from configuration
  - Check configuration changes
    - if new entities configured
      - Data synchronizer initialize orchestrator and start sync
        - for each entity
          - acquire lock
          - mark existing data as staled
          - sync all data by batch
          - marked them not staled anymore
          - acknowledge each processed batch and renew lock
          - update metadata with last synced cursor for entity X
          - release lock
      - remove all remaining staled data
    - if any entities removed from last configuration
      - remove the index data and relations

Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>
This commit is contained in:
Adrien de Peretti
2025-02-05 16:49:18 +00:00
committed by GitHub
co-authored by Carlos R. L. Rodrigues
parent 60f46e07fd
commit a33aebd895
35 changed files with 1677 additions and 727 deletions
@@ -6,11 +6,7 @@ import {
} from "@medusajs/framework"
import { MedusaAppOutput, MedusaModule } from "@medusajs/framework/modules-sdk"
import { IndexTypes } from "@medusajs/framework/types"
import {
ContainerRegistrationKeys,
ModuleRegistrationName,
Modules,
} from "@medusajs/framework/utils"
import { ContainerRegistrationKeys, Modules } from "@medusajs/framework/utils"
import { initDb, TestDatabaseUtils } from "@medusajs/test-utils"
import { EntityManager } from "@mikro-orm/postgresql"
import { IndexData, IndexRelation } from "@models"
@@ -123,11 +119,11 @@ describe("IndexModuleService query", function () {
beforeEach(async () => {
await beforeEach_()
module = medusaApp.sharedContainer!.resolve(ModuleRegistrationName.INDEX)
module = medusaApp.sharedContainer!.resolve(Modules.INDEX)
const manager = (
(medusaApp.sharedContainer!.resolve(ModuleRegistrationName.INDEX) as any)
.container_.manager as EntityManager
(medusaApp.sharedContainer!.resolve(Modules.INDEX) as any).container_
.manager as EntityManager
).fork()
const indexRepository = manager.getRepository(IndexData)
@@ -343,6 +339,65 @@ describe("IndexModuleService query", function () {
])
})
it("should query all products ordered by sku DESC with specific fields", async () => {
const { data } = await module.query({
fields: [
"product.*",
"product.variants.sku",
"product.variants.prices.amount",
],
pagination: {
order: {
product: {
variants: {
sku: "DESC",
},
},
},
},
})
expect(data).toEqual([
{
id: "prod_2",
title: "Product 2 title",
deep: {
a: 1,
obj: {
b: 15,
},
},
variants: [],
},
{
id: "prod_1",
variants: [
{
id: "var_2",
sku: "sku 123",
prices: [
{
id: "money_amount_2",
amount: 10,
},
],
},
{
id: "var_1",
sku: "aaa test aaa",
prices: [
{
id: "money_amount_1",
amount: 100,
},
],
},
],
},
])
})
it("should query all products ordered by price", async () => {
const { data } = await module.query({
fields: ["product.*", "product.variants.*", "product.variants.prices.*"],
@@ -484,6 +539,35 @@ describe("IndexModuleService query", function () {
])
})
it("should query products filtering by variant sku", async () => {
const { data } = await module.query({
fields: ["product.*", "product.variants.*", "product.variants.prices.*"],
joinFilters: {
"product.variants.prices.amount": { $gt: 110 },
},
filters: {
product: {
variants: {
sku: { $like: "aaa%" },
},
},
},
})
expect(data).toEqual([
{
id: "prod_1",
variants: [
{
id: "var_1",
sku: "aaa test aaa",
prices: [],
},
],
},
])
})
it("should query products filtering by price and returning the complete entity", async () => {
const { data } = await module.query({
fields: ["product.*", "product.variants.*", "product.variants.prices.*"],