feat(index): full sync operations (#11178)

Closes: FRMW-2892, FRMW-2893

**What**
Wired up the building block that we merged previously in order to manage data synchronization. The flow is as follow
- On application start
  - Build schema object representation from configuration
  - Check configuration changes
    - if new entities configured
      - Data synchronizer initialize orchestrator and start sync
        - for each entity
          - acquire lock
          - mark existing data as staled
          - sync all data by batch
          - marked them not staled anymore
          - acknowledge each processed batch and renew lock
          - update metadata with last synced cursor for entity X
          - release lock
      - remove all remaining staled data
    - if any entities removed from last configuration
      - remove the index data and relations

Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>
This commit is contained in:
Adrien de Peretti
2025-02-05 17:49:18 +01:00
committed by GitHub
parent 60f46e07fd
commit a33aebd895
35 changed files with 1677 additions and 727 deletions

View File

@@ -68,6 +68,10 @@ function buildWhere(
}
if (["$or", "$and"].includes(prop)) {
if (!Array.isArray(value)) {
throw new Error(`Expected array for ${prop} but got ${value}`)
}
where[prop] = value.map((val) => {
const deepWhere = {}
buildWhere(val, deepWhere, flags)

View File

@@ -273,6 +273,7 @@ export function MedusaInternalService<
{},
sharedContext
)
// Create a pair of entity and data to update
entitiesToUpdate.forEach((entity) => {
toUpdateData.push({
@@ -345,6 +346,10 @@ export function MedusaInternalService<
}
}
if (!toUpdateData.length) {
return []
}
return await this[propertyRepositoryName].update(
toUpdateData,
sharedContext
@@ -458,6 +463,10 @@ export function MedusaInternalService<
})
}
if (!deleteCriteria.$or.length) {
return
}
await this[propertyRepositoryName].delete(deleteCriteria, sharedContext)
}