feat(index): full sync operations (#11178)

Closes: FRMW-2892, FRMW-2893

**What**
Wired up the building block that we merged previously in order to manage data synchronization. The flow is as follow
- On application start
  - Build schema object representation from configuration
  - Check configuration changes
    - if new entities configured
      - Data synchronizer initialize orchestrator and start sync
        - for each entity
          - acquire lock
          - mark existing data as staled
          - sync all data by batch
          - marked them not staled anymore
          - acknowledge each processed batch and renew lock
          - update metadata with last synced cursor for entity X
          - release lock
      - remove all remaining staled data
    - if any entities removed from last configuration
      - remove the index data and relations

Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>
This commit is contained in:
Adrien de Peretti
2025-02-05 17:49:18 +01:00
committed by GitHub
parent 60f46e07fd
commit a33aebd895
35 changed files with 1677 additions and 727 deletions
@@ -37,6 +37,45 @@ export async function createPartitions(
return
}
await manager.execute(partitions.join("; "))
// Create indexes for each partition
const indexCreationCommands = Object.keys(schemaObjectRepresentation)
.filter(
(key) =>
!schemaObjectRepresentationPropertiesToOmit.includes(key) &&
schemaObjectRepresentation[key].listeners.length > 0
)
.map((key) => {
const cName = key.toLowerCase()
const part: string[] = []
part.push(
`CREATE INDEX CONCURRENTLY IF NOT EXISTS "IDX_cat_${cName}_data_gin" ON ${activeSchema}cat_${cName} USING GIN ("data" jsonb_path_ops)`
)
// create child id index on pivot partitions
for (const parent of schemaObjectRepresentation[key].parents) {
const pName = `${parent.ref.entity}${key}`.toLowerCase()
part.push(
`CREATE INDEX CONCURRENTLY IF NOT EXISTS "IDX_cat_pivot_${pName}_child_id" ON ${activeSchema}cat_pivot_${pName} ("child_id")`
)
}
return part
})
.flat()
// Execute index creation commands separately to avoid blocking
for (const cmd of indexCreationCommands) {
try {
await manager.execute(cmd)
} catch (error) {
// Log error but continue with other indexes
console.error(`Failed to create index: ${error.message}`)
}
}
partitions.push(`analyse ${activeSchema}index_data`)
partitions.push(`analyse ${activeSchema}index_relation`)