feat(index): full sync operations (#11178)
Closes: FRMW-2892, FRMW-2893
**What**
Wired up the building block that we merged previously in order to manage data synchronization. The flow is as follow
- On application start
- Build schema object representation from configuration
- Check configuration changes
- if new entities configured
- Data synchronizer initialize orchestrator and start sync
- for each entity
- acquire lock
- mark existing data as staled
- sync all data by batch
- marked them not staled anymore
- acknowledge each processed batch and renew lock
- update metadata with last synced cursor for entity X
- release lock
- remove all remaining staled data
- if any entities removed from last configuration
- remove the index data and relations
Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>
This commit is contained in:
committed by
GitHub
parent
60f46e07fd
commit
a33aebd895
@@ -37,6 +37,45 @@ export async function createPartitions(
|
||||
return
|
||||
}
|
||||
|
||||
await manager.execute(partitions.join("; "))
|
||||
|
||||
// Create indexes for each partition
|
||||
const indexCreationCommands = Object.keys(schemaObjectRepresentation)
|
||||
.filter(
|
||||
(key) =>
|
||||
!schemaObjectRepresentationPropertiesToOmit.includes(key) &&
|
||||
schemaObjectRepresentation[key].listeners.length > 0
|
||||
)
|
||||
.map((key) => {
|
||||
const cName = key.toLowerCase()
|
||||
const part: string[] = []
|
||||
|
||||
part.push(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS "IDX_cat_${cName}_data_gin" ON ${activeSchema}cat_${cName} USING GIN ("data" jsonb_path_ops)`
|
||||
)
|
||||
|
||||
// create child id index on pivot partitions
|
||||
for (const parent of schemaObjectRepresentation[key].parents) {
|
||||
const pName = `${parent.ref.entity}${key}`.toLowerCase()
|
||||
part.push(
|
||||
`CREATE INDEX CONCURRENTLY IF NOT EXISTS "IDX_cat_pivot_${pName}_child_id" ON ${activeSchema}cat_pivot_${pName} ("child_id")`
|
||||
)
|
||||
}
|
||||
|
||||
return part
|
||||
})
|
||||
.flat()
|
||||
|
||||
// Execute index creation commands separately to avoid blocking
|
||||
for (const cmd of indexCreationCommands) {
|
||||
try {
|
||||
await manager.execute(cmd)
|
||||
} catch (error) {
|
||||
// Log error but continue with other indexes
|
||||
console.error(`Failed to create index: ${error.message}`)
|
||||
}
|
||||
}
|
||||
|
||||
partitions.push(`analyse ${activeSchema}index_data`)
|
||||
partitions.push(`analyse ${activeSchema}index_relation`)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user