asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(index): full sync operations (#11178)

Browse Source 
Closes: FRMW-2892, FRMW-2893

**What**
Wired up the building block that we merged previously in order to manage data synchronization. The flow is as follow
- On application start
  - Build schema object representation from configuration
  - Check configuration changes
    - if new entities configured
      - Data synchronizer initialize orchestrator and start sync
        - for each entity
          - acquire lock
          - mark existing data as staled
          - sync all data by batch
          - marked them not staled anymore
          - acknowledge each processed batch and renew lock
          - update metadata with last synced cursor for entity X
          - release lock
      - remove all remaining staled data
    - if any entities removed from last configuration
      - remove the index data and relations

Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>
This commit is contained in:
Adrien de Peretti
2025-02-05 17:49:18 +01:00
committed by GitHub
parent 60f46e07fd
commit a33aebd895
35 changed files with 1677 additions and 727 deletions
Download Patch File Download Diff File Expand all files Collapse all files

141
packages/modules/index/src/services/postgres-provider.ts
View File

@@ -208,14 +208,13 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
}
const { fields, alias } = schemaEntityObjectRepresentation
const graphResult = await this.query_.graph({
const { data: entityData } = await this.query_.graph({
entity: alias,
filters: {
id: ids,
},
fields: [...new Set(["id", ...fields])],
})
const { data: entityData } = graphResult
const argument = {
entity: schemaEntityObjectRepresentation.entity,
@@ -327,9 +326,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
* @protected
*/
@InjectTransactionManager()
protected async onCreate<
TData extends { id: string; [key: string]: unknown }
>(
async onCreate<TData extends { id: string; [key: string]: unknown }>(
{
entity,
data,
@@ -339,13 +336,11 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
data: TData | TData[]
schemaEntityObjectRepresentation: IndexTypes.SchemaObjectEntityRepresentation
},
@MedusaContext() sharedContext: Context = {}
@MedusaContext() sharedContext: Context<SqlEntityManager> = {}
) {
const { transactionManager: em } = sharedContext as {
transactionManager: SqlEntityManager
}
const indexRepository = em.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository: EntityRepository<any> = em.getRepository(
const { transactionManager: em } = sharedContext
const indexRepository = em!.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository: EntityRepository<any> = em!.getRepository(
toMikroORMEntity(IndexRelation)
)
@@ -374,7 +369,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
id: cleanedEntityData.id,
name: entity,
data: cleanedEntityData,
// stale: false,
staled_at: null,
})
/**
@@ -400,7 +395,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
id: (parentData_ as any).id,
name: parentEntity,
data: parentData_,
// stale: false,
staled_at: null,
})
await indexRelationRepository.upsert(
@@ -410,7 +405,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
child_id: cleanedEntityData.id,
child_name: entity,
pivot: `${parentEntity}-${entity}`,
// stale: false,
staled_at: null,
},
{
onConflictAction: "merge",
@@ -437,9 +432,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
* @protected
*/
@InjectTransactionManager()
protected async onUpdate<
TData extends { id: string; [key: string]: unknown }
>(
async onUpdate<TData extends { id: string; [key: string]: unknown }>(
{
entity,
data,
@@ -449,12 +442,10 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
data: TData | TData[]
schemaEntityObjectRepresentation: IndexTypes.SchemaObjectEntityRepresentation
},
@MedusaContext() sharedContext: Context = {}
@MedusaContext() sharedContext: Context<SqlEntityManager> = {}
) {
const { transactionManager: em } = sharedContext as {
transactionManager: SqlEntityManager
}
const indexRepository = em.getRepository(toMikroORMEntity(IndexData))
const { transactionManager: em } = sharedContext
const indexRepository = em!.getRepository(toMikroORMEntity(IndexData))
const { data: data_, entityProperties } = PostgresProvider.parseData(
data,
@@ -470,7 +461,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
acc[property] = entityData[property]
return acc
}, {}),
// stale: false,
staled_at: null,
}
})
)
@@ -485,9 +476,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
* @protected
*/
@InjectTransactionManager()
protected async onDelete<
TData extends { id: string; [key: string]: unknown }
>(
async onDelete<TData extends { id: string; [key: string]: unknown }>(
{
entity,
data,
@@ -497,13 +486,11 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
data: TData | TData[]
schemaEntityObjectRepresentation: IndexTypes.SchemaObjectEntityRepresentation
},
@MedusaContext() sharedContext: Context = {}
@MedusaContext() sharedContext: Context<SqlEntityManager> = {}
) {
const { transactionManager: em } = sharedContext as {
transactionManager: SqlEntityManager
}
const indexRepository = em.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository = em.getRepository(
const { transactionManager: em } = sharedContext
const indexRepository = em!.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository = em!.getRepository(
toMikroORMEntity(IndexRelation)
)
@@ -541,9 +528,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
* @protected
*/
@InjectTransactionManager()
protected async onAttach<
TData extends { id: string; [key: string]: unknown }
>(
async onAttach<TData extends { id: string; [key: string]: unknown }>(
{
entity,
data,
@@ -553,13 +538,11 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
data: TData | TData[]
schemaEntityObjectRepresentation: IndexTypes.SchemaObjectEntityRepresentation
},
@MedusaContext() sharedContext: Context = {}
@MedusaContext() sharedContext: Context<SqlEntityManager> = {}
) {
const { transactionManager: em } = sharedContext as {
transactionManager: SqlEntityManager
}
const indexRepository = em.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository = em.getRepository(
const { transactionManager: em } = sharedContext
const indexRepository = em!.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository = em!.getRepository(
toMikroORMEntity(IndexRelation)
)
@@ -626,34 +609,54 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
id: cleanedEntityData.id,
name: entity,
data: cleanedEntityData,
// stale: false,
staled_at: null,
})
/**
* Create the index relation entries for the parent entity and the child entity
*/
const parentIndexRelationEntry = indexRelationRepository.create({
parent_id: entityData[parentPropertyId] as string,
parent_name: parentEntityName,
child_id: cleanedEntityData.id,
child_name: entity,
pivot: `${parentEntityName}-${entity}`,
// stale: false,
})
await indexRelationRepository.upsert(
{
parent_id: entityData[parentPropertyId] as string,
parent_name: parentEntityName,
child_id: cleanedEntityData.id,
child_name: entity,
pivot: `${parentEntityName}-${entity}`,
staled_at: null,
},
{
onConflictAction: "merge",
onConflictFields: [
"pivot",
"parent_id",
"child_id",
"parent_name",
"child_name",
],
}
)
const childIndexRelationEntry = indexRelationRepository.create({
parent_id: cleanedEntityData.id,
parent_name: entity,
child_id: entityData[childPropertyId] as string,
child_name: childEntityName,
pivot: `${entity}-${childEntityName}`,
// stale: false,
})
indexRelationRepository
.getEntityManager()
.persist([parentIndexRelationEntry, childIndexRelationEntry])
await indexRelationRepository.upsert(
{
parent_id: cleanedEntityData.id,
parent_name: entity,
child_id: entityData[childPropertyId] as string,
child_name: childEntityName,
pivot: `${entity}-${childEntityName}`,
staled_at: null,
},
{
onConflictAction: "merge",
onConflictFields: [
"pivot",
"parent_id",
"child_id",
"parent_name",
"child_name",
],
}
)
}
}
@@ -666,9 +669,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
* @protected
*/
@InjectTransactionManager()
protected async onDetach<
TData extends { id: string; [key: string]: unknown }
>(
async onDetach<TData extends { id: string; [key: string]: unknown }>(
{
entity,
data,
@@ -678,13 +679,11 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
data: TData | TData[]
schemaEntityObjectRepresentation: IndexTypes.SchemaObjectEntityRepresentation
},
@MedusaContext() sharedContext: Context = {}
@MedusaContext() sharedContext: Context<SqlEntityManager> = {}
) {
const { transactionManager: em } = sharedContext as {
transactionManager: SqlEntityManager
}
const indexRepository = em.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository = em.getRepository(
const { transactionManager: em } = sharedContext
const indexRepository = em!.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository = em!.getRepository(
toMikroORMEntity(IndexRelation)
)