asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(index): full sync operations (#11178)

Browse Source 
Closes: FRMW-2892, FRMW-2893

**What**
Wired up the building block that we merged previously in order to manage data synchronization. The flow is as follow
- On application start
  - Build schema object representation from configuration
  - Check configuration changes
    - if new entities configured
      - Data synchronizer initialize orchestrator and start sync
        - for each entity
          - acquire lock
          - mark existing data as staled
          - sync all data by batch
          - marked them not staled anymore
          - acknowledge each processed batch and renew lock
          - update metadata with last synced cursor for entity X
          - release lock
      - remove all remaining staled data
    - if any entities removed from last configuration
      - remove the index data and relations

Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>
This commit is contained in:
Adrien de Peretti
2025-02-05 16:49:18 +00:00
committed by GitHub
co-authored by Carlos R. L. Rodrigues
parent 60f46e07fd
commit a33aebd895
35 changed files with 1677 additions and 727 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/modules/index/src/migrations/.snapshot-medusa-index.json
+137 -50
View File
@@ -1,5 +1,7 @@
{
"namespaces": ["public"],
"namespaces": [
"public"
],
"name": "public",
"tables": [
{
@@ -32,6 +34,16 @@
"default": "'{}'",
"mappedType": "json"
},
"staled_at": {
"name": "staled_at",
"type": "timestamptz",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": true,
"length": 6,
"mappedType": "datetime"
},
"created_at": {
"name": "created_at",
"type": "timestamptz",
@@ -77,36 +89,12 @@
"unique": false,
"expression": "CREATE INDEX IF NOT EXISTS \"IDX_index_data_deleted_at\" ON \"index_data\" (deleted_at) WHERE deleted_at IS NULL"
},
{
"keyName": "IDX_index_data_gin",
"columnNames": [],
"composite": false,
"constraint": false,
"primary": false,
"unique": false,
"expression": "CREATE INDEX IF NOT EXISTS \"IDX_index_data_gin\" ON \"index_data\" USING GIN (data) WHERE deleted_at IS NULL"
},
{
"keyName": "IDX_index_data_id",
"columnNames": [],
"composite": false,
"constraint": false,
"primary": false,
"unique": false,
"expression": "CREATE INDEX IF NOT EXISTS \"IDX_index_data_id\" ON \"index_data\" (id) WHERE deleted_at IS NULL"
},
{
"keyName": "IDX_index_data_name",
"columnNames": [],
"composite": false,
"constraint": false,
"primary": false,
"unique": false,
"expression": "CREATE INDEX IF NOT EXISTS \"IDX_index_data_name\" ON \"index_data\" (name) WHERE deleted_at IS NULL"
},
{
"keyName": "index_data_pkey",
"columnNames": ["id", "name"],
"columnNames": [
"id",
"name"
],
"composite": true,
"constraint": true,
"primary": true,
@@ -163,7 +151,12 @@
"primary": false,
"nullable": false,
"default": "'pending'",
"enumItems": ["pending", "processing", "done", "error"],
"enumItems": [
"pending",
"processing",
"done",
"error"
],
"mappedType": "enum"
},
"created_at": {
@@ -222,7 +215,9 @@
},
{
"keyName": "index_metadata_pkey",
"columnNames": ["id"],
"columnNames": [
"id"
],
"composite": false,
"constraint": true,
"primary": true,
@@ -289,6 +284,16 @@
"nullable": false,
"mappedType": "text"
},
"staled_at": {
"name": "staled_at",
"type": "timestamptz",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": true,
"length": 6,
"mappedType": "datetime"
},
"created_at": {
"name": "created_at",
"type": "timestamptz",
@@ -325,24 +330,6 @@
"name": "index_relation",
"schema": "public",
"indexes": [
{
"keyName": "IDX_index_relation_parent_id",
"columnNames": [],
"composite": false,
"constraint": false,
"primary": false,
"unique": false,
"expression": "CREATE INDEX IF NOT EXISTS \"IDX_index_relation_parent_id\" ON \"index_relation\" (parent_id) WHERE deleted_at IS NULL"
},
{
"keyName": "IDX_index_relation_child_id",
"columnNames": [],
"composite": false,
"constraint": false,
"primary": false,
"unique": false,
"expression": "CREATE INDEX IF NOT EXISTS \"IDX_index_relation_child_id\" ON \"index_relation\" (child_id) WHERE deleted_at IS NULL"
},
{
"keyName": "IDX_index_relation_deleted_at",
"columnNames": [],
@@ -354,7 +341,107 @@
},
{
"keyName": "index_relation_pkey",
"columnNames": ["id"],
"columnNames": [
"id"
],
"composite": false,
"constraint": true,
"primary": true,
"unique": true
}
],
"checks": [],
"foreignKeys": {},
"nativeEnums": {}
},
{
"columns": {
"id": {
"name": "id",
"type": "text",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": false,
"mappedType": "text"
},
"entity": {
"name": "entity",
"type": "text",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": false,
"mappedType": "text"
},
"last_key": {
"name": "last_key",
"type": "text",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": true,
"mappedType": "text"
},
"created_at": {
"name": "created_at",
"type": "timestamptz",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": false,
"length": 6,
"default": "now()",
"mappedType": "datetime"
},
"updated_at": {
"name": "updated_at",
"type": "timestamptz",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": false,
"length": 6,
"default": "now()",
"mappedType": "datetime"
},
"deleted_at": {
"name": "deleted_at",
"type": "timestamptz",
"unsigned": false,
"autoincrement": false,
"primary": false,
"nullable": true,
"length": 6,
"mappedType": "datetime"
}
},
"name": "index_sync",
"schema": "public",
"indexes": [
{
"keyName": "IDX_index_sync_deleted_at",
"columnNames": [],
"composite": false,
"constraint": false,
"primary": false,
"unique": false,
"expression": "CREATE INDEX IF NOT EXISTS \"IDX_index_sync_deleted_at\" ON \"index_sync\" (deleted_at) WHERE deleted_at IS NULL"
},
{
"keyName": "IDX_index_sync_entity",
"columnNames": [],
"composite": false,
"constraint": false,
"primary": false,
"unique": false,
"expression": "CREATE UNIQUE INDEX IF NOT EXISTS \"IDX_index_sync_entity\" ON \"index_sync\" (entity) WHERE deleted_at IS NULL"
},
{
"keyName": "index_sync_pkey",
"columnNames": [
"id"
],
"composite": false,
"constraint": true,
"primary": true,
packages/modules/index/src/migrations/Migration20231019174230.ts
-8
View File
@@ -5,17 +5,9 @@ export class Migration20231019174230 extends Migration {
this.addSql(
`create table "index_data" ("id" text not null, "name" text not null, "data" jsonb not null default '{}', constraint "index_data_pkey" primary key ("id", "name")) PARTITION BY LIST("name");`
)
this.addSql(`create index "IDX_index_data_id" on "index_data" ("id");`)
this.addSql(`create index "IDX_index_data_name" on "index_data" ("name");`)
this.addSql(
`create index "IDX_index_data_gin" on "index_data" using GIN ("data");`
)
this.addSql(
`create table "index_relation" ("id" bigserial, "pivot" text not null, "parent_id" text not null, "parent_name" text not null, "child_id" text not null, "child_name" text not null, constraint "index_relation_pkey" primary key ("id", "pivot")) PARTITION BY LIST("pivot");`
)
this.addSql(
`create index "IDX_index_relation_child_id" on "index_relation" ("child_id");`
)
}
}
packages/modules/index/src/migrations/Migration20250127144442.ts
+23
View File
@@ -0,0 +1,23 @@
import { Migration } from "@mikro-orm/migrations"
export class Migration20250127144442 extends Migration {
override async up(): Promise<void> {
this.addSql(
`alter table if exists "index_data" add column if not exists "staled_at" timestamptz null;`
)
this.addSql(
`alter table if exists "index_relation" add column if not exists "staled_at" timestamptz null;`
)
}
override async down(): Promise<void> {
this.addSql(
`alter table if exists "index_data" drop column if exists "staled_at";`
)
this.addSql(
`alter table if exists "index_relation" drop column if exists "staled_at";`
)
}
}
packages/modules/index/src/migrations/Migration20250128132404.ts
+19
View File
@@ -0,0 +1,19 @@
import { Migration } from "@mikro-orm/migrations"
export class Migration20250128132404 extends Migration {
override async up(): Promise<void> {
this.addSql(
`create table if not exists "index_sync" ("id" text not null, "entity" text not null, "last_key" text null, "created_at" timestamptz not null default now(), "updated_at" timestamptz not null default now(), "deleted_at" timestamptz null, constraint "index_sync_pkey" primary key ("id"));`
)
this.addSql(
`CREATE INDEX IF NOT EXISTS "IDX_index_sync_deleted_at" ON "index_sync" (deleted_at) WHERE deleted_at IS NULL;`
)
this.addSql(
`CREATE UNIQUE INDEX IF NOT EXISTS "IDX_index_sync_entity" ON "index_sync" (entity) WHERE deleted_at IS NULL;`
)
}
override async down(): Promise<void> {
this.addSql(`drop table if exists "index_sync" cascade;`)
}
}
packages/modules/index/src/models/index-data.ts
+6 -21
View File
@@ -1,25 +1,10 @@
import { model } from "@medusajs/framework/utils"
const IndexData = model
.define("IndexData", {
id: model.text().primaryKey(),
name: model.text().primaryKey(),
data: model.json().default({}),
})
.indexes([
{
name: "IDX_index_data_gin",
type: "GIN",
on: ["data"],
},
{
name: "IDX_index_data_id",
on: ["id"],
},
{
name: "IDX_index_data_name",
on: ["name"],
},
])
const IndexData = model.define("IndexData", {
id: model.text().primaryKey(),
name: model.text().primaryKey(),
data: model.json().default({}),
staled_at: model.dateTime().nullable(),
})
export default IndexData
packages/modules/index/src/models/index-metadata.ts
+1 -1
View File
@@ -3,7 +3,7 @@ import { IndexMetadataStatus } from "../utils/index-metadata-status"
const IndexMetadata = model
.define("IndexMetadata", {
id: model.id().primaryKey(),
id: model.id({ prefix: "idxmeta" }).primaryKey(),
entity: model.text(),
fields: model.text(),
fields_hash: model.text(),
packages/modules/index/src/models/index-relation.ts
+3 -2
View File
@@ -4,9 +4,10 @@ const IndexRelation = model.define("IndexRelation", {
id: model.autoincrement().primaryKey(),
pivot: model.text(),
parent_name: model.text(),
parent_id: model.text().index("IDX_index_relation_parent_id"),
parent_id: model.text(),
child_name: model.text(),
child_id: model.text().index("IDX_index_relation_child_id"),
child_id: model.text(),
staled_at: model.dateTime().nullable(),
})
export default IndexRelation
packages/modules/index/src/models/index-sync.ts
+17
View File
@@ -0,0 +1,17 @@
import { model } from "@medusajs/framework/utils"
const IndexSync = model
.define("IndexSync", {
id: model.id({ prefix: "idxsync" }).primaryKey(),
entity: model.text(),
last_key: model.text().nullable(),
})
.indexes([
{
name: "IDX_index_sync_entity",
on: ["entity"],
unique: true,
},
])
export default IndexSync
packages/modules/index/src/models/index.ts
+1
View File
@@ -1,3 +1,4 @@
export { default as IndexData } from "./index-data"
export { default as IndexMetadata } from "./index-metadata"
export { default as IndexRelation } from "./index-relation"
export { default as IndexSync } from "./index-sync"
packages/modules/index/src/services/data-synchronizer.ts
+356
View File
@@ -0,0 +1,356 @@
import {
CommonEvents,
ContainerRegistrationKeys,
groupBy,
Modules,
promiseAll,
} from "@medusajs/framework/utils"
import {
Event,
ILockingModule,
IndexTypes,
Logger,
ModulesSdkTypes,
RemoteQueryFunction,
SchemaObjectEntityRepresentation,
} from "@medusajs/types"
import { IndexMetadataStatus, Orchestrator } from "@utils"
export class DataSynchronizer {
#container: Record<string, any>
#isReady: boolean = false
#schemaObjectRepresentation: IndexTypes.SchemaObjectRepresentation
#storageProvider: IndexTypes.StorageProvider
#orchestrator!: Orchestrator
get #query() {
return this.#container[
ContainerRegistrationKeys.QUERY
] as RemoteQueryFunction
}
get #locking() {
return this.#container[Modules.LOCKING] as ILockingModule
}
get #indexMetadataService(): ModulesSdkTypes.IMedusaInternalService<any> {
return this.#container.indexMetadataService
}
get #indexSyncService(): ModulesSdkTypes.IMedusaInternalService<any> {
return this.#container.indexSyncService
}
get #indexDataService(): ModulesSdkTypes.IMedusaInternalService<any> {
return this.#container.indexDataService
}
// @ts-ignore
get #indexRelationService(): ModulesSdkTypes.IMedusaInternalService<any> {
return this.#container.indexRelationService
}
get #logger(): Logger {
try {
return this.#container.logger
} catch (err) {
return console as unknown as Logger
}
}
constructor(container: Record<string, any>) {
this.#container = container
}
#isReadyOrThrow() {
if (!this.#isReady) {
throw new Error(
"DataSynchronizer is not ready. Call onApplicationStart first."
)
}
}
onApplicationStart({
schemaObjectRepresentation,
storageProvider,
}: {
lockDuration?: number
schemaObjectRepresentation: IndexTypes.SchemaObjectRepresentation
storageProvider: IndexTypes.StorageProvider
}) {
this.#storageProvider = storageProvider
this.#schemaObjectRepresentation = schemaObjectRepresentation
this.#isReady = true
}
async syncEntities(
entities: {
entity: string
fields: string
fields_hash: string
}[],
lockDuration: number = 1000 * 60 * 5
) {
this.#isReadyOrThrow()
const entitiesToSync = entities.map((entity) => entity.entity)
this.#orchestrator = new Orchestrator(this.#locking, entitiesToSync, {
lockDuration,
})
await this.#orchestrator.process(this.#taskRunner.bind(this))
}
async removeEntities(entities: string[], staleOnly: boolean = false) {
this.#isReadyOrThrow()
const staleCondition = staleOnly ? { staled_at: { $ne: null } } : {}
const dataToDelete = await this.#indexDataService.list({
...staleCondition,
name: entities,
})
const toDeleteByEntity = groupBy(dataToDelete, "name")
for (const entity of toDeleteByEntity.keys()) {
const records = toDeleteByEntity.get(entity)
const ids = records?.map(
(record: { data: { id: string } }) => record.data.id
)
if (!ids?.length) {
continue
}
if (this.#schemaObjectRepresentation[entity]) {
// Here we assume that some data have been deleted from from the source and we are cleaning since they are still staled in the index and we remove them from the index
// TODO: expand storage provider interface
await (this.#storageProvider as any).onDelete({
entity,
data: ids,
schemaEntityObjectRepresentation:
this.#schemaObjectRepresentation[entity],
})
} else {
// Here we assume that the entity is not indexed anymore as it is not part of the schema object representation and we are cleaning the index
await promiseAll([
this.#indexDataService.delete({
selector: {
name: entity,
},
}),
this.#indexRelationService.delete({
selector: {
$or: [{ parent_id: entity }, { child_id: entity }],
},
}),
])
}
}
}
async #updatedStatus(entity: string, status: IndexMetadataStatus) {
await this.#indexMetadataService.update({
data: {
status,
},
selector: {
entity,
},
})
}
async #taskRunner(entity: string) {
const [[lastCursor]] = await promiseAll([
this.#indexSyncService.list(
{
entity,
},
{
select: ["last_key"],
}
),
this.#updatedStatus(entity, IndexMetadataStatus.PROCESSING),
this.#indexDataService.update({
data: {
staled_at: new Date(),
},
selector: {
name: entity,
},
}),
])
const finalAcknoledgement = await this.syncEntity({
entityName: entity,
pagination: {
cursor: lastCursor?.last_key,
},
ack: async (ack) => {
const promises: Promise<any>[] = []
if (ack.lastCursor) {
promises.push(
this.#indexSyncService.update({
data: {
last_key: ack.lastCursor,
},
selector: {
entity,
},
})
)
if (!ack.done && !ack.err) {
promises.push(this.#orchestrator.renewLock(entity))
}
}
await promiseAll(promises)
},
})
if (finalAcknoledgement.done) {
await promiseAll([
this.#updatedStatus(entity, IndexMetadataStatus.DONE),
this.#indexSyncService.update({
data: {
last_key: finalAcknoledgement.lastCursor,
},
selector: {
entity,
},
}),
this.removeEntities([entity], true),
])
}
if (finalAcknoledgement.err) {
await this.#updatedStatus(entity, IndexMetadataStatus.ERROR)
}
}
async syncEntity({
entityName,
pagination = {},
ack,
}: {
entityName: string
pagination?: {
cursor?: string
updated_at?: string | Date
limit?: number
batchSize?: number
}
ack: (ack: {
lastCursor: string | null
done?: boolean
err?: Error
}) => Promise<void>
}): Promise<{
lastCursor: string | null
done?: boolean
err?: Error
}> {
this.#isReadyOrThrow()
const schemaEntityObjectRepresentation = this.#schemaObjectRepresentation[
entityName
] as SchemaObjectEntityRepresentation
const { fields, alias, moduleConfig } = schemaEntityObjectRepresentation
const isLink = !!moduleConfig?.isLink
const entityPrimaryKey = fields.find(
(field) => !!moduleConfig?.primaryKeys?.includes(field)
)
if (!entityPrimaryKey) {
// TODO: for now these are skiped
const acknoledgement = {
lastCursor: pagination.cursor ?? null,
done: true,
}
await ack(acknoledgement)
return acknoledgement
}
let processed = 0
let currentCursor = pagination.cursor!
const batchSize = Math.min(pagination.batchSize ?? 100, 100)
const limit = pagination.limit ?? Infinity
let done = false
let error = null
while (processed < limit || !done) {
const filters: Record<string, any> = {}
if (currentCursor) {
filters[entityPrimaryKey] = { $gt: currentCursor }
}
if (pagination.updated_at) {
filters["updated_at"] = { $gt: pagination.updated_at }
}
const { data } = await this.#query.graph({
entity: alias,
fields: [entityPrimaryKey],
filters,
pagination: {
order: {
[entityPrimaryKey]: "asc",
},
take: batchSize,
},
})
done = !data.length
if (done) {
break
}
const envelop: Event = {
data,
name: !isLink
? `*.${CommonEvents.CREATED}`
: `*.${CommonEvents.ATTACHED}`,
}
try {
await this.#storageProvider.consumeEvent(
schemaEntityObjectRepresentation
)(envelop)
currentCursor = data[data.length - 1][entityPrimaryKey]
processed += data.length
await ack({ lastCursor: currentCursor })
} catch (err) {
this.#logger.error(
`Index engine] sync failed for entity ${entityName}`,
err
)
error = err
break
}
}
let acknoledgement: { lastCursor: string; done?: boolean; err?: Error } = {
lastCursor: currentCursor,
done: true,
}
if (error) {
acknoledgement = {
lastCursor: currentCursor,
err: error,
}
await ack(acknoledgement)
return acknoledgement
}
await ack(acknoledgement)
return acknoledgement
}
}
packages/modules/index/src/services/index-data.ts
+4
View File
@@ -0,0 +1,4 @@
import { MedusaInternalService } from "@medusajs/framework/utils"
import { IndexData } from "@models"
export class IndexDataService extends MedusaInternalService(IndexData) {}
packages/modules/index/src/services/index-metadata.ts
+6
View File
@@ -0,0 +1,6 @@
import { MedusaInternalService } from "@medusajs/framework/utils"
import { IndexMetadata } from "@models"
export class IndexMetadataService extends MedusaInternalService(
IndexMetadata
) {}
packages/modules/index/src/services/index-module-service.ts
+47 -129
View File
@@ -3,6 +3,7 @@ import {
IEventBusModuleService,
IndexTypes,
InternalModuleDeclaration,
Logger,
ModulesSdkTypes,
RemoteQueryFunction,
} from "@medusajs/framework/types"
@@ -11,28 +12,30 @@ import {
ContainerRegistrationKeys,
Modules,
ModulesSdkUtils,
simpleHash,
} from "@medusajs/framework/utils"
import { IndexMetadata } from "@models"
import { schemaObjectRepresentationPropertiesToOmit } from "@types"
import { buildSchemaObjectRepresentation } from "../utils/build-config"
import { defaultSchema } from "../utils/default-schema"
import { gqlSchemaToTypes } from "../utils/gql-to-types"
import { IndexMetadataStatus } from "../utils/index-metadata-status"
import {
buildSchemaObjectRepresentation,
Configuration,
defaultSchema,
gqlSchemaToTypes,
} from "@utils"
import { DataSynchronizer } from "./data-synchronizer"
type InjectedDependencies = {
logger: Logger
[Modules.EVENT_BUS]: IEventBusModuleService
storageProviderCtr: Constructor<IndexTypes.StorageProvider>
[ContainerRegistrationKeys.QUERY]: RemoteQueryFunction
storageProviderCtrOptions: unknown
baseRepository: BaseRepository
indexMetadataService: ModulesSdkTypes.IMedusaInternalService<any>
indexSyncService: ModulesSdkTypes.IMedusaInternalService<any>
dataSynchronizer: DataSynchronizer
}
export default class IndexModuleService
extends ModulesSdkUtils.MedusaService({
IndexMetadata,
})
extends ModulesSdkUtils.MedusaService({})
implements IndexTypes.IIndexService
{
private readonly container_: InjectedDependencies
@@ -48,7 +51,25 @@ export default class IndexModuleService
protected storageProvider_: IndexTypes.StorageProvider
private indexMetadataService_: ModulesSdkTypes.IMedusaInternalService<any>
private get indexMetadataService_(): ModulesSdkTypes.IMedusaInternalService<any> {
return this.container_.indexMetadataService
}
private get indexSyncService_(): ModulesSdkTypes.IMedusaInternalService<any> {
return this.container_.indexSyncService
}
private get dataSynchronizer_(): DataSynchronizer {
return this.container_.dataSynchronizer
}
private get logger_(): Logger {
try {
return this.container_.logger
} catch (e) {
return console as unknown as Logger
}
}
constructor(
container: InjectedDependencies,
@@ -62,7 +83,6 @@ export default class IndexModuleService
const {
[Modules.EVENT_BUS]: eventBusModuleService,
indexMetadataService,
storageProviderCtr,
storageProviderCtrOptions,
} = container
@@ -70,8 +90,6 @@ export default class IndexModuleService
this.eventBusModuleService_ = eventBusModuleService
this.storageProviderCtr_ = storageProviderCtr
this.storageProviderCtrOptions_ = storageProviderCtrOptions
this.indexMetadataService_ = indexMetadataService
if (!this.eventBusModuleService_) {
throw new Error(
"EventBusModuleService is required for the IndexModule to work"
@@ -106,124 +124,24 @@ export default class IndexModuleService
await gqlSchemaToTypes(this.moduleOptions_.schema ?? defaultSchema)
const fullSyncRequired = await this.syncIndexConfig()
if (fullSyncRequired.length > 0) {
await this.syncEntities(fullSyncRequired)
this.dataSynchronizer_.onApplicationStart({
schemaObjectRepresentation: this.schemaObjectRepresentation_,
storageProvider: this.storageProvider_,
})
const configurationChecker = new Configuration({
schemaObjectRepresentation: this.schemaObjectRepresentation_,
indexMetadataService: this.indexMetadataService_,
indexSyncService: this.indexSyncService_,
dataSynchronizer: this.dataSynchronizer_,
})
const entitiesMetadataChanged = await configurationChecker.checkChanges()
if (entitiesMetadataChanged.length) {
await this.dataSynchronizer_.syncEntities(entitiesMetadataChanged)
}
} catch (e) {
console.log(e)
}
}
private async syncIndexConfig() {
const schemaObjectRepresentation = (this.schemaObjectRepresentation_ ??
{}) as IndexTypes.SchemaObjectRepresentation
const currentConfig = await this.indexMetadataService_.list()
const currentConfigMap = new Map(
currentConfig.map((c) => [c.entity, c] as const)
)
type modifiedConfig = {
id?: string
entity: string
fields: string[]
fields_hash: string
status?: IndexMetadataStatus
}[]
const entityPresent = new Set<string>()
const newConfig: modifiedConfig = []
const updatedConfig: modifiedConfig = []
const deletedConfig: { entity: string }[] = []
for (const [entityName, schemaEntityObjectRepresentation] of Object.entries(
schemaObjectRepresentation
)) {
if (schemaObjectRepresentationPropertiesToOmit.includes(entityName)) {
continue
}
const entity = schemaEntityObjectRepresentation.entity
const fields = schemaEntityObjectRepresentation.fields.sort().join(",")
const fields_hash = simpleHash(fields)
const existingEntityConfig = currentConfigMap.get(entity)
entityPresent.add(entity)
if (
!existingEntityConfig ||
existingEntityConfig.fields_hash !== fields_hash
) {
const meta = {
id: existingEntityConfig?.id,
entity,
fields,
fields_hash,
}
if (!existingEntityConfig) {
newConfig.push(meta)
} else {
updatedConfig.push({
...meta,
status: IndexMetadataStatus.PENDING,
})
}
}
}
for (const [entity] of currentConfigMap) {
if (!entityPresent.has(entity)) {
deletedConfig.push({ entity })
}
}
if (newConfig.length > 0) {
await this.indexMetadataService_.create(newConfig)
}
if (updatedConfig.length > 0) {
await this.indexMetadataService_.update(updatedConfig)
}
if (deletedConfig.length > 0) {
await this.indexMetadataService_.delete(deletedConfig)
}
return await this.indexMetadataService_.list({
status: [IndexMetadataStatus.PENDING, IndexMetadataStatus.PROCESSING],
})
}
private async syncEntities(
entities: {
entity: string
fields: string[]
fields_hash: string
}[]
) {
const updatedStatus = async (
entity: string,
status: IndexMetadataStatus
) => {
await this.indexMetadataService_.update({
data: {
status,
},
selector: {
entity,
},
})
}
for (const entity of entities) {
await updatedStatus(entity.entity, IndexMetadataStatus.PROCESSING)
try {
// await this.syncEntity(entity)
await updatedStatus(entity.entity, IndexMetadataStatus.DONE)
} catch (e) {
await updatedStatus(entity.entity, IndexMetadataStatus.ERROR)
}
this.logger_.error(e)
}
}
packages/modules/index/src/services/index-relation.ts
+6
View File
@@ -0,0 +1,6 @@
import { MedusaInternalService } from "@medusajs/framework/utils"
import { IndexRelation } from "@models"
export class IndexRelationService extends MedusaInternalService(
IndexRelation
) {}
packages/modules/index/src/services/index-sync.ts
+4
View File
@@ -0,0 +1,4 @@
import { MedusaInternalService } from "@medusajs/framework/utils"
import { IndexSync } from "@models"
export class IndexSyncService extends MedusaInternalService(IndexSync) {}
packages/modules/index/src/services/postgres-provider.ts
+70 -71
View File
@@ -208,14 +208,13 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
}
const { fields, alias } = schemaEntityObjectRepresentation
const graphResult = await this.query_.graph({
const { data: entityData } = await this.query_.graph({
entity: alias,
filters: {
id: ids,
},
fields: [...new Set(["id", ...fields])],
})
const { data: entityData } = graphResult
const argument = {
entity: schemaEntityObjectRepresentation.entity,
@@ -327,9 +326,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
* @protected
*/
@InjectTransactionManager()
protected async onCreate<
TData extends { id: string; [key: string]: unknown }
>(
async onCreate<TData extends { id: string; [key: string]: unknown }>(
{
entity,
data,
@@ -339,13 +336,11 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
data: TData | TData[]
schemaEntityObjectRepresentation: IndexTypes.SchemaObjectEntityRepresentation
},
@MedusaContext() sharedContext: Context = {}
@MedusaContext() sharedContext: Context<SqlEntityManager> = {}
) {
const { transactionManager: em } = sharedContext as {
transactionManager: SqlEntityManager
}
const indexRepository = em.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository: EntityRepository<any> = em.getRepository(
const { transactionManager: em } = sharedContext
const indexRepository = em!.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository: EntityRepository<any> = em!.getRepository(
toMikroORMEntity(IndexRelation)
)
@@ -374,7 +369,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
id: cleanedEntityData.id,
name: entity,
data: cleanedEntityData,
// stale: false,
staled_at: null,
})
/**
@@ -400,7 +395,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
id: (parentData_ as any).id,
name: parentEntity,
data: parentData_,
// stale: false,
staled_at: null,
})
await indexRelationRepository.upsert(
@@ -410,7 +405,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
child_id: cleanedEntityData.id,
child_name: entity,
pivot: `${parentEntity}-${entity}`,
// stale: false,
staled_at: null,
},
{
onConflictAction: "merge",
@@ -437,9 +432,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
* @protected
*/
@InjectTransactionManager()
protected async onUpdate<
TData extends { id: string; [key: string]: unknown }
>(
async onUpdate<TData extends { id: string; [key: string]: unknown }>(
{
entity,
data,
@@ -449,12 +442,10 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
data: TData | TData[]
schemaEntityObjectRepresentation: IndexTypes.SchemaObjectEntityRepresentation
},
@MedusaContext() sharedContext: Context = {}
@MedusaContext() sharedContext: Context<SqlEntityManager> = {}
) {
const { transactionManager: em } = sharedContext as {
transactionManager: SqlEntityManager
}
const indexRepository = em.getRepository(toMikroORMEntity(IndexData))
const { transactionManager: em } = sharedContext
const indexRepository = em!.getRepository(toMikroORMEntity(IndexData))
const { data: data_, entityProperties } = PostgresProvider.parseData(
data,
@@ -470,7 +461,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
acc[property] = entityData[property]
return acc
}, {}),
// stale: false,
staled_at: null,
}
})
)
@@ -485,9 +476,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
* @protected
*/
@InjectTransactionManager()
protected async onDelete<
TData extends { id: string; [key: string]: unknown }
>(
async onDelete<TData extends { id: string; [key: string]: unknown }>(
{
entity,
data,
@@ -497,13 +486,11 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
data: TData | TData[]
schemaEntityObjectRepresentation: IndexTypes.SchemaObjectEntityRepresentation
},
@MedusaContext() sharedContext: Context = {}
@MedusaContext() sharedContext: Context<SqlEntityManager> = {}
) {
const { transactionManager: em } = sharedContext as {
transactionManager: SqlEntityManager
}
const indexRepository = em.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository = em.getRepository(
const { transactionManager: em } = sharedContext
const indexRepository = em!.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository = em!.getRepository(
toMikroORMEntity(IndexRelation)
)
@@ -541,9 +528,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
* @protected
*/
@InjectTransactionManager()
protected async onAttach<
TData extends { id: string; [key: string]: unknown }
>(
async onAttach<TData extends { id: string; [key: string]: unknown }>(
{
entity,
data,
@@ -553,13 +538,11 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
data: TData | TData[]
schemaEntityObjectRepresentation: IndexTypes.SchemaObjectEntityRepresentation
},
@MedusaContext() sharedContext: Context = {}
@MedusaContext() sharedContext: Context<SqlEntityManager> = {}
) {
const { transactionManager: em } = sharedContext as {
transactionManager: SqlEntityManager
}
const indexRepository = em.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository = em.getRepository(
const { transactionManager: em } = sharedContext
const indexRepository = em!.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository = em!.getRepository(
toMikroORMEntity(IndexRelation)
)
@@ -626,34 +609,54 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
id: cleanedEntityData.id,
name: entity,
data: cleanedEntityData,
// stale: false,
staled_at: null,
})
/**
* Create the index relation entries for the parent entity and the child entity
*/
const parentIndexRelationEntry = indexRelationRepository.create({
parent_id: entityData[parentPropertyId] as string,
parent_name: parentEntityName,
child_id: cleanedEntityData.id,
child_name: entity,
pivot: `${parentEntityName}-${entity}`,
// stale: false,
})
await indexRelationRepository.upsert(
{
parent_id: entityData[parentPropertyId] as string,
parent_name: parentEntityName,
child_id: cleanedEntityData.id,
child_name: entity,
pivot: `${parentEntityName}-${entity}`,
staled_at: null,
},
{
onConflictAction: "merge",
onConflictFields: [
"pivot",
"parent_id",
"child_id",
"parent_name",
"child_name",
],
}
)
const childIndexRelationEntry = indexRelationRepository.create({
parent_id: cleanedEntityData.id,
parent_name: entity,
child_id: entityData[childPropertyId] as string,
child_name: childEntityName,
pivot: `${entity}-${childEntityName}`,
// stale: false,
})
indexRelationRepository
.getEntityManager()
.persist([parentIndexRelationEntry, childIndexRelationEntry])
await indexRelationRepository.upsert(
{
parent_id: cleanedEntityData.id,
parent_name: entity,
child_id: entityData[childPropertyId] as string,
child_name: childEntityName,
pivot: `${entity}-${childEntityName}`,
staled_at: null,
},
{
onConflictAction: "merge",
onConflictFields: [
"pivot",
"parent_id",
"child_id",
"parent_name",
"child_name",
],
}
)
}
}
@@ -666,9 +669,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
* @protected
*/
@InjectTransactionManager()
protected async onDetach<
TData extends { id: string; [key: string]: unknown }
>(
async onDetach<TData extends { id: string; [key: string]: unknown }>(
{
entity,
data,
@@ -678,13 +679,11 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
data: TData | TData[]
schemaEntityObjectRepresentation: IndexTypes.SchemaObjectEntityRepresentation
},
@MedusaContext() sharedContext: Context = {}
@MedusaContext() sharedContext: Context<SqlEntityManager> = {}
) {
const { transactionManager: em } = sharedContext as {
transactionManager: SqlEntityManager
}
const indexRepository = em.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository = em.getRepository(
const { transactionManager: em } = sharedContext
const indexRepository = em!.getRepository(toMikroORMEntity(IndexData))
const indexRelationRepository = em!.getRepository(
toMikroORMEntity(IndexRelation)
)
packages/modules/index/src/utils/create-partitions.ts
+39
View File
@@ -37,6 +37,45 @@ export async function createPartitions(
return
}
await manager.execute(partitions.join("; "))
// Create indexes for each partition
const indexCreationCommands = Object.keys(schemaObjectRepresentation)
.filter(
(key) =>
!schemaObjectRepresentationPropertiesToOmit.includes(key) &&
schemaObjectRepresentation[key].listeners.length > 0
)
.map((key) => {
const cName = key.toLowerCase()
const part: string[] = []
part.push(
`CREATE INDEX CONCURRENTLY IF NOT EXISTS "IDX_cat_${cName}_data_gin" ON ${activeSchema}cat_${cName} USING GIN ("data" jsonb_path_ops)`
)
// create child id index on pivot partitions
for (const parent of schemaObjectRepresentation[key].parents) {
const pName = `${parent.ref.entity}${key}`.toLowerCase()
part.push(
`CREATE INDEX CONCURRENTLY IF NOT EXISTS "IDX_cat_pivot_${pName}_child_id" ON ${activeSchema}cat_pivot_${pName} ("child_id")`
)
}
return part
})
.flat()
// Execute index creation commands separately to avoid blocking
for (const cmd of indexCreationCommands) {
try {
await manager.execute(cmd)
} catch (error) {
// Log error but continue with other indexes
console.error(`Failed to create index: ${error.message}`)
}
}
partitions.push(`analyse ${activeSchema}index_data`)
partitions.push(`analyse ${activeSchema}index_relation`)
packages/modules/index/src/utils/default-schema.ts
+9 -2
View File
@@ -5,6 +5,7 @@ export const defaultSchema = `
id: String
title: String
variants: [ProductVariant]
sales_channels: [SalesChannel]
}
type ProductVariant @Listeners(values: ["${Modules.PRODUCT}.product-variant.created", "${Modules.PRODUCT}.product-variant.updated", "${Modules.PRODUCT}.product-variant.deleted"]) {
@@ -15,7 +16,13 @@ export const defaultSchema = `
}
type Price @Listeners(values: ["${Modules.PRICING}.price.created", "${Modules.PRICING}.price.updated", "${Modules.PRICING}.price.deleted"]) {
amount: Int
id: String
amount: Float
currency_code: String
}
}
type SalesChannel @Listeners(values: ["${Modules.SALES_CHANNEL}.sales_channel.created", "${Modules.SALES_CHANNEL}.sales_channel.updated", "${Modules.SALES_CHANNEL}.sales_channel.deleted"]) {
id: String
is_disabled: Boolean
}
`
packages/modules/index/src/utils/index.ts
+5
View File
@@ -1,3 +1,8 @@
export * from "./query-builder"
export * from "./create-partitions"
export * from "./build-config"
export * from "./sync/orchestrator"
export * from "./sync/configuration"
export * from "./index-metadata-status"
export * from "./gql-to-types"
export * from "./default-schema"
packages/modules/index/src/utils/query-builder.ts
+170 -48
View File
@@ -23,6 +23,7 @@ export class QueryBuilder {
private readonly selector: QueryFormat
private readonly options?: QueryOptions
private readonly schema: IndexTypes.SchemaObjectRepresentation
private readonly allSchemaFields: Set<string>
constructor(args: {
schema: IndexTypes.SchemaObjectRepresentation
@@ -37,14 +38,24 @@ export class QueryBuilder {
this.options = args.options
this.knex = args.knex
this.structure = this.selector.select
this.allSchemaFields = new Set(
Object.values(this.schema).flatMap((entity) => entity.fields ?? [])
)
}
private getStructureKeys(structure) {
return Object.keys(structure ?? {}).filter((key) => key !== "entity")
}
private getEntity(path): IndexTypes.SchemaPropertiesMap[0] {
private getEntity(
path,
throwWhenNotFound = true
): IndexTypes.SchemaPropertiesMap[0] | undefined {
if (!this.schema._schemaPropertiesMap[path]) {
if (!throwWhenNotFound) {
return
}
throw new Error(`Could not find entity for path: ${path}`)
}
@@ -52,7 +63,7 @@ export class QueryBuilder {
}
private getGraphQLType(path, field) {
const entity = this.getEntity(path)?.ref?.entity
const entity = this.getEntity(path)?.ref?.entity!
const fieldRef = this.entityMap[entity]._fields[field]
if (!fieldRef) {
throw new Error(`Field ${field} not found in the entityMap.`)
@@ -270,6 +281,18 @@ export class QueryBuilder {
return builder
}
private getShortAlias(aliasMapping, alias: string) {
aliasMapping.__aliasIndex ??= 0
if (aliasMapping[alias]) {
return aliasMapping[alias]
}
aliasMapping[alias] = "t_" + aliasMapping.__aliasIndex++ + "_"
return aliasMapping[alias]
}
private buildQueryParts(
structure: Select,
parentAlias: string,
@@ -281,10 +304,17 @@ export class QueryBuilder {
): string[] {
const currentAliasPath = [...aliasPath, parentProperty].join(".")
const entities = this.getEntity(currentAliasPath)
const isSelectableField = this.allSchemaFields.has(parentProperty)
const entities = this.getEntity(currentAliasPath, false)
if (isSelectableField || !entities) {
// We are currently selecting a specific field of the parent entity or the entity is not found on the index schema
// We don't need to build the query parts for this as there is no join
return []
}
const mainEntity = entities.ref.entity
const mainAlias = mainEntity.toLowerCase() + level
const mainAlias =
this.getShortAlias(aliasMapping, mainEntity.toLowerCase()) + level
const allEntities: any[] = []
if (!entities.shortCutOf) {
@@ -298,7 +328,13 @@ export class QueryBuilder {
const intermediateAlias = entities.shortCutOf.split(".")
for (let i = intermediateAlias.length - 1, x = 0; i >= 0; i--, x++) {
const intermediateEntity = this.getEntity(intermediateAlias.join("."))
const intermediateEntity = this.getEntity(
intermediateAlias.join("."),
false
)
if (!intermediateEntity) {
break
}
intermediateAlias.pop()
@@ -308,14 +344,24 @@ export class QueryBuilder {
const parentIntermediateEntity = this.getEntity(
intermediateAlias.join(".")
)
)!
const alias =
intermediateEntity.ref.entity.toLowerCase() + level + "_" + x
this.getShortAlias(
aliasMapping,
intermediateEntity.ref.entity.toLowerCase()
) +
level +
"_" +
x
const parAlias =
parentIntermediateEntity.ref.entity === parentEntity
? parentAlias
: parentIntermediateEntity.ref.entity.toLowerCase() +
: this.getShortAlias(
aliasMapping,
parentIntermediateEntity.ref.entity.toLowerCase()
) +
level +
"_" +
(x + 1)
@@ -335,62 +381,68 @@ export class QueryBuilder {
let queryParts: string[] = []
for (const join of allEntities) {
const joinBuilder = this.knex.queryBuilder()
const { alias, entity, parEntity, parAlias } = join
aliasMapping[currentAliasPath] = alias
if (level > 0) {
const subQuery = this.knex.queryBuilder()
const knex = this.knex
subQuery
.select(`${alias}.id`, `${alias}.data`)
.from("index_data AS " + alias)
.join(`index_relation AS ${alias}_ref`, function () {
this.on(
`${alias}_ref.pivot`,
"=",
knex.raw("?", [`${parEntity}-${entity}`])
)
.andOn(`${alias}_ref.parent_id`, "=", `${parAlias}.id`)
.andOn(`${alias}.id`, "=", `${alias}_ref.child_id`)
})
.where(`${alias}.name`, "=", knex.raw("?", [entity]))
const cName = entity.toLowerCase()
const pName = `${parEntity}${entity}`.toLowerCase()
let joinTable = `cat_${cName} AS ${alias}`
const pivotTable = `cat_pivot_${pName}`
joinBuilder.leftJoin(
`${pivotTable} AS ${alias}_ref`,
`${alias}_ref.parent_id`,
`${parAlias}.id`
)
joinBuilder.leftJoin(joinTable, `${alias}.id`, `${alias}_ref.child_id`)
const joinWhere = this.selector.joinWhere ?? {}
const joinKey = Object.keys(joinWhere).find((key) => {
const k = key.split(".")
k.pop()
return k.join(".") === currentAliasPath
const curPath = k.join(".")
if (curPath === currentAliasPath) {
const relEntity = this.getEntity(curPath, false)
return relEntity?.ref?.entity === entity
}
return false
})
if (joinKey) {
this.parseWhere(
aliasMapping,
{ [joinKey]: joinWhere[joinKey] },
subQuery
joinBuilder
)
}
queryParts.push(`LEFT JOIN LATERAL (
${subQuery.toQuery()}
) ${alias} ON TRUE`)
queryParts.push(
joinBuilder.toQuery().replace("select * ", "").replace("where", "and")
)
}
}
const children = this.getStructureKeys(structure)
for (const child of children) {
const childStructure = structure[child] as Select
queryParts = queryParts.concat(
this.buildQueryParts(
childStructure,
mainAlias,
mainEntity,
child,
aliasPath.concat(parentProperty),
level + 1,
aliasMapping
queryParts = queryParts
.concat(
this.buildQueryParts(
childStructure,
mainAlias,
mainEntity,
child,
aliasPath.concat(parentProperty),
level + 1,
aliasMapping
)
)
)
.filter(Boolean)
}
return queryParts
@@ -404,7 +456,26 @@ export class QueryBuilder {
selectParts: object = {}
): object {
const currentAliasPath = [...aliasPath, parentProperty].join(".")
const isSelectableField = this.allSchemaFields.has(parentProperty)
if (isSelectableField) {
// We are currently selecting a specific field of the parent entity
// Let's remove the parent alias from the select parts to not select everything entirely
// and add the specific field to the select parts
const parentAliasPath = aliasPath.join(".")
const alias = aliasMapping[parentAliasPath]
delete selectParts[parentAliasPath]
selectParts[currentAliasPath] = this.knex.raw(
`${alias}.data->'${parentProperty}'`
)
return selectParts
}
const alias = aliasMapping[currentAliasPath]
// If the entity is not found in the schema (not indexed), we don't need to build the select parts
if (!alias) {
return selectParts
}
selectParts[currentAliasPath] = `${alias}.data`
selectParts[currentAliasPath + ".id"] = `${alias}.id`
@@ -473,7 +544,8 @@ export class QueryBuilder {
const rootKey = this.getStructureKeys(structure)[0]
const rootStructure = structure[rootKey] as Select
const entity = this.getEntity(rootKey).ref.entity
const entity = this.getEntity(rootKey)!.ref.entity
const rootEntity = entity.toLowerCase()
const aliasMapping: { [path: string]: string } = {}
@@ -494,20 +566,23 @@ export class QueryBuilder {
if (countAllResults) {
selectParts["offset_"] = this.knex.raw(
`DENSE_RANK() OVER (ORDER BY ${rootEntity}0.id)`
`DENSE_RANK() OVER (ORDER BY ${this.getShortAlias(
aliasMapping,
rootEntity
)}.id)`
)
}
queryBuilder.select(selectParts)
queryBuilder.from(`index_data AS ${rootEntity}0`)
queryBuilder.from(
`cat_${rootEntity} AS ${this.getShortAlias(aliasMapping, rootEntity)}`
)
joinParts.forEach((joinPart) => {
queryBuilder.joinRaw(joinPart)
})
queryBuilder.where(`${aliasMapping[rootEntity]}.name`, "=", entity)
// WHERE clause
this.parseWhere(aliasMapping, filter, queryBuilder)
@@ -559,6 +634,11 @@ export class QueryBuilder {
const initializeMaps = (structure: Select, path: string[]) => {
const currentPath = path.join(".")
const entity = this.getEntity(currentPath, false)
if (!entity) {
return
}
maps[currentPath] = {}
if (path.length > 1) {
@@ -566,9 +646,19 @@ export class QueryBuilder {
const parents = path.slice(0, -1)
const parentPath = parents.join(".")
isListMap[currentPath] = !!this.getEntity(currentPath).ref.parents.find(
(p) => p.targetProp === property
)?.isList
// In the case of specific selection
// We dont need to check if the property is a list
const isSelectableField = this.allSchemaFields.has(property)
if (isSelectableField) {
pathDetails[currentPath] = { property, parents, parentPath }
isListMap[currentPath] = false
return
}
isListMap[currentPath] = !!this.getEntity(
currentPath,
false
)?.ref?.parents?.find((p) => p.targetProp === property)?.isList
pathDetails[currentPath] = { property, parents, parentPath }
}
@@ -595,6 +685,20 @@ export class QueryBuilder {
return key + id
}
const columnMap = {}
const columnNames = Object.keys(resultSet[0] ?? {})
for (const property of columnNames) {
const segments = property.split(".")
const field = segments.pop()
const parent = segments.join(".")
columnMap[parent] ??= []
columnMap[parent].push({
field,
property,
})
}
resultSet.forEach((row) => {
for (const path in maps) {
const id = row[`${path}.id`]
@@ -603,6 +707,15 @@ export class QueryBuilder {
if (!pathDetails[path]) {
if (!maps[path][id]) {
maps[path][id] = row[path] || undefined
// If there is an id, but no object values, it means that specific fields were selected
// so we recompose the object with all selected fields. (id will always be selected)
if (!maps[path][id] && id) {
maps[path][id] = {}
for (const column of columnMap[path]) {
maps[path][id][column.field] = row[column.property]
}
}
}
continue
}
@@ -616,6 +729,15 @@ export class QueryBuilder {
maps[path][id] = row[path] || undefined
// If there is an id, but no object values, it means that specific fields were selected
// so we recompose the object with all selected fields. (id will always be selected)
if (!maps[path][id] && id) {
maps[path][id] = {}
for (const column of columnMap[path]) {
maps[path][id][column.field] = row[column.property]
}
}
const parentObj = maps[parentPath][row[`${parentPath}.id`]]
if (!parentObj) {
@@ -623,8 +745,8 @@ export class QueryBuilder {
}
const isList = isListMap[parentPath + "." + property]
if (isList) {
parentObj[property] ??= []
if (isList && !Array.isArray(parentObj[property])) {
parentObj[property] = []
}
if (maps[path][id] !== undefined) {
packages/modules/index/src/utils/sync/configuration.ts
+142
View File
@@ -0,0 +1,142 @@
import { simpleHash } from "@medusajs/framework/utils"
import { IndexTypes, InferEntityType } from "@medusajs/types"
import { IndexMetadata } from "@models"
import { schemaObjectRepresentationPropertiesToOmit } from "@types"
import { DataSynchronizer } from "../../services/data-synchronizer"
import { IndexMetadataService } from "../../services/index-metadata"
import { IndexSyncService } from "../../services/index-sync"
import { IndexMetadataStatus } from "../index-metadata-status"
export class Configuration {
#schemaObjectRepresentation: IndexTypes.SchemaObjectRepresentation
#indexMetadataService: IndexMetadataService
#indexSyncService: IndexSyncService
#dataSynchronizer: DataSynchronizer
constructor({
schemaObjectRepresentation,
indexMetadataService,
indexSyncService,
dataSynchronizer,
}: {
schemaObjectRepresentation: IndexTypes.SchemaObjectRepresentation
indexMetadataService: IndexMetadataService
indexSyncService: IndexSyncService
dataSynchronizer: DataSynchronizer
}) {
this.#schemaObjectRepresentation = schemaObjectRepresentation ?? {}
this.#indexMetadataService = indexMetadataService
this.#indexSyncService = indexSyncService
this.#dataSynchronizer = dataSynchronizer
}
async checkChanges(): Promise<InferEntityType<typeof IndexMetadata>[]> {
const schemaObjectRepresentation = this.#schemaObjectRepresentation
const currentConfig = await this.#indexMetadataService.list()
const currentConfigMap = new Map(
currentConfig.map((c) => [c.entity, c] as const)
)
type modifiedConfig = {
id?: string
entity: string
fields: string[]
fields_hash: string
status?: IndexMetadataStatus
}[]
type dataSyncEntry = {
id?: string
entity: string
last_key: null
}[]
const entityPresent = new Set<string>()
const newConfig: modifiedConfig = []
const updatedConfig: modifiedConfig = []
const deletedConfig: { entity: string }[] = []
const idxSyncData: dataSyncEntry = []
for (const [entityName, schemaEntityObjectRepresentation] of Object.entries(
schemaObjectRepresentation
)) {
if (schemaObjectRepresentationPropertiesToOmit.includes(entityName)) {
continue
}
const entity = schemaEntityObjectRepresentation.entity
const fields = schemaEntityObjectRepresentation.fields.sort().join(",")
const fields_hash = simpleHash(fields)
const existingEntityConfig = currentConfigMap.get(entity)
entityPresent.add(entity)
if (
!existingEntityConfig ||
existingEntityConfig.fields_hash !== fields_hash
) {
const meta = {
id: existingEntityConfig?.id,
entity,
fields,
fields_hash,
}
if (!existingEntityConfig) {
newConfig.push(meta)
} else {
updatedConfig.push({
...meta,
status: IndexMetadataStatus.PENDING,
})
}
idxSyncData.push({
entity,
last_key: null,
})
}
}
for (const [entity] of currentConfigMap) {
if (!entityPresent.has(entity)) {
deletedConfig.push({ entity })
}
}
if (newConfig.length > 0) {
await this.#indexMetadataService.create(newConfig)
}
if (updatedConfig.length > 0) {
await this.#indexMetadataService.update(updatedConfig)
}
if (deletedConfig.length > 0) {
await this.#indexMetadataService.delete(deletedConfig)
await this.#dataSynchronizer.removeEntities(
deletedConfig.map((c) => c.entity)
)
}
if (idxSyncData.length > 0) {
if (updatedConfig.length > 0) {
const ids = await this.#indexSyncService.list({
entity: updatedConfig.map((c) => c.entity),
})
idxSyncData.forEach((sync) => {
const id = ids.find((i) => i.entity === sync.entity)?.id
if (id) {
sync.id = id
}
})
}
await this.#indexSyncService.upsert(idxSyncData)
}
return await this.#indexMetadataService.list({
status: [IndexMetadataStatus.PENDING, IndexMetadataStatus.PROCESSING],
})
}
}
packages/modules/index/src/utils/sync/data-synchronizer.ts
-137
View File
@@ -1,137 +0,0 @@
import {
IndexTypes,
RemoteQueryFunction,
SchemaObjectEntityRepresentation,
Event,
} from "@medusajs/framework/types"
import { CommonEvents } from "@medusajs/framework/utils"
export class DataSynchronizer {
#storageProvider: IndexTypes.StorageProvider
#schemaObjectRepresentation: IndexTypes.SchemaObjectRepresentation
#query: RemoteQueryFunction
constructor({
storageProvider,
schemaObjectRepresentation,
query,
}: {
storageProvider: IndexTypes.StorageProvider
schemaObjectRepresentation: IndexTypes.SchemaObjectRepresentation
query: RemoteQueryFunction
}) {
this.#storageProvider = storageProvider
this.#schemaObjectRepresentation = schemaObjectRepresentation
this.#query = query
}
async sync({
entityName,
pagination = {},
ack,
}: {
entityName: string
pagination?: {
cursor?: string
updated_at?: string | Date
limit?: number
batchSize?: number
}
ack: (ack: {
lastCursor: string | null
done?: boolean
err?: Error
}) => Promise<void>
}) {
const schemaEntityObjectRepresentation = this.#schemaObjectRepresentation[
entityName
] as SchemaObjectEntityRepresentation
const { fields, alias, moduleConfig } = schemaEntityObjectRepresentation
const entityPrimaryKey = fields.find(
(field) => !!moduleConfig.linkableKeys?.[field]
)
if (!entityPrimaryKey) {
void ack({
lastCursor: pagination.cursor ?? null,
err: new Error(
`Entity ${entityName} does not have a linkable primary key`
),
})
return
}
let processed = 0
let currentCursor = pagination.cursor!
const batchSize = pagination.batchSize ?? 1000
const limit = pagination.limit ?? Infinity
let done = false
let error = null
while (processed < limit || !done) {
const filters: Record<string, any> = {}
if (currentCursor) {
filters[entityPrimaryKey] = { $gt: currentCursor }
}
if (pagination.updated_at) {
filters["updated_at"] = { $gt: pagination.updated_at }
}
const { data } = await this.#query.graph({
entity: alias,
fields: [entityPrimaryKey],
filters,
pagination: {
order: {
[entityPrimaryKey]: "asc",
},
take: batchSize,
},
})
done = !data.length
if (done) {
break
}
const envelop: Event = {
data,
name: `*.${CommonEvents.CREATED}`,
}
try {
await this.#storageProvider.consumeEvent(
schemaEntityObjectRepresentation
)(envelop)
currentCursor = data[data.length - 1][entityPrimaryKey]
processed += data.length
void ack({ lastCursor: currentCursor })
} catch (err) {
error = err
break
}
}
let acknoledgement: { lastCursor: string; done?: boolean; err?: Error } = {
lastCursor: currentCursor,
done: true,
}
if (error) {
acknoledgement = {
lastCursor: currentCursor,
err: error,
}
void ack(acknoledgement)
return acknoledgement
}
void ack(acknoledgement)
return acknoledgement
}
}
packages/modules/index/src/orchestrator/index.ts → packages/modules/index/src/utils/sync/orchestrator.ts
+41 -30
View File
@@ -1,4 +1,4 @@
import { ILockingModule, IndexTypes, MedusaContainer } from "@medusajs/types"
import { ILockingModule } from "@medusajs/types"
export class Orchestrator {
/**
@@ -29,18 +29,11 @@ export class Orchestrator {
* - Lock duration is the maximum duration for which to hold the lock.
* After this the lock will be removed.
*
* - Task runner is the implementation function to execute a task.
* Orchestrator has no inbuilt execution logic and it relies on
* the task runner for the same.
*
* The entity is provided to the taskRunner only when the orchestrator
* is able to acquire a lock.
*/
#options: {
lockDuration: number
taskRunner: (
entity: IndexTypes.SchemaObjectEntityRepresentation
) => Promise<void>
}
/**
@@ -53,7 +46,7 @@ export class Orchestrator {
* while an entity is getting synced to avoid multiple processes
* from syncing the same entity
*/
#entities: IndexTypes.SchemaObjectEntityRepresentation[] = []
#entities: string[] = []
/**
* The current state of the orchestrator
@@ -77,16 +70,13 @@ export class Orchestrator {
}
constructor(
container: MedusaContainer,
entities: IndexTypes.SchemaObjectEntityRepresentation[],
lockingModule: ILockingModule,
entities: string[],
options: {
lockDuration: number
taskRunner: (
entity: IndexTypes.SchemaObjectEntityRepresentation
) => Promise<void>
}
) {
this.#lockingModule = container.resolve("locking")
this.#lockingModule = lockingModule
this.#entities = entities
this.#options = options
}
@@ -106,44 +96,65 @@ export class Orchestrator {
}
}
/**
* Acquires or renew the lock for a given key.
*/
async renewLock(forKey: string): Promise<boolean> {
return this.#acquireLock(forKey)
}
/**
* Processes the entity at a given index. If there are no entities
* left, the orchestrator state will be set to completed.
*
* - Task runner is the implementation function to execute a task.
* Orchestrator has no inbuilt execution logic and it relies on
* the task runner for the same.
*/
async #processAtIndex(index: number) {
const entity = this.#entities[index]
if (!entity) {
this.#state = "completed"
return
}
this.#currentIndex = index
const lockAcquired = await this.#acquireLock(entity.entity)
async #processAtIndex(
taskRunner: (entity: string) => Promise<void>,
entity: string
) {
const lockAcquired = await this.#acquireLock(entity)
if (lockAcquired) {
try {
await this.#options.taskRunner(entity)
await taskRunner(entity)
} catch (error) {
this.#state = "error"
throw error
} finally {
await this.#lockingModule.release(entity.entity, {
await this.#lockingModule.release(entity, {
ownerId: this.#lockingOwner,
})
}
}
return this.#processAtIndex(index + 1)
}
/**
* Run the orchestrator to process the entities one by one.
*
* - Task runner is the implementation function to execute a task.
* Orchestrator has no inbuilt execution logic and it relies on
* the task runner for the same.
*/
async process() {
async process(taskRunner: (entity: string) => Promise<void>) {
if (this.state !== "idle") {
throw new Error("Cannot re-run an already running orchestrator instance")
}
this.#state = "processing"
return this.#processAtIndex(0)
for (let i = 0; i < this.#entities.length; i++) {
this.#currentIndex = i
const entity = this.#entities[i]
if (!entity) {
this.#state = "completed"
break
}
await this.#processAtIndex(taskRunner, entity)
}
this.#state = "completed"
}
}