asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(index): full sync operations (#11178)

Browse Source 
Closes: FRMW-2892, FRMW-2893

**What**
Wired up the building block that we merged previously in order to manage data synchronization. The flow is as follow
- On application start
  - Build schema object representation from configuration
  - Check configuration changes
    - if new entities configured
      - Data synchronizer initialize orchestrator and start sync
        - for each entity
          - acquire lock
          - mark existing data as staled
          - sync all data by batch
          - marked them not staled anymore
          - acknowledge each processed batch and renew lock
          - update metadata with last synced cursor for entity X
          - release lock
      - remove all remaining staled data
    - if any entities removed from last configuration
      - remove the index data and relations

Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>
This commit is contained in:
Adrien de Peretti
2025-02-05 17:49:18 +01:00
committed by GitHub
parent 60f46e07fd
commit a33aebd895
35 changed files with 1677 additions and 727 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
packages/modules/index/src/utils/create-partitions.ts
View File

@@ -37,6 +37,45 @@ export async function createPartitions(
return
}
await manager.execute(partitions.join("; "))
// Create indexes for each partition
const indexCreationCommands = Object.keys(schemaObjectRepresentation)
.filter(
(key) =>
!schemaObjectRepresentationPropertiesToOmit.includes(key) &&
schemaObjectRepresentation[key].listeners.length > 0
)
.map((key) => {
const cName = key.toLowerCase()
const part: string[] = []
part.push(
`CREATE INDEX CONCURRENTLY IF NOT EXISTS "IDX_cat_${cName}_data_gin" ON ${activeSchema}cat_${cName} USING GIN ("data" jsonb_path_ops)`
)
// create child id index on pivot partitions
for (const parent of schemaObjectRepresentation[key].parents) {
const pName = `${parent.ref.entity}${key}`.toLowerCase()
part.push(
`CREATE INDEX CONCURRENTLY IF NOT EXISTS "IDX_cat_pivot_${pName}_child_id" ON ${activeSchema}cat_pivot_${pName} ("child_id")`
)
}
return part
})
.flat()
// Execute index creation commands separately to avoid blocking
for (const cmd of indexCreationCommands) {
try {
await manager.execute(cmd)
} catch (error) {
// Log error but continue with other indexes
console.error(`Failed to create index: ${error.message}`)
}
}
partitions.push(`analyse ${activeSchema}index_data`)
partitions.push(`analyse ${activeSchema}index_relation`)

11
packages/modules/index/src/utils/default-schema.ts
View File

@@ -5,6 +5,7 @@ export const defaultSchema = `
id: String
title: String
variants: [ProductVariant]
sales_channels: [SalesChannel]
}
type ProductVariant @Listeners(values: ["${Modules.PRODUCT}.product-variant.created", "${Modules.PRODUCT}.product-variant.updated", "${Modules.PRODUCT}.product-variant.deleted"]) {
@@ -15,7 +16,13 @@ export const defaultSchema = `
}
type Price @Listeners(values: ["${Modules.PRICING}.price.created", "${Modules.PRICING}.price.updated", "${Modules.PRICING}.price.deleted"]) {
amount: Int
id: String
amount: Float
currency_code: String
}
}
type SalesChannel @Listeners(values: ["${Modules.SALES_CHANNEL}.sales_channel.created", "${Modules.SALES_CHANNEL}.sales_channel.updated", "${Modules.SALES_CHANNEL}.sales_channel.deleted"]) {
id: String
is_disabled: Boolean
}
`

5
packages/modules/index/src/utils/index.ts
View File

@@ -1,3 +1,8 @@
export * from "./query-builder"
export * from "./create-partitions"
export * from "./build-config"
export * from "./sync/orchestrator"
export * from "./sync/configuration"
export * from "./index-metadata-status"
export * from "./gql-to-types"
export * from "./default-schema"

218
packages/modules/index/src/utils/query-builder.ts
View File

@@ -23,6 +23,7 @@ export class QueryBuilder {
private readonly selector: QueryFormat
private readonly options?: QueryOptions
private readonly schema: IndexTypes.SchemaObjectRepresentation
private readonly allSchemaFields: Set<string>
constructor(args: {
schema: IndexTypes.SchemaObjectRepresentation
@@ -37,14 +38,24 @@ export class QueryBuilder {
this.options = args.options
this.knex = args.knex
this.structure = this.selector.select
this.allSchemaFields = new Set(
Object.values(this.schema).flatMap((entity) => entity.fields ?? [])
)
}
private getStructureKeys(structure) {
return Object.keys(structure ?? {}).filter((key) => key !== "entity")
}
private getEntity(path): IndexTypes.SchemaPropertiesMap[0] {
private getEntity(
path,
throwWhenNotFound = true
): IndexTypes.SchemaPropertiesMap[0] | undefined {
if (!this.schema._schemaPropertiesMap[path]) {
if (!throwWhenNotFound) {
return
}
throw new Error(`Could not find entity for path: ${path}`)
}
@@ -52,7 +63,7 @@ export class QueryBuilder {
}
private getGraphQLType(path, field) {
const entity = this.getEntity(path)?.ref?.entity
const entity = this.getEntity(path)?.ref?.entity!
const fieldRef = this.entityMap[entity]._fields[field]
if (!fieldRef) {
throw new Error(`Field ${field} not found in the entityMap.`)
@@ -270,6 +281,18 @@ export class QueryBuilder {
return builder
}
private getShortAlias(aliasMapping, alias: string) {
aliasMapping.__aliasIndex ??= 0
if (aliasMapping[alias]) {
return aliasMapping[alias]
}
aliasMapping[alias] = "t_" + aliasMapping.__aliasIndex++ + "_"
return aliasMapping[alias]
}
private buildQueryParts(
structure: Select,
parentAlias: string,
@@ -281,10 +304,17 @@ export class QueryBuilder {
): string[] {
const currentAliasPath = [...aliasPath, parentProperty].join(".")
const entities = this.getEntity(currentAliasPath)
const isSelectableField = this.allSchemaFields.has(parentProperty)
const entities = this.getEntity(currentAliasPath, false)
if (isSelectableField || !entities) {
// We are currently selecting a specific field of the parent entity or the entity is not found on the index schema
// We don't need to build the query parts for this as there is no join
return []
}
const mainEntity = entities.ref.entity
const mainAlias = mainEntity.toLowerCase() + level
const mainAlias =
this.getShortAlias(aliasMapping, mainEntity.toLowerCase()) + level
const allEntities: any[] = []
if (!entities.shortCutOf) {
@@ -298,7 +328,13 @@ export class QueryBuilder {
const intermediateAlias = entities.shortCutOf.split(".")
for (let i = intermediateAlias.length - 1, x = 0; i >= 0; i--, x++) {
const intermediateEntity = this.getEntity(intermediateAlias.join("."))
const intermediateEntity = this.getEntity(
intermediateAlias.join("."),
false
)
if (!intermediateEntity) {
break
}
intermediateAlias.pop()
@@ -308,14 +344,24 @@ export class QueryBuilder {
const parentIntermediateEntity = this.getEntity(
intermediateAlias.join(".")
)
)!
const alias =
intermediateEntity.ref.entity.toLowerCase() + level + "_" + x
this.getShortAlias(
aliasMapping,
intermediateEntity.ref.entity.toLowerCase()
) +
level +
"_" +
x
const parAlias =
parentIntermediateEntity.ref.entity === parentEntity
? parentAlias
: parentIntermediateEntity.ref.entity.toLowerCase() +
: this.getShortAlias(
aliasMapping,
parentIntermediateEntity.ref.entity.toLowerCase()
) +
level +
"_" +
(x + 1)
@@ -335,62 +381,68 @@ export class QueryBuilder {
let queryParts: string[] = []
for (const join of allEntities) {
const joinBuilder = this.knex.queryBuilder()
const { alias, entity, parEntity, parAlias } = join
aliasMapping[currentAliasPath] = alias
if (level > 0) {
const subQuery = this.knex.queryBuilder()
const knex = this.knex
subQuery
.select(`${alias}.id`, `${alias}.data`)
.from("index_data AS " + alias)
.join(`index_relation AS ${alias}_ref`, function () {
this.on(
`${alias}_ref.pivot`,
"=",
knex.raw("?", [`${parEntity}-${entity}`])
)
.andOn(`${alias}_ref.parent_id`, "=", `${parAlias}.id`)
.andOn(`${alias}.id`, "=", `${alias}_ref.child_id`)
})
.where(`${alias}.name`, "=", knex.raw("?", [entity]))
const cName = entity.toLowerCase()
const pName = `${parEntity}${entity}`.toLowerCase()
let joinTable = `cat_${cName} AS ${alias}`
const pivotTable = `cat_pivot_${pName}`
joinBuilder.leftJoin(
`${pivotTable} AS ${alias}_ref`,
`${alias}_ref.parent_id`,
`${parAlias}.id`
)
joinBuilder.leftJoin(joinTable, `${alias}.id`, `${alias}_ref.child_id`)
const joinWhere = this.selector.joinWhere ?? {}
const joinKey = Object.keys(joinWhere).find((key) => {
const k = key.split(".")
k.pop()
return k.join(".") === currentAliasPath
const curPath = k.join(".")
if (curPath === currentAliasPath) {
const relEntity = this.getEntity(curPath, false)
return relEntity?.ref?.entity === entity
}
return false
})
if (joinKey) {
this.parseWhere(
aliasMapping,
{ [joinKey]: joinWhere[joinKey] },
subQuery
joinBuilder
)
}
queryParts.push(`LEFT JOIN LATERAL (
${subQuery.toQuery()}
) ${alias} ON TRUE`)
queryParts.push(
joinBuilder.toQuery().replace("select * ", "").replace("where", "and")
)
}
}
const children = this.getStructureKeys(structure)
for (const child of children) {
const childStructure = structure[child] as Select
queryParts = queryParts.concat(
this.buildQueryParts(
childStructure,
mainAlias,
mainEntity,
child,
aliasPath.concat(parentProperty),
level + 1,
aliasMapping
queryParts = queryParts
.concat(
this.buildQueryParts(
childStructure,
mainAlias,
mainEntity,
child,
aliasPath.concat(parentProperty),
level + 1,
aliasMapping
)
)
)
.filter(Boolean)
}
return queryParts
@@ -404,7 +456,26 @@ export class QueryBuilder {
selectParts: object = {}
): object {
const currentAliasPath = [...aliasPath, parentProperty].join(".")
const isSelectableField = this.allSchemaFields.has(parentProperty)
if (isSelectableField) {
// We are currently selecting a specific field of the parent entity
// Let's remove the parent alias from the select parts to not select everything entirely
// and add the specific field to the select parts
const parentAliasPath = aliasPath.join(".")
const alias = aliasMapping[parentAliasPath]
delete selectParts[parentAliasPath]
selectParts[currentAliasPath] = this.knex.raw(
`${alias}.data->'${parentProperty}'`
)
return selectParts
}
const alias = aliasMapping[currentAliasPath]
// If the entity is not found in the schema (not indexed), we don't need to build the select parts
if (!alias) {
return selectParts
}
selectParts[currentAliasPath] = `${alias}.data`
selectParts[currentAliasPath + ".id"] = `${alias}.id`
@@ -473,7 +544,8 @@ export class QueryBuilder {
const rootKey = this.getStructureKeys(structure)[0]
const rootStructure = structure[rootKey] as Select
const entity = this.getEntity(rootKey).ref.entity
const entity = this.getEntity(rootKey)!.ref.entity
const rootEntity = entity.toLowerCase()
const aliasMapping: { [path: string]: string } = {}
@@ -494,20 +566,23 @@ export class QueryBuilder {
if (countAllResults) {
selectParts["offset_"] = this.knex.raw(
`DENSE_RANK() OVER (ORDER BY ${rootEntity}0.id)`
`DENSE_RANK() OVER (ORDER BY ${this.getShortAlias(
aliasMapping,
rootEntity
)}.id)`
)
}
queryBuilder.select(selectParts)
queryBuilder.from(`index_data AS ${rootEntity}0`)
queryBuilder.from(
`cat_${rootEntity} AS ${this.getShortAlias(aliasMapping, rootEntity)}`
)
joinParts.forEach((joinPart) => {
queryBuilder.joinRaw(joinPart)
})
queryBuilder.where(`${aliasMapping[rootEntity]}.name`, "=", entity)
// WHERE clause
this.parseWhere(aliasMapping, filter, queryBuilder)
@@ -559,6 +634,11 @@ export class QueryBuilder {
const initializeMaps = (structure: Select, path: string[]) => {
const currentPath = path.join(".")
const entity = this.getEntity(currentPath, false)
if (!entity) {
return
}
maps[currentPath] = {}
if (path.length > 1) {
@@ -566,9 +646,19 @@ export class QueryBuilder {
const parents = path.slice(0, -1)
const parentPath = parents.join(".")
isListMap[currentPath] = !!this.getEntity(currentPath).ref.parents.find(
(p) => p.targetProp === property
)?.isList
// In the case of specific selection
// We dont need to check if the property is a list
const isSelectableField = this.allSchemaFields.has(property)
if (isSelectableField) {
pathDetails[currentPath] = { property, parents, parentPath }
isListMap[currentPath] = false
return
}
isListMap[currentPath] = !!this.getEntity(
currentPath,
false
)?.ref?.parents?.find((p) => p.targetProp === property)?.isList
pathDetails[currentPath] = { property, parents, parentPath }
}
@@ -595,6 +685,20 @@ export class QueryBuilder {
return key + id
}
const columnMap = {}
const columnNames = Object.keys(resultSet[0] ?? {})
for (const property of columnNames) {
const segments = property.split(".")
const field = segments.pop()
const parent = segments.join(".")
columnMap[parent] ??= []
columnMap[parent].push({
field,
property,
})
}
resultSet.forEach((row) => {
for (const path in maps) {
const id = row[`${path}.id`]
@@ -603,6 +707,15 @@ export class QueryBuilder {
if (!pathDetails[path]) {
if (!maps[path][id]) {
maps[path][id] = row[path] || undefined
// If there is an id, but no object values, it means that specific fields were selected
// so we recompose the object with all selected fields. (id will always be selected)
if (!maps[path][id] && id) {
maps[path][id] = {}
for (const column of columnMap[path]) {
maps[path][id][column.field] = row[column.property]
}
}
}
continue
}
@@ -616,6 +729,15 @@ export class QueryBuilder {
maps[path][id] = row[path] || undefined
// If there is an id, but no object values, it means that specific fields were selected
// so we recompose the object with all selected fields. (id will always be selected)
if (!maps[path][id] && id) {
maps[path][id] = {}
for (const column of columnMap[path]) {
maps[path][id][column.field] = row[column.property]
}
}
const parentObj = maps[parentPath][row[`${parentPath}.id`]]
if (!parentObj) {
@@ -623,8 +745,8 @@ export class QueryBuilder {
}
const isList = isListMap[parentPath + "." + property]
if (isList) {
parentObj[property] ??= []
if (isList && !Array.isArray(parentObj[property])) {
parentObj[property] = []
}
if (maps[path][id] !== undefined) {

142
packages/modules/index/src/utils/sync/configuration.ts Normal file
View File

@@ -0,0 +1,142 @@
import { simpleHash } from "@medusajs/framework/utils"
import { IndexTypes, InferEntityType } from "@medusajs/types"
import { IndexMetadata } from "@models"
import { schemaObjectRepresentationPropertiesToOmit } from "@types"
import { DataSynchronizer } from "../../services/data-synchronizer"
import { IndexMetadataService } from "../../services/index-metadata"
import { IndexSyncService } from "../../services/index-sync"
import { IndexMetadataStatus } from "../index-metadata-status"
export class Configuration {
#schemaObjectRepresentation: IndexTypes.SchemaObjectRepresentation
#indexMetadataService: IndexMetadataService
#indexSyncService: IndexSyncService
#dataSynchronizer: DataSynchronizer
constructor({
schemaObjectRepresentation,
indexMetadataService,
indexSyncService,
dataSynchronizer,
}: {
schemaObjectRepresentation: IndexTypes.SchemaObjectRepresentation
indexMetadataService: IndexMetadataService
indexSyncService: IndexSyncService
dataSynchronizer: DataSynchronizer
}) {
this.#schemaObjectRepresentation = schemaObjectRepresentation ?? {}
this.#indexMetadataService = indexMetadataService
this.#indexSyncService = indexSyncService
this.#dataSynchronizer = dataSynchronizer
}
async checkChanges(): Promise<InferEntityType<typeof IndexMetadata>[]> {
const schemaObjectRepresentation = this.#schemaObjectRepresentation
const currentConfig = await this.#indexMetadataService.list()
const currentConfigMap = new Map(
currentConfig.map((c) => [c.entity, c] as const)
)
type modifiedConfig = {
id?: string
entity: string
fields: string[]
fields_hash: string
status?: IndexMetadataStatus
}[]
type dataSyncEntry = {
id?: string
entity: string
last_key: null
}[]
const entityPresent = new Set<string>()
const newConfig: modifiedConfig = []
const updatedConfig: modifiedConfig = []
const deletedConfig: { entity: string }[] = []
const idxSyncData: dataSyncEntry = []
for (const [entityName, schemaEntityObjectRepresentation] of Object.entries(
schemaObjectRepresentation
)) {
if (schemaObjectRepresentationPropertiesToOmit.includes(entityName)) {
continue
}
const entity = schemaEntityObjectRepresentation.entity
const fields = schemaEntityObjectRepresentation.fields.sort().join(",")
const fields_hash = simpleHash(fields)
const existingEntityConfig = currentConfigMap.get(entity)
entityPresent.add(entity)
if (
!existingEntityConfig ||
existingEntityConfig.fields_hash !== fields_hash
) {
const meta = {
id: existingEntityConfig?.id,
entity,
fields,
fields_hash,
}
if (!existingEntityConfig) {
newConfig.push(meta)
} else {
updatedConfig.push({
...meta,
status: IndexMetadataStatus.PENDING,
})
}
idxSyncData.push({
entity,
last_key: null,
})
}
}
for (const [entity] of currentConfigMap) {
if (!entityPresent.has(entity)) {
deletedConfig.push({ entity })
}
}
if (newConfig.length > 0) {
await this.#indexMetadataService.create(newConfig)
}
if (updatedConfig.length > 0) {
await this.#indexMetadataService.update(updatedConfig)
}
if (deletedConfig.length > 0) {
await this.#indexMetadataService.delete(deletedConfig)
await this.#dataSynchronizer.removeEntities(
deletedConfig.map((c) => c.entity)
)
}
if (idxSyncData.length > 0) {
if (updatedConfig.length > 0) {
const ids = await this.#indexSyncService.list({
entity: updatedConfig.map((c) => c.entity),
})
idxSyncData.forEach((sync) => {
const id = ids.find((i) => i.entity === sync.entity)?.id
if (id) {
sync.id = id
}
})
}
await this.#indexSyncService.upsert(idxSyncData)
}
return await this.#indexMetadataService.list({
status: [IndexMetadataStatus.PENDING, IndexMetadataStatus.PROCESSING],
})
}
}

137
packages/modules/index/src/utils/sync/data-synchronizer.ts
View File

@@ -1,137 +0,0 @@
import {
IndexTypes,
RemoteQueryFunction,
SchemaObjectEntityRepresentation,
Event,
} from "@medusajs/framework/types"
import { CommonEvents } from "@medusajs/framework/utils"
export class DataSynchronizer {
#storageProvider: IndexTypes.StorageProvider
#schemaObjectRepresentation: IndexTypes.SchemaObjectRepresentation
#query: RemoteQueryFunction
constructor({
storageProvider,
schemaObjectRepresentation,
query,
}: {
storageProvider: IndexTypes.StorageProvider
schemaObjectRepresentation: IndexTypes.SchemaObjectRepresentation
query: RemoteQueryFunction
}) {
this.#storageProvider = storageProvider
this.#schemaObjectRepresentation = schemaObjectRepresentation
this.#query = query
}
async sync({
entityName,
pagination = {},
ack,
}: {
entityName: string
pagination?: {
cursor?: string
updated_at?: string | Date
limit?: number
batchSize?: number
}
ack: (ack: {
lastCursor: string | null
done?: boolean
err?: Error
}) => Promise<void>
}) {
const schemaEntityObjectRepresentation = this.#schemaObjectRepresentation[
entityName
] as SchemaObjectEntityRepresentation
const { fields, alias, moduleConfig } = schemaEntityObjectRepresentation
const entityPrimaryKey = fields.find(
(field) => !!moduleConfig.linkableKeys?.[field]
)
if (!entityPrimaryKey) {
void ack({
lastCursor: pagination.cursor ?? null,
err: new Error(
`Entity ${entityName} does not have a linkable primary key`
),
})
return
}
let processed = 0
let currentCursor = pagination.cursor!
const batchSize = pagination.batchSize ?? 1000
const limit = pagination.limit ?? Infinity
let done = false
let error = null
while (processed < limit || !done) {
const filters: Record<string, any> = {}
if (currentCursor) {
filters[entityPrimaryKey] = { $gt: currentCursor }
}
if (pagination.updated_at) {
filters["updated_at"] = { $gt: pagination.updated_at }
}
const { data } = await this.#query.graph({
entity: alias,
fields: [entityPrimaryKey],
filters,
pagination: {
order: {
[entityPrimaryKey]: "asc",
},
take: batchSize,
},
})
done = !data.length
if (done) {
break
}
const envelop: Event = {
data,
name: `*.${CommonEvents.CREATED}`,
}
try {
await this.#storageProvider.consumeEvent(
schemaEntityObjectRepresentation
)(envelop)
currentCursor = data[data.length - 1][entityPrimaryKey]
processed += data.length
void ack({ lastCursor: currentCursor })
} catch (err) {
error = err
break
}
}
let acknoledgement: { lastCursor: string; done?: boolean; err?: Error } = {
lastCursor: currentCursor,
done: true,
}
if (error) {
acknoledgement = {
lastCursor: currentCursor,
err: error,
}
void ack(acknoledgement)
return acknoledgement
}
void ack(acknoledgement)
return acknoledgement
}
}

160
packages/modules/index/src/utils/sync/orchestrator.ts Normal file
View File

@@ -0,0 +1,160 @@
import { ILockingModule } from "@medusajs/types"
export class Orchestrator {
/**
* Reference to the locking module
*/
#lockingModule: ILockingModule
/**
* Owner id when acquiring locks
*/
#lockingOwner = `index-sync-${process.pid}`
/**
* The current state of the orchestrator
*
* - In "idle" state, one can call the "run" method.
* - In "processing" state, the orchestrator is looping over the entities
* and processing them.
* - In "completed" state, the provided entities have been processed.
* - The "error" state is set when the task runner throws an error.
*/
#state: "idle" | "processing" | "completed" | "error" = "idle"
/**
* Options for the locking module and the task runner to execute the
* task.
*
* - Lock duration is the maximum duration for which to hold the lock.
* After this the lock will be removed.
*
* The entity is provided to the taskRunner only when the orchestrator
* is able to acquire a lock.
*/
#options: {
lockDuration: number
}
/**
* Index of the entity that is currently getting processed.
*/
#currentIndex: number = 0
/**
* Collection of entities to process in sequence. A lock is obtained
* while an entity is getting synced to avoid multiple processes
* from syncing the same entity
*/
#entities: string[] = []
/**
* The current state of the orchestrator
*/
get state() {
return this.#state
}
/**
* Reference to the currently processed entity
*/
get current() {
return this.#entities[this.#currentIndex]
}
/**
* Reference to the number of entities left for processing
*/
get remainingCount() {
return this.#entities.length - (this.#currentIndex + 1)
}
constructor(
lockingModule: ILockingModule,
entities: string[],
options: {
lockDuration: number
}
) {
this.#lockingModule = lockingModule
this.#entities = entities
this.#options = options
}
/**
* Acquires using the lock module.
*/
async #acquireLock(forKey: string): Promise<boolean> {
try {
await this.#lockingModule.acquire(forKey, {
expire: this.#options.lockDuration,
ownerId: this.#lockingOwner,
})
return true
} catch {
return false
}
}
/**
* Acquires or renew the lock for a given key.
*/
async renewLock(forKey: string): Promise<boolean> {
return this.#acquireLock(forKey)
}
/**
* Processes the entity at a given index. If there are no entities
* left, the orchestrator state will be set to completed.
*
* - Task runner is the implementation function to execute a task.
* Orchestrator has no inbuilt execution logic and it relies on
* the task runner for the same.
*/
async #processAtIndex(
taskRunner: (entity: string) => Promise<void>,
entity: string
) {
const lockAcquired = await this.#acquireLock(entity)
if (lockAcquired) {
try {
await taskRunner(entity)
} catch (error) {
this.#state = "error"
throw error
} finally {
await this.#lockingModule.release(entity, {
ownerId: this.#lockingOwner,
})
}
}
}
/**
* Run the orchestrator to process the entities one by one.
*
* - Task runner is the implementation function to execute a task.
* Orchestrator has no inbuilt execution logic and it relies on
* the task runner for the same.
*/
async process(taskRunner: (entity: string) => Promise<void>) {
if (this.state !== "idle") {
throw new Error("Cannot re-run an already running orchestrator instance")
}
this.#state = "processing"
for (let i = 0; i < this.#entities.length; i++) {
this.#currentIndex = i
const entity = this.#entities[i]
if (!entity) {
this.#state = "completed"
break
}
await this.#processAtIndex(taskRunner, entity)
}
this.#state = "completed"
}
}