From 1215a7c09454df5ba734184d41fcba144116a61e Mon Sep 17 00:00:00 2001 From: "Carlos R. L. Rodrigues" <37986729+carlos-r-l-rodrigues@users.noreply.github.com> Date: Fri, 20 Sep 2024 05:30:08 -0300 Subject: [PATCH] chore(orchestrator): remote joiner using entitymap (#9205) --- .../__tests__/modules/remote-query.spec.ts | 2 +- .../src/remote-query/remote-query.ts | 5 +- .../orchestration/src/joiner/remote-joiner.ts | 449 ++++++++++++------ packages/core/types/src/joiner/index.ts | 3 +- .../common/graphql-relations-entity-map.ts | 41 ++ packages/core/utils/src/common/index.ts | 1 + 6 files changed, 361 insertions(+), 140 deletions(-) create mode 100644 packages/core/utils/src/common/graphql-relations-entity-map.ts diff --git a/integration-tests/modules/__tests__/modules/remote-query.spec.ts b/integration-tests/modules/__tests__/modules/remote-query.spec.ts index b9cfd67e00..8208485756 100644 --- a/integration-tests/modules/__tests__/modules/remote-query.spec.ts +++ b/integration-tests/modules/__tests__/modules/remote-query.spec.ts @@ -170,7 +170,7 @@ medusaIntegrationTestRunner({ } ) ).rejects.toThrow( - "Payment id not found: pp_system_default_non_existent" + "PaymentProvider id not found: pp_system_default_non_existent" ) // everything is fine diff --git a/packages/core/modules-sdk/src/remote-query/remote-query.ts b/packages/core/modules-sdk/src/remote-query/remote-query.ts index 909931fa0c..4a19f712d6 100644 --- a/packages/core/modules-sdk/src/remote-query/remote-query.ts +++ b/packages/core/modules-sdk/src/remote-query/remote-query.ts @@ -72,7 +72,10 @@ export class RemoteQuery { this.remoteJoiner = new RemoteJoiner( servicesConfig_ as JoinerServiceConfig[], this.remoteFetchData.bind(this), - { autoCreateServiceNameAlias: false } + { + autoCreateServiceNameAlias: false, + entitiesMap, + } ) } diff --git a/packages/core/orchestration/src/joiner/remote-joiner.ts b/packages/core/orchestration/src/joiner/remote-joiner.ts index faf34a45c6..fa5eb1fd96 100644 --- a/packages/core/orchestration/src/joiner/remote-joiner.ts +++ b/packages/core/orchestration/src/joiner/remote-joiner.ts @@ -9,7 +9,13 @@ import { } from "@medusajs/types" import { RemoteJoinerOptions } from "@medusajs/types" -import { MedusaError, deduplicate, isDefined, isString } from "@medusajs/utils" +import { + MedusaError, + deduplicate, + extractRelationsFromGQL, + isDefined, + isString, +} from "@medusajs/utils" import GraphQLParser from "./graphql-ast" const BASE_PATH = "_root" @@ -35,6 +41,8 @@ export class RemoteJoiner { private serviceConfigCache: Map = new Map() + private entityMap: Map> = new Map() + private static filterFields( data: any, fields?: string[], @@ -142,12 +150,21 @@ export class RemoteJoiner { private remoteFetchData: RemoteFetchDataCallback, private options: { autoCreateServiceNameAlias?: boolean + entitiesMap?: Map } = {} ) { this.options.autoCreateServiceNameAlias ??= true + if (this.options.entitiesMap) { + this.entityMap = extractRelationsFromGQL(this.options.entitiesMap) + } this.serviceConfigs = this.buildReferences( - JSON.parse(JSON.stringify(serviceConfigs)) + JSON.parse(JSON.stringify(serviceConfigs), (key, value) => { + if (key === "schema") { + return + } + return value + }) ) } @@ -158,12 +175,15 @@ export class RemoteJoiner { private buildReferences(serviceConfigs: ModuleJoinerConfig[]) { const expandedRelationships: Map< string, - { fieldAlias; relationships: Map } + { + fieldAlias + relationships: Map + } > = new Map() for (const service of serviceConfigs) { const service_ = service as Omit & { - relationships?: Map + relationships?: Map } if (this.serviceConfigCache.has(service_.serviceName!)) { @@ -233,18 +253,33 @@ export class RemoteJoiner { ? { ...service_.args, ...alias.args } : undefined - service_.relationships?.set(alias.name as string, { - alias: alias.name as string, + const aliasName = alias.name as string + const rel = { + alias: aliasName, entity: alias.entity, foreignKey: alias.name + "_id", primaryKey: "id", serviceName: service_.serviceName!, args, - }) - this.cacheServiceConfig(serviceConfigs, undefined, alias) + } + + if (service_.relationships?.has(aliasName)) { + const existing = service_.relationships.get(aliasName)! + const newRelation = Array.isArray(existing) + ? existing.concat(rel) + : [existing, rel] + + service_.relationships?.set(aliasName, newRelation) + } else { + service_.relationships?.set(aliasName, rel) + } + + this.cacheServiceConfig(serviceConfigs, { serviceAlias: alias }) } - this.cacheServiceConfig(serviceConfigs, service_.serviceName) + this.cacheServiceConfig(serviceConfigs, { + serviceName: service_.serviceName, + }) } for (const extend of service_.extends) { @@ -256,10 +291,20 @@ export class RemoteJoiner { } const service_ = expandedRelationships.get(extend.serviceName)! - service_.relationships.set( - extend.relationship.alias, - extend.relationship - ) + + const aliasName = extend.relationship.alias + const rel = extend.relationship + if (service_.relationships?.has(aliasName)) { + const existing = service_.relationships.get(aliasName)! + const newRelation = Array.isArray(existing) + ? existing.concat(rel) + : [existing, rel] + + service_.relationships?.set(aliasName, newRelation) + } else { + service_.relationships?.set(aliasName, rel) + } + Object.assign(service_.fieldAlias ?? {}, extend.fieldAlias) } } @@ -274,8 +319,19 @@ export class RemoteJoiner { const service_ = this.serviceConfigCache.get(serviceName)! relationships.forEach((relationship, alias) => { - service_.relationships!.set(alias, relationship) + const rel = relationship as JoinerRelationship + if (service_.relationships?.has(alias)) { + const existing = service_.relationships.get(alias)! + const newRelation = Array.isArray(existing) + ? existing.concat(rel) + : [existing, rel] + + service_.relationships?.set(alias, newRelation) + } else { + service_.relationships?.set(alias, rel) + } }) + Object.assign(service_.fieldAlias!, fieldAlias ?? {}) if (Object.keys(service_.fieldAlias!).length) { @@ -296,10 +352,23 @@ export class RemoteJoiner { return serviceConfigs } - private getServiceConfig( - serviceName?: string, + private getServiceConfig({ + serviceName, + serviceAlias, + entity, + }: { + serviceName?: string serviceAlias?: string - ): InternalJoinerServiceConfig | undefined { + entity?: string + }): InternalJoinerServiceConfig | undefined { + if (entity) { + const name = `entity_${entity}` + const serviceConfig = this.serviceConfigCache.get(name) + if (serviceConfig) { + return serviceConfig + } + } + if (serviceAlias) { const name = `alias_${serviceAlias}` return this.serviceConfigCache.get(name) @@ -309,10 +378,14 @@ export class RemoteJoiner { } private cacheServiceConfig( - serviceConfigs, - serviceName?: string, - serviceAlias?: JoinerServiceConfigAlias + serviceConfigs: ModuleJoinerConfig[], + params: { + serviceName?: string + serviceAlias?: JoinerServiceConfigAlias + } ): void { + const { serviceName, serviceAlias } = params + if (serviceAlias) { const name = `alias_${serviceAlias.name}` if (!this.serviceConfigCache.has(name)) { @@ -331,7 +404,19 @@ export class RemoteJoiner { if (aliasConfig) { serviceConfig.args = { ...config?.args, ...aliasConfig?.args } } - this.serviceConfigCache.set(name, serviceConfig) + this.serviceConfigCache.set( + name, + serviceConfig as InternalJoinerServiceConfig + ) + + const entity = serviceAlias.entity + if (entity) { + const name = `entity_${entity}` + this.serviceConfigCache.set( + name, + serviceConfig as InternalJoinerServiceConfig + ) + } } } return @@ -339,20 +424,22 @@ export class RemoteJoiner { const config = serviceConfigs.find( (config) => config.serviceName === serviceName - ) + ) as InternalJoinerServiceConfig this.serviceConfigCache.set(serviceName!, config) } - private async fetchData( - expand: RemoteExpandProperty, - pkField: string, - ids?: (unknown | unknown[])[], - relationship?: any, + private async fetchData(params: { + expand: RemoteExpandProperty + pkField: string + ids?: (unknown | unknown[])[] + relationship?: any options?: RemoteJoinerOptions - ): Promise<{ + }): Promise<{ data: unknown[] | { [path: string]: unknown } path?: string }> { + const { expand, pkField, ids, relationship, options } = params + let uniqueIds = Array.isArray(ids) ? ids : ids ? [ids] : undefined if (uniqueIds) { @@ -372,15 +459,16 @@ export class RemoteJoiner { uniqueIds = uniqueIds.filter((id) => isDefined(id)) } + let pkFieldAdjusted = pkField if (relationship) { - pkField = relationship.inverse + pkFieldAdjusted = relationship.inverse ? relationship.foreignKey.split(".").pop()! : relationship.primaryKey } const response = await this.remoteFetchData( expand, - pkField, + pkFieldAdjusted, uniqueIds, relationship ) @@ -394,14 +482,14 @@ export class RemoteJoiner { : [resData] : [] - this.checkIfKeysExist( + this.checkIfKeysExist({ uniqueIds, resData, expand, - pkField, + pkField: pkFieldAdjusted, relationship, - options - ) + options, + }) const filteredDataArray = resData.map((data: any) => RemoteJoiner.filterFields(data, expand.fields, expand.expands) @@ -416,14 +504,17 @@ export class RemoteJoiner { return response } - private checkIfKeysExist( - uniqueIds: unknown[] | undefined, - resData: any[], - expand: RemoteExpandProperty, - pkField: string, - relationship?: any, + private checkIfKeysExist(params: { + uniqueIds: unknown[] | undefined + resData: any[] + expand: RemoteExpandProperty + pkField: string + relationship?: any options?: RemoteJoinerOptions - ) { + }) { + const { uniqueIds, resData, expand, pkField, relationship, options } = + params + if ( !( isDefined(uniqueIds) && @@ -461,11 +552,13 @@ export class RemoteJoiner { } } - private handleFieldAliases( - items: any[], - parsedExpands: Map, + private handleFieldAliases(params: { + items: any[] + parsedExpands: Map implodeMapping: InternalImplodeMapping[] - ) { + }) { + const { items, parsedExpands, implodeMapping } = params + const getChildren = (item: any, prop: string) => { if (Array.isArray(item)) { return item.flatMap((currentItem) => currentItem[prop]) @@ -541,12 +634,14 @@ export class RemoteJoiner { } } - private async handleExpands( - items: any[], - parsedExpands: Map, - implodeMapping: InternalImplodeMapping[] = [], + private async handleExpands(params: { + items: any[] + parsedExpands: Map + implodeMapping?: InternalImplodeMapping[] options?: RemoteJoinerOptions - ): Promise { + }): Promise { + const { items, parsedExpands, implodeMapping = [], options } = params + if (!parsedExpands) { return } @@ -567,48 +662,87 @@ export class RemoteJoiner { } if (nestedItems.length > 0) { - await this.expandProperty( - nestedItems, - expand.parentConfig!, + await this.expandProperty({ + items: nestedItems, + parentServiceConfig: expand.parentConfig!, expand, - options - ) + options, + }) } } - this.handleFieldAliases(items, parsedExpands, implodeMapping) + this.handleFieldAliases({ items, parsedExpands, implodeMapping }) } - private async expandProperty( - items: any[], - parentServiceConfig: InternalJoinerServiceConfig, - expand?: RemoteExpandProperty, + private getEntityRelationship(params: { + parentServiceConfig: InternalJoinerServiceConfig + property: string + entity?: string + }): JoinerRelationship { + const { parentServiceConfig, property, entity } = params + + const propEntity = entity ?? parentServiceConfig?.entity + const rel = parentServiceConfig?.relationships?.get(property) + + if (Array.isArray(rel)) { + if (!propEntity) { + return rel[0] + } + + const entityRel = rel.find((r) => r.entity === propEntity) + if (entityRel) { + return entityRel + } + + // If entity is not found, return the relationship where the primary key matches + const serviceEntity = this.getServiceConfig({ + entity: propEntity, + })! + + return rel.find((r) => serviceEntity.primaryKeys.includes(r.primaryKey))! + } + + return rel as JoinerRelationship + } + + private async expandProperty(params: { + items: any[] + parentServiceConfig: InternalJoinerServiceConfig + expand?: RemoteExpandProperty options?: RemoteJoinerOptions - ): Promise { + }): Promise { + const { items, parentServiceConfig, expand, options } = params + if (!expand) { return } - const relationship = parentServiceConfig?.relationships?.get( - expand.property - ) + const relationship = this.getEntityRelationship({ + parentServiceConfig, + property: expand.property, + entity: expand.entity, + }) - if (relationship) { - await this.expandRelationshipProperty( - items, - expand, - relationship, - options - ) + if (!relationship) { + return } + + await this.expandRelationshipProperty({ + items, + expand, + relationship, + options, + }) } - private async expandRelationshipProperty( - items: any[], - expand: RemoteExpandProperty, - relationship: JoinerRelationship, + private async expandRelationshipProperty(params: { + items: any[] + expand: RemoteExpandProperty + relationship: JoinerRelationship options?: RemoteJoinerOptions - ): Promise { + }): Promise { + const { items, expand, relationship, options } = params + const field = relationship.inverse ? relationship.primaryKey : relationship.foreignKey.split(".").pop()! @@ -639,13 +773,13 @@ export class RemoteJoiner { return } - const relatedDataArray = await this.fetchData( + const relatedDataArray = await this.fetchData({ expand, - field, - idsToFetch, + pkField: field, + ids: idsToFetch, relationship, - options - ) + options, + }) const joinFields = relationship.inverse ? relationship.foreignKey.split(",") @@ -689,36 +823,46 @@ export class RemoteJoiner { }) } - private parseExpands( - initialService: RemoteExpandProperty, - query: RemoteJoinerQuery, - serviceConfig: InternalJoinerServiceConfig, - expands: RemoteJoinerQuery["expands"], - implodeMapping: InternalImplodeMapping[], + private parseExpands(params: { + initialService: RemoteExpandProperty + query: RemoteJoinerQuery + serviceConfig: InternalJoinerServiceConfig + expands: RemoteJoinerQuery["expands"] + implodeMapping: InternalImplodeMapping[] options?: RemoteJoinerOptions - ): Map { - const parsedExpands = this.parseProperties( + }): Map { + const { initialService, query, serviceConfig, expands, implodeMapping, - options - ) + options, + } = params + + const parsedExpands = this.parseProperties({ + initialService, + query, + serviceConfig, + expands, + implodeMapping, + }) const groupedExpands = this.groupExpands(parsedExpands) return groupedExpands } - private parseProperties( - initialService: RemoteExpandProperty, - query: RemoteJoinerQuery, - serviceConfig: InternalJoinerServiceConfig, - expands: RemoteJoinerQuery["expands"], - implodeMapping: InternalImplodeMapping[], - options?: RemoteJoinerOptions - ): Map { + private parseProperties(params: { + initialService: RemoteExpandProperty + query: RemoteJoinerQuery + serviceConfig: InternalJoinerServiceConfig + expands: RemoteJoinerQuery["expands"] + implodeMapping: InternalImplodeMapping[] + }): Map { + const { initialService, query, serviceConfig, expands, implodeMapping } = + params + const aliasRealPathMap = new Map() const parsedExpands = new Map() parsedExpands.set(BASE_PATH, initialService) @@ -760,7 +904,19 @@ export class RemoteJoiner { const fullPath = [BASE_PATH, ...currentPath, prop].join(".") const fullAliasPath = [BASE_PATH, ...currentAliasPath, prop].join(".") - const relationship = currentServiceConfig.relationships?.get(prop) + let entity = currentServiceConfig.entity + if (entity) { + const completePath = fullPath.split(".") + for (let i = 1; i < completePath.length; i++) { + entity = this.getEntity({ entity, prop: completePath[i] }) ?? entity + } + } + + const relationship = this.getEntityRelationship({ + parentServiceConfig: currentServiceConfig, + property: prop, + entity, + }) const isCurrentProp = fullPath === BASE_PATH + "." + expand.property || @@ -772,7 +928,6 @@ export class RemoteJoiner { if (relationship) { const parentExpand = parsedExpands.get([BASE_PATH, ...currentPath].join(".")) || query - if (parentExpand) { const parRelField = relationship.inverse ? relationship.primaryKey @@ -792,9 +947,10 @@ export class RemoteJoiner { fields = fields.concat(relField.split(",")) } - currentServiceConfig = this.getServiceConfig( - relationship.serviceName - )! + currentServiceConfig = this.getServiceConfig({ + serviceName: relationship.serviceName, + entity: relationship.entity, + })! if (!currentServiceConfig) { throw new Error( @@ -817,6 +973,7 @@ export class RemoteJoiner { parsedExpands.set(fullPath, { property: prop, serviceConfig: currentServiceConfig, + entity: entity, fields, args: isAliasMapping ? forwardArgumentsOnPath.includes(fullPath) @@ -848,6 +1005,10 @@ export class RemoteJoiner { return parsedExpands } + private getEntity({ entity, prop }: { entity: string; prop: string }) { + return this.entityMap.get(entity)?.get(prop) + } + private parseAlias({ aliasPath, aliasRealPathMap, @@ -875,7 +1036,6 @@ export class RemoteJoiner { return parsedExpands.get(fullPath).serviceConfig } - // remove alias from fields const parentPath = [BASE_PATH, ...currentPath].join(".") const parentExpands = parsedExpands.get(parentPath) parentExpands.fields = parentExpands.fields?.filter( @@ -926,10 +1086,31 @@ export class RemoteJoiner { const partialPath: string[] = [] for (const partial of path.split(".")) { - const relationship = currentServiceConfig.relationships?.get(partial) + const completePath = [ + BASE_PATH, + ...currentPath.concat(partialPath), + partial, + ] + const parentPath = completePath.slice(0, -1).join(".") + + let entity = serviceConfig.entity + if (entity) { + for (let i = 1; i < completePath.length; i++) { + entity = this.getEntity({ entity, prop: completePath[i] }) ?? entity + } + } + + const relationship = this.getEntityRelationship({ + parentServiceConfig: currentServiceConfig, + property: partial, + entity, + }) if (relationship) { - currentServiceConfig = this.getServiceConfig(relationship.serviceName)! + currentServiceConfig = this.getServiceConfig({ + serviceName: relationship.serviceName, + entity: relationship.entity, + })! if (!currentServiceConfig) { throw new Error( @@ -938,17 +1119,11 @@ export class RemoteJoiner { } } - const completePath = [ - BASE_PATH, - ...currentPath.concat(partialPath), - partial, - ] - const parentPath = completePath.slice(0, -1).join(".") - partialPath.push(partial) parsedExpands.set(completePath.join("."), { property: partial, serviceConfig: currentServiceConfig, + entity: entity, parent: parentPath, parentConfig: parsedExpands.get(parentPath).serviceConfig, }) @@ -980,7 +1155,6 @@ export class RemoteJoiner { break } - // Merge the current expand into its parent const nestedKeys = path.split(".").slice(parentPath.split(".").length) let targetExpand = parentExpand as Omit< RemoteExpandProperty, @@ -1009,10 +1183,10 @@ export class RemoteJoiner { queryObj: RemoteJoinerQuery, options?: RemoteJoinerOptions ): Promise { - const serviceConfig = this.getServiceConfig( - queryObj.service, - queryObj.alias - ) + const serviceConfig = this.getServiceConfig({ + serviceName: queryObj.service, + serviceAlias: queryObj.alias, + }) if (!serviceConfig) { if (queryObj.alias) { @@ -1035,38 +1209,39 @@ export class RemoteJoiner { ) const implodeMapping: InternalImplodeMapping[] = [] - const parsedExpands = this.parseExpands( - { + const parsedExpands = this.parseExpands({ + initialService: { property: "", parent: "", - serviceConfig: serviceConfig, + serviceConfig, + entity: serviceConfig.entity, fields: queryObj.fields, args: otherArgs, }, - queryObj, + query: queryObj, serviceConfig, - queryObj.expands!, - implodeMapping - ) + expands: queryObj.expands!, + implodeMapping, + options, + }) const root = parsedExpands.get(BASE_PATH)! - const response = await this.fetchData( - root, - pkName, - primaryKeyArg?.value, - undefined, - options - ) + const response = await this.fetchData({ + expand: root, + pkField: pkName, + ids: primaryKeyArg?.value, + options, + }) const data = response.path ? response.data[response.path!] : response.data - await this.handleExpands( - Array.isArray(data) ? data : [data], + await this.handleExpands({ + items: Array.isArray(data) ? data : [data], parsedExpands, implodeMapping, - options - ) + options, + }) return response.data } diff --git a/packages/core/types/src/joiner/index.ts b/packages/core/types/src/joiner/index.ts index 17fb19b2d0..e56da2a935 100644 --- a/packages/core/types/src/joiner/index.ts +++ b/packages/core/types/src/joiner/index.ts @@ -97,7 +97,7 @@ export type InternalJoinerServiceConfig = Omit< JoinerServiceConfig, "relationships" > & { - relationships?: Map + relationships?: Map entity?: string } @@ -106,6 +106,7 @@ export interface RemoteExpandProperty { parent: string parentConfig?: InternalJoinerServiceConfig serviceConfig: InternalJoinerServiceConfig + entity?: string fields?: string[] args?: JoinerArgument[] expands?: RemoteNestedExpands diff --git a/packages/core/utils/src/common/graphql-relations-entity-map.ts b/packages/core/utils/src/common/graphql-relations-entity-map.ts new file mode 100644 index 0000000000..27122f331c --- /dev/null +++ b/packages/core/utils/src/common/graphql-relations-entity-map.ts @@ -0,0 +1,41 @@ +import { isListType, isNonNullType, isObjectType } from "graphql" + +/** + * Extracts only the relation fields from the GraphQL type map. + * @param {Map} typeMap - The GraphQL schema TypeMap. + * @returns {Map>} A map where each key is an entity name, and the values are a map of relation fields and their corresponding entity type. + */ +export function extractRelationsFromGQL( + typeMap: Map +): Map> { + const relationMap = new Map() + + // Extract the actual type + const getBaseType = (type) => { + if (isNonNullType(type) || isListType(type)) { + return getBaseType(type.ofType) + } + return type + } + + for (const [typeName, graphqlType] of Object.entries(typeMap)) { + if (!isObjectType(graphqlType)) { + continue + } + + const fields = graphqlType.getFields() + const entityRelations = new Map() + + for (const [fieldName, fieldConfig] of Object.entries(fields)) { + const fieldType = getBaseType((fieldConfig as any).type) + + // only add relation fields + if (isObjectType(fieldType)) { + entityRelations.set(fieldName, fieldType.name) + } + } + relationMap.set(typeName, entityRelations) + } + + return relationMap +} diff --git a/packages/core/utils/src/common/index.ts b/packages/core/utils/src/common/index.ts index fefffb4fd1..ab70f60feb 100644 --- a/packages/core/utils/src/common/index.ts +++ b/packages/core/utils/src/common/index.ts @@ -26,6 +26,7 @@ export * from "./get-node-version" export * from "./get-selects-and-relations-from-object-array" export * from "./get-set-difference" export * from "./graceful-shutdown-server" +export * from "./graphql-relations-entity-map" export * from "./group-by" export * from "./handle-postgres-database-error" export * from "./is-big-number"