feat: query.index (#11348)

What:
 - `query.index` helper. It queries the index module, and aggregate the rest of requested fields/relations if needed like `query.graph`.
 
Not covered in this PR:
 - Hydrate only sub entities returned by the query. Example: 1 out of 5 variants have returned, it should only hydrate the data of the single entity, currently it will merge all the variants of the product.
 - Generate types of indexed data
 
 example:
 ```ts
 const query = container.resolve(ContainerRegistrationKeys.QUERY)
        
 await query.index({
  entity: "product",
  fields: [
    "id",
    "description",
    "status",
    "variants.sku",
    "variants.barcode",
    "variants.material",
    "variants.options.value",
    "variants.prices.amount",
    "variants.prices.currency_code",
    "variants.inventory_items.inventory.sku",
    "variants.inventory_items.inventory.description",
  ],
  filters: {
    "variants.sku": { $like: "%-1" },
    "variants.prices.amount": { $gt: 30 },
  },
  pagination: {
    order: {
      "variants.prices.amount": "DESC",
    },
  },
})
```
This query return all products where at least one variant has the title ending in `-1` and at least one price bigger than `30`.
 
The Index Module only hold the data used to paginate and filter, and the returned object is:
```json
{
  "id": "prod_01JKEAM2GJZ14K64R0DHK0JE72",
  "title": null,
  "variants": [
    {
      "id": "variant_01JKEAM2HC89GWS95F6GF9C6YA",
      "sku": "extra-variant-1",
      "prices": [
        {
          "id": "price_01JKEAM2JADEWWX72F8QDP6QXT",
          "amount": 80,
          "currency_code": "USD"
        }
      ]
    }
  ]
}
```

All the rest of the fields will be hydrated from their respective modules, and the final result will be:

```json
{
  "id": "prod_01JKEAY2RJTF8TW9A23KTGY1GD",
  "description": "extra description",
  "status": "draft",
  "variants": [
    {
      "sku": "extra-variant-1",
      "barcode": null,
      "material": null,
      "id": "variant_01JKEAY2S945CRZ6X4QZJ7GVBJ",
      "options": [
        {
          "value": "Red"
        }
      ],
      "prices": [
        {
          "amount": 20,
          "currency_code": "CAD",
          "id": "price_01JKEAY2T2EEYSWZHPGG11B7W7"
        },
        {
          "amount": 80,
          "currency_code": "USD",
          "id": "price_01JKEAY2T2NJK2E5468RK84CAR"
        }
      ],
      "inventory_items": [
        {
          "variant_id": "variant_01JKEAY2S945CRZ6X4QZJ7GVBJ",
          "inventory_item_id": "iitem_01JKEAY2SNY2AWEHPZN0DDXVW6",
          "inventory": {
            "sku": "extra-variant-1",
            "description": "extra variant 1",
            "id": "iitem_01JKEAY2SNY2AWEHPZN0DDXVW6"
          }
        }
      ]
    }
  ]
}
```

Co-authored-by: Adrien de Peretti <25098370+adrien2p@users.noreply.github.com>

packages/core/orchestration/src/__fixtures__/joiner/data.ts

@@ -178,7 +178,7 @@ export const remoteJoinerData = {
       date: "2023-04-01T12:00:00Z",
       products: [
         {
-          product_id: [101, 103],
+          product_id: 103,
           variant_id: 993,
           quantity: 4,
         },

packages/core/orchestration/src/__mocks__/joiner/mock_data.ts

@@ -59,7 +59,7 @@ export const serviceConfigs: JoinerServiceConfig[] = [
     fieldAlias: {
       user_shortcut: "product.user",
     },
-    primaryKeys: ["id"],
+    primaryKeys: ["id,product_id"],
     relationships: [
       {
         foreignKey: "product_id",

packages/core/orchestration/src/__tests__/joiner/remote-joiner-data.ts

@@ -291,17 +291,11 @@ describe("RemoteJoiner", () => {
         date: "2023-04-01T12:00:00Z",
         products: [
           {
-            product_id: [101, 103],
-            product: [
-              {
-                name: "Product 1",
-                id: 101,
-              },
-              {
-                name: "Product 3",
-                id: 103,
-              },
-            ],
+            product_id: 103,
+            product: {
+              name: "Product 3",
+              id: 103,
+            },
           },
         ],
         user_id: 1,
@@ -438,19 +432,18 @@ describe("RemoteJoiner", () => {
               number: "ORD-202",
               products: [
                 {
-                  product_id: [101, 103],
+                  product_id: 103,
                   variant_id: 993,
                   quantity: 4,
-                  product: [
-                    {
-                      name: "Product 1",
-                      id: 101,
-                    },
-                    {
-                      name: "Product 3",
-                      id: 103,
-                    },
-                  ],
+                  product: {
+                    name: "Product 3",
+                    id: 103,
+                  },
+                  variant: {
+                    id: 993,
+                    name: "Product variant 33",
+                    product_id: 103,
+                  },
                 },
               ],
               id: 205,
@@ -468,19 +461,18 @@ describe("RemoteJoiner", () => {
             number: "ORD-202",
             products: [
               {
-                product_id: [101, 103],
+                product_id: 103,
                 variant_id: 993,
                 quantity: 4,
-                product: [
-                  {
-                    name: "Product 1",
-                    id: 101,
-                  },
-                  {
-                    name: "Product 3",
-                    id: 103,
-                  },
-                ],
+                product: {
+                  name: "Product 3",
+                  id: 103,
+                },
+                variant: {
+                  id: 993,
+                  name: "Product variant 33",
+                  product_id: 103,
+                },
               },
             ],
             id: 205,
@@ -518,16 +510,10 @@ describe("RemoteJoiner", () => {
         ],
       }),
       expect.objectContaining({
-        product_user_alias: [
-          {
-            email: "janedoe@example.com",
-            id: 2,
-          },
-          {
-            email: "aaa@example.com",
-            id: 3,
-          },
-        ],
+        product_user_alias: {
+          email: "aaa@example.com",
+          id: 3,
+        },
       }),
     ])
     expect(data[0].products[0].product).toEqual(undefined)
@@ -564,16 +550,10 @@ describe("RemoteJoiner", () => {
         ],
       }),
       expect.objectContaining({
-        product_user_alias: [
-          {
-            email: "janedoe@example.com",
-            id: 2,
-          },
-          {
-            email: "aaa@example.com",
-            id: 3,
-          },
-        ],
+        product_user_alias: {
+          email: "aaa@example.com",
+          id: 3,
+        },
       }),
     ])
     expect(data[0].products[0].product).toEqual({
@@ -619,18 +599,11 @@ describe("RemoteJoiner", () => {
         ],
       }),
       expect.objectContaining({
-        product_user_alias: [
-          {
-            name: "Jane Doe",
-            id: 2,
-            email: "janedoe@example.com",
-          },
-          {
-            name: "aaa bbb",
-            id: 3,
-            email: "aaa@example.com",
-          },
-        ],
+        product_user_alias: {
+          name: "aaa bbb",
+          id: 3,
+          email: "aaa@example.com",
+        },
       }),
     ])
     expect(data[0].products[0].product).toEqual({
@@ -725,16 +698,10 @@ describe("RemoteJoiner", () => {
 
     expect(data[1]).toEqual(
       expect.objectContaining({
-        product_user_alias: [
-          {
-            id: 2,
-            name: "Jane Doe",
-          },
-          {
-            id: 3,
-            name: "aaa bbb",
-          },
-        ],
+        product_user_alias: {
+          id: 3,
+          name: "aaa bbb",
+        },
       })
     )
 
@@ -857,10 +824,13 @@ describe("RemoteJoiner", () => {
           id
           number
           products {
-            product {
-              handler
-              user {
-                name
+            variant {
+              name
+              product {
+                handler
+                user {
+                  name
+                }
               }
             }
           }
@@ -878,18 +848,22 @@ describe("RemoteJoiner", () => {
         products: [
           {
             product_id: 101,
+            variant_id: 991,
             color: "red",
-            product: {
-              id: 101,
-              product_extra_field: "extra 101 - red",
+            variant: {
+              id: 991,
+              product_id: 101,
+              variant_extra_field: "extra 101 - var 991 - red",
             },
           },
           {
             product_id: 101,
+            variant_id: 992,
             color: "green",
-            product: {
-              id: 101,
-              product_extra_field: "extra 101 - green",
+            variant: {
+              id: 992,
+              product_id: 101,
+              variant_extra_field: "extra 101 - var 992 - green",
             },
           },
         ],
@@ -897,23 +871,6 @@ describe("RemoteJoiner", () => {
       {
         id: 205,
         extra_field: "extra",
-        products: [
-          {
-            product_id: [101, 103],
-            product: [
-              {
-                id: 101,
-                color: "blue",
-                product_extra_field: "extra 101 - blue",
-              },
-              {
-                id: 103,
-                color: "yellow",
-                product_extra_field: "extra 101 - yellow",
-              },
-            ],
-          },
-        ],
       },
     ]
 
@@ -928,29 +885,41 @@ describe("RemoteJoiner", () => {
         products: [
           {
             product_id: 101,
+            variant_id: 991,
             color: "red",
-            product: {
-              id: 101,
-              product_extra_field: "extra 101 - red",
-              handler: "product-1-handler",
-              user_id: 2,
-              user: {
-                name: "Jane Doe",
-                id: 2,
+            variant: {
+              id: 991,
+              product_id: 101,
+              variant_extra_field: "extra 101 - var 991 - red",
+              name: "Product variant 1",
+              product: {
+                handler: "product-1-handler",
+                id: 101,
+                user_id: 2,
+                user: {
+                  name: "Jane Doe",
+                  id: 2,
+                },
               },
             },
           },
           {
             product_id: 101,
+            variant_id: 992,
             color: "green",
-            product: {
-              id: 101,
-              product_extra_field: "extra 101 - green",
-              handler: "product-1-handler",
-              user_id: 2,
-              user: {
-                name: "Jane Doe",
-                id: 2,
+            variant: {
+              id: 992,
+              product_id: 101,
+              variant_extra_field: "extra 101 - var 992 - green",
+              name: "Product variant 2",
+              product: {
+                handler: "product-1-handler",
+                id: 101,
+                user_id: 2,
+                user: {
+                  name: "Jane Doe",
+                  id: 2,
+                },
               },
             },
           },
@@ -965,31 +934,22 @@ describe("RemoteJoiner", () => {
         number: "ORD-202",
         products: [
           {
-            product_id: [101, 103],
-            product: [
-              {
-                id: 101,
-                color: "blue",
-                product_extra_field: "extra 101 - blue",
-                handler: "product-1-handler",
-                user_id: 2,
-                user: {
-                  name: "Jane Doe",
-                  id: 2,
-                },
-              },
-              {
-                id: 103,
-                color: "yellow",
-                product_extra_field: "extra 101 - yellow",
+            variant_id: 993,
+            product_id: 103,
+            variant: {
+              name: "Product variant 33",
+              id: 993,
+              product_id: 103,
+              product: {
                 handler: "product-3-handler",
+                id: 103,
                 user_id: 3,
                 user: {
                   name: "aaa bbb",
                   id: 3,
                 },
               },
-            ],
+            },
           },
         ],
         extra_field: "extra",

packages/core/orchestration/src/joiner/remote-joiner.ts

@@ -44,6 +44,7 @@ type InternalParseExpandsParams = {
   expands: RemoteJoinerQuery["expands"]
   implodeMapping: InternalImplodeMapping[]
   options?: RemoteJoinerOptions
+  initialData?: any[]
 }
 
 export class RemoteJoiner {
@@ -799,7 +800,9 @@ export class RemoteJoiner {
     })
 
     for (const values of fieldsById.values()) {
-      values.forEach((v) => idsToFetch.add(v))
+      values.forEach((val) => {
+        idsToFetch.add(val)
+      })
     }
 
     if (idsToFetch.size === 0) {
@@ -878,10 +881,17 @@ export class RemoteJoiner {
   private parseExpands(
     params: InternalParseExpandsParams
   ): Map<string, RemoteExpandProperty> {
-    const { initialService, query, serviceConfig, expands, implodeMapping } =
-      params
+    const {
+      initialService,
+      query,
+      serviceConfig,
+      expands,
+      implodeMapping,
+      options,
+      initialData,
+    } = params
 
-    const parsedExpands = this.parseProperties({
+    const { parsedExpands, aliasRealPathMap } = this.parseProperties({
       initialService,
       query,
       serviceConfig,
@@ -889,6 +899,14 @@ export class RemoteJoiner {
       implodeMapping,
     })
 
+    if (initialData?.length) {
+      this.createFilterFromInitialData({
+        initialData: options?.initialData as any,
+        parsedExpands,
+        aliasRealPathMap,
+      })
+    }
+
     const groupedExpands = this.groupExpands(parsedExpands)
 
     return groupedExpands
@@ -900,7 +918,10 @@ export class RemoteJoiner {
     serviceConfig: InternalJoinerServiceConfig
     expands: RemoteJoinerQuery["expands"]
     implodeMapping: InternalImplodeMapping[]
-  }): Map<string, RemoteExpandProperty> {
+  }): {
+    parsedExpands: Map<string, RemoteExpandProperty>
+    aliasRealPathMap: Map<string, string[]>
+  } {
     const { initialService, query, serviceConfig, expands, implodeMapping } =
       params
 
@@ -1039,7 +1060,7 @@ export class RemoteJoiner {
       }
     }
 
-    return parsedExpands
+    return { parsedExpands, aliasRealPathMap }
   }
 
   private getEntity({ entity, prop }: { entity: string; prop: string }) {
@@ -1216,6 +1237,206 @@ export class RemoteJoiner {
     return mergedExpands
   }
 
+  private createFilterFromInitialData({
+    initialData,
+    parsedExpands,
+    aliasRealPathMap,
+  }: {
+    initialData: any[]
+    parsedExpands: Map<string, RemoteExpandProperty>
+    aliasRealPathMap: Map<string, string[]>
+  }): void {
+    if (!initialData.length) {
+      return
+    }
+
+    const getPkValues = ({
+      initialData,
+      serviceConfig,
+      relationship,
+    }: {
+      initialData: any[]
+      serviceConfig: InternalJoinerServiceConfig
+      relationship?: JoinerRelationship
+    }): Record<string, any> => {
+      if (!initialData.length || !relationship || !serviceConfig) {
+        return {}
+      }
+
+      const primaryKeys = relationship.primaryKey
+        ? relationship.primaryKey.split(",")
+        : serviceConfig.primaryKeys
+
+      const filter: Record<string, any> = {}
+
+      // Collect IDs for the current level, considering composed keys
+      primaryKeys.forEach((key) => {
+        filter[key] = Array.from(
+          new Set(initialData.map((dt) => dt[key]).filter(isDefined))
+        )
+      })
+
+      return filter
+    }
+
+    const parsedSegment = new Map<string, any>()
+
+    const aliasReversePathMap = new Map<string, string>(
+      Array.from(aliasRealPathMap).map(([path, realPath]) => [
+        realPath.join("."),
+        path,
+      ])
+    )
+
+    for (let [path, expand] of parsedExpands.entries()) {
+      const serviceConfig = expand.serviceConfig
+      const relationship =
+        this.getEntityRelationship({
+          parentServiceConfig: expand.parentConfig!,
+          property: expand.property,
+        }) ?? serviceConfig.relationships?.get(serviceConfig.serviceName)
+
+      if (!serviceConfig || !relationship) {
+        continue
+      }
+
+      let aliasToPath: string | null = null
+      if (aliasReversePathMap.has(path)) {
+        aliasToPath = path
+        path = aliasReversePathMap.get(path)!
+      }
+
+      const pathSegments = path.split(".")
+      let relevantInitialData = initialData
+      let fullPath: string[] = []
+
+      for (const segment of pathSegments) {
+        fullPath.push(segment)
+        if (segment === BASE_PATH) {
+          continue
+        }
+
+        const pathStr = fullPath.join(".")
+        if (parsedSegment.has(pathStr)) {
+          relevantInitialData = parsedSegment.get(pathStr)
+          continue
+        }
+
+        relevantInitialData =
+          RemoteJoiner.getNestedItems(relevantInitialData, segment) ?? []
+
+        parsedSegment.set(pathStr, relevantInitialData)
+
+        if (!relevantInitialData.length) {
+          break
+        }
+      }
+
+      if (!relevantInitialData.length) {
+        continue
+      }
+
+      const queryPath = expand.parent === "" ? BASE_PATH : aliasToPath ?? path
+      const filter = getPkValues({
+        initialData: relevantInitialData,
+        serviceConfig,
+        relationship,
+      })
+
+      if (!Object.keys(filter).length) {
+        continue
+      }
+
+      const parsed = parsedExpands.get(queryPath)!
+      parsed.args ??= []
+      parsed.args.push({
+        name: "filters",
+        value: filter,
+      })
+    }
+  }
+
+  private mergeInitialData({
+    items,
+    initialData,
+    serviceConfig,
+    path,
+    expands,
+    relationship,
+  }: {
+    items: any[]
+    initialData: any[]
+    serviceConfig: InternalJoinerServiceConfig
+    path: string
+    expands?: RemoteNestedExpands
+    relationship?: JoinerRelationship
+  }) {
+    if (!initialData.length || !relationship) {
+      return items
+    }
+
+    const primaryKeys = relationship?.primaryKey.split(",") || [
+      serviceConfig.primaryKeys[0],
+    ]
+    const expandKeys = Object.keys(expands ?? {})
+
+    const initialDataIndexMap = new Map(
+      initialData.map((dt, index) => [
+        primaryKeys.map((key) => dt[key]).join(","),
+        index,
+      ])
+    )
+    const itemMap = new Map(
+      items.map((item) => [primaryKeys.map((key) => item[key]).join(","), item])
+    )
+
+    const orderedMergedItems = new Array(initialData.length)
+    for (const [key, index] of initialDataIndexMap.entries()) {
+      const iniData = initialData[index]
+      const item = itemMap.get(key)
+
+      if (!item) {
+        orderedMergedItems[index] = iniData
+        continue
+      }
+
+      // Only merge properties that are not relations
+      const shallowProperty = { ...iniData }
+      for (const key of expandKeys) {
+        const isRel = !!this.getEntityRelationship({
+          parentServiceConfig: serviceConfig,
+          property: key,
+        })
+        if (isRel) {
+          delete shallowProperty[key]
+        }
+      }
+
+      Object.assign(item, shallowProperty)
+      orderedMergedItems[index] = item
+    }
+
+    if (expands) {
+      for (const expand of expandKeys) {
+        this.mergeInitialData({
+          items: items.flatMap((dt) => dt[expand] ?? []),
+          initialData: initialData
+            .flatMap((dt) => dt[expand] ?? [])
+            .filter(isDefined),
+          serviceConfig,
+          path: `${path}.${expand}`,
+          expands: expands[expand]?.expands,
+          relationship: this.getEntityRelationship({
+            parentServiceConfig: serviceConfig,
+            property: expand,
+          }),
+        })
+      }
+    }
+
+    return orderedMergedItems
+  }
+
   async query(
     queryObj: RemoteJoinerQuery,
     options?: RemoteJoinerOptions
@@ -1239,20 +1460,35 @@ export class RemoteJoiner {
         : [options.initialData]
       : []
 
-    if (options?.initialData) {
-      let pkName = serviceConfig.primaryKeys[0]
-      queryObj.args ??= []
-      queryObj.args.push({
-        name: pkName,
-        value: iniDataArray.map((dt) => dt[pkName]),
-      })
+    const implodeMapping: InternalImplodeMapping[] = []
+    const parseExpandsConfig: InternalParseExpandsParams = {
+      initialService: {
+        property: "",
+        parent: "",
+        serviceConfig,
+        entity: serviceConfig.entity,
+        fields: queryObj.fields,
+      },
+      query: queryObj,
+      serviceConfig,
+      expands: queryObj.expands!,
+      implodeMapping,
+      options,
+      initialData: iniDataArray,
     }
 
+    const parsedExpands = this.parseExpands(parseExpandsConfig)
+    const root = parsedExpands.get(BASE_PATH)!
+
     const { primaryKeyArg, otherArgs, pkName } = gerPrimaryKeysAndOtherFilters({
       serviceConfig,
       queryObj,
     })
 
+    if (otherArgs) {
+      parseExpandsConfig.initialService.args = otherArgs
+    }
+
     if (options?.throwIfKeyNotFound) {
       if (primaryKeyArg?.value == undefined) {
         if (!primaryKeyArg) {
@@ -1275,30 +1511,6 @@ export class RemoteJoiner {
       }
     }
 
-    const implodeMapping: InternalImplodeMapping[] = []
-    const parseExpandsConfig: InternalParseExpandsParams = {
-      initialService: {
-        property: "",
-        parent: "",
-        serviceConfig,
-        entity: serviceConfig.entity,
-        fields: queryObj.fields,
-      },
-      query: queryObj,
-      serviceConfig,
-      expands: queryObj.expands!,
-      implodeMapping,
-      options,
-    }
-
-    if (otherArgs) {
-      parseExpandsConfig.initialService.args = otherArgs
-    }
-
-    const parsedExpands = this.parseExpands(parseExpandsConfig)
-
-    const root = parsedExpands.get(BASE_PATH)!
-
     const response = await this.fetchData({
       expand: root,
       pkField: pkName,
@@ -1306,28 +1518,40 @@ export class RemoteJoiner {
       options,
     })
 
-    const data = response.path ? response.data[response.path!] : response.data
+    let data = response.path ? response.data[response.path!] : response.data
+    const isDataArray = Array.isArray(data)
+
+    data = isDataArray ? data : [data]
 
     if (options?.initialData) {
-      // merge initial data with fetched data matching the primary key
-      const initialDataMap = new Map(iniDataArray.map((dt) => [dt[pkName], dt]))
-      for (const resData of data) {
-        const iniData = initialDataMap.get(resData[pkName])
+      data = this.mergeInitialData({
+        items: data,
+        initialData: iniDataArray,
+        serviceConfig,
+        path: BASE_PATH,
+        expands: parsedExpands.get(BASE_PATH)?.expands,
+        relationship: serviceConfig.relationships?.get(
+          serviceConfig.serviceName
+        ) as JoinerRelationship,
+      })
 
-        if (iniData) {
-          Object.assign(resData, iniData)
-        }
-      }
       delete options?.initialData
     }
 
     await this.handleExpands({
-      items: Array.isArray(data) ? data : [data],
+      items: data,
       parsedExpands,
       implodeMapping,
       options,
     })
 
+    const retData = isDataArray ? data : data[0]
+    if (response.path) {
+      response.data[response.path] = retData
+    } else {
+      response.data = retData
+    }
+
     return response.data
   }
 }
@@ -1350,10 +1574,10 @@ function gerPrimaryKeysAndOtherFilters({ serviceConfig, queryObj }): {
     (arg) => !serviceConfig.primaryKeys.includes(arg.name)
   )
 
-  const filters =
-    queryObj.args?.find((arg) => arg.name === "filters")?.value ?? {}
-
   if (!primaryKeyArg) {
+    const filters =
+      queryObj.args?.find((arg) => arg.name === "filters")?.value ?? {}
+
     const primaryKeyFilter = Object.keys(filters).find((key) => {
       return serviceConfig.primaryKeys.includes(key)
     })