feat(orchestration): hydrate resultset (#11263)

What:
* Add support for aggregating data into existing resultset.
* `query.graph` new option `initialData` containing the resultset to be hydrated
* It fetches data where the requested fields are not present and merge with the existing resultset

.changeset/honest-ants-guess.md

@@ -0,0 +1,6 @@
+---
+"@medusajs/orchestration": patch
+"@medusajs/types": patch
+---
+
+feat(orchestration): hydrate resultset

packages/core/orchestration/src/__tests__/joiner/remote-joiner-data.ts

@@ -849,4 +849,151 @@ describe("RemoteJoiner", () => {
       "order: Primary key(s) [id] not found in filters"
     )
   })
+
+  it("Should merge initial data with data fetched", async () => {
+    const query = RemoteJoiner.parseQuery(`
+      query {
+        order {
+          id
+          number
+          products {
+            product {
+              handler
+              user {
+                name
+              }
+            }
+          }
+        }
+      }
+    `)
+
+    const initialData = [
+      {
+        id: 201,
+        extra_field: "extra",
+        metadata: {
+          some: "data",
+        },
+        products: [
+          {
+            product_id: 101,
+            color: "red",
+            product: {
+              id: 101,
+              product_extra_field: "extra 101 - red",
+            },
+          },
+          {
+            product_id: 101,
+            color: "green",
+            product: {
+              id: 101,
+              product_extra_field: "extra 101 - green",
+            },
+          },
+        ],
+      },
+      {
+        id: 205,
+        extra_field: "extra",
+        products: [
+          {
+            product_id: [101, 103],
+            product: [
+              {
+                id: 101,
+                color: "blue",
+                product_extra_field: "extra 101 - blue",
+              },
+              {
+                id: 103,
+                color: "yellow",
+                product_extra_field: "extra 101 - yellow",
+              },
+            ],
+          },
+        ],
+      },
+    ]
+
+    const data = await joiner.query(query, {
+      initialData,
+    })
+
+    expect(data).toEqual([
+      {
+        id: 201,
+        number: "ORD-001",
+        products: [
+          {
+            product_id: 101,
+            color: "red",
+            product: {
+              id: 101,
+              product_extra_field: "extra 101 - red",
+              handler: "product-1-handler",
+              user_id: 2,
+              user: {
+                name: "Jane Doe",
+                id: 2,
+              },
+            },
+          },
+          {
+            product_id: 101,
+            color: "green",
+            product: {
+              id: 101,
+              product_extra_field: "extra 101 - green",
+              handler: "product-1-handler",
+              user_id: 2,
+              user: {
+                name: "Jane Doe",
+                id: 2,
+              },
+            },
+          },
+        ],
+        extra_field: "extra",
+        metadata: {
+          some: "data",
+        },
+      },
+      {
+        id: 205,
+        number: "ORD-202",
+        products: [
+          {
+            product_id: [101, 103],
+            product: [
+              {
+                id: 101,
+                color: "blue",
+                product_extra_field: "extra 101 - blue",
+                handler: "product-1-handler",
+                user_id: 2,
+                user: {
+                  name: "Jane Doe",
+                  id: 2,
+                },
+              },
+              {
+                id: 103,
+                color: "yellow",
+                product_extra_field: "extra 101 - yellow",
+                handler: "product-3-handler",
+                user_id: 3,
+                user: {
+                  name: "aaa bbb",
+                  id: 3,
+                },
+              },
+            ],
+          },
+        ],
+        extra_field: "extra",
+      },
+    ])
+  })
 })

packages/core/orchestration/src/joiner/remote-joiner.ts

@@ -765,35 +765,51 @@ export class RemoteJoiner {
       : relationship.foreignKey.split(".").pop()!
     const fieldsArray = field.split(",")
 
-    const idsToFetch: any[] = []
+    const idsToFetch: Set<any> = new Set()
 
+    const requestedFields = new Set(expand.fields ?? [])
+    const fieldsById = new Map<string, string[]>()
     items.forEach((item) => {
       const values = fieldsArray.map((field) => item?.[field])
 
-      if (values.length === fieldsArray.length && !item?.[relationship.alias]) {
-        if (fieldsArray.length === 1) {
-          if (!idsToFetch.includes(values[0])) {
-            idsToFetch.push(values[0])
+      if (values.length === fieldsArray.length) {
+        if (item?.[relationship.alias]) {
+          for (const field of requestedFields.values()) {
+            if (field in item[relationship.alias]) {
+              requestedFields.delete(field)
+              fieldsById.delete(field)
+            } else {
+              if (!fieldsById.has(field)) {
+                fieldsById.set(field, [])
+              }
+
+              fieldsById
+                .get(field)!
+                .push(fieldsArray.length === 1 ? values[0] : values)
+            }
           }
         } else {
-          // composite key
-          const valuesString = values.join(",")
-
-          if (!idsToFetch.some((id) => id.join(",") === valuesString)) {
-            idsToFetch.push(values)
+          if (fieldsArray.length === 1) {
+            idsToFetch.add(values[0])
+          } else {
+            idsToFetch.add(values)
           }
         }
       }
     })
 
-    if (idsToFetch.length === 0) {
+    for (const values of fieldsById.values()) {
+      values.forEach((v) => idsToFetch.add(v))
+    }
+
+    if (idsToFetch.size === 0) {
       return
     }
 
     const relatedDataArray = await this.fetchData({
       expand,
       pkField: field,
-      ids: idsToFetch,
+      ids: Array.from(idsToFetch),
       relationship,
       options,
     })
@@ -812,12 +828,31 @@ export class RemoteJoiner {
     )
 
     items.forEach((item) => {
-      if (!item || item[relationship.alias]) {
+      if (!item) {
         return
       }
 
       const itemKey = fieldsArray.map((field) => item[field]).join(",")
 
+      if (item[relationship.alias]) {
+        if (Array.isArray(item[field])) {
+          for (let i = 0; i < item[relationship.alias].length; i++) {
+            const it = item[relationship.alias][i]
+            item[relationship.alias][i] = Object.assign(
+              it,
+              relatedDataMap[it[relationship.primaryKey]]
+            )
+          }
+          return
+        }
+
+        item[relationship.alias] = Object.assign(
+          item[relationship.alias],
+          relatedDataMap[itemKey]
+        )
+        return
+      }
+
       if (Array.isArray(item[field])) {
         item[relationship.alias] = item[field].map((id) => {
           if (relationship.isList && !Array.isArray(relatedDataMap[id])) {
@@ -874,7 +909,6 @@ export class RemoteJoiner {
     parsedExpands.set(BASE_PATH, initialService)
 
     const forwardArgumentsOnPath: string[] = []
-
     for (const expand of expands || []) {
       const properties = expand.property.split(".")
       const currentPath: string[] = []
@@ -883,7 +917,6 @@ export class RemoteJoiner {
 
       for (const prop of properties) {
         const fieldAlias = currentServiceConfig.fieldAlias ?? {}
-
         if (fieldAlias[prop]) {
           const aliasPath = [BASE_PATH, ...currentPath, prop].join(".")
 
@@ -901,9 +934,7 @@ export class RemoteJoiner {
           })
 
           currentAliasPath.push(prop)
-
           currentServiceConfig = lastServiceConfig
-
           continue
         }
 
@@ -1202,6 +1233,21 @@ export class RemoteJoiner {
       throw new Error(`Service "${queryObj.service}" was not found.`)
     }
 
+    const iniDataArray = options?.initialData
+      ? Array.isArray(options.initialData)
+        ? options.initialData
+        : [options.initialData]
+      : []
+
+    if (options?.initialData) {
+      let pkName = serviceConfig.primaryKeys[0]
+      queryObj.args ??= []
+      queryObj.args.push({
+        name: pkName,
+        value: iniDataArray.map((dt) => dt[pkName]),
+      })
+    }
+
     const { primaryKeyArg, otherArgs, pkName } = gerPrimaryKeysAndOtherFilters({
       serviceConfig,
       queryObj,
@@ -1262,6 +1308,19 @@ export class RemoteJoiner {
 
     const data = response.path ? response.data[response.path!] : response.data
 
+    if (options?.initialData) {
+      // merge initial data with fetched data matching the primary key
+      const initialDataMap = new Map(iniDataArray.map((dt) => [dt[pkName], dt]))
+      for (const resData of data) {
+        const iniData = initialDataMap.get(resData[pkName])
+
+        if (iniData) {
+          Object.assign(resData, iniData)
+        }
+      }
+      delete options?.initialData
+    }
+
     await this.handleExpands({
       items: Array.isArray(data) ? data : [data],
       parsedExpands,

packages/core/types/src/joiner/index.ts

@@ -84,6 +84,7 @@ export interface RemoteJoinerQuery {
 export interface RemoteJoinerOptions {
   throwIfKeyNotFound?: boolean
   throwIfRelationNotFound?: boolean | string[]
+  initialData?: object | object[]
 }
 
 export interface RemoteNestedExpands {