feat(medusa): Rollout index engine behind feature flag (#11431)

**What**
- Add index engine feature flag
- apply it to the `store/products` end point as well as `admin/products`
- Query builder various fixes
- search capabilities on full data of every entities. The `q` search will be applied to all involved joined table for selection/where clauses

Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>

packages/core/modules-sdk/src/remote-query/query.ts

@@ -14,6 +14,7 @@ import {
   MedusaError,
   isObject,
   remoteQueryObjectFromString,
+  unflattenObjectKeys,
 } from "@medusajs/utils"
 import { RemoteQuery } from "./remote-query"
 import { toRemoteQuery } from "./to-remote-query"
@@ -211,7 +212,9 @@ export class Query {
       : ({} as any)
     const pagination = queryOptions.pagination as any
     if (pagination?.order) {
-      pagination.order = { [mainEntity]: pagination.order }
+      pagination.order = {
+        [mainEntity]: unflattenObjectKeys(pagination?.order),
+      }
     }
 
     const indexResponse = (await this.#indexModule.query({

packages/core/modules-sdk/src/remote-query/remote-query.ts

@@ -18,6 +18,8 @@ import { isPresent, isString, toPascalCase } from "@medusajs/utils"
 import { MedusaModule } from "../medusa-module"
 
 const BASE_PREFIX = ""
+const MAX_BATCH_SIZE = 4000
+const MAX_CONCURRENT_REQUESTS = 10
 export class RemoteQuery {
   private remoteJoiner: RemoteJoiner
   private modulesMap: Map<string, LoadedModule> = new Map()
@@ -182,6 +184,102 @@ export class RemoteQuery {
     }
   }
 
+  private async fetchRemoteDataBatched(args: {
+    serviceName: string
+    keyField: string
+    service: any
+    methodName: string
+    filters: any
+    options: any
+    ids: (unknown | unknown[])[]
+  }): Promise<any[]> {
+    const {
+      serviceName,
+      keyField,
+      service,
+      methodName,
+      filters,
+      options,
+      ids,
+    } = args
+
+    const getBatch = function* (
+      idArray: (unknown | unknown[])[],
+      batchSize: number
+    ) {
+      for (let i = 0; i < idArray.length; i += batchSize) {
+        yield idArray.slice(i, i + batchSize)
+      }
+    }
+
+    const idsToFetch = getBatch(ids, MAX_BATCH_SIZE)
+    const results: any[] = []
+    let running = 0
+    const fetchPromises: Promise<void>[] = []
+
+    const processBatch = async (batch: (unknown | unknown[])[]) => {
+      running++
+      const batchFilters = { ...filters, [keyField]: batch }
+      let result
+
+      try {
+        if (RemoteQuery.traceFetchRemoteData) {
+          result = await RemoteQuery.traceFetchRemoteData(
+            async () => service[methodName](batchFilters, options),
+            serviceName,
+            methodName,
+            options
+          )
+        } else {
+          result = await service[methodName](batchFilters, options)
+        }
+        results.push(result)
+      } finally {
+        running--
+        processAllBatches()
+      }
+    }
+
+    let batchesDone: (value: void) => void = () => {}
+    const awaitBatches = new Promise((ok) => {
+      batchesDone = ok
+    })
+    const processAllBatches = async () => {
+      let isDone = false
+      while (running < MAX_CONCURRENT_REQUESTS) {
+        const nextBatch = idsToFetch.next()
+        if (nextBatch.done) {
+          isDone = true
+          break
+        }
+
+        const batch = nextBatch.value
+        fetchPromises.push(processBatch(batch))
+      }
+
+      if (isDone) {
+        await Promise.all(fetchPromises)
+        batchesDone()
+      }
+    }
+
+    processAllBatches()
+    await awaitBatches
+
+    const flattenedResults = results.reduce((acc, result) => {
+      if (
+        Array.isArray(result) &&
+        result.length === 2 &&
+        Array.isArray(result[0])
+      ) {
+        return acc.concat(result[0])
+      }
+      return acc.concat(result)
+    }, [])
+
+    return flattenedResults
+  }
+
   public async remoteFetchData(
     expand: RemoteExpandProperty,
     keyField: string,
@@ -267,6 +365,19 @@ export class RemoteQuery {
       options.take = null
     }
 
+    if (ids && ids.length >= MAX_BATCH_SIZE && !hasPagination) {
+      const data = await this.fetchRemoteDataBatched({
+        serviceName: serviceConfig.serviceName,
+        keyField,
+        service,
+        methodName,
+        filters,
+        options,
+        ids,
+      })
+      return { data }
+    }
+
     let result: any
     if (RemoteQuery.traceFetchRemoteData) {
       result = await RemoteQuery.traceFetchRemoteData(

packages/core/utils/src/common/__tests__/unflatten-object-keys.spec.ts

@@ -0,0 +1,45 @@
+import { unflattenObjectKeys } from "../unflatten-object-keys"
+
+describe("unflattenWhereClauses", () => {
+  it("should unflatten where clauses", () => {
+    const where = {
+      "variants.sku": { $like: "%-1" },
+      "variants.prices.amount": { $gt: 30 },
+      "variants.prices.currency_code": "USD",
+      variants: {
+        prices: {
+          something: "else",
+        },
+      },
+    }
+
+    const result = unflattenObjectKeys(where)
+
+    expect(result).toEqual({
+      variants: {
+        prices: {
+          something: "else",
+          amount: {
+            $gt: 30,
+          },
+          currency_code: "USD",
+        },
+        sku: {
+          $like: "%-1",
+        },
+      },
+    })
+  })
+
+  it("should unflatten obj", () => {
+    const where = {
+      created_at: "ASC",
+    }
+
+    const result = unflattenObjectKeys(where)
+
+    expect(result).toEqual({
+      created_at: "ASC",
+    })
+  })
+})

packages/core/utils/src/common/index.ts

@@ -46,6 +46,8 @@ export * from "./load-env"
 export * from "./lower-case-first"
 export * from "./map-object-to"
 export * from "./medusa-container"
+export * from "./merge-metadata"
+export * from "./merge-plugin-modules"
 export * from "./normalize-import-path-with-source"
 export * from "./object-from-string-path"
 export * from "./object-to-string-path"
@@ -76,10 +78,9 @@ export * from "./to-camel-case"
 export * from "./to-handle"
 export * from "./to-kebab-case"
 export * from "./to-pascal-case"
+export * from "./to-unix-slash"
 export * from "./trim-zeros"
+export * from "./unflatten-object-keys"
 export * from "./upper-case-first"
 export * from "./validate-handle"
 export * from "./wrap-handler"
-export * from "./merge-plugin-modules"
-export * from "./to-unix-slash"
-export * from "./merge-metadata"

packages/core/utils/src/common/unflatten-object-keys.ts

@@ -0,0 +1,68 @@
+import { isObject } from "./is-object"
+
+/**
+ * unFlatten object keys
+ * @example
+ * input: {
+ *   "variants.sku": { $like: "%-1" },
+ *   "variants.prices.amount": { $gt: 30 },
+ *   "variants.prices.currency": "USD"
+ * }
+ *
+ * output: {
+ *   {
+ *       "variants": {
+ *         "sku": {
+ *           "$like": "%-1"
+ *         },
+ *         "prices": {
+ *           "amount": {
+ *             "$gt": 30
+ *           },
+ *           "currency": "USD"
+ *         }
+ *       }
+ *     }
+ * }
+ *
+ * @param input
+ */
+export function unflattenObjectKeys(
+  flattened: Record<string, any>
+): Record<string, any> {
+  const result: Record<string, any> = {}
+
+  for (const key in flattened) {
+    if (!key.includes(".")) {
+      if (isObject(result[key])) {
+        result[key] = { ...result[key], ...flattened[key] }
+      } else {
+        result[key] = flattened[key]
+      }
+    }
+  }
+
+  for (const key in flattened) {
+    if (key.includes(".")) {
+      const value = flattened[key]
+      const keys = key.split(".")
+      let current = result
+
+      for (let i = 0; i < keys.length; i++) {
+        const part = keys[i]
+
+        if (i === keys.length - 1) {
+          if (isObject(value) && current[part]) {
+            current[part] = { ...current[part], ...value }
+          } else {
+            current[part] = value
+          }
+        } else {
+          current = current[part] = current[part] ?? {}
+        }
+      }
+    }
+  }
+
+  return result
+}