feat(medusa): Rollout index engine behind feature flag (#11431)

**What**
- Add index engine feature flag
- apply it to the `store/products` end point as well as `admin/products`
- Query builder various fixes
- search capabilities on full data of every entities. The `q` search will be applied to all involved joined table for selection/where clauses

Co-authored-by: Carlos R. L. Rodrigues <37986729+carlos-r-l-rodrigues@users.noreply.github.com>

packages/medusa/src/api/admin/products/middlewares.ts

@@ -1,4 +1,5 @@
 import {
+  featureFlagRouter,
   validateAndTransformBody,
   validateAndTransformQuery,
 } from "@medusajs/framework"
@@ -35,6 +36,7 @@ import {
   CreateProduct,
   CreateProductVariant,
 } from "./validators"
+import IndexEngineFeatureFlag from "../../../loaders/feature-flags/index-engine"
 
 // TODO: For now we keep the files in memory, as that's how they get passed to the workflows
 // This will need revisiting once we are closer to prod-ready v2, since with workflows and potentially
@@ -50,11 +52,17 @@ export const adminProductRoutesMiddlewares: MiddlewareRoute[] = [
         AdminGetProductsParams,
         QueryConfig.listProductQueryConfig
       ),
-      maybeApplyLinkFilter({
-        entryPoint: "product_sales_channel",
-        resourceId: "product_id",
-        filterableField: "sales_channel_id",
-      }),
+      (req, res, next) => {
+        if (featureFlagRouter.isFeatureEnabled(IndexEngineFeatureFlag.key)) {
+          return next()
+        }
+
+        return maybeApplyLinkFilter({
+          entryPoint: "product_sales_channel",
+          resourceId: "product_id",
+          filterableField: "sales_channel_id",
+        })(req, res, next)
+      },
       maybeApplyPriceListsFilter(),
     ],
   },

packages/medusa/src/api/admin/products/route.ts

@@ -7,11 +7,33 @@ import {
   refetchEntity,
 } from "@medusajs/framework/http"
 import { remapKeysForProduct, remapProductResponse } from "./helpers"
+import IndexEngineFeatureFlag from "../../../loaders/feature-flags/index-engine"
+import { featureFlagRouter } from "@medusajs/framework"
+import { ContainerRegistrationKeys, isPresent } from "@medusajs/framework/utils"
 
 export const GET = async (
   req: AuthenticatedMedusaRequest<HttpTypes.AdminProductListParams>,
   res: MedusaResponse<HttpTypes.AdminProductListResponse>
 ) => {
+  if (featureFlagRouter.isFeatureEnabled(IndexEngineFeatureFlag.key)) {
+    // TODO: These filters are not supported by the index engine yet
+    if (
+      isPresent(req.filterableFields.tags) ||
+      isPresent(req.filterableFields.categories)
+    ) {
+      return await getProducts(req, res)
+    }
+
+    return await getProductsWithIndexEngine(req, res)
+  }
+
+  return await getProducts(req, res)
+}
+
+async function getProducts(
+  req: AuthenticatedMedusaRequest<HttpTypes.AdminProductListParams>,
+  res: MedusaResponse<HttpTypes.AdminProductListResponse>
+) {
   const selectFields = remapKeysForProduct(req.queryConfig.fields ?? [])
 
   const { rows: products, metadata } = await refetchEntities(
@@ -30,6 +52,27 @@ export const GET = async (
   })
 }
 
+async function getProductsWithIndexEngine(
+  req: AuthenticatedMedusaRequest<HttpTypes.AdminProductListParams>,
+  res: MedusaResponse<HttpTypes.AdminProductListResponse>
+) {
+  const query = req.scope.resolve(ContainerRegistrationKeys.QUERY)
+
+  const { data: products, metadata } = await query.index({
+    entity: "product",
+    fields: req.queryConfig.fields ?? [],
+    filters: req.filterableFields,
+    pagination: req.queryConfig.pagination,
+  })
+
+  res.json({
+    products: products.map(remapProductResponse),
+    count: metadata!.count,
+    offset: metadata!.skip,
+    limit: metadata!.take,
+  })
+}
+
 export const POST = async (
   req: AuthenticatedMedusaRequest<
     HttpTypes.AdminCreateProduct & AdditionalData

packages/medusa/src/api/store/products/middlewares.ts

@@ -1,4 +1,7 @@
-import { validateAndTransformQuery } from "@medusajs/framework"
+import {
+  featureFlagRouter,
+  validateAndTransformQuery,
+} from "@medusajs/framework"
 import {
   applyDefaultFilters,
   applyParamsAsFilters,
@@ -16,6 +19,7 @@ import {
 } from "../../utils/middlewares"
 import * as QueryConfig from "./query-config"
 import { StoreGetProductsParams } from "./validators"
+import IndexEngineFeatureFlag from "../../../loaders/feature-flags/index-engine"
 
 export const storeProductRoutesMiddlewares: MiddlewareRoute[] = [
   {
@@ -30,11 +34,17 @@ export const storeProductRoutesMiddlewares: MiddlewareRoute[] = [
         QueryConfig.listProductQueryConfig
       ),
       filterByValidSalesChannels(),
-      maybeApplyLinkFilter({
-        entryPoint: "product_sales_channel",
-        resourceId: "product_id",
-        filterableField: "sales_channel_id",
-      }),
+      (req, res, next) => {
+        if (featureFlagRouter.isFeatureEnabled(IndexEngineFeatureFlag.key)) {
+          return next()
+        }
+
+        return maybeApplyLinkFilter({
+          entryPoint: "product_sales_channel",
+          resourceId: "product_id",
+          filterableField: "sales_channel_id",
+        })(req, res, next)
+      },
       applyDefaultFilters({
         status: ProductStatus.PUBLISHED,
         // TODO: the type here seems off and the implementation does not take into account $and and $or possible filters. Might be worth re working (original type used here was StoreGetProductsParamsType)

packages/medusa/src/api/store/products/route.ts

@@ -1,17 +1,95 @@
+import { featureFlagRouter } from "@medusajs/framework"
+import { MedusaResponse } from "@medusajs/framework/http"
+import { HttpTypes } from "@medusajs/framework/types"
 import {
   ContainerRegistrationKeys,
   isPresent,
+  QueryContext,
   remoteQueryObjectFromString,
 } from "@medusajs/framework/utils"
-import { MedusaResponse } from "@medusajs/framework/http"
+import IndexEngineFeatureFlag from "../../../loaders/feature-flags/index-engine"
 import { wrapVariantsWithInventoryQuantityForSalesChannel } from "../../utils/middlewares"
 import { RequestWithContext, wrapProductsWithTaxPrices } from "./helpers"
-import { HttpTypes } from "@medusajs/framework/types"
 
 export const GET = async (
   req: RequestWithContext<HttpTypes.StoreProductListParams>,
   res: MedusaResponse<HttpTypes.StoreProductListResponse>
 ) => {
+  if (featureFlagRouter.isFeatureEnabled(IndexEngineFeatureFlag.key)) {
+    // TODO: These filters are not supported by the index engine yet
+    if (
+      isPresent(req.filterableFields.tags) ||
+      isPresent(req.filterableFields.categories)
+    ) {
+      return await getProducts(req, res)
+    }
+
+    return await getProductsWithIndexEngine(req, res)
+  }
+
+  return await getProducts(req, res)
+}
+
+async function getProductsWithIndexEngine(
+  req: RequestWithContext<HttpTypes.StoreProductListParams>,
+  res: MedusaResponse<HttpTypes.StoreProductListResponse>
+) {
+  const query = req.scope.resolve(ContainerRegistrationKeys.QUERY)
+
+  const context: object = {}
+  const withInventoryQuantity = req.queryConfig.fields.some((field) =>
+    field.includes("variants.inventory_quantity")
+  )
+
+  if (withInventoryQuantity) {
+    req.queryConfig.fields = req.queryConfig.fields.filter(
+      (field) => !field.includes("variants.inventory_quantity")
+    )
+  }
+
+  if (isPresent(req.pricingContext)) {
+    context["variants"] ??= {}
+    context["variants.calculated_price"] = QueryContext(req.pricingContext!)
+  }
+
+  const filters: Record<string, any> = req.filterableFields
+  if (isPresent(filters.sales_channel_id)) {
+    const salesChannelIds = filters.sales_channel_id
+
+    filters["sales_channels"] ??= {}
+    filters["sales_channels"]["id"] = salesChannelIds
+
+    delete filters.sales_channel_id
+  }
+
+  const { data: products = [], metadata } = await query.index({
+    entity: "product",
+    fields: req.queryConfig.fields,
+    filters,
+    pagination: req.queryConfig.pagination,
+    context,
+  })
+
+  if (withInventoryQuantity) {
+    await wrapVariantsWithInventoryQuantityForSalesChannel(
+      req,
+      products.map((product) => product.variants).flat(1)
+    )
+  }
+
+  await wrapProductsWithTaxPrices(req, products)
+  res.json({
+    products,
+    count: metadata!.count,
+    offset: metadata!.skip,
+    limit: metadata!.take,
+  })
+}
+
+async function getProducts(
+  req: RequestWithContext<HttpTypes.StoreProductListParams>,
+  res: MedusaResponse<HttpTypes.StoreProductListResponse>
+) {
   const remoteQuery = req.scope.resolve(ContainerRegistrationKeys.REMOTE_QUERY)
   const context: object = {}
   const withInventoryQuantity = req.queryConfig.fields.some((field) =>

packages/medusa/src/api/utils/common-validators/products/index.ts

@@ -1,5 +1,5 @@
-import { FilterableProductProps } from "@medusajs/framework/types"
-import { ProductStatus } from "@medusajs/framework/utils"
+import { FilterableProductProps, OperatorMap } from "@medusajs/framework/types"
+import { isPresent, ProductStatus } from "@medusajs/framework/utils"
 import { z } from "zod"
 import { createOperatorMap } from "../../validators"
 import { booleanString } from "../common"
@@ -36,14 +36,19 @@ type HttpProductFilters = FilterableProductProps & {
 export const transformProductParams = (
   data: HttpProductFilters
 ): FilterableProductProps => {
-  const res = {
+  const res: HttpProductFilters = {
     ...data,
-    tags: { id: data.tag_id },
-    categories: { id: data.category_id },
   }
 
-  delete res.tag_id
-  delete res.category_id
+  if (isPresent(data.tag_id)) {
+    res.tags = { id: data.tag_id as string[] }
+    delete res.tag_id
+  }
+
+  if (isPresent(data.category_id)) {
+    res.categories = { id: data.category_id as OperatorMap<string> }
+    delete res.category_id
+  }
 
   return res as FilterableProductProps
 }