fix(index): Apply various fixes to the index engine (#12501)

.changeset/plenty-shirts-smash.md

@@ -0,0 +1,9 @@
+---
+"@medusajs/link-modules": patch
+"@medusajs/index": patch
+"@medusajs/utils": patch
+"@medusajs/modules-sdk": patch
+"@medusajs/types": patch
+---
+
+fix(index): limit partition table name and index enum fields

packages/core/modules-sdk/src/remote-query/query.ts

@@ -232,7 +232,7 @@ export class Query {
       pagination: {
         // We pass through `take` to force the `select-in` query strategy
         //   There might be a better way to do this, but for now this should do
-        take: queryOptions.pagination?.take,
+        take: queryOptions.pagination?.take ?? indexResponse.data.length,
       },
     }
 

packages/core/types/src/index-data/__tests__/index.spec.ts

@@ -23,14 +23,34 @@ describe("IndexQueryConfig", () => {
 
     expectTypeOf<IndexConfig["filters"]>().toEqualTypeOf<
       | {
-          id?: string | string[] | OperatorMap<string>
+          id?: string | string[] | OperatorMap<string | string[] | null> | null
-          title?: string | string[] | OperatorMap<string>
+          title?:
+            | string
+            | string[]
+            | OperatorMap<string | string[] | null>
+            | null
           variants?: {
-            id?: string | string[] | OperatorMap<string>
+            id?:
-            product_id?: string | string[] | OperatorMap<string>
+              | string
-            sku?: string | string[] | OperatorMap<string>
+              | string[]
+              | OperatorMap<string | string[] | null>
+              | null
+            product_id?:
+              | string
+              | string[]
+              | OperatorMap<string | string[] | null>
+              | null
+            sku?:
+              | string
+              | string[]
+              | OperatorMap<string | string[] | null>
+              | null
             prices?: {
-              amount?: number | number[] | OperatorMap<number>
+              amount?:
+                | number
+                | number[]
+                | OperatorMap<number | number[] | null>
+                | null
             }
           }
         }

packages/core/types/src/index-data/index-operator-map.ts

@@ -1,12 +1,12 @@
 export type IndexOperatorMap<T> = {
-  $eq: T
+  $eq?: T
-  $lt: T
+  $lt?: T
-  $lte: T
+  $lte?: T
-  $gt: T
+  $gt?: T
-  $gte: T
+  $gte?: T
-  $ne: T
+  $ne?: T
-  $in: T
+  $in?: T
-  $is: T
+  $is?: T
-  $like: T
+  $like?: T
-  $ilike: T
+  $ilike?: T
 }

packages/core/types/src/index-data/query-config/query-input-config-filters.ts

@@ -19,8 +19,12 @@ type ExtractFiltersOperators<
     ? never
     : Key extends ExcludedProps
     ? never
-    : TypeOnly<T[Key]> extends string | number | boolean | Date
+    : TypeOnly<T[Key]> extends string | number | boolean | Date | null
-    ? TypeOnly<T[Key]> | TypeOnly<T[Key]>[] | OperatorMap<TypeOnly<T[Key]>>
+    ?
+        | TypeOnly<T[Key]>
+        | TypeOnly<T[Key]>[]
+        | OperatorMap<TypeOnly<T[Key]> | TypeOnly<T[Key]>[] | null>
+        | null
     : TypeOnly<T[Key]> extends Array<infer R>
     ? TypeOnly<R> extends { __typename: any }
       ? IndexFilters<Key & string, T, [Key & string, ...Exclusion], Depth[Lim]>

packages/core/types/src/index-data/query-config/query-input-config.ts

@@ -63,7 +63,6 @@ export type IndexQueryConfig<TEntry extends string> = {
   filters?: IndexFilters<TEntry>
   joinFilters?: IndexFilters<TEntry>
   pagination?: Partial<IndexQueryInput<TEntry>["pagination"]>
-  keepFilteredEntities?: boolean
 }
 
 export type QueryFunctionReturnPagination = {

packages/core/utils/src/common/compress-name.ts

@@ -1,4 +1,5 @@
-import { camelToSnakeCase, simpleHash } from "@medusajs/framework/utils"
+import { camelToSnakeCase } from "./camel-to-snake-case"
+import { simpleHash } from "./simple-hash"
 
 export function compressName(name: string, limit = 58) {
   if (name.length <= limit) {

packages/core/utils/src/common/index.ts

@@ -4,6 +4,7 @@ export * from "./array-intersection"
 export * from "./build-query"
 export * from "./build-regexp-if-valid"
 export * from "./camel-to-snake-case"
+export * from "./compress-name"
 export * from "./container"
 export * from "./convert-item-response-to-update-request"
 export * from "./create-container-like"
@@ -80,11 +81,11 @@ export * from "./to-handle"
 export * from "./to-kebab-case"
 export * from "./to-pascal-case"
 export * from "./to-unix-slash"
-export * from "./try-convert-to-number"
 export * from "./trim-zeros"
+export * from "./try-convert-to-boolean"
+export * from "./try-convert-to-number"
 export * from "./unflatten-object-keys"
 export * from "./upper-case-first"
 export * from "./validate-handle"
-export * from "./wrap-handler"
 export * from "./validate-module-name"
-export * from "./try-convert-to-boolean"
+export * from "./wrap-handler"

packages/core/utils/src/event-bus/utils.ts

@@ -44,8 +44,7 @@ export function buildModuleResourceEventName({
   action: string
 }): string {
   const kebabCaseName = lowerCaseFirst(kebabCase(objectName))
-  const conventionalePrefix = prefix && lowerCaseFirst(kebabCase(prefix))
+  return `${prefix ? `${prefix}.` : ""}${kebabCaseName}.${action}`
-  return `${prefix ? `${conventionalePrefix}.` : ""}${kebabCaseName}.${action}`
 }
 
 /**

packages/modules/index/integration-tests/__tests__/query-builder.spec.ts

@@ -284,6 +284,34 @@ describe("IndexModuleService query", function () {
 
   afterEach(afterEach_)
 
+  it("should query all products where sku not null", async () => {
+    const { data } = await module.query({
+      fields: ["product.*", "product.variants.*", "product.variants.prices.*"],
+      filters: {
+        product: {
+          variants: {
+            sku: { $ne: null },
+          },
+        },
+      },
+    })
+
+    expect(data.length).toEqual(1)
+
+    const { data: data2 } = await module.query({
+      fields: ["product.*", "product.variants.*", "product.variants.prices.*"],
+      filters: {
+        product: {
+          variants: {
+            sku: { $eq: null },
+          },
+        },
+      },
+    })
+
+    expect(data2.length).toEqual(0)
+  })
+
   it("should query all products ordered by sku DESC", async () => {
     const { data } = await module.query({
       fields: ["product.*", "product.variants.*", "product.variants.prices.*"],
@@ -710,60 +738,6 @@ describe("IndexModuleService query", function () {
     ])
   })
 
-  it("should query products filtering by price and returning the complete entity", async () => {
-    const { data, metadata } = await module.query({
-      fields: ["product.*", "product.variants.*", "product.variants.prices.*"],
-      filters: {
-        product: {
-          variants: {
-            prices: {
-              amount: { $gt: 50 },
-            },
-          },
-        },
-      },
-      keepFilteredEntities: true,
-      pagination: {
-        take: 100,
-        skip: 0,
-      },
-    })
-
-    expect(metadata).toEqual({
-      estimate_count: expect.any(Number),
-      skip: 0,
-      take: 100,
-    })
-
-    expect(data).toEqual([
-      {
-        id: "prod_1",
-        variants: [
-          {
-            id: "var_1",
-            sku: "aaa test aaa",
-            prices: [
-              {
-                id: "money_amount_1",
-                amount: 100,
-              },
-            ],
-          },
-          {
-            id: "var_2",
-            sku: "sku 123",
-            prices: [
-              {
-                id: "money_amount_2",
-                amount: 10,
-              },
-            ],
-          },
-        ],
-      },
-    ])
-  })
-
   it("should query all products", async () => {
     const { data } = await module.query({
       fields: ["product.*", "product.variants.*", "product.variants.prices.*"],

packages/modules/index/src/migrations/Migration20250515161913.ts

@@ -0,0 +1,59 @@
+import { Migration } from "@mikro-orm/migrations"
+
+export class Migration20250515161913 extends Migration {
+  override async up(): Promise<void> {
+    this.addSql(`
+        DO $$
+        DECLARE
+            r RECORD;
+            protected_tables TEXT[] := ARRAY[
+                'cat_linkproductsaleschannel',
+                'cat_linkproductvariantpriceset',
+                'cat_price',
+                'cat_priceset',
+                'cat_product',
+                'cat_productvariant',
+                'cat_saleschannel',
+                'cat_pivot_linkproductsaleschannelsaleschannel',
+                'cat_pivot_linkproductvariantpricesetpriceset',
+                'cat_pivot_pricesetprice',
+                'cat_pivot_productlinkproductsaleschannel',
+                'cat_pivot_productproductvariant',
+                'cat_pivot_productvariantlinkproductvariantpriceset'
+            ];
+        BEGIN
+            FOR r IN
+                SELECT tablename
+                FROM pg_tables
+                WHERE schemaname = 'public' AND tablename LIKE 'cat\_%'
+            LOOP
+                IF r.tablename <> ALL (protected_tables) THEN
+                    EXECUTE format('DROP TABLE IF EXISTS public.%I CASCADE;', r.tablename);
+                END IF;
+            END LOOP;
+        END $$;
+
+        UPDATE index_sync SET last_key = NULL WHERE entity NOT IN (
+          'Product',
+          'ProductVariant',
+          'LinkProductVariantPriceSet',
+          'LinkProductSalesChannel',
+          'Price',
+          'PriceSet',
+          'SalesChannel'
+        );
+
+        UPDATE index_metadata SET status = 'pending' WHERE entity NOT IN (
+          'Product',
+          'ProductVariant',
+          'LinkProductVariantPriceSet',
+          'LinkProductSalesChannel',
+          'Price',
+          'PriceSet',
+        'SalesChannel'
+        );
+    `)
+  }
+
+  override async down(): Promise<void> {}
+}

packages/modules/index/src/services/postgres-provider.ts

@@ -237,12 +237,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
   ): Promise<IndexTypes.QueryResultSet<TEntry>> {
     await this.#isReady_
 
-    const {
+    const { fields = [], filters = {}, joinFilters = {} } = config
-      keepFilteredEntities,
-      fields = [],
-      filters = {},
-      joinFilters = {},
-    } = config
     const { take, skip, order: inputOrderBy = {} } = config.pagination ?? {}
 
     const select = normalizeFieldsSelection(fields)
@@ -281,7 +276,6 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
       options: {
         skip,
         take,
-        keepFilteredEntities,
         orderBy,
       },
       rawConfig: config,
@@ -290,7 +284,6 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
 
     const { sql, sqlCount } = qb.buildQuery({
       hasPagination,
-      returnIdOnly: !!keepFilteredEntities,
       hasCount,
     })
 
@@ -310,30 +303,6 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
           } as IndexTypes.QueryFunctionReturnPagination)
         : undefined
 
-    if (keepFilteredEntities) {
-      const mainEntity = Object.keys(select)[0]
-
-      const ids = resultSet.map((r) => r[`${mainEntity}.id`])
-      if (ids.length) {
-        const result = await this.query<TEntry>(
-          {
-            fields,
-            joinFilters,
-            filters: {
-              [mainEntity]: {
-                id: ids,
-              },
-            },
-            pagination: undefined,
-            keepFilteredEntities: false,
-          } as IndexTypes.IndexQueryConfig<TEntry>,
-          sharedContext
-        )
-        result.metadata ??= resultMetadata
-        return result
-      }
-    }
-
     return {
       data: qb.buildObjectFromResultset(
         resultSet

packages/modules/index/src/types/index.ts

@@ -29,5 +29,4 @@ export type QueryOptions = {
   skip?: number
   take?: number
   orderBy?: OrderBy | OrderBy[]
-  keepFilteredEntities?: boolean
 }

packages/modules/index/src/utils/build-config.ts

@@ -1180,7 +1180,9 @@ function buildSchemaFromFilterableLinks(
       return
     }
 
-    const fieldType = fieldRef.type.toString()
+    const isEnum =
+      fieldRef.type?.astNode?.kind === GraphQLUtils.Kind.ENUM_TYPE_DEFINITION
+    const fieldType = isEnum ? "String" : fieldRef.type.toString()
     const isArray = fieldType.startsWith("[")
     const currentType = fieldType.replace(/\[|\]|\!/g, "")
 

packages/modules/index/src/utils/create-partitions.ts

@@ -1,6 +1,7 @@
 import { IndexTypes } from "@medusajs/framework/types"
 import { SqlEntityManager } from "@mikro-orm/postgresql"
 import { schemaObjectRepresentationPropertiesToOmit } from "@types"
+import { getPivotTableName, normalizeTableName } from "./normalze-table-name"
 
 export async function createPartitions(
   schemaObjectRepresentation: IndexTypes.SchemaObjectRepresentation,
@@ -18,7 +19,7 @@ export async function createPartitions(
         schemaObjectRepresentation[key].listeners.length > 0
     )
     .map((key) => {
-      const cName = key.toLowerCase()
+      const cName = normalizeTableName(key)
 
       if (createdPartitions.has(cName)) {
         return []
@@ -35,7 +36,8 @@ export async function createPartitions(
           continue
         }
 
-        const pName = `cat_pivot_${parent.ref.entity}${key}`.toLowerCase()
+        const pName = getPivotTableName(`${parent.ref.entity}${key}`)
+
         if (createdPartitions.has(pName)) {
           continue
         }
@@ -63,7 +65,7 @@ export async function createPartitions(
         schemaObjectRepresentation[key].listeners.length > 0
     )
     .map((key) => {
-      const cName = key.toLowerCase()
+      const cName = normalizeTableName(key)
       const part: string[] = []
 
       part.push(
@@ -79,7 +81,8 @@ export async function createPartitions(
           continue
         }
 
-        const pName = `cat_pivot_${parent.ref.entity}${key}`.toLowerCase()
+        const pName = getPivotTableName(`${parent.ref.entity}${key}`)
+
         part.push(
           `CREATE INDEX CONCURRENTLY IF NOT EXISTS "IDX_${pName}_child_id" ON ${activeSchema}${pName} ("child_id")`
         )

packages/modules/index/src/utils/index.ts

@@ -6,3 +6,4 @@ export * from "./sync/configuration"
 export * from "./index-metadata-status"
 export * from "./gql-to-types"
 export * from "./default-schema"
+export * from "./normalze-table-name"

packages/modules/index/src/utils/normalze-table-name.ts

@@ -0,0 +1,10 @@
+import { compressName } from "@medusajs/framework/utils"
+
+export function normalizeTableName(name: string): string {
+  return compressName(name.toLowerCase(), 58).replace(/[^a-z0-9_]/g, "_")
+}
+
+export function getPivotTableName(tableName: string) {
+  const compressedName = normalizeTableName(tableName)
+  return `cat_pivot_${compressedName}`
+}

packages/modules/index/src/utils/query-builder.ts

@@ -7,6 +7,7 @@ import {
 } from "@medusajs/framework/utils"
 import { Knex } from "@mikro-orm/knex"
 import { OrderBy, QueryFormat, QueryOptions, Select } from "@types"
+import { getPivotTableName, normalizeTableName } from "./normalze-table-name"
 
 function escapeJsonPathString(val: string): string {
   // Escape for JSONPath string
@@ -23,6 +24,10 @@ function buildSafeJsonPathQuery(
     jsonPathOperator = "=="
   } else if (operator.toUpperCase().includes("LIKE")) {
     jsonPathOperator = "like_regex"
+  } else if (operator === "IS") {
+    jsonPathOperator = "=="
+  } else if (operator === "IS NOT") {
+    jsonPathOperator = "!="
   }
 
   if (typeof value === "string") {
@@ -32,6 +37,10 @@ function buildSafeJsonPathQuery(
       val = val.replace(/%/g, ".*").replace(/_/g, ".")
     }
     value = `"${escapeJsonPathString(val)}"`
+  } else {
+    if ((operator === "IS" || operator === "IS NOT") && value === null) {
+      value = "null"
+    }
   }
 
   return `$.${field} ${jsonPathOperator} ${value}`
@@ -530,14 +539,14 @@ export class QueryBuilder {
       aliasMapping[currentAliasPath] = alias
 
       if (level > 0) {
-        const cName = entity.ref.entity.toLowerCase()
+        const cName = normalizeTableName(entity.ref.entity)
 
         let joinTable = `cat_${cName} AS ${alias}`
 
         if (entity.isInverse || parEntity.isInverse) {
           const pName =
             `${entity.ref.entity}${parEntity.ref.entity}`.toLowerCase()
-          const pivotTable = `cat_pivot_${pName}`
+          const pivotTable = getPivotTableName(pName)
 
           joinBuilder.leftJoin(
             `${pivotTable} AS ${alias}_ref`,
@@ -552,7 +561,7 @@ export class QueryBuilder {
         } else {
           const pName =
             `${parEntity.ref.entity}${entity.ref.entity}`.toLowerCase()
-          const pivotTable = `cat_pivot_${pName}`
+          const pivotTable = getPivotTableName(pName)
 
           joinBuilder.leftJoin(
             `${pivotTable} AS ${alias}_ref`,
@@ -704,11 +713,9 @@ export class QueryBuilder {
   public buildQuery({
     hasPagination = true,
     hasCount = false,
-    returnIdOnly = false,
   }: {
     hasPagination?: boolean
     hasCount?: boolean
-    returnIdOnly?: boolean
   }): { sql: string; sqlCount?: string } {
     const selectOnlyStructure = this.selector.select
     const structure = this.requestedFields
@@ -816,7 +823,10 @@ export class QueryBuilder {
     }
 
     innerQueryBuilder.from(
-      `cat_${rootEntity} AS ${this.getShortAlias(aliasMapping, rootKey)}`
+      `cat_${normalizeTableName(rootEntity)} AS ${this.getShortAlias(
+        aliasMapping,
+        rootKey
+      )}`
     )
 
     joinParts.forEach((joinPart) => {
@@ -887,7 +897,10 @@ export class QueryBuilder {
     const innerQueryAlias = "paginated_ids"
 
     outerQueryBuilder.from(
-      `cat_${rootEntity} AS ${this.getShortAlias(aliasMapping, rootKey)}`
+      `cat_${normalizeTableName(rootEntity)} AS ${this.getShortAlias(
+        aliasMapping,
+        rootKey
+      )}`
     )
 
     outerQueryBuilder.joinRaw(
@@ -909,13 +922,11 @@ export class QueryBuilder {
       outerQueryBuilder.joinRaw(joinPart)
     })
 
-    const finalSelectParts = !returnIdOnly
+    const finalSelectParts = this.buildSelectParts(
-      ? this.buildSelectParts(
+      selectOnlyStructure[rootKey] as Select,
-          selectOnlyStructure[rootKey] as Select,
+      rootKey,
-          rootKey,
+      aliasMapping
-          aliasMapping
+    )
-        )
-      : { [`${rootKey}.id`]: `${rootAlias}.id` }
 
     outerQueryBuilder.select(finalSelectParts)
 

packages/modules/link-modules/src/utils/generate-entity.ts

@@ -4,13 +4,13 @@ import {
 } from "@medusajs/framework/types"
 import {
   composeTableName,
+  compressName,
   mikroOrmSoftDeletableFilterOptions,
   simpleHash,
   SoftDeletableFilterKey,
 } from "@medusajs/framework/utils"
 
 import { EntitySchema } from "@mikro-orm/core"
-import { compressName } from "./compress-name"
 
 function getClass(...properties) {
   return class LinkModel {