feat(index): $nin and $not operators (#13289)

* feat(index): add $not and $nin operators

* logical operator

* test

* test

* types

* logical

* schema ID

* types and $ilike fix

* index type

.changeset/odd-emus-cheat.md

@@ -0,0 +1,6 @@
+---
+"@medusajs/index": patch
+"@medusajs/types": patch
+---
+
+feat(index): $nin and $not operators

integration-tests/modules/__tests__/index/sync.spec.ts

@@ -160,14 +160,14 @@ medusaIntegrationTestRunner({
         ...(indexEngine as any).moduleOptions_,
         schema: `
   type Product @Listeners(values: ["product.created", "product.updated", "product.deleted"]) {
-    id: String
+    id: ID
     title: String
     handle: String
     variants: [ProductVariant]
   }
   
   type ProductVariant @Listeners(values: ["variant.created", "variant.updated", "variant.deleted"]) {
-    id: String
+    id: ID
     product_id: String
     sku: String
     description: String

packages/core/types/src/index-data/index-operator-map.ts

@@ -5,8 +5,12 @@ export type IndexOperatorMap<T> = {
   $gt?: T
   $gte?: T
   $ne?: T
-  $in?: T
+  $in?: T[]
+  $nin?: T[]
   $is?: T
   $like?: T
   $ilike?: T
+  $and?: T[]
+  $or?: T[]
+  $not?: T | T[]
 }

packages/modules/index/integration-tests/__fixtures__/schema.ts

@@ -1,6 +1,6 @@
 export const schema = `
   type Product @Listeners(values: ["product.created", "product.updated", "product.deleted"]) {
-    id: String
+    id: ID
     title: String
     created_at: DateTime
 
@@ -18,7 +18,7 @@ export const schema = `
   }
   
   type ProductVariant @Listeners(values: ["variant.created", "variant.updated", "variant.deleted"]) {
-    id: String
+    id: ID
     product_id: String
     sku: String
     prices: [Price]

packages/modules/index/integration-tests/__fixtures__/update-removed-schema.ts

@@ -1,13 +1,13 @@
 export const updateRemovedSchema = `
   type Product @Listeners(values: ["product.created", "product.updated", "product.deleted"]) {
-    id: String
+    id: ID
     title: String
     handle: String
     variants: [ProductVariant]
   }
   
   type ProductVariant @Listeners(values: ["variant.created", "variant.updated", "variant.deleted"]) {
-    id: String
+    id: ID
     product_id: String
     sku: String
     description: String

packages/modules/index/integration-tests/__fixtures__/updated-schema.ts

@@ -1,6 +1,6 @@
 export const updatedSchema = `
   type Product @Listeners(values: ["product.created", "product.updated", "product.deleted"]) {
-    id: String
+    id: ID
     title: String
     handle: String
     deep: InternalNested
@@ -17,7 +17,7 @@ export const updatedSchema = `
   }
   
   type ProductVariant @Listeners(values: ["variant.created", "variant.updated", "variant.deleted"]) {
-    id: String
+    id: ID
     product_id: String
     sku: String
     prices: [Price]

packages/modules/index/integration-tests/__tests__/query-builder.spec.ts

@@ -135,6 +135,7 @@ describe("IndexModuleService query", function () {
           name: "Product",
           data: {
             id: "prod_1",
+            title: "Product 1",
           },
         },
         {
@@ -296,7 +297,24 @@ describe("IndexModuleService query", function () {
       },
     })
 
+    const { data: dataNot } = await module.query({
+      fields: ["product.*", "product.variants.*", "product.variants.prices.*"],
+      filters: {
+        product: {
+          variants: {
+            sku: {
+              $not: {
+                $eq: null,
+              },
+            },
+          },
+        },
+      },
+    })
+
     expect(data.length).toEqual(1)
+    expect(dataNot.length).toEqual(1)
+    expect(dataNot).toEqual(data)
 
     const { data: data2 } = await module.query({
       fields: ["product.*", "product.variants.*", "product.variants.prices.*"],
@@ -340,6 +358,7 @@ describe("IndexModuleService query", function () {
       },
       {
         id: "prod_1",
+        title: "Product 1",
         variants: [
           {
             id: "var_2",
@@ -399,6 +418,7 @@ describe("IndexModuleService query", function () {
       },
       {
         id: "prod_1",
+        title: "Product 1",
         variants: [
           {
             id: "var_2",
@@ -456,6 +476,7 @@ describe("IndexModuleService query", function () {
       },
       {
         id: "prod_1",
+        title: "Product 1",
         variants: [
           {
             id: "var_1",
@@ -499,6 +520,7 @@ describe("IndexModuleService query", function () {
     expect(dataAsc).toEqual([
       {
         id: "prod_1",
+        title: "Product 1",
         variants: [
           {
             id: "var_2",
@@ -597,6 +619,7 @@ describe("IndexModuleService query", function () {
     expect(data).toEqual([
       {
         id: "prod_1",
+        title: "Product 1",
         variants: [
           {
             id: "var_1",
@@ -646,6 +669,7 @@ describe("IndexModuleService query", function () {
     expect(data).toEqual([
       {
         id: "prod_1",
+        title: "Product 1",
         variants: [
           {
             id: "var_1",
@@ -782,6 +806,7 @@ describe("IndexModuleService query", function () {
     expect(data).toEqual([
       {
         id: "prod_1",
+        title: "Product 1",
         variants: [
           {
             id: "var_1",
@@ -922,6 +947,7 @@ describe("IndexModuleService query", function () {
     expect(data).toEqual([
       {
         id: "prod_1",
+        title: "Product 1",
         variants: [
           {
             id: "var_1",
@@ -937,4 +963,165 @@ describe("IndexModuleService query", function () {
       },
     ])
   })
+
+  it("should query products filtering product not in [X]", async () => {
+    const expected = [
+      {
+        id: "prod_2",
+        title: "Product 2 title",
+        deep: {
+          a: 1,
+          obj: {
+            b: 15,
+          },
+        },
+      },
+    ]
+
+    const { data } = await module.query({
+      fields: ["product.*"],
+      filters: {
+        product: {
+          $not: [
+            {
+              id: {
+                $in: ["prod_1"],
+              },
+            },
+          ],
+        },
+      },
+    })
+    expect(data).toEqual(expected)
+  })
+
+  it("should query products filtering product not in [X] using $nin", async () => {
+    const expected = [
+      {
+        id: "prod_2",
+        title: "Product 2 title",
+        deep: {
+          a: 1,
+          obj: {
+            b: 15,
+          },
+        },
+      },
+    ]
+
+    const { data } = await module.query({
+      fields: ["product.*"],
+      filters: {
+        product: {
+          id: {
+            $nin: ["prod_1"],
+          },
+        },
+      },
+    })
+    expect(data).toEqual(expected)
+  })
+
+  it("should query products with variants.sku not in [X] and title eq", async () => {
+    const expected = [
+      {
+        id: "prod_2",
+        title: "Product 2 title",
+        deep: {
+          a: 1,
+          obj: {
+            b: 15,
+          },
+        },
+      },
+    ]
+
+    const { data } = await module.query({
+      fields: ["product.*", "variants.*"],
+      filters: {
+        product: {
+          variants: {
+            sku: {
+              $nin: ["sku 123"],
+            },
+          },
+          title: {
+            $eq: "Product 2 title",
+          },
+        },
+      },
+    })
+    expect(data).toEqual(expected)
+  })
+
+  it("should query products filtering title like and not equal specific value", async () => {
+    const expected = [
+      {
+        id: "prod_2",
+        title: "Product 2 title",
+        deep: {
+          a: 1,
+          obj: {
+            b: 15,
+          },
+        },
+      },
+    ]
+
+    const { data } = await module.query({
+      fields: ["product.*"],
+      filters: {
+        product: {
+          $and: [
+            {
+              title: {
+                $like: "Product%",
+              },
+            },
+            {
+              $not: {
+                title: {
+                  $eq: "Product 1",
+                },
+              },
+            },
+          ],
+        },
+      },
+    })
+    expect(data).toEqual(expected)
+  })
+
+  it("should query products filtering title using $ilike", async () => {
+    const expected = [
+      {
+        id: "prod_2",
+        title: "Product 2 title",
+      },
+    ]
+
+    const { data } = await module.query({
+      fields: ["product.id", "product.title"],
+      filters: {
+        product: {
+          title: {
+            $ilike: "PROdUCt 2%",
+          },
+        },
+      },
+    })
+    expect(data).toEqual(expected)
+
+    const { data: sensitive } = await module.query({
+      fields: ["product.id", "product.title"],
+      filters: {
+        product: {
+          title: {
+            $like: "PROdUCt 2%",
+          },
+        },
+      },
+    })
+    expect(sensitive).toEqual([])
+  })
 })

packages/modules/index/src/utils/query-builder.ts

@@ -11,6 +11,7 @@ import { getPivotTableName, normalizeTableName } from "./normalze-table-name"
 
 const AND_OPERATOR = "$and"
 const OR_OPERATOR = "$or"
+const NOT_OPERATOR = "$not"
 
 function escapeJsonPathString(val: string): string {
   // Escape for JSONPath string
@@ -23,10 +24,15 @@ function buildSafeJsonPathQuery(
   value: any
 ): string {
   let jsonPathOperator = operator
+  let caseInsensitiveFlag = ""
   if (operator === "=") {
     jsonPathOperator = "=="
   } else if (operator.toUpperCase().includes("LIKE")) {
     jsonPathOperator = "like_regex"
+
+    if (operator.toUpperCase() === "ILIKE") {
+      caseInsensitiveFlag = ' flag "i"'
+    }
   } else if (operator === "IS") {
     jsonPathOperator = "=="
   } else if (operator === "IS NOT") {
@@ -46,7 +52,7 @@ function buildSafeJsonPathQuery(
     }
   }
 
-  return `$.${field} ${jsonPathOperator} ${value}`
+  return `$.${field} ${jsonPathOperator} ${value}${caseInsensitiveFlag}`
 }
 
 export const OPERATOR_MAP = {
@@ -57,6 +63,7 @@ export const OPERATOR_MAP = {
   $gte: ">=",
   $ne: "!=",
   $in: "IN",
+  $nin: "NOT IN",
   $is: "IS",
   $like: "LIKE",
   $ilike: "ILIKE",
@@ -104,6 +111,10 @@ export class QueryBuilder {
     this.idsOnly = args.idsOnly ?? false
   }
 
+  private isLogicalOperator(key: string) {
+    return key === AND_OPERATOR || key === OR_OPERATOR || key === NOT_OPERATOR
+  }
+
   private getStructureKeys(structure) {
     const collectKeys = (obj: any, keys = new Set<string>()) => {
       if (!isObject(obj)) {
@@ -111,7 +122,7 @@ export class QueryBuilder {
       }
 
       Object.keys(obj).forEach((key) => {
-        if (key === AND_OPERATOR || key === OR_OPERATOR) {
+        if (this.isLogicalOperator(key)) {
           if (Array.isArray(obj[key])) {
             obj[key].forEach((item) => collectKeys(item, keys))
           }
@@ -144,7 +155,7 @@ export class QueryBuilder {
   }
 
   private getGraphQLType(path, field) {
-    if (field === AND_OPERATOR || field === OR_OPERATOR) {
+    if (this.isLogicalOperator(field)) {
       return "JSON"
     }
 
@@ -269,12 +280,11 @@ export class QueryBuilder {
 
     keys.forEach((key) => {
       const pathAsArray = (parentPath ? `${parentPath}.${key}` : key).split(".")
-      const fieldOrLogicalOperator = pathAsArray.pop()
+      const fieldOrLogicalOperator = pathAsArray.pop()!
       let value = obj[key]
 
       if (
-        (fieldOrLogicalOperator === AND_OPERATOR ||
+        this.isLogicalOperator(fieldOrLogicalOperator) &&
-          fieldOrLogicalOperator === OR_OPERATOR) &&
         !Array.isArray(value)
       ) {
         value = [value]
@@ -309,11 +319,35 @@ export class QueryBuilder {
             )
           })
         })
+      } else if (
+        fieldOrLogicalOperator === NOT_OPERATOR &&
+        (Array.isArray(value) || isObject(value))
+      ) {
+        builder.whereNot((qb) => {
+          if (Array.isArray(value)) {
+            value.forEach((cond) => {
+              qb.andWhere((subBuilder) =>
+                this.parseWhere(
+                  aliasMapping,
+                  cond,
+                  subBuilder,
+                  pathAsArray.join(".")
+                )
+              )
+            })
+          } else {
+            this.parseWhere(
+              aliasMapping,
+              value as any,
+              qb,
+              pathAsArray.join(".")
+            )
+          }
+        })
       } else if (
         isObject(value) &&
         !Array.isArray(value) &&
-        fieldOrLogicalOperator !== AND_OPERATOR &&
+        !this.isLogicalOperator(fieldOrLogicalOperator)
-        fieldOrLogicalOperator !== OR_OPERATOR
       ) {
         const currentPath = parentPath ? `${parentPath}.${key}` : key
 
@@ -335,7 +369,10 @@ export class QueryBuilder {
                 value[subKey]
               )
 
-              let val = operator === "IN" ? subValue : [subValue]
+              let val =
+                operator === "IN" || operator === "NOT IN"
+                  ? subValue
+                  : [subValue]
               if (operator === "=" && subValue === null) {
                 operator = "IS"
               } else if (operator === "!=" && subValue === null) {
@@ -355,7 +392,7 @@ export class QueryBuilder {
                     )}'::jsonb`
                   )
                 }
-              } else if (operator === "IN") {
+              } else if (operator === "IN" || operator === "NOT IN") {
                 if (val && !Array.isArray(val)) {
                   val = [val]
                 }
@@ -365,9 +402,12 @@ export class QueryBuilder {
 
                 const inPlaceholders = val.map(() => "?").join(",")
                 const hasId = field[field.length - 1] === "id"
+                const isNegated = operator === "NOT IN"
                 if (hasId) {
                   builder.whereRaw(
-                    `${aliasMapping[attr]}.id IN (${inPlaceholders})`,
+                    `${aliasMapping[attr]}.id ${
+                      isNegated ? "NOT IN" : "IN"
+                    } (${inPlaceholders})`,
                     val
                   )
                 } else {
@@ -379,11 +419,18 @@ export class QueryBuilder {
                     })
                   )
 
+                  if (isNegated) {
+                    builder.whereRaw(
+                      `NOT EXISTS (SELECT 1 FROM unnest(ARRAY[${inPlaceholders}]::JSONB[]) AS v(val) WHERE ${aliasMapping[attr]}.data${nested} @> v.val)`,
+                      jsonbValues
+                    )
+                  } else {
                     builder.whereRaw(
                       `${aliasMapping[attr]}.data${nested} @> ANY(ARRAY[${inPlaceholders}]::JSONB[])`,
                       jsonbValues
                     )
                   }
+                }
               } else {
                 const potentialIdFields = field[field.length - 1]
                 const hasId = potentialIdFields === "id"