feat(utils,types): DML can apply composite indexes (#7842)

**what:**

- DML can apply composite indexes
- Where clause is currently a string, QB version will come as a follow up

```
model.define("user", {
  email: model.text(),
  account: model.text(),
}).indexes([
  {
    name: "IDX-unique-name",
    unique: true,
    on: ["email", "account"],
    where: "email is NOT NULL",
  },
])
```

RESOLVES CORE-2391

packages/core/types/src/dml/index.ts

@@ -1,8 +1,11 @@
 export const IsDmlEntity = Symbol.for("isDmlEntity")
 
-export interface IDmlEntity<
-  Schema extends Record<string, PropertyType<any> | RelationshipType<any>>
-> {
+export type DMLSchema = Record<
+  string,
+  PropertyType<any> | RelationshipType<any>
+>
+
+export interface IDmlEntity<Schema extends DMLSchema> {
   [IsDmlEntity]: true
   schema: Schema
 }
@@ -163,3 +166,23 @@ export type InferTypeOf<T extends IDmlEntity<any>> = InstanceType<Infer<T>>
 export type InferEntityType<T extends any> = T extends IDmlEntity<any>
   ? InferTypeOf<T>
   : T
+
+/**
+ * Infer all indexable properties from a DML entity including inferred foreign keys and excluding relationship
+ */
+export type InferIndexableProperties<T> = keyof (T extends IDmlEntity<
+  infer Schema
+>
+  ? {
+      [K in keyof Schema as Schema[K] extends RelationshipType<any>
+        ? never
+        : K]: string
+    } & InferForeignKeys<T>
+  : never)
+
+export type EntityIndex<TSchema extends DMLSchema = DMLSchema> = {
+  name?: string
+  unique?: boolean
+  on: InferIndexableProperties<IDmlEntity<TSchema>>[]
+  where?: string
+}

packages/core/utils/src/dml/__tests__/entity-builder.spec.ts

@@ -14,6 +14,41 @@ describe("Entity builder", () => {
     MetadataStorage.clear()
   })
 
+  const defaultColumnMetadata = {
+    created_at: {
+      columnType: "timestamptz",
+      defaultRaw: "now()",
+      getter: false,
+      name: "created_at",
+      nullable: false,
+      onCreate: expect.any(Function),
+      reference: "scalar",
+      setter: false,
+      type: "date",
+    },
+    deleted_at: {
+      columnType: "timestamptz",
+      getter: false,
+      name: "deleted_at",
+      nullable: true,
+      reference: "scalar",
+      setter: false,
+      type: "date",
+    },
+    updated_at: {
+      columnType: "timestamptz",
+      defaultRaw: "now()",
+      getter: false,
+      name: "updated_at",
+      nullable: false,
+      onCreate: expect.any(Function),
+      onUpdate: expect.any(Function),
+      reference: "scalar",
+      setter: false,
+      type: "date",
+    },
+  }
+
   describe("Entity builder | properties", () => {
     test("should identify a DML entity correctly", () => {
       const user = model.define("user", {
@@ -2484,6 +2519,145 @@ describe("Entity builder", () => {
     })
   })
 
+  describe("Entity builder | indexes", () => {
+    test("should define indexes for an entity", () => {
+      const group = model.define("group", {
+        id: model.number(),
+        name: model.text(),
+        users: model.hasMany(() => user),
+      })
+
+      const user = model
+        .define("user", {
+          email: model.text(),
+          account: model.text(),
+          organization: model.text(),
+          group: model.belongsTo(() => group, { mappedBy: "users" }),
+        })
+        .indexes([
+          {
+            unique: true,
+            on: ["email", "account"],
+          },
+          { on: ["email", "account"] },
+          {
+            on: ["organization", "account"],
+            where: "email IS NOT NULL",
+          },
+          {
+            name: "IDX_unique-name",
+            unique: true,
+            on: ["organization", "account", "group_id"],
+          },
+        ])
+
+      const User = toMikroORMEntity(user)
+      const metaData = MetadataStorage.getMetadataFromDecorator(User)
+
+      expect(metaData.properties).toEqual({
+        email: {
+          reference: "scalar",
+          type: "string",
+          columnType: "text",
+          name: "email",
+          nullable: false,
+          getter: false,
+          setter: false,
+        },
+        account: {
+          reference: "scalar",
+          type: "string",
+          columnType: "text",
+          name: "account",
+          nullable: false,
+          getter: false,
+          setter: false,
+        },
+        organization: {
+          columnType: "text",
+          getter: false,
+          name: "organization",
+          nullable: false,
+          reference: "scalar",
+          setter: false,
+          type: "string",
+        },
+        group: {
+          entity: "Group",
+          name: "group",
+          nullable: false,
+          persist: false,
+          reference: "m:1",
+        },
+        group_id: {
+          columnType: "text",
+          entity: "Group",
+          fieldName: "group_id",
+          mapToPk: true,
+          name: "group_id",
+          nullable: false,
+          onDelete: undefined,
+          reference: "m:1",
+        },
+        ...defaultColumnMetadata,
+      })
+
+      expect(metaData.indexes).toEqual([
+        {
+          expression:
+            'CREATE UNIQUE INDEX IF NOT EXISTS "IDX_user_email_account_unique" ON "user" (email, account) WHERE deleted_at IS NULL',
+          name: "IDX_user_email_account_unique",
+        },
+        {
+          expression:
+            'CREATE INDEX IF NOT EXISTS "IDX_user_email_account" ON "user" (email, account) WHERE deleted_at IS NULL',
+          name: "IDX_user_email_account",
+        },
+        {
+          expression:
+            'CREATE INDEX IF NOT EXISTS "IDX_user_organization_account" ON "user" (organization, account) WHERE email IS NOT NULL AND deleted_at IS NULL',
+          name: "IDX_user_organization_account",
+        },
+        {
+          expression:
+            'CREATE UNIQUE INDEX IF NOT EXISTS "IDX_unique-name" ON "user" (organization, account, group_id) WHERE deleted_at IS NULL',
+          name: "IDX_unique-name",
+        },
+      ])
+    })
+
+    test("should throw an error if field is unknown for an index", () => {
+      try {
+        const group = model.define("group", {
+          id: model.number(),
+          name: model.text(),
+          users: model.hasMany(() => user),
+        })
+
+        const user = model
+          .define("user", {
+            email: model.text(),
+            account: model.text(),
+            organization: model.text(),
+            group: model.belongsTo(() => group, { mappedBy: "users" }),
+          })
+          .indexes([
+            {
+              on: ["email", "account", "doesnotexist", "anotherdoesnotexist"],
+            },
+          ])
+
+        toMikroORMEntity(user)
+
+        throw "should not reach"
+      } catch (e) {
+        expect(e.message).toEqual(
+          "Fields (doesnotexist, anotherdoesnotexist) are not found when applying indexes from DML entity"
+        )
+      }
+    })
+  })
+
   describe("Entity builder | hasMany", () => {
     test("define hasMany relationship", () => {
       const email = model.define("email", {

packages/core/utils/src/dml/entity-builder.ts

@@ -1,12 +1,9 @@
-import type {
-  PropertyType,
-  RelationshipOptions,
-  RelationshipType,
-} from "@medusajs/types"
+import type { DMLSchema, RelationshipOptions } from "@medusajs/types"
 import { DmlEntity } from "./entity"
 import { createBigNumberProperties } from "./helpers/entity-builder/create-big-number-properties"
 import { createDefaultProperties } from "./helpers/entity-builder/create-default-properties"
 import { inferPrimaryKeyProperties } from "./helpers/entity-builder/infer-primary-key-properties"
+import { ArrayProperty } from "./properties/array"
 import { BigNumberProperty } from "./properties/big-number"
 import { BooleanProperty } from "./properties/boolean"
 import { DateTimeProperty } from "./properties/date-time"
@@ -19,18 +16,12 @@ import { BelongsTo } from "./relations/belongs-to"
 import { HasMany } from "./relations/has-many"
 import { HasOne } from "./relations/has-one"
 import { ManyToMany } from "./relations/many-to-many"
-import { ArrayProperty } from "./properties/array"
 
 /**
  * The implicit properties added by EntityBuilder in every schema
  */
 const IMPLICIT_PROPERTIES = ["created_at", "updated_at", "deleted_at"]
 
-export type DMLSchema = Record<
-  string,
-  PropertyType<any> | RelationshipType<any>
->
-
 type DefineOptions = string | { name?: string; tableName: string }
 
 /**

packages/core/utils/src/dml/entity.ts

@@ -1,14 +1,14 @@
 import {
+  DMLSchema,
   EntityCascades,
+  EntityIndex,
   ExtractEntityRelations,
   IDmlEntity,
   IsDmlEntity,
-  PropertyType,
-  RelationshipType,
 } from "@medusajs/types"
-import { DMLSchema } from "./entity-builder"
-import { BelongsTo } from "./relations/belongs-to"
 import { isObject, isString, toCamelCase } from "../common"
+import { transformIndexWhere } from "./helpers/entity-builder/build-indexes"
+import { BelongsTo } from "./relations/belongs-to"
 
 type Config = string | { name?: string; tableName: string }
 
@@ -54,6 +54,7 @@ export class DmlEntity<Schema extends DMLSchema> implements IDmlEntity<Schema> {
 
   readonly #tableName: string
   #cascades: EntityCascades<string[]> = {}
+  #indexes: EntityIndex<Schema>[] = []
 
   constructor(nameOrConfig: Config, public schema: Schema) {
     const { name, tableName } = extractNameAndTableName(nameOrConfig)
@@ -78,16 +79,16 @@ export class DmlEntity<Schema extends DMLSchema> implements IDmlEntity<Schema> {
   parse(): {
     name: string
     tableName: string
-    schema: PropertyType<any> | RelationshipType<any>
+    schema: DMLSchema
     cascades: EntityCascades<string[]>
+    indexes: EntityIndex<Schema>[]
   } {
     return {
       name: this.name,
       tableName: this.#tableName,
-      schema: this.schema as unknown as
-        | PropertyType<any>
-        | RelationshipType<any>,
+      schema: this.schema,
       cascades: this.#cascades,
+      indexes: this.#indexes,
     }
   }
 
@@ -119,4 +120,14 @@ export class DmlEntity<Schema extends DMLSchema> implements IDmlEntity<Schema> {
     this.#cascades = options
     return this
   }
+
+  indexes(indexes: EntityIndex<Schema>[]) {
+    for (const index of indexes) {
+      index.where = transformIndexWhere(index)
+      index.unique = index.unique ?? false
+    }
+
+    this.#indexes = indexes
+    return this
+  }
 }

packages/core/utils/src/dml/helpers/create-mikro-orm-entity.ts

@@ -2,9 +2,12 @@ import type { EntityConstructor, Infer } from "@medusajs/types"
 import { Entity, Filter } from "@mikro-orm/core"
 import { mikroOrmSoftDeletableFilterOptions } from "../../dal"
 import { DmlEntity } from "../entity"
-import { defineProperty } from "./entity-builder/define-property"
-import { applyIndexes } from "./entity-builder/apply-indexes"
+import {
+  applyEntityIndexes,
+  applyIndexes,
+} from "./entity-builder/apply-indexes"
 import { applySearchable } from "./entity-builder/apply-searchable"
+import { defineProperty } from "./entity-builder/define-property"
 import { defineRelationship } from "./entity-builder/define-relationship"
 import { parseEntityName } from "./entity-builder/parse-entity-name"
 
@@ -37,7 +40,7 @@ export function createMikrORMEntity() {
   return function createEntity<T extends DmlEntity<any>>(entity: T): Infer<T> {
     class MikroORMEntity {}
 
-    const { schema, cascades } = entity.parse()
+    const { schema, cascades, indexes: entityIndexes = [] } = entity.parse()
     const { modelName, tableName } = parseEntityName(entity)
 
     /**
@@ -69,6 +72,10 @@ export function createMikrORMEntity() {
       }
     })
 
+    if (entityIndexes?.length) {
+      applyEntityIndexes(MikroORMEntity, tableName, entityIndexes)
+    }
+
     /**
      * Converting class to a MikroORM entity
      */

packages/core/utils/src/dml/helpers/entity-builder/apply-indexes.ts

@@ -1,5 +1,10 @@
-import { EntityConstructor, PropertyMetadata } from "@medusajs/types"
+import {
+  EntityConstructor,
+  EntityIndex,
+  PropertyMetadata,
+} from "@medusajs/types"
 import { createPsqlIndexStatementHelper } from "../../../common"
+import { validateIndexFields } from "../mikro-orm/build-indexes"
 
 /**
  * Prepares indexes for a given field
@@ -20,3 +25,26 @@ export function applyIndexes(
     providerEntityIdIndexStatement.MikroORMIndex()(MikroORMEntity)
   })
 }
+
+/**
+ * Prepares indexes for a given field
+ */
+export function applyEntityIndexes(
+  MikroORMEntity: EntityConstructor<any>,
+  tableName: string,
+  indexes: EntityIndex[]
+) {
+  indexes.forEach((index) => {
+    validateIndexFields(MikroORMEntity, index)
+
+    const providerEntityIdIndexStatement = createPsqlIndexStatementHelper({
+      tableName,
+      name: index.name,
+      columns: index.on as string[],
+      unique: index.unique,
+      where: index.where,
+    })
+
+    providerEntityIdIndexStatement.MikroORMIndex()(MikroORMEntity)
+  })
+}

packages/core/utils/src/dml/helpers/entity-builder/build-indexes.ts

@@ -0,0 +1,30 @@
+import { DMLSchema, EntityIndex } from "@medusajs/types"
+import { isPresent } from "../../../common"
+
+/*
+  The DML provides an opinionated soft deletable entity as a part of every model
+  We assume that deleted_at would be scoped in indexes in all cases as an index without the scope
+  doesn't seem to be valid. If a case presents itself where one would like to remove the scope, 
+  this will need to be updated to include that case.
+*/
+export function transformIndexWhere<TSchema extends DMLSchema>(
+  index: EntityIndex<TSchema>
+) {
+  let where = index.where
+  const lowerCaseWhere = where?.toLowerCase()
+  const whereIncludesDeleteable =
+    lowerCaseWhere?.includes("deleted_at is null") ||
+    lowerCaseWhere?.includes("deleted_at is not null")
+
+  // If where scope does not include a deleted_at scope, we add a soft deletable scope to it
+  if (where && !whereIncludesDeleteable) {
+    where = where + ` AND deleted_at IS NULL`
+  }
+
+  // If where scope isn't present, we will set an opinionated where scope to the index
+  if (!isPresent(where)) {
+    where = "deleted_at IS NULL"
+  }
+
+  return where
+}

packages/core/utils/src/dml/helpers/entity-builder/create-big-number-properties.ts

@@ -1,4 +1,4 @@
-import { DMLSchema } from "../../entity-builder"
+import { DMLSchema } from "@medusajs/types"
 import { BigNumberProperty } from "../../properties/big-number"
 import { JSONProperty } from "../../properties/json"
 import { NullableModifier } from "../../properties/nullable"

packages/core/utils/src/dml/helpers/entity-builder/infer-primary-key-properties.ts

@@ -1,4 +1,4 @@
-import { DMLSchema } from "../../entity-builder"
+import { DMLSchema } from "@medusajs/types"
 import { IdProperty } from "../../properties/id"
 
 /*

packages/core/utils/src/dml/helpers/mikro-orm/build-indexes.ts

@@ -0,0 +1,32 @@
+import { DMLSchema, EntityConstructor, EntityIndex } from "@medusajs/types"
+import { MetadataStorage } from "@mikro-orm/core"
+import { arrayDifference } from "../../../common"
+
+/*
+  The DML should strictly define indexes where the fields provided for the index are 
+  already present in the schema definition. If not, we throw an error.
+*/
+export function validateIndexFields<TSchema extends DMLSchema>(
+  MikroORMEntity: EntityConstructor<any>,
+  index: EntityIndex<TSchema>
+) {
+  const fields: string[] = index.on
+
+  if (!fields?.length) {
+    throw new Error(
+      `"on" is a required property when applying indexes on a DML entity`
+    )
+  }
+
+  const metaData = MetadataStorage.getMetadataFromDecorator(MikroORMEntity)
+  const entityFields: string[] = Object.keys(metaData.properties)
+  const invalidFields = arrayDifference(fields, entityFields)
+
+  if (invalidFields.length) {
+    throw new Error(
+      `Fields (${invalidFields.join(
+        ", "
+      )}) are not found when applying indexes from DML entity`
+    )
+  }
+}