asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

fix(index): Apply various fixes to the index engine (#12501)

Browse Source
This commit is contained in:
Carlos R. L. Rodrigues
2025-05-19 15:14:25 -03:00
committed by GitHub
parent 32be40a2c0
commit 59bbff62d8
20 changed files with 194 additions and 133 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
.changeset/plenty-shirts-smash.md Normal file
View File

@@ -0,0 +1,9 @@
---
"@medusajs/link-modules": patch
"@medusajs/index": patch
"@medusajs/utils": patch
"@medusajs/modules-sdk": patch
"@medusajs/types": patch
---
fix(index): limit partition table name and index enum fields

2
packages/core/modules-sdk/src/remote-query/query.ts
View File

@@ -232,7 +232,7 @@ export class Query {
pagination: {
// We pass through `take` to force the `select-in` query strategy
// There might be a better way to do this, but for now this should do
take: queryOptions.pagination?.take,
take: queryOptions.pagination?.take ?? indexResponse.data.length,
},
}

32
packages/core/types/src/index-data/__tests__/index.spec.ts
View File

@@ -23,14 +23,34 @@ describe("IndexQueryConfig", () => {
expectTypeOf<IndexConfig["filters"]>().toEqualTypeOf<
| {
id?: string | string[] | OperatorMap<string>
title?: string | string[] | OperatorMap<string>
id?: string | string[] | OperatorMap<string | string[] | null> | null
title?:
| string
| string[]
| OperatorMap<string | string[] | null>
| null
variants?: {
id?: string | string[] | OperatorMap<string>
product_id?: string | string[] | OperatorMap<string>
sku?: string | string[] | OperatorMap<string>
id?:
| string
| string[]
| OperatorMap<string | string[] | null>
| null
product_id?:
| string
| string[]
| OperatorMap<string | string[] | null>
| null
sku?:
| string
| string[]
| OperatorMap<string | string[] | null>
| null
prices?: {
amount?: number | number[] | OperatorMap<number>
amount?:
| number
| number[]
| OperatorMap<number | number[] | null>
| null
}
}
}

20
packages/core/types/src/index-data/index-operator-map.ts
View File

@@ -1,12 +1,12 @@
export type IndexOperatorMap<T> = {
$eq: T
$lt: T
$lte: T
$gt: T
$gte: T
$ne: T
$in: T
$is: T
$like: T
$ilike: T
$eq?: T
$lt?: T
$lte?: T
$gt?: T
$gte?: T
$ne?: T
$in?: T
$is?: T
$like?: T
$ilike?: T
}

8
packages/core/types/src/index-data/query-config/query-input-config-filters.ts
View File

@@ -19,8 +19,12 @@ type ExtractFiltersOperators<
? never
: Key extends ExcludedProps
? never
: TypeOnly<T[Key]> extends string | number | boolean | Date
? TypeOnly<T[Key]> | TypeOnly<T[Key]>[] | OperatorMap<TypeOnly<T[Key]>>
: TypeOnly<T[Key]> extends string | number | boolean | Date | null
?
| TypeOnly<T[Key]>
| TypeOnly<T[Key]>[]
| OperatorMap<TypeOnly<T[Key]> | TypeOnly<T[Key]>[] | null>
| null
: TypeOnly<T[Key]> extends Array<infer R>
? TypeOnly<R> extends { __typename: any }
? IndexFilters<Key & string, T, [Key & string, ...Exclusion], Depth[Lim]>

1
packages/core/types/src/index-data/query-config/query-input-config.ts
View File

@@ -63,7 +63,6 @@ export type IndexQueryConfig<TEntry extends string> = {
filters?: IndexFilters<TEntry>
joinFilters?: IndexFilters<TEntry>
pagination?: Partial<IndexQueryInput<TEntry>["pagination"]>
keepFilteredEntities?: boolean
}
export type QueryFunctionReturnPagination = {

0
packages/modules/link-modules/src/utils/__tests__/compress-name.spec.ts → packages/core/utils/src/common/__tests__/compress-name.spec.ts
View File

3
packages/modules/link-modules/src/utils/compress-name.ts → packages/core/utils/src/common/compress-name.ts
View File

@@ -1,4 +1,5 @@
import { camelToSnakeCase, simpleHash } from "@medusajs/framework/utils"
import { camelToSnakeCase } from "./camel-to-snake-case"
import { simpleHash } from "./simple-hash"
export function compressName(name: string, limit = 58) {
if (name.length <= limit) {

7
packages/core/utils/src/common/index.ts
View File

@@ -4,6 +4,7 @@ export * from "./array-intersection"
export * from "./build-query"
export * from "./build-regexp-if-valid"
export * from "./camel-to-snake-case"
export * from "./compress-name"
export * from "./container"
export * from "./convert-item-response-to-update-request"
export * from "./create-container-like"
@@ -80,11 +81,11 @@ export * from "./to-handle"
export * from "./to-kebab-case"
export * from "./to-pascal-case"
export * from "./to-unix-slash"
export * from "./try-convert-to-number"
export * from "./trim-zeros"
export * from "./try-convert-to-boolean"
export * from "./try-convert-to-number"
export * from "./unflatten-object-keys"
export * from "./upper-case-first"
export * from "./validate-handle"
export * from "./wrap-handler"
export * from "./validate-module-name"
export * from "./try-convert-to-boolean"
export * from "./wrap-handler"

3
packages/core/utils/src/event-bus/utils.ts
View File

@@ -44,8 +44,7 @@ export function buildModuleResourceEventName({
action: string
}): string {
const kebabCaseName = lowerCaseFirst(kebabCase(objectName))
const conventionalePrefix = prefix && lowerCaseFirst(kebabCase(prefix))
return `${prefix ? `${conventionalePrefix}.` : ""}${kebabCaseName}.${action}`
return `${prefix ? `${prefix}.` : ""}${kebabCaseName}.${action}`
}
/**

82
packages/modules/index/integration-tests/__tests__/query-builder.spec.ts
View File

@@ -284,6 +284,34 @@ describe("IndexModuleService query", function () {
afterEach(afterEach_)
it("should query all products where sku not null", async () => {
const { data } = await module.query({
fields: ["product.*", "product.variants.*", "product.variants.prices.*"],
filters: {
product: {
variants: {
sku: { $ne: null },
},
},
},
})
expect(data.length).toEqual(1)
const { data: data2 } = await module.query({
fields: ["product.*", "product.variants.*", "product.variants.prices.*"],
filters: {
product: {
variants: {
sku: { $eq: null },
},
},
},
})
expect(data2.length).toEqual(0)
})
it("should query all products ordered by sku DESC", async () => {
const { data } = await module.query({
fields: ["product.*", "product.variants.*", "product.variants.prices.*"],
@@ -710,60 +738,6 @@ describe("IndexModuleService query", function () {
])
})
it("should query products filtering by price and returning the complete entity", async () => {
const { data, metadata } = await module.query({
fields: ["product.*", "product.variants.*", "product.variants.prices.*"],
filters: {
product: {
variants: {
prices: {
amount: { $gt: 50 },
},
},
},
},
keepFilteredEntities: true,
pagination: {
take: 100,
skip: 0,
},
})
expect(metadata).toEqual({
estimate_count: expect.any(Number),
skip: 0,
take: 100,
})
expect(data).toEqual([
{
id: "prod_1",
variants: [
{
id: "var_1",
sku: "aaa test aaa",
prices: [
{
id: "money_amount_1",
amount: 100,
},
],
},
{
id: "var_2",
sku: "sku 123",
prices: [
{
id: "money_amount_2",
amount: 10,
},
],
},
],
},
])
})
it("should query all products", async () => {
const { data } = await module.query({
fields: ["product.*", "product.variants.*", "product.variants.prices.*"],

59
packages/modules/index/src/migrations/Migration20250515161913.ts Normal file
View File

@@ -0,0 +1,59 @@
import { Migration } from "@mikro-orm/migrations"
export class Migration20250515161913 extends Migration {
override async up(): Promise<void> {
this.addSql(`
DO $$
DECLARE
r RECORD;
protected_tables TEXT[] := ARRAY[
'cat_linkproductsaleschannel',
'cat_linkproductvariantpriceset',
'cat_price',
'cat_priceset',
'cat_product',
'cat_productvariant',
'cat_saleschannel',
'cat_pivot_linkproductsaleschannelsaleschannel',
'cat_pivot_linkproductvariantpricesetpriceset',
'cat_pivot_pricesetprice',
'cat_pivot_productlinkproductsaleschannel',
'cat_pivot_productproductvariant',
'cat_pivot_productvariantlinkproductvariantpriceset'
];
BEGIN
FOR r IN
SELECT tablename
FROM pg_tables
WHERE schemaname = 'public' AND tablename LIKE 'cat\_%'
LOOP
IF r.tablename <> ALL (protected_tables) THEN
EXECUTE format('DROP TABLE IF EXISTS public.%I CASCADE;', r.tablename);
END IF;
END LOOP;
END $$;
UPDATE index_sync SET last_key = NULL WHERE entity NOT IN (
'Product',
'ProductVariant',
'LinkProductVariantPriceSet',
'LinkProductSalesChannel',
'Price',
'PriceSet',
'SalesChannel'
);
UPDATE index_metadata SET status = 'pending' WHERE entity NOT IN (
'Product',
'ProductVariant',
'LinkProductVariantPriceSet',
'LinkProductSalesChannel',
'Price',
'PriceSet',
'SalesChannel'
);
`)
}
override async down(): Promise<void> {}
}

33
packages/modules/index/src/services/postgres-provider.ts
View File

@@ -237,12 +237,7 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
): Promise<IndexTypes.QueryResultSet<TEntry>> {
await this.#isReady_
const {
keepFilteredEntities,
fields = [],
filters = {},
joinFilters = {},
} = config
const { fields = [], filters = {}, joinFilters = {} } = config
const { take, skip, order: inputOrderBy = {} } = config.pagination ?? {}
const select = normalizeFieldsSelection(fields)
@@ -281,7 +276,6 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
options: {
skip,
take,
keepFilteredEntities,
orderBy,
},
rawConfig: config,
@@ -290,7 +284,6 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
const { sql, sqlCount } = qb.buildQuery({
hasPagination,
returnIdOnly: !!keepFilteredEntities,
hasCount,
})
@@ -310,30 +303,6 @@ export class PostgresProvider implements IndexTypes.StorageProvider {
} as IndexTypes.QueryFunctionReturnPagination)
: undefined
if (keepFilteredEntities) {
const mainEntity = Object.keys(select)[0]
const ids = resultSet.map((r) => r[`${mainEntity}.id`])
if (ids.length) {
const result = await this.query<TEntry>(
{
fields,
joinFilters,
filters: {
[mainEntity]: {
id: ids,
},
},
pagination: undefined,
keepFilteredEntities: false,
} as IndexTypes.IndexQueryConfig<TEntry>,
sharedContext
)
result.metadata ??= resultMetadata
return result
}
}
return {
data: qb.buildObjectFromResultset(
resultSet

1
packages/modules/index/src/types/index.ts
View File

@@ -29,5 +29,4 @@ export type QueryOptions = {
skip?: number
take?: number
orderBy?: OrderBy | OrderBy[]
keepFilteredEntities?: boolean
}

4
packages/modules/index/src/utils/build-config.ts
View File

@@ -1180,7 +1180,9 @@ function buildSchemaFromFilterableLinks(
return
}
const fieldType = fieldRef.type.toString()
const isEnum =
fieldRef.type?.astNode?.kind === GraphQLUtils.Kind.ENUM_TYPE_DEFINITION
const fieldType = isEnum ? "String" : fieldRef.type.toString()
const isArray = fieldType.startsWith("[")
const currentType = fieldType.replace(/\[|\]|\!/g, "")

11
packages/modules/index/src/utils/create-partitions.ts
View File

@@ -1,6 +1,7 @@
import { IndexTypes } from "@medusajs/framework/types"
import { SqlEntityManager } from "@mikro-orm/postgresql"
import { schemaObjectRepresentationPropertiesToOmit } from "@types"
import { getPivotTableName, normalizeTableName } from "./normalze-table-name"
export async function createPartitions(
schemaObjectRepresentation: IndexTypes.SchemaObjectRepresentation,
@@ -18,7 +19,7 @@ export async function createPartitions(
schemaObjectRepresentation[key].listeners.length > 0
)
.map((key) => {
const cName = key.toLowerCase()
const cName = normalizeTableName(key)
if (createdPartitions.has(cName)) {
return []
@@ -35,7 +36,8 @@ export async function createPartitions(
continue
}
const pName = `cat_pivot_${parent.ref.entity}${key}`.toLowerCase()
const pName = getPivotTableName(`${parent.ref.entity}${key}`)
if (createdPartitions.has(pName)) {
continue
}
@@ -63,7 +65,7 @@ export async function createPartitions(
schemaObjectRepresentation[key].listeners.length > 0
)
.map((key) => {
const cName = key.toLowerCase()
const cName = normalizeTableName(key)
const part: string[] = []
part.push(
@@ -79,7 +81,8 @@ export async function createPartitions(
continue
}
const pName = `cat_pivot_${parent.ref.entity}${key}`.toLowerCase()
const pName = getPivotTableName(`${parent.ref.entity}${key}`)
part.push(
`CREATE INDEX CONCURRENTLY IF NOT EXISTS "IDX_${pName}_child_id" ON ${activeSchema}${pName} ("child_id")`
)

1
packages/modules/index/src/utils/index.ts
View File

@@ -6,3 +6,4 @@ export * from "./sync/configuration"
export * from "./index-metadata-status"
export * from "./gql-to-types"
export * from "./default-schema"
export * from "./normalze-table-name"

10
packages/modules/index/src/utils/normalze-table-name.ts Normal file
View File

@@ -0,0 +1,10 @@
import { compressName } from "@medusajs/framework/utils"
export function normalizeTableName(name: string): string {
return compressName(name.toLowerCase(), 58).replace(/[^a-z0-9_]/g, "_")
}
export function getPivotTableName(tableName: string) {
const compressedName = normalizeTableName(tableName)
return `cat_pivot_${compressedName}`
}

39
packages/modules/index/src/utils/query-builder.ts
View File

@@ -7,6 +7,7 @@ import {
} from "@medusajs/framework/utils"
import { Knex } from "@mikro-orm/knex"
import { OrderBy, QueryFormat, QueryOptions, Select } from "@types"
import { getPivotTableName, normalizeTableName } from "./normalze-table-name"
function escapeJsonPathString(val: string): string {
// Escape for JSONPath string
@@ -23,6 +24,10 @@ function buildSafeJsonPathQuery(
jsonPathOperator = "=="
} else if (operator.toUpperCase().includes("LIKE")) {
jsonPathOperator = "like_regex"
} else if (operator === "IS") {
jsonPathOperator = "=="
} else if (operator === "IS NOT") {
jsonPathOperator = "!="
}
if (typeof value === "string") {
@@ -32,6 +37,10 @@ function buildSafeJsonPathQuery(
val = val.replace(/%/g, ".*").replace(/_/g, ".")
}
value = `"${escapeJsonPathString(val)}"`
} else {
if ((operator === "IS" || operator === "IS NOT") && value === null) {
value = "null"
}
}
return `$.${field} ${jsonPathOperator} ${value}`
@@ -530,14 +539,14 @@ export class QueryBuilder {
aliasMapping[currentAliasPath] = alias
if (level > 0) {
const cName = entity.ref.entity.toLowerCase()
const cName = normalizeTableName(entity.ref.entity)
let joinTable = `cat_${cName} AS ${alias}`
if (entity.isInverse || parEntity.isInverse) {
const pName =
`${entity.ref.entity}${parEntity.ref.entity}`.toLowerCase()
const pivotTable = `cat_pivot_${pName}`
const pivotTable = getPivotTableName(pName)
joinBuilder.leftJoin(
`${pivotTable} AS ${alias}_ref`,
@@ -552,7 +561,7 @@ export class QueryBuilder {
} else {
const pName =
`${parEntity.ref.entity}${entity.ref.entity}`.toLowerCase()
const pivotTable = `cat_pivot_${pName}`
const pivotTable = getPivotTableName(pName)
joinBuilder.leftJoin(
`${pivotTable} AS ${alias}_ref`,
@@ -704,11 +713,9 @@ export class QueryBuilder {
public buildQuery({
hasPagination = true,
hasCount = false,
returnIdOnly = false,
}: {
hasPagination?: boolean
hasCount?: boolean
returnIdOnly?: boolean
}): { sql: string; sqlCount?: string } {
const selectOnlyStructure = this.selector.select
const structure = this.requestedFields
@@ -816,7 +823,10 @@ export class QueryBuilder {
}
innerQueryBuilder.from(
`cat_${rootEntity} AS ${this.getShortAlias(aliasMapping, rootKey)}`
`cat_${normalizeTableName(rootEntity)} AS ${this.getShortAlias(
aliasMapping,
rootKey
)}`
)
joinParts.forEach((joinPart) => {
@@ -887,7 +897,10 @@ export class QueryBuilder {
const innerQueryAlias = "paginated_ids"
outerQueryBuilder.from(
`cat_${rootEntity} AS ${this.getShortAlias(aliasMapping, rootKey)}`
`cat_${normalizeTableName(rootEntity)} AS ${this.getShortAlias(
aliasMapping,
rootKey
)}`
)
outerQueryBuilder.joinRaw(
@@ -909,13 +922,11 @@ export class QueryBuilder {
outerQueryBuilder.joinRaw(joinPart)
})
const finalSelectParts = !returnIdOnly
? this.buildSelectParts(
selectOnlyStructure[rootKey] as Select,
rootKey,
aliasMapping
)
: { [`${rootKey}.id`]: `${rootAlias}.id` }
const finalSelectParts = this.buildSelectParts(
selectOnlyStructure[rootKey] as Select,
rootKey,
aliasMapping
)
outerQueryBuilder.select(finalSelectParts)

2
packages/modules/link-modules/src/utils/generate-entity.ts
View File

@@ -4,13 +4,13 @@ import {
} from "@medusajs/framework/types"
import {
composeTableName,
compressName,
mikroOrmSoftDeletableFilterOptions,
simpleHash,
SoftDeletableFilterKey,
} from "@medusajs/framework/utils"
import { EntitySchema } from "@mikro-orm/core"
import { compressName } from "./compress-name"
function getClass(...properties) {
return class LinkModel {