diff --git a/integration-tests/http/__tests__/product/admin/product-export.spec.ts b/integration-tests/http/__tests__/product/admin/product-export.spec.ts index 47d16e9354..d9a3e08930 100644 --- a/integration-tests/http/__tests__/product/admin/product-export.spec.ts +++ b/integration-tests/http/__tests__/product/admin/product-export.spec.ts @@ -234,118 +234,6 @@ medusaIntegrationTestRunner({ recursive: true, }) }) - - // it("should be able to import an exported csv file", async () => { - // const api = useApi() - - // const batchPayload = { - // type: "product-export", - // context: { - // batch_size: 1, - // filterable_fields: { collection_id: "test-collection" }, - // order: "created_at", - // }, - // } - - // const batchJobRes = await api.post( - // "/admin/batch-jobs", - // batchPayload, - // adminReqConfig - // ) - // let batchJobId = batchJobRes.data.batch_job.id - - // expect(batchJobId).toBeTruthy() - - // // Pull to check the status until it is completed - // let batchJob - // let shouldContinuePulling = true - // while (shouldContinuePulling) { - // const res = await api.get( - // `/admin/batch-jobs/${batchJobId}`, - // adminReqConfig - // ) - - // await new Promise((resolve, _) => { - // setTimeout(resolve, 1000) - // }) - - // batchJob = res.data.batch_job - // shouldContinuePulling = !( - // batchJob.status === "completed" || batchJob.status === "failed" - // ) - // } - - // expect(batchJob.status).toBe("completed") - - // exportFilePath = path.resolve(__dirname, batchJob.result.file_key) - // const isFileExists = (await fs.stat(exportFilePath)).isFile() - - // expect(isFileExists).toBeTruthy() - - // const data = (await fs.readFile(exportFilePath)).toString() - // const [header, ...lines] = data.split("\r\n").filter((l) => l) - - // expect(lines.length).toBe(4) - - // const csvLine = lines[0].split(";") - // expect(csvLine[0]).toBe("test-product") - // expect(csvLine[2]).toBe("Test product") - - // csvLine[2] = "Updated test product" - // lines.splice(0, 1, csvLine.join(";")) - - // await fs.writeFile(exportFilePath, [header, ...lines].join("\r\n")) - - // const importBatchJobRes = await api.post( - // "/admin/batch-jobs", - // { - // type: "product-import", - // context: { - // fileKey: exportFilePath, - // }, - // }, - // adminReqConfig - // ) - - // batchJobId = importBatchJobRes.data.batch_job.id - - // expect(batchJobId).toBeTruthy() - - // shouldContinuePulling = true - // while (shouldContinuePulling) { - // const res = await api.get( - // `/admin/batch-jobs/${batchJobId}`, - // adminReqConfig - // ) - - // await new Promise((resolve, _) => { - // setTimeout(resolve, 1000) - // }) - - // batchJob = res.data.batch_job - - // shouldContinuePulling = !( - // batchJob.status === "completed" || batchJob.status === "failed" - // ) - // } - - // expect(batchJob.status).toBe("completed") - - // const productsResponse = await api.get( - // "/admin/products", - // adminReqConfig - // ) - // expect(productsResponse.data.count).toBe(5) - // expect(productsResponse.data.products).toEqual( - // expect.arrayContaining([ - // expect.objectContaining({ - // id: csvLine[0], - // handle: csvLine[1], - // title: csvLine[2], - // }), - // ]) - // ) - // }) }) }, }) diff --git a/integration-tests/http/__tests__/product/admin/product-import.spec.ts b/integration-tests/http/__tests__/product/admin/product-import.spec.ts index d17c0dfc68..98e6e82429 100644 --- a/integration-tests/http/__tests__/product/admin/product-import.spec.ts +++ b/integration-tests/http/__tests__/product/admin/product-import.spec.ts @@ -60,7 +60,6 @@ medusaIntegrationTestRunner({ "/admin/products", getProductFixture({ title: "Base product", - type_id: baseType.id, }), adminHeaders ) @@ -72,7 +71,7 @@ medusaIntegrationTestRunner({ }) describe("POST /admin/products/export", () => { - it("should import a products CSV file", async () => { + it("should import a previously exported products CSV file", async () => { const subscriberExecution = TestEventUtils.waitSubscribersExecution( "notification.notification.created", eventBus @@ -83,6 +82,17 @@ medusaIntegrationTestRunner({ { encoding: "utf-8" } ) + fileContent = fileContent.replace( + /prod_01J3CRPNVGRZ01A8GH8FQYK10Z/g, + baseProduct.id + ) + fileContent = fileContent.replace( + /variant_01J3CRPNW5J6EBVVQP1TN33A58/g, + baseProduct.variants[0].id + ) + fileContent = fileContent.replace(/pcol_\w*\d*/g, baseCollection.id) + fileContent = fileContent.replace(/ptyp_\w*\d*/g, baseType.id) + const { form, meta } = getUploadReq({ name: "test.csv", content: fileContent, @@ -108,7 +118,189 @@ medusaIntegrationTestRunner({ }), }) ) + + const dbProducts = (await api.get("/admin/products", adminHeaders)).data + .products + + expect(dbProducts).toHaveLength(2) + expect(dbProducts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: baseProduct.id, + handle: "base-product", + is_giftcard: false, + thumbnail: "test-image.png", + status: "draft", + description: "test-product-description\ntest line 2", + options: [ + expect.objectContaining({ + title: "size", + values: expect.arrayContaining([ + expect.objectContaining({ + value: "small", + }), + expect.objectContaining({ + value: "large", + }), + ]), + }), + expect.objectContaining({ + title: "color", + values: expect.arrayContaining([ + expect.objectContaining({ + value: "green", + }), + ]), + }), + ], + images: expect.arrayContaining([ + expect.objectContaining({ + url: "test-image.png", + }), + expect.objectContaining({ + url: "test-image-2.png", + }), + ]), + tags: [ + expect.objectContaining({ + value: "123", + }), + expect.objectContaining({ + value: "456", + }), + ], + type: expect.objectContaining({ + id: baseType.id, + }), + collection: expect.objectContaining({ + id: baseCollection.id, + }), + variants: [ + expect.objectContaining({ + title: "Test variant", + allow_backorder: false, + manage_inventory: true, + prices: [ + expect.objectContaining({ + currency_code: "usd", + amount: 100, + }), + expect.objectContaining({ + currency_code: "eur", + amount: 45, + }), + expect.objectContaining({ + currency_code: "dkk", + amount: 30, + }), + ], + options: [ + expect.objectContaining({ + value: "large", + }), + expect.objectContaining({ + value: "green", + }), + ], + }), + expect.objectContaining({ + title: "Test variant 2", + allow_backorder: false, + manage_inventory: true, + // TODO: Since we are doing a product update, there won't be any prices created for the variant + options: [ + expect.objectContaining({ + value: "small", + }), + expect.objectContaining({ + value: "green", + }), + ], + }), + ], + created_at: expect.any(String), + updated_at: expect.any(String), + }), + expect.objectContaining({ + id: expect.any(String), + handle: "proposed-product", + is_giftcard: false, + thumbnail: "test-image.png", + status: "proposed", + description: "test-product-description", + options: [ + expect.objectContaining({ + title: "size", + values: expect.arrayContaining([ + expect.objectContaining({ + value: "large", + }), + ]), + }), + expect.objectContaining({ + title: "color", + values: expect.arrayContaining([ + expect.objectContaining({ + value: "green", + }), + ]), + }), + ], + images: expect.arrayContaining([ + expect.objectContaining({ + url: "test-image.png", + }), + expect.objectContaining({ + url: "test-image-2.png", + }), + ]), + tags: [ + expect.objectContaining({ + value: "new-tag", + }), + ], + type: expect.objectContaining({ + id: baseType.id, + }), + collection: null, + variants: [ + expect.objectContaining({ + title: "Test variant", + allow_backorder: false, + manage_inventory: true, + prices: [ + expect.objectContaining({ + currency_code: "usd", + amount: 100, + }), + expect.objectContaining({ + currency_code: "eur", + amount: 45, + }), + expect.objectContaining({ + currency_code: "dkk", + amount: 30, + }), + ], + options: [ + expect.objectContaining({ + value: "large", + }), + expect.objectContaining({ + value: "green", + }), + ], + }), + ], + created_at: expect.any(String), + updated_at: expect.any(String), + }), + ]) + ) }) + + it("should fail on invalid prices being present in the CSV", async () => {}) + it("should fail on non-existent fields being present in the CSV", async () => {}) }) }, }) diff --git a/packages/core/core-flows/src/product/helpers/normalize-for-import.ts b/packages/core/core-flows/src/product/helpers/normalize-for-import.ts new file mode 100644 index 0000000000..99d4020163 --- /dev/null +++ b/packages/core/core-flows/src/product/helpers/normalize-for-import.ts @@ -0,0 +1,216 @@ +import { HttpTypes } from "@medusajs/types" +import { MedusaError, lowerCaseFirst } from "@medusajs/utils" + +// We want to convert the csv data format to a standard DTO format. +export const normalizeForImport = ( + rawProducts: object[] +): HttpTypes.AdminCreateProduct[] => { + const productMap = new Map< + string, + { + product: HttpTypes.AdminCreateProduct + variants: HttpTypes.AdminCreateProductVariant[] + } + >() + + rawProducts.forEach((rawProduct) => { + const productInMap = productMap.get(rawProduct["Product Handle"]) + if (!productInMap) { + productMap.set(rawProduct["Product Handle"], { + product: normalizeProductForImport(rawProduct), + variants: [normalizeVariantForImport(rawProduct)], + }) + return + } + + productMap.set(rawProduct["Product Handle"], { + product: productInMap.product, + variants: [ + ...productInMap.variants, + normalizeVariantForImport(rawProduct), + ], + }) + }) + + return Array.from(productMap.values()).map((p) => { + const options = p.variants.reduce( + (agg: Record>, variant) => { + Object.entries(variant.options ?? {}).forEach(([key, value]) => { + if (!agg[key]) { + agg[key] = new Set() + } + + agg[key].add(value as string) + }) + + return agg + }, + {} + ) + + return { + ...p.product, + options: Object.entries(options).map(([key, value]) => ({ + title: key, + values: Array.from(value), + })), + variants: p.variants, + } + }) +} + +const productFieldsToOmit = new Map() +const variantFieldsToOmit = new Map([["variant_product_id", true]]) + +// We use an array here as we do a substring matching as a check. +// These fields can have a numeric value, but they are stored as string in the DB so we need to normalize them +const stringFields = [ + "product_tag_", + "variant_barcode", + "variant_sku", + "variant_ean", + "variant_upc", + "variant_hs_code", + "variant_mid_code", +] + +const normalizeProductForImport = ( + rawProduct: object +): HttpTypes.AdminCreateProduct => { + const response = {} + + Object.entries(rawProduct).forEach(([key, value]) => { + const normalizedKey = snakecaseKey(key) + const normalizedValue = getNormalizedValue(normalizedKey, value) + + // We have no way of telling if a field is set as an empty string or it was undefined, so we completely omit empty fields. + if (normalizedValue === "") { + return + } + + if (normalizedKey.startsWith("product_image_")) { + response["images"] = [ + ...(response["images"] || []), + { url: normalizedValue }, + ] + return + } + + if (normalizedKey.startsWith("product_tag_")) { + response["tags"] = [ + ...(response["tags"] || []), + { value: normalizedValue }, + ] + return + } + + if (normalizedKey.startsWith("product_sales_channel_")) { + response["sales_channels"] = [ + ...(response["sales_channels"] || []), + { id: normalizedValue }, + ] + return + } + + if ( + normalizedKey.startsWith("product_") && + !productFieldsToOmit.has(normalizedKey) + ) { + response[normalizedKey.replace("product_", "")] = normalizedValue + return + } + }) + + return response as HttpTypes.AdminCreateProduct +} + +const normalizeVariantForImport = ( + rawProduct: object +): HttpTypes.AdminCreateProductVariant => { + const response = {} + const options = new Map() + + Object.entries(rawProduct).forEach(([key, value]) => { + const normalizedKey = snakecaseKey(key) + const normalizedValue = getNormalizedValue(normalizedKey, value) + + // We have no way of telling if a field is set as an empty string or it was undefined, so we completely omit empty fields. + if (normalizedValue === "") { + return + } + + if (normalizedKey.startsWith("variant_price_")) { + const priceKey = normalizedKey.replace("variant_price_", "") + // Note: If we start using the region name instead of ID, this check might not always work. + if (priceKey.length === 3) { + response["prices"] = [ + ...(response["prices"] || []), + { currency_code: priceKey.toLowerCase(), amount: normalizedValue }, + ] + } else { + response["prices"] = [ + ...(response["prices"] || []), + { + amount: normalizedValue, + rules: { region_id: priceKey }, + }, + ] + } + return + } + + if (normalizedKey.startsWith("variant_option_")) { + const keyBase = normalizedKey.replace("variant_option_", "") + const optionIndex = parseInt(keyBase.split("_")[0]) + const optionType = keyBase.split("_")[1] + + options.set(optionIndex, { + ...options.get(optionIndex), + [optionType]: normalizedValue, + }) + return + } + + if ( + normalizedKey.startsWith("variant_") && + !variantFieldsToOmit.has(normalizedKey) + ) { + response[normalizedKey.replace("variant_", "")] = normalizedValue + return + } + }) + + response["options"] = Array.from(options.values()).reduce( + (agg: Record, option) => { + if (!option.name) { + throw new MedusaError( + MedusaError.Types.INVALID_DATA, + `Missing option name for product with handle ${rawProduct["Product Handle"]}` + ) + } + + if (!option.value) { + throw new MedusaError( + MedusaError.Types.INVALID_DATA, + `Missing option value for product with handle ${rawProduct["Product Handle"]} and option ${option.name}` + ) + } + + agg[option.name] = option.value + return agg + }, + {} + ) + + return response as HttpTypes.AdminCreateProductVariant +} + +const getNormalizedValue = (key: string, value: any): any => { + return stringFields.some((field) => key.startsWith(field)) + ? value.toString() + : value +} + +const snakecaseKey = (key: string): string => { + return key.split(" ").map(lowerCaseFirst).join("_") +} diff --git a/packages/core/core-flows/src/product/steps/group-products-for-batch.ts b/packages/core/core-flows/src/product/steps/group-products-for-batch.ts new file mode 100644 index 0000000000..3d1c1c49f9 --- /dev/null +++ b/packages/core/core-flows/src/product/steps/group-products-for-batch.ts @@ -0,0 +1,63 @@ +import { HttpTypes, IProductModuleService, ProductTypes } from "@medusajs/types" +import { MedusaError, ModuleRegistrationName } from "@medusajs/utils" +import { StepResponse, createStep } from "@medusajs/workflows-sdk" + +export const groupProductsForBatchStepId = "group-products-for-batch" +export const groupProductsForBatchStep = createStep( + groupProductsForBatchStepId, + async (data: HttpTypes.AdminCreateProduct[], { container }) => { + const service = container.resolve( + ModuleRegistrationName.PRODUCT + ) + + const existingProducts = await service.listProducts( + { + // We already validate that there is handle in a previous step + handle: data.map((product) => product.handle) as string[], + }, + { take: null, select: ["handle"] } + ) + const existingProductsMap = new Map( + existingProducts.map((p) => [p.handle, true]) + ) + + const { toUpdate, toCreate } = data.reduce( + ( + acc: { + toUpdate: (HttpTypes.AdminUpdateProduct & { id: string })[] + toCreate: HttpTypes.AdminCreateProduct[] + }, + product + ) => { + // There are few data normalizations to do if we are dealing with an update. + if (existingProductsMap.has(product.handle!)) { + if (!(product as any).id) { + throw new MedusaError( + MedusaError.Types.INVALID_DATA, + "Product id is required when updating products in import" + ) + } + + // TODO: Currently the update product workflow doesn't update variant pricing, but we should probably add support for it. + product.variants?.forEach((variant: any) => { + delete variant.prices + }) + + acc.toUpdate.push( + product as HttpTypes.AdminUpdateProduct & { id: string } + ) + return acc + } + + // New products will be created with a new ID, even if there is one present in the CSV. + // To add support for creating with predefined IDs we will need to do changes to the upsert method. + delete (product as any).id + acc.toCreate.push(product) + return acc + }, + { toUpdate: [], toCreate: [] } + ) + + return new StepResponse({ create: toCreate, update: toUpdate }) + } +) diff --git a/packages/core/core-flows/src/product/steps/index.ts b/packages/core/core-flows/src/product/steps/index.ts index f3ff03dc5d..a5b59dfe0d 100644 --- a/packages/core/core-flows/src/product/steps/index.ts +++ b/packages/core/core-flows/src/product/steps/index.ts @@ -21,3 +21,5 @@ export * from "./create-product-tags" export * from "./update-product-tags" export * from "./delete-product-tags" export * from "./generate-product-csv" +export * from "./parse-product-csv" +export * from "./group-products-for-batch" diff --git a/packages/core/core-flows/src/product/steps/parse-product-csv.ts b/packages/core/core-flows/src/product/steps/parse-product-csv.ts new file mode 100644 index 0000000000..4bee5227ac --- /dev/null +++ b/packages/core/core-flows/src/product/steps/parse-product-csv.ts @@ -0,0 +1,23 @@ +import { MedusaError, convertCsvToJson } from "@medusajs/utils" +import { StepResponse, createStep } from "@medusajs/workflows-sdk" +import { normalizeForImport } from "../helpers/normalize-for-import" + +export const parseProductCsvStepId = "parse-product-csv" +export const parseProductCsvStep = createStep( + parseProductCsvStepId, + async (fileContent: string) => { + const csvProducts = convertCsvToJson(fileContent) + + csvProducts.forEach((product: any) => { + if (!product["Product Handle"]) { + throw new MedusaError( + MedusaError.Types.INVALID_DATA, + "Product handle is required when importing products" + ) + } + }) + + const normalizedData = normalizeForImport(csvProducts) + return new StepResponse(normalizedData) + } +) diff --git a/packages/core/core-flows/src/product/workflows/import-products.ts b/packages/core/core-flows/src/product/workflows/import-products.ts index cf724cc954..e3ecb58133 100644 --- a/packages/core/core-flows/src/product/workflows/import-products.ts +++ b/packages/core/core-flows/src/product/workflows/import-products.ts @@ -5,6 +5,8 @@ import { } from "@medusajs/workflows-sdk" import { WorkflowTypes } from "@medusajs/types" import { sendNotificationsStep } from "../../notification" +import { groupProductsForBatchStep, parseProductCsvStep } from "../steps" +import { batchProductsWorkflow } from "./batch-products" export const importProductsWorkflowId = "import-products" export const importProductsWorkflow = createWorkflow( @@ -12,7 +14,12 @@ export const importProductsWorkflow = createWorkflow( ( input: WorkflowData ): WorkflowData => { - // validateImportCsvStep(input.fileContent) + const products = parseProductCsvStep(input.fileContent) + const batchRequest = groupProductsForBatchStep(products) + + // TODO: Add async confirmation step here + + batchProductsWorkflow.runAsStep({ input: batchRequest }) const notifications = transform({ input }, (data) => { return [ diff --git a/packages/core/utils/src/csv/csvtojson.ts b/packages/core/utils/src/csv/csvtojson.ts new file mode 100644 index 0000000000..2c2f971867 --- /dev/null +++ b/packages/core/utils/src/csv/csvtojson.ts @@ -0,0 +1,12 @@ +import { csv2json } from "json-2-csv" + +export interface ConvertCsvToJsonOptions {} + +export const convertCsvToJson = ( + data: string, + options?: ConvertCsvToJsonOptions +): T[] => { + return csv2json(data, { + preventCsvInjection: true, + }) as T[] +} diff --git a/packages/core/utils/src/csv/index.ts b/packages/core/utils/src/csv/index.ts index 9da5d2c13b..322f7b3339 100644 --- a/packages/core/utils/src/csv/index.ts +++ b/packages/core/utils/src/csv/index.ts @@ -1 +1,2 @@ export * from "./jsontocsv" +export * from "./csvtojson" diff --git a/packages/core/utils/src/csv/jsontocsv.ts b/packages/core/utils/src/csv/jsontocsv.ts index a8adc7eba7..dbcd5318ef 100644 --- a/packages/core/utils/src/csv/jsontocsv.ts +++ b/packages/core/utils/src/csv/jsontocsv.ts @@ -12,6 +12,7 @@ export const convertJsonToCsv = ( expandNestedObjects: true, expandArrayObjects: true, unwindArrays: false, + preventCsvInjection: true, emptyFieldValue: "", }) }