From 8a6e172dec9dedddd2808482f3e74937f78771e0 Mon Sep 17 00:00:00 2001 From: Stevche Radevski Date: Wed, 31 Jul 2024 14:03:05 +0300 Subject: [PATCH] feat:Make product import v1 compatible (#8362) --- ...oducts.csv => exported-products-comma.csv} | 0 .../exported-products-semicolon.csv | 6 + .../admin/__fixtures__/invalid-prices.csv | 2 +- .../admin/__fixtures__/v1-template.csv | 6 + .../product/admin/product-export.spec.ts | 2 +- .../product/admin/product-import.spec.ts | 748 ++++++++++++------ .../product/helpers/normalize-for-import.ts | 17 +- .../product/helpers/normalize-v1-import.ts | 142 ++++ .../product/steps/group-products-for-batch.ts | 28 +- .../src/product/steps/parse-product-csv.ts | 35 +- packages/core/utils/src/csv/csvtojson.ts | 17 + 11 files changed, 734 insertions(+), 269 deletions(-) rename integration-tests/http/__tests__/product/admin/__fixtures__/{exported-products.csv => exported-products-comma.csv} (100%) create mode 100644 integration-tests/http/__tests__/product/admin/__fixtures__/exported-products-semicolon.csv create mode 100644 integration-tests/http/__tests__/product/admin/__fixtures__/v1-template.csv create mode 100644 packages/core/core-flows/src/product/helpers/normalize-v1-import.ts diff --git a/integration-tests/http/__tests__/product/admin/__fixtures__/exported-products.csv b/integration-tests/http/__tests__/product/admin/__fixtures__/exported-products-comma.csv similarity index 100% rename from integration-tests/http/__tests__/product/admin/__fixtures__/exported-products.csv rename to integration-tests/http/__tests__/product/admin/__fixtures__/exported-products-comma.csv diff --git a/integration-tests/http/__tests__/product/admin/__fixtures__/exported-products-semicolon.csv b/integration-tests/http/__tests__/product/admin/__fixtures__/exported-products-semicolon.csv new file mode 100644 index 0000000000..b8aac43e12 --- /dev/null +++ b/integration-tests/http/__tests__/product/admin/__fixtures__/exported-products-semicolon.csv @@ -0,0 +1,6 @@ +Product Id;Product Title;Product Subtitle;Product Status;Product External Id;Product Description;Product Handle;Product Is Giftcard;Product Discountable;Product Thumbnail;Product Collection Id;Product Type Id;Product Weight;Product Length;Product Height;Product Width;Product Hs Code;Product Origin Country;Product Mid Code;Product Material;Product Created At;Product Updated At;Product Deleted At;Product Image 1;Product Image 2;Product Tag 1;Product Tag 2;Variant Id;Variant Title;Variant Sku;Variant Barcode;Variant Ean;Variant Upc;Variant Allow Backorder;Variant Manage Inventory;Variant Hs Code;Variant Origin Country;Variant Mid Code;Variant Material;Variant Weight;Variant Length;Variant Height;Variant Width;Variant Metadata;Variant Variant Rank;Variant Product Id;Variant Created At;Variant Updated At;Variant Deleted At;Variant Price USD;Variant Price EUR;Variant Price DKK;Variant Option 1 Name;Variant Option 1 Value;Variant Option 2 Name;Variant Option 2 Value +prod_01J3CRPNVGRZ01A8GH8FQYK10Z;Base product;;draft;;"test-product-description +test line 2";base-product;false;true;test-image.png;pcol_01J3CRPNT6A0G5GG34MWHWE7QD;ptyp_01J3CRPNV39E51BGGWSKT674C5;;;;;;;;;2024-07-22T08:25:06.158Z;2024-07-22T08:25:06.158Z;;test-image.png;test-image-2.png;123;456;variant_01J3CRPNW5J6EBVVQP1TN33A58;Test variant;;;;;false;true;;;;;;;;;;0;prod_01J3CRPNVGRZ01A8GH8FQYK10Z;2024-07-22T08:25:06.182Z;2024-07-22T08:25:06.182Z;;100;45;30;size;large;color;green +prod_01J3CRPNVGRZ01A8GH8FQYK10Z;Base product;;draft;;"test-product-description +test line 2";base-product;false;true;test-image.png;pcol_01J3CRPNT6A0G5GG34MWHWE7QD;ptyp_01J3CRPNV39E51BGGWSKT674C5;;;;;;;;;2024-07-22T08:25:06.158Z;2024-07-22T08:25:06.158Z;;test-image.png;test-image-2.png;123;456;variant_01J3CRPNW6NES6EN14X93F6YYB;Test variant 2;;;;;false;true;;;;;;;;;;0;prod_01J3CRPNVGRZ01A8GH8FQYK10Z;2024-07-22T08:25:06.182Z;2024-07-22T08:25:06.182Z;;200;65;50;size;small;color;green +prod_01J3CRPNYJTCAV1QKRF6H0BY3M;Proposed product;;proposed;;test-product-description;proposed-product;false;true;test-image.png;;ptyp_01J3CRPNV39E51BGGWSKT674C5;;;;;;;;;2024-07-22T08:25:06.256Z;2024-07-22T08:25:06.256Z;;test-image.png;test-image-2.png;new-tag;;variant_01J3CRPNYZ6VZ5FVJ7WHJABV54;Test variant;;;;;false;true;;;;;;;;;;0;prod_01J3CRPNYJTCAV1QKRF6H0BY3M;2024-07-22T08:25:06.271Z;2024-07-22T08:25:06.271Z;;100;45;30;size;large;color;green \ No newline at end of file diff --git a/integration-tests/http/__tests__/product/admin/__fixtures__/invalid-prices.csv b/integration-tests/http/__tests__/product/admin/__fixtures__/invalid-prices.csv index 4239fe9065..85b62739f1 100644 --- a/integration-tests/http/__tests__/product/admin/__fixtures__/invalid-prices.csv +++ b/integration-tests/http/__tests__/product/admin/__fixtures__/invalid-prices.csv @@ -1,2 +1,2 @@ -Product Id,Product Title,Product Subtitle,Product Status,Product External Id,Product Description,Product Handle,Product Is Giftcard,Product Discountable,Product Thumbnail,Product Collection Id,Product Type Id,Product Weight,Product Length,Product Height,Product Width,Product Hs Code,Product Origin Country,Product Mid Code,Product Material,Product Created At,Product Updated At,Product Deleted At,Product Image 1,Product Image 2,Product Tag 1,Variant Id,Variant Title,Variant Sku,Variant Barcode,Variant Ean,Variant Upc,Variant Allow Backorder,Variant Manage Inventory,Variant Hs Code,Variant Origin Country,Variant Mid Code,Variant Material,Variant Weight,Variant Length,Variant Height,Variant Width,Variant Metadata,Variant Variant Rank,Variant Product Id,Variant Created At,Variant Updated At,Variant Deleted At,Variant Price USD,Variant Price EUR,Variant Price reg_nonexistent,Variant Option 1 Name,Variant Option 1 Value,Variant Option 2 Name,Variant Option 2 Value +Product Id,Product Title,Product Subtitle,Product Status,Product External Id,Product Description,Product Handle,Product Is Giftcard,Product Discountable,Product Thumbnail,Product Collection Id,Product Type Id,Product Weight,Product Length,Product Height,Product Width,Product Hs Code,Product Origin Country,Product Mid Code,Product Material,Product Created At,Product Updated At,Product Deleted At,Product Image 1,Product Image 2,Product Tag 1,Variant Id,Variant Title,Variant Sku,Variant Barcode,Variant Ean,Variant Upc,Variant Allow Backorder,Variant Manage Inventory,Variant Hs Code,Variant Origin Country,Variant Mid Code,Variant Material,Variant Weight,Variant Length,Variant Height,Variant Width,Variant Metadata,Variant Variant Rank,Variant Product Id,Variant Created At,Variant Updated At,Variant Deleted At,Variant Price USD,Variant Price EUR,Variant Price nonexistent [EUR],Variant Option 1 Name,Variant Option 1 Value,Variant Option 2 Name,Variant Option 2 Value prod_01J3CSN791SN1RN7X155Z8S9CN,Proposed product,,proposed,,test-product-description,proposed-product,false,true,test-image.png,,ptyp_01J3CSN76GCRSCDV9V489B5FWQ,,,,,,,,,2024-07-22T08:41:47.040Z,2024-07-22T08:41:47.040Z,,test-image.png,test-image-2.png,new-tag,variant_01J3CSN79CQ2ND94SRJSXMEMNH,Test variant,,,,,false,true,,,,,,,,,,0,prod_01J3CSN791SN1RN7X155Z8S9CN,2024-07-22T08:41:47.053Z,2024-07-22T08:41:47.053Z,,100,45,30,size,large,color,green \ No newline at end of file diff --git a/integration-tests/http/__tests__/product/admin/__fixtures__/v1-template.csv b/integration-tests/http/__tests__/product/admin/__fixtures__/v1-template.csv new file mode 100644 index 0000000000..337cc96044 --- /dev/null +++ b/integration-tests/http/__tests__/product/admin/__fixtures__/v1-template.csv @@ -0,0 +1,6 @@ +Product Id,Product Handle,Product Title,Product Subtitle,Product Description,Product Status,Product Thumbnail,Product Weight,Product Length,Product Width,Product Height,Product HS Code,Product Origin Country,Product MID Code,Product Material,Product Collection Title,Product Collection Handle,Product Type,Product Tags,Product Discountable,Product External Id,Variant Id,Variant Title,Variant SKU,Variant Barcode,Variant Inventory Quantity,Variant Allow Backorder,Variant Manage Inventory,Variant Weight,Variant Length,Variant Width,Variant Height,Variant HS Code,Variant Origin Country,Variant MID Code,Variant Material,Price Test region [USD],Price USD,Option 1 Name,Option 1 Value,Option 2 Name,Option 2 Value,Image 1 Url +,test-product-product-1,Test product,,"Hopper Stripes Bedding, available as duvet cover, pillow sham and sheet.\n100% organic cotton, soft and crisp to the touch. Made in Portugal.",draft,,,,,,,,,,Test collection 1,test-collection1,,123_1,TRUE,,,Test variant,test-sku-1,test-barcode-1,10,FALSE,TRUE,,,,,,,,,1.00,1.10,test-option-1,option 1 value red,test-option-2,option 2 value 1,test-image.png +,test-product-product-1-1,Test product,,"Hopper Stripes Bedding, available as duvet cover, pillow sham and sheet.\n100% organic cotton, soft and crisp to the touch. Made in Portugal.",draft,,,,,,,,,,Test collection 1,test-collection1,,,TRUE,,,Test variant,test-sku-1-1,test-barcode-1-1,10,FALSE,TRUE,,,,,,,,,1.00,1.10,test-option-1,option 1 value red,test-option-2,option 2 value 1,test-image.png +existing-product-id,test-product-product-2,Test product,,test-product-description,draft,test-image.png,,,,,,,,,Test collection,test-collection2,test-type,123,TRUE,,,Test variant,test-sku-2,test-barcode-2,10,FALSE,TRUE,,,,,,,,,,1.10,Size,Small,,,test-image.png +existing-product-id,test-product-product-2,Test product,,test-product-description,draft,,,,,,,,,,Test collection,test-collection2,test-type,123,TRUE,,,Test variant,test-sku-3,test-barcode-3,10,FALSE,TRUE,,,,,,,,,,1.20,Size,Medium,,,test-image.png +existing-product-id,test-product-product-2,Test product,,test-product-description,draft,,,,,,,,,,Test collection,test-collection2,test-type,123,TRUE,,existing-variant-id,Test variant changed,test-sku-4,test-barcode-4,10,FALSE,TRUE,,,,,,,,,,,Size,Large,,,test-image.png \ No newline at end of file diff --git a/integration-tests/http/__tests__/product/admin/product-export.spec.ts b/integration-tests/http/__tests__/product/admin/product-export.spec.ts index 91655030cf..3350012672 100644 --- a/integration-tests/http/__tests__/product/admin/product-export.spec.ts +++ b/integration-tests/http/__tests__/product/admin/product-export.spec.ts @@ -194,7 +194,7 @@ medusaIntegrationTestRunner({ await compareCSVs( notifications[0].data.file.url, - path.join(__dirname, "__fixtures__", "exported-products.csv") + path.join(__dirname, "__fixtures__", "exported-products-comma.csv") ) await fs.rm(path.dirname(notifications[0].data.file.url), { force: true, diff --git a/integration-tests/http/__tests__/product/admin/product-import.spec.ts b/integration-tests/http/__tests__/product/admin/product-import.spec.ts index 829b331fff..371d56f844 100644 --- a/integration-tests/http/__tests__/product/admin/product-import.spec.ts +++ b/integration-tests/http/__tests__/product/admin/product-import.spec.ts @@ -31,6 +31,7 @@ medusaIntegrationTestRunner({ let baseCollection let baseType let baseProduct + let baseRegion let eventBus: IEventBusModuleService beforeAll(async () => { @@ -64,6 +65,17 @@ medusaIntegrationTestRunner({ adminHeaders ) ).data.product + + baseRegion = ( + await api.post( + "/admin/regions", + { + name: "Test region", + currency_code: "USD", + }, + adminHeaders + ) + ).data.region }) afterEach(() => { @@ -71,255 +83,268 @@ medusaIntegrationTestRunner({ }) describe("POST /admin/products/export", () => { - it("should import a previously exported products CSV file", async () => { - const subscriberExecution = TestEventUtils.waitSubscribersExecution( - "notification.notification.created", - eventBus - ) + // We want to ensure files with different delimiters are supported + ;[ + { file: "exported-products-comma.csv", name: "delimited with comma" }, + { + file: "exported-products-semicolon.csv", + name: "delimited with semicolon", + }, + ].forEach((testcase) => { + it(`should import a previously exported products CSV file ${testcase.name}`, async () => { + const subscriberExecution = TestEventUtils.waitSubscribersExecution( + "notification.notification.created", + eventBus + ) - let fileContent = await fs.readFile( - path.join(__dirname, "__fixtures__", "exported-products.csv"), - { encoding: "utf-8" } - ) + let fileContent = await fs.readFile( + path.join(__dirname, "__fixtures__", testcase.file), + { encoding: "utf-8" } + ) - fileContent = fileContent.replace( - /prod_01J3CRPNVGRZ01A8GH8FQYK10Z/g, - baseProduct.id - ) - fileContent = fileContent.replace( - /variant_01J3CRPNW5J6EBVVQP1TN33A58/g, - baseProduct.variants[0].id - ) - fileContent = fileContent.replace(/pcol_\w*\d*/g, baseCollection.id) - fileContent = fileContent.replace(/ptyp_\w*\d*/g, baseType.id) + fileContent = fileContent.replace( + /prod_01J3CRPNVGRZ01A8GH8FQYK10Z/g, + baseProduct.id + ) + fileContent = fileContent.replace( + /variant_01J3CRPNW5J6EBVVQP1TN33A58/g, + baseProduct.variants[0].id + ) + fileContent = fileContent.replace(/pcol_\w*\d*/g, baseCollection.id) + fileContent = fileContent.replace(/ptyp_\w*\d*/g, baseType.id) - const { form, meta } = getUploadReq({ - name: "test.csv", - content: fileContent, - }) - - // BREAKING: The batch endpoints moved to the domain routes (admin/batch-jobs -> /admin/products/import). The payload and response changed as well. - const batchJobRes = await api.post("/admin/products/import", form, meta) - - const transactionId = batchJobRes.data.transaction_id - expect(transactionId).toBeTruthy() - expect(batchJobRes.data.summary).toEqual({ - toCreate: 1, - toUpdate: 1, - }) - - await api.post( - `/admin/products/import/${transactionId}/confirm`, - {}, - meta - ) - - await subscriberExecution - const notifications = ( - await api.get("/admin/notifications", adminHeaders) - ).data.notifications - - expect(notifications.length).toBe(1) - expect(notifications[0]).toEqual( - expect.objectContaining({ - data: expect.objectContaining({ - title: "Product import", - description: `Product import of file test.csv completed successfully!`, - }), + const { form, meta } = getUploadReq({ + name: "test.csv", + content: fileContent, }) - ) - const dbProducts = (await api.get("/admin/products", adminHeaders)).data - .products + // BREAKING: The batch endpoints moved to the domain routes (admin/batch-jobs -> /admin/products/import). The payload and response changed as well. + const batchJobRes = await api.post( + "/admin/products/import", + form, + meta + ) - expect(dbProducts).toHaveLength(2) - expect(dbProducts).toEqual( - expect.arrayContaining([ + const transactionId = batchJobRes.data.transaction_id + expect(transactionId).toBeTruthy() + expect(batchJobRes.data.summary).toEqual({ + toCreate: 1, + toUpdate: 1, + }) + + await api.post( + `/admin/products/import/${transactionId}/confirm`, + {}, + meta + ) + + await subscriberExecution + const notifications = ( + await api.get("/admin/notifications", adminHeaders) + ).data.notifications + + expect(notifications.length).toBe(1) + expect(notifications[0]).toEqual( expect.objectContaining({ - id: baseProduct.id, - handle: "base-product", - is_giftcard: false, - thumbnail: "test-image.png", - status: "draft", - description: "test-product-description\ntest line 2", - options: [ - expect.objectContaining({ - title: "size", - values: expect.arrayContaining([ - expect.objectContaining({ - value: "small", - }), - expect.objectContaining({ - value: "large", - }), - ]), - }), - expect.objectContaining({ - title: "color", - values: expect.arrayContaining([ - expect.objectContaining({ - value: "green", - }), - ]), - }), - ], - images: expect.arrayContaining([ - expect.objectContaining({ - url: "test-image.png", - }), - expect.objectContaining({ - url: "test-image-2.png", - }), - ]), - tags: [ - expect.objectContaining({ - value: "123", - }), - expect.objectContaining({ - value: "456", - }), - ], - type: expect.objectContaining({ - id: baseType.id, + data: expect.objectContaining({ + title: "Product import", + description: `Product import of file test.csv completed successfully!`, }), - collection: expect.objectContaining({ - id: baseCollection.id, + }) + ) + + const dbProducts = (await api.get("/admin/products", adminHeaders)) + .data.products + + expect(dbProducts).toHaveLength(2) + expect(dbProducts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: baseProduct.id, + handle: "base-product", + is_giftcard: false, + thumbnail: "test-image.png", + status: "draft", + description: "test-product-description\ntest line 2", + options: [ + expect.objectContaining({ + title: "size", + values: expect.arrayContaining([ + expect.objectContaining({ + value: "small", + }), + expect.objectContaining({ + value: "large", + }), + ]), + }), + expect.objectContaining({ + title: "color", + values: expect.arrayContaining([ + expect.objectContaining({ + value: "green", + }), + ]), + }), + ], + images: expect.arrayContaining([ + expect.objectContaining({ + url: "test-image.png", + }), + expect.objectContaining({ + url: "test-image-2.png", + }), + ]), + tags: [ + expect.objectContaining({ + value: "123", + }), + expect.objectContaining({ + value: "456", + }), + ], + type: expect.objectContaining({ + id: baseType.id, + }), + collection: expect.objectContaining({ + id: baseCollection.id, + }), + variants: [ + expect.objectContaining({ + title: "Test variant", + allow_backorder: false, + manage_inventory: true, + prices: [ + expect.objectContaining({ + currency_code: "usd", + amount: 100, + }), + expect.objectContaining({ + currency_code: "eur", + amount: 45, + }), + expect.objectContaining({ + currency_code: "dkk", + amount: 30, + }), + ], + options: [ + expect.objectContaining({ + value: "large", + }), + expect.objectContaining({ + value: "green", + }), + ], + }), + expect.objectContaining({ + title: "Test variant 2", + allow_backorder: false, + manage_inventory: true, + prices: [ + expect.objectContaining({ + currency_code: "usd", + amount: 200, + }), + expect.objectContaining({ + currency_code: "eur", + amount: 65, + }), + expect.objectContaining({ + currency_code: "dkk", + amount: 50, + }), + ], + options: [ + expect.objectContaining({ + value: "small", + }), + expect.objectContaining({ + value: "green", + }), + ], + }), + ], + created_at: expect.any(String), + updated_at: expect.any(String), }), - variants: [ - expect.objectContaining({ - title: "Test variant", - allow_backorder: false, - manage_inventory: true, - prices: [ - expect.objectContaining({ - currency_code: "usd", - amount: 100, - }), - expect.objectContaining({ - currency_code: "eur", - amount: 45, - }), - expect.objectContaining({ - currency_code: "dkk", - amount: 30, - }), - ], - options: [ - expect.objectContaining({ - value: "large", - }), - expect.objectContaining({ - value: "green", - }), - ], + expect.objectContaining({ + id: expect.any(String), + handle: "proposed-product", + is_giftcard: false, + thumbnail: "test-image.png", + status: "proposed", + description: "test-product-description", + options: [ + expect.objectContaining({ + title: "size", + values: expect.arrayContaining([ + expect.objectContaining({ + value: "large", + }), + ]), + }), + expect.objectContaining({ + title: "color", + values: expect.arrayContaining([ + expect.objectContaining({ + value: "green", + }), + ]), + }), + ], + images: expect.arrayContaining([ + expect.objectContaining({ + url: "test-image.png", + }), + expect.objectContaining({ + url: "test-image-2.png", + }), + ]), + tags: [ + expect.objectContaining({ + value: "new-tag", + }), + ], + type: expect.objectContaining({ + id: baseType.id, }), - expect.objectContaining({ - title: "Test variant 2", - allow_backorder: false, - manage_inventory: true, - prices: [ - expect.objectContaining({ - currency_code: "usd", - amount: 200, - }), - expect.objectContaining({ - currency_code: "eur", - amount: 65, - }), - expect.objectContaining({ - currency_code: "dkk", - amount: 50, - }), - ], - options: [ - expect.objectContaining({ - value: "small", - }), - expect.objectContaining({ - value: "green", - }), - ], - }), - ], - created_at: expect.any(String), - updated_at: expect.any(String), - }), - expect.objectContaining({ - id: expect.any(String), - handle: "proposed-product", - is_giftcard: false, - thumbnail: "test-image.png", - status: "proposed", - description: "test-product-description", - options: [ - expect.objectContaining({ - title: "size", - values: expect.arrayContaining([ - expect.objectContaining({ - value: "large", - }), - ]), - }), - expect.objectContaining({ - title: "color", - values: expect.arrayContaining([ - expect.objectContaining({ - value: "green", - }), - ]), - }), - ], - images: expect.arrayContaining([ - expect.objectContaining({ - url: "test-image.png", - }), - expect.objectContaining({ - url: "test-image-2.png", - }), - ]), - tags: [ - expect.objectContaining({ - value: "new-tag", - }), - ], - type: expect.objectContaining({ - id: baseType.id, + collection: null, + variants: [ + expect.objectContaining({ + title: "Test variant", + allow_backorder: false, + manage_inventory: true, + prices: [ + expect.objectContaining({ + currency_code: "usd", + amount: 100, + }), + expect.objectContaining({ + currency_code: "eur", + amount: 45, + }), + expect.objectContaining({ + currency_code: "dkk", + amount: 30, + }), + ], + options: [ + expect.objectContaining({ + value: "large", + }), + expect.objectContaining({ + value: "green", + }), + ], + }), + ], + created_at: expect.any(String), + updated_at: expect.any(String), }), - collection: null, - variants: [ - expect.objectContaining({ - title: "Test variant", - allow_backorder: false, - manage_inventory: true, - prices: [ - expect.objectContaining({ - currency_code: "usd", - amount: 100, - }), - expect.objectContaining({ - currency_code: "eur", - amount: 45, - }), - expect.objectContaining({ - currency_code: "dkk", - amount: 30, - }), - ], - options: [ - expect.objectContaining({ - value: "large", - }), - expect.objectContaining({ - value: "green", - }), - ], - }), - ], - created_at: expect.any(String), - updated_at: expect.any(String), - }), - ]) - ) + ]) + ) + }) }) it("should fail on invalid region in prices being present in the CSV", async () => { @@ -337,7 +362,7 @@ medusaIntegrationTestRunner({ .post("/admin/products/import", form, meta) .catch((e) => e) expect(err.response.data.message).toEqual( - "Region with ID reg_nonexistent not found" + "Region with name nonexistent not found" ) }) @@ -439,6 +464,251 @@ medusaIntegrationTestRunner({ }) ) }) + + it("supports importing the v1 template", async () => { + const subscriberExecution = TestEventUtils.waitSubscribersExecution( + "notification.notification.created", + eventBus + ) + + let fileContent = await fs.readFile( + path.join(__dirname, "__fixtures__", "v1-template.csv"), + { encoding: "utf-8" } + ) + + fileContent = fileContent.replace( + /existing-product-id/g, + baseProduct.id + ) + fileContent = fileContent.replace( + /existing-variant-id/g, + baseProduct.variants[0].id + ) + fileContent = fileContent.replace(/test-type/g, baseType.value) + fileContent = fileContent.replace( + /test-collection1/g, + baseCollection.handle + ) + fileContent = fileContent.replace( + /test-collection2/g, + baseCollection.handle + ) + + const { form, meta } = getUploadReq({ + name: "test.csv", + content: fileContent, + }) + + const batchJobRes = await api.post("/admin/products/import", form, meta) + const transactionId = batchJobRes.data.transaction_id + expect(transactionId).toBeTruthy() + expect(batchJobRes.data.summary).toEqual({ + toCreate: 2, + toUpdate: 1, + }) + + await api.post( + `/admin/products/import/${transactionId}/confirm`, + {}, + meta + ) + + await subscriberExecution + const dbProducts = (await api.get("/admin/products", adminHeaders)).data + .products + + expect(dbProducts).toHaveLength(3) + expect(dbProducts).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: baseProduct.id, + handle: "test-product-product-2", + title: "Test product", + is_giftcard: false, + thumbnail: "test-image.png", + status: "draft", + description: "test-product-description", + options: [ + expect.objectContaining({ + title: "Size", + values: expect.arrayContaining([ + expect.objectContaining({ + value: "Small", + }), + expect.objectContaining({ + value: "Medium", + }), + expect.objectContaining({ + value: "Large", + }), + ]), + }), + ], + images: expect.arrayContaining([ + expect.objectContaining({ + url: "test-image.png", + }), + ]), + tags: [ + expect.objectContaining({ + value: "123", + }), + ], + type: expect.objectContaining({ + id: baseType.id, + }), + collection: expect.objectContaining({ + id: baseCollection.id, + }), + variants: [ + expect.objectContaining({ + title: "Test variant", + sku: "test-sku-2", + barcode: "test-barcode-2", + allow_backorder: false, + manage_inventory: true, + prices: [ + expect.objectContaining({ + currency_code: "usd", + amount: 1.1, + }), + ], + options: [ + expect.objectContaining({ + value: "Small", + }), + ], + }), + expect.objectContaining({ + title: "Test variant", + sku: "test-sku-3", + barcode: "test-barcode-3", + allow_backorder: false, + manage_inventory: true, + prices: [ + expect.objectContaining({ + currency_code: "usd", + amount: 1.2, + }), + ], + options: [ + expect.objectContaining({ + value: "Medium", + }), + ], + }), + expect.objectContaining({ + id: baseProduct.variants[0].id, + title: "Test variant changed", + sku: "test-sku-4", + barcode: "test-barcode-4", + allow_backorder: false, + manage_inventory: true, + prices: [], + options: [ + expect.objectContaining({ + value: "Large", + }), + ], + }), + ], + created_at: expect.any(String), + updated_at: expect.any(String), + }), + expect.objectContaining({ + id: expect.any(String), + title: "Test product", + handle: "test-product-product-1-1", + is_giftcard: false, + thumbnail: "test-image.png", + status: "draft", + description: + "Hopper Stripes Bedding, available as duvet cover, pillow sham and sheet.\\n100% organic cotton, soft and crisp to the touch. Made in Portugal.", + options: [ + expect.objectContaining({ + title: "test-option-1", + values: expect.arrayContaining([ + expect.objectContaining({ + value: "option 1 value red", + }), + ]), + }), + expect.objectContaining({ + title: "test-option-2", + values: expect.arrayContaining([ + expect.objectContaining({ + value: "option 2 value 1", + }), + ]), + }), + ], + images: expect.arrayContaining([ + expect.objectContaining({ + url: "test-image.png", + }), + ]), + tags: [], + type: null, + variants: [ + expect.objectContaining({ + title: "Test variant", + sku: "test-sku-1-1", + barcode: "test-barcode-1-1", + allow_backorder: true, + manage_inventory: true, + prices: [ + expect.objectContaining({ + currency_code: "usd", + rules: { + region_id: baseRegion.id, + }, + amount: 1, + }), + expect.objectContaining({ + currency_code: "usd", + amount: 1.1, + }), + ], + options: [ + expect.objectContaining({ + value: "option 1 value red", + }), + expect.objectContaining({ + value: "option 2 value 1", + }), + ], + }), + ], + created_at: expect.any(String), + updated_at: expect.any(String), + }), + expect.objectContaining({ + id: expect.any(String), + title: "Test product", + handle: "test-product-product-1", + }), + ]) + ) + }) + + it("should fail when a v1 import has non existent collection", async () => { + let fileContent = await fs.readFile( + path.join(__dirname, "__fixtures__", "v1-template.csv"), + { encoding: "utf-8" } + ) + + const { form, meta } = getUploadReq({ + name: "test.csv", + content: fileContent, + }) + + const err = await api + .post("/admin/products/import", form, meta) + .catch((e) => e) + expect(err.response.data.message).toEqual( + "Product collection with handle 'test-collection1' does not exist" + ) + }) }) }, }) diff --git a/packages/core/core-flows/src/product/helpers/normalize-for-import.ts b/packages/core/core-flows/src/product/helpers/normalize-for-import.ts index adf13b26a5..e2801556a8 100644 --- a/packages/core/core-flows/src/product/helpers/normalize-for-import.ts +++ b/packages/core/core-flows/src/product/helpers/normalize-for-import.ts @@ -13,7 +13,9 @@ export const normalizeForImport = ( variants: HttpTypes.AdminCreateProductVariant[] } >() - const regionsMap = new Map(regions.map((r) => [r.id, r])) + + // Currently region names are treated as case-insensitive. + const regionsMap = new Map(regions.map((r) => [r.name.toLowerCase(), r])) rawProducts.forEach((rawProduct) => { const productInMap = productMap.get(rawProduct["Product Handle"]) @@ -144,18 +146,19 @@ const normalizeVariantForImport = ( if (normalizedKey.startsWith("variant_price_")) { const priceKey = normalizedKey.replace("variant_price_", "") - // Note: If we start using the region name instead of ID, this check might not always work. - if (priceKey.length === 3) { + // Note: Region prices should always have the currency in brackets, eg. "variant_price_region_name_[EUR]" + if (!priceKey.endsWith("]")) { response["prices"] = [ ...(response["prices"] || []), { currency_code: priceKey.toLowerCase(), amount: normalizedValue }, ] } else { - const region = regionsMap.get(priceKey) + const regionName = priceKey.split("_").slice(0, -1).join(" ") + const region = regionsMap.get(regionName) if (!region) { throw new MedusaError( MedusaError.Types.INVALID_DATA, - `Region with ID ${priceKey} not found` + `Region with name ${regionName} not found` ) } @@ -164,7 +167,7 @@ const normalizeVariantForImport = ( { amount: normalizedValue, currency_code: region.currency_code, - rules: { region_id: priceKey }, + rules: { region_id: region.id }, }, ] } @@ -219,7 +222,7 @@ const normalizeVariantForImport = ( const getNormalizedValue = (key: string, value: any): any => { return stringFields.some((field) => key.startsWith(field)) - ? value.toString() + ? value?.toString() : value } diff --git a/packages/core/core-flows/src/product/helpers/normalize-v1-import.ts b/packages/core/core-flows/src/product/helpers/normalize-v1-import.ts new file mode 100644 index 0000000000..7cb43ca6c1 --- /dev/null +++ b/packages/core/core-flows/src/product/helpers/normalize-v1-import.ts @@ -0,0 +1,142 @@ +import { ProductTypes, SalesChannelTypes } from "@medusajs/types" +import { MedusaError } from "@medusajs/utils" + +const basicFieldsToOmit = [ + // Fields with slightly different naming + "Product MID Code", + "Product HS Code", + "Variant MID Code", + "Variant HS Code", + "Variant EAN", + "Variant UPC", + "Variant SKU", + + // Fields no longer present in v2 + "Variant Inventory Quantity", + "Product Profile Name", + "Product Profile Type", + + // Fields that are remapped + "Product Collection Handle", + "Product Collection Title", + "Product Type", + "Product Tags", +] + +// This is primarily to have backwards compatibility with v1 exports +// Although it also makes v2 import template more dynamic +// it's better to not expose eg. "Product MID Code" as an available public API so we can remove this code at some point. +export const normalizeV1Products = ( + rawProducts: object[], + supportingData: { + productTypes: ProductTypes.ProductTypeDTO[] + productCollections: ProductTypes.ProductCollectionDTO[] + salesChannels: SalesChannelTypes.SalesChannelDTO[] + } +): object[] => { + const productTypesMap = new Map( + supportingData.productTypes.map((pt) => [pt.value, pt.id]) + ) + const productCollectionsMap = new Map( + supportingData.productCollections.map((pc) => [pc.handle, pc.id]) + ) + const salesChannelsMap = new Map( + supportingData.salesChannels.map((sc) => [sc.name, sc.id]) + ) + + return rawProducts.map((product) => { + let finalRes = { + ...product, + "Product Mid Code": + product["Product MID Code"] ?? product["Product Mid Code"], + "Product Hs Code": + product["Product HS Code"] ?? product["Product Hs Code"], + "Variant MID Code": + product["Variant MID Code"] ?? product["Variant Mid Code"], + "Variant Hs Code": + product["Variant HS Code"] ?? product["Variant Hs Code"], + "Variant Ean": product["Variant EAN"] ?? product["Variant Ean"], + "Variant Upc": product["Variant UPC"] ?? product["Variant Upc"], + "Variant Sku": product["Variant SKU"] ?? product["Variant Sku"], + } + + basicFieldsToOmit.forEach((field) => { + delete finalRes[field] + }) + + // You can either pass "Product Tags" or "Product Tag ", but not both + const tags = product["Product Tags"]?.toString()?.split(",") + if (tags) { + finalRes = { + ...finalRes, + ...tags.reduce((agg, tag, i) => { + agg[`Product Tag ${i + 1}`] = tag + return agg + }, {}), + } + } + + const productTypeValue = product["Product Type"] + if (productTypeValue) { + if (!productTypesMap.has(productTypeValue)) { + throw new MedusaError( + MedusaError.Types.INVALID_DATA, + `Product type with value '${productTypeValue}' does not exist` + ) + } + + finalRes["Product Type Id"] = productTypesMap.get(productTypeValue) + } + + const productCollectionHandle = product["Product Collection Handle"] + if (productCollectionHandle) { + if (!productCollectionsMap.has(productCollectionHandle)) { + throw new MedusaError( + MedusaError.Types.INVALID_DATA, + `Product collection with handle '${productCollectionHandle}' does not exist` + ) + } + + finalRes["Product Collection Id"] = productCollectionsMap.get( + productCollectionHandle + ) + } + + // We have to iterate over all fields for the ones that are index-based + Object.entries(finalRes).forEach(([key, value]) => { + if (key.startsWith("Price")) { + delete finalRes[key] + finalRes[`Variant ${key}`] = value + } + + if (key.startsWith("Option")) { + delete finalRes[key] + finalRes[`Variant ${key}`] = value + } + + if (key.startsWith("Image")) { + delete finalRes[key] + finalRes[`Product Image ${key.split(" ")[1]}`] = value + } + + if (key.startsWith("Sales Channel")) { + delete finalRes[key] + if (key.endsWith("Id")) { + if (!salesChannelsMap.has(value)) { + throw new MedusaError( + MedusaError.Types.INVALID_DATA, + `Sales channel with name '${value}' does not exist` + ) + } + + finalRes[`Product Sales Channel ${key.split(" ")[2]}`] = + salesChannelsMap.get(value) + } + } + + // Note: Product categories from v1 are not imported to v2 + }) + + return finalRes + }) +} diff --git a/packages/core/core-flows/src/product/steps/group-products-for-batch.ts b/packages/core/core-flows/src/product/steps/group-products-for-batch.ts index 3b860ed171..64344b27cf 100644 --- a/packages/core/core-flows/src/product/steps/group-products-for-batch.ts +++ b/packages/core/core-flows/src/product/steps/group-products-for-batch.ts @@ -1,25 +1,26 @@ -import { HttpTypes, IProductModuleService, ProductTypes } from "@medusajs/types" -import { MedusaError, ModuleRegistrationName } from "@medusajs/utils" +import { HttpTypes, IProductModuleService } from "@medusajs/types" +import { ModuleRegistrationName } from "@medusajs/utils" import { StepResponse, createStep } from "@medusajs/workflows-sdk" export const groupProductsForBatchStepId = "group-products-for-batch" export const groupProductsForBatchStep = createStep( groupProductsForBatchStepId, - async (data: HttpTypes.AdminCreateProduct[], { container }) => { + async ( + data: (HttpTypes.AdminCreateProduct & { id?: string })[], + { container } + ) => { const service = container.resolve( ModuleRegistrationName.PRODUCT ) const existingProducts = await service.listProducts( { - // We already validate that there is handle in a previous step - handle: data.map((product) => product.handle) as string[], + // We use the ID to do product updates + id: data.map((product) => product.id).filter(Boolean) as string[], }, { take: null, select: ["handle"] } ) - const existingProductsMap = new Map( - existingProducts.map((p) => [p.handle, true]) - ) + const existingProductsSet = new Set(existingProducts.map((p) => p.id)) const { toUpdate, toCreate } = data.reduce( ( @@ -30,14 +31,7 @@ export const groupProductsForBatchStep = createStep( product ) => { // There are few data normalizations to do if we are dealing with an update. - if (existingProductsMap.has(product.handle!)) { - if (!(product as any).id) { - throw new MedusaError( - MedusaError.Types.INVALID_DATA, - "Product id is required when updating products in import" - ) - } - + if (product.id && existingProductsSet.has(product.id)) { acc.toUpdate.push( product as HttpTypes.AdminUpdateProduct & { id: string } ) @@ -46,7 +40,7 @@ export const groupProductsForBatchStep = createStep( // New products will be created with a new ID, even if there is one present in the CSV. // To add support for creating with predefined IDs we will need to do changes to the upsert method. - delete (product as any).id + delete product.id acc.toCreate.push(product) return acc }, diff --git a/packages/core/core-flows/src/product/steps/parse-product-csv.ts b/packages/core/core-flows/src/product/steps/parse-product-csv.ts index 0bbc1dbb70..49aaadea0a 100644 --- a/packages/core/core-flows/src/product/steps/parse-product-csv.ts +++ b/packages/core/core-flows/src/product/steps/parse-product-csv.ts @@ -5,7 +5,12 @@ import { } from "@medusajs/utils" import { StepResponse, createStep } from "@medusajs/workflows-sdk" import { normalizeForImport } from "../helpers/normalize-for-import" -import { IRegionModuleService } from "@medusajs/types" +import { + IProductModuleService, + IRegionModuleService, + ISalesChannelModuleService, +} from "@medusajs/types" +import { normalizeV1Products } from "../helpers/normalize-v1-import" export const parseProductCsvStepId = "parse-product-csv" export const parseProductCsvStep = createStep( @@ -14,9 +19,31 @@ export const parseProductCsvStep = createStep( const regionService = container.resolve( ModuleRegistrationName.REGION ) + const productService = container.resolve( + ModuleRegistrationName.PRODUCT + ) + const salesChannelService = container.resolve( + ModuleRegistrationName.SALES_CHANNEL + ) + const csvProducts = convertCsvToJson(fileContent) - csvProducts.forEach((product: any) => { + const [productTypes, productCollections, salesChannels] = await Promise.all( + [ + productService.listProductTypes({}, { take: null }), + productService.listProductCollections({}, { take: null }), + salesChannelService.listSalesChannels({}, { take: null }), + ] + ) + + const v1Normalized = normalizeV1Products(csvProducts, { + productTypes, + productCollections, + salesChannels, + }) + + // We use the handle to group products and variants correctly. + v1Normalized.forEach((product: any) => { if (!product["Product Handle"]) { throw new MedusaError( MedusaError.Types.INVALID_DATA, @@ -27,10 +54,10 @@ export const parseProductCsvStep = createStep( const allRegions = await regionService.listRegions( {}, - { select: ["id", "currency_code"], take: null } + { select: ["id", "name", "currency_code"], take: null } ) - const normalizedData = normalizeForImport(csvProducts, allRegions) + const normalizedData = normalizeForImport(v1Normalized, allRegions) return new StepResponse(normalizedData) } ) diff --git a/packages/core/utils/src/csv/csvtojson.ts b/packages/core/utils/src/csv/csvtojson.ts index 2c2f971867..90f6926e97 100644 --- a/packages/core/utils/src/csv/csvtojson.ts +++ b/packages/core/utils/src/csv/csvtojson.ts @@ -8,5 +8,22 @@ export const convertCsvToJson = ( ): T[] => { return csv2json(data, { preventCsvInjection: true, + delimiter: { field: detectDelimiter(data) }, }) as T[] } + +const delimiters = [",", ";", "|"] + +const detectDelimiter = (data: string) => { + let delimiter = "," + const header = data.split("\n")[0] + + for (const del of delimiters) { + if (header.split(del).length > 1) { + delimiter = del + break + } + } + + return delimiter +}