asyavee/medusa-store
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat: process import from pre-processed chunks (#12527)

Browse Source 
Fixes: FRMW-2968

In this PR we have done two major things.

- First, we remove storing CSV contents within the workflow storage and neither store the JSON payloads to be created/updated in workflows. Earlier, they all were workflow inputs, hence were stored in the workflow
- Introduce a naive concept of chunks and process chunks one by one. The next PR making chunking a bit more robust while using streams, adding ability to resume from the failed chunk and so on.

> [!IMPORTANT]  
> The new endpoint `/admin/product/imports` is not in use yet. But it will be after the next (final) PR.

## Old context in workflow storage

![CleanShot 2025-05-19 at 17 11 08@2x](https://github.com/user-attachments/assets/798bdcc9-a368-4c1f-afdd-2a77f5ce43e0)

## New context in workflow storage

![CleanShot 2025-05-19 at 17 15 08@2x](https://github.com/user-attachments/assets/0463d035-403f-4600-a9cd-5af24d5fee7c)
This commit is contained in:
Harminder Virk
2025-05-20 12:33:18 +00:00
committed by GitHub
parent 3e5794d657
commit fca5ad77b4
13 changed files with 868 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
integration-tests/http/__tests__/product/admin/product-export.spec.ts
+2 -1
View File
@@ -1,5 +1,6 @@
import { IEventBusModuleService } from "@medusajs/types"
import { CommonEvents, Modules } from "@medusajs/utils"
import os from "os"
import fs from "fs/promises"
import {
TestEventUtils,
@@ -16,7 +17,7 @@ import { csv2json } from "json-2-csv"
jest.setTimeout(50000)
const getCSVContents = async (filePath: string) => {
const asLocalPath = filePath.replace("http://localhost:9000", process.cwd())
const asLocalPath = filePath.replace("http://localhost:9000", os.tmpdir())
const fileContent = await fs.readFile(asLocalPath, { encoding: "utf-8" })
await fs.rm(path.dirname(asLocalPath), { recursive: true, force: true })
const csvRows = csv2json(fileContent)
integration-tests/http/__tests__/product/admin/product-import.spec.ts
-1
View File
@@ -55,7 +55,6 @@ function prepareCSVForImport(fileContents: string, delimiter: string = ",") {
}
medusaIntegrationTestRunner({
dbName: "bulk-uploads-local",
testSuite: ({ dbConnection, getContainer, api }) => {
let baseCollection
let baseType
integration-tests/http/__tests__/product/admin/product-imports.spec.ts
+610
View File
@@ -0,0 +1,610 @@
import fs from "fs/promises"
import path, { extname } from "path"
import { csv2json, json2csv } from "json-2-csv"
import { CommonEvents, Modules } from "@medusajs/utils"
import { IEventBusModuleService, IFileModuleService } from "@medusajs/types"
import {
TestEventUtils,
medusaIntegrationTestRunner,
} from "@medusajs/test-utils"
import {
adminHeaders,
createAdminUser,
} from "../../../../helpers/create-admin-user"
import { getProductFixture } from "../../../../helpers/fixtures"
const UNALLOWED_EXPORTED_COLUMNS = [
"Product Is Giftcard",
"Product Created At",
"Product Updated At",
"Product Deleted At",
"Variant Product Id",
"Variant Created At",
"Variant Updated At",
"Variant Deleted At",
]
jest.setTimeout(50000)
const getUploadReq = (file: { key: string; name: string; size: number }) => {
return {
body: {
file_key: file.key,
originalname: file.name,
extension: extname(file.name),
size: file.size,
mime_type: "text/csv",
},
meta: {
headers: {
...adminHeaders.headers,
},
},
}
}
function prepareCSVForImport(fileContents: string, delimiter: string = ",") {
const CSVFileAsJSON = csv2json(fileContents, {
delimiter: { field: delimiter },
})
CSVFileAsJSON.forEach((row) => {
UNALLOWED_EXPORTED_COLUMNS.forEach((col) => {
delete row[col]
})
})
return json2csv(CSVFileAsJSON)
}
medusaIntegrationTestRunner({
testSuite: ({ dbConnection, getContainer, api }) => {
let baseCollection
let baseType
let baseProduct
let baseRegion
let baseCategory
let baseTag1
let baseTag2
let baseTag3
let newTag
let shippingProfile
let eventBus: IEventBusModuleService
let fileModule: IFileModuleService
beforeAll(async () => {
eventBus = getContainer().resolve(Modules.EVENT_BUS)
fileModule = getContainer().resolve(Modules.FILE)
})
beforeEach(async () => {
await createAdminUser(dbConnection, adminHeaders, getContainer())
baseCollection = (
await api.post(
"/admin/collections",
{ title: "base-collection" },
adminHeaders
)
).data.collection
baseType = (
await api.post(
"/admin/product-types",
{ value: "test-type" },
adminHeaders
)
).data.product_type
baseTag1 = (
await api.post(
"/admin/product-tags",
{ value: "tag-123" },
adminHeaders
)
).data.product_tag
baseTag2 = (
await api.post(
"/admin/product-tags",
{ value: "tag-123_1" },
adminHeaders
)
).data.product_tag
baseTag3 = (
await api.post(
"/admin/product-tags",
{ value: "tag-456" },
adminHeaders
)
).data.product_tag
newTag = (
await api.post(
"/admin/product-tags",
{ value: "new-tag" },
adminHeaders
)
).data.product_tag
shippingProfile = (
await api.post(
`/admin/shipping-profiles`,
{ name: "Test", type: "default" },
adminHeaders
)
).data.shipping_profile
baseProduct = (
await api.post(
"/admin/products",
getProductFixture({
title: "Base product",
tags: [{ id: baseTag1.id }, { id: baseTag2.id }],
shipping_profile_id: shippingProfile.id,
}),
adminHeaders
)
).data.product
baseRegion = (
await api.post(
"/admin/regions",
{
name: "Test region",
currency_code: "USD",
},
adminHeaders
)
).data.region
baseCategory = (
await api.post(
"/admin/product-categories",
{ name: "Test", is_internal: false, is_active: true },
adminHeaders
)
).data.product_category
})
afterEach(() => {
;(eventBus as any).eventEmitter_.removeAllListeners()
})
describe("POST /admin/products/imports", () => {
// We want to ensure files with different delimiters are supported
;[
{
file: "products-comma.csv",
name: "delimited with comma",
delimiter: ",",
},
{
file: "products-semicolon.csv",
name: "delimited with semicolon",
delimiter: ";",
},
].forEach((testcase) => {
it(`should import a previously exported products CSV file ${testcase.name}`, async () => {
const subscriberExecution = TestEventUtils.waitSubscribersExecution(
`${Modules.NOTIFICATION}.notification.${CommonEvents.CREATED}`,
eventBus
)
let fileContent = await fs.readFile(
path.join(__dirname, "__fixtures__", testcase.file),
{ encoding: "utf-8" }
)
fileContent = fileContent.replace(
/prod_01J44RRJZ3M5F63NY82434RNM5/g,
baseProduct.id
)
fileContent = fileContent.replace(
/variant_01J44RRJZW1T9KQB6XG7Q6K61F/g,
baseProduct.variants[0].id
)
fileContent = fileContent.replace(/pcol_\w*\d*/g, baseCollection.id)
fileContent = fileContent.replace(/ptyp_\w*\d*/g, baseType.id)
fileContent = fileContent.replace(/tag-123/g, baseTag1.id)
fileContent = fileContent.replace(/tag-456/g, baseTag3.id)
fileContent = fileContent.replace(/new-tag/g, newTag.id)
fileContent = fileContent.replace(
/import-shipping-profile*/g,
shippingProfile.id
)
const csvContents = prepareCSVForImport(
fileContent,
testcase.delimiter
)
const { id } = await fileModule.createFiles({
filename: "test.csv",
content: csvContents,
mimeType: "text/csv",
})
const { body, meta } = getUploadReq({
name: "test.csv",
key: id,
size: csvContents.length,
})
// BREAKING: The batch endpoints moved to the domain routes (admin/batch-jobs -> /admin/products/import). The payload and response changed as well.
const batchJobRes = await api.post(
"/admin/products/imports",
body,
meta
)
const transactionId = batchJobRes.data.transaction_id
expect(transactionId).toBeTruthy()
expect(batchJobRes.data.summary).toEqual({
toCreate: 1,
toUpdate: 1,
})
await api.post(
`/admin/products/imports/${transactionId}/confirm`,
{},
meta
)
await subscriberExecution
const notifications = (
await api.get("/admin/notifications", adminHeaders)
).data.notifications
expect(notifications.length).toBe(1)
expect(notifications[0]).toEqual(
expect.objectContaining({
data: expect.objectContaining({
title: "Product import",
description: `Product import of file test.csv completed successfully!`,
}),
})
)
const dbProducts = (await api.get("/admin/products", adminHeaders))
.data.products
expect(dbProducts).toHaveLength(2)
expect(dbProducts[0]).toEqual(
expect.objectContaining({
id: baseProduct.id,
handle: "base-product",
is_giftcard: false,
thumbnail: "test-image.png",
status: "draft",
description: "test-product-description\ntest line 2",
options: expect.arrayContaining([
expect.objectContaining({
title: "size",
values: expect.arrayContaining([
expect.objectContaining({
value: "large",
}),
expect.objectContaining({
value: "small",
}),
]),
}),
expect.objectContaining({
title: "color",
values: expect.arrayContaining([
expect.objectContaining({
value: "green",
}),
]),
}),
]),
images: expect.arrayContaining([
expect.objectContaining({
url: "test-image.png",
}),
expect.objectContaining({
url: "test-image-2.png",
}),
]),
tags: [
expect.objectContaining({
id: baseTag1.id,
}),
expect.objectContaining({
id: baseTag3.id,
}),
],
type: expect.objectContaining({
id: baseType.id,
}),
collection: expect.objectContaining({
id: baseCollection.id,
}),
variants: expect.arrayContaining([
expect.objectContaining({
title: "Test variant",
allow_backorder: false,
manage_inventory: true,
prices: expect.arrayContaining([
expect.objectContaining({
currency_code: "dkk",
amount: 30,
}),
expect.objectContaining({
currency_code: "eur",
amount: 45,
}),
expect.objectContaining({
currency_code: "usd",
amount: 100,
}),
]),
options: expect.arrayContaining([
expect.objectContaining({
value: "large",
}),
expect.objectContaining({
value: "green",
}),
]),
}),
expect.objectContaining({
title: "Test variant 2",
allow_backorder: false,
manage_inventory: true,
prices: expect.arrayContaining([
expect.objectContaining({
currency_code: "dkk",
amount: 50,
}),
expect.objectContaining({
currency_code: "eur",
amount: 65,
}),
expect.objectContaining({
currency_code: "usd",
amount: 200,
}),
]),
options: expect.arrayContaining([
expect.objectContaining({
value: "small",
}),
expect.objectContaining({
value: "green",
}),
]),
}),
]),
created_at: expect.any(String),
updated_at: expect.any(String),
})
)
expect(dbProducts[1]).toEqual(
expect.objectContaining({
id: expect.any(String),
handle: "proposed-product",
is_giftcard: false,
thumbnail: "test-image.png",
status: "proposed",
description: "test-product-description",
options: expect.arrayContaining([
expect.objectContaining({
title: "size",
values: expect.arrayContaining([
expect.objectContaining({
value: "large",
}),
]),
}),
expect.objectContaining({
title: "color",
values: expect.arrayContaining([
expect.objectContaining({
value: "green",
}),
]),
}),
]),
images: expect.arrayContaining([
expect.objectContaining({
url: "test-image.png",
}),
expect.objectContaining({
url: "test-image-2.png",
}),
]),
tags: [
expect.objectContaining({
value: "new-tag",
}),
],
type: expect.objectContaining({
id: baseType.id,
}),
collection: null,
variants: expect.arrayContaining([
expect.objectContaining({
title: "Test variant",
allow_backorder: false,
manage_inventory: true,
prices: expect.arrayContaining([
expect.objectContaining({
currency_code: "dkk",
amount: 30,
}),
expect.objectContaining({
currency_code: "eur",
amount: 45,
}),
expect.objectContaining({
currency_code: "usd",
amount: 100,
}),
]),
options: expect.arrayContaining([
expect.objectContaining({
value: "large",
}),
expect.objectContaining({
value: "green",
}),
]),
}),
]),
created_at: expect.any(String),
updated_at: expect.any(String),
})
)
})
})
it("should import product with categories", async () => {
const subscriberExecution = TestEventUtils.waitSubscribersExecution(
`${Modules.NOTIFICATION}.notification.${CommonEvents.CREATED}`,
eventBus
)
let fileContent = await fs.readFile(
path.join(__dirname, "__fixtures__", "product-with-categories.csv"),
{ encoding: "utf-8" }
)
fileContent = fileContent.replace(/prod_\w*\d*/g, baseProduct.id)
fileContent = fileContent.replace(/pcol_\w*\d*/g, baseCollection.id)
fileContent = fileContent.replace(/ptyp_\w*\d*/g, baseType.id)
fileContent = fileContent.replace(/pcat_\w*\d*/g, baseCategory.id)
fileContent = fileContent.replace(/tag-123/g, baseTag1.id)
fileContent = fileContent.replace(/tag-456/g, baseTag3.id)
fileContent = fileContent.replace(/new-tag/g, newTag.id)
fileContent = fileContent.replace(
/import-shipping-profile*/g,
shippingProfile.id
)
const csvContents = prepareCSVForImport(fileContent, ",")
const { id } = await fileModule.createFiles({
filename: "test.csv",
content: csvContents,
mimeType: "text/csv",
})
const { body, meta } = getUploadReq({
name: "test.csv",
key: id,
size: csvContents.length,
})
const batchJobRes = await api.post(
"/admin/products/imports",
body,
meta
)
const transactionId = batchJobRes.data.transaction_id
expect(transactionId).toBeTruthy()
expect(batchJobRes.data.summary).toEqual({
toCreate: 0,
toUpdate: 1,
})
await api.post(
`/admin/products/imports/${transactionId}/confirm`,
{},
meta
)
await subscriberExecution
const dbProducts = (
await api.get("/admin/products?fields=*categories", adminHeaders)
).data.products
expect(dbProducts).toHaveLength(1)
expect(dbProducts[0]).toEqual(
expect.objectContaining({
id: baseProduct.id,
categories: [expect.objectContaining({ id: baseCategory.id })],
})
)
})
it("should complain about non-existent fields being present in the CSV", async () => {
let fileContent = await fs.readFile(
path.join(__dirname, "__fixtures__", "unrelated-column.csv"),
{ encoding: "utf-8" }
)
fileContent = fileContent.replace(/pcol_\w*\d*/g, baseCollection.id)
fileContent = fileContent.replace(/ptyp_\w*\d*/g, baseType.id)
fileContent = fileContent.replace(/tag-123/g, baseTag1.id)
fileContent = fileContent.replace(/tag-456/g, baseTag3.id)
fileContent = fileContent.replace(/new-tag/g, newTag.id)
fileContent = fileContent.replace(
/import-shipping-profile*/g,
shippingProfile.id
)
const csvContents = prepareCSVForImport(fileContent, ",")
const { id } = await fileModule.createFiles({
filename: "test.csv",
content: csvContents,
mimeType: "text/csv",
})
const { body, meta } = getUploadReq({
name: "test.csv",
key: id,
size: csvContents.length,
})
const batchJobRes = await api
.post("/admin/products/imports", body, meta)
.catch((e) => e)
expect(batchJobRes.response.data.message).toEqual(
'Invalid column name(s) "Some field"'
)
})
it("should successfully skip non-existent product fields being present in the CSV", async () => {
let fileContent = await fs.readFile(
path.join(__dirname, "__fixtures__", "invalid-column.csv"),
{ encoding: "utf-8" }
)
fileContent = fileContent.replace(/pcol_\w*\d*/g, baseCollection.id)
fileContent = fileContent.replace(/ptyp_\w*\d*/g, baseType.id)
fileContent = fileContent.replace(
/import-shipping-profile*/g,
shippingProfile.id
)
const csvContents = prepareCSVForImport(fileContent, ",")
const { id } = await fileModule.createFiles({
filename: "test.csv",
content: csvContents,
mimeType: "text/csv",
})
const { body, meta } = getUploadReq({
name: "test.csv",
key: id,
size: csvContents.length,
})
const batchJobRes = await api
.post("/admin/products/imports", body, meta)
.catch((e) => e)
expect(batchJobRes.response.data.message).toEqual(
'Invalid column name(s) "Product field"'
)
})
})
},
})
integration-tests/http/medusa-config.js
+1
View File
@@ -65,6 +65,7 @@ module.exports = defineConfig({
options: {
// This is the directory where we can reliably write in CI environments
upload_dir: path.join(os.tmpdir(), "uploads"),
private_upload_dir: path.join(os.tmpdir(), "static"),
},
},
],