feat: process import from pre-processed chunks (#12527)

Fixes: FRMW-2968

In this PR we have done two major things.

- First, we remove storing CSV contents within the workflow storage and neither store the JSON payloads to be created/updated in workflows. Earlier, they all were workflow inputs, hence were stored in the workflow
- Introduce a naive concept of chunks and process chunks one by one. The next PR making chunking a bit more robust while using streams, adding ability to resume from the failed chunk and so on.

> [!IMPORTANT]  
> The new endpoint `/admin/product/imports` is not in use yet. But it will be after the next (final) PR.

## Old context in workflow storage

![CleanShot 2025-05-19 at 17 11 08@2x](https://github.com/user-attachments/assets/798bdcc9-a368-4c1f-afdd-2a77f5ce43e0)

## New context in workflow storage

![CleanShot 2025-05-19 at 17 15 08@2x](https://github.com/user-attachments/assets/0463d035-403f-4600-a9cd-5af24d5fee7c)
This commit is contained in:
Harminder Virk
2025-05-20 18:03:18 +05:30
committed by GitHub
parent 3e5794d657
commit fca5ad77b4
13 changed files with 868 additions and 12 deletions

View File

@@ -4,7 +4,7 @@ import {
} from "@medusajs/framework/http"
import {
importProductsWorkflowId,
importProductsAsChunksWorkflowId,
waitConfirmationProductImportStepId,
} from "@medusajs/core-flows"
import { IWorkflowEngineService } from "@medusajs/framework/types"
@@ -28,7 +28,7 @@ export const POST = async (
action: TransactionHandlerType.INVOKE,
transactionId,
stepId: waitConfirmationProductImportStepId,
workflowId: importProductsWorkflowId,
workflowId: importProductsAsChunksWorkflowId,
},
stepResponse: new StepResponse(true),
})

View File

@@ -2,10 +2,9 @@ import {
MedusaResponse,
AuthenticatedMedusaRequest,
} from "@medusajs/framework/http"
import { Modules } from "@medusajs/framework/utils"
import type { HttpTypes } from "@medusajs/framework/types"
import { importProductsWorkflow } from "@medusajs/core-flows"
import type { AdminImportProductsType } from "../validators"
import { importProductsAsChunksWorkflow } from "@medusajs/core-flows"
/**
* @version 2.8.0
@@ -14,13 +13,12 @@ export const POST = async (
req: AuthenticatedMedusaRequest<AdminImportProductsType>,
res: MedusaResponse<HttpTypes.AdminImportProductResponse>
) => {
const fileProvider = req.scope.resolve(Modules.FILE)
const file = await fileProvider.getAsBuffer(req.validatedBody.file_key)
const { result, transaction } = await importProductsWorkflow(req.scope).run({
const { result, transaction } = await importProductsAsChunksWorkflow(
req.scope
).run({
input: {
filename: req.validatedBody.originalname,
fileContent: file.toString("utf-8"),
fileKey: req.validatedBody.file_key,
},
})