feat(medusa): Add batch strategy for order exports (#1603)

This commit is contained in:
Philip Korsholm
2022-06-29 09:54:37 +02:00
committed by GitHub
parent c0f624ad3b
commit bf47d1aecd
18 changed files with 1156 additions and 115 deletions
@@ -24,12 +24,16 @@ export default (app) => {
route.post("/", middlewares.wrap(require("./create-batch-job").default))
const batchJobRouter = Router({ mergeParams: true })
route.use("/:id", getRequestedBatchJob, canAccessBatchJob, batchJobRouter)
batchJobRouter.get("/", middlewares.wrap(require("./get-batch-job").default))
batchJobRouter.post(
"/confirm",
middlewares.wrap(require("./confirm-batch-job").default)
)
batchJobRouter.post(
"/cancel",
middlewares.wrap(require("./cancel-batch-job").default)
+8 -4
View File
@@ -5,12 +5,16 @@ import { ConfigModule, MedusaContainer } from "../types/global"
import { Logger } from "../types/global"
type Options = {
container: MedusaContainer;
configModule: ConfigModule;
logger: Logger;
container: MedusaContainer
configModule: ConfigModule
logger: Logger
}
async function redisLoader({ container, configModule, logger }: Options): Promise<void> {
async function redisLoader({
container,
configModule,
logger,
}: Options): Promise<void> {
if (configModule.projectConfig.redis_url) {
// Economical way of dealing with redis clients
const client = new RealRedis(configModule.projectConfig.redis_url)
+1 -1
View File
@@ -1,6 +1,6 @@
import glob from "glob"
import path from "path"
import { AwilixContainer, asFunction, aliasTo } from "awilix"
import { asFunction, aliasTo } from "awilix"
import formatRegistrationName from "../utils/format-registration-name"
import { isBatchJobStrategy } from "../interfaces"
+15 -4
View File
@@ -1,5 +1,16 @@
import { AfterLoad, BeforeInsert, Column, Entity, JoinColumn, ManyToOne } from "typeorm"
import { BatchJobResultError, BatchJobResultStatDescriptor, BatchJobStatus } from "../types/batch-job"
import {
AfterLoad,
BeforeInsert,
Column,
Entity,
JoinColumn,
ManyToOne,
} from "typeorm"
import {
BatchJobResultError,
BatchJobResultStatDescriptor,
BatchJobStatus,
} from "../types/batch-job"
import { DbAwareColumn, resolveDbType } from "../utils/db-aware-column"
import { SoftDeletableEntity } from "../interfaces/models/soft-deletable-entity"
import { generateEntityId } from "../utils/generate-entity-id"
@@ -33,7 +44,7 @@ export class BatchJob extends SoftDeletableEntity {
} & Record<string, unknown>
@Column({ type: "boolean", nullable: false, default: false })
dry_run: boolean = false;
dry_run = false
@Column({ type: resolveDbType("timestamptz"), nullable: true })
pre_processed_at?: Date
@@ -141,7 +152,7 @@ export class BatchJob extends SoftDeletableEntity {
* description: "The date of the completion."
* type: string
* format: date-time
* canceled_at:
* canceled_at:
* description: "The date of the concellation."
* type: string
* format: date-time
@@ -0,0 +1,74 @@
import { DeepPartial } from "typeorm"
import {
FulfillmentStatus,
Order,
OrderStatus,
PaymentStatus,
} from "../../models"
const createdAtDate = new Date("2019-01-01T00:00:00.000Z")
export const ordersToExport: DeepPartial<Order>[] = [
{
id: "order_1",
created_at: createdAtDate,
display_id: 123,
status: OrderStatus.PENDING,
fulfillment_status: FulfillmentStatus.PARTIALLY_FULFILLED,
payment_status: PaymentStatus.CAPTURED,
subtotal: 10,
shipping_total: 10,
discount_total: 0,
gift_card_total: 0,
refunded_total: 0,
tax_total: 5,
total: 25,
currency_code: "usd",
region_id: "region_1",
shipping_address: {
id: "address_1",
address_1: "123 Main St",
address_2: "",
city: "New York",
country_code: "US",
postal_code: "10001",
},
customer: {
id: "customer_1",
first_name: "John",
last_name: "Doe",
email: "John@Doe.com",
},
},
{
id: "order_2",
created_at: createdAtDate,
display_id: 124,
status: OrderStatus.COMPLETED,
fulfillment_status: FulfillmentStatus.FULFILLED,
payment_status: PaymentStatus.CAPTURED,
subtotal: 125,
shipping_total: 10,
discount_total: 0,
gift_card_total: 0,
refunded_total: 0,
tax_total: 0,
total: 135,
currency_code: "eur",
region_id: "region_2",
shipping_address: {
id: "address_2",
address_1: "Hovedgaden 1",
address_2: "",
city: "Copenhagen",
country_code: "DK",
postal_code: "1150",
},
customer: {
id: "customer_2",
first_name: "Jane",
last_name: "Doe",
email: "Jane@Doe.com",
},
},
]
@@ -0,0 +1,12 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP
exports[`Order export strategy should process the batch job and generate the appropriate output 1`] = `
Array [
"Order_ID;Display_ID;Order status;Date;Customer First name;Customer Last name;Customer Email;Customer ID;Shipping Address 1;Shipping Address 2;Shipping Country Code;Shipping City;Shipping Postal Code;Shipping Region ID;Fulfillment Status;Payment Status;Subtotal;Shipping Total;Discount Total;Gift Card Total;Refunded Total;Tax Total;Total;Currency Code
",
"order_1;123;pending;Tue, 01 Jan 2019 00:00:00 GMT;John;Doe;John@Doe.com;customer_1;123 Main St;;US;New York;10001;region_1;partially_fulfilled;captured;10;10;0;0;0;5;25;usd
",
"order_2;124;completed;Tue, 01 Jan 2019 00:00:00 GMT;Jane;Doe;Jane@Doe.com;customer_2;Hovedgaden 1;;DK;Copenhagen;1150;region_2;fulfilled;captured;125;10;0;0;0;0;135;eur
",
]
`;
@@ -0,0 +1,138 @@
import OrderExportStrategy from "../../../batch-jobs/order/export"
import { IdMap, MockManager } from "medusa-test-utils"
import { User } from "../../../../models"
import { BatchJobStatus } from "../../../../types/batch-job"
import { ordersToExport } from "../../../__fixtures__/order-export-data"
const outputDataStorage: string[] = []
let fakeJob = {
id: IdMap.getId("order-export-job"),
type: "order-export",
context: {
params: {},
list_config: {
select: [
"id",
"display_id",
"status",
"created_at",
"fulfillment_status",
"payment_status",
"subtotal",
"shipping_total",
"discount_total",
"gift_card_total",
"refunded_total",
"tax_total",
"total",
"currency_code",
"region_id",
],
relations: ["customer", "shipping_address"],
},
},
created_by: IdMap.getId("order-export-job-creator"),
created_by_user: {} as User,
result: {},
dry_run: false,
status: BatchJobStatus.PROCESSING,
}
const fileServiceMock = {
delete: jest.fn(),
withTransaction: function () {
return this
},
getUploadStreamDescriptor: jest.fn().mockImplementation(() => {
return Promise.resolve({
writeStream: {
write: (data: string) => {
outputDataStorage.push(data)
},
end: () => void 0,
},
promise: Promise.resolve(),
fileKey: "product-export.csv",
})
}),
}
const batchJobServiceMock = {
withTransaction: function (): any {
return this
},
update: jest.fn().mockImplementation(async (data) => {
fakeJob = {
...fakeJob,
...data,
}
return fakeJob
}),
complete: jest.fn().mockImplementation(async () => {
fakeJob.status = BatchJobStatus.COMPLETED
return fakeJob
}),
ready: jest.fn().mockImplementation(async () => {
fakeJob.status = BatchJobStatus.READY
return fakeJob
}),
retrieve: jest.fn().mockImplementation(async () => {
return fakeJob
}),
}
const orderServiceMock = {
withTransaction: function (): any {
return this
},
listAndCount: jest
.fn()
.mockImplementation(() =>
Promise.resolve([ordersToExport, ordersToExport.length])
),
list: jest.fn().mockImplementation(() => Promise.resolve(ordersToExport)),
}
describe("Order export strategy", () => {
const orderExportStrategy = new OrderExportStrategy({
batchJobService: batchJobServiceMock as any,
fileService: fileServiceMock as any,
orderService: orderServiceMock as any,
manager: MockManager,
})
it("Should generate header as template", async () => {
const template = await orderExportStrategy.buildTemplate()
expect(template.split(";")).toEqual([
"Order_ID",
"Display_ID",
"Order status",
"Date",
"Customer First name",
"Customer Last name",
"Customer Email",
"Customer ID",
"Shipping Address 1",
"Shipping Address 2",
"Shipping Country Code",
"Shipping City",
"Shipping Postal Code",
"Shipping Region ID",
"Fulfillment Status",
"Payment Status",
"Subtotal",
"Shipping Total",
"Discount Total",
"Gift Card Total",
"Refunded Total",
"Tax Total",
"Total",
"Currency Code\r\n",
])
})
it("should process the batch job and generate the appropriate output", async () => {
await orderExportStrategy.processJob(fakeJob.id)
expect(outputDataStorage).toMatchSnapshot()
})
})
@@ -0,0 +1,295 @@
import { EntityManager } from "typeorm"
import {
OrderDescriptor,
OrderExportBatchJob,
OrderExportBatchJobContext,
orderExportPropertiesDescriptors,
} from "."
import { AdminPostBatchesReq } from "../../../api/routes/admin/batch/create-batch-job"
import { IFileService } from "../../../interfaces"
import { AbstractBatchJobStrategy } from "../../../interfaces/batch-job-strategy"
import { Order } from "../../../models"
import { OrderService } from "../../../services"
import BatchJobService from "../../../services/batch-job"
import { BatchJobStatus } from "../../../types/batch-job"
import { prepareListQuery } from "../../../utils/get-query-config"
type InjectedDependencies = {
fileService: IFileService<any>
orderService: OrderService
batchJobService: BatchJobService
manager: EntityManager
}
class OrderExportStrategy extends AbstractBatchJobStrategy<OrderExportStrategy> {
public static identifier = "order-export-strategy"
public static batchType = "order-export"
public defaultMaxRetry = 3
protected readonly DEFAULT_LIMIT = 100
protected readonly NEWLINE = "\r\n"
protected readonly DELIMITER = ";"
protected manager_: EntityManager
protected transactionManager_: EntityManager | undefined
protected readonly fileService_: IFileService<any>
protected readonly batchJobService_: BatchJobService
protected readonly orderService_: OrderService
protected readonly defaultRelations_ = ["customer", "shipping_address"]
protected readonly defaultFields_ = [
"id",
"display_id",
"status",
"created_at",
"fulfillment_status",
"payment_status",
"subtotal",
"shipping_total",
"discount_total",
"gift_card_total",
"refunded_total",
"tax_total",
"total",
"currency_code",
"region_id",
]
constructor({
fileService,
batchJobService,
orderService,
manager,
}: InjectedDependencies) {
// eslint-disable-next-line prefer-rest-params
super(arguments[0])
this.manager_ = manager
this.fileService_ = fileService
this.batchJobService_ = batchJobService
this.orderService_ = orderService
}
async prepareBatchJobForProcessing(
batchJob: AdminPostBatchesReq,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
req: Express.Request
): Promise<AdminPostBatchesReq> {
const {
limit,
offset,
order,
fields,
expand,
filterable_fields,
...context
} = batchJob.context as OrderExportBatchJobContext
const listConfig = prepareListQuery(
{
limit,
offset,
order,
fields,
expand,
},
{
isList: true,
defaultRelations: this.defaultRelations_,
defaultFields: this.defaultFields_,
}
)
batchJob.context = {
...(context ?? {}),
list_config: listConfig,
filterable_fields,
}
return batchJob
}
async preProcessBatchJob(batchJobId: string): Promise<void> {
return await this.atomicPhase_(async (transactionManager) => {
const batchJob = (await this.batchJobService_
.withTransaction(transactionManager)
.retrieve(batchJobId)) as OrderExportBatchJob
const offset = batchJob.context?.list_config?.skip ?? 0
const limit = batchJob.context?.list_config?.take ?? this.DEFAULT_LIMIT
const { list_config = {}, filterable_fields = {} } = batchJob.context
let count = batchJob.context?.batch_size
if (!count) {
const [, orderCount] = await this.orderService_
.withTransaction(transactionManager)
.listAndCount(filterable_fields, {
...(list_config ?? {}),
skip: offset as number,
order: { created_at: "DESC" },
take: Math.min(batchJob.context.batch_size ?? Infinity, limit),
})
count = orderCount
}
await this.batchJobService_
.withTransaction(transactionManager)
.update(batchJob, {
result: {
stat_descriptors: [
{
key: "order-export-count",
name: "Order count to export",
message: `There will be ${count} orders exported by this action`,
},
],
},
})
})
}
async processJob(batchJobId: string): Promise<void> {
let offset = 0
let limit = this.DEFAULT_LIMIT
let advancementCount = 0
let orderCount = 0
let approximateFileSize = 0
return await this.atomicPhase_(
async (transactionManager) => {
let batchJob = (await this.batchJobService_
.withTransaction(transactionManager)
.retrieve(batchJobId)) as OrderExportBatchJob
const { writeStream, fileKey, promise } = await this.fileService_
.withTransaction(transactionManager)
.getUploadStreamDescriptor({
name: `exports/orders/order-export-${Date.now()}`,
ext: "csv",
})
advancementCount =
batchJob.result?.advancement_count ?? advancementCount
offset = (batchJob.context?.list_config?.skip ?? 0) + advancementCount
limit = batchJob.context?.list_config?.take ?? limit
const { list_config = {}, filterable_fields = {} } = batchJob.context
const [, count] = await this.orderService_.listAndCount(
filterable_fields,
{
...list_config,
order: { created_at: "DESC" },
skip: offset,
take: Math.min(batchJob.context.batch_size ?? Infinity, limit),
}
)
const lineDescriptor = this.getLineDescriptor(
list_config.select as string[],
list_config.relations as string[]
)
const header = this.buildHeader(lineDescriptor)
approximateFileSize += Buffer.from(header).byteLength
writeStream.write(header)
orderCount = batchJob.context?.batch_size ?? count
let orders = []
while (offset < orderCount) {
orders = await this.orderService_
.withTransaction(transactionManager)
.list(filterable_fields, {
...list_config,
skip: offset,
take: Math.min(orderCount - offset, limit),
})
orders.forEach((order) => {
const line = this.buildCSVLine(order, lineDescriptor)
approximateFileSize += Buffer.from(line).byteLength
writeStream.write(line)
})
advancementCount += orders.length
offset += orders.length
batchJob = (await this.batchJobService_
.withTransaction(transactionManager)
.update(batchJobId, {
result: {
file_key: fileKey,
file_size: approximateFileSize,
count: orderCount,
advancement_count: advancementCount,
progress: advancementCount / orderCount,
},
})) as OrderExportBatchJob
if (batchJob.status === BatchJobStatus.CANCELED) {
writeStream.end()
await this.fileService_
.withTransaction(transactionManager)
.delete({ key: fileKey })
return
}
}
writeStream.end()
await promise
},
"REPEATABLE READ",
async (err: Error) => {
this.handleProcessingError(batchJobId, err, {
offset,
count: orderCount,
progress: offset / orderCount,
})
}
)
}
public async buildTemplate(): Promise<string> {
return this.buildHeader(
this.getLineDescriptor(this.defaultFields_, this.defaultRelations_)
)
}
private buildHeader(
lineDescriptor: OrderDescriptor[] = orderExportPropertiesDescriptors
): string {
return (
[...lineDescriptor.map(({ title }) => title)].join(this.DELIMITER) +
this.NEWLINE
)
}
private buildCSVLine(
order: Order,
lineDescriptor: OrderDescriptor[]
): string {
return (
[...lineDescriptor.map(({ accessor }) => accessor(order))].join(
this.DELIMITER
) + this.NEWLINE
)
}
private getLineDescriptor(
fields: string[],
relations: string[]
): OrderDescriptor[] {
return orderExportPropertiesDescriptors.filter(
({ fieldName }) =>
fields.indexOf(fieldName) !== -1 || relations.indexOf(fieldName) !== -1
)
}
}
export default OrderExportStrategy
@@ -0,0 +1,153 @@
import { BatchJob, Order } from "../../../models"
import { Selector } from "../../../types/common"
export type OrderExportBatchJobContext = {
offset?: number
limit?: number
order?: string
fields?: string
expand?: string
list_config?: {
select?: string[]
relations?: string[]
skip?: number
take?: number
order?: Record<string, "ASC" | "DESC">
}
filterable_fields?: Selector<Order>
retry_count?: number
max_retry?: number
batch_size?: number
}
export type OrderExportBatchJob = BatchJob & {
context: OrderExportBatchJobContext
}
export type OrderDescriptor = {
fieldName: string
title: string
accessor: (entity: Order) => string
}
export const orderExportPropertiesDescriptors: OrderDescriptor[] = [
{
fieldName: "id",
title: "Order_ID",
accessor: (order: Order): string => order.id,
},
{
fieldName: "display_id",
title: "Display_ID",
accessor: (order: Order): string => order.display_id.toString(),
},
{
fieldName: "status",
title: "Order status",
accessor: (order: Order): string => order.status.toString(),
},
{
fieldName: "created_at",
title: "Date",
accessor: (order: Order): string => order.created_at.toUTCString(),
},
{
fieldName: "customer",
title: [
"Customer First name",
"Customer Last name",
"Customer Email",
"Customer ID",
].join(";"),
accessor: (order: Order): string =>
[
order.customer.first_name,
order.customer.last_name,
order.customer.email,
order.customer.id,
].join(";"),
},
{
fieldName: "shipping_address",
title: [
"Shipping Address 1",
"Shipping Address 2",
"Shipping Country Code",
"Shipping City",
"Shipping Postal Code",
"Shipping Region ID",
].join(";"),
accessor: (order: Order): string =>
[
order.shipping_address?.address_1,
order.shipping_address?.address_2,
order.shipping_address?.country_code,
order.shipping_address?.city,
order.shipping_address?.postal_code,
order.region_id,
].join(";"),
},
{
fieldName: "fulfillment_status",
title: "Fulfillment Status",
accessor: (order: Order): string => order.fulfillment_status,
},
{
fieldName: "payment_status",
title: "Payment Status",
accessor: (order: Order): string => order.payment_status,
},
{
fieldName: "subtotal",
title: "Subtotal",
accessor: (order: Order): string => order.subtotal.toString(),
},
{
fieldName: "shipping_total",
title: "Shipping Total",
accessor: (order: Order): string => order.shipping_total.toString(),
},
{
fieldName: "discount_total",
title: "Discount Total",
accessor: (order: Order): string => order.discount_total.toString(),
},
{
fieldName: "gift_card_total",
title: "Gift Card Total",
accessor: (order: Order): string => order.gift_card_total.toString(),
},
{
fieldName: "refunded_total",
title: "Refunded Total",
accessor: (order: Order): string => order.refunded_total.toString(),
},
{
fieldName: "tax_total",
title: "Tax Total",
accessor: (order: Order): string => order.tax_total.toString(),
},
{
fieldName: "total",
title: "Total",
accessor: (order: Order): string => order.total.toString(),
},
{
fieldName: "currency_code",
title: "Currency Code",
accessor: (order: Order): string => order.currency_code,
},
]
@@ -78,6 +78,44 @@ export default class ProductExportStrategy extends AbstractBatchJobStrategy<
return ""
}
async prepareBatchJobForProcessing(
batchJob: CreateBatchJobInput,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
req: Express.Request
): Promise<CreateBatchJobInput> {
const {
limit,
offset,
order,
fields,
expand,
filterable_fields,
...context
} = (batchJob?.context ?? {}) as ProductExportBatchJobContext
const listConfig = prepareListQuery(
{
limit,
offset,
order,
fields,
expand,
},
{
isList: true,
defaultRelations: this.defaultRelations_,
}
)
batchJob.context = {
...(context ?? {}),
list_config: listConfig,
filterable_fields,
}
return batchJob
}
async preProcessBatchJob(batchJobId: string): Promise<void> {
return await this.atomicPhase_(async (transactionManager) => {
const batchJob = (await this.batchJobService_
@@ -177,44 +215,6 @@ export default class ProductExportStrategy extends AbstractBatchJobStrategy<
})
}
async prepareBatchJobForProcessing(
batchJob: CreateBatchJobInput,
// eslint-disable-next-line @typescript-eslint/no-unused-vars
req: Express.Request
): Promise<CreateBatchJobInput> {
const {
limit,
offset,
order,
fields,
expand,
filterable_fields,
...context
} = (batchJob?.context ?? {}) as ProductExportBatchJobContext
const listConfig = prepareListQuery(
{
limit,
offset,
order,
fields,
expand,
},
{
isList: true,
defaultRelations: this.defaultRelations_,
}
)
batchJob.context = {
...(context ?? {}),
list_config: listConfig,
filterable_fields,
}
return batchJob
}
async processJob(batchJobId: string): Promise<void> {
let offset = 0
let limit = this.DEFAULT_LIMIT
@@ -231,7 +231,7 @@ export default class ProductExportStrategy extends AbstractBatchJobStrategy<
const { writeStream, fileKey, promise } = await this.fileService_
.withTransaction(transactionManager)
.getUploadStreamDescriptor({
name: `product-export-${Date.now()}`,
name: `exports/products/product-export-${Date.now()}`,
ext: "csv",
})