feat: order export and upload stream (#14243)

* feat: order export

* Merge branch 'develop' of https://github.com/medusajs/medusa into feat/order-export

* normalize status

* rm util

* serialize totals

* test

* lock

* comments

* configurable order list

packages/modules/file/src/services/file-module-service.ts

@@ -1,19 +1,19 @@
-import type { Readable } from "stream"
 import {
   Context,
   CreateFileDTO,
-  GetUploadFileUrlDTO,
   FileDTO,
-  UploadFileUrlDTO,
   FileTypes,
   FilterableFileProps,
   FindConfig,
+  GetUploadFileUrlDTO,
   ModuleJoinerConfig,
+  UploadFileUrlDTO,
 } from "@medusajs/framework/types"
+import type { Readable, Writable } from "stream"
 
+import { MedusaError } from "@medusajs/framework/utils"
 import { joinerConfig } from "../joiner-config"
 import FileProviderService from "./file-provider-service"
-import { MedusaError } from "@medusajs/framework/utils"
 
 type InjectedDependencies = {
   fileProviderService: FileProviderService
@@ -172,4 +172,25 @@ export default class FileModuleService implements FileTypes.IFileModuleService {
   getAsBuffer(id: string): Promise<Buffer> {
     return this.fileProviderService_.getAsBuffer({ fileKey: id })
   }
+
+  /**
+   * Get a writeable stream to upload a file.
+   *
+   * @example
+   * const { writeStream, promise } = await fileModuleService.getUploadStream({
+   *   filename: "test.csv",
+   *   mimeType: "text/csv",
+   * })
+   *
+   * stream.pipe(writeStream)
+   * const result = await promise
+   */
+  getUploadStream(data: FileTypes.ProviderUploadStreamDTO): Promise<{
+    writeStream: Writable
+    promise: Promise<FileTypes.ProviderFileResultDTO>
+    url: string
+    fileKey: string
+  }> {
+    return this.fileProviderService_.getUploadStream(data)
+  }
 }

packages/modules/file/src/services/file-provider-service.ts

@@ -1,7 +1,7 @@
-import type { Readable } from "stream"
 import { Constructor, FileTypes } from "@medusajs/framework/types"
 import { MedusaError } from "@medusajs/framework/utils"
 import { FileProviderRegistrationPrefix } from "@types"
+import type { Readable, Writable } from "stream"
 
 type InjectedDependencies = {
   [
@@ -81,4 +81,13 @@ export default class FileProviderService {
   getAsBuffer(fileData: FileTypes.ProviderGetFileDTO): Promise<Buffer> {
     return this.fileProvider_.getAsBuffer(fileData)
   }
+
+  getUploadStream(fileData: FileTypes.ProviderUploadStreamDTO): Promise<{
+    writeStream: Writable
+    promise: Promise<FileTypes.ProviderFileResultDTO>
+    url: string
+    fileKey: string
+  }> {
+    return this.fileProvider_.getUploadStream(fileData)
+  }
 }

packages/modules/providers/file-local/integration-tests/__tests__/services.spec.ts

@@ -0,0 +1,119 @@
+import { FileSystem } from "@medusajs/utils"
+import fs from "fs/promises"
+import path from "path"
+import { LocalFileService } from "../../src/services/local-file"
+
+jest.setTimeout(10000)
+
+describe("Local File Plugin", () => {
+  let localService: LocalFileService
+
+  const fixtureImagePath =
+    process.cwd() + "/integration-tests/__fixtures__/catphoto.jpg"
+
+  const uploadDir = path.join(
+    process.cwd(),
+    "integration-tests/__tests__/uploads"
+  )
+
+  const fileSystem = new FileSystem(uploadDir)
+
+  beforeAll(async () => {
+    localService = new LocalFileService(
+      {
+        logger: console as any,
+      },
+      {
+        upload_dir: uploadDir,
+        backend_url: "http://localhost:9000/static",
+      }
+    )
+  })
+
+  afterAll(async () => {
+    await fileSystem.cleanup()
+  })
+
+  it(`should upload, read, and then delete a public file successfully`, async () => {
+    const fileContent = await fs.readFile(fixtureImagePath)
+    const fixtureAsBase64 = fileContent.toString("base64")
+
+    const resp = await localService.upload({
+      filename: "catphoto.jpg",
+      mimeType: "image/jpeg",
+      content: fileContent as any,
+      access: "public",
+    })
+
+    expect(resp).toEqual({
+      key: expect.stringMatching(/catphoto.*\.jpg/),
+      url: expect.stringMatching(
+        /http:\/\/localhost:9000\/static\/.*catphoto.*\.jpg/
+      ),
+    })
+
+    // For local file provider, we can verify the file exists on disk
+    const fileKey = resp.key
+    const baseDir = uploadDir
+    const filePath = path.join(baseDir, fileKey)
+
+    const fileOnDisk = await fs.readFile(filePath)
+
+    const fileOnDiskAsBase64 = fileOnDisk.toString("base64")
+
+    expect(fileOnDiskAsBase64).toEqual(fixtureAsBase64)
+
+    const signedUrl = await localService.getPresignedDownloadUrl({
+      fileKey: resp.key,
+    })
+
+    expect(signedUrl).toEqual(resp.url)
+
+    const buffer = await localService.getAsBuffer({ fileKey: resp.key })
+    expect(buffer).toEqual(fileContent)
+
+    await localService.delete({ fileKey: resp.key })
+
+    await expect(fs.access(filePath)).rejects.toThrow()
+  })
+
+  it("uploads using stream", async () => {
+    const fileContent = await fs.readFile(fixtureImagePath)
+
+    const { writeStream, promise } = await localService.getUploadStream({
+      filename: "catphoto-stream.jpg",
+      mimeType: "image/jpeg",
+      access: "public",
+    })
+
+    writeStream.write(fileContent)
+    writeStream.end()
+
+    const resp = await promise
+
+    expect(resp).toEqual({
+      key: expect.stringMatching(/catphoto-stream.*\.jpg/),
+      url: expect.stringMatching(
+        /http:\/\/localhost:9000\/static\/.*catphoto-stream.*\.jpg/
+      ),
+    })
+
+    const fileKey = resp.key
+    const filePath = path.join(uploadDir, fileKey)
+
+    const fileOnDisk = await fs.readFile(filePath)
+    expect(fileOnDisk).toEqual(fileContent)
+
+    const signedUrl = await localService.getPresignedDownloadUrl({
+      fileKey: resp.key,
+    })
+
+    expect(signedUrl).toEqual(resp.url)
+
+    const buffer = await localService.getAsBuffer({ fileKey: resp.key })
+    expect(buffer).toEqual(fileContent)
+
+    await localService.delete({ fileKey: resp.key })
+    await expect(fs.access(filePath)).rejects.toThrow()
+  })
+})

packages/modules/providers/file-local/package.json

@@ -21,6 +21,7 @@
   "license": "MIT",
   "scripts": {
     "test": "../../../../node_modules/.bin/jest --passWithNoTests src",
+    "test:integration": "../../../../node_modules/.bin/jest --passWithNoTests --forceExit --testPathPattern=\"integration-tests/__tests__/[^/]*\\.spec\\.ts\"",
     "build": "yarn run -T rimraf dist && yarn run -T tsc --build ./tsconfig.json",
     "watch": "yarn run -T tsc --watch"
   },

packages/modules/providers/file-local/src/services/local-file.ts

@@ -3,10 +3,10 @@ import {
   AbstractFileProviderService,
   MedusaError,
 } from "@medusajs/framework/utils"
-import { createReadStream } from "fs"
+import { createReadStream, createWriteStream } from "fs"
 import fs from "fs/promises"
 import path from "path"
-import type { Readable } from "stream"
+import type { Readable, Writable } from "stream"
 
 export class LocalFileService extends AbstractFileProviderService {
   static identifier = "localfs"
@@ -78,6 +78,59 @@ export class LocalFileService extends AbstractFileProviderService {
     }
   }
 
+  async getUploadStream(fileData: FileTypes.ProviderUploadStreamDTO): Promise<{
+    writeStream: Writable
+    promise: Promise<FileTypes.ProviderFileResultDTO>
+    url: string
+    fileKey: string
+  }> {
+    if (!fileData.filename) {
+      throw new MedusaError(
+        MedusaError.Types.INVALID_DATA,
+        `No filename provided`
+      )
+    }
+
+    const parsedFilename = path.parse(fileData.filename)
+    const baseDir =
+      fileData.access === "public" ? this.uploadDir_ : this.privateUploadDir_
+    await this.ensureDirExists(baseDir, parsedFilename.dir)
+
+    const fileKey = path.join(
+      parsedFilename.dir,
+      // We prepend "private" to the file key so deletions and presigned URLs can know which folder to look into
+      `${fileData.access === "public" ? "" : "private-"}${Date.now()}-${
+        parsedFilename.base
+      }`
+    )
+
+    const filePath = this.getUploadFilePath(baseDir, fileKey)
+    const fileUrl = this.getUploadFileUrl(fileKey)
+
+    const writeStream = createWriteStream(filePath)
+
+    const promise = new Promise<FileTypes.ProviderFileResultDTO>(
+      (resolve, reject) => {
+        writeStream.on("finish", () => {
+          resolve({
+            url: fileUrl,
+            key: fileKey,
+          })
+        })
+        writeStream.on("error", (err) => {
+          reject(err)
+        })
+      }
+    )
+
+    return {
+      writeStream,
+      promise,
+      url: fileUrl,
+      fileKey,
+    }
+  }
+
   async delete(
     files: FileTypes.ProviderDeleteFileDTO | FileTypes.ProviderDeleteFileDTO[]
   ): Promise<void> {

packages/modules/providers/file-s3/integration-tests/__tests__/services.spec.ts

@@ -49,7 +49,7 @@ describe.skip("S3 File Plugin", () => {
 
       expect(resp).toEqual({
         key: expect.stringMatching(/tests\/catphoto.*\.jpg/),
-        url: expect.stringMatching(/https:\/\/.*\.jpg/),
+        url: expect.stringMatching(/https?:\/\/.*\.jpg/),
       })
 
       const urlResp = await axios.get(resp.url).catch((e) => e.response)
@@ -95,7 +95,7 @@ describe.skip("S3 File Plugin", () => {
 
     expect(resp).toEqual({
       key: expect.stringMatching(/tests\/catphoto-か.*\.jpg/),
-      url: expect.stringMatching(/https:\/\/.*\/catphoto-%E3%81%8B.*\.jpg/),
+      url: expect.stringMatching(/https?:\/\/.*\/catphoto-%E3%81%8B.*\.jpg/),
     })
   })
 
@@ -112,7 +112,7 @@ describe.skip("S3 File Plugin", () => {
 
     expect(resp).toEqual({
       key: expect.stringMatching(/tests\/catphoto.*\.jpg/),
-      url: expect.stringMatching(/https:\/\/.*\/cat%3Fphoto.*\.jpg/),
+      url: expect.stringMatching(/https?:\/\/.*\/cat%3Fphoto.*\.jpg/),
     })
   })
 
@@ -128,7 +128,7 @@ describe.skip("S3 File Plugin", () => {
 
     expect(resp).toEqual({
       key: expect.stringMatching(/tests\/catphoto.*\.jpg/),
-      url: expect.stringMatching(/https:\/\/.*catphoto\.jpg/),
+      url: expect.stringMatching(/https?:\/\/.*catphoto\.jpg/),
     })
 
     const uploadResp = await axios.put(resp.url, fileContent, {
@@ -169,7 +169,7 @@ describe.skip("S3 File Plugin", () => {
 
     expect(resp).toEqual({
       key: expect.stringMatching(/tests\/testfolder\/catphoto.*\.jpg/),
-      url: expect.stringMatching(/https:\/\/.*testfolder\/catphoto\.jpg/),
+      url: expect.stringMatching(/https?:\/\/.*testfolder\/catphoto\.jpg/),
     })
 
     const uploadResp = await axios.put(resp.url, fileContent, {
@@ -221,4 +221,42 @@ describe.skip("S3 File Plugin", () => {
       { fileKey: cat2.key },
     ])
   })
+
+  it("uploads using stream", async () => {
+    const fileContent = await fs.readFile(fixtureImagePath)
+    const fixtureAsBinary = fileContent.toString("binary")
+
+    const { writeStream, promise } = await s3Service.getUploadStream({
+      filename: "catphoto-stream.jpg",
+      mimeType: "image/jpeg",
+      access: "public",
+    })
+
+    writeStream.write(fileContent)
+    writeStream.end()
+
+    const resp = await promise
+
+    expect(resp).toEqual({
+      key: expect.stringMatching(/tests\/catphoto-stream.*\.jpg/),
+      url: expect.stringMatching(/https?:\/\/.*\.jpg/),
+    })
+
+    const urlResp = await axios.get(resp.url).catch((e) => e.response)
+    expect(urlResp.status).toEqual(200)
+
+    const signedUrl = await s3Service.getPresignedDownloadUrl({
+      fileKey: resp.key,
+    })
+
+    const signedUrlFile = Buffer.from(
+      await axios
+        .get(signedUrl, { responseType: "arraybuffer" })
+        .then((r) => r.data)
+    )
+
+    expect(signedUrlFile.toString("binary")).toEqual(fixtureAsBinary)
+
+    await s3Service.delete({ fileKey: resp.key })
+  })
 })

packages/modules/providers/file-s3/package.json

@@ -30,6 +30,7 @@
   },
   "dependencies": {
     "@aws-sdk/client-s3": "^3.556.0",
+    "@aws-sdk/lib-storage": "^3.556.0",
     "@aws-sdk/s3-request-presigner": "^3.556.0",
     "ulid": "^2.3.0"
   },

packages/modules/providers/file-s3/src/services/s3-file.ts

@@ -7,6 +7,7 @@ import {
   S3Client,
   S3ClientConfigType,
 } from "@aws-sdk/client-s3"
+import { Upload } from "@aws-sdk/lib-storage"
 import { getSignedUrl } from "@aws-sdk/s3-request-presigner"
 import {
   FileTypes,
@@ -18,7 +19,7 @@ import {
   MedusaError,
 } from "@medusajs/framework/utils"
 import path from "path"
-import { Readable } from "stream"
+import { PassThrough, Readable, Writable } from "stream"
 import { ulid } from "ulid"
 
 type InjectedDependencies = {
@@ -165,6 +166,53 @@ export class S3FileService extends AbstractFileProviderService {
     }
   }
 
+  async getUploadStream(fileData: FileTypes.ProviderUploadStreamDTO): Promise<{
+    writeStream: Writable
+    promise: Promise<FileTypes.ProviderFileResultDTO>
+    url: string
+    fileKey: string
+  }> {
+    if (!fileData.filename) {
+      throw new MedusaError(
+        MedusaError.Types.INVALID_DATA,
+        `No filename provided`
+      )
+    }
+
+    const parsedFilename = path.parse(fileData.filename)
+    const fileKey = `${this.config_.prefix}${parsedFilename.name}-${ulid()}${
+      parsedFilename.ext
+    }`
+
+    const pass = new PassThrough()
+    const upload = new Upload({
+      client: this.client_,
+      params: {
+        ACL: fileData.access === "public" ? "public-read" : "private",
+        Bucket: this.config_.bucket,
+        Key: fileKey,
+        Body: pass,
+        ContentType: fileData.mimeType,
+        CacheControl: this.config_.cacheControl,
+        Metadata: {
+          "original-filename": encodeURIComponent(fileData.filename),
+        },
+      },
+    })
+
+    const promise = upload.done().then(() => ({
+      url: `${this.config_.fileUrl}/${fileKey}`,
+      key: fileKey,
+    }))
+
+    return {
+      writeStream: pass,
+      promise,
+      url: `${this.config_.fileUrl}/${fileKey}`,
+      fileKey,
+    }
+  }
+
   async delete(
     files: FileTypes.ProviderDeleteFileDTO | FileTypes.ProviderDeleteFileDTO[]
   ): Promise<void> {
@@ -207,7 +255,7 @@ export class S3FileService extends AbstractFileProviderService {
       Key: `${fileData.fileKey}`,
     })
 
-    return await getSignedUrl(this.client_, command, {
+    return await getSignedUrl(this.client_ as any, command as any, {
       expiresIn: this.config_.downloadFileDuration,
     })
   }
@@ -238,7 +286,7 @@ export class S3FileService extends AbstractFileProviderService {
       Key: fileKey,
     })
 
-    const signedUrl = await getSignedUrl(this.client_, command, {
+    const signedUrl = await getSignedUrl(this.client_ as any, command as any, {
       expiresIn:
         fileData.expiresIn ?? DEFAULT_UPLOAD_EXPIRATION_DURATION_SECONDS,
     })