feat: introduce bulkDelete method for IFileProvider (#12614)

Fixes: FRMW-2974

Currently during the product imports, we create multiple chunks that must be deleted after the import has finished (either successfully or with an error). Deleting files one by one leads to multiple network calls and slows down everything.

The `bulkDelete` method deletes multiple files (with their fileKey) in one go

.changeset/brown-fans-draw.md

@@ -0,0 +1,9 @@
+---
+"@medusajs/file": patch
+"@medusajs/file-local": patch
+"@medusajs/file-s3": patch
+"@medusajs/types": patch
+"@medusajs/utils": patch
+---
+
+feat: introduce bulkDelete method for IFileProvider

packages/core/types/src/file/provider.ts

@@ -119,14 +119,17 @@ export interface IFileProvider {
    *
    */
   upload(file: ProviderUploadFileDTO): Promise<ProviderFileResultDTO>
+
   /**
-   * This method is used to delete a file from storage.
+   * This method is used to delete one or more files from the storage
    *
-   * @param {ProviderDeleteFileDTO} fileData - The details of the file to remove.
+   * @param {ProviderDeleteFileDTO | ProviderDeleteFileDTO[]} fileData - The details of the file to remove.
    * @returns {Promise<void>} Resolves when the file is deleted successfully.
    *
    */
-  delete(fileData: ProviderDeleteFileDTO): Promise<void>
+  delete(
+    fileData: ProviderDeleteFileDTO | ProviderDeleteFileDTO[]
+  ): Promise<void>
 
   /**
    * This method is used to retrieve a download URL of the file. For some file services, such as S3, a presigned URL indicates a temporary URL to get access to a file.

packages/core/utils/src/file/abstract-file-provider.ts

@@ -125,10 +125,10 @@ export class AbstractFileProviderService implements IFileProvider {
   }
 
   /**
-   * This method deletes the file from storage. It's used when an admin user deletes a product image,
+   * This method deletes one or more files from the storage. It's used when an admin user
-   * or other custom file deletions.
+   * deletes a product image, or other custom file deletions.
    *
-   * @param {FileTypes.ProviderDeleteFileDTO} file - The details of the file to delete.
+   * @param {FileTypes.ProviderDeleteFileDTO | FileTypes.ProviderDeleteFileDTO[]} files - The details of the file(s) to delete.
    * @returns {Promise<void>} Resolves when the file is deleted.
    *
    * @example
@@ -143,7 +143,9 @@ export class AbstractFileProviderService implements IFileProvider {
    *   }
    * }
    */
-  async delete(file: FileTypes.ProviderDeleteFileDTO): Promise<void> {
+  async delete(
+    files: FileTypes.ProviderDeleteFileDTO | FileTypes.ProviderDeleteFileDTO[]
+  ): Promise<void> {
     throw Error("delete must be overridden by the child class")
   }
 

packages/modules/file/src/services/file-module-service.ts

@@ -81,11 +81,11 @@ export default class FileModuleService implements FileTypes.IFileModuleService {
   async deleteFiles(id: string, sharedContext?: Context): Promise<void>
   async deleteFiles(ids: string[] | string): Promise<void> {
     const input = Array.isArray(ids) ? ids : [ids]
-    await Promise.all(
+    await this.fileProviderService_.delete(
-      input.map((id) => this.fileProviderService_.delete({ fileKey: id }))
+      input.map((id) => {
+        return { fileKey: id }
+      })
     )
-
-    return
   }
 
   async retrieveFile(id: string): Promise<FileDTO> {

packages/modules/file/src/services/file-provider-service.ts

@@ -40,7 +40,11 @@ export default class FileProviderService {
     return this.fileProvider_.upload(file)
   }
 
-  delete(fileData: FileTypes.ProviderDeleteFileDTO): Promise<void> {
+  delete(
+    fileData:
+      | FileTypes.ProviderDeleteFileDTO
+      | FileTypes.ProviderDeleteFileDTO[]
+  ): Promise<void> {
     return this.fileProvider_.delete(fileData)
   }
 

packages/modules/providers/file-local/src/services/local-file.ts

@@ -66,21 +66,29 @@ export class LocalFileService extends AbstractFileProviderService {
     }
   }
 
-  async delete(file: FileTypes.ProviderDeleteFileDTO): Promise<void> {
+  async delete(
-    const baseDir = file.fileKey.startsWith("private-")
+    files: FileTypes.ProviderDeleteFileDTO | FileTypes.ProviderDeleteFileDTO[]
-      ? this.privateUploadDir_
+  ): Promise<void> {
-      : this.uploadDir_
+    files = Array.isArray(files) ? files : [files]
 
-    const filePath = this.getUploadFilePath(baseDir, file.fileKey)
+    await Promise.all(
-    try {
+      files.map(async (file) => {
-      await fs.access(filePath, fs.constants.W_OK)
+        const baseDir = file.fileKey.startsWith("private-")
-      await fs.unlink(filePath)
+          ? this.privateUploadDir_
-    } catch (e) {
+          : this.uploadDir_
-      // The file does not exist, we don't do anything
+
-      if (e.code !== "ENOENT") {
+        const filePath = this.getUploadFilePath(baseDir, file.fileKey)
-        throw e
+        try {
-      }
+          await fs.access(filePath, fs.constants.W_OK)
-    }
+          await fs.unlink(filePath)
+        } catch (e) {
+          // The file does not exist, we don't do anything
+          if (e.code !== "ENOENT") {
+            throw e
+          }
+        }
+      })
+    )
 
     return
   }

packages/modules/providers/file-s3/integration-tests/__tests__/services.spec.ts

@@ -160,4 +160,31 @@ describe.skip("S3 File Plugin", () => {
 
     await s3Service.delete({ fileKey: resp.key })
   })
+
+  it("deletes multiple files in bulk", async () => {
+    const fileContent = await fs.readFile(fixtureImagePath)
+    const fixtureAsBinary = fileContent.toString("binary")
+
+    const cat = await s3Service.upload({
+      filename: "catphoto.jpg",
+      mimeType: "image/jpeg",
+      content: fixtureAsBinary,
+    })
+    const cat1 = await s3Service.upload({
+      filename: "catphoto-1.jpg",
+      mimeType: "image/jpeg",
+      content: fixtureAsBinary,
+    })
+    const cat2 = await s3Service.upload({
+      filename: "catphoto-2.jpg",
+      mimeType: "image/jpeg",
+      content: fixtureAsBinary,
+    })
+
+    await s3Service.delete([
+      { fileKey: cat.key },
+      { fileKey: cat1.key },
+      { fileKey: cat2.key },
+    ])
+  })
 })

packages/modules/providers/file-s3/src/services/s3-file.ts

@@ -1,5 +1,6 @@
 import {
   DeleteObjectCommand,
+  DeleteObjectsCommand,
   GetObjectCommand,
   ObjectCannedACL,
   PutObjectCommand,
@@ -152,14 +153,33 @@ export class S3FileService extends AbstractFileProviderService {
     }
   }
 
-  async delete(file: FileTypes.ProviderDeleteFileDTO): Promise<void> {
+  async delete(
-    const command = new DeleteObjectCommand({
+    files: FileTypes.ProviderDeleteFileDTO | FileTypes.ProviderDeleteFileDTO[]
-      Bucket: this.config_.bucket,
+  ): Promise<void> {
-      Key: file.fileKey,
-    })
-
     try {
-      await this.client_.send(command)
+      /**
+       * Bulk delete files
+       */
+      if (Array.isArray(files)) {
+        await this.client_.send(
+          new DeleteObjectsCommand({
+            Bucket: this.config_.bucket,
+            Delete: {
+              Objects: files.map((file) => ({
+                Key: file.fileKey,
+              })),
+              Quiet: true,
+            },
+          })
+        )
+      } else {
+        await this.client_.send(
+          new DeleteObjectCommand({
+            Bucket: this.config_.bucket,
+            Key: files.fileKey,
+          })
+        )
+      }
     } catch (e) {
       // TODO: Rethrow depending on the error (eg. a file not found error is fine, but a failed request should be rethrown)
       this.logger_.error(e)