feat: introduce bulkDelete method for IFileProvider (#12614)

Fixes: FRMW-2974

Currently during the product imports, we create multiple chunks that must be deleted after the import has finished (either successfully or with an error). Deleting files one by one leads to multiple network calls and slows down everything.

The `bulkDelete` method deletes multiple files (with their fileKey) in one go

packages/modules/file/src/services/file-module-service.ts

@@ -81,11 +81,11 @@ export default class FileModuleService implements FileTypes.IFileModuleService {
   async deleteFiles(id: string, sharedContext?: Context): Promise<void>
   async deleteFiles(ids: string[] | string): Promise<void> {
     const input = Array.isArray(ids) ? ids : [ids]
-    await Promise.all(
-      input.map((id) => this.fileProviderService_.delete({ fileKey: id }))
+    await this.fileProviderService_.delete(
+      input.map((id) => {
+        return { fileKey: id }
+      })
     )
-
-    return
   }
 
   async retrieveFile(id: string): Promise<FileDTO> {

packages/modules/file/src/services/file-provider-service.ts

@@ -40,7 +40,11 @@ export default class FileProviderService {
     return this.fileProvider_.upload(file)
   }
 
-  delete(fileData: FileTypes.ProviderDeleteFileDTO): Promise<void> {
+  delete(
+    fileData:
+      | FileTypes.ProviderDeleteFileDTO
+      | FileTypes.ProviderDeleteFileDTO[]
+  ): Promise<void> {
     return this.fileProvider_.delete(fileData)
   }
 

packages/modules/providers/file-local/src/services/local-file.ts

@@ -66,21 +66,29 @@ export class LocalFileService extends AbstractFileProviderService {
     }
   }
 
-  async delete(file: FileTypes.ProviderDeleteFileDTO): Promise<void> {
-    const baseDir = file.fileKey.startsWith("private-")
-      ? this.privateUploadDir_
-      : this.uploadDir_
+  async delete(
+    files: FileTypes.ProviderDeleteFileDTO | FileTypes.ProviderDeleteFileDTO[]
+  ): Promise<void> {
+    files = Array.isArray(files) ? files : [files]
 
-    const filePath = this.getUploadFilePath(baseDir, file.fileKey)
-    try {
-      await fs.access(filePath, fs.constants.W_OK)
-      await fs.unlink(filePath)
-    } catch (e) {
-      // The file does not exist, we don't do anything
-      if (e.code !== "ENOENT") {
-        throw e
-      }
-    }
+    await Promise.all(
+      files.map(async (file) => {
+        const baseDir = file.fileKey.startsWith("private-")
+          ? this.privateUploadDir_
+          : this.uploadDir_
+
+        const filePath = this.getUploadFilePath(baseDir, file.fileKey)
+        try {
+          await fs.access(filePath, fs.constants.W_OK)
+          await fs.unlink(filePath)
+        } catch (e) {
+          // The file does not exist, we don't do anything
+          if (e.code !== "ENOENT") {
+            throw e
+          }
+        }
+      })
+    )
 
     return
   }

packages/modules/providers/file-s3/integration-tests/__tests__/services.spec.ts

@@ -160,4 +160,31 @@ describe.skip("S3 File Plugin", () => {
 
     await s3Service.delete({ fileKey: resp.key })
   })
+
+  it("deletes multiple files in bulk", async () => {
+    const fileContent = await fs.readFile(fixtureImagePath)
+    const fixtureAsBinary = fileContent.toString("binary")
+
+    const cat = await s3Service.upload({
+      filename: "catphoto.jpg",
+      mimeType: "image/jpeg",
+      content: fixtureAsBinary,
+    })
+    const cat1 = await s3Service.upload({
+      filename: "catphoto-1.jpg",
+      mimeType: "image/jpeg",
+      content: fixtureAsBinary,
+    })
+    const cat2 = await s3Service.upload({
+      filename: "catphoto-2.jpg",
+      mimeType: "image/jpeg",
+      content: fixtureAsBinary,
+    })
+
+    await s3Service.delete([
+      { fileKey: cat.key },
+      { fileKey: cat1.key },
+      { fileKey: cat2.key },
+    ])
+  })
 })

packages/modules/providers/file-s3/src/services/s3-file.ts

@@ -1,5 +1,6 @@
 import {
   DeleteObjectCommand,
+  DeleteObjectsCommand,
   GetObjectCommand,
   ObjectCannedACL,
   PutObjectCommand,
@@ -152,14 +153,33 @@ export class S3FileService extends AbstractFileProviderService {
     }
   }
 
-  async delete(file: FileTypes.ProviderDeleteFileDTO): Promise<void> {
-    const command = new DeleteObjectCommand({
-      Bucket: this.config_.bucket,
-      Key: file.fileKey,
-    })
-
+  async delete(
+    files: FileTypes.ProviderDeleteFileDTO | FileTypes.ProviderDeleteFileDTO[]
+  ): Promise<void> {
     try {
-      await this.client_.send(command)
+      /**
+       * Bulk delete files
+       */
+      if (Array.isArray(files)) {
+        await this.client_.send(
+          new DeleteObjectsCommand({
+            Bucket: this.config_.bucket,
+            Delete: {
+              Objects: files.map((file) => ({
+                Key: file.fileKey,
+              })),
+              Quiet: true,
+            },
+          })
+        )
+      } else {
+        await this.client_.send(
+          new DeleteObjectCommand({
+            Bucket: this.config_.bucket,
+            Key: files.fileKey,
+          })
+        )
+      }
     } catch (e) {
       // TODO: Rethrow depending on the error (eg. a file not found error is fine, but a failed request should be rethrown)
       this.logger_.error(e)