From 72dc676cb8785642df222bf17186fefa5488728c Mon Sep 17 00:00:00 2001
From: tiefseetauchner <lena@tauchner.me>
Date: Thu, 15 Aug 2024 23:10:23 +0200
Subject: [PATCH] fix issues with 2,000,000 files being uploaded

---
 cli/src/commands/asset.ts | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/cli/src/commands/asset.ts b/cli/src/commands/asset.ts
index 496a0c79e730f..a624bd91ce1d5 100644
--- a/cli/src/commands/asset.ts
+++ b/cli/src/commands/asset.ts
@@ -127,17 +127,22 @@ export const checkForDuplicates = async (files: string[], { concurrency, skipHas
 
   while (retries < maxRetries) {
     try {
-      const response = await checkBulkUpload({ assetBulkUploadCheckDto: { assets: results } });
+      const chunks = chunk(results, 1000);
 
-      for (const { id: filepath, assetId, action } of response.results) {
-        if (action === Action.Accept) {
-          newFiles.push(filepath);
-        } else {
-          // rejects are always duplicates
-          duplicates.push({ id: assetId as string, filepath });
-        }
-      }
+      await Promise.all(
+        chunks.map(async (chunk) => {
+          const response = await checkBulkUpload({ assetBulkUploadCheckDto: { assets: chunk } });
 
+          for (const { id: filepath, assetId, action } of response.results) {
+            if (action === Action.Accept) {
+              newFiles.push(filepath);
+            } else {
+              // rejects are always duplicates
+              duplicates.push({ id: assetId as string, filepath });
+            }
+          }
+        }),
+      );
       break;
     } catch (error: any) {
       retries++;