fix issues with 2,000,000 files being uploaded

cli/src/commands/asset.ts

@@ -127,17 +127,22 @@ export const checkForDuplicates = async (files: string[], { concurrency, skipHas
 
   while (retries < maxRetries) {
     try {
-      const response = await checkBulkUpload({ assetBulkUploadCheckDto: { assets: results } });
+      const chunks = chunk(results, 1000);
 
-      for (const { id: filepath, assetId, action } of response.results) {
+      await Promise.all(
-        if (action === Action.Accept) {
+        chunks.map(async (chunk) => {
-          newFiles.push(filepath);
+          const response = await checkBulkUpload({ assetBulkUploadCheckDto: { assets: chunk } });
-        } else {
-          // rejects are always duplicates
-          duplicates.push({ id: assetId as string, filepath });
-        }
-      }
 
+          for (const { id: filepath, assetId, action } of response.results) {
+            if (action === Action.Accept) {
+              newFiles.push(filepath);
+            } else {
+              // rejects are always duplicates
+              duplicates.push({ id: assetId as string, filepath });
+            }
+          }
+        }),
+      );
       break;
     } catch (error: any) {
       retries++;