feat(web): determine duplication of upload on client (#8825)

* web upload duplicate verification on client

* _

* fix formating

* chore: clean up

---------

Co-authored-by: Jason Rasmussen <jrasm91@gmail.com>

web/src/lib/stores/upload.ts

@@ -1,4 +1,4 @@
-import { derived, writable } from 'svelte/store';
+import { derived, get, writable } from 'svelte/store';
 import { UploadState, type UploadAsset } from '../models/upload-asset';
 
 function createUploadStore() {
@@ -22,6 +22,11 @@ function createUploadStore() {
   );
 
   const addNewUploadAsset = (newAsset: UploadAsset) => {
+    const assets = get(uploadAssets);
+    const duplicate = assets.find((asset) => asset.id === newAsset.id);
+    if (duplicate) {
+      uploadAssets.update((assets) => assets.map((asset) => (asset.id === newAsset.id ? newAsset : asset)));
+    } else {
       totalUploadCounter.update((c) => c + 1);
       uploadAssets.update((assets) => [
         ...assets,
@@ -33,6 +38,7 @@ function createUploadStore() {
           eta: 0,
         },
       ]);
+    }
   };
 
   const updateProgress = (id: string, loaded: number, total: number) => {

web/src/lib/utils/file-uploader.ts

@@ -3,7 +3,14 @@ import { uploadAssetsStore } from '$lib/stores/upload';
 import { getKey, uploadRequest } from '$lib/utils';
 import { addAssetsToAlbum } from '$lib/utils/asset-utils';
 import { ExecutorQueue } from '$lib/utils/executor-queue';
-import { defaults, getSupportedMediaTypes, type AssetFileUploadResponseDto } from '@immich/sdk';
+import {
+  Action,
+  checkBulkUpload,
+  defaults,
+  getSupportedMediaTypes,
+  type AssetFileUploadResponseDto,
+} from '@immich/sdk';
+import { tick } from 'svelte';
 import { getServerErrorMessage, handleError } from './handle-error';
 
 let _extensions: string[];
@@ -70,8 +77,9 @@ async function fileUploader(asset: File, albumId: string | undefined = undefined
   const fileCreatedAt = new Date(asset.lastModified).toISOString();
   const deviceAssetId = getDeviceAssetId(asset);
 
-  return new Promise((resolve) => resolve(uploadAssetsStore.markStarted(deviceAssetId)))
-    .then(() => {
+  uploadAssetsStore.markStarted(deviceAssetId);
+
+  try {
     const formData = new FormData();
     for (const [key, value] of Object.entries({
       deviceAssetId,
@@ -85,45 +93,66 @@ async function fileUploader(asset: File, albumId: string | undefined = undefined
       formData.append(key, value);
     }
 
+    let responseData: AssetFileUploadResponseDto | undefined;
     const key = getKey();
+    if (crypto?.subtle?.digest && !key) {
+      uploadAssetsStore.updateAsset(deviceAssetId, { message: 'Hashing...' });
+      await tick();
+      try {
+        const bytes = await asset.arrayBuffer();
+        const hash = await crypto.subtle.digest('SHA-1', bytes);
+        const checksum = Array.from(new Uint8Array(hash))
+          .map((b) => b.toString(16).padStart(2, '0'))
+          .join('');
 
-      return uploadRequest<AssetFileUploadResponseDto>({
+        const {
+          results: [checkUploadResult],
+        } = await checkBulkUpload({ assetBulkUploadCheckDto: { assets: [{ id: asset.name, checksum }] } });
+        if (checkUploadResult.action === Action.Reject && checkUploadResult.assetId) {
+          responseData = { duplicate: true, id: checkUploadResult.assetId };
+        }
+      } catch (error) {
+        console.error(`Error calculating sha1 file=${asset.name})`, error);
+      }
+    }
+
+    if (!responseData) {
+      uploadAssetsStore.updateAsset(deviceAssetId, { message: 'Uploading...' });
+      const response = await uploadRequest<AssetFileUploadResponseDto>({
         url: defaults.baseUrl + '/asset/upload' + (key ? `?key=${key}` : ''),
         data: formData,
         onUploadProgress: (event) => uploadAssetsStore.updateProgress(deviceAssetId, event.loaded, event.total),
       });
-    })
-    .then(async (response) => {
-      if (response.status == 200 || response.status == 201) {
-        const res: AssetFileUploadResponseDto = response.data;
+      if (![200, 201].includes(response.status)) {
+        throw new Error('Failed to upload file');
+      }
+      responseData = response.data;
+    }
+    const { duplicate, id: assetId } = responseData;
 
-        if (res.duplicate) {
+    if (duplicate) {
       uploadAssetsStore.duplicateCounter.update((count) => count + 1);
     } else {
       uploadAssetsStore.successCounter.update((c) => c + 1);
     }
 
-        if (albumId && res.id) {
+    if (albumId && assetId) {
       uploadAssetsStore.updateAsset(deviceAssetId, { message: 'Adding to album...' });
-          await addAssetsToAlbum(albumId, [res.id]);
+      await addAssetsToAlbum(albumId, [assetId]);
       uploadAssetsStore.updateAsset(deviceAssetId, { message: 'Added to album' });
     }
 
-        uploadAssetsStore.updateAsset(deviceAssetId, {
-          state: res.duplicate ? UploadState.DUPLICATED : UploadState.DONE,
-        });
+    uploadAssetsStore.updateAsset(deviceAssetId, { state: duplicate ? UploadState.DUPLICATED : UploadState.DONE });
 
     setTimeout(() => {
       uploadAssetsStore.removeUploadAsset(deviceAssetId);
     }, 1000);
 
-        return res.id;
-      }
-    })
-    .catch((error) => {
+    return assetId;
+  } catch (error) {
     handleError(error, 'Unable to upload file');
     const reason = getServerErrorMessage(error) || error;
     uploadAssetsStore.updateAsset(deviceAssetId, { state: UploadState.ERROR, error: reason });
-      return undefined;
-    });
+    return;
+  }
 }