diff --git a/cli/src/commands/asset.spec.ts b/cli/src/commands/asset.spec.ts index 21137a3296..7dce135985 100644 --- a/cli/src/commands/asset.spec.ts +++ b/cli/src/commands/asset.spec.ts @@ -271,7 +271,7 @@ describe('startWatch', () => { }); }); - it('should filger out ignored patterns', async () => { + it('should filter out ignored patterns', async () => { const testFilePath = path.join(testFolder, 'test.jpg'); const ignoredPattern = 'ignored'; const ignoredFolder = path.join(testFolder, ignoredPattern); diff --git a/cli/src/commands/asset.ts b/cli/src/commands/asset.ts index 2af3cf8d5e..ff7b609eef 100644 --- a/cli/src/commands/asset.ts +++ b/cli/src/commands/asset.ts @@ -37,6 +37,7 @@ export interface UploadOptionsDto { dryRun?: boolean; skipHash?: boolean; delete?: boolean; + deleteDuplicates?: boolean; album?: boolean; albumName?: string; includeHidden?: boolean; @@ -70,10 +71,8 @@ const uploadBatch = async (files: string[], options: UploadOptionsDto) => { console.log(JSON.stringify({ newFiles, duplicates, newAssets }, undefined, 4)); } await updateAlbums([...newAssets, ...duplicates], options); - await deleteFiles( - newAssets.map(({ filepath }) => filepath), - options, - ); + + await deleteFiles(newAssets, duplicates, options); }; export const startWatch = async ( @@ -406,28 +405,46 @@ const uploadFile = async (input: string, stats: Stats): Promise => { - if (!options.delete) { - return; +const deleteFiles = async (uploaded: Asset[], duplicates: Asset[], options: UploadOptionsDto): Promise => { + let fileCount = 0; + if (options.delete) { + fileCount += uploaded.length; + } + + if (options.deleteDuplicates) { + fileCount += duplicates.length; } if (options.dryRun) { - console.log(`Would have deleted ${files.length} local asset${s(files.length)}`); + console.log(`Would have deleted ${fileCount} local asset${s(fileCount)}`); + return; + } + + if (fileCount === 0) { return; } console.log('Deleting assets that have been uploaded...'); - const deletionProgress = new SingleBar( { format: 'Deleting local assets | {bar} | {percentage}% | ETA: {eta}s | {value}/{total} assets' }, Presets.shades_classic, ); - deletionProgress.start(files.length, 0); + deletionProgress.start(fileCount, 0); + + const chunkDelete = async (files: Asset[]) => { + for (const assetBatch of chunk(files, options.concurrency)) { + await Promise.all(assetBatch.map((input: Asset) => unlink(input.filepath))); + deletionProgress.update(assetBatch.length); + } + }; try { - for (const assetBatch of chunk(files, options.concurrency)) { - await Promise.all(assetBatch.map((input: string) => unlink(input))); - deletionProgress.update(assetBatch.length); + if (options.delete) { + await chunkDelete(uploaded); + } + + if (options.deleteDuplicates) { + await chunkDelete(duplicates); } } finally { deletionProgress.stop(); diff --git a/cli/src/index.ts b/cli/src/index.ts index cd49d0e830..0bc2e5de37 100644 --- a/cli/src/index.ts +++ b/cli/src/index.ts @@ -75,6 +75,11 @@ program .default(false), ) .addOption(new Option('--delete', 'Delete local assets after upload').env('IMMICH_DELETE_ASSETS')) + .addOption( + new Option('--delete-duplicates', 'Delete local assets that are duplicates (already exist on server)').env( + 'IMMICH_DELETE_DUPLICATES', + ), + ) .addOption(new Option('--no-progress', 'Hide progress bars').env('IMMICH_PROGRESS_BAR').default(true)) .addOption( new Option('--watch', 'Watch for changes and upload automatically') diff --git a/docs/docs/features/command-line-interface.md b/docs/docs/features/command-line-interface.md index 2b35d62ce8..9a00cb50e1 100644 --- a/docs/docs/features/command-line-interface.md +++ b/docs/docs/features/command-line-interface.md @@ -103,6 +103,7 @@ Options: -c, --concurrency Number of assets to upload at the same time (default: 4, env: IMMICH_UPLOAD_CONCURRENCY) -j, --json-output Output detailed information in json format (default: false, env: IMMICH_JSON_OUTPUT) --delete Delete local assets after upload (env: IMMICH_DELETE_ASSETS) + --delete-duplicates Delete local assets that are duplicates (already exist on server) (env: IMMICH_DELETE_DUPLICATES) --no-progress Hide progress bars (env: IMMICH_PROGRESS_BAR) --watch Watch for changes and upload automatically (default: false, env: IMMICH_WATCH_CHANGES) --help display help for command diff --git a/e2e/src/cli/specs/upload.e2e-spec.ts b/e2e/src/cli/specs/upload.e2e-spec.ts index 8249b9b360..b53b4403f8 100644 --- a/e2e/src/cli/specs/upload.e2e-spec.ts +++ b/e2e/src/cli/specs/upload.e2e-spec.ts @@ -442,6 +442,176 @@ describe(`immich upload`, () => { }); }); + describe('immich upload --delete-duplicates', () => { + it('should delete local duplicate files', async () => { + const { + stderr: firstStderr, + stdout: firstStdout, + exitCode: firstExitCode, + } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]); + expect(firstStderr).toContain('{message}'); + expect(firstStdout.split('\n')).toEqual( + expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]), + ); + expect(firstExitCode).toBe(0); + + await mkdir(`/tmp/albums/nature`, { recursive: true }); + await symlink(`${testAssetDir}/albums/nature/silver_fir.jpg`, `/tmp/albums/nature/silver_fir.jpg`); + + // Upload with --delete-duplicates flag + const { stderr, stdout, exitCode } = await immichCli([ + 'upload', + `/tmp/albums/nature/silver_fir.jpg`, + '--delete-duplicates', + ]); + + // Check that the duplicate file was deleted + const files = await readdir(`/tmp/albums/nature`); + await rm(`/tmp/albums/nature`, { recursive: true }); + expect(files.length).toBe(0); + + expect(stdout.split('\n')).toEqual( + expect.arrayContaining([ + expect.stringContaining('Found 0 new files and 1 duplicate'), + expect.stringContaining('All assets were already uploaded, nothing to do'), + ]), + ); + expect(stderr).toContain('{message}'); + expect(exitCode).toBe(0); + + // Verify no new assets were uploaded + const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) }); + expect(assets.total).toBe(1); + }); + + it('should have accurate dry run with --delete-duplicates', async () => { + const { + stderr: firstStderr, + stdout: firstStdout, + exitCode: firstExitCode, + } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]); + expect(firstStderr).toContain('{message}'); + expect(firstStdout.split('\n')).toEqual( + expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]), + ); + expect(firstExitCode).toBe(0); + + await mkdir(`/tmp/albums/nature`, { recursive: true }); + await symlink(`${testAssetDir}/albums/nature/silver_fir.jpg`, `/tmp/albums/nature/silver_fir.jpg`); + + // Upload with --delete-duplicates and --dry-run flags + const { stderr, stdout, exitCode } = await immichCli([ + 'upload', + `/tmp/albums/nature/silver_fir.jpg`, + '--delete-duplicates', + '--dry-run', + ]); + + // Check that the duplicate file was NOT deleted in dry run mode + const files = await readdir(`/tmp/albums/nature`); + await rm(`/tmp/albums/nature`, { recursive: true }); + expect(files.length).toBe(1); + + expect(stdout.split('\n')).toEqual( + expect.arrayContaining([ + expect.stringContaining('Found 0 new files and 1 duplicate'), + expect.stringContaining('Would have deleted 1 local asset'), + ]), + ); + expect(stderr).toContain('{message}'); + expect(exitCode).toBe(0); + + // Verify no new assets were uploaded + const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) }); + expect(assets.total).toBe(1); + }); + + it('should work with both --delete and --delete-duplicates flags', async () => { + // First, upload a file to create a duplicate on the server + const { + stderr: firstStderr, + stdout: firstStdout, + exitCode: firstExitCode, + } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]); + expect(firstStderr).toContain('{message}'); + expect(firstStdout.split('\n')).toEqual( + expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]), + ); + expect(firstExitCode).toBe(0); + + // Both new and duplicate files + await mkdir(`/tmp/albums/nature`, { recursive: true }); + await symlink(`${testAssetDir}/albums/nature/silver_fir.jpg`, `/tmp/albums/nature/silver_fir.jpg`); // duplicate + await symlink(`${testAssetDir}/albums/nature/el_torcal_rocks.jpg`, `/tmp/albums/nature/el_torcal_rocks.jpg`); // new + + // Upload with both --delete and --delete-duplicates flags + const { stderr, stdout, exitCode } = await immichCli([ + 'upload', + `/tmp/albums/nature`, + '--delete', + '--delete-duplicates', + ]); + + // Check that both files were deleted (new file due to --delete, duplicate due to --delete-duplicates) + const files = await readdir(`/tmp/albums/nature`); + await rm(`/tmp/albums/nature`, { recursive: true }); + expect(files.length).toBe(0); + + expect(stdout.split('\n')).toEqual( + expect.arrayContaining([ + expect.stringContaining('Found 1 new files and 1 duplicate'), + expect.stringContaining('Successfully uploaded 1 new asset'), + expect.stringContaining('Deleting assets that have been uploaded'), + ]), + ); + expect(stderr).toContain('{message}'); + expect(exitCode).toBe(0); + + // Verify one new asset was uploaded (total should be 2 now) + const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) }); + expect(assets.total).toBe(2); + }); + + it('should only delete duplicates when --delete-duplicates is used without --delete', async () => { + const { + stderr: firstStderr, + stdout: firstStdout, + exitCode: firstExitCode, + } = await immichCli(['upload', `${testAssetDir}/albums/nature/silver_fir.jpg`]); + expect(firstStderr).toContain('{message}'); + expect(firstStdout.split('\n')).toEqual( + expect.arrayContaining([expect.stringContaining('Successfully uploaded 1 new asset')]), + ); + expect(firstExitCode).toBe(0); + + // Both new and duplicate files + await mkdir(`/tmp/albums/nature`, { recursive: true }); + await symlink(`${testAssetDir}/albums/nature/silver_fir.jpg`, `/tmp/albums/nature/silver_fir.jpg`); // duplicate + await symlink(`${testAssetDir}/albums/nature/el_torcal_rocks.jpg`, `/tmp/albums/nature/el_torcal_rocks.jpg`); // new + + // Upload with only --delete-duplicates flag + const { stderr, stdout, exitCode } = await immichCli(['upload', `/tmp/albums/nature`, '--delete-duplicates']); + + // Check that only the duplicate was deleted, new file should remain + const files = await readdir(`/tmp/albums/nature`); + await rm(`/tmp/albums/nature`, { recursive: true }); + expect(files).toEqual(['el_torcal_rocks.jpg']); + + expect(stdout.split('\n')).toEqual( + expect.arrayContaining([ + expect.stringContaining('Found 1 new files and 1 duplicate'), + expect.stringContaining('Successfully uploaded 1 new asset'), + ]), + ); + expect(stderr).toContain('{message}'); + expect(exitCode).toBe(0); + + // Verify one new asset was uploaded (total should be 2 now) + const assets = await getAssetStatistics({}, { headers: asKeyAuth(key) }); + expect(assets.total).toBe(2); + }); + }); + describe('immich upload --skip-hash', () => { it('should skip hashing', async () => { const filename = `albums/nature/silver_fir.jpg`;