Merge branch 'main' of https://github.com/immich-app/immich into feat/offline-files-job

server/package-lock.json

@@ -47,6 +47,7 @@
         "js-yaml": "^4.1.0",
         "lodash": "^4.17.21",
         "luxon": "^3.4.2",
+        "mnemonist": "^0.39.8",
         "nest-commander": "^3.11.1",
         "nestjs-otel": "^5.1.5",
         "node-addon-api": "^7.0.0",
@@ -10426,6 +10427,14 @@
       "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
       "dev": true
     },
+    "node_modules/mnemonist": {
+      "version": "0.39.8",
+      "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.8.tgz",
+      "integrity": "sha512-vyWo2K3fjrUw8YeeZ1zF0fy6Mu59RHokURlld8ymdUPjMlD9EC9ov1/YPqTgqRvUN9nTr3Gqfz29LYAmu0PHPQ==",
+      "dependencies": {
+        "obliterator": "^2.0.1"
+      }
+    },
     "node_modules/mock-fs": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/mock-fs/-/mock-fs-5.2.0.tgz",
@@ -10819,6 +10828,11 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/obliterator": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.4.tgz",
+      "integrity": "sha512-lgHwxlxV1qIg1Eap7LgIeoBWIMFibOjbrYPIPJZcI1mmGAI2m3lNYpK12Y+GBdPQ0U1hRwSord7GIaawz962qQ=="
+    },
     "node_modules/obuf": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz",
@@ -22037,6 +22051,14 @@
       "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
       "dev": true
     },
+    "mnemonist": {
+      "version": "0.39.8",
+      "resolved": "https://registry.npmjs.org/mnemonist/-/mnemonist-0.39.8.tgz",
+      "integrity": "sha512-vyWo2K3fjrUw8YeeZ1zF0fy6Mu59RHokURlld8ymdUPjMlD9EC9ov1/YPqTgqRvUN9nTr3Gqfz29LYAmu0PHPQ==",
+      "requires": {
+        "obliterator": "^2.0.1"
+      }
+    },
     "mock-fs": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/mock-fs/-/mock-fs-5.2.0.tgz",
@@ -22349,6 +22371,11 @@
       "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.3.tgz",
       "integrity": "sha512-geUvdk7c+eizMNUDkRpW1wJwgfOiOeHbxBR/hLXK1aT6zmVSO0jsQcs7fj6MGw89jC/cjGfLcNOrtMYtGqm81g=="
     },
+    "obliterator": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/obliterator/-/obliterator-2.0.4.tgz",
+      "integrity": "sha512-lgHwxlxV1qIg1Eap7LgIeoBWIMFibOjbrYPIPJZcI1mmGAI2m3lNYpK12Y+GBdPQ0U1hRwSord7GIaawz962qQ=="
+    },
     "obuf": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/obuf/-/obuf-1.1.2.tgz",

server/package.json

@@ -71,6 +71,7 @@
     "js-yaml": "^4.1.0",
     "lodash": "^4.17.21",
     "luxon": "^3.4.2",
+    "mnemonist": "^0.39.8",
     "nest-commander": "^3.11.1",
     "nestjs-otel": "^5.1.5",
     "node-addon-api": "^7.0.0",

server/src/domain/download/download.service.spec.ts

@@ -164,7 +164,7 @@ describe(DownloadService.name, () => {
       const assetIds = ['asset-1', 'asset-2'];
       await expect(sut.getDownloadInfo(authStub.admin, { assetIds })).resolves.toEqual(downloadResponse);
 
-      expect(assetMock.getByIds).toHaveBeenCalledWith(['asset-1', 'asset-2']);
+      expect(assetMock.getByIds).toHaveBeenCalledWith(['asset-1', 'asset-2'], { exifInfo: true });
     });
 
     it('should return a list of archives (albumId)', async () => {
@@ -228,10 +228,10 @@ describe(DownloadService.name, () => {
 
       accessMock.asset.checkOwnerAccess.mockResolvedValue(new Set(assetIds));
       when(assetMock.getByIds)
-        .calledWith([assetStub.livePhotoStillAsset.id])
+        .calledWith([assetStub.livePhotoStillAsset.id], { exifInfo: true })
         .mockResolvedValue([assetStub.livePhotoStillAsset]);
       when(assetMock.getByIds)
-        .calledWith([assetStub.livePhotoMotionAsset.id])
+        .calledWith([assetStub.livePhotoMotionAsset.id], { exifInfo: true })
         .mockResolvedValue([assetStub.livePhotoMotionAsset]);
 
       await expect(sut.getDownloadInfo(authStub.admin, { assetIds })).resolves.toEqual({

server/src/domain/download/download.service.ts

@@ -50,7 +50,7 @@ export class DownloadService {
       // motion part of live photos
       const motionIds = assets.map((asset) => asset.livePhotoVideoId).filter<string>((id): id is string => !!id);
       if (motionIds.length > 0) {
-        assets.push(...(await this.assetRepository.getByIds(motionIds)));
+        assets.push(...(await this.assetRepository.getByIds(motionIds, { exifInfo: true })));
       }
 
       for (const asset of assets) {
@@ -114,7 +114,7 @@ export class DownloadService {
     if (dto.assetIds) {
       const assetIds = dto.assetIds;
       await this.access.requirePermission(auth, Permission.ASSET_DOWNLOAD, assetIds);
-      const assets = await this.assetRepository.getByIds(assetIds);
+      const assets = await this.assetRepository.getByIds(assetIds, { exifInfo: true });
       return usePagination(PAGINATION_SIZE, () => ({ hasNextPage: false, items: assets }));
     }
 

server/src/domain/job/job.service.spec.ts

@@ -330,8 +330,6 @@ describe(JobService.name, () => {
           } else {
             assetMock.getByIds.mockResolvedValue([assetStub.livePhotoMotionAsset]);
           }
-        } else {
-          assetMock.getByIds.mockResolvedValue([]);
         }
 
         await sut.init(makeMockHandlers(true));

server/src/domain/job/job.service.ts

@@ -214,7 +214,7 @@ export class JobService {
 
       case JobName.METADATA_EXTRACTION: {
         if (item.data.source === 'sidecar-write') {
-          const [asset] = await this.assetRepository.getByIds([item.data.id]);
+          const [asset] = await this.assetRepository.getByIdsWithAllRelations([item.data.id]);
           if (asset) {
             this.communicationRepository.send(ClientEvent.ASSET_UPDATE, asset.ownerId, mapAsset(asset));
           }
@@ -272,7 +272,7 @@ export class JobService {
           break;
         }
 
-        const [asset] = await this.assetRepository.getByIds([item.data.id]);
+        const [asset] = await this.assetRepository.getByIdsWithAllRelations([item.data.id]);
 
         // Only live-photo motion part will be marked as not visible immediately on upload. Skip notifying clients
         if (asset && asset.isVisible) {

server/src/domain/library/library.service.spec.ts

@@ -155,7 +155,10 @@ describe(LibraryService.name, () => {
       };
 
       libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
-      storageMock.crawl.mockResolvedValue(['/data/user1/photo.jpg']);
+      // eslint-disable-next-line @typescript-eslint/require-await
+      storageMock.walk.mockImplementation(async function* generator() {
+        yield '/data/user1/photo.jpg';
+      });
       assetMock.getLibraryAssetPaths.mockResolvedValue({ items: [], hasNextPage: false });
 
       await sut.handleQueueAssetRefresh(mockLibraryJob);
@@ -181,7 +184,10 @@ describe(LibraryService.name, () => {
       };
 
       libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
-      storageMock.crawl.mockResolvedValue(['/data/user1/photo.jpg']);
+      // eslint-disable-next-line @typescript-eslint/require-await
+      storageMock.walk.mockImplementation(async function* generator() {
+        yield '/data/user1/photo.jpg';
+      });
       assetMock.getLibraryAssetPaths.mockResolvedValue({ items: [], hasNextPage: false });
 
       await sut.handleQueueAssetRefresh(mockLibraryJob);
@@ -231,17 +237,37 @@ describe(LibraryService.name, () => {
       };
 
       libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1);
-      storageMock.crawl.mockResolvedValue([]);
       assetMock.getLibraryAssetPaths.mockResolvedValue({ items: [], hasNextPage: false });
 
       await sut.handleQueueAssetRefresh(mockLibraryJob);
 
-      expect(storageMock.crawl).toHaveBeenCalledWith({
+      expect(storageMock.walk).toHaveBeenCalledWith({
         pathsToCrawl: [libraryStub.externalLibraryWithImportPaths1.importPaths[1]],
         exclusionPatterns: [],
       });
     });
 
+    it('should set missing assets offline', async () => {
+      const mockLibraryJob: ILibraryRefreshJob = {
+        id: libraryStub.externalLibrary1.id,
+        refreshModifiedFiles: false,
+        refreshAllFiles: false,
+      };
+
+      libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
+      storageMock.crawl.mockResolvedValue([]);
+      assetMock.getLibraryAssetPaths.mockResolvedValue({
+        items: [assetStub.image],
+        hasNextPage: false,
+      });
+
+      await sut.handleQueueAssetRefresh(mockLibraryJob);
+
+      expect(assetMock.updateAll).toHaveBeenCalledWith([assetStub.image.id], { isOffline: true });
+      expect(assetMock.updateAll).not.toHaveBeenCalledWith(expect.anything(), { isOffline: false });
+      expect(jobMock.queueAll).not.toHaveBeenCalled();
+    });
+
     it('should set crawled assets that were previously offline back online', async () => {
       const mockLibraryJob: ILibraryRefreshJob = {
         id: libraryStub.externalLibrary1.id,
@@ -250,7 +276,10 @@ describe(LibraryService.name, () => {
       };
 
       libraryMock.get.mockResolvedValue(libraryStub.externalLibrary1);
-      storageMock.crawl.mockResolvedValue([assetStub.offline.originalPath]);
+      // eslint-disable-next-line @typescript-eslint/require-await
+      storageMock.walk.mockImplementation(async function* generator() {
+        yield assetStub.offline.originalPath;
+      });
       assetMock.getLibraryAssetPaths.mockResolvedValue({
         items: [assetStub.offline],
         hasNextPage: false,

server/src/domain/library/library.service.ts

@@ -1,6 +1,7 @@
-import { AssetType, LibraryType } from '@app/infra/entities';
+import { AssetType, LibraryEntity, LibraryType } from '@app/infra/entities';
 import { ImmichLogger } from '@app/infra/logger';
 import { BadRequestException, Inject, Injectable } from '@nestjs/common';
+import { Trie } from 'mnemonist';
 import { R_OK } from 'node:constants';
 import { EventEmitter } from 'node:events';
 import { Stats } from 'node:fs';
@@ -11,7 +12,6 @@ import { AuthDto } from '../auth';
 import { mimeTypes } from '../domain.constant';
 import { handlePromiseError, usePagination, validateCronExpression } from '../domain.util';
 import { IBaseJob, IEntityJob, ILibraryFileJob, ILibraryRefreshJob, JOBS_ASSET_PAGINATION_SIZE, JobName } from '../job';
-
 import {
   DatabaseLock,
   IAccessRepository,
@@ -39,6 +39,8 @@ import {
   mapLibrary,
 } from './library.dto';
 
+const LIBRARY_SCAN_BATCH_SIZE = 5000;
+
 @Injectable()
 export class LibraryService extends EventEmitter {
   readonly logger = new ImmichLogger(LibraryService.name);
@@ -665,6 +667,58 @@ export class LibraryService extends EventEmitter {
 
     this.logger.verbose(`Refreshing library: ${job.id}`);
 
+    const crawledAssetPaths = await this.getPathTrie(library);
+    this.logger.debug(`Found ${crawledAssetPaths.size} asset(s) when crawling import paths ${library.importPaths}`);
+
+    const assetIdsToMarkOnline = [];
+    const pagination = usePagination(LIBRARY_SCAN_BATCH_SIZE, (pagination) =>
+      this.assetRepository.getLibraryAssetPaths(pagination, library.id),
+    );
+
+    const shouldScanAll = job.refreshAllFiles || job.refreshModifiedFiles;
+    for await (const page of pagination) {
+      for (const asset of page) {
+        if (asset.isOffline) {
+          assetIdsToMarkOnline.push(asset.id);
+        }
+
+        if (!shouldScanAll) {
+          crawledAssetPaths.delete(asset.originalPath);
+        }
+      }
+    }
+
+    if (assetIdsToMarkOnline.length > 0) {
+      this.logger.debug(`Found ${assetIdsToMarkOnline.length} online asset(s) previously marked as offline`);
+      await this.assetRepository.updateAll(assetIdsToMarkOnline, { isOffline: false });
+    }
+
+    if (crawledAssetPaths.size > 0) {
+      if (!shouldScanAll) {
+        this.logger.debug(`Will import ${crawledAssetPaths.size} new asset(s)`);
+      }
+
+      const batch = [];
+      for (const assetPath of crawledAssetPaths) {
+        batch.push(assetPath);
+
+        if (batch.length >= LIBRARY_SCAN_BATCH_SIZE) {
+          await this.scanAssets(job.id, batch, library.ownerId, job.refreshAllFiles ?? false);
+          batch.length = 0;
+        }
+      }
+
+      if (batch.length > 0) {
+        await this.scanAssets(job.id, batch, library.ownerId, job.refreshAllFiles ?? false);
+      }
+    }
+
+    await this.repository.update({ id: job.id, refreshedAt: new Date() });
+
+    return true;
+  }
+
+  private async getPathTrie(library: LibraryEntity): Promise<Trie<string>> {
     const pathValidation = await Promise.all(
       library.importPaths.map(async (importPath) => await this.validateImportPath(importPath)),
     );
@@ -679,50 +733,17 @@ export class LibraryService extends EventEmitter {
       .filter((validation) => validation.isValid)
       .map((validation) => validation.importPath);
 
-    let rawPaths = await this.storageRepository.crawl({
+    const generator = this.storageRepository.walk({
       pathsToCrawl: validImportPaths,
       exclusionPatterns: library.exclusionPatterns,
     });
-    const crawledAssetPaths = new Set<string>(rawPaths);
 
-    const shouldScanAll = job.refreshAllFiles || job.refreshModifiedFiles;
+    const trie = new Trie<string>();
-    let pathsToScan: string[] = shouldScanAll ? rawPaths : [];
+    for await (const filePath of generator) {
-    rawPaths = [];
+      trie.add(filePath);
-
-    this.logger.debug(`Found ${crawledAssetPaths.size} asset(s) when crawling import paths ${library.importPaths}`);
-
-    const assetIdsToMarkOnline = [];
-    const pagination = usePagination(5000, (pagination) =>
-      this.assetRepository.getLibraryAssetPaths(pagination, library.id),
-    );
-
-    for await (const page of pagination) {
-      for (const asset of page) {
-        if (asset.isOffline) {
-          assetIdsToMarkOnline.push(asset.id);
-        }
-
-        crawledAssetPaths.delete(asset.originalPath);
-      }
     }
 
-    if (assetIdsToMarkOnline.length > 0) {
+    return trie;
-      this.logger.debug(`Found ${assetIdsToMarkOnline.length} online asset(s) previously marked as offline`);
-      await this.assetRepository.updateAll(assetIdsToMarkOnline, { isOffline: false });
-    }
-
-    if (!shouldScanAll) {
-      pathsToScan = [...crawledAssetPaths];
-      this.logger.debug(`Will import ${pathsToScan.length} new asset(s)`);
-    }
-
-    if (pathsToScan.length > 0) {
-      await this.scanAssets(job.id, pathsToScan, library.ownerId, job.refreshAllFiles ?? false);
-    }
-
-    await this.repository.update({ id: job.id, refreshedAt: new Date() });
-
-    return true;
   }
 
   private async findOrFail(id: string) {

server/src/domain/media/media.service.ts

@@ -165,7 +165,7 @@ export class MediaService {
   }
 
   async handleGenerateJpegThumbnail({ id }: IEntityJob) {
-    const [asset] = await this.assetRepository.getByIds([id]);
+    const [asset] = await this.assetRepository.getByIds([id], { exifInfo: true });
     if (!asset) {
       return false;
     }
@@ -215,7 +215,7 @@ export class MediaService {
   }
 
   async handleGenerateWebpThumbnail({ id }: IEntityJob) {
-    const [asset] = await this.assetRepository.getByIds([id]);
+    const [asset] = await this.assetRepository.getByIds([id], { exifInfo: true });
     if (!asset) {
       return false;
     }

server/src/domain/metadata/metadata.service.spec.ts

@@ -114,7 +114,7 @@ describe(MetadataService.name, () => {
   describe('handleLivePhotoLinking', () => {
     it('should handle an asset that could not be found', async () => {
       await expect(sut.handleLivePhotoLinking({ id: assetStub.image.id })).resolves.toBe(false);
-      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
+      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
       expect(assetMock.findLivePhotoMatch).not.toHaveBeenCalled();
       expect(assetMock.save).not.toHaveBeenCalled();
       expect(albumMock.removeAsset).not.toHaveBeenCalled();
@@ -124,7 +124,7 @@ describe(MetadataService.name, () => {
       assetMock.getByIds.mockResolvedValue([{ ...assetStub.image, exifInfo: undefined }]);
 
       await expect(sut.handleLivePhotoLinking({ id: assetStub.image.id })).resolves.toBe(false);
-      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
+      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
       expect(assetMock.findLivePhotoMatch).not.toHaveBeenCalled();
       expect(assetMock.save).not.toHaveBeenCalled();
       expect(albumMock.removeAsset).not.toHaveBeenCalled();
@@ -134,7 +134,7 @@ describe(MetadataService.name, () => {
       assetMock.getByIds.mockResolvedValue([{ ...assetStub.image }]);
 
       await expect(sut.handleLivePhotoLinking({ id: assetStub.image.id })).resolves.toBe(true);
-      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
+      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
       expect(assetMock.findLivePhotoMatch).not.toHaveBeenCalled();
       expect(assetMock.save).not.toHaveBeenCalled();
       expect(albumMock.removeAsset).not.toHaveBeenCalled();
@@ -149,7 +149,7 @@ describe(MetadataService.name, () => {
       ]);
 
       await expect(sut.handleLivePhotoLinking({ id: assetStub.livePhotoMotionAsset.id })).resolves.toBe(true);
-      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id]);
+      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id], { exifInfo: true });
       expect(assetMock.findLivePhotoMatch).toHaveBeenCalledWith({
         livePhotoCID: assetStub.livePhotoStillAsset.id,
         ownerId: assetStub.livePhotoMotionAsset.ownerId,
@@ -170,7 +170,7 @@ describe(MetadataService.name, () => {
       assetMock.findLivePhotoMatch.mockResolvedValue(assetStub.livePhotoMotionAsset);
 
       await expect(sut.handleLivePhotoLinking({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe(true);
-      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id]);
+      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id], { exifInfo: true });
       expect(assetMock.findLivePhotoMatch).toHaveBeenCalledWith({
         livePhotoCID: assetStub.livePhotoMotionAsset.id,
         ownerId: assetStub.livePhotoStillAsset.ownerId,

server/src/domain/metadata/metadata.service.ts

@@ -153,7 +153,7 @@ export class MetadataService {
 
   async handleLivePhotoLinking(job: IEntityJob) {
     const { id } = job;
-    const [asset] = await this.assetRepository.getByIds([id]);
+    const [asset] = await this.assetRepository.getByIds([id], { exifInfo: true });
     if (!asset?.exifInfo) {
       return false;
     }

server/src/domain/repositories/asset.repository.ts

@@ -122,6 +122,7 @@ export interface IAssetRepository {
     relations?: FindOptionsRelations<AssetEntity>,
     select?: FindOptionsSelect<AssetEntity>,
   ): Promise<AssetEntity[]>;
+  getByIdsWithAllRelations(ids: string[]): Promise<AssetEntity[]>;
   getByDayOfYear(ownerId: string, monthDay: MonthDay): Promise<AssetEntity[]>;
   getByChecksum(userId: string, checksum: Buffer): Promise<AssetEntity | null>;
   getByAlbumId(pagination: PaginationOptions, albumId: string): Paginated<AssetEntity>;

server/src/domain/repositories/storage.repository.ts

@@ -53,6 +53,7 @@ export interface IStorageRepository {
   readdir(folder: string): Promise<string[]>;
   stat(filepath: string): Promise<Stats>;
   crawl(crawlOptions: CrawlOptionsDto): Promise<string[]>;
+  walk(crawlOptions: CrawlOptionsDto): AsyncGenerator<string>;
   copyFile(source: string, target: string): Promise<void>;
   rename(source: string, target: string): Promise<void>;
   watch(paths: string[], options: WatchOptions, events: Partial<WatchEvents>): () => Promise<void>;

server/src/domain/search/search.service.spec.ts

@@ -76,7 +76,7 @@ describe(SearchService.name, () => {
         fieldName: 'smartInfo.tags',
         items: [{ value: 'train', data: assetStub.imageFrom2015.id }],
       });
-      assetMock.getByIds.mockResolvedValueOnce([assetStub.image, assetStub.imageFrom2015]);
+      assetMock.getByIdsWithAllRelations.mockResolvedValueOnce([assetStub.image, assetStub.imageFrom2015]);
       const expectedResponse = [
         { fieldName: 'exifInfo.city', items: [{ value: 'Paris', data: mapAsset(assetStub.image) }] },
         { fieldName: 'smartInfo.tags', items: [{ value: 'train', data: mapAsset(assetStub.imageFrom2015) }] },

server/src/domain/search/search.service.ts

@@ -60,7 +60,7 @@ export class SearchService {
       this.assetRepository.getAssetIdByTag(auth.user.id, options),
     ]);
     const assetIds = new Set<string>(results.flatMap((field) => field.items.map((item) => item.data)));
-    const assets = await this.assetRepository.getByIds([...assetIds]);
+    const assets = await this.assetRepository.getByIdsWithAllRelations([...assetIds]);
     const assetMap = new Map<string, AssetResponseDto>(assets.map((asset) => [asset.id, mapAsset(asset)]));
 
     return results.map(({ fieldName, items }) => ({

server/src/domain/smart-info/smart-info.service.ts

@@ -76,6 +76,10 @@ export class SmartInfoService {
     }
 
     const [asset] = await this.assetRepository.getByIds([id]);
+    if (!asset) {
+      return false;
+    }
+
     if (!asset.resizePath) {
       return false;
     }

server/src/domain/storage-template/storage-template.service.spec.ts

@@ -101,11 +101,11 @@ describe(StorageTemplateService.name, () => {
         .mockResolvedValue(assetStub.livePhotoMotionAsset);
 
       when(assetMock.getByIds)
-        .calledWith([assetStub.livePhotoStillAsset.id])
+        .calledWith([assetStub.livePhotoStillAsset.id], { exifInfo: true })
         .mockResolvedValue([assetStub.livePhotoStillAsset]);
 
       when(assetMock.getByIds)
-        .calledWith([assetStub.livePhotoMotionAsset.id])
+        .calledWith([assetStub.livePhotoMotionAsset.id], { exifInfo: true })
         .mockResolvedValue([assetStub.livePhotoMotionAsset]);
 
       when(moveMock.create)
@@ -140,8 +140,8 @@ describe(StorageTemplateService.name, () => {
 
       await expect(sut.handleMigrationSingle({ id: assetStub.livePhotoStillAsset.id })).resolves.toBe(true);
 
-      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id]);
+      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoStillAsset.id], { exifInfo: true });
-      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id]);
+      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.livePhotoMotionAsset.id], { exifInfo: true });
       expect(storageMock.checkFileExists).toHaveBeenCalledTimes(2);
       expect(assetMock.save).toHaveBeenCalledWith({
         id: assetStub.livePhotoStillAsset.id,
@@ -172,7 +172,9 @@ describe(StorageTemplateService.name, () => {
         .calledWith({ id: assetStub.image.id, originalPath: newPath })
         .mockResolvedValue(assetStub.image);
 
-      when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
+      when(assetMock.getByIds)
+        .calledWith([assetStub.image.id], { exifInfo: true })
+        .mockResolvedValue([assetStub.image]);
 
       when(moveMock.update)
         .calledWith({
@@ -190,7 +192,7 @@ describe(StorageTemplateService.name, () => {
 
       await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
 
-      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
+      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
       expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
       expect(storageMock.rename).toHaveBeenCalledWith(assetStub.image.originalPath, newPath);
       expect(moveMock.update).toHaveBeenCalledWith({
@@ -227,7 +229,9 @@ describe(StorageTemplateService.name, () => {
         .calledWith({ id: assetStub.image.id, originalPath: newPath })
         .mockResolvedValue(assetStub.image);
 
-      when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
+      when(assetMock.getByIds)
+        .calledWith([assetStub.image.id], { exifInfo: true })
+        .mockResolvedValue([assetStub.image]);
 
       when(moveMock.update)
         .calledWith({
@@ -245,7 +249,7 @@ describe(StorageTemplateService.name, () => {
 
       await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
 
-      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
+      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
       expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
       expect(storageMock.stat).toHaveBeenCalledWith(previousFailedNewPath);
       expect(storageMock.rename).toHaveBeenCalledWith(previousFailedNewPath, newPath);
@@ -275,7 +279,9 @@ describe(StorageTemplateService.name, () => {
         .calledWith({ id: assetStub.image.id, originalPath: newPath })
         .mockResolvedValue(assetStub.image);
 
-      when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
+      when(assetMock.getByIds)
+        .calledWith([assetStub.image.id], { exifInfo: true })
+        .mockResolvedValue([assetStub.image]);
 
       when(moveMock.create)
         .calledWith({
@@ -294,7 +300,7 @@ describe(StorageTemplateService.name, () => {
 
       await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
 
-      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
+      expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
       expect(storageMock.checkFileExists).toHaveBeenCalledTimes(1);
       expect(storageMock.stat).toHaveBeenCalledWith(newPath);
       expect(moveMock.create).toHaveBeenCalledWith({
@@ -340,7 +346,9 @@ describe(StorageTemplateService.name, () => {
           .calledWith({ id: assetStub.image.id, originalPath: newPath })
           .mockResolvedValue(assetStub.image);
 
-        when(assetMock.getByIds).calledWith([assetStub.image.id]).mockResolvedValue([assetStub.image]);
+        when(assetMock.getByIds)
+          .calledWith([assetStub.image.id], { exifInfo: true })
+          .mockResolvedValue([assetStub.image]);
 
         when(moveMock.update)
           .calledWith({
@@ -358,7 +366,7 @@ describe(StorageTemplateService.name, () => {
 
         await expect(sut.handleMigrationSingle({ id: assetStub.image.id })).resolves.toBe(true);
 
-        expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id]);
+        expect(assetMock.getByIds).toHaveBeenCalledWith([assetStub.image.id], { exifInfo: true });
         expect(storageMock.checkFileExists).toHaveBeenCalledTimes(3);
         expect(storageMock.stat).toHaveBeenCalledWith(previousFailedNewPath);
         expect(storageMock.rename).not.toHaveBeenCalled();

server/src/domain/storage-template/storage-template.service.ts

@@ -92,7 +92,10 @@ export class StorageTemplateService {
       return true;
     }
 
-    const [asset] = await this.assetRepository.getByIds([id]);
+    const [asset] = await this.assetRepository.getByIds([id], { exifInfo: true });
+    if (!asset) {
+      return false;
+    }
 
     const user = await this.userRepository.get(asset.ownerId, {});
     const storageLabel = user?.storageLabel || null;
@@ -101,7 +104,10 @@ export class StorageTemplateService {
 
     // move motion part of live photo
     if (asset.livePhotoVideoId) {
-      const [livePhotoVideo] = await this.assetRepository.getByIds([asset.livePhotoVideoId]);
+      const [livePhotoVideo] = await this.assetRepository.getByIds([asset.livePhotoVideoId], { exifInfo: true });
+      if (!livePhotoVideo) {
+        return false;
+      }
       const motionFilename = getLivePhotoMotionFilename(filename, livePhotoVideo.originalPath);
       await this.moveAsset(livePhotoVideo, { storageLabel, filename: motionFilename });
     }

server/src/infra/entities/asset-face.entity.ts

@@ -3,6 +3,7 @@ import { AssetEntity } from './asset.entity';
 import { PersonEntity } from './person.entity';
 
 @Entity('asset_faces', { synchronize: false })
+@Index('IDX_asset_faces_assetId_personId', ['assetId', 'personId'])
 @Index(['personId', 'assetId'])
 export class AssetFaceEntity {
   @PrimaryGeneratedColumn('uuid')

server/src/infra/entities/asset.entity.ts

@@ -35,6 +35,7 @@ export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_library_checksum';
 @Index('IDX_day_of_month', { synchronize: false })
 @Index('IDX_month', { synchronize: false })
 @Index('IDX_originalPath_libraryId', ['originalPath', 'libraryId'])
+@Index('IDX_asset_id_stackId', ['id', 'stackId'])
 @Index('idx_originalFileName_trigram', { synchronize: false })
 // For all assets, each originalpath must be unique per user and library
 export class AssetEntity {
@@ -145,7 +146,7 @@ export class AssetEntity {
   smartSearch?: SmartSearchEntity;
 
   @ManyToMany(() => TagEntity, (tag) => tag.assets, { cascade: true })
-  @JoinTable({ name: 'tag_asset' })
+  @JoinTable({ name: 'tag_asset', synchronize: false })
   tags!: TagEntity[];
 
   @ManyToMany(() => SharedLinkEntity, (link) => link.assets, { cascade: true })

server/src/infra/migrations/1710293990203-AddAssetRelationIndices.ts

@@ -0,0 +1,15 @@
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddAssetRelationIndices1710293990203 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`CREATE INDEX "IDX_asset_id_stackId" on assets ("id", "stackId")`);
+        await queryRunner.query(`CREATE INDEX "IDX_tag_asset_assetsId_tagsId" on tag_asset ("assetsId", "tagsId")`);
+        await queryRunner.query(`CREATE INDEX "IDX_asset_faces_assetId_personId" on asset_faces ("assetId", "personId")`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`DROP INDEX "IDX_asset_id_stackId" on assets ("id", "stackId")`);
+        await queryRunner.query(`DROP INDEX "IDX_tag_asset_assetsId_tagsId" on tag_asset ("assetsId", "tagsId")`);
+        await queryRunner.query(`DROP INDEX "IDX_asset_faces_assetId_personId" on asset_faces ("assetId", "personId")`);
+    }
+}

server/src/infra/repositories/asset.repository.ts

@@ -137,8 +137,20 @@ export class AssetRepository implements IAssetRepository {
     relations?: FindOptionsRelations<AssetEntity>,
     select?: FindOptionsSelect<AssetEntity>,
   ): Promise<AssetEntity[]> {
-    if (!relations) {
+    return this.repository.find({
-      relations = {
+      where: { id: In(ids) },
+      relations,
+      select,
+      withDeleted: true,
+    });
+  }
+
+  @GenerateSql({ params: [[DummyValue.UUID]] })
+  @ChunkedArray()
+  getByIdsWithAllRelations(ids: string[]): Promise<AssetEntity[]> {
+    return this.repository.find({
+      where: { id: In(ids) },
+      relations: {
         exifInfo: true,
         smartInfo: true,
         tags: true,
@@ -148,13 +160,7 @@ export class AssetRepository implements IAssetRepository {
         stack: {
           assets: true,
         },
-      };
+      },
-    }
-
-    return this.repository.find({
-      where: { id: In(ids) },
-      relations,
-      select,
       withDeleted: true,
     });
   }

server/src/infra/repositories/filesystem.provider.ts

@@ -11,7 +11,7 @@ import {
 import { ImmichLogger } from '@app/infra/logger';
 import archiver from 'archiver';
 import chokidar, { WatchOptions } from 'chokidar';
-import { glob } from 'fast-glob';
+import { glob, globStream } from 'fast-glob';
 import { constants, createReadStream, existsSync, mkdirSync } from 'node:fs';
 import fs from 'node:fs/promises';
 import path from 'node:path';
@@ -141,10 +141,7 @@ export class FilesystemProvider implements IStorageRepository {
       return Promise.resolve([]);
     }
 
-    const base = pathsToCrawl.length === 1 ? pathsToCrawl[0] : `{${pathsToCrawl.join(',')}}`;
+    return glob(this.asGlob(pathsToCrawl), {
-    const extensions = `*{${mimeTypes.getSupportedFileExtensions().join(',')}}`;
-
-    return glob(`${base}/**/${extensions}`, {
       absolute: true,
       caseSensitiveMatch: false,
       onlyFiles: true,
@@ -153,6 +150,26 @@ export class FilesystemProvider implements IStorageRepository {
     });
   }
 
+  async *walk(crawlOptions: CrawlOptionsDto): AsyncGenerator<string> {
+    const { pathsToCrawl, exclusionPatterns, includeHidden } = crawlOptions;
+    if (pathsToCrawl.length === 0) {
+      async function* emptyGenerator() {}
+      return emptyGenerator();
+    }
+
+    const stream = globStream(this.asGlob(pathsToCrawl), {
+      absolute: true,
+      caseSensitiveMatch: false,
+      onlyFiles: true,
+      dot: includeHidden,
+      ignore: exclusionPatterns,
+    });
+
+    for await (const value of stream) {
+      yield value as string;
+    }
+  }
+
   watch(paths: string[], options: WatchOptions, events: Partial<WatchEvents>) {
     const watcher = chokidar.watch(paths, options);
 
@@ -164,4 +181,10 @@ export class FilesystemProvider implements IStorageRepository {
 
     return () => watcher.close();
   }
+
+  private asGlob(pathsToCrawl: string[]): string {
+    const base = pathsToCrawl.length === 1 ? pathsToCrawl[0] : `{${pathsToCrawl.join(',')}}`;
+    const extensions = `*{${mimeTypes.getSupportedFileExtensions().join(',')}}`;
+    return `${base}/**/${extensions}`;
+  }
 }

server/src/infra/sql/asset.repository.sql

@@ -160,6 +160,42 @@ ORDER BY
   "entity"."localDateTime" DESC
 
 -- AssetRepository.getByIds
+SELECT
+  "AssetEntity"."id" AS "AssetEntity_id",
+  "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId",
+  "AssetEntity"."ownerId" AS "AssetEntity_ownerId",
+  "AssetEntity"."libraryId" AS "AssetEntity_libraryId",
+  "AssetEntity"."deviceId" AS "AssetEntity_deviceId",
+  "AssetEntity"."type" AS "AssetEntity_type",
+  "AssetEntity"."originalPath" AS "AssetEntity_originalPath",
+  "AssetEntity"."resizePath" AS "AssetEntity_resizePath",
+  "AssetEntity"."webpPath" AS "AssetEntity_webpPath",
+  "AssetEntity"."thumbhash" AS "AssetEntity_thumbhash",
+  "AssetEntity"."encodedVideoPath" AS "AssetEntity_encodedVideoPath",
+  "AssetEntity"."createdAt" AS "AssetEntity_createdAt",
+  "AssetEntity"."updatedAt" AS "AssetEntity_updatedAt",
+  "AssetEntity"."deletedAt" AS "AssetEntity_deletedAt",
+  "AssetEntity"."fileCreatedAt" AS "AssetEntity_fileCreatedAt",
+  "AssetEntity"."localDateTime" AS "AssetEntity_localDateTime",
+  "AssetEntity"."fileModifiedAt" AS "AssetEntity_fileModifiedAt",
+  "AssetEntity"."isFavorite" AS "AssetEntity_isFavorite",
+  "AssetEntity"."isArchived" AS "AssetEntity_isArchived",
+  "AssetEntity"."isExternal" AS "AssetEntity_isExternal",
+  "AssetEntity"."isReadOnly" AS "AssetEntity_isReadOnly",
+  "AssetEntity"."isOffline" AS "AssetEntity_isOffline",
+  "AssetEntity"."checksum" AS "AssetEntity_checksum",
+  "AssetEntity"."duration" AS "AssetEntity_duration",
+  "AssetEntity"."isVisible" AS "AssetEntity_isVisible",
+  "AssetEntity"."livePhotoVideoId" AS "AssetEntity_livePhotoVideoId",
+  "AssetEntity"."originalFileName" AS "AssetEntity_originalFileName",
+  "AssetEntity"."sidecarPath" AS "AssetEntity_sidecarPath",
+  "AssetEntity"."stackId" AS "AssetEntity_stackId"
+FROM
+  "assets" "AssetEntity"
+WHERE
+  (("AssetEntity"."id" IN ($1)))
+
+-- AssetRepository.getByIdsWithAllRelations
 SELECT
   "AssetEntity"."id" AS "AssetEntity_id",
   "AssetEntity"."deviceAssetId" AS "AssetEntity_deviceAssetId",

server/test/repositories/asset.repository.mock.ts

@@ -8,6 +8,7 @@ export const newAssetRepositoryMock = (): jest.Mocked<IAssetRepository> => {
     getByDate: jest.fn(),
     getByDayOfYear: jest.fn(),
     getByIds: jest.fn().mockResolvedValue([]),
+    getByIdsWithAllRelations: jest.fn().mockResolvedValue([]),
     getByAlbumId: jest.fn(),
     getByUserId: jest.fn(),
     getById: jest.fn(),

server/test/repositories/storage.repository.mock.ts

@@ -56,6 +56,7 @@ export const newStorageRepositoryMock = (reset = true): jest.Mocked<IStorageRepo
     readdir: jest.fn(),
     stat: jest.fn(),
     crawl: jest.fn(),
+    walk: jest.fn().mockImplementation(async function* () {}),
     rename: jest.fn(),
     copyFile: jest.fn(),
     utimes: jest.fn(),

web/src/lib/components/shared-components/fullscreen-container.svelte

@@ -10,7 +10,7 @@
     class="flex w-full max-w-lg flex-col gap-4 rounded-3xl border bg-white p-8 shadow-sm dark:border-immich-dark-gray dark:bg-immich-dark-gray"
   >
     <div class="flex flex-col place-content-center place-items-center gap-4 py-4">
-      <ImmichLogo class="h-24 w-24" />
+      <ImmichLogo noText class="h-24 w-24" />
       <h1 class="text-2xl font-medium text-immich-primary dark:text-immich-dark-primary">
         {title}
       </h1>