WIP

docs/docs/administration/server-stats.md

@@ -3,7 +3,7 @@
 Server statistics to show the total number of videos, photos, and usage per user.
 
 :::info
-If a storage quota has been defined for the user, the usage number will be displayed as a percentage of the total storage quota allocated to him.
+If a storage quota has been defined for the user, the usage number will be displayed as a percentage of the total storage quota allocated to them.
 :::
 
 :::info External library

e2e/src/api/specs/library.e2e-spec.ts

@@ -6,7 +6,7 @@ import {
   getAllLibraries,
   scanLibrary,
 } from '@immich/sdk';
-import { existsSync, rmdirSync } from 'node:fs';
+import { cpSync, existsSync, readdirSync } from 'node:fs';
 import { Socket } from 'socket.io-client';
 import { userDto, uuidDto } from 'src/fixtures';
 import { errorDto } from 'src/responses';
@@ -33,14 +33,12 @@ describe('/library', () => {
 
   afterAll(() => {
     utils.disconnectWebsocket(websocket);
+    utils.deleteTempFolder();
   });
 
-  beforeEach(() => {
+  beforeEach(async () => {
     utils.resetEvents();
-    const tempDir = `${testAssetDir}/temp`;
-    if (existsSync(tempDir)) {
-      rmdirSync(tempDir, { recursive: true });
-    }
+    utils.deleteTempFolder();
     utils.createImageFile(`${testAssetDir}/temp/directoryA/assetA.png`);
     utils.createImageFile(`${testAssetDir}/temp/directoryB/assetB.png`);
   });
@@ -357,95 +355,6 @@ describe('/library', () => {
     });
   });
 
-  describe('DELETE /library/:id', () => {
-    it('should require authentication', async () => {
-      const { status, body } = await request(app).delete(`/library/${uuidDto.notFound}`);
-
-      expect(status).toBe(401);
-      expect(body).toEqual(errorDto.unauthorized);
-    });
-
-    it('should not delete the last upload library', async () => {
-      const libraries = await getAllLibraries(
-        { $type: LibraryType.Upload },
-        { headers: asBearerAuth(admin.accessToken) },
-      );
-
-      const adminLibraries = libraries.filter((library) => library.ownerId === admin.userId);
-      expect(adminLibraries.length).toBeGreaterThanOrEqual(1);
-      const lastLibrary = adminLibraries.pop() as LibraryResponseDto;
-
-      // delete all but the last upload library
-      for (const library of adminLibraries) {
-        const { status } = await request(app)
-          .delete(`/library/${library.id}`)
-          .set('Authorization', `Bearer ${admin.accessToken}`);
-        expect(status).toBe(204);
-      }
-
-      const { status, body } = await request(app)
-        .delete(`/library/${lastLibrary.id}`)
-        .set('Authorization', `Bearer ${admin.accessToken}`);
-
-      expect(body).toEqual(errorDto.noDeleteUploadLibrary);
-      expect(status).toBe(400);
-    });
-
-    it('should delete an external library', async () => {
-      const library = await utils.createLibrary(admin.accessToken, {
-        ownerId: admin.userId,
-        type: LibraryType.External,
-      });
-
-      const { status, body } = await request(app)
-        .delete(`/library/${library.id}`)
-        .set('Authorization', `Bearer ${admin.accessToken}`);
-
-      expect(status).toBe(204);
-      expect(body).toEqual({});
-
-      const libraries = await getAllLibraries({}, { headers: asBearerAuth(admin.accessToken) });
-      expect(libraries).not.toEqual(
-        expect.arrayContaining([
-          expect.objectContaining({
-            id: library.id,
-          }),
-        ]),
-      );
-    });
-
-    it('should delete an external library with assets', async () => {
-      const library = await utils.createLibrary(admin.accessToken, {
-        ownerId: admin.userId,
-        type: LibraryType.External,
-        importPaths: [`${testAssetDirInternal}/temp`],
-      });
-
-      await scan(admin.accessToken, library.id);
-      await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 2 });
-
-      const { status, body } = await request(app)
-        .delete(`/library/${library.id}`)
-        .set('Authorization', `Bearer ${admin.accessToken}`);
-
-      expect(status).toBe(204);
-      expect(body).toEqual({});
-
-      const libraries = await getAllLibraries({}, { headers: asBearerAuth(admin.accessToken) });
-      expect(libraries).not.toEqual(
-        expect.arrayContaining([
-          expect.objectContaining({
-            id: library.id,
-          }),
-        ]),
-      );
-
-      // ensure no files get deleted
-      expect(existsSync(`${testAssetDir}/temp/directoryA/assetA.png`)).toBe(true);
-      expect(existsSync(`${testAssetDir}/temp/directoryB/assetB.png`)).toBe(true);
-    });
-  });
-
   describe('GET /library/:id/statistics', () => {
     it('should require authentication', async () => {
       const { status, body } = await request(app).get(`/library/${uuidDto.notFound}/statistics`);
@@ -550,6 +459,51 @@ describe('/library', () => {
 
       expect(newAssets.count).toBe(3);
     });
+
+    it('should offline missing files', async () => {
+      cpSync(`${testAssetDir}/albums/nature`, `${testAssetDir}/temp`, {
+        recursive: true,
+      });
+      console.log(readdirSync(`${testAssetDir}/temp`));
+
+      const library = await utils.createLibrary(admin.accessToken, {
+        ownerId: admin.userId,
+        type: LibraryType.External,
+        importPaths: [`${testAssetDirInternal}/temp`],
+      });
+
+      await scan(admin.accessToken, library.id, { refreshAllFiles: true });
+      // await setTimeout(8000);
+      await utils.waitForQueueFinish(admin.accessToken, 'library');
+      // await utils.waitForQueueFinish(admin.accessToken, 'metadataExtraction');
+      // await utils.waitForQueueFinish(admin.accessToken, 'thumbnailGeneration');
+
+      const onlineAssets = await utils.getAllAssets(admin.accessToken);
+      console.log(onlineAssets);
+      // expect(onlineAssets.length).toBeGreaterThan(1);
+      // console.log(onlineAssets)
+
+      utils.deleteTempFolder();
+
+      await scan(admin.accessToken, library.id);
+      await utils.waitForQueueFinish(admin.accessToken, 'library');
+
+      const assets = await utils.getAllAssets(admin.accessToken);
+      console.log(assets);
+
+      expect(assets).toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            isOffline: true,
+            originalFileName: 'el_torcal_rocks.jpg',
+          }),
+          expect.objectContaining({
+            isOffline: true,
+            originalFileName: 'tanners_ridge.jpg',
+          }),
+        ]),
+      );
+    });
   });
 
   describe('POST /library/:id/removeOffline', () => {
@@ -608,4 +562,93 @@ describe('/library', () => {
       });
     });
   });
+
+  describe('DELETE /library/:id', () => {
+    it('should require authentication', async () => {
+      const { status, body } = await request(app).delete(`/library/${uuidDto.notFound}`);
+
+      expect(status).toBe(401);
+      expect(body).toEqual(errorDto.unauthorized);
+    });
+
+    it('should not delete the last upload library', async () => {
+      const libraries = await getAllLibraries(
+        { $type: LibraryType.Upload },
+        { headers: asBearerAuth(admin.accessToken) },
+      );
+
+      const adminLibraries = libraries.filter((library) => library.ownerId === admin.userId);
+      expect(adminLibraries.length).toBeGreaterThanOrEqual(1);
+      const lastLibrary = adminLibraries.pop() as LibraryResponseDto;
+
+      // delete all but the last upload library
+      for (const library of adminLibraries) {
+        const { status } = await request(app)
+          .delete(`/library/${library.id}`)
+          .set('Authorization', `Bearer ${admin.accessToken}`);
+        expect(status).toBe(204);
+      }
+
+      const { status, body } = await request(app)
+        .delete(`/library/${lastLibrary.id}`)
+        .set('Authorization', `Bearer ${admin.accessToken}`);
+
+      expect(body).toEqual(errorDto.noDeleteUploadLibrary);
+      expect(status).toBe(400);
+    });
+
+    it('should delete an external library', async () => {
+      const library = await utils.createLibrary(admin.accessToken, {
+        ownerId: admin.userId,
+        type: LibraryType.External,
+      });
+
+      const { status, body } = await request(app)
+        .delete(`/library/${library.id}`)
+        .set('Authorization', `Bearer ${admin.accessToken}`);
+
+      expect(status).toBe(204);
+      expect(body).toEqual({});
+
+      const libraries = await getAllLibraries({}, { headers: asBearerAuth(admin.accessToken) });
+      expect(libraries).not.toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            id: library.id,
+          }),
+        ]),
+      );
+    });
+
+    it('should delete an external library with assets', async () => {
+      const library = await utils.createLibrary(admin.accessToken, {
+        ownerId: admin.userId,
+        type: LibraryType.External,
+        importPaths: [`${testAssetDirInternal}/temp`],
+      });
+
+      await scan(admin.accessToken, library.id);
+      await utils.waitForWebsocketEvent({ event: 'assetUpload', total: 2 });
+
+      const { status, body } = await request(app)
+        .delete(`/library/${library.id}`)
+        .set('Authorization', `Bearer ${admin.accessToken}`);
+
+      expect(status).toBe(204);
+      expect(body).toEqual({});
+
+      const libraries = await getAllLibraries({}, { headers: asBearerAuth(admin.accessToken) });
+      expect(libraries).not.toEqual(
+        expect.arrayContaining([
+          expect.objectContaining({
+            id: library.id,
+          }),
+        ]),
+      );
+
+      // ensure no files get deleted
+      expect(existsSync(`${testAssetDir}/temp/directoryA/assetA.png`)).toBe(true);
+      expect(existsSync(`${testAssetDir}/temp/directoryB/assetB.png`)).toBe(true);
+    });
+  });
 });

e2e/src/utils.ts

@@ -1,4 +1,5 @@
 import {
+  AllJobStatusResponseDto,
   AssetFileUploadResponseDto,
   AssetResponseDto,
   CreateAlbumDto,
@@ -18,6 +19,7 @@ import {
   defaults,
   deleteAssets,
   getAllAssets,
+  getAllJobsStatus,
   getAssetInfo,
   login,
   searchMetadata,
@@ -31,6 +33,7 @@ import { createHash } from 'node:crypto';
 import { existsSync, mkdirSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import path, { dirname } from 'node:path';
+import { setTimeout as setAsyncTimeout } from 'node:timers/promises';
 import { promisify } from 'node:util';
 import pg from 'pg';
 import { io, type Socket } from 'socket.io-client';
@@ -406,6 +409,35 @@ export const utils = {
       },
     ]),
 
+  deleteTempFolder: () => {
+    rmSync(`${testAssetDir}/temp`, { recursive: true, force: true });
+  },
+
+  isQueueEmpty: async (accessToken: string, queue: keyof AllJobStatusResponseDto) => {
+    const queues = await getAllJobsStatus({ headers: asBearerAuth(accessToken) });
+    const jobCounts = queues[queue].jobCounts;
+    console.log(jobCounts);
+    return !jobCounts.active && !jobCounts.waiting;
+  },
+
+  waitForQueueFinish: (accessToken: string, queue: keyof AllJobStatusResponseDto, ms?: number) => {
+    return new Promise<void>(async (resolve, reject) => {
+      const timeout = setTimeout(() => reject(new Error('Timed out waiting for queue to empty')), ms || 10_000);
+
+      while (true) {
+        const done = await utils.isQueueEmpty(accessToken, queue);
+        if (done) {
+          break;
+        }
+        await setAsyncTimeout(300);
+      }
+
+      clearTimeout(timeout);
+      // await setAsyncTimeout(5000);
+      resolve();
+    });
+  },
+
   cliLogin: async (accessToken: string) => {
     const key = await utils.createApiKey(accessToken);
     await immichCli(['login', app, `${key.secret}`]);

e2e/vitest.config.ts

@@ -12,7 +12,7 @@ export default defineConfig({
   test: {
     include: ['src/{api,cli}/specs/*.e2e-spec.ts'],
     globalSetup,
-    testTimeout: 10_000,
+    testTimeout: 15_000,
     poolOptions: {
       threads: {
         singleThread: true,

mobile/android/fastlane/report.xml

@@ -5,17 +5,17 @@
     
     
       
-      <testcase classname="fastlane.lanes" name="0: default_platform" time="0.000235">
+      <testcase classname="fastlane.lanes" name="0: default_platform" time="0.000219">
         
       </testcase>
     
       
-      <testcase classname="fastlane.lanes" name="1: bundleRelease" time="71.774783">
+      <testcase classname="fastlane.lanes" name="1: bundleRelease" time="67.515419">
         
       </testcase>
     
       
-      <testcase classname="fastlane.lanes" name="2: upload_to_play_store" time="32.283066">
+      <testcase classname="fastlane.lanes" name="2: upload_to_play_store" time="35.431743">
         
       </testcase>
     

mobile/ios/Podfile.lock

@@ -17,6 +17,9 @@ PODS:
   - fluttertoast (0.0.2):
     - Flutter
     - Toast
+  - FMDB (2.7.5):
+    - FMDB/standard (= 2.7.5)
+  - FMDB/standard (2.7.5)
   - geolocator_apple (1.2.0):
     - Flutter
   - image_picker_ios (0.0.1):
@@ -36,7 +39,7 @@ PODS:
     - FlutterMacOS
   - path_provider_ios (0.0.1):
     - Flutter
-  - permission_handler_apple (9.3.0):
+  - permission_handler_apple (9.1.1):
     - Flutter
   - photo_manager (2.0.0):
     - Flutter
@@ -50,7 +53,7 @@ PODS:
     - FlutterMacOS
   - sqflite (0.0.3):
     - Flutter
-    - FlutterMacOS
+    - FMDB (>= 2.7.5)
   - Toast (4.0.0)
   - url_launcher_ios (0.0.1):
     - Flutter
@@ -81,13 +84,14 @@ DEPENDENCIES:
   - photo_manager (from `.symlinks/plugins/photo_manager/ios`)
   - share_plus (from `.symlinks/plugins/share_plus/ios`)
   - shared_preferences_foundation (from `.symlinks/plugins/shared_preferences_foundation/darwin`)
-  - sqflite (from `.symlinks/plugins/sqflite/darwin`)
+  - sqflite (from `.symlinks/plugins/sqflite/ios`)
   - url_launcher_ios (from `.symlinks/plugins/url_launcher_ios/ios`)
   - video_player_avfoundation (from `.symlinks/plugins/video_player_avfoundation/darwin`)
   - wakelock_plus (from `.symlinks/plugins/wakelock_plus/ios`)
 
 SPEC REPOS:
   trunk:
+    - FMDB
     - MapLibre
     - ReachabilitySwift
     - SAMKeychain
@@ -135,7 +139,7 @@ EXTERNAL SOURCES:
   shared_preferences_foundation:
     :path: ".symlinks/plugins/shared_preferences_foundation/darwin"
   sqflite:
-    :path: ".symlinks/plugins/sqflite/darwin"
+    :path: ".symlinks/plugins/sqflite/ios"
   url_launcher_ios:
     :path: ".symlinks/plugins/url_launcher_ios/ios"
   video_player_avfoundation:
@@ -151,23 +155,24 @@ SPEC CHECKSUMS:
   flutter_native_splash: 52501b97d1c0a5f898d687f1646226c1f93c56ef
   flutter_udid: a2482c67a61b9c806ef59dd82ed8d007f1b7ac04
   flutter_web_auth: c25208760459cec375a3c39f6a8759165ca0fa4d
-  fluttertoast: 31b00dabfa7fb7bacd9e7dbee580d7a2ff4bf265
+  fluttertoast: fafc4fa4d01a6a9e4f772ecd190ffa525e9e2d9c
+  FMDB: 2ce00b547f966261cd18927a3ddb07cb6f3db82a
   geolocator_apple: 9157311f654584b9bb72686c55fc02a97b73f461
-  image_picker_ios: 99dfe1854b4fa34d0364e74a78448a0151025425
+  image_picker_ios: 4a8aadfbb6dc30ad5141a2ce3832af9214a705b5
   integration_test: 13825b8a9334a850581300559b8839134b124670
   isar_flutter_libs: b69f437aeab9c521821c3f376198c4371fa21073
   MapLibre: 620fc933c1d6029b33738c905c1490d024e5d4ef
   maplibre_gl: a2efec727dd340e4c65e26d2b03b584f14881fd9
   package_info_plus: 115f4ad11e0698c8c1c5d8a689390df880f47e85
-  path_provider_foundation: 3784922295ac71e43754bd15e0653ccfd36a147c
+  path_provider_foundation: 29f094ae23ebbca9d3d0cec13889cd9060c0e943
   path_provider_ios: 14f3d2fd28c4fdb42f44e0f751d12861c43cee02
-  permission_handler_apple: 036b856153a2b1f61f21030ff725f3e6fece2b78
+  permission_handler_apple: e76247795d700c14ea09e3a2d8855d41ee80a2e6
   photo_manager: 4f6810b7dfc4feb03b461ac1a70dacf91fba7604
   ReachabilitySwift: 985039c6f7b23a1da463388634119492ff86c825
   SAMKeychain: 483e1c9f32984d50ca961e26818a534283b4cd5c
   share_plus: c3fef564749587fc939ef86ffb283ceac0baf9f5
-  shared_preferences_foundation: b4c3b4cddf1c21f02770737f147a3f5da9d39695
-  sqflite: 673a0e54cc04b7d6dba8d24fb8095b31c3a99eec
+  shared_preferences_foundation: 5b919d13b803cadd15ed2dc053125c68730e5126
+  sqflite: 31f7eba61e3074736dff8807a9b41581e4f7f15a
   Toast: 91b396c56ee72a5790816f40d3a94dd357abc196
   url_launcher_ios: bbd758c6e7f9fd7b5b1d4cde34d2b95fcce5e812
   video_player_avfoundation: 02011213dab73ae3687df27ce441fbbcc82b5579
@@ -175,4 +180,4 @@ SPEC CHECKSUMS:
 
 PODFILE CHECKSUM: 64c9b5291666c0ca3caabdfe9865c141ac40321d
 
-COCOAPODS: 1.15.2
+COCOAPODS: 1.12.1

mobile/ios/Runner.xcodeproj/project.pbxproj

@@ -383,7 +383,7 @@
 				CODE_SIGN_ENTITLEMENTS = Runner/RunnerProfile.entitlements;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 146;
+				CURRENT_PROJECT_VERSION = 147;
 				DEVELOPMENT_TEAM = 2F67MQ8R79;
 				ENABLE_BITCODE = NO;
 				INFOPLIST_FILE = Runner/Info.plist;
@@ -525,7 +525,7 @@
 				CLANG_ENABLE_MODULES = YES;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 146;
+				CURRENT_PROJECT_VERSION = 147;
 				DEVELOPMENT_TEAM = 2F67MQ8R79;
 				ENABLE_BITCODE = NO;
 				INFOPLIST_FILE = Runner/Info.plist;
@@ -553,7 +553,7 @@
 				CLANG_ENABLE_MODULES = YES;
 				CODE_SIGN_IDENTITY = "Apple Development";
 				CODE_SIGN_STYLE = Automatic;
-				CURRENT_PROJECT_VERSION = 146;
+				CURRENT_PROJECT_VERSION = 147;
 				DEVELOPMENT_TEAM = 2F67MQ8R79;
 				ENABLE_BITCODE = NO;
 				INFOPLIST_FILE = Runner/Info.plist;

mobile/ios/Runner/Info.plist

@@ -55,11 +55,11 @@
 		<key>CFBundlePackageType</key>
 		<string>APPL</string>
 		<key>CFBundleShortVersionString</key>
-		<string>1.100.0</string>
+		<string>1.101.0</string>
 		<key>CFBundleSignature</key>
 		<string>????</string>
 		<key>CFBundleVersion</key>
-		<string>146</string>
+		<string>147</string>
 		<key>FLTEnableImpeller</key>
 		<true/>
 		<key>ITSAppUsesNonExemptEncryption</key>

mobile/ios/fastlane/report.xml

@@ -5,32 +5,32 @@
     
     
       
-      <testcase classname="fastlane.lanes" name="0: default_platform" time="0.000226">
+      <testcase classname="fastlane.lanes" name="0: default_platform" time="0.000242">
         
       </testcase>
     
       
-      <testcase classname="fastlane.lanes" name="1: increment_version_number" time="0.183824">
+      <testcase classname="fastlane.lanes" name="1: increment_version_number" time="0.761829">
         
       </testcase>
     
       
-      <testcase classname="fastlane.lanes" name="2: latest_testflight_build_number" time="3.799845">
+      <testcase classname="fastlane.lanes" name="2: latest_testflight_build_number" time="4.47461">
         
       </testcase>
     
       
-      <testcase classname="fastlane.lanes" name="3: increment_build_number" time="0.185425">
+      <testcase classname="fastlane.lanes" name="3: increment_build_number" time="0.179512">
         
       </testcase>
     
       
-      <testcase classname="fastlane.lanes" name="4: build_app" time="111.245268">
+      <testcase classname="fastlane.lanes" name="4: build_app" time="165.636347">
         
       </testcase>
     
       
-      <testcase classname="fastlane.lanes" name="5: upload_to_testflight" time="72.572736">
+      <testcase classname="fastlane.lanes" name="5: upload_to_testflight" time="77.651963">
         
       </testcase>
     

server/e2e/jobs/specs/library.e2e-spec.ts

@@ -31,42 +31,6 @@ describe(`${LibraryController.name} (e2e)`, () => {
   });
 
   describe('POST /library/:id/scan', () => {
-    it('should offline missing files', async () => {
-      await fs.promises.cp(`${IMMICH_TEST_ASSET_PATH}/albums/nature`, `${IMMICH_TEST_ASSET_TEMP_PATH}/albums/nature`, {
-        recursive: true,
-      });
-
-      const library = await api.libraryApi.create(server, admin.accessToken, {
-        ownerId: admin.userId,
-        type: LibraryType.EXTERNAL,
-        importPaths: [`${IMMICH_TEST_ASSET_TEMP_PATH}`],
-      });
-
-      await api.libraryApi.scanLibrary(server, admin.accessToken, library.id);
-
-      const onlineAssets = await api.assetApi.getAllAssets(server, admin.accessToken);
-      expect(onlineAssets.length).toBeGreaterThan(1);
-
-      await restoreTempFolder();
-
-      await api.libraryApi.scanLibrary(server, admin.accessToken, library.id);
-
-      const assets = await api.assetApi.getAllAssets(server, admin.accessToken);
-
-      expect(assets).toEqual(
-        expect.arrayContaining([
-          expect.objectContaining({
-            isOffline: true,
-            originalFileName: 'el_torcal_rocks.jpg',
-          }),
-          expect.objectContaining({
-            isOffline: true,
-            originalFileName: 'tanners_ridge.jpg',
-          }),
-        ]),
-      );
-    });
-
     it('should scan new files', async () => {
       const library = await api.libraryApi.create(server, admin.accessToken, {
         ownerId: admin.userId,