From a85b147b3aab1681ab812db937178d459498efc4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 07:55:46 -0400 Subject: [PATCH 1/4] chore(deps): bump docker/build-push-action from 5.2.0 to 5.3.0 (#7947) Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 5.2.0 to 5.3.0. - [Release notes](https://github.com/docker/build-push-action/releases) - [Commits](https://github.com/docker/build-push-action/compare/v5.2.0...v5.3.0) --- updated-dependencies: - dependency-name: docker/build-push-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/cli.yml | 2 +- .github/workflows/docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 13c2df740..c5cd4d15d 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -87,7 +87,7 @@ jobs: type=raw,value=latest,enable=${{ github.event_name == 'workflow_dispatch' }} - name: Build and push image - uses: docker/build-push-action@v5.2.0 + uses: docker/build-push-action@v5.3.0 with: file: cli/Dockerfile platforms: linux/amd64,linux/arm64 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 5b92e44ff..21ecb1f08 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -121,7 +121,7 @@ jobs: fi - name: Build and push image - uses: docker/build-push-action@v5.2.0 + uses: docker/build-push-action@v5.3.0 with: context: ${{ matrix.context }} file: ${{ matrix.file }} From 2f78bff97cc6d437f00275d92f2f62e7fc10db0e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 14 Mar 2024 07:55:55 -0400 Subject: [PATCH 2/4] chore(deps): bump docker/setup-buildx-action from 3.1.0 to 3.2.0 (#7948) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 3.1.0 to 3.2.0. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v3.1.0...v3.2.0) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/cli.yml | 2 +- .github/workflows/docker.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index c5cd4d15d..c3c3fea01 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -58,7 +58,7 @@ jobs: uses: docker/setup-qemu-action@v3.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.1.0 + uses: docker/setup-buildx-action@v3.2.0 - name: Login to GitHub Container Registry uses: docker/login-action@v3 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 21ecb1f08..fd7904415 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -66,7 +66,7 @@ jobs: uses: docker/setup-qemu-action@v3.0.0 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.1.0 + uses: docker/setup-buildx-action@v3.2.0 # Workaround to fix error: # failed to push: failed to copy: io: read/write on closed pipe # See https://github.com/docker/build-push-action/issues/761 From 428b7b0c4ed09a5131710af8b04562b791eb7c72 Mon Sep 17 00:00:00 2001 From: thielepaul Date: Thu, 14 Mar 2024 13:28:18 +0100 Subject: [PATCH 3/4] fix(mobile): make elements scrollable to avoid overflow in landscale (#7933) fix(mobile): make elements scrollable to avoid overflow in landscape --- mobile/lib/modules/album/ui/album_viewer_appbar.dart | 4 ++-- mobile/lib/modules/album/views/album_options_part.dart | 5 ++--- .../views/select_additional_user_for_sharing_page.dart | 4 ++-- .../album/views/select_user_for_sharing_page.dart | 4 ++-- .../shared_link/views/shared_link_edit_page.dart | 10 ++++++---- 5 files changed, 14 insertions(+), 13 deletions(-) diff --git a/mobile/lib/modules/album/ui/album_viewer_appbar.dart b/mobile/lib/modules/album/ui/album_viewer_appbar.dart index 996e25d0f..b1a8d4c54 100644 --- a/mobile/lib/modules/album/ui/album_viewer_appbar.dart +++ b/mobile/lib/modules/album/ui/album_viewer_appbar.dart @@ -211,8 +211,8 @@ class AlbumViewerAppbar extends HookConsumerWidget return SafeArea( child: Padding( padding: const EdgeInsets.only(top: 24.0), - child: Column( - mainAxisSize: MainAxisSize.min, + child: ListView( + shrinkWrap: true, children: [ ...buildBottomSheetActions(), if (onAddPhotos != null) ...commonActions, diff --git a/mobile/lib/modules/album/views/album_options_part.dart b/mobile/lib/modules/album/views/album_options_part.dart index 832db7cb8..2f831e430 100644 --- a/mobile/lib/modules/album/views/album_options_part.dart +++ b/mobile/lib/modules/album/views/album_options_part.dart @@ -142,6 +142,7 @@ class AlbumOptionsPage extends HookConsumerWidget { buildSharedUsersList() { return ListView.builder( + primary: false, shrinkWrap: true, itemCount: sharedUsers.value.length, itemBuilder: (context, index) { @@ -188,9 +189,7 @@ class AlbumOptionsPage extends HookConsumerWidget { centerTitle: true, title: Text("translated_text_options".tr()), ), - body: Column( - mainAxisAlignment: MainAxisAlignment.start, - crossAxisAlignment: CrossAxisAlignment.start, + body: ListView( children: [ if (isOwner && album.shared) SwitchListTile.adaptive( diff --git a/mobile/lib/modules/album/views/select_additional_user_for_sharing_page.dart b/mobile/lib/modules/album/views/select_additional_user_for_sharing_page.dart index cb17e6638..0a883791a 100644 --- a/mobile/lib/modules/album/views/select_additional_user_for_sharing_page.dart +++ b/mobile/lib/modules/album/views/select_additional_user_for_sharing_page.dart @@ -63,8 +63,7 @@ class SelectAdditionalUserForSharingPage extends HookConsumerWidget { ), ); } - return Column( - crossAxisAlignment: CrossAxisAlignment.start, + return ListView( children: [ Wrap( children: [...usersChip], @@ -81,6 +80,7 @@ class SelectAdditionalUserForSharingPage extends HookConsumerWidget { ), ), ListView.builder( + primary: false, shrinkWrap: true, itemBuilder: ((context, index) { return ListTile( diff --git a/mobile/lib/modules/album/views/select_user_for_sharing_page.dart b/mobile/lib/modules/album/views/select_user_for_sharing_page.dart index 61550e04f..763f53c65 100644 --- a/mobile/lib/modules/album/views/select_user_for_sharing_page.dart +++ b/mobile/lib/modules/album/views/select_user_for_sharing_page.dart @@ -90,8 +90,7 @@ class SelectUserForSharingPage extends HookConsumerWidget { ), ); } - return Column( - crossAxisAlignment: CrossAxisAlignment.start, + return ListView( children: [ Wrap( children: [...usersChip], @@ -108,6 +107,7 @@ class SelectUserForSharingPage extends HookConsumerWidget { ).tr(), ), ListView.builder( + primary: false, shrinkWrap: true, itemBuilder: ((context, index) { return ListTile( diff --git a/mobile/lib/modules/shared_link/views/shared_link_edit_page.dart b/mobile/lib/modules/shared_link/views/shared_link_edit_page.dart index 288fdae33..b6d892f92 100644 --- a/mobile/lib/modules/shared_link/views/shared_link_edit_page.dart +++ b/mobile/lib/modules/shared_link/views/shared_link_edit_page.dart @@ -428,10 +428,8 @@ class SharedLinkEditPage extends HookConsumerWidget { leading: const CloseButton(), centerTitle: false, ), - resizeToAvoidBottomInset: false, body: SafeArea( - child: Column( - crossAxisAlignment: CrossAxisAlignment.start, + child: ListView( children: [ Padding( padding: const EdgeInsets.all(padding), @@ -487,7 +485,10 @@ class SharedLinkEditPage extends HookConsumerWidget { Align( alignment: Alignment.bottomRight, child: Padding( - padding: const EdgeInsets.only(right: padding + 10), + padding: const EdgeInsets.only( + right: padding + 10, + bottom: padding, + ), child: ElevatedButton( onPressed: existingLink != null ? handleEditLink : handleNewLink, @@ -508,6 +509,7 @@ class SharedLinkEditPage extends HookConsumerWidget { padding: const EdgeInsets.only( left: padding, right: padding, + bottom: padding, ), child: buildNewLinkField(), ), From ba38713fbc40fc714baf9478c3fa8a26f757fecc Mon Sep 17 00:00:00 2001 From: Jonathan Jogenfors Date: Thu, 14 Mar 2024 14:43:05 +0100 Subject: [PATCH 4/4] fix(server): queue library asset refresh in batches (#7914) * add debug logs * scan assets in batches * Cleanup * don't normalize * Removing extra log * remove unneeded code * change log levels --- server/src/domain/library/library.service.ts | 47 +++++++++++--------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/server/src/domain/library/library.service.ts b/server/src/domain/library/library.service.ts index b354c71cd..7615a97dc 100644 --- a/server/src/domain/library/library.service.ts +++ b/server/src/domain/library/library.service.ts @@ -300,17 +300,26 @@ export class LibraryService extends EventEmitter { } private async scanAssets(libraryId: string, assetPaths: string[], ownerId: string, force = false) { - await this.jobRepository.queueAll( - assetPaths.map((assetPath) => ({ - name: JobName.LIBRARY_SCAN_ASSET, - data: { - id: libraryId, - assetPath: path.normalize(assetPath), - ownerId, - force, - }, - })), - ); + this.logger.verbose(`Queuing refresh of ${assetPaths.length} asset(s)`); + + // We perform this in batches to save on memory when performing large refreshes (greater than 1M assets) + const batchSize = 5000; + for (let i = 0; i < assetPaths.length; i += batchSize) { + const batch = assetPaths.slice(i, i + batchSize); + await this.jobRepository.queueAll( + batch.map((assetPath) => ({ + name: JobName.LIBRARY_SCAN_ASSET, + data: { + id: libraryId, + assetPath: assetPath, + ownerId, + force, + }, + })), + ); + } + + this.logger.debug('Asset refresh queue completed'); } private async validateImportPath(importPath: string): Promise { @@ -611,14 +620,6 @@ export class LibraryService extends EventEmitter { return true; } - // Check if a given path is in a user's external path. Both arguments are assumed to be normalized - private isInExternalPath(filePath: string, externalPath: string | null): boolean { - if (externalPath === null) { - return false; - } - return filePath.startsWith(externalPath); - } - async handleQueueAssetRefresh(job: ILibraryRefreshJob): Promise { const library = await this.repository.get(job.id); if (!library || library.type !== LibraryType.EXTERNAL) { @@ -626,7 +627,7 @@ export class LibraryService extends EventEmitter { return false; } - this.logger.verbose(`Refreshing library: ${job.id}`); + this.logger.log(`Refreshing library: ${job.id}`); const crawledAssetPaths = await this.getPathTrie(library); this.logger.debug(`Found ${crawledAssetPaths.size} asset(s) when crawling import paths ${library.importPaths}`); @@ -637,16 +638,20 @@ export class LibraryService extends EventEmitter { this.assetRepository.getLibraryAssetPaths(pagination, library.id), ); + this.logger.verbose(`Crawled asset paths paginated`); + const shouldScanAll = job.refreshAllFiles || job.refreshModifiedFiles; for await (const page of pagination) { for (const asset of page) { const isOffline = !crawledAssetPaths.has(asset.originalPath); if (isOffline && !asset.isOffline) { assetIdsToMarkOffline.push(asset.id); + this.logger.verbose(`Added to mark-offline list: ${asset.originalPath}`); } if (!isOffline && asset.isOffline) { assetIdsToMarkOnline.push(asset.id); + this.logger.verbose(`Added to mark-online list: ${asset.originalPath}`); } if (!shouldScanAll) { @@ -655,6 +660,8 @@ export class LibraryService extends EventEmitter { } } + this.logger.verbose(`Crawled assets have been checked for online/offline status`); + if (assetIdsToMarkOffline.length > 0) { this.logger.debug(`Found ${assetIdsToMarkOffline.length} offline asset(s) previously marked as online`); await this.assetRepository.updateAll(assetIdsToMarkOffline, { isOffline: true });