From bc062da11b1885be94c352b8655fecf5aaf0f91b Mon Sep 17 00:00:00 2001 From: Mert <101130780+mertalev@users.noreply.github.com> Date: Tue, 17 Jun 2025 10:20:14 -0400 Subject: [PATCH] feat(web): wasm justified layout (#19150) * wasm justified layout * fix tests * redundant layout generation * raw position --- web/eslint.config.js | 1 + web/package-lock.json | 7 + web/package.json | 1 + .../gallery-viewer/gallery-viewer.svelte | 123 +++++------------- .../timeline-manager/day-group.svelte.ts | 5 +- .../timeline-manager.svelte.spec.ts | 14 +- .../timeline-manager.svelte.ts | 6 +- .../timeline-manager/viewer-asset.svelte.ts | 2 +- web/src/lib/utils/layout-utils.ts | 76 ++++++----- web/src/lib/utils/server.ts | 6 +- web/src/lib/utils/timeline-util.ts | 6 +- web/src/lib/utils/tunables.ts | 2 +- web/src/test-data/factories/asset-factory.ts | 2 +- 13 files changed, 106 insertions(+), 145 deletions(-) diff --git a/web/eslint.config.js b/web/eslint.config.js index 6b7b343ad1..78b87d24ef 100644 --- a/web/eslint.config.js +++ b/web/eslint.config.js @@ -118,6 +118,7 @@ export default typescriptEslint.config( 'unicorn/filename-case': 'off', 'unicorn/prefer-top-level-await': 'off', 'unicorn/import-style': 'off', + 'unicorn/no-for-loop': 'off', 'svelte/button-has-type': 'error', '@typescript-eslint/await-thenable': 'error', '@typescript-eslint/no-floating-promises': 'error', diff --git a/web/package-lock.json b/web/package-lock.json index 72b9eb6a7c..2888dfb71b 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -10,6 +10,7 @@ "license": "GNU Affero General Public License version 3", "dependencies": { "@formatjs/icu-messageformat-parser": "^2.9.8", + "@immich/justified-layout-wasm": "^0.3.0", "@immich/sdk": "file:../open-api/typescript-sdk", "@immich/ui": "^0.22.7", "@mapbox/mapbox-gl-rtl-text": "0.2.3", @@ -1328,6 +1329,12 @@ "url": "https://opencollective.com/libvips" } }, + "node_modules/@immich/justified-layout-wasm": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@immich/justified-layout-wasm/-/justified-layout-wasm-0.3.0.tgz", + "integrity": "sha512-eiKaPHHVsm0YL8SZVUuEs8miTT2uF3b9Tggve7QvIh+KF1Vq41EZFUeDI0RzLvoXAUnoAh37iFvYxaA9iXMHZA==", + "license": "AGPL-3" + }, "node_modules/@immich/sdk": { "resolved": "../open-api/typescript-sdk", "link": true diff --git a/web/package.json b/web/package.json index a06b12f826..2665904c62 100644 --- a/web/package.json +++ b/web/package.json @@ -27,6 +27,7 @@ }, "dependencies": { "@formatjs/icu-messageformat-parser": "^2.9.8", + "@immich/justified-layout-wasm": "^0.3.0", "@immich/sdk": "file:../open-api/typescript-sdk", "@immich/ui": "^0.22.7", "@mapbox/mapbox-gl-rtl-text": "0.2.3", diff --git a/web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte b/web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte index 09998ed060..eb1869cfbb 100644 --- a/web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte +++ b/web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte @@ -16,7 +16,7 @@ import { archiveAssets, cancelMultiselect } from '$lib/utils/asset-utils'; import { moveFocus } from '$lib/utils/focus-util'; import { handleError } from '$lib/utils/handle-error'; - import { getJustifiedLayoutFromAssets, type CommonJustifiedLayout } from '$lib/utils/layout-utils'; + import { getJustifiedLayoutFromAssets } from '$lib/utils/layout-utils'; import { navigate } from '$lib/utils/navigation'; import { isTimelineAsset, toTimelineAsset } from '$lib/utils/timeline-util'; import { AssetVisibility, type AssetResponseDto } from '@immich/sdk'; @@ -27,7 +27,7 @@ import Portal from '../portal/portal.svelte'; interface Props { - assets: (TimelineAsset | AssetResponseDto)[]; + assets: TimelineAsset[] | AssetResponseDto[]; assetInteraction: AssetInteraction; disableAssetSelect?: boolean; showArchiveIcon?: boolean; @@ -62,91 +62,39 @@ let { isViewing: isViewerOpen, asset: viewingAsset, setAssetId } = assetViewingStore; - let geometry: CommonJustifiedLayout | undefined = $state(); - - $effect(() => { - const _assets = assets; - updateSlidingWindow(); - - const rowWidth = Math.floor(viewport.width); - const rowHeight = rowWidth < 850 ? 100 : 235; - - geometry = getJustifiedLayoutFromAssets(_assets, { + const geometry = $derived( + getJustifiedLayoutFromAssets(assets, { spacing: 2, - heightTolerance: 0.15, - rowHeight, - rowWidth, - }); - }); - - let assetLayouts = $derived.by(() => { - const assetLayout = []; - let containerHeight = 0; - let containerWidth = 0; - if (geometry) { - containerHeight = geometry.containerHeight; - containerWidth = geometry.containerWidth; - for (const [index, asset] of assets.entries()) { - const top = geometry.getTop(index); - const left = geometry.getLeft(index); - const width = geometry.getWidth(index); - const height = geometry.getHeight(index); - - const layoutTopWithOffset = top + pageHeaderOffset; - const layoutBottom = layoutTopWithOffset + height; - - const display = layoutTopWithOffset < slidingWindow.bottom && layoutBottom > slidingWindow.top; - - const layout = { - asset, - top, - left, - width, - height, - display, - }; - - assetLayout.push(layout); - } - } - - return { - assetLayout, - containerHeight, - containerWidth, - }; - }); + heightTolerance: 0.3, + rowHeight: Math.floor(viewport.width) < 850 ? 100 : 235, + rowWidth: Math.floor(viewport.width), + }), + ); let currentViewAssetIndex = 0; let shiftKeyIsDown = $state(false); let lastAssetMouseEvent: TimelineAsset | null = $state(null); - let slidingWindow = $state({ top: 0, bottom: 0 }); + let slidingTop = $state(0); + let slidingBottom = $state(0); const updateSlidingWindow = () => { const v = $state.snapshot(viewport); const top = (document.scrollingElement?.scrollTop || 0) - slidingWindowOffset; - const bottom = top + v.height; - const w = { - top, - bottom, - }; - slidingWindow = w; + slidingTop = top; + slidingBottom = top + v.height; }; + $effect(updateSlidingWindow); const debouncedOnIntersected = debounce(() => onIntersected?.(), 750, { maxWait: 100, leading: true }); let lastIntersectedHeight = 0; $effect(() => { // notify we got to (near) the end of scroll const scrollPercentage = - ((slidingWindow.bottom - viewport.height) / (viewport.height - (document.scrollingElement?.clientHeight || 0))) * - 100; + (slidingBottom - viewport.height) / (viewport.height - (document.scrollingElement?.clientHeight || 0)); - if (scrollPercentage > 90) { - const intersectedHeight = geometry?.containerHeight || 0; - if (lastIntersectedHeight !== intersectedHeight) { - debouncedOnIntersected(); - lastIntersectedHeight = intersectedHeight; - } + if (scrollPercentage > 0.9 && lastIntersectedHeight !== geometry.containerHeight) { + debouncedOnIntersected(); + lastIntersectedHeight = geometry.containerHeight; } }); const viewAssetHandler = async (asset: TimelineAsset) => { @@ -256,7 +204,7 @@ isShowDeleteConfirmation = false; await deleteAssets( !(isTrashEnabled && !force), - (assetIds) => (assets = assets.filter((asset) => !assetIds.includes(asset.id))), + (assetIds) => (assets = assets.filter((asset) => !assetIds.includes(asset.id)) as TimelineAsset[]), assetInteraction.selectedAssets, onReload, ); @@ -269,7 +217,7 @@ assetInteraction.isAllArchived ? AssetVisibility.Timeline : AssetVisibility.Archive, ); if (ids) { - assets = assets.filter((asset) => !ids.includes(asset.id)); + assets = assets.filter((asset) => !ids.includes(asset.id)) as TimelineAsset[]; deselectAllAssets(); } }; @@ -454,7 +402,7 @@ onkeyup={onKeyUp} onselectstart={onSelectStart} use:shortcuts={shortcutList} - onscroll={() => updateSlidingWindow()} + onscroll={updateSlidingWindow} /> {#if isShowDeleteConfirmation} @@ -468,13 +416,12 @@ {#if assets.length > 0}
- {#each assetLayouts.assetLayout as layout, layoutIndex (layout.asset.id + '-' + layoutIndex)} - {@const currentAsset = layout.asset} - - {#if layout.display} + {#each assets as asset, i (asset.id + i)} + {#if geometry.getTop(i) + pageHeaderOffset < slidingBottom && geometry.getTop(i) + pageHeaderOffset + geometry.getHeight(i) > slidingTop} + {@const layout = geometry.getPosition(i)}
{ if (assetInteraction.selectionActive) { - handleSelectAssets(toTimelineAsset(currentAsset)); + handleSelectAssets(toTimelineAsset(asset)); return; } - void viewAssetHandler(toTimelineAsset(currentAsset)); + void viewAssetHandler(toTimelineAsset(asset)); }} - onSelect={() => handleSelectAssets(toTimelineAsset(currentAsset))} - onMouseEvent={() => assetMouseEventHandler(toTimelineAsset(currentAsset))} + onSelect={() => handleSelectAssets(toTimelineAsset(asset))} + onMouseEvent={() => assetMouseEventHandler(toTimelineAsset(asset))} {showArchiveIcon} - asset={toTimelineAsset(currentAsset)} - selected={assetInteraction.hasSelectedAsset(currentAsset.id)} - selectionCandidate={assetInteraction.hasSelectionCandidate(currentAsset.id)} + asset={toTimelineAsset(asset)} + selected={assetInteraction.hasSelectedAsset(asset.id)} + selectionCandidate={assetInteraction.hasSelectionCandidate(asset.id)} thumbnailWidth={layout.width} thumbnailHeight={layout.height} /> - {#if showAssetName && !isTimelineAsset(currentAsset)} + {#if showAssetName && !isTimelineAsset(asset)}
- {currentAsset.originalFileName} + {asset.originalFileName}
{/if}
diff --git a/web/src/lib/managers/timeline-manager/day-group.svelte.ts b/web/src/lib/managers/timeline-manager/day-group.svelte.ts index 2a949499ec..a43a71c511 100644 --- a/web/src/lib/managers/timeline-manager/day-group.svelte.ts +++ b/web/src/lib/managers/timeline-manager/day-group.svelte.ts @@ -1,7 +1,7 @@ import { AssetOrder } from '@immich/sdk'; import type { CommonLayoutOptions } from '$lib/utils/layout-utils'; -import { getJustifiedLayoutFromAssets, getPosition } from '$lib/utils/layout-utils'; +import { getJustifiedLayoutFromAssets } from '$lib/utils/layout-utils'; import { plainDateTimeCompare } from '$lib/utils/timeline-util'; import type { MonthGroup } from './month-group.svelte'; @@ -153,8 +153,7 @@ export class DayGroup { this.width = geometry.containerWidth; this.height = assets.length === 0 ? 0 : geometry.containerHeight; for (let i = 0; i < this.viewerAssets.length; i++) { - const position = getPosition(geometry, i); - this.viewerAssets[i].position = position; + this.viewerAssets[i].position = geometry.getPosition(i); } } diff --git a/web/src/lib/managers/timeline-manager/timeline-manager.svelte.spec.ts b/web/src/lib/managers/timeline-manager/timeline-manager.svelte.spec.ts index 05f8b7c7f7..9aa06d940d 100644 --- a/web/src/lib/managers/timeline-manager/timeline-manager.svelte.spec.ts +++ b/web/src/lib/managers/timeline-manager/timeline-manager.svelte.spec.ts @@ -2,8 +2,10 @@ import { sdkMock } from '$lib/__mocks__/sdk.mock'; import { getMonthGroupByDate } from '$lib/managers/timeline-manager/internal/search-support.svelte'; import { AbortError } from '$lib/utils'; import { fromISODateTimeUTCToObject } from '$lib/utils/timeline-util'; +import { initSync } from '@immich/justified-layout-wasm'; import { type AssetResponseDto, type TimeBucketAssetResponseDto } from '@immich/sdk'; import { timelineAssetFactory, toResponseDto } from '@test-data/factories/asset-factory'; +import { readFile } from 'node:fs/promises'; import { TimelineManager } from './timeline-manager.svelte'; import type { TimelineAsset } from './types'; @@ -23,6 +25,12 @@ function deriveLocalDateTimeFromFileCreatedAt(arg: TimelineAsset): TimelineAsset } describe('TimelineManager', () => { + beforeAll(async () => { + // needed for Node.js + const file = await readFile('node_modules/@immich/justified-layout-wasm/pkg/justified-layout-wasm_bg.wasm'); + initSync({ module: file }); + }); + beforeEach(() => { vi.resetAllMocks(); }); @@ -80,15 +88,15 @@ describe('TimelineManager', () => { expect(plainMonths).toEqual( expect.arrayContaining([ - expect.objectContaining({ year: 2024, month: 3, height: 185.5 }), - expect.objectContaining({ year: 2024, month: 2, height: 12_016 }), + expect.objectContaining({ year: 2024, month: 3, height: 353.5 }), + expect.objectContaining({ year: 2024, month: 2, height: 7786.452_636_718_75 }), expect.objectContaining({ year: 2024, month: 1, height: 286 }), ]), ); }); it('calculates timeline height', () => { - expect(timelineManager.timelineHeight).toBe(12_487.5); + expect(timelineManager.timelineHeight).toBe(8425.952_636_718_75); }); }); diff --git a/web/src/lib/managers/timeline-manager/timeline-manager.svelte.ts b/web/src/lib/managers/timeline-manager/timeline-manager.svelte.ts index 8aacd0a90a..e17082c816 100644 --- a/web/src/lib/managers/timeline-manager/timeline-manager.svelte.ts +++ b/web/src/lib/managers/timeline-manager/timeline-manager.svelte.ts @@ -377,13 +377,11 @@ export class TimelineManager { } createLayoutOptions() { - const viewportWidth = this.viewportWidth; - return { spacing: 2, - heightTolerance: 0.15, + heightTolerance: 0.3, rowHeight: this.#rowHeight, - rowWidth: Math.floor(viewportWidth), + rowWidth: Math.floor(this.viewportWidth), }; } diff --git a/web/src/lib/managers/timeline-manager/viewer-asset.svelte.ts b/web/src/lib/managers/timeline-manager/viewer-asset.svelte.ts index b6e28df576..161cc049f1 100644 --- a/web/src/lib/managers/timeline-manager/viewer-asset.svelte.ts +++ b/web/src/lib/managers/timeline-manager/viewer-asset.svelte.ts @@ -18,7 +18,7 @@ export class ViewerAsset { return calculateViewerAssetIntersecting(store, positionTop, this.position.height); }); - position: CommonPosition | undefined = $state(); + position: CommonPosition | undefined = $state.raw(); asset: TimelineAsset = $state(); id: string = $derived(this.asset.id); diff --git a/web/src/lib/utils/layout-utils.ts b/web/src/lib/utils/layout-utils.ts index e60fa3b9e1..090a7168d5 100644 --- a/web/src/lib/utils/layout-utils.ts +++ b/web/src/lib/utils/layout-utils.ts @@ -1,16 +1,13 @@ -// import { TUNABLES } from '$lib/utils/tunables'; -// note: it's important that this is not imported in more than one file due to https://github.com/sveltejs/kit/issues/7805 -// import { JustifiedLayout, type LayoutOptions } from '@immich/justified-layout-wasm'; +import { TUNABLES } from '$lib/utils/tunables'; import type { TimelineAsset } from '$lib/managers/timeline-manager/types'; import { getAssetRatio } from '$lib/utils/asset-utils'; -import { isTimelineAsset } from '$lib/utils/timeline-util'; +import { isTimelineAsset, isTimelineAssets } from '$lib/utils/timeline-util'; +import { JustifiedLayout, type LayoutOptions } from '@immich/justified-layout-wasm'; import type { AssetResponseDto } from '@immich/sdk'; import createJustifiedLayout from 'justified-layout'; -export type getJustifiedLayoutFromAssetsFunction = typeof getJustifiedLayoutFromAssets; - -// let useWasm = TUNABLES.LAYOUT.WASM; +const useWasm = TUNABLES.LAYOUT.WASM; export type CommonJustifiedLayout = { containerWidth: number; @@ -19,6 +16,12 @@ export type CommonJustifiedLayout = { getLeft(boxIdx: number): number; getWidth(boxIdx: number): number; getHeight(boxIdx: number): number; + getPosition(boxIdx: number): { + top: number; + left: number; + width: number; + height: number; + }; }; export type CommonLayoutOptions = { @@ -29,25 +32,32 @@ export type CommonLayoutOptions = { }; export function getJustifiedLayoutFromAssets( - assets: (TimelineAsset | AssetResponseDto)[], + assets: AssetResponseDto[] | TimelineAsset[], options: CommonLayoutOptions, -): CommonJustifiedLayout { - // if (useWasm) { - // return wasmJustifiedLayout(assets, options); - // } +) { + if (useWasm) { + return isTimelineAssets(assets) ? wasmLayoutFromTimeline(assets, options) : wasmLayoutFromDto(assets, options); + } + return justifiedLayout(assets, options); } -// commented out until a solution for top level awaits on safari is fixed -// function wasmJustifiedLayout(assets: AssetResponseDto[], options: LayoutOptions) { -// const aspectRatios = new Float32Array(assets.length); -// // eslint-disable-next-line unicorn/no-for-loop -// for (let i = 0; i < assets.length; i++) { -// const { width, height } = getAssetRatio(assets[i]); -// aspectRatios[i] = width / height; -// } -// return new JustifiedLayout(aspectRatios, options); -// } +function wasmLayoutFromTimeline(assets: TimelineAsset[], options: LayoutOptions) { + const aspectRatios = new Float32Array(assets.length); + for (let i = 0; i < assets.length; i++) { + aspectRatios[i] = assets[i].ratio; + } + return new JustifiedLayout(aspectRatios, options); +} + +function wasmLayoutFromDto(assets: AssetResponseDto[], options: LayoutOptions) { + const aspectRatios = new Float32Array(assets.length); + for (let i = 0; i < assets.length; i++) { + const { width, height } = getAssetRatio(assets[i]); + aspectRatios[i] = width / height; + } + return new JustifiedLayout(aspectRatios, options); +} type Geometry = ReturnType; class Adapter { @@ -88,9 +98,13 @@ class Adapter { getHeight(boxIdx: number) { return this.result.boxes[boxIdx]?.height; } + + getPosition(boxIdx: number): CommonPosition { + return this.result.boxes[boxIdx]; + } } -export function justifiedLayout(assets: (TimelineAsset | AssetResponseDto)[], options: CommonLayoutOptions) { +export function justifiedLayout(assets: TimelineAsset[] | AssetResponseDto[], options: CommonLayoutOptions) { const adapter = { targetRowHeight: options.rowHeight, containerWidth: options.rowWidth, @@ -105,25 +119,9 @@ export function justifiedLayout(assets: (TimelineAsset | AssetResponseDto)[], op return new Adapter(result); } -export const emptyGeometry = () => - new Adapter({ - containerHeight: 0, - widowCount: 0, - boxes: [], - }); - export type CommonPosition = { top: number; left: number; width: number; height: number; }; - -export function getPosition(geometry: CommonJustifiedLayout, boxIdx: number): CommonPosition { - const top = geometry.getTop(boxIdx); - const left = geometry.getLeft(boxIdx); - const width = geometry.getWidth(boxIdx); - const height = geometry.getHeight(boxIdx); - - return { top, left, width, height }; -} diff --git a/web/src/lib/utils/server.ts b/web/src/lib/utils/server.ts index 1c52274d23..b99ab80ec5 100644 --- a/web/src/lib/utils/server.ts +++ b/web/src/lib/utils/server.ts @@ -1,17 +1,17 @@ import { retrieveServerConfig } from '$lib/stores/server-config.store'; import { initLanguage } from '$lib/utils'; +import { init as initLayout } from '@immich/justified-layout-wasm'; import { defaults } from '@immich/sdk'; import { memoize } from 'lodash-es'; type Fetch = typeof fetch; -async function _init(fetch: Fetch) { +function _init(fetch: Fetch) { // set event.fetch on the fetch-client used by @immich/sdk // https://kit.svelte.dev/docs/load#making-fetch-requests // https://github.com/oazapfts/oazapfts/blob/main/README.md#fetch-options defaults.fetch = fetch; - await initLanguage(); - await retrieveServerConfig(); + return Promise.all([initLayout(), initLanguage(), retrieveServerConfig()]); } export const init = memoize(_init, () => 'singlevalue'); diff --git a/web/src/lib/utils/timeline-util.ts b/web/src/lib/utils/timeline-util.ts index c3e41c01be..ca9dece6b2 100644 --- a/web/src/lib/utils/timeline-util.ts +++ b/web/src/lib/utils/timeline-util.ts @@ -190,8 +190,10 @@ export const toTimelineAsset = (unknownAsset: AssetResponseDto | TimelineAsset): }; }; -export const isTimelineAsset = (unknownAsset: AssetResponseDto | TimelineAsset): unknownAsset is TimelineAsset => - (unknownAsset as TimelineAsset).ratio !== undefined; +export const isTimelineAsset = (asset: AssetResponseDto | TimelineAsset): asset is TimelineAsset => 'ratio' in asset; + +export const isTimelineAssets = (assets: AssetResponseDto[] | TimelineAsset[]): assets is TimelineAsset[] => + assets.length === 0 || 'ratio' in assets[0]; export const plainDateTimeCompare = (ascending: boolean, a: TimelinePlainDateTime, b: TimelinePlainDateTime) => { const [aDateTime, bDateTime] = ascending ? [a, b] : [b, a]; diff --git a/web/src/lib/utils/tunables.ts b/web/src/lib/utils/tunables.ts index 6ce64ed041..c586e11957 100644 --- a/web/src/lib/utils/tunables.ts +++ b/web/src/lib/utils/tunables.ts @@ -19,7 +19,7 @@ const storage = browser }; export const TUNABLES = { LAYOUT: { - WASM: getBoolean(storage.getItem('LAYOUT.WASM'), false), + WASM: getBoolean(storage.getItem('LAYOUT.WASM'), true), }, TIMELINE: { INTERSECTION_EXPAND_TOP: getNumber(storage.getItem('TIMELINE_INTERSECTION_EXPAND_TOP'), 500), diff --git a/web/src/test-data/factories/asset-factory.ts b/web/src/test-data/factories/asset-factory.ts index c2f03f9c6a..273bb6f97b 100644 --- a/web/src/test-data/factories/asset-factory.ts +++ b/web/src/test-data/factories/asset-factory.ts @@ -31,7 +31,7 @@ export const assetFactory = Sync.makeFactory({ export const timelineAssetFactory = Sync.makeFactory({ id: Sync.each(() => faker.string.uuid()), - ratio: Sync.each(() => faker.number.int()), + ratio: Sync.each((i) => 0.2 + ((i * 0.618_034) % 3.8)), // deterministic random float between 0.2 and 4.0 ownerId: Sync.each(() => faker.string.uuid()), thumbhash: Sync.each(() => faker.string.alphanumeric(28)), localDateTime: Sync.each(() => fromISODateTimeUTCToObject(faker.date.past().toISOString())),