feat(web): wasm justified layout (#19150)

* wasm justified layout

* fix tests

* redundant layout generation

* raw position

web/eslint.config.js

@@ -118,6 +118,7 @@ export default typescriptEslint.config(
       'unicorn/filename-case': 'off',
       'unicorn/prefer-top-level-await': 'off',
       'unicorn/import-style': 'off',
+      'unicorn/no-for-loop': 'off',
       'svelte/button-has-type': 'error',
       '@typescript-eslint/await-thenable': 'error',
       '@typescript-eslint/no-floating-promises': 'error',

web/package-lock.json

@@ -10,6 +10,7 @@
       "license": "GNU Affero General Public License version 3",
       "dependencies": {
         "@formatjs/icu-messageformat-parser": "^2.9.8",
+        "@immich/justified-layout-wasm": "^0.3.0",
         "@immich/sdk": "file:../open-api/typescript-sdk",
         "@immich/ui": "^0.22.7",
         "@mapbox/mapbox-gl-rtl-text": "0.2.3",
@@ -1328,6 +1329,12 @@
         "url": "https://opencollective.com/libvips"
       }
     },
+    "node_modules/@immich/justified-layout-wasm": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/@immich/justified-layout-wasm/-/justified-layout-wasm-0.3.0.tgz",
+      "integrity": "sha512-eiKaPHHVsm0YL8SZVUuEs8miTT2uF3b9Tggve7QvIh+KF1Vq41EZFUeDI0RzLvoXAUnoAh37iFvYxaA9iXMHZA==",
+      "license": "AGPL-3"
+    },
     "node_modules/@immich/sdk": {
       "resolved": "../open-api/typescript-sdk",
       "link": true

web/package.json

@@ -27,6 +27,7 @@
   },
   "dependencies": {
     "@formatjs/icu-messageformat-parser": "^2.9.8",
+    "@immich/justified-layout-wasm": "^0.3.0",
     "@immich/sdk": "file:../open-api/typescript-sdk",
     "@immich/ui": "^0.22.7",
     "@mapbox/mapbox-gl-rtl-text": "0.2.3",

web/src/lib/components/shared-components/gallery-viewer/gallery-viewer.svelte

@@ -16,7 +16,7 @@
   import { archiveAssets, cancelMultiselect } from '$lib/utils/asset-utils';
   import { moveFocus } from '$lib/utils/focus-util';
   import { handleError } from '$lib/utils/handle-error';
-  import { getJustifiedLayoutFromAssets, type CommonJustifiedLayout } from '$lib/utils/layout-utils';
+  import { getJustifiedLayoutFromAssets } from '$lib/utils/layout-utils';
   import { navigate } from '$lib/utils/navigation';
   import { isTimelineAsset, toTimelineAsset } from '$lib/utils/timeline-util';
   import { AssetVisibility, type AssetResponseDto } from '@immich/sdk';
@@ -27,7 +27,7 @@
   import Portal from '../portal/portal.svelte';
 
   interface Props {
-    assets: (TimelineAsset | AssetResponseDto)[];
+    assets: TimelineAsset[] | AssetResponseDto[];
     assetInteraction: AssetInteraction;
     disableAssetSelect?: boolean;
     showArchiveIcon?: boolean;
@@ -62,91 +62,39 @@
 
   let { isViewing: isViewerOpen, asset: viewingAsset, setAssetId } = assetViewingStore;
 
-  let geometry: CommonJustifiedLayout | undefined = $state();
-
-  $effect(() => {
-    const _assets = assets;
-    updateSlidingWindow();
-
-    const rowWidth = Math.floor(viewport.width);
-    const rowHeight = rowWidth < 850 ? 100 : 235;
-
-    geometry = getJustifiedLayoutFromAssets(_assets, {
+  const geometry = $derived(
+    getJustifiedLayoutFromAssets(assets, {
       spacing: 2,
-      heightTolerance: 0.15,
-      rowHeight,
-      rowWidth,
-    });
-  });
-
-  let assetLayouts = $derived.by(() => {
-    const assetLayout = [];
-    let containerHeight = 0;
-    let containerWidth = 0;
-    if (geometry) {
-      containerHeight = geometry.containerHeight;
-      containerWidth = geometry.containerWidth;
-      for (const [index, asset] of assets.entries()) {
-        const top = geometry.getTop(index);
-        const left = geometry.getLeft(index);
-        const width = geometry.getWidth(index);
-        const height = geometry.getHeight(index);
-
-        const layoutTopWithOffset = top + pageHeaderOffset;
-        const layoutBottom = layoutTopWithOffset + height;
-
-        const display = layoutTopWithOffset < slidingWindow.bottom && layoutBottom > slidingWindow.top;
-
-        const layout = {
-          asset,
-          top,
-          left,
-          width,
-          height,
-          display,
-        };
-
-        assetLayout.push(layout);
-      }
-    }
-
-    return {
-      assetLayout,
-      containerHeight,
-      containerWidth,
-    };
-  });
+      heightTolerance: 0.3,
+      rowHeight: Math.floor(viewport.width) < 850 ? 100 : 235,
+      rowWidth: Math.floor(viewport.width),
+    }),
+  );
 
   let currentViewAssetIndex = 0;
   let shiftKeyIsDown = $state(false);
   let lastAssetMouseEvent: TimelineAsset | null = $state(null);
-  let slidingWindow = $state({ top: 0, bottom: 0 });
+  let slidingTop = $state(0);
+  let slidingBottom = $state(0);
 
   const updateSlidingWindow = () => {
     const v = $state.snapshot(viewport);
     const top = (document.scrollingElement?.scrollTop || 0) - slidingWindowOffset;
-    const bottom = top + v.height;
-    const w = {
-      top,
-      bottom,
-    };
-    slidingWindow = w;
+    slidingTop = top;
+    slidingBottom = top + v.height;
   };
+  $effect(updateSlidingWindow);
   const debouncedOnIntersected = debounce(() => onIntersected?.(), 750, { maxWait: 100, leading: true });
 
   let lastIntersectedHeight = 0;
   $effect(() => {
     // notify we got to (near) the end of scroll
     const scrollPercentage =
-      ((slidingWindow.bottom - viewport.height) / (viewport.height - (document.scrollingElement?.clientHeight || 0))) *
-      100;
+      (slidingBottom - viewport.height) / (viewport.height - (document.scrollingElement?.clientHeight || 0));
 
-    if (scrollPercentage > 90) {
-      const intersectedHeight = geometry?.containerHeight || 0;
-      if (lastIntersectedHeight !== intersectedHeight) {
-        debouncedOnIntersected();
-        lastIntersectedHeight = intersectedHeight;
-      }
+    if (scrollPercentage > 0.9 && lastIntersectedHeight !== geometry.containerHeight) {
+      debouncedOnIntersected();
+      lastIntersectedHeight = geometry.containerHeight;
     }
   });
   const viewAssetHandler = async (asset: TimelineAsset) => {
@@ -256,7 +204,7 @@
     isShowDeleteConfirmation = false;
     await deleteAssets(
       !(isTrashEnabled && !force),
-      (assetIds) => (assets = assets.filter((asset) => !assetIds.includes(asset.id))),
+      (assetIds) => (assets = assets.filter((asset) => !assetIds.includes(asset.id)) as TimelineAsset[]),
       assetInteraction.selectedAssets,
       onReload,
     );
@@ -269,7 +217,7 @@
       assetInteraction.isAllArchived ? AssetVisibility.Timeline : AssetVisibility.Archive,
     );
     if (ids) {
-      assets = assets.filter((asset) => !ids.includes(asset.id));
+      assets = assets.filter((asset) => !ids.includes(asset.id)) as TimelineAsset[];
       deselectAllAssets();
     }
   };
@@ -454,7 +402,7 @@
   onkeyup={onKeyUp}
   onselectstart={onSelectStart}
   use:shortcuts={shortcutList}
-  onscroll={() => updateSlidingWindow()}
+  onscroll={updateSlidingWindow}
 />
 
 {#if isShowDeleteConfirmation}
@@ -468,13 +416,12 @@
 {#if assets.length > 0}
   <div
     style:position="relative"
-    style:height={assetLayouts.containerHeight + 'px'}
-    style:width={assetLayouts.containerWidth - 1 + 'px'}
+    style:height={geometry.containerHeight + 'px'}
+    style:width={geometry.containerWidth + 'px'}
   >
-    {#each assetLayouts.assetLayout as layout, layoutIndex (layout.asset.id + '-' + layoutIndex)}
-      {@const currentAsset = layout.asset}
-
-      {#if layout.display}
+    {#each assets as asset, i (asset.id + i)}
+      {#if geometry.getTop(i) + pageHeaderOffset < slidingBottom && geometry.getTop(i) + pageHeaderOffset + geometry.getHeight(i) > slidingTop}
+        {@const layout = geometry.getPosition(i)}
         <div
           class="absolute"
           style:overflow="clip"
@@ -484,25 +431,25 @@
             readonly={disableAssetSelect}
             onClick={() => {
               if (assetInteraction.selectionActive) {
-                handleSelectAssets(toTimelineAsset(currentAsset));
+                handleSelectAssets(toTimelineAsset(asset));
                 return;
               }
-              void viewAssetHandler(toTimelineAsset(currentAsset));
+              void viewAssetHandler(toTimelineAsset(asset));
             }}
-            onSelect={() => handleSelectAssets(toTimelineAsset(currentAsset))}
-            onMouseEvent={() => assetMouseEventHandler(toTimelineAsset(currentAsset))}
+            onSelect={() => handleSelectAssets(toTimelineAsset(asset))}
+            onMouseEvent={() => assetMouseEventHandler(toTimelineAsset(asset))}
             {showArchiveIcon}
-            asset={toTimelineAsset(currentAsset)}
-            selected={assetInteraction.hasSelectedAsset(currentAsset.id)}
-            selectionCandidate={assetInteraction.hasSelectionCandidate(currentAsset.id)}
+            asset={toTimelineAsset(asset)}
+            selected={assetInteraction.hasSelectedAsset(asset.id)}
+            selectionCandidate={assetInteraction.hasSelectionCandidate(asset.id)}
             thumbnailWidth={layout.width}
             thumbnailHeight={layout.height}
           />
-          {#if showAssetName && !isTimelineAsset(currentAsset)}
+          {#if showAssetName && !isTimelineAsset(asset)}
             <div
               class="absolute text-center p-1 text-xs font-mono font-semibold w-full bottom-0 bg-linear-to-t bg-slate-50/75 dark:bg-slate-800/75 overflow-clip text-ellipsis whitespace-pre-wrap"
             >
-              {currentAsset.originalFileName}
+              {asset.originalFileName}
             </div>
           {/if}
         </div>

web/src/lib/managers/timeline-manager/day-group.svelte.ts

@@ -1,7 +1,7 @@
 import { AssetOrder } from '@immich/sdk';
 
 import type { CommonLayoutOptions } from '$lib/utils/layout-utils';
-import { getJustifiedLayoutFromAssets, getPosition } from '$lib/utils/layout-utils';
+import { getJustifiedLayoutFromAssets } from '$lib/utils/layout-utils';
 import { plainDateTimeCompare } from '$lib/utils/timeline-util';
 
 import type { MonthGroup } from './month-group.svelte';
@@ -153,8 +153,7 @@ export class DayGroup {
     this.width = geometry.containerWidth;
     this.height = assets.length === 0 ? 0 : geometry.containerHeight;
     for (let i = 0; i < this.viewerAssets.length; i++) {
-      const position = getPosition(geometry, i);
-      this.viewerAssets[i].position = position;
+      this.viewerAssets[i].position = geometry.getPosition(i);
     }
   }
 

web/src/lib/managers/timeline-manager/timeline-manager.svelte.spec.ts

@@ -2,8 +2,10 @@ import { sdkMock } from '$lib/__mocks__/sdk.mock';
 import { getMonthGroupByDate } from '$lib/managers/timeline-manager/internal/search-support.svelte';
 import { AbortError } from '$lib/utils';
 import { fromISODateTimeUTCToObject } from '$lib/utils/timeline-util';
+import { initSync } from '@immich/justified-layout-wasm';
 import { type AssetResponseDto, type TimeBucketAssetResponseDto } from '@immich/sdk';
 import { timelineAssetFactory, toResponseDto } from '@test-data/factories/asset-factory';
+import { readFile } from 'node:fs/promises';
 import { TimelineManager } from './timeline-manager.svelte';
 import type { TimelineAsset } from './types';
 
@@ -23,6 +25,12 @@ function deriveLocalDateTimeFromFileCreatedAt(arg: TimelineAsset): TimelineAsset
 }
 
 describe('TimelineManager', () => {
+  beforeAll(async () => {
+    // needed for Node.js
+    const file = await readFile('node_modules/@immich/justified-layout-wasm/pkg/justified-layout-wasm_bg.wasm');
+    initSync({ module: file });
+  });
+
   beforeEach(() => {
     vi.resetAllMocks();
   });
@@ -80,15 +88,15 @@ describe('TimelineManager', () => {
 
       expect(plainMonths).toEqual(
         expect.arrayContaining([
-          expect.objectContaining({ year: 2024, month: 3, height: 185.5 }),
-          expect.objectContaining({ year: 2024, month: 2, height: 12_016 }),
+          expect.objectContaining({ year: 2024, month: 3, height: 353.5 }),
+          expect.objectContaining({ year: 2024, month: 2, height: 7786.452_636_718_75 }),
           expect.objectContaining({ year: 2024, month: 1, height: 286 }),
         ]),
       );
     });
 
     it('calculates timeline height', () => {
-      expect(timelineManager.timelineHeight).toBe(12_487.5);
+      expect(timelineManager.timelineHeight).toBe(8425.952_636_718_75);
     });
   });
 

web/src/lib/managers/timeline-manager/timeline-manager.svelte.ts

@@ -377,13 +377,11 @@ export class TimelineManager {
   }
 
   createLayoutOptions() {
-    const viewportWidth = this.viewportWidth;
-
     return {
       spacing: 2,
-      heightTolerance: 0.15,
+      heightTolerance: 0.3,
       rowHeight: this.#rowHeight,
-      rowWidth: Math.floor(viewportWidth),
+      rowWidth: Math.floor(this.viewportWidth),
     };
   }
 

web/src/lib/managers/timeline-manager/viewer-asset.svelte.ts

@@ -18,7 +18,7 @@ export class ViewerAsset {
     return calculateViewerAssetIntersecting(store, positionTop, this.position.height);
   });
 
-  position: CommonPosition | undefined = $state();
+  position: CommonPosition | undefined = $state.raw();
   asset: TimelineAsset = <TimelineAsset>$state();
   id: string = $derived(this.asset.id);
 

web/src/lib/utils/layout-utils.ts

@@ -1,16 +1,13 @@
-// import { TUNABLES } from '$lib/utils/tunables';
-// note: it's important that this is not imported in more than one file due to https://github.com/sveltejs/kit/issues/7805
-// import { JustifiedLayout, type LayoutOptions } from '@immich/justified-layout-wasm';
+import { TUNABLES } from '$lib/utils/tunables';
 
 import type { TimelineAsset } from '$lib/managers/timeline-manager/types';
 import { getAssetRatio } from '$lib/utils/asset-utils';
-import { isTimelineAsset } from '$lib/utils/timeline-util';
+import { isTimelineAsset, isTimelineAssets } from '$lib/utils/timeline-util';
+import { JustifiedLayout, type LayoutOptions } from '@immich/justified-layout-wasm';
 import type { AssetResponseDto } from '@immich/sdk';
 import createJustifiedLayout from 'justified-layout';
 
-export type getJustifiedLayoutFromAssetsFunction = typeof getJustifiedLayoutFromAssets;
-
-// let useWasm = TUNABLES.LAYOUT.WASM;
+const useWasm = TUNABLES.LAYOUT.WASM;
 
 export type CommonJustifiedLayout = {
   containerWidth: number;
@@ -19,6 +16,12 @@ export type CommonJustifiedLayout = {
   getLeft(boxIdx: number): number;
   getWidth(boxIdx: number): number;
   getHeight(boxIdx: number): number;
+  getPosition(boxIdx: number): {
+    top: number;
+    left: number;
+    width: number;
+    height: number;
+  };
 };
 
 export type CommonLayoutOptions = {
@@ -29,25 +32,32 @@ export type CommonLayoutOptions = {
 };
 
 export function getJustifiedLayoutFromAssets(
-  assets: (TimelineAsset | AssetResponseDto)[],
+  assets: AssetResponseDto[] | TimelineAsset[],
   options: CommonLayoutOptions,
-): CommonJustifiedLayout {
-  // if (useWasm) {
-  //   return wasmJustifiedLayout(assets, options);
-  // }
+) {
+  if (useWasm) {
+    return isTimelineAssets(assets) ? wasmLayoutFromTimeline(assets, options) : wasmLayoutFromDto(assets, options);
+  }
+
   return justifiedLayout(assets, options);
 }
 
-// commented out until a solution for top level awaits on safari is fixed
-// function wasmJustifiedLayout(assets: AssetResponseDto[], options: LayoutOptions) {
-//   const aspectRatios = new Float32Array(assets.length);
-//   // eslint-disable-next-line unicorn/no-for-loop
-//   for (let i = 0; i < assets.length; i++) {
-//     const { width, height } = getAssetRatio(assets[i]);
-//     aspectRatios[i] = width / height;
-//   }
-//   return new JustifiedLayout(aspectRatios, options);
-// }
+function wasmLayoutFromTimeline(assets: TimelineAsset[], options: LayoutOptions) {
+  const aspectRatios = new Float32Array(assets.length);
+  for (let i = 0; i < assets.length; i++) {
+    aspectRatios[i] = assets[i].ratio;
+  }
+  return new JustifiedLayout(aspectRatios, options);
+}
+
+function wasmLayoutFromDto(assets: AssetResponseDto[], options: LayoutOptions) {
+  const aspectRatios = new Float32Array(assets.length);
+  for (let i = 0; i < assets.length; i++) {
+    const { width, height } = getAssetRatio(assets[i]);
+    aspectRatios[i] = width / height;
+  }
+  return new JustifiedLayout(aspectRatios, options);
+}
 
 type Geometry = ReturnType<typeof createJustifiedLayout>;
 class Adapter {
@@ -88,9 +98,13 @@ class Adapter {
   getHeight(boxIdx: number) {
     return this.result.boxes[boxIdx]?.height;
   }
+
+  getPosition(boxIdx: number): CommonPosition {
+    return this.result.boxes[boxIdx];
+  }
 }
 
-export function justifiedLayout(assets: (TimelineAsset | AssetResponseDto)[], options: CommonLayoutOptions) {
+export function justifiedLayout(assets: TimelineAsset[] | AssetResponseDto[], options: CommonLayoutOptions) {
   const adapter = {
     targetRowHeight: options.rowHeight,
     containerWidth: options.rowWidth,
@@ -105,25 +119,9 @@ export function justifiedLayout(assets: (TimelineAsset | AssetResponseDto)[], op
   return new Adapter(result);
 }
 
-export const emptyGeometry = () =>
-  new Adapter({
-    containerHeight: 0,
-    widowCount: 0,
-    boxes: [],
-  });
-
 export type CommonPosition = {
   top: number;
   left: number;
   width: number;
   height: number;
 };
-
-export function getPosition(geometry: CommonJustifiedLayout, boxIdx: number): CommonPosition {
-  const top = geometry.getTop(boxIdx);
-  const left = geometry.getLeft(boxIdx);
-  const width = geometry.getWidth(boxIdx);
-  const height = geometry.getHeight(boxIdx);
-
-  return { top, left, width, height };
-}

web/src/lib/utils/server.ts

@@ -1,17 +1,17 @@
 import { retrieveServerConfig } from '$lib/stores/server-config.store';
 import { initLanguage } from '$lib/utils';
+import { init as initLayout } from '@immich/justified-layout-wasm';
 import { defaults } from '@immich/sdk';
 import { memoize } from 'lodash-es';
 
 type Fetch = typeof fetch;
 
-async function _init(fetch: Fetch) {
+function _init(fetch: Fetch) {
   // set event.fetch on the fetch-client used by @immich/sdk
   // https://kit.svelte.dev/docs/load#making-fetch-requests
   // https://github.com/oazapfts/oazapfts/blob/main/README.md#fetch-options
   defaults.fetch = fetch;
-  await initLanguage();
-  await retrieveServerConfig();
+  return Promise.all([initLayout(), initLanguage(), retrieveServerConfig()]);
 }
 
 export const init = memoize(_init, () => 'singlevalue');

web/src/lib/utils/timeline-util.ts

@@ -190,8 +190,10 @@ export const toTimelineAsset = (unknownAsset: AssetResponseDto | TimelineAsset):
   };
 };
 
-export const isTimelineAsset = (unknownAsset: AssetResponseDto | TimelineAsset): unknownAsset is TimelineAsset =>
-  (unknownAsset as TimelineAsset).ratio !== undefined;
+export const isTimelineAsset = (asset: AssetResponseDto | TimelineAsset): asset is TimelineAsset => 'ratio' in asset;
+
+export const isTimelineAssets = (assets: AssetResponseDto[] | TimelineAsset[]): assets is TimelineAsset[] =>
+  assets.length === 0 || 'ratio' in assets[0];
 
 export const plainDateTimeCompare = (ascending: boolean, a: TimelinePlainDateTime, b: TimelinePlainDateTime) => {
   const [aDateTime, bDateTime] = ascending ? [a, b] : [b, a];

web/src/lib/utils/tunables.ts

@@ -19,7 +19,7 @@ const storage = browser
     };
 export const TUNABLES = {
   LAYOUT: {
-    WASM: getBoolean(storage.getItem('LAYOUT.WASM'), false),
+    WASM: getBoolean(storage.getItem('LAYOUT.WASM'), true),
   },
   TIMELINE: {
     INTERSECTION_EXPAND_TOP: getNumber(storage.getItem('TIMELINE_INTERSECTION_EXPAND_TOP'), 500),

web/src/test-data/factories/asset-factory.ts

@@ -31,7 +31,7 @@ export const assetFactory = Sync.makeFactory<AssetResponseDto>({
 
 export const timelineAssetFactory = Sync.makeFactory<TimelineAsset>({
   id: Sync.each(() => faker.string.uuid()),
-  ratio: Sync.each(() => faker.number.int()),
+  ratio: Sync.each((i) => 0.2 + ((i * 0.618_034) % 3.8)), // deterministic random float between 0.2 and 4.0
   ownerId: Sync.each(() => faker.string.uuid()),
   thumbhash: Sync.each(() => faker.string.alphanumeric(28)),
   localDateTime: Sync.each(() => fromISODateTimeUTCToObject(faker.date.past().toISOString())),