refactor(web): decouple FaceEditor from DOM element

Change-Id: I71f07ba8d0bc2d829c0b2af4da5ee5bc6a6a6964

web/src/lib/components/AdaptiveImage.svelte

@@ -74,8 +74,6 @@
     onError?: () => void;
     ref?: HTMLDivElement;
     imgRef?: HTMLImageElement;
-    imgNaturalSize?: Size;
-    imgScaledSize?: Size;
     backdrop?: Snippet;
     overlays?: Snippet;
   };
@@ -84,10 +82,6 @@
     ref = $bindable(),
     // eslint-disable-next-line no-useless-assignment
     imgRef = $bindable(),
-    // eslint-disable-next-line no-useless-assignment
-    imgNaturalSize = $bindable(),
-    // eslint-disable-next-line no-useless-assignment
-    imgScaledSize = $bindable(),
     asset,
     sharedLink,
     objectFit = 'contain',
@@ -155,22 +149,10 @@
     return { width: 1, height: 1 };
   });
 
-  $effect(() => {
-    imgNaturalSize = imageDimensions;
-  });
-
-  const scaledDimensions = $derived.by(() => {
-    const scaleFn = objectFit === 'cover' ? scaleToCover : scaleToFit;
-    return scaleFn(imageDimensions, container);
-  });
-
-  $effect(() => {
-    imgScaledSize = scaledDimensions;
-  });
-
   const { insetInlineStart, top, displayWidth, displayHeight, rasterWidth, rasterHeight, rasterScale } = $derived.by(
     () => {
-      const { width, height } = scaledDimensions;
+      const scaleFn = objectFit === 'cover' ? scaleToCover : scaleToFit;
+      const { width, height } = scaleFn(imageDimensions, container);
       if (maxRasterPixels === 0) {
         return {
           insetInlineStart: (container.width - width) / 2 + 'px',

web/src/lib/components/asset-viewer/PhotoViewer.svelte

@@ -13,7 +13,7 @@
   import { SlideshowLook, SlideshowState, slideshowStore } from '$lib/stores/slideshow.store';
   import { handlePromiseError } from '$lib/utils';
   import { canCopyImageToClipboard, copyImageToClipboard } from '$lib/utils/asset-utils';
-  import type { Size } from '$lib/utils/container-utils';
+  import { getNaturalSize, scaleToFit, type Size } from '$lib/utils/container-utils';
   import { handleError } from '$lib/utils/handle-error';
   import { getOcrBoundingBoxes } from '$lib/utils/ocr-utils';
   import { getBoundingBox, type BoundingBox } from '$lib/utils/people-utils';
@@ -67,9 +67,13 @@
     height: containerHeight,
   });
 
-  let scaledDimensions = $state<Size>({ width: 0, height: 0 });
+  const overlaySize = $derived.by((): Size => {
+    if (!assetViewerManager.imgRef || !visibleImageReady) {
+      return { width: 0, height: 0 };
+    }
 
-  const overlaySize = $derived(visibleImageReady ? scaledDimensions : { width: 0, height: 0 });
+    return scaleToFit(getNaturalSize(assetViewerManager.imgRef), { width: containerWidth, height: containerHeight });
+  });
 
   const highlightedBoxes = $derived(getBoundingBox(assetViewerManager.highlightedFaces, overlaySize));
   const isHighlighting = $derived(highlightedBoxes.length > 0);
@@ -231,7 +235,6 @@
       onReady?.();
     }}
     bind:imgRef={assetViewerManager.imgRef}
-    bind:imgScaledSize={scaledDimensions}
     bind:ref={adaptiveImage}
   >
     {#snippet backdrop()}
@@ -283,7 +286,11 @@
     {/snippet}
   </AdaptiveImage>
 
-  {#if assetViewerManager.isFaceEditMode && assetViewerManager.imgRef}
-    <FaceEditor htmlElement={assetViewerManager.imgRef} {containerWidth} {containerHeight} assetId={asset.id} />
+  {#if assetViewerManager.isFaceEditMode && assetViewerManager.imgRef && asset.width && asset.height}
+    <FaceEditor
+      imageSize={{ width: asset.width, height: asset.height }}
+      containerSize={{ width: containerWidth, height: containerHeight }}
+      assetId={asset.id}
+    />
   {/if}
 </div>

web/src/lib/components/asset-viewer/VideoNativeViewer.svelte

@@ -308,9 +308,30 @@
   let containerHeight = $state(0);
 
   $effect(() => {
-    if (assetViewerManager.isFaceEditMode) {
-      videoPlayer?.pause();
+    if (!assetViewerManager.isFaceEditMode || !videoPlayer) {
+      return;
     }
+    videoPlayer.pause();
+
+    const { videoWidth, videoHeight } = videoPlayer;
+    if (videoWidth === 0 || videoHeight === 0) {
+      return;
+    }
+
+    const canvas = document.createElement('canvas');
+    canvas.width = videoWidth;
+    canvas.height = videoHeight;
+    canvas.getContext('2d')?.drawImage(videoPlayer, 0, 0);
+
+    const img = new Image();
+    img.onload = () => (assetViewerManager.imgRef = img);
+    img.src = canvas.toDataURL('image/png');
+
+    return () => {
+      img.onload = null;
+      img.src = '';
+      assetViewerManager.imgRef = undefined;
+    };
   });
 
   // The time is only refreshed on HLS fragment decode by default,
@@ -454,8 +475,12 @@
         </div>
       {/if}
 
-      {#if assetViewerManager.isFaceEditMode && videoPlayer}
-        <FaceEditor htmlElement={videoPlayer} {containerWidth} {containerHeight} {assetId} />
+      {#if assetViewerManager.isFaceEditMode}
+        <FaceEditor
+          imageSize={{ width: asset.width ?? 0, height: asset.height ?? 0 }}
+          containerSize={{ width: containerWidth, height: containerHeight }}
+          {assetId}
+        />
       {/if}
     {/if}
   </div>

web/src/lib/components/asset-viewer/face-editor/FaceEditor.svelte

@@ -4,7 +4,7 @@
   import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';
   import FaceCreateTagModal from '$lib/modals/CreateFaceModal.svelte';
   import { getPeopleThumbnailUrl } from '$lib/utils';
-  import { getNaturalSize, scaleToFit } from '$lib/utils/container-utils';
+  import { computeContentMetrics, mapContentRectToNatural, type Size } from '$lib/utils/container-utils';
   import { handleError } from '$lib/utils/handle-error';
   import { createFace, getAllPeople, type PersonResponseDto } from '@immich/sdk';
   import { Button, Input, modalManager, toastManager } from '@immich/ui';
@@ -14,13 +14,12 @@
   import { t } from 'svelte-i18n';
 
   type Props = {
-    htmlElement: HTMLImageElement | HTMLVideoElement;
-    containerWidth: number;
-    containerHeight: number;
+    imageSize: Size;
+    containerSize: Size;
     assetId: string;
   };
 
-  let { htmlElement, containerWidth, containerHeight, assetId }: Props = $props();
+  let { imageSize, containerSize, assetId }: Props = $props();
 
   let canvasEl: HTMLCanvasElement | undefined = $state();
   let canvas: Canvas | undefined = $state();
@@ -54,7 +53,7 @@
   };
 
   const setupCanvas = () => {
-    if (!canvasEl || !htmlElement) {
+    if (!canvasEl) {
       return;
     }
 
@@ -86,17 +85,7 @@
     searchInputEl?.focus();
   });
 
-  const imageContentMetrics = $derived.by(() => {
-    const natural = getNaturalSize(htmlElement);
-    const container = { width: containerWidth, height: containerHeight };
-    const { width: contentWidth, height: contentHeight } = scaleToFit(natural, container);
-    return {
-      contentWidth,
-      contentHeight,
-      offsetX: (containerWidth - contentWidth) / 2,
-      offsetY: (containerHeight - contentHeight) / 2,
-    };
-  });
+  const imageContentMetrics = $derived(computeContentMetrics(imageSize, containerSize));
 
   const setDefaultFaceRectanglePosition = (faceRect: Rect) => {
     const { offsetX, offsetY } = imageContentMetrics;
@@ -116,8 +105,8 @@
     }
 
     canvas.setDimensions({
-      width: containerWidth,
-      height: containerHeight,
+      width: containerSize.width,
+      height: containerSize.height,
     });
 
     if (!faceRect) {
@@ -175,11 +164,11 @@
     };
     const selectorWidth = faceSelectorEl.offsetWidth;
     const chromeHeight = faceSelectorEl.offsetHeight - scrollableListEl.offsetHeight;
-    const listHeight = Math.min(MAX_LIST_HEIGHT, containerHeight - gap * 2 - chromeHeight);
+    const listHeight = Math.min(MAX_LIST_HEIGHT, containerSize.height - gap * 2 - chromeHeight);
     const selectorHeight = listHeight + chromeHeight;
 
-    const clampTop = (top: number) => clamp(top, gap, containerHeight - selectorHeight - gap);
-    const clampLeft = (left: number) => clamp(left, gap, containerWidth - selectorWidth - gap);
+    const clampTop = (top: number) => clamp(top, gap, containerSize.height - selectorHeight - gap);
+    const clampLeft = (left: number) => clamp(left, gap, containerSize.width - selectorWidth - gap);
 
     const overlapArea = (position: { top: number; left: number }) => {
       const selectorRight = position.left + selectorWidth;
@@ -238,61 +227,42 @@
   });
 
   const getFaceCroppedCoordinates = () => {
-    if (!faceRect || !htmlElement) {
+    if (!faceRect || imageContentMetrics.contentWidth === 0) {
       return;
     }
 
-    const { left, top, width, height } = faceRect.getBoundingRect();
-    const { offsetX, offsetY, contentWidth, contentHeight } = imageContentMetrics;
-    const natural = getNaturalSize(htmlElement);
-
-    const scaleX = natural.width / contentWidth;
-    const scaleY = natural.height / contentHeight;
-    const imageX = (left - offsetX) * scaleX;
-    const imageY = (top - offsetY) * scaleY;
+    const imageRect = mapContentRectToNatural(faceRect.getBoundingRect(), imageContentMetrics, imageSize);
 
     return {
-      imageWidth: natural.width,
-      imageHeight: natural.height,
-      x: Math.floor(imageX),
-      y: Math.floor(imageY),
-      width: Math.floor(width * scaleX),
-      height: Math.floor(height * scaleY),
+      imageWidth: imageSize.width,
+      imageHeight: imageSize.height,
+      x: Math.floor(imageRect.left),
+      y: Math.floor(imageRect.top),
+      width: Math.floor(imageRect.width),
+      height: Math.floor(imageRect.height),
     };
   };
 
   type FaceCoordinates = NonNullable<ReturnType<typeof getFaceCroppedCoordinates>>;
 
   const getFacePreviewUrl = (data: FaceCoordinates) => {
-    if (!htmlElement) {
+    const imgRef = assetViewerManager.imgRef;
+    if (!imgRef || imageContentMetrics.contentWidth === 0) {
       return;
     }
 
-    const natural = getNaturalSize(htmlElement);
-    if (natural.width <= 0 || natural.height <= 0) {
-      return;
-    }
-
-    const x = clamp(data.x, 0, natural.width - 1);
-    const y = clamp(data.y, 0, natural.height - 1);
-    const width = clamp(data.width, 1, natural.width - x);
-    const height = clamp(data.height, 1, natural.height - y);
-
-    if (width <= 0 || height <= 0) {
-      return;
-    }
+    const scaleX = imgRef.naturalWidth / imageSize.width;
+    const scaleY = imgRef.naturalHeight / imageSize.height;
+    const x = clamp(Math.floor(data.x * scaleX), 0, imgRef.naturalWidth - 1);
+    const y = clamp(Math.floor(data.y * scaleY), 0, imgRef.naturalHeight - 1);
+    const width = clamp(Math.floor(data.width * scaleX), 1, imgRef.naturalWidth - x);
+    const height = clamp(Math.floor(data.height * scaleY), 1, imgRef.naturalHeight - y);
 
     const canvas = document.createElement('canvas');
     canvas.width = width;
     canvas.height = height;
-
-    const context = canvas.getContext('2d');
-    if (!context) {
-      return;
-    }
-
     try {
-      context.drawImage(htmlElement, x, y, width, height, 0, 0, width, height);
+      canvas.getContext('2d')?.drawImage(imgRef, x, y, width, height, 0, 0, width, height);
       return canvas.toDataURL('image/png');
     } catch {
       return;

web/src/lib/utils/container-utils.spec.ts

@@ -1,18 +1,15 @@
 import {
-  getContentMetrics,
+  computeContentMetrics,
   getNaturalSize,
+  mapContentRectToNatural,
   mapNormalizedRectToContent,
   mapNormalizedToContent,
   scaleToCover,
   scaleToFit,
 } from '$lib/utils/container-utils';
 
-const mockImage = (props: {
-  naturalWidth: number;
-  naturalHeight: number;
-  width: number;
-  height: number;
-}): HTMLImageElement => props as unknown as HTMLImageElement;
+const mockImage = (props: { naturalWidth: number; naturalHeight: number }): HTMLImageElement =>
+  props as unknown as HTMLImageElement;
 
 const mockVideo = (props: {
   videoWidth: number;
@@ -49,48 +46,85 @@ describe('scaleToFit', () => {
   });
 });
 
-describe('getContentMetrics', () => {
-  it('should compute zero offsets when aspect ratios match', () => {
-    const img = mockImage({ naturalWidth: 1600, naturalHeight: 900, width: 800, height: 450 });
-    expect(getContentMetrics(img)).toEqual({
+describe('computeContentMetrics', () => {
+  it('should return zero metrics for zero-width content', () => {
+    expect(computeContentMetrics({ width: 0, height: 1080 }, { width: 800, height: 600 })).toEqual({
+      contentWidth: 0,
+      contentHeight: 0,
+      offsetX: 0,
+      offsetY: 0,
+    });
+  });
+
+  it('should return zero metrics for zero-height content', () => {
+    expect(computeContentMetrics({ width: 1920, height: 0 }, { width: 800, height: 600 })).toEqual({
+      contentWidth: 0,
+      contentHeight: 0,
+      offsetX: 0,
+      offsetY: 0,
+    });
+  });
+
+  it('should center wide content vertically', () => {
+    expect(computeContentMetrics({ width: 2000, height: 1000 }, { width: 800, height: 600 })).toEqual({
+      contentWidth: 800,
+      contentHeight: 400,
+      offsetX: 0,
+      offsetY: 100,
+    });
+  });
+
+  it('should center tall content horizontally', () => {
+    expect(computeContentMetrics({ width: 1000, height: 2000 }, { width: 800, height: 600 })).toEqual({
+      contentWidth: 300,
+      contentHeight: 600,
+      offsetX: 250,
+      offsetY: 0,
+    });
+  });
+
+  it('should produce zero offsets when aspect ratios match', () => {
+    expect(computeContentMetrics({ width: 1600, height: 900 }, { width: 800, height: 450 })).toEqual({
       contentWidth: 800,
       contentHeight: 450,
       offsetX: 0,
       offsetY: 0,
     });
   });
+});
 
-  it('should compute horizontal letterbox offsets for tall image', () => {
-    const img = mockImage({ naturalWidth: 1000, naturalHeight: 2000, width: 800, height: 600 });
-    const metrics = getContentMetrics(img);
-    expect(metrics.contentWidth).toBe(300);
-    expect(metrics.contentHeight).toBe(600);
-    expect(metrics.offsetX).toBe(250);
-    expect(metrics.offsetY).toBe(0);
+describe('mapContentRectToNatural', () => {
+  it('should map a full-content rect back to natural size', () => {
+    const metrics = { contentWidth: 800, contentHeight: 400, offsetX: 0, offsetY: 100 };
+    const rect = mapContentRectToNatural({ left: 0, top: 100, width: 800, height: 400 }, metrics, {
+      width: 2000,
+      height: 1000,
+    });
+    expect(rect).toEqual({ left: 0, top: 0, width: 2000, height: 1000 });
   });
 
-  it('should compute vertical letterbox offsets for wide image', () => {
-    const img = mockImage({ naturalWidth: 2000, naturalHeight: 1000, width: 800, height: 600 });
-    const metrics = getContentMetrics(img);
-    expect(metrics.contentWidth).toBe(800);
-    expect(metrics.contentHeight).toBe(400);
-    expect(metrics.offsetX).toBe(0);
-    expect(metrics.offsetY).toBe(100);
+  it('should map a centered sub-rect to natural coordinates', () => {
+    const metrics = { contentWidth: 800, contentHeight: 400, offsetX: 0, offsetY: 100 };
+    const rect = mapContentRectToNatural({ left: 200, top: 200, width: 400, height: 200 }, metrics, {
+      width: 2000,
+      height: 1000,
+    });
+    expect(rect).toEqual({ left: 500, top: 250, width: 1000, height: 500 });
   });
 
-  it('should use clientWidth/clientHeight for video elements', () => {
-    const video = mockVideo({ videoWidth: 1920, videoHeight: 1080, clientWidth: 800, clientHeight: 600 });
-    const metrics = getContentMetrics(video);
-    expect(metrics.contentWidth).toBe(800);
-    expect(metrics.contentHeight).toBe(450);
-    expect(metrics.offsetX).toBe(0);
-    expect(metrics.offsetY).toBe(75);
+  it('should handle letterboxed content with horizontal offset', () => {
+    const metrics = { contentWidth: 300, contentHeight: 600, offsetX: 250, offsetY: 0 };
+    const rect = mapContentRectToNatural({ left: 250, top: 0, width: 300, height: 600 }, metrics, {
+      width: 1000,
+      height: 2000,
+    });
+    expect(rect).toEqual({ left: 0, top: 0, width: 1000, height: 2000 });
   });
 });
 
 describe('getNaturalSize', () => {
   it('should return naturalWidth/naturalHeight for images', () => {
-    const img = mockImage({ naturalWidth: 4000, naturalHeight: 3000, width: 800, height: 600 });
+    const img = mockImage({ naturalWidth: 4000, naturalHeight: 3000 });
     expect(getNaturalSize(img)).toEqual({ width: 4000, height: 3000 });
   });
 

web/src/lib/utils/container-utils.ts

@@ -49,13 +49,6 @@ export const scaleToFit = (dimensions: Size, container: Size): Size => {
   };
 };
 
-const getElementSize = (element: HTMLImageElement | HTMLVideoElement): Size => {
-  if (element instanceof HTMLVideoElement) {
-    return { width: element.clientWidth, height: element.clientHeight };
-  }
-  return { width: element.width, height: element.height };
-};
-
 export const getNaturalSize = (element: HTMLImageElement | HTMLVideoElement): Size => {
   if (element instanceof HTMLVideoElement) {
     return { width: element.videoWidth, height: element.videoHeight };
@@ -63,17 +56,18 @@ export const getNaturalSize = (element: HTMLImageElement | HTMLVideoElement): Si
   return { width: element.naturalWidth, height: element.naturalHeight };
 };
 
-export const getContentMetrics = (element: HTMLImageElement | HTMLVideoElement): ContentMetrics => {
-  const natural = getNaturalSize(element);
-  const client = getElementSize(element);
-  const { width: contentWidth, height: contentHeight } = scaleToFit(natural, client);
+export function computeContentMetrics(imageSize: Size, containerSize: Size): ContentMetrics {
+  if (imageSize.width === 0 || imageSize.height === 0) {
+    return { contentWidth: 0, contentHeight: 0, offsetX: 0, offsetY: 0 };
+  }
+  const { width: contentWidth, height: contentHeight } = scaleToFit(imageSize, containerSize);
   return {
     contentWidth,
     contentHeight,
-    offsetX: (client.width - contentWidth) / 2,
-    offsetY: (client.height - contentHeight) / 2,
+    offsetX: (containerSize.width - contentWidth) / 2,
+    offsetY: (containerSize.height - contentHeight) / 2,
   };
-};
+}
 
 export function mapNormalizedToContent(point: Point, sizeOrMetrics: Size | ContentMetrics): Point {
   if ('contentWidth' in sizeOrMetrics) {
@@ -109,3 +103,25 @@ export function mapNormalizedRectToContent(
     height: br.y - tl.y,
   };
 }
+
+export function mapContentToNatural(point: Point, metrics: ContentMetrics, naturalSize: Size): Point {
+  return {
+    x: ((point.x - metrics.offsetX) / metrics.contentWidth) * naturalSize.width,
+    y: ((point.y - metrics.offsetY) / metrics.contentHeight) * naturalSize.height,
+  };
+}
+
+export function mapContentRectToNatural(rect: Rect, metrics: ContentMetrics, naturalSize: Size): Rect {
+  const topLeft = mapContentToNatural({ x: rect.left, y: rect.top }, metrics, naturalSize);
+  const bottomRight = mapContentToNatural(
+    { x: rect.left + rect.width, y: rect.top + rect.height },
+    metrics,
+    naturalSize,
+  );
+  return {
+    top: topLeft.y,
+    left: topLeft.x,
+    width: bottomRight.x - topLeft.x,
+    height: bottomRight.y - topLeft.y,
+  };
+}