From d952b620531ad9c874509f3365cee2c11aadbb7e Mon Sep 17 00:00:00 2001
From: Mees Frensel <33722705+meesfrensel@users.noreply.github.com>
Date: Fri, 21 Nov 2025 16:11:47 +0100
Subject: [PATCH] feat(web): show detected faces in spherical photos (#23974)

---
 pnpm-lock.yaml                                | 22 ++++--
 web/package.json                              | 11 +--
 .../photo-sphere-viewer-adapter.svelte        | 71 ++++++++++++++++++-
 3 files changed, 93 insertions(+), 11 deletions(-)

diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 4a00b6755a..1228fcd55f 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -726,19 +726,22 @@ importers:
         specifier: ^7.4.47
         version: 7.4.47
       '@photo-sphere-viewer/core':
-        specifier: ^5.11.5
+        specifier: ^5.14.0
         version: 5.14.0
       '@photo-sphere-viewer/equirectangular-video-adapter':
-        specifier: ^5.11.5
+        specifier: ^5.14.0
         version: 5.14.0(@photo-sphere-viewer/core@5.14.0)(@photo-sphere-viewer/video-plugin@5.14.0(@photo-sphere-viewer/core@5.14.0))
+      '@photo-sphere-viewer/markers-plugin':
+        specifier: ^5.14.0
+        version: 5.14.0(@photo-sphere-viewer/core@5.14.0)
       '@photo-sphere-viewer/resolution-plugin':
-        specifier: ^5.11.5
+        specifier: ^5.14.0
         version: 5.14.0(@photo-sphere-viewer/core@5.14.0)(@photo-sphere-viewer/settings-plugin@5.14.0(@photo-sphere-viewer/core@5.14.0))
       '@photo-sphere-viewer/settings-plugin':
-        specifier: ^5.11.5
+        specifier: ^5.14.0
         version: 5.14.0(@photo-sphere-viewer/core@5.14.0)
       '@photo-sphere-viewer/video-plugin':
-        specifier: ^5.11.5
+        specifier: ^5.14.0
         version: 5.14.0(@photo-sphere-viewer/core@5.14.0)
       '@types/geojson':
         specifier: ^7946.0.16
@@ -3573,6 +3576,11 @@ packages:
       '@photo-sphere-viewer/core': 5.14.0
       '@photo-sphere-viewer/video-plugin': 5.14.0
 
+  '@photo-sphere-viewer/markers-plugin@5.14.0':
+    resolution: {integrity: sha512-w7txVHtLxXMS61m0EbNjgvdNXQYRh6Aa0oatft5oruKgoXLg/UlCu1mG6Btg+zrNsG05W2zl4gRM3fcWoVdneA==}
+    peerDependencies:
+      '@photo-sphere-viewer/core': 5.14.0
+
   '@photo-sphere-viewer/resolution-plugin@5.14.0':
     resolution: {integrity: sha512-PvDMX1h+8FzWdySxiorQ2bSmyBGTPsZjNNFRBqIfmb5C+01aWCIE7kuXodXGHwpXQNcOojsVX9IiX0Vz4CiW4A==}
     peerDependencies:
@@ -15374,6 +15382,10 @@ snapshots:
       '@photo-sphere-viewer/video-plugin': 5.14.0(@photo-sphere-viewer/core@5.14.0)
       three: 0.180.0
 
+  '@photo-sphere-viewer/markers-plugin@5.14.0(@photo-sphere-viewer/core@5.14.0)':
+    dependencies:
+      '@photo-sphere-viewer/core': 5.14.0
+
   '@photo-sphere-viewer/resolution-plugin@5.14.0(@photo-sphere-viewer/core@5.14.0)(@photo-sphere-viewer/settings-plugin@5.14.0(@photo-sphere-viewer/core@5.14.0))':
     dependencies:
       '@photo-sphere-viewer/core': 5.14.0
diff --git a/web/package.json b/web/package.json
index 27454f7e88..1afe94ac07 100644
--- a/web/package.json
+++ b/web/package.json
@@ -31,11 +31,12 @@
     "@immich/ui": "^0.43.0",
     "@mapbox/mapbox-gl-rtl-text": "0.2.3",
     "@mdi/js": "^7.4.47",
-    "@photo-sphere-viewer/core": "^5.11.5",
-    "@photo-sphere-viewer/equirectangular-video-adapter": "^5.11.5",
-    "@photo-sphere-viewer/resolution-plugin": "^5.11.5",
-    "@photo-sphere-viewer/settings-plugin": "^5.11.5",
-    "@photo-sphere-viewer/video-plugin": "^5.11.5",
+    "@photo-sphere-viewer/core": "^5.14.0",
+    "@photo-sphere-viewer/equirectangular-video-adapter": "^5.14.0",
+    "@photo-sphere-viewer/markers-plugin": "^5.14.0",
+    "@photo-sphere-viewer/resolution-plugin": "^5.14.0",
+    "@photo-sphere-viewer/settings-plugin": "^5.14.0",
+    "@photo-sphere-viewer/video-plugin": "^5.14.0",
     "@types/geojson": "^7946.0.16",
     "@zoom-image/core": "^0.41.0",
     "@zoom-image/svelte": "^0.3.0",
diff --git a/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte b/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte
index cd9a010f78..00f3a6795f 100644
--- a/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte
+++ b/web/src/lib/components/asset-viewer/photo-sphere-viewer-adapter.svelte
@@ -1,4 +1,5 @@
 <script lang="ts">
+  import { boundingBoxesArray, type Faces } from '$lib/stores/people.store';
   import { alwaysLoadOriginalFile } from '$lib/stores/preferences.store';
   import {
     EquirectangularAdapter,
@@ -8,11 +9,21 @@
     type PluginConstructor,
   } from '@photo-sphere-viewer/core';
   import '@photo-sphere-viewer/core/index.css';
+  import { MarkersPlugin } from '@photo-sphere-viewer/markers-plugin';
+  import '@photo-sphere-viewer/markers-plugin/index.css';
   import { ResolutionPlugin } from '@photo-sphere-viewer/resolution-plugin';
   import { SettingsPlugin } from '@photo-sphere-viewer/settings-plugin';
   import '@photo-sphere-viewer/settings-plugin/index.css';
   import { onDestroy, onMount } from 'svelte';
 
+  // Adapted as well as possible from classlist 'border-solid border-white border-3 rounded-lg'
+  const FACE_BOX_SVG_STYLE = {
+    fill: 'rgba(0, 0, 0, 0)',
+    stroke: '#ffffff',
+    strokeWidth: '3px',
+    strokeLinejoin: 'round',
+  };
+
   interface Props {
     panorama: string | { source: string };
     originalPanorama?: string | { source: string };
@@ -26,6 +37,62 @@
   let container: HTMLDivElement | undefined = $state();
   let viewer: Viewer;
 
+  let animationInProgress: { cancel: () => void } | undefined;
+  let previousFaces: Faces[] = [];
+
+  const boundingBoxesUnsubscribe = boundingBoxesArray.subscribe((faces: Faces[]) => {
+    // Debounce; don't do anything when the data didn't actually change.
+    if (faces === previousFaces) {
+      return;
+    }
+    previousFaces = faces;
+
+    if (animationInProgress) {
+      animationInProgress.cancel();
+      animationInProgress = undefined;
+    }
+    if (!viewer || !viewer.state.textureData || !viewer.getPlugin(MarkersPlugin)) {
+      return;
+    }
+    const markersPlugin = viewer.getPlugin<MarkersPlugin>(MarkersPlugin);
+
+    // croppedWidth is the size of the texture, which might be cropped to be less than 360/180 degrees.
+    // This is what we want because the facial recognition is done on the image, not the sphere.
+    const currentTextureWidth = viewer.state.textureData.panoData.croppedWidth;
+
+    markersPlugin.clearMarkers();
+    for (const [index, face] of faces.entries()) {
+      const { boundingBoxX1: x1, boundingBoxY1: y1, boundingBoxX2: x2, boundingBoxY2: y2 } = face;
+      const ratio = currentTextureWidth / face.imageWidth;
+      // Pixel values are translated to spherical coordinates and only then added to the panorama;
+      // no need to recalculate when the texture image changes to the original size.
+      markersPlugin.addMarker({
+        id: `face_${index}`,
+        polygonPixels: [
+          [x1 * ratio, y1 * ratio],
+          [x2 * ratio, y1 * ratio],
+          [x2 * ratio, y2 * ratio],
+          [x1 * ratio, y2 * ratio],
+        ],
+        svgStyle: FACE_BOX_SVG_STYLE,
+      });
+    }
+
+    // Smoothly pan to the highlighted (hovered-over) face.
+    if (faces.length === 1) {
+      const { boundingBoxX1: x1, boundingBoxY1: y1, boundingBoxX2: x2, boundingBoxY2: y2, imageWidth: w } = faces[0];
+      const ratio = currentTextureWidth / w;
+      const x = ((x1 + x2) * ratio) / 2;
+      const y = ((y1 + y2) * ratio) / 2;
+      animationInProgress = viewer.animate({
+        textureX: x,
+        textureY: y,
+        zoom: Math.min(viewer.getZoomLevel(), 75),
+        speed: 500, // duration in ms
+      });
+    }
+  });
+
   onMount(() => {
     if (!container) {
       return;
@@ -34,6 +101,7 @@
     viewer = new Viewer({
       adapter,
       plugins: [
+        MarkersPlugin,
         SettingsPlugin,
         [
           ResolutionPlugin,
@@ -68,7 +136,7 @@
       zoomSpeed: 0.5,
       fisheye: false,
     });
-    const resolutionPlugin = viewer.getPlugin(ResolutionPlugin) as ResolutionPlugin;
+    const resolutionPlugin = viewer.getPlugin<ResolutionPlugin>(ResolutionPlugin);
     const zoomHandler = ({ zoomLevel }: events.ZoomUpdatedEvent) => {
       // zoomLevel range: [0, 100]
       if (Math.round(zoomLevel) >= 75) {
@@ -89,6 +157,7 @@
     if (viewer) {
       viewer.destroy();
     }
+    boundingBoxesUnsubscribe();
   });
 </script>