feat: create new person in face editor (#27364)

* feat: create new person in face editor

* add delay

* fix: test

* i18n

* fix: unit test

* pr feedback

i18n/en.json

@@ -849,9 +849,12 @@
   "create_link_to_share": "Create link to share",
   "create_link_to_share_description": "Let anyone with the link see the selected photo(s)",
   "create_new": "CREATE NEW",
+  "create_new_face": "Create new face",
   "create_new_person": "Create new person",
   "create_new_person_hint": "Assign selected assets to a new person",
   "create_new_user": "Create new user",
+  "create_person": "Create person",
+  "create_person_subtitle": "Add a name to the selected face to create and tag the new person",
   "create_shared_album_page_share_add_assets": "ADD ASSETS",
   "create_shared_album_page_share_select_photos": "Select Photos",
   "create_shared_link": "Create shared link",
@@ -2214,6 +2217,7 @@
   "tag": "Tag",
   "tag_assets": "Tag assets",
   "tag_created": "Created tag: {tag}",
+  "tag_face": "Tag face",
   "tag_feature_description": "Browsing photos and videos grouped by logical tag topics",
   "tag_not_found_question": "Cannot find a tag? <link>Create a new tag.</link>",
   "tag_people": "Tag People",

server/src/services/person.service.spec.ts

@@ -12,7 +12,13 @@ import { PersonFactory } from 'test/factories/person.factory';
 import { UserFactory } from 'test/factories/user.factory';
 import { authStub } from 'test/fixtures/auth.stub';
 import { systemConfigStub } from 'test/fixtures/system-config.stub';
-import { getAsDetectedFace, getForAssetFace, getForDetectedFaces, getForFacialRecognitionJob } from 'test/mappers';
+import {
+  getAsDetectedFace,
+  getForAsset,
+  getForAssetFace,
+  getForDetectedFaces,
+  getForFacialRecognitionJob,
+} from 'test/mappers';
 import { newDate, newUuid } from 'test/small.factory';
 import { makeStream, newTestService, ServiceMocks } from 'test/utils';
 
@@ -370,6 +376,86 @@ describe(PersonService.name, () => {
     });
   });
 
+  describe('createFace', () => {
+    it('should create a manual face and initialize the person feature photo creation', async () => {
+      const auth = AuthFactory.create();
+      const asset = AssetFactory.create();
+      const person = PersonFactory.create({ faceAssetId: null });
+      const featureFace = AssetFaceFactory.create({
+        assetId: asset.id,
+        personId: person.id,
+        sourceType: SourceType.Manual,
+      });
+
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([asset.id]));
+      mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id]));
+      mocks.asset.getById.mockResolvedValue(getForAsset(asset));
+      mocks.person.getById.mockResolvedValue(person);
+      mocks.person.getRandomFace.mockResolvedValue(featureFace);
+      mocks.person.update.mockResolvedValue({ ...person, faceAssetId: featureFace.id });
+
+      await expect(
+        sut.createFace(auth, {
+          assetId: asset.id,
+          personId: person.id,
+          imageHeight: 500,
+          imageWidth: 400,
+          x: 10,
+          y: 20,
+          width: 100,
+          height: 110,
+        }),
+      ).resolves.toBeUndefined();
+
+      expect(mocks.asset.getById).toHaveBeenCalledWith(asset.id, { edits: true, exifInfo: true });
+      expect(mocks.person.createAssetFace).toHaveBeenCalledWith({
+        assetId: asset.id,
+        personId: person.id,
+        imageHeight: 500,
+        imageWidth: 400,
+        boundingBoxX1: 10,
+        boundingBoxX2: 110,
+        boundingBoxY1: 20,
+        boundingBoxY2: 130,
+        sourceType: SourceType.Manual,
+      });
+      expect(mocks.person.getRandomFace).toHaveBeenCalledWith(person.id);
+      expect(mocks.person.update).toHaveBeenCalledWith({ id: person.id, faceAssetId: featureFace.id });
+      expect(mocks.job.queueAll).toHaveBeenCalledWith([
+        { name: JobName.PersonGenerateThumbnail, data: { id: person.id } },
+      ]);
+    });
+
+    it('should not update the person feature photo if one already exists', async () => {
+      const auth = AuthFactory.create();
+      const asset = AssetFactory.create();
+      const person = PersonFactory.create({ faceAssetId: newUuid() });
+
+      mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([asset.id]));
+      mocks.access.person.checkOwnerAccess.mockResolvedValue(new Set([person.id]));
+      mocks.asset.getById.mockResolvedValue(getForAsset(asset));
+      mocks.person.getById.mockResolvedValue(person);
+
+      await expect(
+        sut.createFace(auth, {
+          assetId: asset.id,
+          personId: person.id,
+          imageHeight: 500,
+          imageWidth: 400,
+          x: 10,
+          y: 20,
+          width: 100,
+          height: 110,
+        }),
+      ).resolves.toBeUndefined();
+
+      expect(mocks.person.createAssetFace).toHaveBeenCalledOnce();
+      expect(mocks.person.getRandomFace).not.toHaveBeenCalled();
+      expect(mocks.person.update).not.toHaveBeenCalled();
+      expect(mocks.job.queueAll).not.toHaveBeenCalled();
+    });
+  });
+
   describe('createNewFeaturePhoto', () => {
     it('should change person feature photo', async () => {
       const person = PersonFactory.create();

server/src/services/person.service.ts

@@ -631,7 +631,11 @@ export class PersonService extends BaseService {
       this.requireAccess({ auth, permission: Permission.PersonRead, ids: [dto.personId] }),
     ]);
 
-    const asset = await this.assetRepository.getById(dto.assetId, { edits: true, exifInfo: true });
+    const [asset, person] = await Promise.all([
+      this.assetRepository.getById(dto.assetId, { edits: true, exifInfo: true }),
+      this.findOrFail(dto.personId),
+    ]);
+
     if (!asset) {
       throw new NotFoundException('Asset not found');
     }
@@ -689,6 +693,10 @@ export class PersonService extends BaseService {
       boundingBoxY2: Math.round(bottomRight.y),
       sourceType: SourceType.Manual,
     });
+
+    if (!person.faceAssetId) {
+      await this.createNewFeaturePhoto([person.id]);
+    }
   }
 
   async deleteFace(auth: AuthDto, id: string, dto: AssetFaceDeleteDto): Promise<void> {

server/test/medium/specs/services/person.service.spec.ts

@@ -5,6 +5,7 @@ import { AccessRepository } from 'src/repositories/access.repository';
 import { AssetEditRepository } from 'src/repositories/asset-edit.repository';
 import { AssetRepository } from 'src/repositories/asset.repository';
 import { DatabaseRepository } from 'src/repositories/database.repository';
+import { JobRepository } from 'src/repositories/job.repository';
 import { LoggingRepository } from 'src/repositories/logging.repository';
 import { PersonRepository } from 'src/repositories/person.repository';
 import { StorageRepository } from 'src/repositories/storage.repository';
@@ -20,7 +21,7 @@ const setup = (db?: Kysely<DB>) => {
   return newMediumService(PersonService, {
     database: db || defaultDatabase,
     real: [AccessRepository, DatabaseRepository, PersonRepository, AssetRepository, AssetEditRepository],
-    mock: [LoggingRepository, StorageRepository],
+    mock: [JobRepository, LoggingRepository, StorageRepository],
   });
 };
 
@@ -89,6 +90,7 @@ describe(PersonService.name, () => {
       const { person } = await ctx.newPerson({ ownerId: user.id });
       const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 200, height: 200 });
       await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 200 });
+      ctx.getMock(JobRepository).queueAll.mockResolvedValue();
 
       const auth = factory.auth({ user });
 
@@ -128,6 +130,7 @@ describe(PersonService.name, () => {
       const { person } = await ctx.newPerson({ ownerId: user.id });
       const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 150, height: 200 });
       await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 200 });
+      ctx.getMock(JobRepository).queueAll.mockResolvedValue();
 
       await ctx.newEdits(asset.id, {
         edits: [
@@ -199,6 +202,7 @@ describe(PersonService.name, () => {
       const { person } = await ctx.newPerson({ ownerId: user.id });
       const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 100, height: 200 });
       await ctx.newExif({ assetId: asset.id, exifImageWidth: 200, exifImageHeight: 100 });
+      ctx.getMock(JobRepository).queueAll.mockResolvedValue();
 
       await ctx.newEdits(asset.id, {
         edits: [
@@ -263,6 +267,7 @@ describe(PersonService.name, () => {
       const { person } = await ctx.newPerson({ ownerId: user.id });
       const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 200, height: 100 });
       await ctx.newExif({ assetId: asset.id, exifImageHeight: 100, exifImageWidth: 200 });
+      ctx.getMock(JobRepository).queueAll.mockResolvedValue();
 
       await ctx.newEdits(asset.id, {
         edits: [
@@ -327,6 +332,7 @@ describe(PersonService.name, () => {
       const { person } = await ctx.newPerson({ ownerId: user.id });
       const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 200, height: 150 });
       await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 200 });
+      ctx.getMock(JobRepository).queueAll.mockResolvedValue();
 
       await ctx.newEdits(asset.id, {
         edits: [
@@ -400,6 +406,7 @@ describe(PersonService.name, () => {
       const { person } = await ctx.newPerson({ ownerId: user.id });
       const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 150, height: 100 });
       await ctx.newExif({ assetId: asset.id, exifImageHeight: 100, exifImageWidth: 200 });
+      ctx.getMock(JobRepository).queueAll.mockResolvedValue();
 
       await ctx.newEdits(asset.id, {
         edits: [
@@ -473,6 +480,7 @@ describe(PersonService.name, () => {
       const { person } = await ctx.newPerson({ ownerId: user.id });
       const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 200, height: 150 });
       await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 150 });
+      ctx.getMock(JobRepository).queueAll.mockResolvedValue();
 
       await ctx.newEdits(asset.id, {
         edits: [
@@ -543,6 +551,7 @@ describe(PersonService.name, () => {
       const { person } = await ctx.newPerson({ ownerId: user.id });
       const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 150, height: 100 });
       await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 200 });
+      ctx.getMock(JobRepository).queueAll.mockResolvedValue();
 
       await ctx.newEdits(asset.id, {
         edits: [
@@ -622,6 +631,7 @@ describe(PersonService.name, () => {
       const { person } = await ctx.newPerson({ ownerId: user.id });
       const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 100, height: 100 });
       await ctx.newExif({ assetId: asset.id, exifImageHeight: 100, exifImageWidth: 100 });
+      ctx.getMock(JobRepository).queueAll.mockResolvedValue();
 
       await ctx.newEdits(asset.id, {
         edits: [
@@ -692,6 +702,7 @@ describe(PersonService.name, () => {
       const { person } = await ctx.newPerson({ ownerId: user.id });
       const { asset } = await ctx.newAsset({ id: factory.uuid(), ownerId: user.id, width: 100, height: 100 });
       await ctx.newExif({ assetId: asset.id, exifImageHeight: 200, exifImageWidth: 100, orientation: '6' });
+      ctx.getMock(JobRepository).queueAll.mockResolvedValue();
 
       await ctx.newEdits(asset.id, {
         edits: [

web/src/lib/components/asset-viewer/face-editor/face-editor.svelte

@@ -2,6 +2,7 @@
   import { shortcut } from '$lib/actions/shortcut';
   import ImageThumbnail from '$lib/components/assets/thumbnail/image-thumbnail.svelte';
   import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';
+  import FaceCreateTagModal from '$lib/modals/CreateFaceModal.svelte';
   import { getPeopleThumbnailUrl } from '$lib/utils';
   import { getNaturalSize, scaleToFit } from '$lib/utils/container-utils';
   import { handleError } from '$lib/utils/handle-error';
@@ -260,6 +261,44 @@
     };
   };
 
+  type FaceCoordinates = NonNullable<ReturnType<typeof getFaceCroppedCoordinates>>;
+
+  const getFacePreviewUrl = (data: FaceCoordinates) => {
+    if (!htmlElement) {
+      return;
+    }
+
+    const natural = getNaturalSize(htmlElement);
+    if (natural.width <= 0 || natural.height <= 0) {
+      return;
+    }
+
+    const x = clamp(data.x, 0, natural.width - 1);
+    const y = clamp(data.y, 0, natural.height - 1);
+    const width = clamp(data.width, 1, natural.width - x);
+    const height = clamp(data.height, 1, natural.height - y);
+
+    if (width <= 0 || height <= 0) {
+      return;
+    }
+
+    const canvas = document.createElement('canvas');
+    canvas.width = width;
+    canvas.height = height;
+
+    const context = canvas.getContext('2d');
+    if (!context) {
+      return;
+    }
+
+    try {
+      context.drawImage(htmlElement, x, y, width, height, 0, 0, width, height);
+      return canvas.toDataURL('image/png');
+    } catch {
+      return;
+    }
+  };
+
   const tagFace = async (person: PersonResponseDto) => {
     try {
       const data = getFaceCroppedCoordinates();
@@ -294,6 +333,28 @@
     }
   };
 
+  const showCreateFaceModal = async () => {
+    try {
+      const data = getFaceCroppedCoordinates();
+      if (!data) {
+        return;
+      }
+
+      const created = await modalManager.show(FaceCreateTagModal, {
+        assetId,
+        ...data,
+        previewUrl: getFacePreviewUrl(data),
+      });
+      if (!created) {
+        return;
+      }
+
+      onClose();
+    } catch (error) {
+      handleError(error, 'Error creating and tagging face');
+    }
+  };
+
   onDestroy(() => {
     onClose();
   });
@@ -303,19 +364,19 @@
 
 <div
   id="face-editor-data"
-  class="absolute start-0 top-0 z-5 h-full w-full overflow-hidden"
+  class="absolute inset-s-0 top-0 z-5 h-full w-full overflow-hidden"
   data-overlay-interactive
   data-face-left={faceBoxPosition.left}
   data-face-top={faceBoxPosition.top}
   data-face-width={faceBoxPosition.width}
   data-face-height={faceBoxPosition.height}
 >
-  <canvas bind:this={canvasEl} id="face-editor" class="absolute top-0 start-0"></canvas>
+  <canvas bind:this={canvasEl} id="face-editor" class="absolute top-0 inset-s-0"></canvas>
 
   <div
     id="face-selector"
     bind:this={faceSelectorEl}
-    class="absolute top-[calc(50%-250px)] start-[calc(50%-125px)] max-w-[250px] w-[250px] bg-white dark:bg-immich-dark-gray dark:text-immich-dark-fg backdrop-blur-sm px-2 py-4 rounded-xl border border-gray-200 dark:border-gray-800 transition-[top,left] duration-200 ease-out"
+    class="absolute top-[calc(50%-250px)] inset-s-[calc(50%-125px)] max-w-62.5 w-62.5 bg-white dark:bg-immich-dark-gray dark:text-immich-dark-fg backdrop-blur-sm px-2 py-4 rounded-xl border border-gray-200 dark:border-gray-800 transition-[top,left] duration-200 ease-out"
   >
     <p class="text-center text-sm">{$t('select_person_to_tag')}</p>
 
@@ -354,6 +415,12 @@
       {/if}
     </div>
 
-    <Button size="small" fullWidth onclick={onClose} color="danger" class="mt-2">{$t('cancel')}</Button>
+    <Button size="small" fullWidth onclick={showCreateFaceModal} variant="outline" class="mt-2">
+      {$t('create_person')}
+    </Button>
+
+    <Button size="small" fullWidth onclick={onClose} color="danger" class="mt-2">
+      {$t('cancel')}
+    </Button>
   </div>
 </div>

web/src/lib/modals/CreateFaceModal.svelte

@@ -0,0 +1,91 @@
+<script lang="ts">
+  import { assetViewerManager } from '$lib/managers/asset-viewer-manager.svelte';
+  import { delay } from '$lib/utils/asset-utils';
+  import { handleError } from '$lib/utils/handle-error';
+  import { createFace, createPerson } from '@immich/sdk';
+  import { Field, FormModal, Input, LoadingSpinner, Text } from '@immich/ui';
+  import { t } from 'svelte-i18n';
+
+  type Props = {
+    assetId: string;
+    imageWidth: number;
+    imageHeight: number;
+    x: number;
+    y: number;
+    width: number;
+    height: number;
+    onClose: (created?: boolean) => void;
+    previewUrl?: string;
+  };
+
+  let { assetId, imageWidth, imageHeight, x, y, width, height, onClose, previewUrl }: Props = $props();
+  let personName = $state('');
+  let isSubmitting = $state(false);
+
+  const getTrimmedName = () => personName.trim();
+
+  const onSubmit = async () => {
+    const name = getTrimmedName();
+    if (!name) {
+      return;
+    }
+
+    try {
+      isSubmitting = true;
+
+      const person = await createPerson({
+        personCreateDto: {
+          name,
+        },
+      });
+
+      await createFace({
+        assetFaceCreateDto: {
+          assetId,
+          imageHeight,
+          imageWidth,
+          personId: person.id,
+          x,
+          y,
+          width,
+          height,
+        },
+      });
+
+      await delay(1500);
+      await assetViewerManager.setAssetId(assetId);
+      onClose(true);
+    } catch (error) {
+      handleError(error, 'Error creating and tagging face');
+    } finally {
+      isSubmitting = false;
+    }
+  };
+</script>
+
+<FormModal
+  size="small"
+  title={$t('create_person')}
+  submitText={$t('tag_face')}
+  disabled={!getTrimmedName() || isSubmitting}
+  {onClose}
+  {onSubmit}
+>
+  <Text size="tiny" class="mb-4" color="muted">{$t('create_person_subtitle')}</Text>
+  {#if previewUrl}
+    <Field label={$t('preview')}>
+      <div class="flex justify-center rounded-xl bg-gray-50 p-3 dark:border-gray-700 dark:bg-black/20 relative">
+        <img src={previewUrl} alt={$t('preview')} class="max-h-48 rounded-lg object-contain shadow-sm" />
+        {#if isSubmitting}
+          <div class="flex place-items-center place-content-center absolute inset-0 bg-black/20 rounded-lg">
+            <LoadingSpinner />
+          </div>
+        {/if}
+      </div>
+    </Field>
+  {/if}
+
+  <Field label={$t('name')} required class="mt-3">
+    <Input autofocus bind:value={personName} disabled={isSubmitting} />
+  </Field>
+</FormModal>