chore: face cluster abstraction

open-api/immich-openapi-specs.json

@@ -8409,7 +8409,7 @@
           "content": {
             "application/json": {
               "schema": {
-                "$ref": "#/components/schemas/MergePersonDto"
+                "$ref": "#/components/schemas/MergeFaceClusterDto"
               }
             }
           },
@@ -18878,10 +18878,10 @@
         },
         "type": "object"
       },
-      "MergePersonDto": {
+      "MergeFaceClusterDto": {
         "properties": {
           "ids": {
-            "description": "Person IDs to merge",
+            "description": "Face cluster IDs to merge",
             "items": {
               "format": "uuid",
               "type": "string"
@@ -23055,6 +23055,70 @@
         ],
         "type": "object"
       },
+      "SyncAssetFaceV3": {
+        "properties": {
+          "assetId": {
+            "description": "Asset ID",
+            "type": "string"
+          },
+          "boundingBoxX1": {
+            "type": "integer"
+          },
+          "boundingBoxX2": {
+            "type": "integer"
+          },
+          "boundingBoxY1": {
+            "type": "integer"
+          },
+          "boundingBoxY2": {
+            "type": "integer"
+          },
+          "deletedAt": {
+            "description": "Face deleted at",
+            "format": "date-time",
+            "nullable": true,
+            "type": "string"
+          },
+          "faceClusterId": {
+            "description": "Face cluster ID",
+            "nullable": true,
+            "type": "string"
+          },
+          "id": {
+            "description": "Asset face ID",
+            "type": "string"
+          },
+          "imageHeight": {
+            "type": "integer"
+          },
+          "imageWidth": {
+            "type": "integer"
+          },
+          "isVisible": {
+            "description": "Is the face visible in the asset",
+            "type": "boolean"
+          },
+          "sourceType": {
+            "description": "Source type",
+            "type": "string"
+          }
+        },
+        "required": [
+          "assetId",
+          "boundingBoxX1",
+          "boundingBoxX2",
+          "boundingBoxY1",
+          "boundingBoxY2",
+          "deletedAt",
+          "faceClusterId",
+          "id",
+          "imageHeight",
+          "imageWidth",
+          "isVisible",
+          "sourceType"
+        ],
+        "type": "object"
+      },
       "SyncAssetMetadataDeleteV1": {
         "properties": {
           "assetId": {
@@ -23348,10 +23412,14 @@
           "StackV1",
           "StackDeleteV1",
           "PersonV1",
+          "PersonV2",
           "PersonDeleteV1",
           "AssetFaceV1",
           "AssetFaceV2",
+          "AssetFaceV3",
           "AssetFaceDeleteV1",
+          "FaceClusterV1",
+          "FaceClusterDeleteV1",
           "UserMetadataV1",
           "UserMetadataDeleteV1",
           "SyncAckV1",
@@ -23360,6 +23428,47 @@
         ],
         "type": "string"
       },
+      "SyncFaceClusterDeleteV1": {
+        "properties": {
+          "faceClusterId": {
+            "description": "Face cluster ID",
+            "type": "string"
+          }
+        },
+        "required": [
+          "faceClusterId"
+        ],
+        "type": "object"
+      },
+      "SyncFaceClusterV1": {
+        "properties": {
+          "createdAt": {
+            "description": "Created at",
+            "format": "date-time",
+            "type": "string"
+          },
+          "id": {
+            "description": "Face cluster ID",
+            "type": "string"
+          },
+          "ownerId": {
+            "description": "Owner ID",
+            "type": "string"
+          },
+          "updatedAt": {
+            "description": "Updated at",
+            "format": "date-time",
+            "type": "string"
+          }
+        },
+        "required": [
+          "createdAt",
+          "id",
+          "ownerId",
+          "updatedAt"
+        ],
+        "type": "object"
+      },
       "SyncMemoryAssetDeleteV1": {
         "properties": {
           "assetId": {
@@ -23602,6 +23711,75 @@
         ],
         "type": "object"
       },
+      "SyncPersonV2": {
+        "properties": {
+          "birthDate": {
+            "description": "Birth date",
+            "format": "date-time",
+            "nullable": true,
+            "type": "string"
+          },
+          "color": {
+            "description": "Color",
+            "nullable": true,
+            "type": "string"
+          },
+          "createdAt": {
+            "description": "Created at",
+            "format": "date-time",
+            "type": "string"
+          },
+          "faceAssetId": {
+            "description": "Face asset ID",
+            "nullable": true,
+            "type": "string"
+          },
+          "faceClusterId": {
+            "description": "Face cluster ID",
+            "nullable": true,
+            "type": "string"
+          },
+          "id": {
+            "description": "Person ID",
+            "type": "string"
+          },
+          "isFavorite": {
+            "description": "Is favorite",
+            "type": "boolean"
+          },
+          "isHidden": {
+            "description": "Is hidden",
+            "type": "boolean"
+          },
+          "name": {
+            "description": "Person name",
+            "type": "string"
+          },
+          "ownerId": {
+            "description": "Owner ID",
+            "type": "string"
+          },
+          "updatedAt": {
+            "description": "Updated at",
+            "format": "date-time",
+            "type": "string"
+          }
+        },
+        "required": [
+          "birthDate",
+          "color",
+          "createdAt",
+          "faceAssetId",
+          "faceClusterId",
+          "id",
+          "isFavorite",
+          "isHidden",
+          "name",
+          "ownerId",
+          "updatedAt"
+        ],
+        "type": "object"
+      },
       "SyncRequestType": {
         "description": "Sync request types",
         "enum": [
@@ -23624,8 +23802,11 @@
           "StacksV1",
           "UsersV1",
           "PeopleV1",
+          "PeopleV2",
           "AssetFacesV1",
           "AssetFacesV2",
+          "AssetFacesV3",
+          "FaceClusterV1",
           "UserMetadataV1"
         ],
         "type": "string"

server/src/controllers/person.controller.ts

@@ -19,7 +19,7 @@ import { BulkIdResponseDto, BulkIdsDto } from 'src/dtos/asset-ids.response.dto';
 import { AuthDto } from 'src/dtos/auth.dto';
 import {
   AssetFaceUpdateDto,
-  MergePersonDto,
+  MergeFaceClusterDto,
   PeopleResponseDto,
   PeopleUpdateDto,
   PersonCreateDto,
@@ -182,7 +182,7 @@ export class PersonController {
   mergePerson(
     @Auth() auth: AuthDto,
     @Param() { id }: UUIDParamDto,
-    @Body() dto: MergePersonDto,
+    @Body() dto: MergeFaceClusterDto,
   ): Promise<BulkIdResponseDto[]> {
     return this.service.mergePerson(auth, id, dto);
   }

server/src/database.ts

@@ -269,7 +269,7 @@ export type AssetFace = {
   boundingBoxY2: number;
   imageHeight: number;
   imageWidth: number;
-  personId: string | null;
+  faceClusterId: string | null;
   sourceType: SourceType;
   person?: ShallowDehydrateObject<Person> | null;
   updatedAt: Date;

server/src/dtos/person.dto.ts

@@ -67,8 +67,8 @@ export class PeopleUpdateItem extends PersonUpdateDto {
   id!: string;
 }
 
-export class MergePersonDto {
+export class MergeFaceClusterDto {
-  @ValidateUUID({ each: true, description: 'Person IDs to merge' })
+  @ValidateUUID({ each: true, description: 'Face cluster IDs to merge' })
   ids!: string[];
 }
 

server/src/dtos/sync.dto.ts

@@ -411,6 +411,18 @@ export class SyncPersonV1 {
   faceAssetId!: string | null;
 }
 
+@ExtraModel()
+export class SyncPersonV2 extends SyncPersonV1 {
+  @ApiProperty({ description: 'Face cluster ID' })
+  faceClusterId!: string | null;
+}
+
+export function syncPersonV2ToV1(personV2: SyncPersonV2): SyncPersonV1 {
+  const { faceClusterId: _, ...personV1 } = personV2;
+
+  return personV1;
+}
+
 @ExtraModel()
 export class SyncPersonDeleteV1 {
   @ApiProperty({ description: 'Person ID' })
@@ -449,6 +461,40 @@ export class SyncAssetFaceV2 extends SyncAssetFaceV1 {
   isVisible!: boolean;
 }
 
+@ExtraModel()
+export class SyncAssetFaceV3 {
+  @ApiProperty({ description: 'Asset face ID' })
+  id!: string;
+  @ApiProperty({ description: 'Asset ID' })
+  assetId!: string;
+  @ApiProperty({ description: 'Face cluster ID' })
+  faceClusterId!: string | null;
+  @ApiProperty({ type: 'integer' })
+  imageWidth!: number;
+  @ApiProperty({ type: 'integer' })
+  imageHeight!: number;
+  @ApiProperty({ type: 'integer' })
+  boundingBoxX1!: number;
+  @ApiProperty({ type: 'integer' })
+  boundingBoxY1!: number;
+  @ApiProperty({ type: 'integer' })
+  boundingBoxX2!: number;
+  @ApiProperty({ type: 'integer' })
+  boundingBoxY2!: number;
+  @ApiProperty({ description: 'Source type' })
+  sourceType!: string;
+  @ApiProperty({ description: 'Face deleted at' })
+  deletedAt!: Date | null;
+  @ApiProperty({ description: 'Is the face visible in the asset' })
+  isVisible!: boolean;
+}
+
+export function syncAssetFaceV3ToV2(faceV3: SyncAssetFaceV3, personId: string | null): SyncAssetFaceV2 {
+  const { faceClusterId: _, ...face } = faceV3;
+
+  return { ...face, personId };
+}
+
 export function syncAssetFaceV2ToV1(faceV2: SyncAssetFaceV2): SyncAssetFaceV1 {
   const { deletedAt: _, isVisible: __, ...faceV1 } = faceV2;
 
@@ -461,6 +507,24 @@ export class SyncAssetFaceDeleteV1 {
   assetFaceId!: string;
 }
 
+@ExtraModel()
+export class SyncFaceClusterV1 {
+  @ApiProperty({ description: 'Face cluster ID' })
+  id!: string;
+  @ApiProperty({ description: 'Created at' })
+  createdAt!: Date;
+  @ApiProperty({ description: 'Updated at' })
+  updatedAt!: Date;
+  @ApiProperty({ description: 'Owner ID' })
+  ownerId!: string;
+}
+
+@ExtraModel()
+export class SyncFaceClusterDeleteV1 {
+  @ApiProperty({ description: 'Face cluster ID' })
+  faceClusterId!: string;
+}
+
 @ExtraModel()
 export class SyncUserMetadataV1 {
   @ApiProperty({ description: 'User ID' })
@@ -530,10 +594,14 @@ export type SyncItem = {
   [SyncEntityType.PartnerStackDeleteV1]: SyncStackDeleteV1;
   [SyncEntityType.PartnerStackV1]: SyncStackV1;
   [SyncEntityType.PersonV1]: SyncPersonV1;
+  [SyncEntityType.PersonV2]: SyncPersonV2;
   [SyncEntityType.PersonDeleteV1]: SyncPersonDeleteV1;
   [SyncEntityType.AssetFaceV1]: SyncAssetFaceV1;
   [SyncEntityType.AssetFaceV2]: SyncAssetFaceV2;
+  [SyncEntityType.AssetFaceV3]: SyncAssetFaceV3;
   [SyncEntityType.AssetFaceDeleteV1]: SyncAssetFaceDeleteV1;
+  [SyncEntityType.FaceClusterV1]: SyncFaceClusterV1;
+  [SyncEntityType.FaceClusterDeleteV1]: SyncFaceClusterDeleteV1;
   [SyncEntityType.UserMetadataV1]: SyncUserMetadataV1;
   [SyncEntityType.UserMetadataDeleteV1]: SyncUserMetadataDeleteV1;
   [SyncEntityType.SyncAckV1]: SyncAckV1;

server/src/enum.ts

@@ -735,8 +735,11 @@ export enum SyncRequestType {
   StacksV1 = 'StacksV1',
   UsersV1 = 'UsersV1',
   PeopleV1 = 'PeopleV1',
+  PeopleV2 = 'PeopleV2',
   AssetFacesV1 = 'AssetFacesV1',
   AssetFacesV2 = 'AssetFacesV2',
+  AssetFacesV3 = 'AssetFacesV3',
+  FaceClusterV1 = 'FaceClusterV1',
   UserMetadataV1 = 'UserMetadataV1',
 }
 
@@ -794,12 +797,17 @@ export enum SyncEntityType {
   StackDeleteV1 = 'StackDeleteV1',
 
   PersonV1 = 'PersonV1',
+  PersonV2 = 'PersonV2',
   PersonDeleteV1 = 'PersonDeleteV1',
 
   AssetFaceV1 = 'AssetFaceV1',
   AssetFaceV2 = 'AssetFaceV2',
+  AssetFaceV3 = 'AssetFaceV3',
   AssetFaceDeleteV1 = 'AssetFaceDeleteV1',
 
+  FaceClusterV1 = 'FaceClusterV1',
+  FaceClusterDeleteV1 = 'FaceClusterDeleteV1',
+
   UserMetadataV1 = 'UserMetadataV1',
   UserMetadataDeleteV1 = 'UserMetadataDeleteV1',
 

server/src/repositories/person.repository.ts

@@ -33,9 +33,9 @@ export interface AssetFaceId {
 }
 
 export interface UpdateFacesData {
-  oldPersonId?: string;
+  oldFaceClusterId?: string;
   faceIds?: string[];
-  newPersonId: string;
+  newFaceClusterId: string;
 }
 
 export interface PersonStatistics {
@@ -54,7 +54,7 @@ export interface GetAllPeopleOptions {
 }
 
 export interface GetAllFacesOptions {
-  personId?: string | null;
+  faceClusterId?: string | null;
   assetId?: string;
   sourceType?: SourceType;
 }
@@ -65,7 +65,7 @@ export type SelectFaceOptions = (keyof Selectable<AssetFaceTable>)[];
 
 const withPerson = (eb: ExpressionBuilder<DB, 'asset_face'>) => {
   return jsonObjectFrom(
-    eb.selectFrom('person').selectAll('person').whereRef('person.id', '=', 'asset_face.personId'),
+    eb.selectFrom('person').selectAll('person').whereRef('person.faceClusterId', '=', 'asset_face.faceClusterId'),
   ).as('person');
 };
 
@@ -80,11 +80,11 @@ export class PersonRepository {
   constructor(@InjectKysely() private db: Kysely<DB>) {}
 
   @GenerateSql({ params: [{ oldPersonId: DummyValue.UUID, newPersonId: DummyValue.UUID }] })
-  async reassignFaces({ oldPersonId, faceIds, newPersonId }: UpdateFacesData): Promise<number> {
+  async reassignFaces({ oldFaceClusterId, faceIds, newFaceClusterId }: UpdateFacesData): Promise<number> {
     const result = await this.db
       .updateTable('asset_face')
-      .set({ personId: newPersonId })
+      .set({ faceClusterId: newFaceClusterId })
-      .$if(!!oldPersonId, (qb) => qb.where('asset_face.personId', '=', oldPersonId!))
+      .$if(!!oldFaceClusterId, (qb) => qb.where('asset_face.faceClusterId', '=', oldFaceClusterId!))
       .$if(!!faceIds, (qb) => qb.where('asset_face.id', 'in', faceIds!))
       .executeTakeFirst();
 
@@ -94,7 +94,7 @@ export class PersonRepository {
   async unassignFaces({ sourceType }: UnassignFacesOptions): Promise<void> {
     await this.db
       .updateTable('asset_face')
-      .set({ personId: null })
+      .set({ faceClusterId: null })
       .where('asset_face.sourceType', '=', sourceType)
       .execute();
   }
@@ -117,8 +117,8 @@ export class PersonRepository {
     return this.db
       .selectFrom('asset_face')
       .selectAll('asset_face')
-      .$if(options.personId === null, (qb) => qb.where('asset_face.personId', 'is', null))
+      .$if(options.faceClusterId === null, (qb) => qb.where('asset_face.faceClusterId', 'is', null))
-      .$if(!!options.personId, (qb) => qb.where('asset_face.personId', '=', options.personId!))
+      .$if(!!options.faceClusterId, (qb) => qb.where('asset_face.faceClusterId', '=', options.faceClusterId!))
       .$if(!!options.sourceType, (qb) => qb.where('asset_face.sourceType', '=', options.sourceType!))
       .$if(!!options.assetId, (qb) => qb.where('asset_face.assetId', '=', options.assetId!))
       .where('asset_face.deletedAt', 'is', null)
@@ -153,7 +153,7 @@ export class PersonRepository {
     const items = await this.db
       .selectFrom('person')
       .selectAll('person')
-      .innerJoin('asset_face', 'asset_face.personId', 'person.id')
+      .innerJoin('asset_face', 'asset_face.faceClusterId', 'person.faceClusterId')
       .innerJoin('asset', (join) =>
         join
           .onRef('asset_face.assetId', '=', 'asset.id')
@@ -209,7 +209,7 @@ export class PersonRepository {
     return this.db
       .selectFrom('person')
       .selectAll('person')
-      .leftJoin('asset_face', 'asset_face.personId', 'person.id')
+      .leftJoin('asset_face', 'asset_face.faceClusterId', 'person.faceClusterId')
       .where('asset_face.deletedAt', 'is', null)
       .where('asset_face.isVisible', 'is', true)
       .having((eb) => eb.fn.count('asset_face.assetId'), '=', 0)
@@ -248,7 +248,7 @@ export class PersonRepository {
   getFaceForFacialRecognitionJob(id: string) {
     return this.db
       .selectFrom('asset_face')
-      .select(['asset_face.id', 'asset_face.personId', 'asset_face.sourceType'])
+      .select(['asset_face.id', 'asset_face.faceClusterId', 'asset_face.sourceType'])
       .select((eb) =>
         jsonObjectFrom(
           eb
@@ -297,10 +297,10 @@ export class PersonRepository {
   }
 
   @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] })
-  async reassignFace(assetFaceId: string, newPersonId: string): Promise<number> {
+  async reassignFace(assetFaceId: string, newFaceClusterId: string): Promise<number> {
     const result = await this.db
       .updateTable('asset_face')
-      .set({ personId: newPersonId })
+      .set({ faceClusterId: newFaceClusterId })
       .where('asset_face.id', '=', assetFaceId)
       .executeTakeFirst();
 
@@ -346,13 +346,13 @@ export class PersonRepository {
   }
 
   @GenerateSql({ params: [DummyValue.UUID] })
-  async getStatistics(personId: string): Promise<PersonStatistics> {
+  async getStatistics(faceClusterId: string): Promise<PersonStatistics> {
     const result = await this.db
       .selectFrom('asset_face')
       .leftJoin('asset', (join) =>
         join
           .onRef('asset.id', '=', 'asset_face.assetId')
-          .on('asset_face.personId', '=', personId)
+          .on('asset_face.faceClusterId', '=', faceClusterId)
           .on('asset.visibility', '=', sql.lit(AssetVisibility.Timeline))
           .on('asset.deletedAt', 'is', null),
       )
@@ -375,7 +375,7 @@ export class PersonRepository {
         eb.exists((eb) =>
           eb
             .selectFrom('asset_face')
-            .whereRef('asset_face.personId', '=', 'person.id')
+            .whereRef('asset_face.faceClusterId', '=', 'person.faceClusterId')
             .where('asset_face.deletedAt', 'is', null)
             .where('asset_face.isVisible', '=', true)
             .where((eb) =>
@@ -395,7 +395,16 @@ export class PersonRepository {
       .executeTakeFirstOrThrow();
   }
 
-  create(person: Insertable<PersonTable>) {
+  async create(person: Insertable<PersonTable>) {
+    if (!person.faceClusterId) {
+      const { id } = await this.db
+        .insertInto('face_cluster')
+        .values({ ownerId: person.ownerId })
+        .returning('id')
+        .executeTakeFirstOrThrow();
+      person.faceClusterId = id;
+    }
+
     return this.db.insertInto('person').values(person).returningAll().executeTakeFirstOrThrow();
   }
 
@@ -486,18 +495,19 @@ export class PersonRepository {
       .selectFrom('asset_face')
       .selectAll('asset_face')
       .select(withPerson)
+      .innerJoin('person', (join) => join.onRef('person.faceClusterId', '=', 'asset_face.faceClusterId'))
+      .where('person.id', 'in', personIds)
       .where('asset_face.assetId', 'in', assetIds)
-      .where('asset_face.personId', 'in', personIds)
       .where('asset_face.deletedAt', 'is', null)
       .execute();
   }
 
   @GenerateSql({ params: [DummyValue.UUID] })
-  getRandomFace(personId: string) {
+  getRandomFace(faceClusterId: string) {
     return this.db
       .selectFrom('asset_face')
       .selectAll('asset_face')
-      .where('asset_face.personId', '=', personId)
+      .where('asset_face.faceClusterId', '=', faceClusterId)
       .where('asset_face.deletedAt', 'is', null)
       .where('asset_face.isVisible', 'is', true)
       .executeTakeFirst();
@@ -584,7 +594,9 @@ export class PersonRepository {
       .selectFrom('asset_face')
       .select('asset_face.id')
       .where('asset_face.assetId', '=', assetId)
-      .where('asset_face.personId', '=', personId)
+      .innerJoin('person', (join) =>
+        join.onRef('person.faceClusterId', '=', 'asset_face.faceClusterId').on('person.id', '=', personId),
+      )
       .innerJoin('asset', (join) => join.onRef('asset.id', '=', 'asset_face.assetId').on('asset.isOffline', '=', false))
       .executeTakeFirst();
   }

server/src/repositories/search.repository.ts

@@ -338,15 +338,15 @@ export class SearchRepository {
             .selectFrom('asset_face')
             .select([
               'asset_face.id',
-              'asset_face.personId',
+              'asset_face.faceClusterId',
               sql<number>`face_search.embedding <=> ${embedding}`.as('distance'),
             ])
             .innerJoin('asset', 'asset.id', 'asset_face.assetId')
             .innerJoin('face_search', 'face_search.faceId', 'asset_face.id')
-            .leftJoin('person', 'person.id', 'asset_face.personId')
+            .leftJoin('person', 'person.faceClusterId', 'asset_face.faceClusterId')
             .where('asset.ownerId', '=', anyUuid(userIds))
             .where('asset.deletedAt', 'is', null)
-            .$if(!!hasPerson, (qb) => qb.where('asset_face.personId', 'is not', null))
+            .$if(!!hasPerson, (qb) => qb.where('asset_face.faceClusterId', 'is not', null))
             .$if(!!minBirthDate, (qb) =>
               qb.where((eb) =>
                 eb.or([eb('person.birthDate', 'is', null), eb('person.birthDate', '<=', minBirthDate!)]),

server/src/repositories/sync.repository.ts

@@ -57,6 +57,7 @@ export class SyncRepository {
   assetFace: AssetFaceSync;
   assetMetadata: AssetMetadataSync;
   authUser: AuthUserSync;
+  faceCluster: FaceClusterSync;
   memory: MemorySync;
   memoryToAsset: MemoryToAssetSync;
   partner: PartnerSync;
@@ -80,6 +81,7 @@ export class SyncRepository {
     this.assetFace = new AssetFaceSync(this.db);
     this.assetMetadata = new AssetMetadataSync(this.db);
     this.authUser = new AuthUserSync(this.db);
+    this.faceCluster = new FaceClusterSync(this.db);
     this.memory = new MemorySync(this.db);
     this.memoryToAsset = new MemoryToAssetSync(this.db);
     this.partner = new PartnerSync(this.db);
@@ -447,6 +449,7 @@ class PersonSync extends BaseSync {
         'color',
         'updateId',
         'faceAssetId',
+        'faceClusterId',
       ])
       .where('ownerId', '=', options.userId)
       .stream();
@@ -473,7 +476,7 @@ class AssetFaceSync extends BaseSync {
       .select([
         'asset_face.id',
         'assetId',
-        'personId',
+        'faceClusterId',
         'imageWidth',
         'imageHeight',
         'boundingBoxX1',
@@ -485,6 +488,8 @@ class AssetFaceSync extends BaseSync {
         'asset_face.deletedAt',
         'asset_face.updateId',
       ])
+      .leftJoin('person', 'person.faceClusterId', 'asset_face.faceClusterId')
+      .select('person.id as personId')
       .leftJoin('asset', 'asset.id', 'asset_face.assetId')
       .where('asset.ownerId', '=', options.userId)
       .where('asset_face.isVisible', '=', true)
@@ -492,6 +497,35 @@ class AssetFaceSync extends BaseSync {
   }
 }
 
+class FaceClusterSync extends BaseSync {
+  @GenerateSql({ params: [dummyQueryOptions], stream: true })
+  getDeletes(options: SyncQueryOptions) {
+    return this.auditQuery('face_cluster_audit', options)
+      .select(['face_cluster_audit.id', 'face_cluster_audit.faceClusterId'])
+      .leftJoin('face_cluster', 'face_cluster.id', 'face_cluster_audit.id')
+      .where('face_cluster.ownerId', '=', options.userId)
+      .stream();
+  }
+
+  cleanupAuditTable(daysAgo: number) {
+    return this.auditCleanup('face_cluster_audit', daysAgo);
+  }
+
+  @GenerateSql({ params: [dummyQueryOptions], stream: true })
+  getUpserts(options: SyncQueryOptions) {
+    return this.upsertQuery('face_cluster', options)
+      .select([
+        'face_cluster.id',
+        'face_cluster.createdAt',
+        'face_cluster.updatedAt',
+        'face_cluster.ownerId',
+        'face_cluster.updateId',
+      ])
+      .where('face_cluster.ownerId', '=', options.userId)
+      .stream();
+  }
+}
+
 class AssetExifSync extends BaseSync {
   @GenerateSql({ params: [dummyQueryOptions], stream: true })
   getUpserts(options: SyncQueryOptions) {

server/src/schema/functions.ts

@@ -299,3 +299,16 @@ export const asset_edit_audit = registerFunction({
       RETURN NULL;
     END`,
 });
+
+export const face_cluster_delete_audit = registerFunction({
+  name: 'face_cluster_delete_audit',
+  returnType: 'TRIGGER',
+  language: 'PLPGSQL',
+  body: `
+    BEGIN
+      INSERT INTO face_cluster_audit ("faceClusterId", "ownerId")
+      SELECT "id", "ownerId"
+      FROM OLD;
+      RETURN NULL;
+    END`,
+});

server/src/schema/index.ts

@@ -41,6 +41,8 @@ import { AssetMetadataTable } from 'src/schema/tables/asset-metadata.table';
 import { AssetOcrTable } from 'src/schema/tables/asset-ocr.table';
 import { AssetTable } from 'src/schema/tables/asset.table';
 import { AuditTable } from 'src/schema/tables/audit.table';
+import { FaceClusterAuditTable } from 'src/schema/tables/face-cluster-audit.table';
+import { FaceClusterTable } from 'src/schema/tables/face-cluster.table';
 import { FaceSearchTable } from 'src/schema/tables/face-search.table';
 import { GeodataPlacesTable } from 'src/schema/tables/geodata-places.table';
 import { LibraryTable } from 'src/schema/tables/library.table';
@@ -199,6 +201,8 @@ export interface DB {
 
   audit: AuditTable;
 
+  face_cluster: FaceClusterTable;
+  face_cluster_audit: FaceClusterAuditTable;
   face_search: FaceSearchTable;
 
   geodata_places: GeodataPlacesTable;

server/src/schema/tables/asset-face.table.ts

@@ -15,7 +15,7 @@ import { SourceType } from 'src/enum';
 import { asset_face_source_type } from 'src/schema/enums';
 import { asset_face_audit } from 'src/schema/functions';
 import { AssetTable } from 'src/schema/tables/asset.table';
-import { PersonTable } from 'src/schema/tables/person.table';
+import { FaceClusterTable } from 'src/schema/tables/face-cluster.table';
 
 @Table({ name: 'asset_face' })
 @UpdatedAtTrigger('asset_face_updatedAt')
@@ -26,13 +26,13 @@ import { PersonTable } from 'src/schema/tables/person.table';
   when: 'pg_trigger_depth() = 0',
 })
 // schemaFromDatabase does not preserve column order
-@Index({ name: 'asset_face_assetId_personId_idx', columns: ['assetId', 'personId'] })
+@Index({ name: 'asset_face_assetId_faceClusterId_idx', columns: ['assetId', 'faceClusterId'] })
 @Index({
-  name: 'asset_face_personId_assetId_notDeleted_isVisible_idx',
+  name: 'asset_face_faceClusterId_assetId_notDeleted_isVisible_idx',
-  columns: ['personId', 'assetId'],
+  columns: ['faceClusterId', 'assetId'],
   where: '"deletedAt" IS NULL AND "isVisible" IS TRUE',
 })
-@Index({ columns: ['personId', 'assetId'] })
+@Index({ columns: ['faceClusterId', 'assetId'] })
 export class AssetFaceTable {
   @PrimaryGeneratedColumn()
   id!: Generated<string>;
@@ -45,14 +45,14 @@ export class AssetFaceTable {
   })
   assetId!: string;
 
-  @ForeignKeyColumn(() => PersonTable, {
+  @ForeignKeyColumn(() => FaceClusterTable, {
     onDelete: 'SET NULL',
     onUpdate: 'CASCADE',
     nullable: true,
-    // [personId, assetId] makes this redundant
+    // [faceClusterId, assetId] makes this redundant
     index: false,
   })
-  personId!: string | null;
+  faceClusterId!: string | null;
 
   @Column({ default: 0, type: 'integer' })
   imageWidth!: Generated<number>;

server/src/schema/tables/face-cluster-audit.table.ts

@@ -0,0 +1,17 @@
+import { Column, CreateDateColumn, Generated, Table, Timestamp } from '@immich/sql-tools';
+import { PrimaryGeneratedUuidV7Column } from 'src/decorators';
+
+@Table('face_cluster_audit')
+export class FaceClusterAuditTable {
+  @PrimaryGeneratedUuidV7Column()
+  id!: Generated<string>;
+
+  @Column({ type: 'uuid', index: true })
+  faceClusterId!: string;
+
+  @Column({ type: 'uuid', index: true })
+  ownerId!: string;
+
+  @CreateDateColumn({ default: () => 'clock_timestamp()', index: true })
+  deletedAt!: Generated<Timestamp>;
+}

server/src/schema/tables/face-cluster.table.ts

@@ -0,0 +1,38 @@
+import {
+  AfterDeleteTrigger,
+  CreateDateColumn,
+  ForeignKeyColumn,
+  Generated,
+  PrimaryGeneratedColumn,
+  Table,
+  Timestamp,
+  UpdateDateColumn,
+} from '@immich/sql-tools';
+import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators';
+import { face_cluster_delete_audit } from 'src/schema/functions';
+import { UserTable } from 'src/schema/tables/user.table';
+
+@Table('face_cluster')
+@UpdatedAtTrigger('face_cluster_updatedAt')
+@AfterDeleteTrigger({
+  scope: 'statement',
+  function: face_cluster_delete_audit,
+  referencingOldTableAs: 'old',
+  when: 'pg_trigger_depth() = 0',
+})
+export class FaceClusterTable {
+  @PrimaryGeneratedColumn('uuid')
+  id!: Generated<string>;
+
+  @CreateDateColumn()
+  createdAt!: Generated<Timestamp>;
+
+  @UpdateDateColumn()
+  updatedAt!: Generated<Timestamp>;
+
+  @UpdateIdColumn({ index: true })
+  updateId!: Generated<string>;
+
+  @ForeignKeyColumn(() => UserTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: false })
+  ownerId!: string;
+}

server/src/schema/tables/person.table.ts

@@ -13,6 +13,7 @@ import {
 import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators';
 import { person_delete_audit } from 'src/schema/functions';
 import { AssetFaceTable } from 'src/schema/tables/asset-face.table';
+import { FaceClusterTable } from 'src/schema/tables/face-cluster.table';
 import { UserTable } from 'src/schema/tables/user.table';
 
 @Table('person')
@@ -60,4 +61,7 @@ export class PersonTable {
 
   @UpdateIdColumn({ index: true })
   updateId!: Generated<string>;
+
+  @ForeignKeyColumn(() => FaceClusterTable, { onDelete: 'CASCADE', onUpdate: 'CASCADE', nullable: true, index: true })
+  faceClusterId!: string | null;
 }

server/src/services/metadata.service.ts

@@ -876,7 +876,7 @@ export class MetadataService extends BaseService {
     }
 
     if (missing.length > 0) {
-      this.logger.debugFn(() => `Creating missing persons: ${missing.map((p) => `${p.name}/${p.id}`)}`);
+      this.logger.debug(`Creating missing people: ${missing.map((p) => `${p.name}/${p.id}`)}`);
       const newPersonIds = await this.personRepository.createAll(missing);
       const jobs = newPersonIds.map((id) => ({ name: JobName.PersonGenerateThumbnail, data: { id } }) as const);
       await this.jobRepository.queueAll(jobs);

server/src/services/person.service.ts

@@ -13,7 +13,7 @@ import {
   FaceDto,
   mapFaces,
   mapPerson,
-  MergePersonDto,
+  MergeFaceClusterDto,
   PeopleResponseDto,
   PeopleUpdateDto,
   PersonCreateDto,
@@ -438,7 +438,7 @@ export class PersonService extends BaseService {
 
     const lastRun = new Date().toISOString();
     const facePagination = this.personRepository.getAllFaces(
-      force ? undefined : { personId: null, sourceType: SourceType.MachineLearning },
+      force ? undefined : { faceClusterId: null, sourceType: SourceType.MachineLearning },
     );
 
     let jobs: { name: JobName.FacialRecognition; data: { id: string; deferred: false } }[] = [];
@@ -481,8 +481,8 @@ export class PersonService extends BaseService {
       return JobStatus.Failed;
     }
 
-    if (face.personId) {
+    if (face.faceClusterId) {
-      this.logger.debug(`Face ${id} already has a person assigned`);
+      this.logger.debug(`Face ${id} already belongs to a face cluster`);
       return JobStatus.Skipped;
     }
 
@@ -511,8 +511,8 @@ export class PersonService extends BaseService {
       return JobStatus.Skipped;
     }
 
-    let personId = matches.find((match) => match.personId)?.personId;
+    let faceClusterId = matches.find((match) => match.faceClusterId)?.faceClusterId;
-    if (!personId) {
+    if (!faceClusterId) {
       const matchWithPerson = await this.searchRepository.searchFaces({
         userIds: [face.asset.ownerId],
         embedding: face.faceSearch.embedding,
@@ -523,20 +523,20 @@ export class PersonService extends BaseService {
       });
 
       if (matchWithPerson.length > 0) {
-        personId = matchWithPerson[0].personId;
+        faceClusterId = matchWithPerson[0].faceClusterId;
       }
     }
 
-    if (isCore && !personId) {
+    if (isCore && !faceClusterId) {
       this.logger.log(`Creating new person for face ${id}`);
       const newPerson = await this.personRepository.create({ ownerId: face.asset.ownerId, faceAssetId: face.id });
       await this.jobRepository.queue({ name: JobName.PersonGenerateThumbnail, data: { id: newPerson.id } });
-      personId = newPerson.id;
+      faceClusterId = newPerson.faceClusterId;
     }
 
-    if (personId) {
+    if (faceClusterId) {
-      this.logger.debug(`Assigning face ${id} to person ${personId}`);
+      this.logger.debug(`Assigning face ${id} to face cluster ${faceClusterId}`);
-      await this.personRepository.reassignFaces({ faceIds: [id], newPersonId: personId });
+      await this.personRepository.reassignFaces({ faceIds: [id], newFaceClusterId: faceClusterId });
     }
 
     return JobStatus.Success;
@@ -554,7 +554,7 @@ export class PersonService extends BaseService {
     return JobStatus.Success;
   }
 
-  async mergePerson(auth: AuthDto, id: string, dto: MergePersonDto): Promise<BulkIdResponseDto[]> {
+  async mergePerson(auth: AuthDto, id: string, dto: MergeFaceClusterDto): Promise<BulkIdResponseDto[]> {
     const mergeIds = dto.ids;
     if (mergeIds.includes(id)) {
       throw new BadRequestException('Cannot merge a person into themselves');
@@ -600,7 +600,7 @@ export class PersonService extends BaseService {
         }
 
         const mergeName = mergePerson.name || mergePerson.id;
-        const mergeData: UpdateFacesData = { oldPersonId: mergeId, newPersonId: id };
+        const mergeData: UpdateFacesData = { oldFaceClusterId: mergeId, newFaceClusterId: id };
         this.logger.log(`Merging ${mergeName} into ${primaryName}`);
 
         await this.personRepository.reassignFaces(mergeData);
@@ -678,8 +678,14 @@ export class PersonService extends BaseService {
       dto.imageHeight = originalDimensions.height;
     }
 
+    const person = await this.personRepository.getById(dto.personId);
+
+    if (!person?.faceClusterId) {
+      throw new Error('Person must already have some recognized faces and belong to a face cluster');
+    }
+
     await this.personRepository.createAssetFace({
-      personId: dto.personId,
+      faceClusterId: person.faceClusterId,
       assetId: dto.assetId,
       imageHeight: dto.imageHeight,
       imageWidth: dto.imageWidth,

server/src/services/sync.service.ts

@@ -13,8 +13,10 @@ import {
   SyncAckDeleteDto,
   SyncAckSetDto,
   syncAssetFaceV2ToV1,
+  syncAssetFaceV3ToV2,
   SyncAssetV1,
   SyncItem,
+  syncPersonV2ToV1,
   SyncStreamDto,
 } from 'src/dtos/sync.dto';
 import {
@@ -192,8 +194,11 @@ export class SyncService extends BaseService {
       [SyncRequestType.StacksV1]: () => this.syncStackV1(options, response, checkpointMap),
       [SyncRequestType.PartnerStacksV1]: () => this.syncPartnerStackV1(options, response, checkpointMap, session.id),
       [SyncRequestType.PeopleV1]: () => this.syncPeopleV1(options, response, checkpointMap),
+      [SyncRequestType.PeopleV2]: () => this.syncPeopleV2(options, response, checkpointMap),
       [SyncRequestType.AssetFacesV1]: async () => this.syncAssetFacesV1(options, response, checkpointMap),
       [SyncRequestType.AssetFacesV2]: async () => this.syncAssetFacesV2(options, response, checkpointMap),
+      [SyncRequestType.AssetFacesV3]: async () => this.syncAssetFacesV3(options, response, checkpointMap),
+      [SyncRequestType.FaceClusterV1]: async () => this.syncFaceClusterV1(options, response, checkpointMap),
       [SyncRequestType.UserMetadataV1]: () => this.syncUserMetadataV1(options, response, checkpointMap),
     };
 
@@ -796,6 +801,20 @@ export class SyncService extends BaseService {
 
     const upsertType = SyncEntityType.PersonV1;
     const upserts = this.syncRepository.person.getUpserts({ ...options, ack: checkpointMap[upsertType] });
+    for await (const { updateId, ...data } of upserts) {
+      send(response, { type: upsertType, ids: [updateId], data: syncPersonV2ToV1(data) });
+    }
+  }
+
+  private async syncPeopleV2(options: SyncQueryOptions, response: Writable, checkpointMap: CheckpointMap) {
+    const deleteType = SyncEntityType.PersonDeleteV1;
+    const deletes = this.syncRepository.person.getDeletes({ ...options, ack: checkpointMap[deleteType] });
+    for await (const { id, ...data } of deletes) {
+      send(response, { type: deleteType, ids: [id], data });
+    }
+
+    const upsertType = SyncEntityType.PersonV2;
+    const upserts = this.syncRepository.person.getUpserts({ ...options, ack: checkpointMap[upsertType] });
     for await (const { updateId, ...data } of upserts) {
       send(response, { type: upsertType, ids: [updateId], data });
     }
@@ -810,8 +829,8 @@ export class SyncService extends BaseService {
 
     const upsertType = SyncEntityType.AssetFaceV1;
     const upserts = this.syncRepository.assetFace.getUpserts({ ...options, ack: checkpointMap[upsertType] });
-    for await (const { updateId, ...data } of upserts) {
+    for await (const { updateId, personId, ...data } of upserts) {
-      const v1 = syncAssetFaceV2ToV1(data);
+      const v1 = syncAssetFaceV2ToV1(syncAssetFaceV3ToV2(data, personId));
       send(response, { type: upsertType, ids: [updateId], data: v1 });
     }
   }
@@ -825,6 +844,34 @@ export class SyncService extends BaseService {
 
     const upsertType = SyncEntityType.AssetFaceV2;
     const upserts = this.syncRepository.assetFace.getUpserts({ ...options, ack: checkpointMap[upsertType] });
+    for await (const { updateId, personId, ...data } of upserts) {
+      send(response, { type: upsertType, ids: [updateId], data: syncAssetFaceV3ToV2(data, personId) });
+    }
+  }
+
+  private async syncAssetFacesV3(options: SyncQueryOptions, response: Writable, checkpointMap: CheckpointMap) {
+    const deleteType = SyncEntityType.AssetFaceDeleteV1;
+    const deletes = this.syncRepository.assetFace.getDeletes({ ...options, ack: checkpointMap[deleteType] });
+    for await (const { id, ...data } of deletes) {
+      send(response, { type: deleteType, ids: [id], data });
+    }
+
+    const upsertType = SyncEntityType.AssetFaceV3;
+    const upserts = this.syncRepository.assetFace.getUpserts({ ...options, ack: checkpointMap[upsertType] });
+    for await (const { updateId, personId: _, ...data } of upserts) {
+      send(response, { type: upsertType, ids: [updateId], data });
+    }
+  }
+
+  private async syncFaceClusterV1(options: SyncQueryOptions, response: Writable, checkpointMap: CheckpointMap) {
+    const deleteType = SyncEntityType.FaceClusterDeleteV1;
+    const deletes = this.syncRepository.faceCluster.getDeletes({ ...options, ack: checkpointMap[deleteType] });
+    for await (const { id, ...data } of deletes) {
+      send(response, { type: deleteType, ids: [id], data });
+    }
+
+    const upsertType = SyncEntityType.FaceClusterV1;
+    const upserts = this.syncRepository.faceCluster.getUpserts({ ...options, ack: checkpointMap[upsertType] });
     for await (const { updateId, ...data } of upserts) {
       send(response, { type: upsertType, ids: [updateId], data });
     }

server/src/utils/database.ts

@@ -144,7 +144,12 @@ export function withFacesAndPeople(
       .selectFrom('asset_face')
       .leftJoinLateral(
         (eb) =>
-          eb.selectFrom('person').selectAll('person').whereRef('asset_face.personId', '=', 'person.id').as('person'),
+          eb
+            .selectFrom('face_cluster')
+            .where('face_cluster.id', '=', 'asset_face.faceClusterId')
+            .innerJoin('person', 'person.faceClusterId', 'face_cluster.id')
+            .selectAll('person')
+            .as('person'),
         (join) => join.onTrue(),
       )
       .selectAll('asset_face')
@@ -161,11 +166,12 @@ export function hasPeople<O>(qb: SelectQueryBuilder<DB, 'asset', O>, personIds:
       eb
         .selectFrom('asset_face')
         .select('assetId')
-        .where('personId', '=', anyUuid(personIds!))
+        .innerJoin('person', 'person.faceClusterId', 'asset_face.faceClusterId')
+        .where('person.id', '=', anyUuid(personIds!))
         .where('deletedAt', 'is', null)
         .where('isVisible', 'is', true)
         .groupBy('assetId')
-        .having((eb) => eb.fn.count('personId').distinct(), '=', personIds.length)
+        .having((eb) => eb.fn.count('person.id').distinct(), '=', personIds.length)
         .as('has_people'),
     (join) => join.onRef('has_people.assetId', '=', 'asset.id'),
   );