1
0
forked from Cutlery/immich

add web placeholder, server endpoint, migration, various fixes

This commit is contained in:
mertalev 2024-04-02 00:24:06 -04:00
parent a27c72a426
commit 7d9d9f9e97
No known key found for this signature in database
GPG Key ID: 9181CD92C0A1C5E3
24 changed files with 303 additions and 38 deletions

View File

@ -96,6 +96,7 @@ Class | Method | HTTP request | Description
*AssetApi* | [**deleteAssets**](doc//AssetApi.md#deleteassets) | **DELETE** /asset | *AssetApi* | [**deleteAssets**](doc//AssetApi.md#deleteassets) | **DELETE** /asset |
*AssetApi* | [**getAllAssets**](doc//AssetApi.md#getallassets) | **GET** /asset | *AssetApi* | [**getAllAssets**](doc//AssetApi.md#getallassets) | **GET** /asset |
*AssetApi* | [**getAllUserAssetsByDeviceId**](doc//AssetApi.md#getalluserassetsbydeviceid) | **GET** /asset/device/{deviceId} | *AssetApi* | [**getAllUserAssetsByDeviceId**](doc//AssetApi.md#getalluserassetsbydeviceid) | **GET** /asset/device/{deviceId} |
*AssetApi* | [**getAssetDuplicates**](doc//AssetApi.md#getassetduplicates) | **GET** /asset/duplicates |
*AssetApi* | [**getAssetInfo**](doc//AssetApi.md#getassetinfo) | **GET** /asset/{id} | *AssetApi* | [**getAssetInfo**](doc//AssetApi.md#getassetinfo) | **GET** /asset/{id} |
*AssetApi* | [**getAssetSearchTerms**](doc//AssetApi.md#getassetsearchterms) | **GET** /asset/search-terms | *AssetApi* | [**getAssetSearchTerms**](doc//AssetApi.md#getassetsearchterms) | **GET** /asset/search-terms |
*AssetApi* | [**getAssetStatistics**](doc//AssetApi.md#getassetstatistics) | **GET** /asset/statistics | *AssetApi* | [**getAssetStatistics**](doc//AssetApi.md#getassetstatistics) | **GET** /asset/statistics |

View File

@ -14,6 +14,7 @@ Method | HTTP request | Description
[**deleteAssets**](AssetApi.md#deleteassets) | **DELETE** /asset | [**deleteAssets**](AssetApi.md#deleteassets) | **DELETE** /asset |
[**getAllAssets**](AssetApi.md#getallassets) | **GET** /asset | [**getAllAssets**](AssetApi.md#getallassets) | **GET** /asset |
[**getAllUserAssetsByDeviceId**](AssetApi.md#getalluserassetsbydeviceid) | **GET** /asset/device/{deviceId} | [**getAllUserAssetsByDeviceId**](AssetApi.md#getalluserassetsbydeviceid) | **GET** /asset/device/{deviceId} |
[**getAssetDuplicates**](AssetApi.md#getassetduplicates) | **GET** /asset/duplicates |
[**getAssetInfo**](AssetApi.md#getassetinfo) | **GET** /asset/{id} | [**getAssetInfo**](AssetApi.md#getassetinfo) | **GET** /asset/{id} |
[**getAssetSearchTerms**](AssetApi.md#getassetsearchterms) | **GET** /asset/search-terms | [**getAssetSearchTerms**](AssetApi.md#getassetsearchterms) | **GET** /asset/search-terms |
[**getAssetStatistics**](AssetApi.md#getassetstatistics) | **GET** /asset/statistics | [**getAssetStatistics**](AssetApi.md#getassetstatistics) | **GET** /asset/statistics |
@ -328,6 +329,57 @@ Name | Type | Description | Notes
[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md) [[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
# **getAssetDuplicates**
> List<AssetResponseDto> getAssetDuplicates()
### Example
```dart
import 'package:openapi/api.dart';
// TODO Configure API key authorization: cookie
//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKey = 'YOUR_API_KEY';
// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKeyPrefix = 'Bearer';
// TODO Configure API key authorization: api_key
//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKey = 'YOUR_API_KEY';
// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKeyPrefix = 'Bearer';
// TODO Configure HTTP Bearer authorization: bearer
// Case 1. Use String Token
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken('YOUR_ACCESS_TOKEN');
// Case 2. Use Function which generate token.
// String yourTokenGeneratorFunction() { ... }
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken(yourTokenGeneratorFunction);
final api_instance = AssetApi();
try {
final result = api_instance.getAssetDuplicates();
print(result);
} catch (e) {
print('Exception when calling AssetApi->getAssetDuplicates: $e\n');
}
```
### Parameters
This endpoint does not need any parameter.
### Return type
[**List<AssetResponseDto>**](AssetResponseDto.md)
### Authorization
[cookie](../README.md#cookie), [api_key](../README.md#api_key), [bearer](../README.md#bearer)
### HTTP request headers
- **Content-Type**: Not defined
- **Accept**: application/json
[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
# **getAssetInfo** # **getAssetInfo**
> AssetResponseDto getAssetInfo(id, key) > AssetResponseDto getAssetInfo(id, key)

View File

@ -326,6 +326,50 @@ class AssetApi {
return null; return null;
} }
/// Performs an HTTP 'GET /asset/duplicates' operation and returns the [Response].
Future<Response> getAssetDuplicatesWithHttpInfo() async {
// ignore: prefer_const_declarations
final path = r'/asset/duplicates';
// ignore: prefer_final_locals
Object? postBody;
final queryParams = <QueryParam>[];
final headerParams = <String, String>{};
final formParams = <String, String>{};
const contentTypes = <String>[];
return apiClient.invokeAPI(
path,
'GET',
queryParams,
postBody,
headerParams,
formParams,
contentTypes.isEmpty ? null : contentTypes.first,
);
}
Future<List<AssetResponseDto>?> getAssetDuplicates() async {
final response = await getAssetDuplicatesWithHttpInfo();
if (response.statusCode >= HttpStatus.badRequest) {
throw ApiException(response.statusCode, await _decodeBodyBytes(response));
}
// When a remote server returns no body with a status of 204, we shall not decode it.
// At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
// FormatException when trying to decode an empty string.
if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
final responseBody = await _decodeBodyBytes(response);
return (await apiClient.deserializeAsync(responseBody, 'List<AssetResponseDto>') as List)
.cast<AssetResponseDto>()
.toList(growable: false);
}
return null;
}
/// Performs an HTTP 'GET /asset/{id}' operation and returns the [Response]. /// Performs an HTTP 'GET /asset/{id}' operation and returns the [Response].
/// Parameters: /// Parameters:
/// ///

View File

@ -50,6 +50,11 @@ void main() {
// TODO // TODO
}); });
//Future<List<AssetResponseDto>> getAssetDuplicates() async
test('test getAssetDuplicates', () async {
// TODO
});
//Future<AssetResponseDto> getAssetInfo(String id, { String key }) async //Future<AssetResponseDto> getAssetInfo(String id, { String key }) async
test('test getAssetInfo', () async { test('test getAssetInfo', () async {
// TODO // TODO

View File

@ -1213,6 +1213,41 @@
] ]
} }
}, },
"/asset/duplicates": {
"get": {
"operationId": "getAssetDuplicates",
"parameters": [],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"items": {
"$ref": "#/components/schemas/AssetResponseDto"
},
"type": "array"
}
}
},
"description": ""
}
},
"security": [
{
"bearer": []
},
{
"cookie": []
},
{
"api_key": []
}
],
"tags": [
"Asset"
]
}
},
"/asset/exist": { "/asset/exist": {
"post": { "post": {
"description": "Checks if multiple assets exist on the server and returns all existing - used by background backup", "description": "Checks if multiple assets exist on the server and returns all existing - used by background backup",

View File

@ -1323,6 +1323,14 @@ export function getAllUserAssetsByDeviceId({ deviceId }: {
...opts ...opts
})); }));
} }
export function getAssetDuplicates(opts?: Oazapfts.RequestOpts) {
return oazapfts.ok(oazapfts.fetchJson<{
status: 200;
data: AssetResponseDto[];
}>("/asset/duplicates", {
...opts
}));
}
/** /**
* Checks if multiple assets exist on the server and returns all existing - used by background backup * Checks if multiple assets exist on the server and returns all existing - used by background backup
*/ */

View File

@ -70,6 +70,11 @@ export class AssetController {
return this.service.getStatistics(auth, dto); return this.service.getStatistics(auth, dto);
} }
@Get('duplicates')
getAssetDuplicates(@Auth() auth: AuthDto): Promise<AssetResponseDto[]> {
return this.service.getDuplicates(auth);
}
@Post('jobs') @Post('jobs')
@HttpCode(HttpStatus.NO_CONTENT) @HttpCode(HttpStatus.NO_CONTENT)
runAssetJobs(@Auth() auth: AuthDto, @Body() dto: AssetJobsDto): Promise<void> { runAssetJobs(@Auth() auth: AuthDto, @Body() dto: AssetJobsDto): Promise<void> {

View File

@ -71,7 +71,7 @@ export const defaults = Object.freeze<SystemConfig>({
clip: { clip: {
enabled: true, enabled: true,
modelName: 'ViT-B-32__openai', modelName: 'ViT-B-32__openai',
duplicateThreshold: 0.01, duplicateThreshold: 0.03,
}, },
facialRecognition: { facialRecognition: {
enabled: true, enabled: true,

View File

@ -4,7 +4,7 @@ import { Entity, Index, JoinColumn, OneToMany, PrimaryColumn } from 'typeorm';
@Entity('asset_duplicates') @Entity('asset_duplicates')
@Index('asset_duplicates_assetId_uindex', ['assetId'], { unique: true }) @Index('asset_duplicates_assetId_uindex', ['assetId'], { unique: true })
export class AssetDuplicateEntity { export class AssetDuplicateEntity {
@OneToMany(() => AssetEntity, (asset) => asset.duplicates) @OneToMany(() => AssetEntity, (asset) => asset.duplicates, { onDelete: 'CASCADE', onUpdate: 'CASCADE' })
@JoinColumn({ name: 'assetId', referencedColumnName: 'id' }) @JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
assets!: AssetEntity; assets!: AssetEntity;

View File

@ -1,4 +1,5 @@
import { AlbumEntity } from 'src/entities/album.entity'; import { AlbumEntity } from 'src/entities/album.entity';
import { AssetDuplicateEntity } from 'src/entities/asset-duplicate.entity';
import { AssetFaceEntity } from 'src/entities/asset-face.entity'; import { AssetFaceEntity } from 'src/entities/asset-face.entity';
import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity'; import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
import { AssetStackEntity } from 'src/entities/asset-stack.entity'; import { AssetStackEntity } from 'src/entities/asset-stack.entity';
@ -24,7 +25,6 @@ import {
PrimaryGeneratedColumn, PrimaryGeneratedColumn,
UpdateDateColumn, UpdateDateColumn,
} from 'typeorm'; } from 'typeorm';
import { AssetDuplicateEntity } from './asset-duplicate.entity';
export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_library_checksum'; export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_library_checksum';
@ -173,7 +173,7 @@ export class AssetEntity {
@Column({ nullable: true }) @Column({ nullable: true })
duplicateId?: string | null; duplicateId?: string | null;
@ManyToOne(() => AssetDuplicateEntity, { nullable: true, onDelete: 'SET NULL', onUpdate: 'CASCADE' }) @ManyToOne(() => AssetDuplicateEntity, { nullable: true })
@JoinColumn({ name: 'duplicateId' }) @JoinColumn({ name: 'duplicateId' })
duplicates?: AssetDuplicateEntity | null; duplicates?: AssetDuplicateEntity | null;
} }

View File

@ -61,6 +61,7 @@ export interface AssetBuilderOptions {
isArchived?: boolean; isArchived?: boolean;
isFavorite?: boolean; isFavorite?: boolean;
isTrashed?: boolean; isTrashed?: boolean;
isDuplicate?: boolean;
albumId?: string; albumId?: string;
personId?: string; personId?: string;
userIds?: string[]; userIds?: string[];
@ -172,8 +173,9 @@ export interface IAssetRepository {
getTimeBuckets(options: TimeBucketOptions): Promise<TimeBucketItem[]>; getTimeBuckets(options: TimeBucketOptions): Promise<TimeBucketItem[]>;
getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise<AssetEntity[]>; getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise<AssetEntity[]>;
upsertExif(exif: Partial<ExifEntity>): Promise<void>; upsertExif(exif: Partial<ExifEntity>): Promise<void>;
upsertJobStatus(jobStatus: Partial<AssetJobStatusEntity>): Promise<void>; upsertJobStatus(jobStatus: Partial<AssetJobStatusEntity> | Partial<AssetJobStatusEntity>[]): Promise<void>;
getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>; getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>; getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
getDuplicates(options: AssetBuilderOptions): Promise<AssetEntity[]>;
searchMetadata(query: string, userIds: string[], options: MetadataSearchOptions): Promise<AssetEntity[]>; searchMetadata(query: string, userIds: string[], options: MetadataSearchOptions): Promise<AssetEntity[]>;
} }

View File

@ -179,6 +179,7 @@ export interface FaceEmbeddingSearch extends SearchEmbeddingOptions {
export interface AssetDuplicateSearch { export interface AssetDuplicateSearch {
assetId: string; assetId: string;
embedding: Embedding;
userIds: string[]; userIds: string[];
maxDistance?: number; maxDistance?: number;
} }

View File

@ -0,0 +1,36 @@
import { MigrationInterface, QueryRunner } from 'typeorm';
export class CreateAssetDuplicateTable1711989989911 implements MigrationInterface {
public async up(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`
CREATE TABLE asset_duplicates (
id uuid,
"assetId" uuid REFERENCES assets ON UPDATE CASCADE ON DELETE CASCADE,
PRIMARY KEY (id, "assetId")
);
`);
await queryRunner.query(`ALTER TABLE assets ADD COLUMN "duplicateId" uuid`);
await queryRunner.query(`ALTER TABLE asset_job_status ADD COLUMN "duplicatesDetectedAt" timestamptz`);
await queryRunner.query(`
ALTER TABLE assets
ADD CONSTRAINT asset_duplicates_id
FOREIGN KEY ("duplicateId", id)
REFERENCES asset_duplicates DEFERRABLE INITIALLY DEFERRED
`);
await queryRunner.query(`
CREATE UNIQUE INDEX "asset_duplicates_assetId_uindex"
ON asset_duplicates ("assetId")
`);
await queryRunner.query(`CREATE INDEX "IDX_assets_duplicateId" ON assets ("duplicateId")`);
}
public async down(queryRunner: QueryRunner): Promise<void> {
await queryRunner.query(`ALTER TABLE assets DROP COLUMN "duplicateId"`);
await queryRunner.query(`DROP TABLE asset_duplicates`);
}
}

View File

@ -22,7 +22,7 @@ export class AssetDuplicateRepository implements IAssetDuplicateRepository {
await manager.update(AssetDuplicateEntity, { id: In(oldDuplicateIds) }, { id }); await manager.update(AssetDuplicateEntity, { id: In(oldDuplicateIds) }, { id });
} }
await manager.update(AssetEntity, { id: In(assetIds) }, { duplicateId: id }); await manager.update(AssetEntity, { id: In(assetIds) }, { duplicateId: id });
await manager.update(AssetEntity, { duplicateId: In(oldDuplicateIds) }, { duplicateId: id }); // TODO: cascade should handle this, but it doesn't seem to await manager.update(AssetEntity, { duplicateId: In(oldDuplicateIds) }, { duplicateId: id });
}); });
} }

View File

@ -67,7 +67,7 @@ export class AssetRepository implements IAssetRepository {
await this.exifRepository.upsert(exif, { conflictPaths: ['assetId'] }); await this.exifRepository.upsert(exif, { conflictPaths: ['assetId'] });
} }
async upsertJobStatus(jobStatus: Partial<AssetJobStatusEntity>): Promise<void> { async upsertJobStatus(jobStatus: Partial<AssetJobStatusEntity> | Partial<AssetJobStatusEntity>[]): Promise<void> {
await this.jobStatusRepository.upsert(jobStatus, { conflictPaths: ['assetId'] }); await this.jobStatusRepository.upsert(jobStatus, { conflictPaths: ['assetId'] });
} }
@ -201,6 +201,7 @@ export class AssetRepository implements IAssetRepository {
let builder = this.repository.createQueryBuilder('asset'); let builder = this.repository.createQueryBuilder('asset');
builder = searchAssetBuilder(builder, options); builder = searchAssetBuilder(builder, options);
builder.orderBy('asset.createdAt', options.orderDirection ?? 'ASC'); builder.orderBy('asset.createdAt', options.orderDirection ?? 'ASC');
builder.innerJoin('asset.smartSearch', 'smartSearch');
return paginatedBuilder<AssetEntity>(builder, { return paginatedBuilder<AssetEntity>(builder, {
mode: PaginationMode.SKIP_TAKE, mode: PaginationMode.SKIP_TAKE,
skip: pagination.skip, skip: pagination.skip,
@ -591,6 +592,13 @@ export class AssetRepository implements IAssetRepository {
); );
} }
@GenerateSql({ params: [{ userIds: [DummyValue.UUID, DummyValue.UUID] }] })
getDuplicates(options: AssetBuilderOptions): Promise<AssetEntity[]> {
return this.getBuilder({ ...options, isDuplicate: true })
.orderBy('asset.duplicateId')
.getMany();
}
@GenerateSql({ params: [DummyValue.UUID, { minAssetsPerField: 5, maxFields: 12 }] }) @GenerateSql({ params: [DummyValue.UUID, { minAssetsPerField: 5, maxFields: 12 }] })
async getAssetIdByCity( async getAssetIdByCity(
ownerId: string, ownerId: string,
@ -650,16 +658,14 @@ export class AssetRepository implements IAssetRepository {
} }
private getBuilder(options: AssetBuilderOptions) { private getBuilder(options: AssetBuilderOptions) {
const { isArchived, isFavorite, isTrashed, albumId, personId, userIds, withStacked, exifInfo, assetType } = options;
let builder = this.repository.createQueryBuilder('asset').where('asset.isVisible = true'); let builder = this.repository.createQueryBuilder('asset').where('asset.isVisible = true');
if (assetType !== undefined) { if (options.assetType !== undefined) {
builder = builder.andWhere('asset.type = :assetType', { assetType }); builder = builder.andWhere('asset.type = :assetType', { assetType: options.assetType });
} }
let stackJoined = false; let stackJoined = false;
if (exifInfo !== false) { if (options.exifInfo !== false) {
stackJoined = true; stackJoined = true;
builder = builder builder = builder
.leftJoinAndSelect('asset.exifInfo', 'exifInfo') .leftJoinAndSelect('asset.exifInfo', 'exifInfo')
@ -667,34 +673,38 @@ export class AssetRepository implements IAssetRepository {
.leftJoinAndSelect('stack.assets', 'stackedAssets'); .leftJoinAndSelect('stack.assets', 'stackedAssets');
} }
if (albumId) { if (options.albumId) {
builder = builder.leftJoin('asset.albums', 'album').andWhere('album.id = :albumId', { albumId }); builder = builder.leftJoin('asset.albums', 'album').andWhere('album.id = :albumId', { albumId: options.albumId });
} }
if (userIds) { if (options.userIds) {
builder = builder.andWhere('asset.ownerId IN (:...userIds )', { userIds }); builder = builder.andWhere('asset.ownerId IN (:...userIds )', { userIds: options.userIds });
} }
if (isArchived !== undefined) { if (options.isArchived !== undefined) {
builder = builder.andWhere('asset.isArchived = :isArchived', { isArchived }); builder = builder.andWhere('asset.isArchived = :isArchived', { isArchived: options.isArchived });
} }
if (isFavorite !== undefined) { if (options.isFavorite !== undefined) {
builder = builder.andWhere('asset.isFavorite = :isFavorite', { isFavorite }); builder = builder.andWhere('asset.isFavorite = :isFavorite', { isFavorite: options.isFavorite });
} }
if (isTrashed !== undefined) { if (options.isTrashed !== undefined) {
builder = builder.andWhere(`asset.deletedAt ${isTrashed ? 'IS NOT NULL' : 'IS NULL'}`).withDeleted(); builder = builder.andWhere(`asset.deletedAt ${options.isTrashed ? 'IS NOT NULL' : 'IS NULL'}`).withDeleted();
} }
if (personId !== undefined) { if (options.isDuplicate !== undefined) {
builder = builder.andWhere(`asset.duplicateId ${options.isDuplicate ? 'IS NOT NULL' : 'IS NULL'}`);
}
if (options.personId !== undefined) {
builder = builder builder = builder
.innerJoin('asset.faces', 'faces') .innerJoin('asset.faces', 'faces')
.innerJoin('faces.person', 'person') .innerJoin('faces.person', 'person')
.andWhere('person.id = :personId', { personId }); .andWhere('person.id = :personId', { personId: options.personId });
} }
if (withStacked) { if (options.withStacked) {
if (!stackJoined) { if (!stackJoined) {
builder = builder.leftJoinAndSelect('asset.stack', 'stack').leftJoinAndSelect('stack.assets', 'stackedAssets'); builder = builder.leftJoinAndSelect('asset.stack', 'stack').leftJoinAndSelect('stack.assets', 'stackedAssets');
} }

View File

@ -248,6 +248,11 @@ export class AssetService {
return data; return data;
} }
async getDuplicates(auth: AuthDto): Promise<AssetResponseDto[]> {
const res = await this.assetRepository.getDuplicates({ userIds: [auth.user.id] });
return res.map((a) => mapAsset(a, { auth }));
}
async update(auth: AuthDto, id: string, dto: UpdateAssetDto): Promise<AssetResponseDto> { async update(auth: AuthDto, id: string, dto: UpdateAssetDto): Promise<AssetResponseDto> {
await this.access.requirePermission(auth, Permission.ASSET_UPDATE, id); await this.access.requirePermission(auth, Permission.ASSET_UPDATE, id);

View File

@ -179,7 +179,11 @@ export class SearchService {
async handleSearchDuplicates({ id }: IEntityJob): Promise<JobStatus> { async handleSearchDuplicates({ id }: IEntityJob): Promise<JobStatus> {
const { machineLearning } = await this.configCore.getConfig(); const { machineLearning } = await this.configCore.getConfig();
const asset = await this.assetRepository.getById(id); const [asset] = await this.assetRepository.getByIds(
[id],
{ smartSearch: true },
{ smartSearch: { assetId: true, embedding: true } },
);
if (!asset) { if (!asset) {
this.logger.error(`Asset ${id} not found`); this.logger.error(`Asset ${id} not found`);
return JobStatus.FAILED; return JobStatus.FAILED;
@ -187,10 +191,6 @@ export class SearchService {
if (!asset.isVisible) { if (!asset.isVisible) {
this.logger.debug(`Asset ${id} is not visible, skipping`); this.logger.debug(`Asset ${id} is not visible, skipping`);
await this.assetRepository.upsertJobStatus({
assetId: asset.id,
duplicatesDetectedAt: new Date(),
});
return JobStatus.SKIPPED; return JobStatus.SKIPPED;
} }
@ -204,28 +204,35 @@ export class SearchService {
return JobStatus.FAILED; return JobStatus.FAILED;
} }
if (!asset.smartSearch?.embedding) {
this.logger.debug(`Asset ${id} is missing embedding`);
return JobStatus.FAILED;
}
const duplicateAssets = await this.searchRepository.searchDuplicates({ const duplicateAssets = await this.searchRepository.searchDuplicates({
assetId: asset.id, assetId: asset.id,
embedding: asset.smartSearch.embedding,
maxDistance: machineLearning.clip.duplicateThreshold, maxDistance: machineLearning.clip.duplicateThreshold,
userIds: [asset.ownerId], userIds: [asset.ownerId],
}); });
const duplicateAssetIds = [asset.id];
if (duplicateAssets.length > 0) { if (duplicateAssets.length > 0) {
this.logger.debug(`Found ${duplicateAssets.length} duplicates for asset ${asset.id}`); this.logger.debug(
`Found ${duplicateAssets.length} duplicate${duplicateAssets.length === 1 ? '' : 's'} for asset ${asset.id}`,
);
const duplicateIds = duplicateAssets.map((duplicate) => duplicate.duplicateId).filter(Boolean); const duplicateIds = duplicateAssets.map((duplicate) => duplicate.duplicateId).filter(Boolean);
const duplicateId = duplicateIds[0] || this.cryptoRepository.randomUUID(); const duplicateId = duplicateIds[0] || this.cryptoRepository.randomUUID();
const duplicateAssetIds = duplicateAssets.map((duplicate) => duplicate.assetId); duplicateAssetIds.push(...duplicateAssets.map((duplicate) => duplicate.assetId));
duplicateAssetIds.push(asset.id);
await this.assetDuplicateRepository.upsert(duplicateId, duplicateAssetIds, duplicateIds); await this.assetDuplicateRepository.upsert(duplicateId, duplicateAssetIds, duplicateIds);
} }
await this.assetRepository.upsertJobStatus({ const duplicatesDetectedAt = new Date();
assetId: asset.id, await this.assetRepository.upsertJobStatus(duplicateAssetIds.map((assetId) => ({ assetId, duplicatesDetectedAt })));
duplicatesDetectedAt: new Date(),
});
return JobStatus.SUCCESS; return JobStatus.SUCCESS;
} }

View File

@ -74,7 +74,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
clip: { clip: {
enabled: true, enabled: true,
modelName: 'ViT-B-32__openai', modelName: 'ViT-B-32__openai',
duplicateThreshold: 0.01, duplicateThreshold: 0.03,
}, },
facialRecognition: { facialRecognition: {
enabled: true, enabled: true,

View File

@ -131,6 +131,19 @@
</svelte:fragment> </svelte:fragment>
</SideBarLink> </SideBarLink>
<SideBarLink title="Duplicates" routeId="/(user)/duplicates" icon={mdiArchiveArrowDownOutline}>
<svelte:fragment slot="moreInformation">
{#await getStats({ isArchived: true })}
<LoadingSpinner />
{:then data}
<div>
<p>{data.videos.toLocaleString($locale)} Videos</p>
<p>{data.images.toLocaleString($locale)} Photos</p>
</div>
{/await}
</svelte:fragment>
</SideBarLink>
{#if $featureFlags.trash} {#if $featureFlags.trash}
<SideBarLink title="Trash" routeId="/(user)/trash" icon={mdiTrashCanOutline}> <SideBarLink title="Trash" routeId="/(user)/trash" icon={mdiTrashCanOutline}>
<svelte:fragment slot="moreInformation"> <svelte:fragment slot="moreInformation">

View File

@ -20,6 +20,7 @@ export enum AppRoute {
ALBUMS = '/albums', ALBUMS = '/albums',
LIBRARIES = '/libraries', LIBRARIES = '/libraries',
ARCHIVE = '/archive', ARCHIVE = '/archive',
DUPLICATES = '/duplicates',
FAVORITES = '/favorites', FAVORITES = '/favorites',
PEOPLE = '/people', PEOPLE = '/people',
PLACES = '/places', PLACES = '/places',

View File

@ -0,0 +1,21 @@
<script lang="ts">
import GalleryViewer from '$lib/components/shared-components/gallery-viewer/gallery-viewer.svelte';
import { getAssetDuplicates, type AssetResponseDto } from '@immich/sdk';
import type { Viewport } from '$lib/stores/assets.store';
import { onMount } from 'svelte';
let assets: AssetResponseDto[] = [];
const viewport: Viewport = { width: 0, height: 0 };
onMount(async () => {
assets = await getAssetDuplicates();
});
</script>
<section
class="relative mb-12 bg-immich-bg dark:bg-immich-dark-bg m-4"
bind:clientHeight={viewport.height}
bind:clientWidth={viewport.width}
>
<GalleryViewer {assets} {viewport}></GalleryViewer>
</section>

View File

@ -0,0 +1,12 @@
import { authenticate } from '$lib/utils/auth';
import type { PageLoad } from './$types';
export const load = (async () => {
await authenticate();
return {
meta: {
title: 'Duplicates',
},
};
}) satisfies PageLoad;

View File

@ -0,0 +1,7 @@
import { AppRoute } from '$lib/constants';
import { redirect } from '@sveltejs/kit';
import type { PageLoad } from './$types';
export const load: PageLoad = () => {
redirect(302, AppRoute.DUPLICATES);
};