forked from Cutlery/immich
add web placeholder, server endpoint, migration, various fixes
This commit is contained in:
parent
a27c72a426
commit
7d9d9f9e97
1
mobile/openapi/README.md
generated
1
mobile/openapi/README.md
generated
@ -96,6 +96,7 @@ Class | Method | HTTP request | Description
|
|||||||
*AssetApi* | [**deleteAssets**](doc//AssetApi.md#deleteassets) | **DELETE** /asset |
|
*AssetApi* | [**deleteAssets**](doc//AssetApi.md#deleteassets) | **DELETE** /asset |
|
||||||
*AssetApi* | [**getAllAssets**](doc//AssetApi.md#getallassets) | **GET** /asset |
|
*AssetApi* | [**getAllAssets**](doc//AssetApi.md#getallassets) | **GET** /asset |
|
||||||
*AssetApi* | [**getAllUserAssetsByDeviceId**](doc//AssetApi.md#getalluserassetsbydeviceid) | **GET** /asset/device/{deviceId} |
|
*AssetApi* | [**getAllUserAssetsByDeviceId**](doc//AssetApi.md#getalluserassetsbydeviceid) | **GET** /asset/device/{deviceId} |
|
||||||
|
*AssetApi* | [**getAssetDuplicates**](doc//AssetApi.md#getassetduplicates) | **GET** /asset/duplicates |
|
||||||
*AssetApi* | [**getAssetInfo**](doc//AssetApi.md#getassetinfo) | **GET** /asset/{id} |
|
*AssetApi* | [**getAssetInfo**](doc//AssetApi.md#getassetinfo) | **GET** /asset/{id} |
|
||||||
*AssetApi* | [**getAssetSearchTerms**](doc//AssetApi.md#getassetsearchterms) | **GET** /asset/search-terms |
|
*AssetApi* | [**getAssetSearchTerms**](doc//AssetApi.md#getassetsearchterms) | **GET** /asset/search-terms |
|
||||||
*AssetApi* | [**getAssetStatistics**](doc//AssetApi.md#getassetstatistics) | **GET** /asset/statistics |
|
*AssetApi* | [**getAssetStatistics**](doc//AssetApi.md#getassetstatistics) | **GET** /asset/statistics |
|
||||||
|
52
mobile/openapi/doc/AssetApi.md
generated
52
mobile/openapi/doc/AssetApi.md
generated
@ -14,6 +14,7 @@ Method | HTTP request | Description
|
|||||||
[**deleteAssets**](AssetApi.md#deleteassets) | **DELETE** /asset |
|
[**deleteAssets**](AssetApi.md#deleteassets) | **DELETE** /asset |
|
||||||
[**getAllAssets**](AssetApi.md#getallassets) | **GET** /asset |
|
[**getAllAssets**](AssetApi.md#getallassets) | **GET** /asset |
|
||||||
[**getAllUserAssetsByDeviceId**](AssetApi.md#getalluserassetsbydeviceid) | **GET** /asset/device/{deviceId} |
|
[**getAllUserAssetsByDeviceId**](AssetApi.md#getalluserassetsbydeviceid) | **GET** /asset/device/{deviceId} |
|
||||||
|
[**getAssetDuplicates**](AssetApi.md#getassetduplicates) | **GET** /asset/duplicates |
|
||||||
[**getAssetInfo**](AssetApi.md#getassetinfo) | **GET** /asset/{id} |
|
[**getAssetInfo**](AssetApi.md#getassetinfo) | **GET** /asset/{id} |
|
||||||
[**getAssetSearchTerms**](AssetApi.md#getassetsearchterms) | **GET** /asset/search-terms |
|
[**getAssetSearchTerms**](AssetApi.md#getassetsearchterms) | **GET** /asset/search-terms |
|
||||||
[**getAssetStatistics**](AssetApi.md#getassetstatistics) | **GET** /asset/statistics |
|
[**getAssetStatistics**](AssetApi.md#getassetstatistics) | **GET** /asset/statistics |
|
||||||
@ -328,6 +329,57 @@ Name | Type | Description | Notes
|
|||||||
|
|
||||||
[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
|
[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
|
||||||
|
|
||||||
|
# **getAssetDuplicates**
|
||||||
|
> List<AssetResponseDto> getAssetDuplicates()
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
### Example
|
||||||
|
```dart
|
||||||
|
import 'package:openapi/api.dart';
|
||||||
|
// TODO Configure API key authorization: cookie
|
||||||
|
//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKey = 'YOUR_API_KEY';
|
||||||
|
// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
|
||||||
|
//defaultApiClient.getAuthentication<ApiKeyAuth>('cookie').apiKeyPrefix = 'Bearer';
|
||||||
|
// TODO Configure API key authorization: api_key
|
||||||
|
//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKey = 'YOUR_API_KEY';
|
||||||
|
// uncomment below to setup prefix (e.g. Bearer) for API key, if needed
|
||||||
|
//defaultApiClient.getAuthentication<ApiKeyAuth>('api_key').apiKeyPrefix = 'Bearer';
|
||||||
|
// TODO Configure HTTP Bearer authorization: bearer
|
||||||
|
// Case 1. Use String Token
|
||||||
|
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken('YOUR_ACCESS_TOKEN');
|
||||||
|
// Case 2. Use Function which generate token.
|
||||||
|
// String yourTokenGeneratorFunction() { ... }
|
||||||
|
//defaultApiClient.getAuthentication<HttpBearerAuth>('bearer').setAccessToken(yourTokenGeneratorFunction);
|
||||||
|
|
||||||
|
final api_instance = AssetApi();
|
||||||
|
|
||||||
|
try {
|
||||||
|
final result = api_instance.getAssetDuplicates();
|
||||||
|
print(result);
|
||||||
|
} catch (e) {
|
||||||
|
print('Exception when calling AssetApi->getAssetDuplicates: $e\n');
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Parameters
|
||||||
|
This endpoint does not need any parameter.
|
||||||
|
|
||||||
|
### Return type
|
||||||
|
|
||||||
|
[**List<AssetResponseDto>**](AssetResponseDto.md)
|
||||||
|
|
||||||
|
### Authorization
|
||||||
|
|
||||||
|
[cookie](../README.md#cookie), [api_key](../README.md#api_key), [bearer](../README.md#bearer)
|
||||||
|
|
||||||
|
### HTTP request headers
|
||||||
|
|
||||||
|
- **Content-Type**: Not defined
|
||||||
|
- **Accept**: application/json
|
||||||
|
|
||||||
|
[[Back to top]](#) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to Model list]](../README.md#documentation-for-models) [[Back to README]](../README.md)
|
||||||
|
|
||||||
# **getAssetInfo**
|
# **getAssetInfo**
|
||||||
> AssetResponseDto getAssetInfo(id, key)
|
> AssetResponseDto getAssetInfo(id, key)
|
||||||
|
|
||||||
|
44
mobile/openapi/lib/api/asset_api.dart
generated
44
mobile/openapi/lib/api/asset_api.dart
generated
@ -326,6 +326,50 @@ class AssetApi {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Performs an HTTP 'GET /asset/duplicates' operation and returns the [Response].
|
||||||
|
Future<Response> getAssetDuplicatesWithHttpInfo() async {
|
||||||
|
// ignore: prefer_const_declarations
|
||||||
|
final path = r'/asset/duplicates';
|
||||||
|
|
||||||
|
// ignore: prefer_final_locals
|
||||||
|
Object? postBody;
|
||||||
|
|
||||||
|
final queryParams = <QueryParam>[];
|
||||||
|
final headerParams = <String, String>{};
|
||||||
|
final formParams = <String, String>{};
|
||||||
|
|
||||||
|
const contentTypes = <String>[];
|
||||||
|
|
||||||
|
|
||||||
|
return apiClient.invokeAPI(
|
||||||
|
path,
|
||||||
|
'GET',
|
||||||
|
queryParams,
|
||||||
|
postBody,
|
||||||
|
headerParams,
|
||||||
|
formParams,
|
||||||
|
contentTypes.isEmpty ? null : contentTypes.first,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
Future<List<AssetResponseDto>?> getAssetDuplicates() async {
|
||||||
|
final response = await getAssetDuplicatesWithHttpInfo();
|
||||||
|
if (response.statusCode >= HttpStatus.badRequest) {
|
||||||
|
throw ApiException(response.statusCode, await _decodeBodyBytes(response));
|
||||||
|
}
|
||||||
|
// When a remote server returns no body with a status of 204, we shall not decode it.
|
||||||
|
// At the time of writing this, `dart:convert` will throw an "Unexpected end of input"
|
||||||
|
// FormatException when trying to decode an empty string.
|
||||||
|
if (response.body.isNotEmpty && response.statusCode != HttpStatus.noContent) {
|
||||||
|
final responseBody = await _decodeBodyBytes(response);
|
||||||
|
return (await apiClient.deserializeAsync(responseBody, 'List<AssetResponseDto>') as List)
|
||||||
|
.cast<AssetResponseDto>()
|
||||||
|
.toList(growable: false);
|
||||||
|
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
/// Performs an HTTP 'GET /asset/{id}' operation and returns the [Response].
|
/// Performs an HTTP 'GET /asset/{id}' operation and returns the [Response].
|
||||||
/// Parameters:
|
/// Parameters:
|
||||||
///
|
///
|
||||||
|
5
mobile/openapi/test/asset_api_test.dart
generated
5
mobile/openapi/test/asset_api_test.dart
generated
@ -50,6 +50,11 @@ void main() {
|
|||||||
// TODO
|
// TODO
|
||||||
});
|
});
|
||||||
|
|
||||||
|
//Future<List<AssetResponseDto>> getAssetDuplicates() async
|
||||||
|
test('test getAssetDuplicates', () async {
|
||||||
|
// TODO
|
||||||
|
});
|
||||||
|
|
||||||
//Future<AssetResponseDto> getAssetInfo(String id, { String key }) async
|
//Future<AssetResponseDto> getAssetInfo(String id, { String key }) async
|
||||||
test('test getAssetInfo', () async {
|
test('test getAssetInfo', () async {
|
||||||
// TODO
|
// TODO
|
||||||
|
@ -1213,6 +1213,41 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/asset/duplicates": {
|
||||||
|
"get": {
|
||||||
|
"operationId": "getAssetDuplicates",
|
||||||
|
"parameters": [],
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/components/schemas/AssetResponseDto"
|
||||||
|
},
|
||||||
|
"type": "array"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": ""
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"bearer": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"cookie": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"api_key": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"tags": [
|
||||||
|
"Asset"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
"/asset/exist": {
|
"/asset/exist": {
|
||||||
"post": {
|
"post": {
|
||||||
"description": "Checks if multiple assets exist on the server and returns all existing - used by background backup",
|
"description": "Checks if multiple assets exist on the server and returns all existing - used by background backup",
|
||||||
|
@ -1323,6 +1323,14 @@ export function getAllUserAssetsByDeviceId({ deviceId }: {
|
|||||||
...opts
|
...opts
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
export function getAssetDuplicates(opts?: Oazapfts.RequestOpts) {
|
||||||
|
return oazapfts.ok(oazapfts.fetchJson<{
|
||||||
|
status: 200;
|
||||||
|
data: AssetResponseDto[];
|
||||||
|
}>("/asset/duplicates", {
|
||||||
|
...opts
|
||||||
|
}));
|
||||||
|
}
|
||||||
/**
|
/**
|
||||||
* Checks if multiple assets exist on the server and returns all existing - used by background backup
|
* Checks if multiple assets exist on the server and returns all existing - used by background backup
|
||||||
*/
|
*/
|
||||||
|
@ -70,6 +70,11 @@ export class AssetController {
|
|||||||
return this.service.getStatistics(auth, dto);
|
return this.service.getStatistics(auth, dto);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Get('duplicates')
|
||||||
|
getAssetDuplicates(@Auth() auth: AuthDto): Promise<AssetResponseDto[]> {
|
||||||
|
return this.service.getDuplicates(auth);
|
||||||
|
}
|
||||||
|
|
||||||
@Post('jobs')
|
@Post('jobs')
|
||||||
@HttpCode(HttpStatus.NO_CONTENT)
|
@HttpCode(HttpStatus.NO_CONTENT)
|
||||||
runAssetJobs(@Auth() auth: AuthDto, @Body() dto: AssetJobsDto): Promise<void> {
|
runAssetJobs(@Auth() auth: AuthDto, @Body() dto: AssetJobsDto): Promise<void> {
|
||||||
|
@ -71,7 +71,7 @@ export const defaults = Object.freeze<SystemConfig>({
|
|||||||
clip: {
|
clip: {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
modelName: 'ViT-B-32__openai',
|
modelName: 'ViT-B-32__openai',
|
||||||
duplicateThreshold: 0.01,
|
duplicateThreshold: 0.03,
|
||||||
},
|
},
|
||||||
facialRecognition: {
|
facialRecognition: {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
|
@ -4,7 +4,7 @@ import { Entity, Index, JoinColumn, OneToMany, PrimaryColumn } from 'typeorm';
|
|||||||
@Entity('asset_duplicates')
|
@Entity('asset_duplicates')
|
||||||
@Index('asset_duplicates_assetId_uindex', ['assetId'], { unique: true })
|
@Index('asset_duplicates_assetId_uindex', ['assetId'], { unique: true })
|
||||||
export class AssetDuplicateEntity {
|
export class AssetDuplicateEntity {
|
||||||
@OneToMany(() => AssetEntity, (asset) => asset.duplicates)
|
@OneToMany(() => AssetEntity, (asset) => asset.duplicates, { onDelete: 'CASCADE', onUpdate: 'CASCADE' })
|
||||||
@JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
|
@JoinColumn({ name: 'assetId', referencedColumnName: 'id' })
|
||||||
assets!: AssetEntity;
|
assets!: AssetEntity;
|
||||||
|
|
||||||
|
@ -1,4 +1,5 @@
|
|||||||
import { AlbumEntity } from 'src/entities/album.entity';
|
import { AlbumEntity } from 'src/entities/album.entity';
|
||||||
|
import { AssetDuplicateEntity } from 'src/entities/asset-duplicate.entity';
|
||||||
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
|
import { AssetFaceEntity } from 'src/entities/asset-face.entity';
|
||||||
import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
|
import { AssetJobStatusEntity } from 'src/entities/asset-job-status.entity';
|
||||||
import { AssetStackEntity } from 'src/entities/asset-stack.entity';
|
import { AssetStackEntity } from 'src/entities/asset-stack.entity';
|
||||||
@ -24,7 +25,6 @@ import {
|
|||||||
PrimaryGeneratedColumn,
|
PrimaryGeneratedColumn,
|
||||||
UpdateDateColumn,
|
UpdateDateColumn,
|
||||||
} from 'typeorm';
|
} from 'typeorm';
|
||||||
import { AssetDuplicateEntity } from './asset-duplicate.entity';
|
|
||||||
|
|
||||||
export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_library_checksum';
|
export const ASSET_CHECKSUM_CONSTRAINT = 'UQ_assets_owner_library_checksum';
|
||||||
|
|
||||||
@ -173,7 +173,7 @@ export class AssetEntity {
|
|||||||
@Column({ nullable: true })
|
@Column({ nullable: true })
|
||||||
duplicateId?: string | null;
|
duplicateId?: string | null;
|
||||||
|
|
||||||
@ManyToOne(() => AssetDuplicateEntity, { nullable: true, onDelete: 'SET NULL', onUpdate: 'CASCADE' })
|
@ManyToOne(() => AssetDuplicateEntity, { nullable: true })
|
||||||
@JoinColumn({ name: 'duplicateId' })
|
@JoinColumn({ name: 'duplicateId' })
|
||||||
duplicates?: AssetDuplicateEntity | null;
|
duplicates?: AssetDuplicateEntity | null;
|
||||||
}
|
}
|
||||||
|
@ -61,6 +61,7 @@ export interface AssetBuilderOptions {
|
|||||||
isArchived?: boolean;
|
isArchived?: boolean;
|
||||||
isFavorite?: boolean;
|
isFavorite?: boolean;
|
||||||
isTrashed?: boolean;
|
isTrashed?: boolean;
|
||||||
|
isDuplicate?: boolean;
|
||||||
albumId?: string;
|
albumId?: string;
|
||||||
personId?: string;
|
personId?: string;
|
||||||
userIds?: string[];
|
userIds?: string[];
|
||||||
@ -172,8 +173,9 @@ export interface IAssetRepository {
|
|||||||
getTimeBuckets(options: TimeBucketOptions): Promise<TimeBucketItem[]>;
|
getTimeBuckets(options: TimeBucketOptions): Promise<TimeBucketItem[]>;
|
||||||
getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise<AssetEntity[]>;
|
getTimeBucket(timeBucket: string, options: TimeBucketOptions): Promise<AssetEntity[]>;
|
||||||
upsertExif(exif: Partial<ExifEntity>): Promise<void>;
|
upsertExif(exif: Partial<ExifEntity>): Promise<void>;
|
||||||
upsertJobStatus(jobStatus: Partial<AssetJobStatusEntity>): Promise<void>;
|
upsertJobStatus(jobStatus: Partial<AssetJobStatusEntity> | Partial<AssetJobStatusEntity>[]): Promise<void>;
|
||||||
getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
|
getAssetIdByCity(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
|
||||||
getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
|
getAssetIdByTag(userId: string, options: AssetExploreFieldOptions): Promise<SearchExploreItem<string>>;
|
||||||
|
getDuplicates(options: AssetBuilderOptions): Promise<AssetEntity[]>;
|
||||||
searchMetadata(query: string, userIds: string[], options: MetadataSearchOptions): Promise<AssetEntity[]>;
|
searchMetadata(query: string, userIds: string[], options: MetadataSearchOptions): Promise<AssetEntity[]>;
|
||||||
}
|
}
|
||||||
|
@ -179,6 +179,7 @@ export interface FaceEmbeddingSearch extends SearchEmbeddingOptions {
|
|||||||
|
|
||||||
export interface AssetDuplicateSearch {
|
export interface AssetDuplicateSearch {
|
||||||
assetId: string;
|
assetId: string;
|
||||||
|
embedding: Embedding;
|
||||||
userIds: string[];
|
userIds: string[];
|
||||||
maxDistance?: number;
|
maxDistance?: number;
|
||||||
}
|
}
|
||||||
|
@ -0,0 +1,36 @@
|
|||||||
|
import { MigrationInterface, QueryRunner } from 'typeorm';
|
||||||
|
|
||||||
|
export class CreateAssetDuplicateTable1711989989911 implements MigrationInterface {
|
||||||
|
public async up(queryRunner: QueryRunner): Promise<void> {
|
||||||
|
await queryRunner.query(`
|
||||||
|
CREATE TABLE asset_duplicates (
|
||||||
|
id uuid,
|
||||||
|
"assetId" uuid REFERENCES assets ON UPDATE CASCADE ON DELETE CASCADE,
|
||||||
|
PRIMARY KEY (id, "assetId")
|
||||||
|
);
|
||||||
|
`);
|
||||||
|
|
||||||
|
await queryRunner.query(`ALTER TABLE assets ADD COLUMN "duplicateId" uuid`);
|
||||||
|
|
||||||
|
await queryRunner.query(`ALTER TABLE asset_job_status ADD COLUMN "duplicatesDetectedAt" timestamptz`);
|
||||||
|
|
||||||
|
await queryRunner.query(`
|
||||||
|
ALTER TABLE assets
|
||||||
|
ADD CONSTRAINT asset_duplicates_id
|
||||||
|
FOREIGN KEY ("duplicateId", id)
|
||||||
|
REFERENCES asset_duplicates DEFERRABLE INITIALLY DEFERRED
|
||||||
|
`);
|
||||||
|
|
||||||
|
await queryRunner.query(`
|
||||||
|
CREATE UNIQUE INDEX "asset_duplicates_assetId_uindex"
|
||||||
|
ON asset_duplicates ("assetId")
|
||||||
|
`);
|
||||||
|
|
||||||
|
await queryRunner.query(`CREATE INDEX "IDX_assets_duplicateId" ON assets ("duplicateId")`);
|
||||||
|
}
|
||||||
|
|
||||||
|
public async down(queryRunner: QueryRunner): Promise<void> {
|
||||||
|
await queryRunner.query(`ALTER TABLE assets DROP COLUMN "duplicateId"`);
|
||||||
|
await queryRunner.query(`DROP TABLE asset_duplicates`);
|
||||||
|
}
|
||||||
|
}
|
@ -22,7 +22,7 @@ export class AssetDuplicateRepository implements IAssetDuplicateRepository {
|
|||||||
await manager.update(AssetDuplicateEntity, { id: In(oldDuplicateIds) }, { id });
|
await manager.update(AssetDuplicateEntity, { id: In(oldDuplicateIds) }, { id });
|
||||||
}
|
}
|
||||||
await manager.update(AssetEntity, { id: In(assetIds) }, { duplicateId: id });
|
await manager.update(AssetEntity, { id: In(assetIds) }, { duplicateId: id });
|
||||||
await manager.update(AssetEntity, { duplicateId: In(oldDuplicateIds) }, { duplicateId: id }); // TODO: cascade should handle this, but it doesn't seem to
|
await manager.update(AssetEntity, { duplicateId: In(oldDuplicateIds) }, { duplicateId: id });
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -67,7 +67,7 @@ export class AssetRepository implements IAssetRepository {
|
|||||||
await this.exifRepository.upsert(exif, { conflictPaths: ['assetId'] });
|
await this.exifRepository.upsert(exif, { conflictPaths: ['assetId'] });
|
||||||
}
|
}
|
||||||
|
|
||||||
async upsertJobStatus(jobStatus: Partial<AssetJobStatusEntity>): Promise<void> {
|
async upsertJobStatus(jobStatus: Partial<AssetJobStatusEntity> | Partial<AssetJobStatusEntity>[]): Promise<void> {
|
||||||
await this.jobStatusRepository.upsert(jobStatus, { conflictPaths: ['assetId'] });
|
await this.jobStatusRepository.upsert(jobStatus, { conflictPaths: ['assetId'] });
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -201,6 +201,7 @@ export class AssetRepository implements IAssetRepository {
|
|||||||
let builder = this.repository.createQueryBuilder('asset');
|
let builder = this.repository.createQueryBuilder('asset');
|
||||||
builder = searchAssetBuilder(builder, options);
|
builder = searchAssetBuilder(builder, options);
|
||||||
builder.orderBy('asset.createdAt', options.orderDirection ?? 'ASC');
|
builder.orderBy('asset.createdAt', options.orderDirection ?? 'ASC');
|
||||||
|
builder.innerJoin('asset.smartSearch', 'smartSearch');
|
||||||
return paginatedBuilder<AssetEntity>(builder, {
|
return paginatedBuilder<AssetEntity>(builder, {
|
||||||
mode: PaginationMode.SKIP_TAKE,
|
mode: PaginationMode.SKIP_TAKE,
|
||||||
skip: pagination.skip,
|
skip: pagination.skip,
|
||||||
@ -591,6 +592,13 @@ export class AssetRepository implements IAssetRepository {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@GenerateSql({ params: [{ userIds: [DummyValue.UUID, DummyValue.UUID] }] })
|
||||||
|
getDuplicates(options: AssetBuilderOptions): Promise<AssetEntity[]> {
|
||||||
|
return this.getBuilder({ ...options, isDuplicate: true })
|
||||||
|
.orderBy('asset.duplicateId')
|
||||||
|
.getMany();
|
||||||
|
}
|
||||||
|
|
||||||
@GenerateSql({ params: [DummyValue.UUID, { minAssetsPerField: 5, maxFields: 12 }] })
|
@GenerateSql({ params: [DummyValue.UUID, { minAssetsPerField: 5, maxFields: 12 }] })
|
||||||
async getAssetIdByCity(
|
async getAssetIdByCity(
|
||||||
ownerId: string,
|
ownerId: string,
|
||||||
@ -650,16 +658,14 @@ export class AssetRepository implements IAssetRepository {
|
|||||||
}
|
}
|
||||||
|
|
||||||
private getBuilder(options: AssetBuilderOptions) {
|
private getBuilder(options: AssetBuilderOptions) {
|
||||||
const { isArchived, isFavorite, isTrashed, albumId, personId, userIds, withStacked, exifInfo, assetType } = options;
|
|
||||||
|
|
||||||
let builder = this.repository.createQueryBuilder('asset').where('asset.isVisible = true');
|
let builder = this.repository.createQueryBuilder('asset').where('asset.isVisible = true');
|
||||||
if (assetType !== undefined) {
|
if (options.assetType !== undefined) {
|
||||||
builder = builder.andWhere('asset.type = :assetType', { assetType });
|
builder = builder.andWhere('asset.type = :assetType', { assetType: options.assetType });
|
||||||
}
|
}
|
||||||
|
|
||||||
let stackJoined = false;
|
let stackJoined = false;
|
||||||
|
|
||||||
if (exifInfo !== false) {
|
if (options.exifInfo !== false) {
|
||||||
stackJoined = true;
|
stackJoined = true;
|
||||||
builder = builder
|
builder = builder
|
||||||
.leftJoinAndSelect('asset.exifInfo', 'exifInfo')
|
.leftJoinAndSelect('asset.exifInfo', 'exifInfo')
|
||||||
@ -667,34 +673,38 @@ export class AssetRepository implements IAssetRepository {
|
|||||||
.leftJoinAndSelect('stack.assets', 'stackedAssets');
|
.leftJoinAndSelect('stack.assets', 'stackedAssets');
|
||||||
}
|
}
|
||||||
|
|
||||||
if (albumId) {
|
if (options.albumId) {
|
||||||
builder = builder.leftJoin('asset.albums', 'album').andWhere('album.id = :albumId', { albumId });
|
builder = builder.leftJoin('asset.albums', 'album').andWhere('album.id = :albumId', { albumId: options.albumId });
|
||||||
}
|
}
|
||||||
|
|
||||||
if (userIds) {
|
if (options.userIds) {
|
||||||
builder = builder.andWhere('asset.ownerId IN (:...userIds )', { userIds });
|
builder = builder.andWhere('asset.ownerId IN (:...userIds )', { userIds: options.userIds });
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isArchived !== undefined) {
|
if (options.isArchived !== undefined) {
|
||||||
builder = builder.andWhere('asset.isArchived = :isArchived', { isArchived });
|
builder = builder.andWhere('asset.isArchived = :isArchived', { isArchived: options.isArchived });
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isFavorite !== undefined) {
|
if (options.isFavorite !== undefined) {
|
||||||
builder = builder.andWhere('asset.isFavorite = :isFavorite', { isFavorite });
|
builder = builder.andWhere('asset.isFavorite = :isFavorite', { isFavorite: options.isFavorite });
|
||||||
}
|
}
|
||||||
|
|
||||||
if (isTrashed !== undefined) {
|
if (options.isTrashed !== undefined) {
|
||||||
builder = builder.andWhere(`asset.deletedAt ${isTrashed ? 'IS NOT NULL' : 'IS NULL'}`).withDeleted();
|
builder = builder.andWhere(`asset.deletedAt ${options.isTrashed ? 'IS NOT NULL' : 'IS NULL'}`).withDeleted();
|
||||||
}
|
}
|
||||||
|
|
||||||
if (personId !== undefined) {
|
if (options.isDuplicate !== undefined) {
|
||||||
|
builder = builder.andWhere(`asset.duplicateId ${options.isDuplicate ? 'IS NOT NULL' : 'IS NULL'}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (options.personId !== undefined) {
|
||||||
builder = builder
|
builder = builder
|
||||||
.innerJoin('asset.faces', 'faces')
|
.innerJoin('asset.faces', 'faces')
|
||||||
.innerJoin('faces.person', 'person')
|
.innerJoin('faces.person', 'person')
|
||||||
.andWhere('person.id = :personId', { personId });
|
.andWhere('person.id = :personId', { personId: options.personId });
|
||||||
}
|
}
|
||||||
|
|
||||||
if (withStacked) {
|
if (options.withStacked) {
|
||||||
if (!stackJoined) {
|
if (!stackJoined) {
|
||||||
builder = builder.leftJoinAndSelect('asset.stack', 'stack').leftJoinAndSelect('stack.assets', 'stackedAssets');
|
builder = builder.leftJoinAndSelect('asset.stack', 'stack').leftJoinAndSelect('stack.assets', 'stackedAssets');
|
||||||
}
|
}
|
||||||
|
@ -248,6 +248,11 @@ export class AssetService {
|
|||||||
return data;
|
return data;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async getDuplicates(auth: AuthDto): Promise<AssetResponseDto[]> {
|
||||||
|
const res = await this.assetRepository.getDuplicates({ userIds: [auth.user.id] });
|
||||||
|
return res.map((a) => mapAsset(a, { auth }));
|
||||||
|
}
|
||||||
|
|
||||||
async update(auth: AuthDto, id: string, dto: UpdateAssetDto): Promise<AssetResponseDto> {
|
async update(auth: AuthDto, id: string, dto: UpdateAssetDto): Promise<AssetResponseDto> {
|
||||||
await this.access.requirePermission(auth, Permission.ASSET_UPDATE, id);
|
await this.access.requirePermission(auth, Permission.ASSET_UPDATE, id);
|
||||||
|
|
||||||
|
@ -179,7 +179,11 @@ export class SearchService {
|
|||||||
async handleSearchDuplicates({ id }: IEntityJob): Promise<JobStatus> {
|
async handleSearchDuplicates({ id }: IEntityJob): Promise<JobStatus> {
|
||||||
const { machineLearning } = await this.configCore.getConfig();
|
const { machineLearning } = await this.configCore.getConfig();
|
||||||
|
|
||||||
const asset = await this.assetRepository.getById(id);
|
const [asset] = await this.assetRepository.getByIds(
|
||||||
|
[id],
|
||||||
|
{ smartSearch: true },
|
||||||
|
{ smartSearch: { assetId: true, embedding: true } },
|
||||||
|
);
|
||||||
if (!asset) {
|
if (!asset) {
|
||||||
this.logger.error(`Asset ${id} not found`);
|
this.logger.error(`Asset ${id} not found`);
|
||||||
return JobStatus.FAILED;
|
return JobStatus.FAILED;
|
||||||
@ -187,10 +191,6 @@ export class SearchService {
|
|||||||
|
|
||||||
if (!asset.isVisible) {
|
if (!asset.isVisible) {
|
||||||
this.logger.debug(`Asset ${id} is not visible, skipping`);
|
this.logger.debug(`Asset ${id} is not visible, skipping`);
|
||||||
await this.assetRepository.upsertJobStatus({
|
|
||||||
assetId: asset.id,
|
|
||||||
duplicatesDetectedAt: new Date(),
|
|
||||||
});
|
|
||||||
return JobStatus.SKIPPED;
|
return JobStatus.SKIPPED;
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -204,28 +204,35 @@ export class SearchService {
|
|||||||
return JobStatus.FAILED;
|
return JobStatus.FAILED;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (!asset.smartSearch?.embedding) {
|
||||||
|
this.logger.debug(`Asset ${id} is missing embedding`);
|
||||||
|
return JobStatus.FAILED;
|
||||||
|
}
|
||||||
|
|
||||||
const duplicateAssets = await this.searchRepository.searchDuplicates({
|
const duplicateAssets = await this.searchRepository.searchDuplicates({
|
||||||
assetId: asset.id,
|
assetId: asset.id,
|
||||||
|
embedding: asset.smartSearch.embedding,
|
||||||
maxDistance: machineLearning.clip.duplicateThreshold,
|
maxDistance: machineLearning.clip.duplicateThreshold,
|
||||||
userIds: [asset.ownerId],
|
userIds: [asset.ownerId],
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const duplicateAssetIds = [asset.id];
|
||||||
|
|
||||||
if (duplicateAssets.length > 0) {
|
if (duplicateAssets.length > 0) {
|
||||||
this.logger.debug(`Found ${duplicateAssets.length} duplicates for asset ${asset.id}`);
|
this.logger.debug(
|
||||||
|
`Found ${duplicateAssets.length} duplicate${duplicateAssets.length === 1 ? '' : 's'} for asset ${asset.id}`,
|
||||||
|
);
|
||||||
|
|
||||||
const duplicateIds = duplicateAssets.map((duplicate) => duplicate.duplicateId).filter(Boolean);
|
const duplicateIds = duplicateAssets.map((duplicate) => duplicate.duplicateId).filter(Boolean);
|
||||||
const duplicateId = duplicateIds[0] || this.cryptoRepository.randomUUID();
|
const duplicateId = duplicateIds[0] || this.cryptoRepository.randomUUID();
|
||||||
|
|
||||||
const duplicateAssetIds = duplicateAssets.map((duplicate) => duplicate.assetId);
|
duplicateAssetIds.push(...duplicateAssets.map((duplicate) => duplicate.assetId));
|
||||||
duplicateAssetIds.push(asset.id);
|
|
||||||
|
|
||||||
await this.assetDuplicateRepository.upsert(duplicateId, duplicateAssetIds, duplicateIds);
|
await this.assetDuplicateRepository.upsert(duplicateId, duplicateAssetIds, duplicateIds);
|
||||||
}
|
}
|
||||||
|
|
||||||
await this.assetRepository.upsertJobStatus({
|
const duplicatesDetectedAt = new Date();
|
||||||
assetId: asset.id,
|
await this.assetRepository.upsertJobStatus(duplicateAssetIds.map((assetId) => ({ assetId, duplicatesDetectedAt })));
|
||||||
duplicatesDetectedAt: new Date(),
|
|
||||||
});
|
|
||||||
|
|
||||||
return JobStatus.SUCCESS;
|
return JobStatus.SUCCESS;
|
||||||
}
|
}
|
||||||
|
@ -74,7 +74,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
|
|||||||
clip: {
|
clip: {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
modelName: 'ViT-B-32__openai',
|
modelName: 'ViT-B-32__openai',
|
||||||
duplicateThreshold: 0.01,
|
duplicateThreshold: 0.03,
|
||||||
},
|
},
|
||||||
facialRecognition: {
|
facialRecognition: {
|
||||||
enabled: true,
|
enabled: true,
|
||||||
|
@ -131,6 +131,19 @@
|
|||||||
</svelte:fragment>
|
</svelte:fragment>
|
||||||
</SideBarLink>
|
</SideBarLink>
|
||||||
|
|
||||||
|
<SideBarLink title="Duplicates" routeId="/(user)/duplicates" icon={mdiArchiveArrowDownOutline}>
|
||||||
|
<svelte:fragment slot="moreInformation">
|
||||||
|
{#await getStats({ isArchived: true })}
|
||||||
|
<LoadingSpinner />
|
||||||
|
{:then data}
|
||||||
|
<div>
|
||||||
|
<p>{data.videos.toLocaleString($locale)} Videos</p>
|
||||||
|
<p>{data.images.toLocaleString($locale)} Photos</p>
|
||||||
|
</div>
|
||||||
|
{/await}
|
||||||
|
</svelte:fragment>
|
||||||
|
</SideBarLink>
|
||||||
|
|
||||||
{#if $featureFlags.trash}
|
{#if $featureFlags.trash}
|
||||||
<SideBarLink title="Trash" routeId="/(user)/trash" icon={mdiTrashCanOutline}>
|
<SideBarLink title="Trash" routeId="/(user)/trash" icon={mdiTrashCanOutline}>
|
||||||
<svelte:fragment slot="moreInformation">
|
<svelte:fragment slot="moreInformation">
|
||||||
|
@ -20,6 +20,7 @@ export enum AppRoute {
|
|||||||
ALBUMS = '/albums',
|
ALBUMS = '/albums',
|
||||||
LIBRARIES = '/libraries',
|
LIBRARIES = '/libraries',
|
||||||
ARCHIVE = '/archive',
|
ARCHIVE = '/archive',
|
||||||
|
DUPLICATES = '/duplicates',
|
||||||
FAVORITES = '/favorites',
|
FAVORITES = '/favorites',
|
||||||
PEOPLE = '/people',
|
PEOPLE = '/people',
|
||||||
PLACES = '/places',
|
PLACES = '/places',
|
||||||
|
21
web/src/routes/(user)/duplicates/+page.svelte
Normal file
21
web/src/routes/(user)/duplicates/+page.svelte
Normal file
@ -0,0 +1,21 @@
|
|||||||
|
<script lang="ts">
|
||||||
|
import GalleryViewer from '$lib/components/shared-components/gallery-viewer/gallery-viewer.svelte';
|
||||||
|
import { getAssetDuplicates, type AssetResponseDto } from '@immich/sdk';
|
||||||
|
import type { Viewport } from '$lib/stores/assets.store';
|
||||||
|
import { onMount } from 'svelte';
|
||||||
|
|
||||||
|
let assets: AssetResponseDto[] = [];
|
||||||
|
const viewport: Viewport = { width: 0, height: 0 };
|
||||||
|
|
||||||
|
onMount(async () => {
|
||||||
|
assets = await getAssetDuplicates();
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
|
||||||
|
<section
|
||||||
|
class="relative mb-12 bg-immich-bg dark:bg-immich-dark-bg m-4"
|
||||||
|
bind:clientHeight={viewport.height}
|
||||||
|
bind:clientWidth={viewport.width}
|
||||||
|
>
|
||||||
|
<GalleryViewer {assets} {viewport}></GalleryViewer>
|
||||||
|
</section>
|
12
web/src/routes/(user)/duplicates/+page.ts
Normal file
12
web/src/routes/(user)/duplicates/+page.ts
Normal file
@ -0,0 +1,12 @@
|
|||||||
|
import { authenticate } from '$lib/utils/auth';
|
||||||
|
import type { PageLoad } from './$types';
|
||||||
|
|
||||||
|
export const load = (async () => {
|
||||||
|
await authenticate();
|
||||||
|
|
||||||
|
return {
|
||||||
|
meta: {
|
||||||
|
title: 'Duplicates',
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}) satisfies PageLoad;
|
@ -0,0 +1,7 @@
|
|||||||
|
import { AppRoute } from '$lib/constants';
|
||||||
|
import { redirect } from '@sveltejs/kit';
|
||||||
|
import type { PageLoad } from './$types';
|
||||||
|
|
||||||
|
export const load: PageLoad = () => {
|
||||||
|
redirect(302, AppRoute.DUPLICATES);
|
||||||
|
};
|
Loading…
x
Reference in New Issue
Block a user