diff --git a/cli/src/api/open-api/api.ts b/cli/src/api/open-api/api.ts index b185f4d0c..56167aaf3 100644 --- a/cli/src/api/open-api/api.ts +++ b/cli/src/api/open-api/api.ts @@ -1046,6 +1046,20 @@ export interface ClassificationConfig { } +/** + * + * @export + * @enum {string} + */ + +export const Colorspace = { + Srgb: 'srgb', + P3: 'p3' +} as const; + +export type Colorspace = typeof Colorspace[keyof typeof Colorspace]; + + /** * * @export @@ -3184,12 +3198,24 @@ export interface SystemConfigTemplateStorageOptionDto { * @interface SystemConfigThumbnailDto */ export interface SystemConfigThumbnailDto { + /** + * + * @type {Colorspace} + * @memberof SystemConfigThumbnailDto + */ + 'colorspace': Colorspace; /** * * @type {number} * @memberof SystemConfigThumbnailDto */ 'jpegSize': number; + /** + * + * @type {number} + * @memberof SystemConfigThumbnailDto + */ + 'quality': number; /** * * @type {number} @@ -3197,6 +3223,8 @@ export interface SystemConfigThumbnailDto { */ 'webpSize': number; } + + /** * * @export diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index 7f2f8e0ff..5cb7fa651 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -44,6 +44,7 @@ doc/CheckDuplicateAssetResponseDto.md doc/CheckExistingAssetsDto.md doc/CheckExistingAssetsResponseDto.md doc/ClassificationConfig.md +doc/Colorspace.md doc/CreateAlbumDto.md doc/CreateProfileImageResponseDto.md doc/CreateTagDto.md @@ -199,6 +200,7 @@ lib/model/check_existing_assets_response_dto.dart lib/model/classification_config.dart lib/model/clip_config.dart lib/model/clip_mode.dart +lib/model/colorspace.dart lib/model/cq_mode.dart lib/model/create_album_dto.dart lib/model/create_profile_image_response_dto.dart @@ -326,6 +328,7 @@ test/check_existing_assets_response_dto_test.dart test/classification_config_test.dart test/clip_config_test.dart test/clip_mode_test.dart +test/colorspace_test.dart test/cq_mode_test.dart test/create_album_dto_test.dart test/create_profile_image_response_dto_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index f5ffe5319..fc0e4fd33 100644 --- a/mobile/openapi/README.md +++ b/mobile/openapi/README.md @@ -218,6 +218,7 @@ Class | Method | HTTP request | Description - [CheckExistingAssetsDto](doc//CheckExistingAssetsDto.md) - [CheckExistingAssetsResponseDto](doc//CheckExistingAssetsResponseDto.md) - [ClassificationConfig](doc//ClassificationConfig.md) + - [Colorspace](doc//Colorspace.md) - [CreateAlbumDto](doc//CreateAlbumDto.md) - [CreateProfileImageResponseDto](doc//CreateProfileImageResponseDto.md) - [CreateTagDto](doc//CreateTagDto.md) diff --git a/mobile/openapi/doc/Colorspace.md b/mobile/openapi/doc/Colorspace.md new file mode 100644 index 000000000..6f49da91f --- /dev/null +++ b/mobile/openapi/doc/Colorspace.md @@ -0,0 +1,14 @@ +# openapi.model.Colorspace + +## Load the model package +```dart +import 'package:openapi/api.dart'; +``` + +## Properties +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/mobile/openapi/doc/SystemConfigThumbnailDto.md b/mobile/openapi/doc/SystemConfigThumbnailDto.md index 892b863b3..491bf9f12 100644 --- a/mobile/openapi/doc/SystemConfigThumbnailDto.md +++ b/mobile/openapi/doc/SystemConfigThumbnailDto.md @@ -8,7 +8,9 @@ import 'package:openapi/api.dart'; ## Properties Name | Type | Description | Notes ------------ | ------------- | ------------- | ------------- +**colorspace** | [**Colorspace**](Colorspace.md) | | **jpegSize** | **int** | | +**quality** | **int** | | **webpSize** | **int** | | [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index bd921bf67..73a6d84e1 100644 --- a/mobile/openapi/lib/api.dart +++ b/mobile/openapi/lib/api.dart @@ -80,6 +80,7 @@ part 'model/check_duplicate_asset_response_dto.dart'; part 'model/check_existing_assets_dto.dart'; part 'model/check_existing_assets_response_dto.dart'; part 'model/classification_config.dart'; +part 'model/colorspace.dart'; part 'model/create_album_dto.dart'; part 'model/create_profile_image_response_dto.dart'; part 'model/create_tag_dto.dart'; diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index b4ca5a716..38ffa7f1b 100644 --- a/mobile/openapi/lib/api_client.dart +++ b/mobile/openapi/lib/api_client.dart @@ -253,6 +253,8 @@ class ApiClient { return CheckExistingAssetsResponseDto.fromJson(value); case 'ClassificationConfig': return ClassificationConfig.fromJson(value); + case 'Colorspace': + return ColorspaceTypeTransformer().decode(value); case 'CreateAlbumDto': return CreateAlbumDto.fromJson(value); case 'CreateProfileImageResponseDto': diff --git a/mobile/openapi/lib/api_helper.dart b/mobile/openapi/lib/api_helper.dart index c64bacac8..b3205a0c9 100644 --- a/mobile/openapi/lib/api_helper.dart +++ b/mobile/openapi/lib/api_helper.dart @@ -70,6 +70,9 @@ String parameterToString(dynamic value) { if (value is CQMode) { return CQModeTypeTransformer().encode(value).toString(); } + if (value is Colorspace) { + return ColorspaceTypeTransformer().encode(value).toString(); + } if (value is DeleteAssetStatus) { return DeleteAssetStatusTypeTransformer().encode(value).toString(); } diff --git a/mobile/openapi/lib/model/colorspace.dart b/mobile/openapi/lib/model/colorspace.dart new file mode 100644 index 000000000..8ed4303ee --- /dev/null +++ b/mobile/openapi/lib/model/colorspace.dart @@ -0,0 +1,85 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +part of openapi.api; + + +class Colorspace { + /// Instantiate a new enum with the provided [value]. + const Colorspace._(this.value); + + /// The underlying value of this enum member. + final String value; + + @override + String toString() => value; + + String toJson() => value; + + static const srgb = Colorspace._(r'srgb'); + static const p3 = Colorspace._(r'p3'); + + /// List of all possible values in this [enum][Colorspace]. + static const values = [ + srgb, + p3, + ]; + + static Colorspace? fromJson(dynamic value) => ColorspaceTypeTransformer().decode(value); + + static List? listFromJson(dynamic json, {bool growable = false,}) { + final result = []; + if (json is List && json.isNotEmpty) { + for (final row in json) { + final value = Colorspace.fromJson(row); + if (value != null) { + result.add(value); + } + } + } + return result.toList(growable: growable); + } +} + +/// Transformation class that can [encode] an instance of [Colorspace] to String, +/// and [decode] dynamic data back to [Colorspace]. +class ColorspaceTypeTransformer { + factory ColorspaceTypeTransformer() => _instance ??= const ColorspaceTypeTransformer._(); + + const ColorspaceTypeTransformer._(); + + String encode(Colorspace data) => data.value; + + /// Decodes a [dynamic value][data] to a Colorspace. + /// + /// If [allowNull] is true and the [dynamic value][data] cannot be decoded successfully, + /// then null is returned. However, if [allowNull] is false and the [dynamic value][data] + /// cannot be decoded successfully, then an [UnimplementedError] is thrown. + /// + /// The [allowNull] is very handy when an API changes and a new enum value is added or removed, + /// and users are still using an old app with the old code. + Colorspace? decode(dynamic data, {bool allowNull = true}) { + if (data != null) { + switch (data) { + case r'srgb': return Colorspace.srgb; + case r'p3': return Colorspace.p3; + default: + if (!allowNull) { + throw ArgumentError('Unknown enum value to decode: $data'); + } + } + } + return null; + } + + /// Singleton [ColorspaceTypeTransformer] instance. + static ColorspaceTypeTransformer? _instance; +} + diff --git a/mobile/openapi/lib/model/system_config_thumbnail_dto.dart b/mobile/openapi/lib/model/system_config_thumbnail_dto.dart index 54360074e..46d5fe7c1 100644 --- a/mobile/openapi/lib/model/system_config_thumbnail_dto.dart +++ b/mobile/openapi/lib/model/system_config_thumbnail_dto.dart @@ -13,31 +13,43 @@ part of openapi.api; class SystemConfigThumbnailDto { /// Returns a new [SystemConfigThumbnailDto] instance. SystemConfigThumbnailDto({ + required this.colorspace, required this.jpegSize, + required this.quality, required this.webpSize, }); + Colorspace colorspace; + int jpegSize; + int quality; + int webpSize; @override bool operator ==(Object other) => identical(this, other) || other is SystemConfigThumbnailDto && + other.colorspace == colorspace && other.jpegSize == jpegSize && + other.quality == quality && other.webpSize == webpSize; @override int get hashCode => // ignore: unnecessary_parenthesis + (colorspace.hashCode) + (jpegSize.hashCode) + + (quality.hashCode) + (webpSize.hashCode); @override - String toString() => 'SystemConfigThumbnailDto[jpegSize=$jpegSize, webpSize=$webpSize]'; + String toString() => 'SystemConfigThumbnailDto[colorspace=$colorspace, jpegSize=$jpegSize, quality=$quality, webpSize=$webpSize]'; Map toJson() { final json = {}; + json[r'colorspace'] = this.colorspace; json[r'jpegSize'] = this.jpegSize; + json[r'quality'] = this.quality; json[r'webpSize'] = this.webpSize; return json; } @@ -50,7 +62,9 @@ class SystemConfigThumbnailDto { final json = value.cast(); return SystemConfigThumbnailDto( + colorspace: Colorspace.fromJson(json[r'colorspace'])!, jpegSize: mapValueOfType(json, r'jpegSize')!, + quality: mapValueOfType(json, r'quality')!, webpSize: mapValueOfType(json, r'webpSize')!, ); } @@ -99,7 +113,9 @@ class SystemConfigThumbnailDto { /// The list of required keys that must be present in a JSON. static const requiredKeys = { + 'colorspace', 'jpegSize', + 'quality', 'webpSize', }; } diff --git a/mobile/openapi/test/colorspace_test.dart b/mobile/openapi/test/colorspace_test.dart new file mode 100644 index 000000000..f689d519e --- /dev/null +++ b/mobile/openapi/test/colorspace_test.dart @@ -0,0 +1,21 @@ +// +// AUTO-GENERATED FILE, DO NOT MODIFY! +// +// @dart=2.12 + +// ignore_for_file: unused_element, unused_import +// ignore_for_file: always_put_required_named_parameters_first +// ignore_for_file: constant_identifier_names +// ignore_for_file: lines_longer_than_80_chars + +import 'package:openapi/api.dart'; +import 'package:test/test.dart'; + +// tests for Colorspace +void main() { + + group('test Colorspace', () { + + }); + +} diff --git a/mobile/openapi/test/system_config_thumbnail_dto_test.dart b/mobile/openapi/test/system_config_thumbnail_dto_test.dart index 3dd82cff7..3cc66f467 100644 --- a/mobile/openapi/test/system_config_thumbnail_dto_test.dart +++ b/mobile/openapi/test/system_config_thumbnail_dto_test.dart @@ -16,11 +16,21 @@ void main() { // final instance = SystemConfigThumbnailDto(); group('test SystemConfigThumbnailDto', () { + // Colorspace colorspace + test('to test the property `colorspace`', () async { + // TODO + }); + // int jpegSize test('to test the property `jpegSize`', () async { // TODO }); + // int quality + test('to test the property `quality`', () async { + // TODO + }); + // int webpSize test('to test the property `webpSize`', () async { // TODO diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index a61a16a76..c38db8cbf 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -5523,6 +5523,13 @@ ], "type": "object" }, + "Colorspace": { + "enum": [ + "srgb", + "p3" + ], + "type": "string" + }, "CreateAlbumDto": { "properties": { "albumName": { @@ -7284,16 +7291,24 @@ }, "SystemConfigThumbnailDto": { "properties": { + "colorspace": { + "$ref": "#/components/schemas/Colorspace" + }, "jpegSize": { "type": "integer" }, + "quality": { + "type": "integer" + }, "webpSize": { "type": "integer" } }, "required": [ "webpSize", - "jpegSize" + "jpegSize", + "quality", + "colorspace" ], "type": "object" }, diff --git a/server/src/domain/facial-recognition/facial-recognition.service.spec.ts b/server/src/domain/facial-recognition/facial-recognition.service.spec.ts index 0b117d6a2..1ca3ed89b 100644 --- a/server/src/domain/facial-recognition/facial-recognition.service.spec.ts +++ b/server/src/domain/facial-recognition/facial-recognition.service.spec.ts @@ -1,3 +1,4 @@ +import { Colorspace } from '@app/infra/entities'; import { assetStub, faceStub, @@ -115,7 +116,6 @@ describe(FacialRecognitionService.name, () => { personMock = newPersonRepositoryMock(); searchMock = newSearchRepositoryMock(); storageMock = newStorageRepositoryMock(); - configMock = newSystemConfigRepositoryMock(); mediaMock.crop.mockResolvedValue(croppedFace); @@ -292,6 +292,8 @@ describe(FacialRecognitionService.name, () => { expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { format: 'jpeg', size: 250, + quality: 80, + colorspace: Colorspace.P3, }); expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', @@ -313,6 +315,8 @@ describe(FacialRecognitionService.name, () => { expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { format: 'jpeg', size: 250, + quality: 80, + colorspace: Colorspace.P3, }); }); @@ -330,6 +334,8 @@ describe(FacialRecognitionService.name, () => { expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { format: 'jpeg', size: 250, + quality: 80, + colorspace: Colorspace.P3, }); }); }); diff --git a/server/src/domain/facial-recognition/facial-recognition.services.ts b/server/src/domain/facial-recognition/facial-recognition.services.ts index ad90dca24..471d21ba4 100644 --- a/server/src/domain/facial-recognition/facial-recognition.services.ts +++ b/server/src/domain/facial-recognition/facial-recognition.services.ts @@ -162,8 +162,15 @@ export class FacialRecognitionService { height: newHalfSize * 2, }; + const { thumbnail } = await this.configCore.getConfig(); const croppedOutput = await this.mediaRepository.crop(asset.resizePath, cropOptions); - await this.mediaRepository.resize(croppedOutput, output, { size: FACE_THUMBNAIL_SIZE, format: 'jpeg' }); + const thumbnailOptions = { + format: 'jpeg', + size: FACE_THUMBNAIL_SIZE, + colorspace: thumbnail.colorspace, + quality: thumbnail.quality, + } as const; + await this.mediaRepository.resize(croppedOutput, output, thumbnailOptions); await this.personRepository.update({ id: personId, thumbnailPath: output }); return true; diff --git a/server/src/domain/media/media.repository.ts b/server/src/domain/media/media.repository.ts index 25486cc4e..cf2b8fa81 100644 --- a/server/src/domain/media/media.repository.ts +++ b/server/src/domain/media/media.repository.ts @@ -1,10 +1,13 @@ import { VideoCodec } from '@app/infra/entities'; +import { Writable } from 'stream'; export const IMediaRepository = 'IMediaRepository'; export interface ResizeOptions { size: number; format: 'webp' | 'jpeg'; + colorspace: string; + quality: number; } export interface VideoStreamInfo { @@ -73,5 +76,5 @@ export interface IMediaRepository { // video probe(input: string): Promise; - transcode(input: string, output: string, options: TranscodeOptions): Promise; + transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise; } diff --git a/server/src/domain/media/media.service.spec.ts b/server/src/domain/media/media.service.spec.ts index 355ba7873..dc9359aa4 100644 --- a/server/src/domain/media/media.service.spec.ts +++ b/server/src/domain/media/media.service.spec.ts @@ -1,5 +1,6 @@ import { AssetType, + Colorspace, SystemConfigKey, ToneMapping, TranscodeHWAccel, @@ -134,6 +135,8 @@ describe(MediaService.name, () => { expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.jpeg', { size: 1440, format: 'jpeg', + quality: 80, + colorspace: Colorspace.P3, }); expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', @@ -148,12 +151,11 @@ describe(MediaService.name, () => { expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id'); expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', { - inputOptions: [], + inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'], outputOptions: [ - '-ss 00:00:00.000', '-frames:v 1', '-v verbose', - '-vf scale=-2:1440:out_color_matrix=bt601:out_range=pc,format=yuv420p', + '-vf scale=-2:1440:flags=lanczos+accurate_rnd+bitexact+full_chroma_int:out_color_matrix=601:out_range=pc,format=yuv420p', ], twoPass: false, }); @@ -170,12 +172,11 @@ describe(MediaService.name, () => { expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id'); expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', { - inputOptions: [], + inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'], outputOptions: [ - '-ss 00:00:00.000', '-frames:v 1', '-v verbose', - '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt470bg:t=601:m=bt470bg:range=pc,format=yuv420p', + '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p', ], twoPass: false, }); @@ -209,12 +210,13 @@ describe(MediaService.name, () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); await sut.handleGenerateWebpThumbnail({ id: assetStub.image.id }); - expect(mediaMock.resize).toHaveBeenCalledWith( - '/uploads/user-id/thumbs/path.jpg', - '/uploads/user-id/thumbs/path.webp', - { format: 'webp', size: 250 }, - ); - expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', webpPath: '/uploads/user-id/thumbs/path.webp' }); + expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.webp', { + format: 'webp', + size: 250, + quality: 80, + colorspace: Colorspace.P3, + }); + expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', webpPath: 'upload/thumbs/user-id/asset-id.webp' }); }); }); diff --git a/server/src/domain/media/media.service.ts b/server/src/domain/media/media.service.ts index 732aadde7..a09b77cbc 100644 --- a/server/src/domain/media/media.service.ts +++ b/server/src/domain/media/media.service.ts @@ -9,7 +9,6 @@ import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config import { SystemConfigCore } from '../system-config/system-config.core'; import { AudioStreamInfo, IMediaRepository, VideoCodecHWConfig, VideoStreamInfo } from './media.repository'; import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, ThumbnailConfig, VAAPIConfig, VP9Config } from './media.util'; - @Injectable() export class MediaService { private logger = new Logger(MediaService.name); @@ -21,9 +20,9 @@ export class MediaService { @Inject(IJobRepository) private jobRepository: IJobRepository, @Inject(IMediaRepository) private mediaRepository: IMediaRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, - @Inject(ISystemConfigRepository) systemConfig: ISystemConfigRepository, + @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, ) { - this.configCore = new SystemConfigCore(systemConfig); + this.configCore = new SystemConfigCore(configRepository); } async handleQueueGenerateThumbnails(job: IBaseJob) { @@ -59,38 +58,53 @@ export class MediaService { return false; } - const resizePath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId); - this.storageRepository.mkdirSync(resizePath); - const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`); - const { thumbnail } = await this.configCore.getConfig(); + const resizePath = await this.generateThumbnail(asset, 'jpeg'); + await this.assetRepository.save({ id: asset.id, resizePath }); + return true; + } + async generateThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') { + let path; switch (asset.type) { case AssetType.IMAGE: - await this.mediaRepository.resize(asset.originalPath, jpegThumbnailPath, { - size: thumbnail.jpegSize, - format: 'jpeg', - }); - this.logger.log(`Successfully generated image thumbnail ${asset.id}`); + path = await this.generateImageThumbnail(asset, format); break; case AssetType.VIDEO: - const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath); - const mainVideoStream = this.getMainStream(videoStreams); - if (!mainVideoStream) { - this.logger.error(`Could not extract thumbnail for asset ${asset.id}: no video streams found`); - return false; - } - const mainAudioStream = this.getMainStream(audioStreams); - const { ffmpeg } = await this.configCore.getConfig(); - const config = { ...ffmpeg, targetResolution: thumbnail.jpegSize.toString(), twoPass: false }; - const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream); - await this.mediaRepository.transcode(asset.originalPath, jpegThumbnailPath, options); - this.logger.log(`Successfully generated video thumbnail ${asset.id}`); + path = await this.generateVideoThumbnail(asset, format); break; + default: + throw new UnsupportedMediaTypeException(`Unsupported asset type for thumbnail generation: ${asset.type}`); } + this.logger.log( + `Successfully generated ${format.toUpperCase()} ${asset.type.toLowerCase()} thumbnail for asset ${asset.id}`, + ); + return path; + } - await this.assetRepository.save({ id: asset.id, resizePath: jpegThumbnailPath }); + async generateImageThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') { + const { thumbnail } = await this.configCore.getConfig(); + const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize; + const thumbnailOptions = { format, size, colorspace: thumbnail.colorspace, quality: thumbnail.quality }; + const path = this.ensureThumbnailPath(asset, format); + await this.mediaRepository.resize(asset.originalPath, path, thumbnailOptions); + return path; + } - return true; + async generateVideoThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') { + const { ffmpeg, thumbnail } = await this.configCore.getConfig(); + const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize; + const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath); + const mainVideoStream = this.getMainStream(videoStreams); + if (!mainVideoStream) { + this.logger.warn(`Skipped thumbnail generation for asset ${asset.id}: no video streams found`); + return; + } + const mainAudioStream = this.getMainStream(audioStreams); + const path = this.ensureThumbnailPath(asset, format); + const config = { ...ffmpeg, targetResolution: size.toString() }; + const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream); + await this.mediaRepository.transcode(asset.originalPath, path, options); + return path; } async handleGenerateWebpThumbnail({ id }: IEntityJob) { @@ -99,12 +113,8 @@ export class MediaService { return false; } - const webpPath = asset.resizePath.replace('jpeg', 'webp').replace('jpg', 'webp'); - - const { thumbnail } = await this.configCore.getConfig(); - await this.mediaRepository.resize(asset.resizePath, webpPath, { size: thumbnail.webpSize, format: 'webp' }); + const webpPath = await this.generateThumbnail(asset, 'webp'); await this.assetRepository.save({ id: asset.id, webpPath }); - return true; } @@ -289,4 +299,10 @@ export class MediaService { return handler; } + + ensureThumbnailPath(asset: AssetEntity, extension: string): string { + const folderPath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId); + this.storageRepository.mkdirSync(folderPath); + return join(folderPath, `${asset.id}.${extension}`); + } } diff --git a/server/src/domain/media/media.util.ts b/server/src/domain/media/media.util.ts index b276f20d0..75d202094 100644 --- a/server/src/domain/media/media.util.ts +++ b/server/src/domain/media/media.util.ts @@ -263,8 +263,11 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig { } export class ThumbnailConfig extends BaseConfig { + getBaseInputOptions(): string[] { + return ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int']; + } getBaseOutputOptions() { - return ['-ss 00:00:00.000', '-frames:v 1']; + return ['-frames:v 1']; } getPresetOptions() { @@ -277,16 +280,16 @@ export class ThumbnailConfig extends BaseConfig { getScaling(videoStream: VideoStreamInfo) { let options = super.getScaling(videoStream); + options += ':flags=lanczos+accurate_rnd+bitexact+full_chroma_int'; if (!this.shouldToneMap(videoStream)) { - options += ':out_color_matrix=bt601:out_range=pc'; + options += ':out_color_matrix=601:out_range=pc'; } return options; } getColors() { return { - // jpeg and webp only support bt.601, so we need to convert to that directly when tone-mapping to avoid color shifts - primaries: 'bt470bg', + primaries: 'bt709', transfer: '601', matrix: 'bt470bg', }; diff --git a/server/src/domain/system-config/dto/system-config-thumbnail.dto.ts b/server/src/domain/system-config/dto/system-config-thumbnail.dto.ts index 53d9d64a5..c389ef77a 100644 --- a/server/src/domain/system-config/dto/system-config-thumbnail.dto.ts +++ b/server/src/domain/system-config/dto/system-config-thumbnail.dto.ts @@ -1,15 +1,29 @@ +import { Colorspace } from '@app/infra/entities'; import { ApiProperty } from '@nestjs/swagger'; import { Type } from 'class-transformer'; -import { IsInt } from 'class-validator'; +import { IsEnum, IsInt, Max, Min } from 'class-validator'; export class SystemConfigThumbnailDto { @IsInt() + @Min(1) @Type(() => Number) @ApiProperty({ type: 'integer' }) webpSize!: number; @IsInt() + @Min(1) @Type(() => Number) @ApiProperty({ type: 'integer' }) jpegSize!: number; + + @IsInt() + @Min(1) + @Max(100) + @Type(() => Number) + @ApiProperty({ type: 'integer' }) + quality!: number; + + @IsEnum(Colorspace) + @ApiProperty({ enumName: 'Colorspace', enum: Colorspace }) + colorspace!: Colorspace; } diff --git a/server/src/domain/system-config/system-config.core.ts b/server/src/domain/system-config/system-config.core.ts index 1236c15e8..cfe64b0a7 100644 --- a/server/src/domain/system-config/system-config.core.ts +++ b/server/src/domain/system-config/system-config.core.ts @@ -1,6 +1,7 @@ import { AudioCodec, CQMode, + Colorspace, SystemConfig, SystemConfigEntity, SystemConfigKey, @@ -98,6 +99,8 @@ export const defaults = Object.freeze({ thumbnail: { webpSize: 250, jpegSize: 1440, + quality: 80, + colorspace: Colorspace.P3, }, }); diff --git a/server/src/domain/system-config/system-config.service.spec.ts b/server/src/domain/system-config/system-config.service.spec.ts index 28016c20d..6b9ef2d9d 100644 --- a/server/src/domain/system-config/system-config.service.spec.ts +++ b/server/src/domain/system-config/system-config.service.spec.ts @@ -1,6 +1,7 @@ import { AudioCodec, CQMode, + Colorspace, SystemConfig, SystemConfigEntity, SystemConfigKey, @@ -94,6 +95,8 @@ const updatedConfig = Object.freeze({ thumbnail: { webpSize: 250, jpegSize: 1440, + quality: 80, + colorspace: Colorspace.P3, }, }); diff --git a/server/src/infra/entities/system-config.entity.ts b/server/src/infra/entities/system-config.entity.ts index 941058959..2da905d10 100644 --- a/server/src/infra/entities/system-config.entity.ts +++ b/server/src/infra/entities/system-config.entity.ts @@ -76,6 +76,8 @@ export enum SystemConfigKey { THUMBNAIL_WEBP_SIZE = 'thumbnail.webpSize', THUMBNAIL_JPEG_SIZE = 'thumbnail.jpegSize', + THUMBNAIL_QUALITY = 'thumbnail.quality', + THUMBNAIL_COLORSPACE = 'thumbnail.colorspace', } export enum TranscodePolicy { @@ -117,6 +119,11 @@ export enum CQMode { ICQ = 'icq', } +export enum Colorspace { + SRGB = 'srgb', + P3 = 'p3', +} + export interface SystemConfig { ffmpeg: { crf: number; @@ -179,5 +186,7 @@ export interface SystemConfig { thumbnail: { webpSize: number; jpegSize: number; + quality: number; + colorspace: Colorspace; }; } diff --git a/server/src/infra/repositories/media.repository.ts b/server/src/infra/repositories/media.repository.ts index 4012cbf13..2d711a530 100644 --- a/server/src/infra/repositories/media.repository.ts +++ b/server/src/infra/repositories/media.repository.ts @@ -1,8 +1,10 @@ import { CropOptions, IMediaRepository, ResizeOptions, TranscodeOptions, VideoInfo } from '@app/domain'; +import { Colorspace } from '@app/infra/entities'; import { Logger } from '@nestjs/common'; import ffmpeg, { FfprobeData } from 'fluent-ffmpeg'; import fs from 'fs/promises'; import sharp from 'sharp'; +import { Writable } from 'stream'; import { promisify } from 'util'; const probe = promisify(ffmpeg.ffprobe); @@ -11,7 +13,7 @@ sharp.concurrency(0); export class MediaRepository implements IMediaRepository { private logger = new Logger(MediaRepository.name); - crop(input: string, options: CropOptions): Promise { + crop(input: string | Buffer, options: CropOptions): Promise { return sharp(input, { failOn: 'none' }) .extract({ left: options.left, @@ -23,10 +25,25 @@ export class MediaRepository implements IMediaRepository { } async resize(input: string | Buffer, output: string, options: ResizeOptions): Promise { - await sharp(input, { failOn: 'none' }) + let colorProfile = options.colorspace; + if (options.colorspace !== Colorspace.SRGB) { + try { + const { space } = await sharp(input).metadata(); + // if the image is already in srgb, keep it that way + if (space === 'srgb') { + colorProfile = Colorspace.SRGB; + } + } catch (err) { + this.logger.warn(`Could not determine colorspace of image, defaulting to ${colorProfile} profile`); + } + } + const chromaSubsampling = options.quality >= 80 ? '4:4:4' : '4:2:0'; // this is default in libvips (except the threshold is 90), but we need to set it manually in sharp + sharp(input, { failOn: 'none' }) + .pipelineColorspace('rgb16') .resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true }) .rotate() - .toFormat(options.format) + .withMetadata({ icc: colorProfile }) + .toFormat(options.format, { quality: options.quality, chromaSubsampling }) .toFile(output); } @@ -61,7 +78,7 @@ export class MediaRepository implements IMediaRepository { }; } - transcode(input: string, output: string, options: TranscodeOptions): Promise { + transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise { if (!options.twoPass) { return new Promise((resolve, reject) => { ffmpeg(input, { niceness: 10 }) @@ -77,6 +94,10 @@ export class MediaRepository implements IMediaRepository { }); } + if (typeof output !== 'string') { + throw new Error('Two-pass transcoding does not support writing to a stream'); + } + // two-pass allows for precise control of bitrate at the cost of running twice // recommended for vp9 for better quality and compression return new Promise((resolve, reject) => { diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index b185f4d0c..56167aaf3 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -1046,6 +1046,20 @@ export interface ClassificationConfig { } +/** + * + * @export + * @enum {string} + */ + +export const Colorspace = { + Srgb: 'srgb', + P3: 'p3' +} as const; + +export type Colorspace = typeof Colorspace[keyof typeof Colorspace]; + + /** * * @export @@ -3184,12 +3198,24 @@ export interface SystemConfigTemplateStorageOptionDto { * @interface SystemConfigThumbnailDto */ export interface SystemConfigThumbnailDto { + /** + * + * @type {Colorspace} + * @memberof SystemConfigThumbnailDto + */ + 'colorspace': Colorspace; /** * * @type {number} * @memberof SystemConfigThumbnailDto */ 'jpegSize': number; + /** + * + * @type {number} + * @memberof SystemConfigThumbnailDto + */ + 'quality': number; /** * * @type {number} @@ -3197,6 +3223,8 @@ export interface SystemConfigThumbnailDto { */ 'webpSize': number; } + + /** * * @export diff --git a/web/src/lib/components/admin-page/settings/setting-switch.svelte b/web/src/lib/components/admin-page/settings/setting-switch.svelte index 6cc4166e9..8cd1f8483 100644 --- a/web/src/lib/components/admin-page/settings/setting-switch.svelte +++ b/web/src/lib/components/admin-page/settings/setting-switch.svelte @@ -1,12 +1,16 @@
@@ -29,7 +33,13 @@