feat(server): use tonemapx for software tone-mapping (#13785)

docs/docs/install/config-file.md

@@ -26,7 +26,6 @@ The default configuration looks like this:
     "bframes": -1,
     "refs": 0,
     "gopSize": 0,
-    "npl": 0,
     "temporalAQ": false,
     "cqMode": "auto",
     "twoPass": false,

i18n/en.json

@@ -305,8 +305,6 @@
     "transcoding_threads_description": "Higher values lead to faster encoding, but leave less room for the server to process other tasks while active. This value should not be more than the number of CPU cores. Maximizes utilization if set to 0.",
     "transcoding_tone_mapping": "Tone-mapping",
     "transcoding_tone_mapping_description": "Attempts to preserve the appearance of HDR videos when converted to SDR. Each algorithm makes different tradeoffs for color, detail and brightness. Hable preserves detail, Mobius preserves color, and Reinhard preserves brightness.",
-    "transcoding_tone_mapping_npl": "Tone-mapping NPL",
-    "transcoding_tone_mapping_npl_description": "Colors will be adjusted to look normal for a display of this brightness. Counter-intuitively, lower values increase the brightness of the video and vice versa since it compensates for the brightness of the display. 0 sets this value automatically.",
     "transcoding_transcode_policy": "Transcode policy",
     "transcoding_transcode_policy_description": "Policy for when a video should be transcoded. HDR videos will always be transcoded (except if transcoding is disabled).",
     "transcoding_two_pass_encoding": "Two-pass encoding",

mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart

@@ -23,7 +23,6 @@ class SystemConfigFFmpegDto {
     required this.crf,
     required this.gopSize,
     required this.maxBitrate,
-    required this.npl,
     required this.preferredHwDevice,
     required this.preset,
     required this.refs,
@@ -62,9 +61,6 @@ class SystemConfigFFmpegDto {
 
   String maxBitrate;
 
-  /// Minimum value: 0
-  int npl;
-
   String preferredHwDevice;
 
   String preset;
@@ -102,7 +98,6 @@ class SystemConfigFFmpegDto {
     other.crf == crf &&
     other.gopSize == gopSize &&
     other.maxBitrate == maxBitrate &&
-    other.npl == npl &&
     other.preferredHwDevice == preferredHwDevice &&
     other.preset == preset &&
     other.refs == refs &&
@@ -128,7 +123,6 @@ class SystemConfigFFmpegDto {
     (crf.hashCode) +
     (gopSize.hashCode) +
     (maxBitrate.hashCode) +
-    (npl.hashCode) +
     (preferredHwDevice.hashCode) +
     (preset.hashCode) +
     (refs.hashCode) +
@@ -142,7 +136,7 @@ class SystemConfigFFmpegDto {
     (twoPass.hashCode);
 
   @override
-  String toString() => 'SystemConfigFFmpegDto[accel=$accel, accelDecode=$accelDecode, acceptedAudioCodecs=$acceptedAudioCodecs, acceptedContainers=$acceptedContainers, acceptedVideoCodecs=$acceptedVideoCodecs, bframes=$bframes, cqMode=$cqMode, crf=$crf, gopSize=$gopSize, maxBitrate=$maxBitrate, npl=$npl, preferredHwDevice=$preferredHwDevice, preset=$preset, refs=$refs, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, targetVideoCodec=$targetVideoCodec, temporalAQ=$temporalAQ, threads=$threads, tonemap=$tonemap, transcode=$transcode, twoPass=$twoPass]';
+  String toString() => 'SystemConfigFFmpegDto[accel=$accel, accelDecode=$accelDecode, acceptedAudioCodecs=$acceptedAudioCodecs, acceptedContainers=$acceptedContainers, acceptedVideoCodecs=$acceptedVideoCodecs, bframes=$bframes, cqMode=$cqMode, crf=$crf, gopSize=$gopSize, maxBitrate=$maxBitrate, preferredHwDevice=$preferredHwDevice, preset=$preset, refs=$refs, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, targetVideoCodec=$targetVideoCodec, temporalAQ=$temporalAQ, threads=$threads, tonemap=$tonemap, transcode=$transcode, twoPass=$twoPass]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
@@ -156,7 +150,6 @@ class SystemConfigFFmpegDto {
       json[r'crf'] = this.crf;
       json[r'gopSize'] = this.gopSize;
       json[r'maxBitrate'] = this.maxBitrate;
-      json[r'npl'] = this.npl;
       json[r'preferredHwDevice'] = this.preferredHwDevice;
       json[r'preset'] = this.preset;
       json[r'refs'] = this.refs;
@@ -190,7 +183,6 @@ class SystemConfigFFmpegDto {
         crf: mapValueOfType<int>(json, r'crf')!,
         gopSize: mapValueOfType<int>(json, r'gopSize')!,
         maxBitrate: mapValueOfType<String>(json, r'maxBitrate')!,
-        npl: mapValueOfType<int>(json, r'npl')!,
         preferredHwDevice: mapValueOfType<String>(json, r'preferredHwDevice')!,
         preset: mapValueOfType<String>(json, r'preset')!,
         refs: mapValueOfType<int>(json, r'refs')!,
@@ -259,7 +251,6 @@ class SystemConfigFFmpegDto {
     'crf',
     'gopSize',
     'maxBitrate',
-    'npl',
     'preferredHwDevice',
     'preset',
     'refs',

open-api/immich-openapi-specs.json

@@ -11621,10 +11621,6 @@
           "maxBitrate": {
             "type": "string"
           },
-          "npl": {
-            "minimum": 0,
-            "type": "integer"
-          },
           "preferredHwDevice": {
             "type": "string"
           },
@@ -11673,7 +11669,6 @@
           "crf",
           "gopSize",
           "maxBitrate",
-          "npl",
           "preferredHwDevice",
           "preset",
           "refs",

open-api/typescript-sdk/src/fetch-client.ts

@@ -1104,7 +1104,6 @@ export type SystemConfigFFmpegDto = {
     crf: number;
     gopSize: number;
     maxBitrate: string;
-    npl: number;
     preferredHwDevice: string;
     preset: string;
     refs: number;

server/src/config.ts

@@ -36,7 +36,6 @@ export interface SystemConfig {
     bframes: number;
     refs: number;
     gopSize: number;
-    npl: number;
     temporalAQ: boolean;
     cqMode: CQMode;
     twoPass: boolean;
@@ -178,7 +177,6 @@ export const defaults = Object.freeze<SystemConfig>({
     bframes: -1,
     refs: 0,
     gopSize: 0,
-    npl: 0,
     temporalAQ: false,
     cqMode: CQMode.AUTO,
     twoPass: false,

server/src/dtos/system-config.dto.ts

@@ -134,12 +134,6 @@ export class SystemConfigFFmpegDto {
   @ApiProperty({ type: 'integer' })
   gopSize!: number;
 
-  @IsInt()
-  @Min(0)
-  @Type(() => Number)
-  @ApiProperty({ type: 'integer' })
-  npl!: number;
-
   @ValidateBoolean()
   temporalAQ!: boolean;
 

server/src/interfaces/media.interface.ts

@@ -59,6 +59,7 @@ export interface VideoStreamInfo {
   frameCount: number;
   isHDR: boolean;
   bitrate: number;
+  pixelFormat: string;
 }
 
 export interface AudioStreamInfo {

server/src/migrations/1730227312171-RemoveNplFromSystemConfig.ts

@@ -0,0 +1,12 @@
+import { MigrationInterface, QueryRunner } from 'typeorm';
+
+export class RemoveNplFromSystemConfig1730227312171 implements MigrationInterface {
+  public async up(queryRunner: QueryRunner): Promise<void> {
+    await queryRunner.query(`
+        update system_metadata
+        set value = value #- '{ffmpeg,npl}'
+        where key = 'system-config' and value->'ffmpeg'->'npl' is not null`);
+  }
+
+  public async down(): Promise<void> {}
+}

server/src/repositories/media.repository.ts

@@ -126,6 +126,7 @@ export class MediaRepository implements IMediaRepository {
           rotation: this.parseInt(stream.rotation),
           isHDR: stream.color_transfer === 'smpte2084' || stream.color_transfer === 'arib-std-b67',
           bitrate: this.parseInt(stream.bit_rate),
+          pixelFormat: stream.pix_fmt || 'yuv420p',
         })),
       audioStreams: results.streams
         .filter((stream) => stream.codec_type === 'audio')

server/src/services/media.service.spec.ts

@@ -9,7 +9,6 @@ import {
   AudioCodec,
   Colorspace,
   ImageFormat,
-  ToneMapping,
   TranscodeHWAccel,
   TranscodePolicy,
   VideoCodec,
@@ -410,7 +409,7 @@ describe(MediaService.name, () => {
             '-frames:v 1',
             '-update 1',
             '-v verbose',
-            String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:1440:flags=lanczos+accurate_rnd+full_chroma_int:out_color_matrix=bt601:out_range=pc,format=yuv420p`,
+            String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:1440:flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc`,
           ],
           twoPass: false,
         }),
@@ -445,7 +444,7 @@ describe(MediaService.name, () => {
             '-frames:v 1',
             '-update 1',
             '-v verbose',
-            String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=601:m=470bg:range=pc,format=yuv420p`,
+            String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`,
           ],
           twoPass: false,
         }),
@@ -482,7 +481,7 @@ describe(MediaService.name, () => {
             '-frames:v 1',
             '-update 1',
             '-v verbose',
-            String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=601:m=470bg:range=pc,format=yuv420p`,
+            String.raw`-vf fps=12:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`,
           ],
           twoPass: false,
         }),
@@ -1328,7 +1327,7 @@ describe(MediaService.name, () => {
             '-map 0:0',
             '-map 0:1',
             '-v verbose',
-            '-vf scale=-2:720,format=yuv420p',
+            '-vf scale=-2:720',
             '-preset 12',
             '-crf 23',
           ]),
@@ -1454,7 +1453,7 @@ describe(MediaService.name, () => {
             '-map 0:1',
             '-g 256',
             '-v verbose',
-            '-vf format=nv12,hwupload_cuda,scale_cuda=-2:720',
+            '-vf hwupload_cuda,scale_cuda=-2:720:format=nv12',
             '-preset p1',
             '-cq:v 23',
           ]),
@@ -1586,7 +1585,7 @@ describe(MediaService.name, () => {
           inputOptions: expect.arrayContaining(['-hwaccel cuda', '-hwaccel_output_format cuda']),
           outputOptions: expect.arrayContaining([
             expect.stringContaining(
-              'tonemap_cuda=desat=0:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709:format=nv12',
+              'tonemap_cuda=desat=0:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:tonemap_mode=lum:transfer=bt709:peak=100:format=nv12',
             ),
           ]),
           twoPass: false,
@@ -1594,6 +1593,24 @@ describe(MediaService.name, () => {
       );
     });
 
+    it('should set format to nv12 for nvenc if input is not yuv420p', async () => {
+      mediaMock.probe.mockResolvedValue(probeStub.videoStream10Bit);
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.NVENC, accelDecode: true },
+      });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        expect.objectContaining({
+          inputOptions: expect.arrayContaining(['-hwaccel cuda', '-hwaccel_output_format cuda']),
+          outputOptions: expect.arrayContaining([expect.stringContaining('format=nv12')]),
+          twoPass: false,
+        }),
+      );
+    });
+
     it('should set options for qsv', async () => {
       storageMock.readdir.mockResolvedValue(['renderD128']);
       mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
@@ -1616,7 +1633,7 @@ describe(MediaService.name, () => {
             '-refs 5',
             '-g 256',
             '-v verbose',
-            '-vf format=nv12,hwupload=extra_hw_frames=64,scale_qsv=-1:720:mode=hq',
+            '-vf hwupload=extra_hw_frames=64,scale_qsv=-1:720:mode=hq:format=nv12',
             '-preset 7',
             '-global_quality:v 23',
             '-maxrate 10000k',
@@ -1748,7 +1765,7 @@ describe(MediaService.name, () => {
           ]),
           outputOptions: expect.arrayContaining([
             expect.stringContaining(
-              'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709,hwmap=derive_device=qsv:reverse=1,format=qsv',
+              'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:transfer=bt709:range=pc:tonemap=hable:tonemap_mode=lum:peak=100,hwmap=derive_device=qsv:reverse=1,format=qsv',
             ),
           ]),
           twoPass: false,
@@ -1776,6 +1793,32 @@ describe(MediaService.name, () => {
       );
     });
 
+    it('should set format to nv12 for qsv if input is not yuv420p', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.videoStream10Bit);
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.QSV, accelDecode: true },
+      });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        expect.objectContaining({
+          inputOptions: expect.arrayContaining([
+            '-hwaccel qsv',
+            '-hwaccel_output_format qsv',
+            '-async_depth 4',
+            '-threads 1',
+          ]),
+          outputOptions: expect.arrayContaining([expect.stringContaining('format=nv12')]),
+          twoPass: false,
+        }),
+      );
+    });
+
     it('should set options for vaapi', async () => {
       storageMock.readdir.mockResolvedValue(['renderD128']);
       mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
@@ -1799,7 +1842,7 @@ describe(MediaService.name, () => {
             '-map 0:1',
             '-g 256',
             '-v verbose',
-            '-vf format=nv12,hwupload,scale_vaapi=-2:720:mode=hq:out_range=pc',
+            '-vf hwupload=extra_hw_frames=64,scale_vaapi=-2:720:mode=hq:out_range=pc:format=nv12',
             '-compression_level 7',
             '-rc_mode 1',
           ]),
@@ -1970,7 +2013,7 @@ describe(MediaService.name, () => {
       );
     });
 
-    it('should use hardware tone-mapping for qsv if hardware decoding is enabled and should tone map', async () => {
+    it('should use hardware tone-mapping for vaapi if hardware decoding is enabled and should tone map', async () => {
       storageMock.readdir.mockResolvedValue(['renderD128']);
       mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
       systemMock.get.mockResolvedValue({
@@ -1987,7 +2030,7 @@ describe(MediaService.name, () => {
           inputOptions: expect.arrayContaining(['-hwaccel vaapi', '-hwaccel_output_format vaapi', '-threads 1']),
           outputOptions: expect.arrayContaining([
             expect.stringContaining(
-              'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:range=pc:tonemap=hable:transfer=bt709,hwmap=derive_device=vaapi:reverse=1,format=vaapi',
+              'hwmap=derive_device=opencl,tonemap_opencl=desat=0:format=nv12:matrix=bt709:primaries=bt709:transfer=bt709:range=pc:tonemap=hable:tonemap_mode=lum:peak=100,hwmap=derive_device=vaapi:reverse=1,format=vaapi',
             ),
           ]),
           twoPass: false,
@@ -1995,6 +2038,27 @@ describe(MediaService.name, () => {
       );
     });
 
+    it('should set format to nv12 for vaapi if input is not yuv420p', async () => {
+      storageMock.readdir.mockResolvedValue(['renderD128']);
+      mediaMock.probe.mockResolvedValue(probeStub.videoStream10Bit);
+      systemMock.get.mockResolvedValue({
+        ffmpeg: { accel: TranscodeHWAccel.VAAPI, accelDecode: true },
+      });
+      assetMock.getByIds.mockResolvedValue([assetStub.video]);
+
+      await sut.handleVideoConversion({ id: assetStub.video.id });
+
+      expect(mediaMock.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        'upload/encoded-video/user-id/as/se/asset-id.mp4',
+        expect.objectContaining({
+          inputOptions: expect.arrayContaining(['-hwaccel vaapi', '-hwaccel_output_format vaapi', '-threads 1']),
+          outputOptions: expect.arrayContaining([expect.stringContaining('format=nv12')]),
+          twoPass: false,
+        }),
+      );
+    });
+
     it('should use preferred device for vaapi when hardware decoding', async () => {
       storageMock.readdir.mockResolvedValue(['renderD128', 'renderD129', 'renderD130']);
       mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer);
@@ -2140,7 +2204,7 @@ describe(MediaService.name, () => {
           inputOptions: expect.arrayContaining(['-hwaccel rkmpp', '-hwaccel_output_format drm_prime', '-afbc rga']),
           outputOptions: expect.arrayContaining([
             expect.stringContaining(
-              'scale_rkrga=-2:720:format=p010:afbc=1,hwmap=derive_device=opencl:mode=read,tonemap_opencl=format=nv12:r=pc:p=bt709:t=bt709:m=bt709:tonemap=hable:desat=0,hwmap=derive_device=rkmpp:mode=write:reverse=1,format=drm_prime',
+              'scale_rkrga=-2:720:format=p010:afbc=1,hwmap=derive_device=opencl:mode=read,tonemap_opencl=format=nv12:r=pc:p=bt709:t=bt709:m=bt709:tonemap=hable:desat=0:tonemap_mode=lum:peak=100,hwmap=derive_device=rkmpp:mode=write:reverse=1,format=drm_prime',
             ),
           ]),
           twoPass: false,
@@ -2164,7 +2228,7 @@ describe(MediaService.name, () => {
           inputOptions: [],
           outputOptions: expect.arrayContaining([
             expect.stringContaining(
-              'zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p',
+              'tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p',
             ),
           ]),
           twoPass: false,
@@ -2188,7 +2252,7 @@ describe(MediaService.name, () => {
           inputOptions: [],
           outputOptions: expect.arrayContaining([
             expect.stringContaining(
-              'zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p',
+              'tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p',
             ),
           ]),
           twoPass: false,
@@ -2209,7 +2273,7 @@ describe(MediaService.name, () => {
           outputOptions: expect.arrayContaining([
             '-c:v h264',
             '-c:a copy',
-            '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p',
+            '-vf tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p',
           ]),
           twoPass: false,
         }),
@@ -2229,16 +2293,16 @@ describe(MediaService.name, () => {
           outputOptions: expect.arrayContaining([
             '-c:v h264',
             '-c:a copy',
-            '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p',
+            '-vf tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p',
           ]),
           twoPass: false,
         }),
       );
     });
 
-    it('should set npl to 250 for reinhard and mobius tone-mapping algorithms', async () => {
+    it('should transcode when policy is required and video is not yuv420p', async () => {
-      mediaMock.probe.mockResolvedValue(probeStub.videoStreamHDR);
+      mediaMock.probe.mockResolvedValue(probeStub.videoStream10Bit);
-      systemMock.get.mockResolvedValue({ ffmpeg: { tonemap: ToneMapping.MOBIUS } });
+      systemMock.get.mockResolvedValue({ ffmpeg: { transcode: TranscodePolicy.REQUIRED } });
       assetMock.getByIds.mockResolvedValue([assetStub.video]);
       await sut.handleVideoConversion({ id: assetStub.video.id });
       expect(mediaMock.transcode).toHaveBeenCalledWith(
@@ -2246,11 +2310,7 @@ describe(MediaService.name, () => {
         'upload/encoded-video/user-id/as/se/asset-id.mp4',
         expect.objectContaining({
           inputOptions: expect.any(Array),
-          outputOptions: expect.arrayContaining([
+          outputOptions: expect.arrayContaining(['-c:v h264', '-c:a copy', '-vf format=yuv420p']),
-            '-c:v h264',
-            '-c:a copy',
-            '-vf zscale=t=linear:npl=250,tonemap=mobius:desat=0,zscale=p=709:t=709:m=709:range=pc,format=yuv420p',
-          ]),
           twoPass: false,
         }),
       );

server/src/services/media.service.ts

@@ -413,7 +413,7 @@ export class MediaService extends BaseService {
     const isLargerThanTargetBitrate = stream.bitrate > this.parseBitrateToBps(ffmpegConfig.maxBitrate);
 
     const isTargetVideoCodec = ffmpegConfig.acceptedVideoCodecs.includes(stream.codecName as VideoCodec);
-    const isRequired = !isTargetVideoCodec || stream.isHDR;
+    const isRequired = !isTargetVideoCodec || !stream.pixelFormat.endsWith('420p');
 
     switch (ffmpegConfig.transcode) {
       case TranscodePolicy.DISABLED: {

server/src/services/system-config.service.spec.ts

@@ -65,7 +65,6 @@ const updatedConfig = Object.freeze<SystemConfig>({
     bframes: -1,
     refs: 0,
     gopSize: 0,
-    npl: 0,
     temporalAQ: false,
     cqMode: CQMode.AUTO,
     twoPass: false,

server/src/utils/media.ts

@@ -149,7 +149,11 @@ export class BaseConfig implements VideoCodecSWConfig {
       options.push(`scale=${this.getScaling(videoStream)}`);
     }
 
-    options.push(...this.getToneMapping(videoStream), 'format=yuv420p');
+    options.push(...this.getToneMapping(videoStream));
+    if (options.length === 0 && !videoStream.pixelFormat.endsWith('420p')) {
+      options.push(`format=yuv420p`);
+    }
+
     return options;
   }
 
@@ -271,33 +275,20 @@ export class BaseConfig implements VideoCodecSWConfig {
 
   getColors() {
     return {
-      primaries: '709',
+      primaries: 'bt709',
-      transfer: '709',
+      transfer: 'bt709',
-      matrix: '709',
+      matrix: 'bt709',
     };
   }
 
-  getNPL() {
-    if (this.config.npl <= 0) {
-      // since hable already outputs a darker image, we use a lower npl value for it
-      return this.config.tonemap === ToneMapping.HABLE ? 100 : 250;
-    } else {
-      return this.config.npl;
-    }
-  }
-
   getToneMapping(videoStream: VideoStreamInfo) {
     if (!this.shouldToneMap(videoStream)) {
       return [];
     }
 
-    const colors = this.getColors();
+    const { primaries, transfer, matrix } = this.getColors();
-
+    const options = `tonemapx=tonemap=${this.config.tonemap}:desat=0:p=${primaries}:t=${transfer}:m=${matrix}:r=pc:peak=100:format=yuv420p`;
-    return [
+    return [options];
-      `zscale=t=linear:npl=${this.getNPL()}`,
-      `tonemap=${this.config.tonemap}:desat=0`,
-      `zscale=p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:range=pc`,
-    ];
   }
 
   getAudioCodec(): string {
@@ -395,19 +386,14 @@ export class ThumbnailConfig extends BaseConfig {
   }
 
   getFilterOptions(videoStream: VideoStreamInfo): string[] {
-    const options = [
+    return [
       'fps=12:eof_action=pass:round=down',
       'thumbnail=12',
       String.raw`select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20)`,
       'trim=end_frame=2',
       'reverse',
+      ...super.getFilterOptions(videoStream),
     ];
-    if (this.shouldScale(videoStream)) {
-      options.push(`scale=${this.getScaling(videoStream)}`);
-    }
-
-    options.push(...this.getToneMapping(videoStream), 'format=yuv420p');
-    return options;
   }
 
   getPresetOptions() {
@@ -423,19 +409,7 @@ export class ThumbnailConfig extends BaseConfig {
   }
 
   getScaling(videoStream: VideoStreamInfo) {
-    let options = super.getScaling(videoStream) + ':flags=lanczos+accurate_rnd+full_chroma_int';
+    return super.getScaling(videoStream) + ':flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc';
-    if (!this.shouldToneMap(videoStream)) {
-      options += ':out_color_matrix=bt601:out_range=pc';
-    }
-    return options;
-  }
-
-  getColors() {
-    return {
-      primaries: '709',
-      transfer: '601',
-      matrix: '470bg',
-    };
   }
 }
 
@@ -559,9 +533,9 @@ export class NvencSwDecodeConfig extends BaseHWConfig {
 
   getFilterOptions(videoStream: VideoStreamInfo) {
     const options = this.getToneMapping(videoStream);
-    options.push('format=nv12', 'hwupload_cuda');
+    options.push('hwupload_cuda');
     if (this.shouldScale(videoStream)) {
-      options.push(`scale_cuda=${this.getScaling(videoStream)}`);
+      options.push(`scale_cuda=${this.getScaling(videoStream)}:format=nv12`);
     }
 
     return options;
@@ -622,6 +596,8 @@ export class NvencHwDecodeConfig extends NvencSwDecodeConfig {
     options.push(...this.getToneMapping(videoStream));
     if (options.length > 0) {
       options[options.length - 1] += ':format=nv12';
+    } else if (!videoStream.pixelFormat.endsWith('420p')) {
+      options.push('format=nv12');
     }
     return options;
   }
@@ -631,14 +607,16 @@ export class NvencHwDecodeConfig extends NvencSwDecodeConfig {
       return [];
     }
 
-    const colors = this.getColors();
+    const { matrix, primaries, transfer } = this.getColors();
     const tonemapOptions = [
       'desat=0',
-      `matrix=${colors.matrix}`,
+      `matrix=${matrix}`,
-      `primaries=${colors.primaries}`,
+      `primaries=${primaries}`,
       'range=pc',
       `tonemap=${this.config.tonemap}`,
-      `transfer=${colors.transfer}`,
+      'tonemap_mode=lum',
+      `transfer=${transfer}`,
+      'peak=100',
     ];
 
     return [`tonemap_cuda=${tonemapOptions.join(':')}`];
@@ -651,14 +629,6 @@ export class NvencHwDecodeConfig extends NvencSwDecodeConfig {
   getOutputThreadOptions() {
     return [];
   }
-
-  getColors() {
-    return {
-      primaries: 'bt709',
-      transfer: 'bt709',
-      matrix: 'bt709',
-    };
-  }
 }
 
 export class QsvSwDecodeConfig extends BaseHWConfig {
@@ -687,9 +657,9 @@ export class QsvSwDecodeConfig extends BaseHWConfig {
 
   getFilterOptions(videoStream: VideoStreamInfo) {
     const options = this.getToneMapping(videoStream);
-    options.push('format=nv12', 'hwupload=extra_hw_frames=64');
+    options.push('hwupload=extra_hw_frames=64');
     if (this.shouldScale(videoStream)) {
-      options.push(`scale_qsv=${this.getScaling(videoStream)}:mode=hq`);
+      options.push(`scale_qsv=${this.getScaling(videoStream)}:mode=hq:format=nv12`);
     }
     return options;
   }
@@ -764,15 +734,18 @@ export class QsvHwDecodeConfig extends QsvSwDecodeConfig {
 
   getFilterOptions(videoStream: VideoStreamInfo) {
     const options = [];
-    if (this.shouldScale(videoStream) || !this.shouldToneMap(videoStream)) {
+    const tonemapOptions = this.getToneMapping(videoStream);
+    if (this.shouldScale(videoStream) || tonemapOptions.length === 0) {
       let scaling = `scale_qsv=${this.getScaling(videoStream)}:async_depth=4:mode=hq`;
-      if (!this.shouldToneMap(videoStream)) {
+      if (tonemapOptions.length === 0) {
         scaling += ':format=nv12';
       }
       options.push(scaling);
     }
-
+    options.push(...tonemapOptions);
-    options.push(...this.getToneMapping(videoStream));
+    if (options.length === 0 && !videoStream.pixelFormat.endsWith('420p')) {
+      options.push('format=nv12');
+    }
     return options;
   }
 
@@ -781,15 +754,17 @@ export class QsvHwDecodeConfig extends QsvSwDecodeConfig {
       return [];
     }
 
-    const colors = this.getColors();
+    const { matrix, primaries, transfer } = this.getColors();
     const tonemapOptions = [
       'desat=0',
       'format=nv12',
-      `matrix=${colors.matrix}`,
+      `matrix=${matrix}`,
-      `primaries=${colors.primaries}`,
+      `primaries=${primaries}`,
+      `transfer=${transfer}`,
       'range=pc',
       `tonemap=${this.config.tonemap}`,
-      `transfer=${colors.transfer}`,
+      'tonemap_mode=lum',
+      'peak=100',
     ];
 
     return [
@@ -802,14 +777,6 @@ export class QsvHwDecodeConfig extends QsvSwDecodeConfig {
   getInputThreadOptions() {
     return [`-threads 1`];
   }
-
-  getColors() {
-    return {
-      primaries: 'bt709',
-      transfer: 'bt709',
-      matrix: 'bt709',
-    };
-  }
 }
 
 export class VaapiSwDecodeConfig extends BaseHWConfig {
@@ -828,9 +795,9 @@ export class VaapiSwDecodeConfig extends BaseHWConfig {
 
   getFilterOptions(videoStream: VideoStreamInfo) {
     const options = this.getToneMapping(videoStream);
-    options.push('format=nv12', 'hwupload');
+    options.push('hwupload=extra_hw_frames=64');
     if (this.shouldScale(videoStream)) {
-      options.push(`scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc`);
+      options.push(`scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc:format=nv12`);
     }
 
     return options;
@@ -901,15 +868,18 @@ export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig {
 
   getFilterOptions(videoStream: VideoStreamInfo) {
     const options = [];
-    if (this.shouldScale(videoStream) || !this.shouldToneMap(videoStream)) {
+    const tonemapOptions = this.getToneMapping(videoStream);
+    if (this.shouldScale(videoStream) || tonemapOptions.length === 0) {
       let scaling = `scale_vaapi=${this.getScaling(videoStream)}:mode=hq:out_range=pc`;
-      if (!this.shouldToneMap(videoStream)) {
+      if (tonemapOptions.length === 0) {
         scaling += ':format=nv12';
       }
       options.push(scaling);
     }
-
+    options.push(...tonemapOptions);
-    options.push(...this.getToneMapping(videoStream));
+    if (options.length === 0 && !videoStream.pixelFormat.endsWith('420p')) {
+      options.push('format=nv12');
+    }
     return options;
   }
 
@@ -918,15 +888,17 @@ export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig {
       return [];
     }
 
-    const colors = this.getColors();
+    const { matrix, primaries, transfer } = this.getColors();
     const tonemapOptions = [
       'desat=0',
       'format=nv12',
-      `matrix=${colors.matrix}`,
+      `matrix=${matrix}`,
-      `primaries=${colors.primaries}`,
+      `primaries=${primaries}`,
+      `transfer=${transfer}`,
       'range=pc',
       `tonemap=${this.config.tonemap}`,
-      `transfer=${colors.transfer}`,
+      'tonemap_mode=lum',
+      'peak=100',
     ];
 
     return [
@@ -939,14 +911,6 @@ export class VaapiHwDecodeConfig extends VaapiSwDecodeConfig {
   getInputThreadOptions() {
     return [`-threads 1`];
   }
-
-  getColors() {
-    return {
-      primaries: 'bt709',
-      transfer: 'bt709',
-      matrix: 'bt709',
-    };
-  }
 }
 
 export class RkmppSwDecodeConfig extends BaseHWConfig {
@@ -1014,11 +978,11 @@ export class RkmppHwDecodeConfig extends RkmppSwDecodeConfig {
 
   getFilterOptions(videoStream: VideoStreamInfo) {
     if (this.shouldToneMap(videoStream)) {
-      const colors = this.getColors();
+      const { primaries, transfer, matrix } = this.getColors();
       return [
         `scale_rkrga=${this.getScaling(videoStream)}:format=p010:afbc=1`,
         'hwmap=derive_device=opencl:mode=read',
-        `tonemap_opencl=format=nv12:r=pc:p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:tonemap=${this.config.tonemap}:desat=0`,
+        `tonemap_opencl=format=nv12:r=pc:p=${primaries}:t=${transfer}:m=${matrix}:tonemap=${this.config.tonemap}:desat=0:tonemap_mode=lum:peak=100`,
         'hwmap=derive_device=rkmpp:mode=write:reverse=1',
         'format=drm_prime',
       ];
@@ -1027,12 +991,4 @@ export class RkmppHwDecodeConfig extends RkmppSwDecodeConfig {
     }
     return [];
   }
-
-  getColors() {
-    return {
-      primaries: 'bt709',
-      transfer: 'bt709',
-      matrix: 'bt709',
-    };
-  }
 }

server/test/fixtures/media.stub.ts

@@ -17,6 +17,7 @@ const probeStubDefaultVideoStream: VideoStreamInfo[] = [
     rotation: 0,
     isHDR: false,
     bitrate: 0,
+    pixelFormat: 'yuv420p',
   },
 ];
 
@@ -43,6 +44,7 @@ export const probeStub = {
         rotation: 0,
         isHDR: false,
         bitrate: 0,
+        pixelFormat: 'yuv420p',
       },
       {
         index: 1,
@@ -53,6 +55,7 @@ export const probeStub = {
         rotation: 0,
         isHDR: false,
         bitrate: 0,
+        pixelFormat: 'yuv420p',
       },
     ],
   }),
@@ -68,6 +71,7 @@ export const probeStub = {
         rotation: 0,
         isHDR: false,
         bitrate: 0,
+        pixelFormat: 'yuv420p',
       },
     ],
   }),
@@ -83,6 +87,7 @@ export const probeStub = {
         rotation: 0,
         isHDR: false,
         bitrate: 0,
+        pixelFormat: 'yuv420p',
       },
     ],
   }),
@@ -102,6 +107,23 @@ export const probeStub = {
         rotation: 0,
         isHDR: true,
         bitrate: 0,
+        pixelFormat: 'yuv420p10le',
+      },
+    ],
+  }),
+  videoStream10Bit: Object.freeze<VideoInfo>({
+    ...probeStubDefault,
+    videoStreams: [
+      {
+        index: 0,
+        height: 480,
+        width: 480,
+        codecName: 'h264',
+        frameCount: 100,
+        rotation: 0,
+        isHDR: false,
+        bitrate: 0,
+        pixelFormat: 'yuv420p10le',
       },
     ],
   }),
@@ -117,6 +139,7 @@ export const probeStub = {
         rotation: 90,
         isHDR: false,
         bitrate: 0,
+        pixelFormat: 'yuv420p',
       },
     ],
   }),
@@ -132,6 +155,7 @@ export const probeStub = {
         rotation: 0,
         isHDR: false,
         bitrate: 0,
+        pixelFormat: 'yuv420p',
       },
     ],
   }),
@@ -147,6 +171,7 @@ export const probeStub = {
         rotation: 0,
         isHDR: false,
         bitrate: 0,
+        pixelFormat: 'yuv420p',
       },
     ],
   }),

web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte

@@ -343,15 +343,6 @@
           subtitle={$t('admin.transcoding_advanced_options_description')}
         >
           <div class="ml-4 mt-4 flex flex-col gap-4">
-            <SettingInputField
-              inputType={SettingInputFieldType.NUMBER}
-              label={$t('admin.transcoding_tone_mapping_npl')}
-              desc={$t('admin.transcoding_tone_mapping_npl_description')}
-              bind:value={config.ffmpeg.npl}
-              isEdited={config.ffmpeg.npl !== savedConfig.ffmpeg.npl}
-              {disabled}
-            />
-
             <SettingInputField
               inputType={SettingInputFieldType.NUMBER}
               label={$t('admin.transcoding_max_b_frames')}