feat(all): ffmpeg quality options improvements (#2161)

* feat: change target scaling to resolution in ffmpeg config

* feat(microservices): scale vertical video correctly, only scale if video is larger than target

mobile/openapi/doc/SystemConfigFFmpegDto.md

@@ -12,7 +12,7 @@ Name | Type | Description | Notes
 **preset** | **String** |  | 
 **targetVideoCodec** | **String** |  | 
 **targetAudioCodec** | **String** |  | 
-**targetScaling** | **String** |  | 
+**targetResolution** | **String** |  | 
 **transcode** | **String** |  | 
 
 [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md)

mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart

@@ -17,7 +17,7 @@ class SystemConfigFFmpegDto {
     required this.preset,
     required this.targetVideoCodec,
     required this.targetAudioCodec,
-    required this.targetScaling,
+    required this.targetResolution,
     required this.transcode,
   });
 
@@ -29,7 +29,7 @@ class SystemConfigFFmpegDto {
 
   String targetAudioCodec;
 
-  String targetScaling;
+  String targetResolution;
 
   SystemConfigFFmpegDtoTranscodeEnum transcode;
 
@@ -39,7 +39,7 @@ class SystemConfigFFmpegDto {
      other.preset == preset &&
      other.targetVideoCodec == targetVideoCodec &&
      other.targetAudioCodec == targetAudioCodec &&
-     other.targetScaling == targetScaling &&
+     other.targetResolution == targetResolution &&
      other.transcode == transcode;
 
   @override
@@ -49,11 +49,11 @@ class SystemConfigFFmpegDto {
     (preset.hashCode) +
     (targetVideoCodec.hashCode) +
     (targetAudioCodec.hashCode) +
-    (targetScaling.hashCode) +
+    (targetResolution.hashCode) +
     (transcode.hashCode);
 
   @override
-  String toString() => 'SystemConfigFFmpegDto[crf=$crf, preset=$preset, targetVideoCodec=$targetVideoCodec, targetAudioCodec=$targetAudioCodec, targetScaling=$targetScaling, transcode=$transcode]';
+  String toString() => 'SystemConfigFFmpegDto[crf=$crf, preset=$preset, targetVideoCodec=$targetVideoCodec, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, transcode=$transcode]';
 
   Map<String, dynamic> toJson() {
     final json = <String, dynamic>{};
@@ -61,7 +61,7 @@ class SystemConfigFFmpegDto {
       json[r'preset'] = this.preset;
       json[r'targetVideoCodec'] = this.targetVideoCodec;
       json[r'targetAudioCodec'] = this.targetAudioCodec;
-      json[r'targetScaling'] = this.targetScaling;
+      json[r'targetResolution'] = this.targetResolution;
       json[r'transcode'] = this.transcode;
     return json;
   }
@@ -89,7 +89,7 @@ class SystemConfigFFmpegDto {
         preset: mapValueOfType<String>(json, r'preset')!,
         targetVideoCodec: mapValueOfType<String>(json, r'targetVideoCodec')!,
         targetAudioCodec: mapValueOfType<String>(json, r'targetAudioCodec')!,
-        targetScaling: mapValueOfType<String>(json, r'targetScaling')!,
+        targetResolution: mapValueOfType<String>(json, r'targetResolution')!,
         transcode: SystemConfigFFmpegDtoTranscodeEnum.fromJson(json[r'transcode'])!,
       );
     }
@@ -144,7 +144,7 @@ class SystemConfigFFmpegDto {
     'preset',
     'targetVideoCodec',
     'targetAudioCodec',
-    'targetScaling',
+    'targetResolution',
     'transcode',
   };
 }

mobile/openapi/test/system_config_f_fmpeg_dto_test.dart

@@ -36,8 +36,8 @@ void main() {
       // TODO
     });
 
-    // String targetScaling
+    // String targetResolution
-    test('to test the property `targetScaling`', () async {
+    test('to test the property `targetResolution`', () async {
       // TODO
     });
 

server/apps/microservices/src/processors/video-transcode.processor.ts

@@ -16,7 +16,7 @@ import { AssetEntity, AssetType, TranscodePreset } from '@app/infra/entities';
 import { Process, Processor } from '@nestjs/bull';
 import { Inject, Logger } from '@nestjs/common';
 import { Job } from 'bull';
-import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
+import ffmpeg, { FfprobeData, FfprobeStream } from 'fluent-ffmpeg';
 import { join } from 'path';
 
 @Processor(QueueName.VIDEO_CONVERSION)
@@ -74,22 +74,22 @@ export class VideoTranscodeProcessor {
 
   async runVideoEncode(asset: AssetEntity, savedEncodedPath: string): Promise<void> {
     const config = await this.systemConfigService.getConfig();
+    const videoStream = await this.getVideoStream(asset);
 
-    const transcode = await this.needsTranscoding(asset, config.ffmpeg);
+    const transcode = await this.needsTranscoding(videoStream, config.ffmpeg);
     if (transcode) {
       //TODO: If video or audio are already the correct format, don't re-encode, copy the stream
-      return this.runFFMPEGPipeLine(asset, savedEncodedPath);
+      return this.runFFMPEGPipeLine(asset, videoStream, savedEncodedPath);
     }
   }
 
-  async needsTranscoding(asset: AssetEntity, ffmpegConfig: SystemConfigFFmpegDto): Promise<boolean> {
+  async needsTranscoding(videoStream: FfprobeStream, ffmpegConfig: SystemConfigFFmpegDto): Promise<boolean> {
     switch (ffmpegConfig.transcode) {
       case TranscodePreset.ALL:
         return true;
 
       case TranscodePreset.REQUIRED:
         {
-          const videoStream = await this.getVideoStream(asset);
           if (videoStream.codec_name !== ffmpegConfig.targetVideoCodec) {
             return true;
           }
@@ -97,12 +97,13 @@ export class VideoTranscodeProcessor {
         break;
 
       case TranscodePreset.OPTIMAL: {
-        const videoStream = await this.getVideoStream(asset);
         if (videoStream.codec_name !== ffmpegConfig.targetVideoCodec) {
           return true;
         }
 
-        const videoHeightThreshold = 1080;
+        const config = await this.systemConfigService.getConfig();
+
+        const videoHeightThreshold = Number.parseInt(config.ffmpeg.targetResolution);
         return !videoStream.height || videoStream.height > videoHeightThreshold;
       }
     }
@@ -125,22 +126,45 @@ export class VideoTranscodeProcessor {
     return longestVideoStream;
   }
 
-  async runFFMPEGPipeLine(asset: AssetEntity, savedEncodedPath: string): Promise<void> {
+  async runFFMPEGPipeLine(asset: AssetEntity, videoStream: FfprobeStream, savedEncodedPath: string): Promise<void> {
     const config = await this.systemConfigService.getConfig();
 
-    return new Promise((resolve, reject) => {
+    const ffmpegOptions = [
-      ffmpeg(asset.originalPath)
-        .outputOptions([
       `-crf ${config.ffmpeg.crf}`,
       `-preset ${config.ffmpeg.preset}`,
       `-vcodec ${config.ffmpeg.targetVideoCodec}`,
       `-acodec ${config.ffmpeg.targetAudioCodec}`,
-          `-vf scale=${config.ffmpeg.targetScaling}`,
-
       // Makes a second pass moving the moov atom to the beginning of
       // the file for improved playback speed.
       `-movflags faststart`,
-        ])
+    ];
+
+    if (!videoStream.height || !videoStream.width) {
+      this.logger.error('Height or width undefined for video stream');
+      return;
+    }
+
+    const streamHeight = videoStream.height;
+    const streamWidth = videoStream.width;
+
+    const targetResolution = Number.parseInt(config.ffmpeg.targetResolution);
+
+    let scaling = `-2:${targetResolution}`;
+    const shouldScale = Math.min(streamHeight, streamWidth) > targetResolution;
+
+    const videoIsRotated = Math.abs(Number.parseInt(`${videoStream.rotation ?? 0}`)) === 90;
+
+    if (streamHeight > streamWidth || videoIsRotated) {
+      scaling = `${targetResolution}:-2`;
+    }
+
+    if (shouldScale) {
+      ffmpegOptions.push(`-vf scale=${scaling}`);
+    }
+
+    return new Promise((resolve, reject) => {
+      ffmpeg(asset.originalPath)
+        .outputOptions(ffmpegOptions)
         .output(savedEncodedPath)
         .on('start', () => {
           this.logger.log('Start Converting Video');

server/immich-openapi-specs.json

@@ -4644,7 +4644,7 @@
           "targetAudioCodec": {
             "type": "string"
           },
-          "targetScaling": {
+          "targetResolution": {
             "type": "string"
           },
           "transcode": {
@@ -4661,7 +4661,7 @@
           "preset",
           "targetVideoCodec",
           "targetAudioCodec",
-          "targetScaling",
+          "targetResolution",
           "transcode"
         ]
       },

server/libs/domain/src/system-config/dto/system-config-ffmpeg.dto.ts

@@ -15,7 +15,7 @@ export class SystemConfigFFmpegDto {
   targetAudioCodec!: string;
 
   @IsString()
-  targetScaling!: string;
+  targetResolution!: string;
 
   @IsEnum(TranscodePreset)
   transcode!: TranscodePreset;

server/libs/domain/src/system-config/system-config.core.ts

@@ -13,7 +13,7 @@ const defaults: SystemConfig = Object.freeze({
     preset: 'ultrafast',
     targetVideoCodec: 'h264',
     targetAudioCodec: 'aac',
-    targetScaling: '1280:-2',
+    targetResolution: '720',
     transcode: TranscodePreset.REQUIRED,
   },
   oauth: {

server/libs/domain/src/system-config/system-config.service.spec.ts

@@ -16,7 +16,7 @@ const updatedConfig = Object.freeze({
     crf: 'a new value',
     preset: 'ultrafast',
     targetAudioCodec: 'aac',
-    targetScaling: '1280:-2',
+    targetResolution: '720',
     targetVideoCodec: 'h264',
     transcode: TranscodePreset.REQUIRED,
   },

server/libs/domain/test/fixtures.ts

@@ -401,7 +401,7 @@ export const systemConfigStub = {
       crf: '23',
       preset: 'ultrafast',
       targetAudioCodec: 'aac',
-      targetScaling: '1280:-2',
+      targetResolution: '720',
       targetVideoCodec: 'h264',
       transcode: TranscodePreset.REQUIRED,
     },

server/libs/infra/src/entities/system-config.entity.ts

@@ -17,7 +17,7 @@ export enum SystemConfigKey {
   FFMPEG_PRESET = 'ffmpeg.preset',
   FFMPEG_TARGET_VIDEO_CODEC = 'ffmpeg.targetVideoCodec',
   FFMPEG_TARGET_AUDIO_CODEC = 'ffmpeg.targetAudioCodec',
-  FFMPEG_TARGET_SCALING = 'ffmpeg.targetScaling',
+  FFMPEG_TARGET_RESOLUTION = 'ffmpeg.targetResolution',
   FFMPEG_TRANSCODE = 'ffmpeg.transcode',
   OAUTH_ENABLED = 'oauth.enabled',
   OAUTH_ISSUER_URL = 'oauth.issuerUrl',
@@ -45,7 +45,7 @@ export interface SystemConfig {
     preset: string;
     targetVideoCodec: string;
     targetAudioCodec: string;
-    targetScaling: string;
+    targetResolution: string;
     transcode: TranscodePreset;
   };
   oauth: {

web/src/api/open-api/api.ts

@@ -2034,7 +2034,7 @@ export interface SystemConfigFFmpegDto {
      * @type {string}
      * @memberof SystemConfigFFmpegDto
      */
-    'targetScaling': string;
+    'targetResolution': string;
     /**
      * 
      * @type {string}

web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte

@@ -113,12 +113,18 @@
 						isEdited={!(ffmpegConfig.targetVideoCodec == savedConfig.targetVideoCodec)}
 					/>
 
-					<SettingInputField
+					<SettingSelect
-						inputType={SettingInputFieldType.TEXT}
+						label="TARGET RESOLUTION"
-						label="SCALING (-vf scale=)"
+						bind:value={ffmpegConfig.targetResolution}
-						bind:value={ffmpegConfig.targetScaling}
+						options={[
-						required={true}
+							{ value: '2160', text: '4k' },
-						isEdited={!(ffmpegConfig.targetScaling == savedConfig.targetScaling)}
+							{ value: '1440', text: '1440p' },
+							{ value: '1080', text: '1080p' },
+							{ value: '720', text: '720p' },
+							{ value: '480', text: '480p' }
+						]}
+						name="resolution"
+						isEdited={!(ffmpegConfig.targetResolution == savedConfig.targetResolution)}
 					/>
 
 					<SettingSelect
@@ -129,7 +135,7 @@
 							{ value: SystemConfigFFmpegDtoTranscodeEnum.All, text: 'All videos' },
 							{
 								value: SystemConfigFFmpegDtoTranscodeEnum.Optimal,
-								text: 'Videos higher than 1080p or not in the desired format'
+								text: 'Videos higher than target resolution or not in the desired format'
 							},
 							{
 								value: SystemConfigFFmpegDtoTranscodeEnum.Required,