From 808d6423be90e48b968c23d719638cc6c02fa459 Mon Sep 17 00:00:00 2001 From: Zack Pollard Date: Tue, 4 Apr 2023 02:42:53 +0100 Subject: [PATCH] feat(all): ffmpeg quality options improvements (#2161) * feat: change target scaling to resolution in ffmpeg config * feat(microservices): scale vertical video correctly, only scale if video is larger than target --- mobile/openapi/doc/SystemConfigFFmpegDto.md | 2 +- .../lib/model/system_config_f_fmpeg_dto.dart | 16 ++--- .../test/system_config_f_fmpeg_dto_test.dart | 4 +- .../processors/video-transcode.processor.ts | 62 +++++++++++++------ server/immich-openapi-specs.json | 4 +- .../dto/system-config-ffmpeg.dto.ts | 2 +- .../src/system-config/system-config.core.ts | 2 +- .../system-config.service.spec.ts | 2 +- server/libs/domain/test/fixtures.ts | 2 +- .../src/entities/system-config.entity.ts | 4 +- web/src/api/open-api/api.ts | 2 +- .../settings/ffmpeg/ffmpeg-settings.svelte | 20 +++--- 12 files changed, 76 insertions(+), 46 deletions(-) diff --git a/mobile/openapi/doc/SystemConfigFFmpegDto.md b/mobile/openapi/doc/SystemConfigFFmpegDto.md index 11ccb1fa0b..d442d43f8c 100644 --- a/mobile/openapi/doc/SystemConfigFFmpegDto.md +++ b/mobile/openapi/doc/SystemConfigFFmpegDto.md @@ -12,7 +12,7 @@ Name | Type | Description | Notes **preset** | **String** | | **targetVideoCodec** | **String** | | **targetAudioCodec** | **String** | | -**targetScaling** | **String** | | +**targetResolution** | **String** | | **transcode** | **String** | | [[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) diff --git a/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart b/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart index d9e1ad6696..a99e855d7e 100644 --- a/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart +++ b/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart @@ -17,7 +17,7 @@ class SystemConfigFFmpegDto { required this.preset, required this.targetVideoCodec, required this.targetAudioCodec, - required this.targetScaling, + required this.targetResolution, required this.transcode, }); @@ -29,7 +29,7 @@ class SystemConfigFFmpegDto { String targetAudioCodec; - String targetScaling; + String targetResolution; SystemConfigFFmpegDtoTranscodeEnum transcode; @@ -39,7 +39,7 @@ class SystemConfigFFmpegDto { other.preset == preset && other.targetVideoCodec == targetVideoCodec && other.targetAudioCodec == targetAudioCodec && - other.targetScaling == targetScaling && + other.targetResolution == targetResolution && other.transcode == transcode; @override @@ -49,11 +49,11 @@ class SystemConfigFFmpegDto { (preset.hashCode) + (targetVideoCodec.hashCode) + (targetAudioCodec.hashCode) + - (targetScaling.hashCode) + + (targetResolution.hashCode) + (transcode.hashCode); @override - String toString() => 'SystemConfigFFmpegDto[crf=$crf, preset=$preset, targetVideoCodec=$targetVideoCodec, targetAudioCodec=$targetAudioCodec, targetScaling=$targetScaling, transcode=$transcode]'; + String toString() => 'SystemConfigFFmpegDto[crf=$crf, preset=$preset, targetVideoCodec=$targetVideoCodec, targetAudioCodec=$targetAudioCodec, targetResolution=$targetResolution, transcode=$transcode]'; Map toJson() { final json = {}; @@ -61,7 +61,7 @@ class SystemConfigFFmpegDto { json[r'preset'] = this.preset; json[r'targetVideoCodec'] = this.targetVideoCodec; json[r'targetAudioCodec'] = this.targetAudioCodec; - json[r'targetScaling'] = this.targetScaling; + json[r'targetResolution'] = this.targetResolution; json[r'transcode'] = this.transcode; return json; } @@ -89,7 +89,7 @@ class SystemConfigFFmpegDto { preset: mapValueOfType(json, r'preset')!, targetVideoCodec: mapValueOfType(json, r'targetVideoCodec')!, targetAudioCodec: mapValueOfType(json, r'targetAudioCodec')!, - targetScaling: mapValueOfType(json, r'targetScaling')!, + targetResolution: mapValueOfType(json, r'targetResolution')!, transcode: SystemConfigFFmpegDtoTranscodeEnum.fromJson(json[r'transcode'])!, ); } @@ -144,7 +144,7 @@ class SystemConfigFFmpegDto { 'preset', 'targetVideoCodec', 'targetAudioCodec', - 'targetScaling', + 'targetResolution', 'transcode', }; } diff --git a/mobile/openapi/test/system_config_f_fmpeg_dto_test.dart b/mobile/openapi/test/system_config_f_fmpeg_dto_test.dart index 62297cb2bb..dfbb791244 100644 --- a/mobile/openapi/test/system_config_f_fmpeg_dto_test.dart +++ b/mobile/openapi/test/system_config_f_fmpeg_dto_test.dart @@ -36,8 +36,8 @@ void main() { // TODO }); - // String targetScaling - test('to test the property `targetScaling`', () async { + // String targetResolution + test('to test the property `targetResolution`', () async { // TODO }); diff --git a/server/apps/microservices/src/processors/video-transcode.processor.ts b/server/apps/microservices/src/processors/video-transcode.processor.ts index 76b4b5feec..067bd49cc2 100644 --- a/server/apps/microservices/src/processors/video-transcode.processor.ts +++ b/server/apps/microservices/src/processors/video-transcode.processor.ts @@ -16,7 +16,7 @@ import { AssetEntity, AssetType, TranscodePreset } from '@app/infra/entities'; import { Process, Processor } from '@nestjs/bull'; import { Inject, Logger } from '@nestjs/common'; import { Job } from 'bull'; -import ffmpeg, { FfprobeData } from 'fluent-ffmpeg'; +import ffmpeg, { FfprobeData, FfprobeStream } from 'fluent-ffmpeg'; import { join } from 'path'; @Processor(QueueName.VIDEO_CONVERSION) @@ -74,22 +74,22 @@ export class VideoTranscodeProcessor { async runVideoEncode(asset: AssetEntity, savedEncodedPath: string): Promise { const config = await this.systemConfigService.getConfig(); + const videoStream = await this.getVideoStream(asset); - const transcode = await this.needsTranscoding(asset, config.ffmpeg); + const transcode = await this.needsTranscoding(videoStream, config.ffmpeg); if (transcode) { //TODO: If video or audio are already the correct format, don't re-encode, copy the stream - return this.runFFMPEGPipeLine(asset, savedEncodedPath); + return this.runFFMPEGPipeLine(asset, videoStream, savedEncodedPath); } } - async needsTranscoding(asset: AssetEntity, ffmpegConfig: SystemConfigFFmpegDto): Promise { + async needsTranscoding(videoStream: FfprobeStream, ffmpegConfig: SystemConfigFFmpegDto): Promise { switch (ffmpegConfig.transcode) { case TranscodePreset.ALL: return true; case TranscodePreset.REQUIRED: { - const videoStream = await this.getVideoStream(asset); if (videoStream.codec_name !== ffmpegConfig.targetVideoCodec) { return true; } @@ -97,12 +97,13 @@ export class VideoTranscodeProcessor { break; case TranscodePreset.OPTIMAL: { - const videoStream = await this.getVideoStream(asset); if (videoStream.codec_name !== ffmpegConfig.targetVideoCodec) { return true; } - const videoHeightThreshold = 1080; + const config = await this.systemConfigService.getConfig(); + + const videoHeightThreshold = Number.parseInt(config.ffmpeg.targetResolution); return !videoStream.height || videoStream.height > videoHeightThreshold; } } @@ -125,22 +126,45 @@ export class VideoTranscodeProcessor { return longestVideoStream; } - async runFFMPEGPipeLine(asset: AssetEntity, savedEncodedPath: string): Promise { + async runFFMPEGPipeLine(asset: AssetEntity, videoStream: FfprobeStream, savedEncodedPath: string): Promise { const config = await this.systemConfigService.getConfig(); + const ffmpegOptions = [ + `-crf ${config.ffmpeg.crf}`, + `-preset ${config.ffmpeg.preset}`, + `-vcodec ${config.ffmpeg.targetVideoCodec}`, + `-acodec ${config.ffmpeg.targetAudioCodec}`, + // Makes a second pass moving the moov atom to the beginning of + // the file for improved playback speed. + `-movflags faststart`, + ]; + + if (!videoStream.height || !videoStream.width) { + this.logger.error('Height or width undefined for video stream'); + return; + } + + const streamHeight = videoStream.height; + const streamWidth = videoStream.width; + + const targetResolution = Number.parseInt(config.ffmpeg.targetResolution); + + let scaling = `-2:${targetResolution}`; + const shouldScale = Math.min(streamHeight, streamWidth) > targetResolution; + + const videoIsRotated = Math.abs(Number.parseInt(`${videoStream.rotation ?? 0}`)) === 90; + + if (streamHeight > streamWidth || videoIsRotated) { + scaling = `${targetResolution}:-2`; + } + + if (shouldScale) { + ffmpegOptions.push(`-vf scale=${scaling}`); + } + return new Promise((resolve, reject) => { ffmpeg(asset.originalPath) - .outputOptions([ - `-crf ${config.ffmpeg.crf}`, - `-preset ${config.ffmpeg.preset}`, - `-vcodec ${config.ffmpeg.targetVideoCodec}`, - `-acodec ${config.ffmpeg.targetAudioCodec}`, - `-vf scale=${config.ffmpeg.targetScaling}`, - - // Makes a second pass moving the moov atom to the beginning of - // the file for improved playback speed. - `-movflags faststart`, - ]) + .outputOptions(ffmpegOptions) .output(savedEncodedPath) .on('start', () => { this.logger.log('Start Converting Video'); diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index 70b11afa94..3eb8159577 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -4644,7 +4644,7 @@ "targetAudioCodec": { "type": "string" }, - "targetScaling": { + "targetResolution": { "type": "string" }, "transcode": { @@ -4661,7 +4661,7 @@ "preset", "targetVideoCodec", "targetAudioCodec", - "targetScaling", + "targetResolution", "transcode" ] }, diff --git a/server/libs/domain/src/system-config/dto/system-config-ffmpeg.dto.ts b/server/libs/domain/src/system-config/dto/system-config-ffmpeg.dto.ts index 07b1723797..77dca9f49e 100644 --- a/server/libs/domain/src/system-config/dto/system-config-ffmpeg.dto.ts +++ b/server/libs/domain/src/system-config/dto/system-config-ffmpeg.dto.ts @@ -15,7 +15,7 @@ export class SystemConfigFFmpegDto { targetAudioCodec!: string; @IsString() - targetScaling!: string; + targetResolution!: string; @IsEnum(TranscodePreset) transcode!: TranscodePreset; diff --git a/server/libs/domain/src/system-config/system-config.core.ts b/server/libs/domain/src/system-config/system-config.core.ts index d91ffa1e51..713af9ef3b 100644 --- a/server/libs/domain/src/system-config/system-config.core.ts +++ b/server/libs/domain/src/system-config/system-config.core.ts @@ -13,7 +13,7 @@ const defaults: SystemConfig = Object.freeze({ preset: 'ultrafast', targetVideoCodec: 'h264', targetAudioCodec: 'aac', - targetScaling: '1280:-2', + targetResolution: '720', transcode: TranscodePreset.REQUIRED, }, oauth: { diff --git a/server/libs/domain/src/system-config/system-config.service.spec.ts b/server/libs/domain/src/system-config/system-config.service.spec.ts index 32562c5b89..2600e384bd 100644 --- a/server/libs/domain/src/system-config/system-config.service.spec.ts +++ b/server/libs/domain/src/system-config/system-config.service.spec.ts @@ -16,7 +16,7 @@ const updatedConfig = Object.freeze({ crf: 'a new value', preset: 'ultrafast', targetAudioCodec: 'aac', - targetScaling: '1280:-2', + targetResolution: '720', targetVideoCodec: 'h264', transcode: TranscodePreset.REQUIRED, }, diff --git a/server/libs/domain/test/fixtures.ts b/server/libs/domain/test/fixtures.ts index e5383703f2..1c2f02746c 100644 --- a/server/libs/domain/test/fixtures.ts +++ b/server/libs/domain/test/fixtures.ts @@ -401,7 +401,7 @@ export const systemConfigStub = { crf: '23', preset: 'ultrafast', targetAudioCodec: 'aac', - targetScaling: '1280:-2', + targetResolution: '720', targetVideoCodec: 'h264', transcode: TranscodePreset.REQUIRED, }, diff --git a/server/libs/infra/src/entities/system-config.entity.ts b/server/libs/infra/src/entities/system-config.entity.ts index 6e0237b428..db5576d9c0 100644 --- a/server/libs/infra/src/entities/system-config.entity.ts +++ b/server/libs/infra/src/entities/system-config.entity.ts @@ -17,7 +17,7 @@ export enum SystemConfigKey { FFMPEG_PRESET = 'ffmpeg.preset', FFMPEG_TARGET_VIDEO_CODEC = 'ffmpeg.targetVideoCodec', FFMPEG_TARGET_AUDIO_CODEC = 'ffmpeg.targetAudioCodec', - FFMPEG_TARGET_SCALING = 'ffmpeg.targetScaling', + FFMPEG_TARGET_RESOLUTION = 'ffmpeg.targetResolution', FFMPEG_TRANSCODE = 'ffmpeg.transcode', OAUTH_ENABLED = 'oauth.enabled', OAUTH_ISSUER_URL = 'oauth.issuerUrl', @@ -45,7 +45,7 @@ export interface SystemConfig { preset: string; targetVideoCodec: string; targetAudioCodec: string; - targetScaling: string; + targetResolution: string; transcode: TranscodePreset; }; oauth: { diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index 3853e55758..b3c465e1ae 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -2034,7 +2034,7 @@ export interface SystemConfigFFmpegDto { * @type {string} * @memberof SystemConfigFFmpegDto */ - 'targetScaling': string; + 'targetResolution': string; /** * * @type {string} diff --git a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte index be1775ef21..bab12c62c2 100644 --- a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte +++ b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte @@ -113,12 +113,18 @@ isEdited={!(ffmpegConfig.targetVideoCodec == savedConfig.targetVideoCodec)} /> -