diff --git a/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart b/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart index a99e855d7..66eaca436 100644 --- a/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart +++ b/mobile/openapi/lib/model/system_config_f_fmpeg_dto.dart @@ -165,12 +165,14 @@ class SystemConfigFFmpegDtoTranscodeEnum { static const all = SystemConfigFFmpegDtoTranscodeEnum._(r'all'); static const optimal = SystemConfigFFmpegDtoTranscodeEnum._(r'optimal'); static const required_ = SystemConfigFFmpegDtoTranscodeEnum._(r'required'); + static const disabled = SystemConfigFFmpegDtoTranscodeEnum._(r'disabled'); /// List of all possible values in this [enum][SystemConfigFFmpegDtoTranscodeEnum]. static const values = [ all, optimal, required_, + disabled, ]; static SystemConfigFFmpegDtoTranscodeEnum? fromJson(dynamic value) => SystemConfigFFmpegDtoTranscodeEnumTypeTransformer().decode(value); @@ -212,6 +214,7 @@ class SystemConfigFFmpegDtoTranscodeEnumTypeTransformer { case r'all': return SystemConfigFFmpegDtoTranscodeEnum.all; case r'optimal': return SystemConfigFFmpegDtoTranscodeEnum.optimal; case r'required': return SystemConfigFFmpegDtoTranscodeEnum.required_; + case r'disabled': return SystemConfigFFmpegDtoTranscodeEnum.disabled; default: if (!allowNull) { throw ArgumentError('Unknown enum value to decode: $data'); diff --git a/server/apps/microservices/src/processors.ts b/server/apps/microservices/src/processors.ts index 882c61aa6..0cfe32d5b 100644 --- a/server/apps/microservices/src/processors.ts +++ b/server/apps/microservices/src/processors.ts @@ -163,7 +163,7 @@ export class VideoTranscodeProcessor { await this.mediaService.handleQueueVideoConversion(job.data); } - @Process({ name: JobName.VIDEO_CONVERSION, concurrency: 2 }) + @Process({ name: JobName.VIDEO_CONVERSION, concurrency: 1 }) async onVideoConversion(job: Job) { await this.mediaService.handleVideoConversion(job.data); } diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index 147a7f733..72d4be3c3 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -4679,7 +4679,8 @@ "enum": [ "all", "optimal", - "required" + "required", + "disabled" ] } }, diff --git a/server/libs/domain/src/media/media.repository.ts b/server/libs/domain/src/media/media.repository.ts index 931b5b7a2..ff2c2e045 100644 --- a/server/libs/domain/src/media/media.repository.ts +++ b/server/libs/domain/src/media/media.repository.ts @@ -14,8 +14,21 @@ export interface VideoStreamInfo { frameCount: number; } +export interface AudioStreamInfo { + codecName?: string; + codecType?: string; +} + +export interface VideoFormat { + formatName?: string; + formatLongName?: string; + duration: number; +} + export interface VideoInfo { - streams: VideoStreamInfo[]; + format: VideoFormat; + videoStreams: VideoStreamInfo[]; + audioStreams: AudioStreamInfo[]; } export interface IMediaRepository { diff --git a/server/libs/domain/src/media/media.service.spec.ts b/server/libs/domain/src/media/media.service.spec.ts index 93e0900d0..bdb6b1eef 100644 --- a/server/libs/domain/src/media/media.service.spec.ts +++ b/server/libs/domain/src/media/media.service.spec.ts @@ -222,7 +222,7 @@ describe(MediaService.name, () => { }); it('should transcode the longest stream', async () => { - mediaMock.probe.mockResolvedValue(probeStub.multiple); + mediaMock.probe.mockResolvedValue(probeStub.multipleVideoStreams); await sut.handleVideoConversion({ asset: assetEntityStub.video }); @@ -237,7 +237,7 @@ describe(MediaService.name, () => { }); it('should skip a video without any streams', async () => { - mediaMock.probe.mockResolvedValue(probeStub.empty); + mediaMock.probe.mockResolvedValue(probeStub.noVideoStreams); await sut.handleVideoConversion({ asset: assetEntityStub.video }); expect(mediaMock.transcode).not.toHaveBeenCalled(); }); @@ -249,7 +249,7 @@ describe(MediaService.name, () => { }); it('should transcode when set to all', async () => { - mediaMock.probe.mockResolvedValue(probeStub.multiple); + mediaMock.probe.mockResolvedValue(probeStub.multipleVideoStreams); configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'all' }]); await sut.handleVideoConversion({ asset: assetEntityStub.video }); expect(mediaMock.transcode).toHaveBeenCalledWith( @@ -260,7 +260,40 @@ describe(MediaService.name, () => { }); it('should transcode when optimal and too big', async () => { - mediaMock.probe.mockResolvedValue(probeStub.tooBig); + mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p); + configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'optimal' }]); + await sut.handleVideoConversion({ asset: assetEntityStub.video }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + ['-crf 23', '-preset ultrafast', '-vcodec h264', '-acodec aac', '-movflags faststart', '-vf scale=-2:720'], + ); + }); + + it('should transcode with alternate scaling video is vertical', async () => { + mediaMock.probe.mockResolvedValue(probeStub.videoStreamVertical2160p); + configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'optimal' }]); + await sut.handleVideoConversion({ asset: assetEntityStub.video }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + ['-crf 23', '-preset ultrafast', '-vcodec h264', '-acodec aac', '-movflags faststart', '-vf scale=720:-2'], + ); + }); + + it('should transcode when audio doesnt match target', async () => { + mediaMock.probe.mockResolvedValue(probeStub.audioStreamMp3); + configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'optimal' }]); + await sut.handleVideoConversion({ asset: assetEntityStub.video }); + expect(mediaMock.transcode).toHaveBeenCalledWith( + '/original/path.ext', + 'upload/encoded-video/user-id/asset-id.mp4', + ['-crf 23', '-preset ultrafast', '-vcodec h264', '-acodec aac', '-movflags faststart', '-vf scale=-2:720'], + ); + }); + + it('should transcode when container doesnt match target', async () => { + mediaMock.probe.mockResolvedValue(probeStub.matroskaContainer); configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'optimal' }]); await sut.handleVideoConversion({ asset: assetEntityStub.video }); expect(mediaMock.transcode).toHaveBeenCalledWith( @@ -271,7 +304,7 @@ describe(MediaService.name, () => { }); it('should not transcode an invalid transcode value', async () => { - mediaMock.probe.mockResolvedValue(probeStub.tooBig); + mediaMock.probe.mockResolvedValue(probeStub.videoStream2160p); configMock.load.mockResolvedValue([{ key: SystemConfigKey.FFMPEG_TRANSCODE, value: 'invalid' }]); await sut.handleVideoConversion({ asset: assetEntityStub.video }); expect(mediaMock.transcode).not.toHaveBeenCalled(); diff --git a/server/libs/domain/src/media/media.service.ts b/server/libs/domain/src/media/media.service.ts index 1388d1cad..764b1448d 100644 --- a/server/libs/domain/src/media/media.service.ts +++ b/server/libs/domain/src/media/media.service.ts @@ -7,7 +7,7 @@ import { IAssetJob, IBaseJob, IJobRepository, JobName } from '../job'; import { IStorageRepository, StorageCore, StorageFolder } from '../storage'; import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config'; import { SystemConfigCore } from '../system-config/system-config.core'; -import { IMediaRepository, VideoStreamInfo } from './media.repository'; +import { AudioStreamInfo, IMediaRepository, VideoStreamInfo } from './media.repository'; @Injectable() export class MediaService { @@ -127,23 +127,27 @@ export class MediaService { const output = join(outputFolder, `${asset.id}.mp4`); this.storageRepository.mkdirSync(outputFolder); - const { streams } = await this.mediaRepository.probe(input); - const stream = await this.getLongestStream(streams); - if (!stream) { + const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input); + const mainVideoStream = this.getMainVideoStream(videoStreams); + const mainAudioStream = this.getMainAudioStream(audioStreams); + const containerExtension = format.formatName; + if (!mainVideoStream || !mainAudioStream || !containerExtension) { return; } const { ffmpeg: config } = await this.configCore.getConfig(); - const required = this.isTranscodeRequired(stream, config); + const required = this.isTranscodeRequired(mainVideoStream, mainAudioStream, containerExtension, config); if (!required) { return; } - const options = this.getFfmpegOptions(stream, config); + const options = this.getFfmpegOptions(mainVideoStream, config); + + this.logger.log(`Start encoding video ${asset.id} ${options}`); await this.mediaRepository.transcode(input, output, options); - this.logger.log(`Converting Success ${asset.id}`); + this.logger.log(`Encoding success ${asset.id}`); await this.assetRepository.save({ id: asset.id, encodedVideoPath: output }); } catch (error: any) { @@ -151,32 +155,48 @@ export class MediaService { } } - private getLongestStream(streams: VideoStreamInfo[]): VideoStreamInfo | null { - return streams - .filter((stream) => stream.codecType === 'video') - .sort((stream1, stream2) => stream2.frameCount - stream1.frameCount)[0]; + private getMainVideoStream(streams: VideoStreamInfo[]): VideoStreamInfo | null { + return streams.sort((stream1, stream2) => stream2.frameCount - stream1.frameCount)[0]; } - private isTranscodeRequired(stream: VideoStreamInfo, ffmpegConfig: SystemConfigFFmpegDto): boolean { - if (!stream.height || !stream.width) { + private getMainAudioStream(streams: AudioStreamInfo[]): AudioStreamInfo | null { + return streams[0]; + } + + private isTranscodeRequired( + videoStream: VideoStreamInfo, + audioStream: AudioStreamInfo, + containerExtension: string, + ffmpegConfig: SystemConfigFFmpegDto, + ): boolean { + if (!videoStream.height || !videoStream.width) { this.logger.error('Skipping transcode, height or width undefined for video stream'); return false; } - const isTargetVideoCodec = stream.codecName === ffmpegConfig.targetVideoCodec; + const isTargetVideoCodec = videoStream.codecName === ffmpegConfig.targetVideoCodec; + const isTargetAudioCodec = audioStream.codecName === ffmpegConfig.targetAudioCodec; + const isTargetContainer = ['mov,mp4,m4a,3gp,3g2,mj2', 'mp4', 'mov'].includes(containerExtension); + + this.logger.debug(audioStream.codecName, audioStream.codecType, containerExtension); + + const allTargetsMatching = isTargetVideoCodec && isTargetAudioCodec && isTargetContainer; const targetResolution = Number.parseInt(ffmpegConfig.targetResolution); - const isLargerThanTargetResolution = Math.min(stream.height, stream.width) > targetResolution; + const isLargerThanTargetResolution = Math.min(videoStream.height, videoStream.width) > targetResolution; switch (ffmpegConfig.transcode) { + case TranscodePreset.DISABLED: + return false; + case TranscodePreset.ALL: return true; case TranscodePreset.REQUIRED: - return !isTargetVideoCodec; + return !allTargetsMatching; case TranscodePreset.OPTIMAL: - return !isTargetVideoCodec || isLargerThanTargetResolution; + return !allTargetsMatching || isLargerThanTargetResolution; default: return false; @@ -184,8 +204,6 @@ export class MediaService { } private getFfmpegOptions(stream: VideoStreamInfo, ffmpeg: SystemConfigFFmpegDto) { - // TODO: If video or audio are already the correct format, don't re-encode, copy the stream - const options = [ `-crf ${ffmpeg.crf}`, `-preset ${ffmpeg.preset}`, diff --git a/server/libs/domain/test/fixtures.ts b/server/libs/domain/test/fixtures.ts index cc1d7976b..af5ca08ec 100644 --- a/server/libs/domain/test/fixtures.ts +++ b/server/libs/domain/test/fixtures.ts @@ -13,12 +13,15 @@ import { import { AlbumResponseDto, AssetResponseDto, + AudioStreamInfo, AuthUserDto, ExifResponseDto, mapUser, SearchResult, SharedLinkResponseDto, + VideoFormat, VideoInfo, + VideoStreamInfo, } from '../src'; const today = new Date(); @@ -706,10 +709,29 @@ export const searchStub = { }), }; +const probeStubDefaultFormat: VideoFormat = { + formatName: 'mov,mp4,m4a,3gp,3g2,mj2', + formatLongName: 'QuickTime / MOV', + duration: 0, +}; + +const probeStubDefaultVideoStream: VideoStreamInfo[] = [ + { height: 1080, width: 1920, codecName: 'h265', codecType: 'video', frameCount: 100, rotation: 0 }, +]; + +const probeStubDefaultAudioStream: AudioStreamInfo[] = [{ codecName: 'aac', codecType: 'audio' }]; + +const probeStubDefault: VideoInfo = { + format: probeStubDefaultFormat, + videoStreams: probeStubDefaultVideoStream, + audioStreams: probeStubDefaultAudioStream, +}; + export const probeStub = { - empty: { streams: [] }, - multiple: Object.freeze({ - streams: [ + noVideoStreams: Object.freeze({ ...probeStubDefault, videoStreams: [] }), + multipleVideoStreams: Object.freeze({ + ...probeStubDefault, + videoStreams: [ { height: 1080, width: 400, @@ -729,7 +751,8 @@ export const probeStub = { ], }), noHeight: Object.freeze({ - streams: [ + ...probeStubDefault, + videoStreams: [ { height: 0, width: 400, @@ -740,11 +763,12 @@ export const probeStub = { }, ], }), - tooBig: Object.freeze({ - streams: [ + videoStream2160p: Object.freeze({ + ...probeStubDefault, + videoStreams: [ { - height: 10000, - width: 10000, + height: 2160, + width: 3840, codecName: 'h264', codecType: 'video', frameCount: 100, @@ -752,4 +776,29 @@ export const probeStub = { }, ], }), + videoStreamVertical2160p: Object.freeze({ + ...probeStubDefault, + videoStreams: [ + { + height: 2160, + width: 3840, + codecName: 'h264', + codecType: 'video', + frameCount: 100, + rotation: 90, + }, + ], + }), + audioStreamMp3: Object.freeze({ + ...probeStubDefault, + audioStreams: [{ codecType: 'audio', codecName: 'aac' }], + }), + matroskaContainer: Object.freeze({ + ...probeStubDefault, + format: { + formatName: 'matroska,webm', + formatLongName: 'Matroska / WebM', + duration: 0, + }, + }), }; diff --git a/server/libs/infra/src/entities/system-config.entity.ts b/server/libs/infra/src/entities/system-config.entity.ts index db5576d9c..c24374f55 100644 --- a/server/libs/infra/src/entities/system-config.entity.ts +++ b/server/libs/infra/src/entities/system-config.entity.ts @@ -37,6 +37,7 @@ export enum TranscodePreset { ALL = 'all', OPTIMAL = 'optimal', REQUIRED = 'required', + DISABLED = 'disabled', } export interface SystemConfig { diff --git a/server/libs/infra/src/repositories/media.repository.ts b/server/libs/infra/src/repositories/media.repository.ts index b96fb0c5d..94d5f70d3 100644 --- a/server/libs/infra/src/repositories/media.repository.ts +++ b/server/libs/infra/src/repositories/media.repository.ts @@ -50,20 +50,33 @@ export class MediaRepository implements IMediaRepository { const results = await probe(input); return { - streams: results.streams.map((stream) => ({ - height: stream.height || 0, - width: stream.width || 0, - codecName: stream.codec_name, - codecType: stream.codec_type, - frameCount: Number.parseInt(stream.nb_frames ?? '0'), - rotation: Number.parseInt(`${stream.rotation ?? 0}`), - })), + format: { + formatName: results.format.format_name, + formatLongName: results.format.format_long_name, + duration: results.format.duration || 0, + }, + videoStreams: results.streams + .filter((stream) => stream.codec_type === 'video') + .map((stream) => ({ + height: stream.height || 0, + width: stream.width || 0, + codecName: stream.codec_name, + codecType: stream.codec_type, + frameCount: Number.parseInt(stream.nb_frames ?? '0'), + rotation: Number.parseInt(`${stream.rotation ?? 0}`), + })), + audioStreams: results.streams + .filter((stream) => stream.codec_type === 'audio') + .map((stream) => ({ + codecType: stream.codec_type, + codecName: stream.codec_name, + })), }; } transcode(input: string, output: string, options: string[]): Promise { return new Promise((resolve, reject) => { - ffmpeg(input) + ffmpeg(input, { niceness: 10 }) // .outputOptions(options) .output(output) diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index b3c465e1a..9d1ab6d70 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -2046,7 +2046,8 @@ export interface SystemConfigFFmpegDto { export const SystemConfigFFmpegDtoTranscodeEnum = { All: 'all', Optimal: 'optimal', - Required: 'required' + Required: 'required', + Disabled: 'disabled' } as const; export type SystemConfigFFmpegDtoTranscodeEnum = typeof SystemConfigFFmpegDtoTranscodeEnum[keyof typeof SystemConfigFFmpegDtoTranscodeEnum]; diff --git a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte index bab12c62c..ae1ac58b6 100644 --- a/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte +++ b/web/src/lib/components/admin-page/settings/ffmpeg/ffmpeg-settings.svelte @@ -93,16 +93,20 @@ isEdited={!(ffmpegConfig.preset == savedConfig.preset)} /> -