From 4a384bca86c8e8d461e520e92a94a2fc67036996 Mon Sep 17 00:00:00 2001 From: Sergey Katsubo Date: Sat, 7 Mar 2026 21:08:42 +0300 Subject: [PATCH] fix(server): opus handling as accepted audio codec in transcode policy (#26736) * Fix opus handling as accepted audio codec in transcode policy Fix the issue when opus is among accepted audio codecs in transcode policy (which is default) but it still triggers transcoding because the codec name from ffprobe (opus) does not match `libopus` literal in Immich. Make a distinction between a codec name and encoder: - codec name: switch to `opus` as the audio codec name. This matches what ffprobe returns for a media file with opus audio. - encoder: continue using the `libopus` encoder in ffmpeg. * Add unit tests for accepted audio codecs and for libopus encoder * Add db migration for ffmpeg.targetAudioCodec opus * backward compatibility * tweak * noisy logs * full mapping * make check happy * mark deprecated * update api * indexOf --------- Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com> --- docs/docs/administration/system-settings.md | 2 +- docs/docs/install/config-file.md | 2 +- mobile/openapi/lib/model/audio_codec.dart | 3 + open-api/immich-openapi-specs.json | 1 + open-api/typescript-sdk/src/fetch-client.ts | 1 + server/src/config.ts | 2 +- server/src/constants.ts | 10 ++- server/src/dtos/system-config.dto.ts | 12 +++- server/src/enum.ts | 4 +- .../1772609167000-UpdateOpusCodecName.ts | 65 +++++++++++++++++++ server/src/services/media.service.spec.ts | 44 +++++++++++++ .../services/system-config.service.spec.ts | 2 +- server/src/utils/media.ts | 7 +- server/test/fixtures/media.stub.ts | 8 +++ .../admin-settings/FFmpegSettings.svelte | 4 +- 15 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 server/src/schema/migrations/1772609167000-UpdateOpusCodecName.ts diff --git a/docs/docs/administration/system-settings.md b/docs/docs/administration/system-settings.md index fdfdad29ea..7dc9c08db3 100644 --- a/docs/docs/administration/system-settings.md +++ b/docs/docs/administration/system-settings.md @@ -230,7 +230,7 @@ The default value is `ultrafast`. ### Audio codec (`ffmpeg.targetAudioCodec`) {#ffmpeg.targetAudioCodec} -Which audio codec to use when the audio stream is being transcoded. Can be one of `mp3`, `aac`, `libopus`. +Which audio codec to use when the audio stream is being transcoded. Can be one of `mp3`, `aac`, `opus`. The default value is `aac`. diff --git a/docs/docs/install/config-file.md b/docs/docs/install/config-file.md index bf815521ef..3355750603 100644 --- a/docs/docs/install/config-file.md +++ b/docs/docs/install/config-file.md @@ -27,7 +27,7 @@ The default configuration looks like this: "ffmpeg": { "accel": "disabled", "accelDecode": false, - "acceptedAudioCodecs": ["aac", "mp3", "libopus"], + "acceptedAudioCodecs": ["aac", "mp3", "opus"], "acceptedContainers": ["mov", "ogg", "webm"], "acceptedVideoCodecs": ["h264"], "bframes": -1, diff --git a/mobile/openapi/lib/model/audio_codec.dart b/mobile/openapi/lib/model/audio_codec.dart index 095c616995..be1ff0dcb9 100644 --- a/mobile/openapi/lib/model/audio_codec.dart +++ b/mobile/openapi/lib/model/audio_codec.dart @@ -26,6 +26,7 @@ class AudioCodec { static const mp3 = AudioCodec._(r'mp3'); static const aac = AudioCodec._(r'aac'); static const libopus = AudioCodec._(r'libopus'); + static const opus = AudioCodec._(r'opus'); static const pcmS16le = AudioCodec._(r'pcm_s16le'); /// List of all possible values in this [enum][AudioCodec]. @@ -33,6 +34,7 @@ class AudioCodec { mp3, aac, libopus, + opus, pcmS16le, ]; @@ -75,6 +77,7 @@ class AudioCodecTypeTransformer { case r'mp3': return AudioCodec.mp3; case r'aac': return AudioCodec.aac; case r'libopus': return AudioCodec.libopus; + case r'opus': return AudioCodec.opus; case r'pcm_s16le': return AudioCodec.pcmS16le; default: if (!allowNull) { diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 38e1fe8e01..d2eb322009 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -17260,6 +17260,7 @@ "mp3", "aac", "libopus", + "opus", "pcm_s16le" ], "type": "string" diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 1ae12cd091..5c8ac6dbc1 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -7324,6 +7324,7 @@ export enum AudioCodec { Mp3 = "mp3", Aac = "aac", Libopus = "libopus", + Opus = "opus", PcmS16Le = "pcm_s16le" } export enum VideoContainer { diff --git a/server/src/config.ts b/server/src/config.ts index 2a43b51187..e6134df477 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -206,7 +206,7 @@ export const defaults = Object.freeze({ targetVideoCodec: VideoCodec.H264, acceptedVideoCodecs: [VideoCodec.H264], targetAudioCodec: AudioCodec.Aac, - acceptedAudioCodecs: [AudioCodec.Aac, AudioCodec.Mp3, AudioCodec.LibOpus], + acceptedAudioCodecs: [AudioCodec.Aac, AudioCodec.Mp3, AudioCodec.Opus], acceptedContainers: [VideoContainer.Mov, VideoContainer.Ogg, VideoContainer.Webm], targetResolution: '720', maxBitrate: '0', diff --git a/server/src/constants.ts b/server/src/constants.ts index 9ea5e134b6..e24057beba 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -2,7 +2,7 @@ import { Duration } from 'luxon'; import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { SemVer } from 'semver'; -import { ApiTag, DatabaseExtension, ExifOrientation, VectorIndex } from 'src/enum'; +import { ApiTag, AudioCodec, DatabaseExtension, ExifOrientation, VectorIndex } from 'src/enum'; export const ErrorMessages = { InconsistentMediaLocation: @@ -201,3 +201,11 @@ export const endpointTags: Record = { [ApiTag.Workflows]: 'A workflow is a set of actions that run whenever a triggering event occurs. Workflows also can include filters to further limit execution.', }; + +export const AUDIO_ENCODER: Record = { + [AudioCodec.Aac]: 'aac', + [AudioCodec.Mp3]: 'mp3', + [AudioCodec.Libopus]: 'libopus', + [AudioCodec.Opus]: 'libopus', + [AudioCodec.PcmS16le]: 'pcm_s16le', +}; diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts index 7a0dcb6f3a..a214dbc467 100644 --- a/server/src/dtos/system-config.dto.ts +++ b/server/src/dtos/system-config.dto.ts @@ -1,5 +1,5 @@ import { ApiProperty } from '@nestjs/swagger'; -import { Type } from 'class-transformer'; +import { Transform, Type } from 'class-transformer'; import { ArrayMinSize, IsInt, @@ -92,6 +92,16 @@ export class SystemConfigFFmpegDto { targetAudioCodec!: AudioCodec; @ValidateEnum({ enum: AudioCodec, name: 'AudioCodec', each: true, description: 'Accepted audio codecs' }) + @Transform(({ value }) => { + if (Array.isArray(value)) { + const libopusIndex = value.indexOf('libopus'); + if (libopusIndex !== -1) { + value[libopusIndex] = 'opus'; + } + } + + return value; + }) acceptedAudioCodecs!: AudioCodec[]; @ValidateEnum({ enum: VideoContainer, name: 'VideoContainer', each: true, description: 'Accepted containers' }) diff --git a/server/src/enum.ts b/server/src/enum.ts index 2aa9bd2aa6..887c8fa93c 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -409,7 +409,9 @@ export enum VideoCodec { export enum AudioCodec { Mp3 = 'mp3', Aac = 'aac', - LibOpus = 'libopus', + /** @deprecated Use `Opus` instead */ + Libopus = 'libopus', + Opus = 'opus', PcmS16le = 'pcm_s16le', } diff --git a/server/src/schema/migrations/1772609167000-UpdateOpusCodecName.ts b/server/src/schema/migrations/1772609167000-UpdateOpusCodecName.ts new file mode 100644 index 0000000000..9fa5f7d788 --- /dev/null +++ b/server/src/schema/migrations/1772609167000-UpdateOpusCodecName.ts @@ -0,0 +1,65 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await sql` + UPDATE system_metadata + SET value = jsonb_set( + value, + '{ffmpeg,acceptedAudioCodecs}', + ( + SELECT jsonb_agg( + CASE + WHEN elem = 'libopus' THEN 'opus' + ELSE elem + END + ) + FROM jsonb_array_elements_text(value->'ffmpeg'->'acceptedAudioCodecs') elem + ) + ) + WHERE key = 'system-config' + AND value->'ffmpeg'->'acceptedAudioCodecs' ? 'libopus'; + `.execute(db); + + await sql` + UPDATE system_metadata + SET value = jsonb_set( + value, + '{ffmpeg,targetAudioCodec}', + '"opus"'::jsonb + ) + WHERE key = 'system-config' + AND value->'ffmpeg'->>'targetAudioCodec' = 'libopus'; + `.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql` + UPDATE system_metadata + SET value = jsonb_set( + value, + '{ffmpeg,acceptedAudioCodecs}', + ( + SELECT jsonb_agg( + CASE + WHEN elem = 'opus' THEN 'libopus' + ELSE elem + END + ) + FROM jsonb_array_elements_text(value->'ffmpeg'->'acceptedAudioCodecs') elem + ) + ) + WHERE key = 'system-config' + AND value->'ffmpeg'->'acceptedAudioCodecs' ? 'opus'; + `.execute(db); + + await sql` + UPDATE system_metadata + SET value = jsonb_set( + value, + '{ffmpeg,targetAudioCodec}', + '"libopus"'::jsonb + ) + WHERE key = 'system-config' + AND value->'ffmpeg'->>'targetAudioCodec' = 'opus'; + `.execute(db); +} diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index 12440fb263..cd61d7b45b 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -2571,6 +2571,50 @@ describe(MediaService.name, () => { expect(mocks.media.transcode).not.toHaveBeenCalled(); }); + describe('should skip transcoding for accepted audio codecs with optimal policy if video is fine', () => { + const acceptedCodecs = [ + { codec: 'aac', probeStub: probeStub.audioStreamAac }, + { codec: 'mp3', probeStub: probeStub.audioStreamMp3 }, + { codec: 'opus', probeStub: probeStub.audioStreamOpus }, + ]; + + beforeEach(() => { + mocks.systemMetadata.get.mockResolvedValue({ + ffmpeg: { + targetVideoCodec: VideoCodec.Hevc, + transcode: TranscodePolicy.Optimal, + targetResolution: '1080p', + }, + }); + }); + + it.each(acceptedCodecs)('should skip $codec', async ({ probeStub }) => { + mocks.media.probe.mockResolvedValue(probeStub); + await sut.handleVideoConversion({ id: 'video-id' }); + expect(mocks.media.transcode).not.toHaveBeenCalled(); + }); + }); + + it('should use libopus audio encoder when target audio is opus', async () => { + mocks.media.probe.mockResolvedValue(probeStub.audioStreamAac); + mocks.systemMetadata.get.mockResolvedValue({ + ffmpeg: { + targetAudioCodec: AudioCodec.Opus, + transcode: TranscodePolicy.All, + }, + }); + await sut.handleVideoConversion({ id: 'video-id' }); + expect(mocks.media.transcode).toHaveBeenCalledWith( + '/original/path.ext', + expect.any(String), + expect.objectContaining({ + inputOptions: expect.any(Array), + outputOptions: expect.arrayContaining(['-c:a libopus']), + twoPass: false, + }), + ); + }); + it('should fail if hwaccel is enabled for an unsupported codec', async () => { mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer); mocks.systemMetadata.get.mockResolvedValue({ diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index 1c93c9d7d3..b346906fc8 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -55,7 +55,7 @@ const updatedConfig = Object.freeze({ threads: 0, preset: 'ultrafast', targetAudioCodec: AudioCodec.Aac, - acceptedAudioCodecs: [AudioCodec.Aac, AudioCodec.Mp3, AudioCodec.LibOpus], + acceptedAudioCodecs: [AudioCodec.Aac, AudioCodec.Mp3, AudioCodec.Opus], targetResolution: '720', targetVideoCodec: VideoCodec.H264, acceptedVideoCodecs: [VideoCodec.H264], diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index b2ffb9ac8b..ce185305bd 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -1,3 +1,4 @@ +import { AUDIO_ENCODER } from 'src/constants'; import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto'; import { CQMode, ToneMapping, TranscodeHardwareAcceleration, TranscodeTarget, VideoCodec } from 'src/enum'; import { @@ -117,7 +118,7 @@ export class BaseConfig implements VideoCodecSWConfig { getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { const videoCodec = [TranscodeTarget.All, TranscodeTarget.Video].includes(target) ? this.getVideoCodec() : 'copy'; - const audioCodec = [TranscodeTarget.All, TranscodeTarget.Audio].includes(target) ? this.getAudioCodec() : 'copy'; + const audioCodec = [TranscodeTarget.All, TranscodeTarget.Audio].includes(target) ? this.getAudioEncoder() : 'copy'; const options = [ `-c:v ${videoCodec}`, @@ -305,8 +306,8 @@ export class BaseConfig implements VideoCodecSWConfig { return [options]; } - getAudioCodec(): string { - return this.config.targetAudioCodec; + getAudioEncoder(): string { + return AUDIO_ENCODER[this.config.targetAudioCodec]; } getVideoCodec(): string { diff --git a/server/test/fixtures/media.stub.ts b/server/test/fixtures/media.stub.ts index f80ad70c8f..23617fcaf0 100644 --- a/server/test/fixtures/media.stub.ts +++ b/server/test/fixtures/media.stub.ts @@ -221,6 +221,14 @@ export const probeStub = { ...probeStubDefault, audioStreams: [{ index: 1, codecName: 'aac', bitrate: 100 }], }), + audioStreamMp3: Object.freeze({ + ...probeStubDefault, + audioStreams: [{ index: 1, codecName: 'mp3', bitrate: 100 }], + }), + audioStreamOpus: Object.freeze({ + ...probeStubDefault, + audioStreams: [{ index: 1, codecName: 'opus', bitrate: 100 }], + }), audioStreamUnknown: Object.freeze({ ...probeStubDefault, audioStreams: [ diff --git a/web/src/lib/components/admin-settings/FFmpegSettings.svelte b/web/src/lib/components/admin-settings/FFmpegSettings.svelte index 83596069f9..e062b616b3 100644 --- a/web/src/lib/components/admin-settings/FFmpegSettings.svelte +++ b/web/src/lib/components/admin-settings/FFmpegSettings.svelte @@ -115,7 +115,7 @@ options={[ { value: AudioCodec.Aac, text: 'AAC' }, { value: AudioCodec.Mp3, text: 'MP3' }, - { value: AudioCodec.Libopus, text: 'Opus' }, + { value: AudioCodec.Opus, text: 'Opus' }, { value: AudioCodec.PcmS16Le, text: 'PCM (16 bit)' }, ]} isEdited={!isEqual( @@ -174,7 +174,7 @@ options={[ { value: AudioCodec.Aac, text: 'aac' }, { value: AudioCodec.Mp3, text: 'mp3' }, - { value: AudioCodec.Libopus, text: 'opus' }, + { value: AudioCodec.Opus, text: 'opus' }, ]} name="acodec" isEdited={configToEdit.ffmpeg.targetAudioCodec !== config.ffmpeg.targetAudioCodec}