From 4a384bca86c8e8d461e520e92a94a2fc67036996 Mon Sep 17 00:00:00 2001 From: Sergey Katsubo Date: Sat, 7 Mar 2026 21:08:42 +0300 Subject: [PATCH] fix(server): opus handling as accepted audio codec in transcode policy (#26736) * Fix opus handling as accepted audio codec in transcode policy Fix the issue when opus is among accepted audio codecs in transcode policy (which is default) but it still triggers transcoding because the codec name from ffprobe (opus) does not match `libopus` literal in Immich. Make a distinction between a codec name and encoder: - codec name: switch to `opus` as the audio codec name. This matches what ffprobe returns for a media file with opus audio. - encoder: continue using the `libopus` encoder in ffmpeg. * Add unit tests for accepted audio codecs and for libopus encoder * Add db migration for ffmpeg.targetAudioCodec opus * backward compatibility * tweak * noisy logs * full mapping * make check happy * mark deprecated * update api * indexOf --------- Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com> --- docs/docs/administration/system-settings.md | 2 +- docs/docs/install/config-file.md | 2 +- mobile/openapi/lib/model/audio_codec.dart | 3 + open-api/immich-openapi-specs.json | 1 + open-api/typescript-sdk/src/fetch-client.ts | 1 + server/src/config.ts | 2 +- server/src/constants.ts | 10 ++- server/src/dtos/system-config.dto.ts | 12 +++- server/src/enum.ts | 4 +- .../1772609167000-UpdateOpusCodecName.ts | 65 +++++++++++++++++++ server/src/services/media.service.spec.ts | 44 +++++++++++++ .../services/system-config.service.spec.ts | 2 +- server/src/utils/media.ts | 7 +- server/test/fixtures/media.stub.ts | 8 +++ .../admin-settings/FFmpegSettings.svelte | 4 +- 15 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 server/src/schema/migrations/1772609167000-UpdateOpusCodecName.ts diff --git a/docs/docs/administration/system-settings.md b/docs/docs/administration/system-settings.md index fdfdad29eaa1..7dc9c08db3cc 100644 --- a/docs/docs/administration/system-settings.md +++ b/docs/docs/administration/system-settings.md @@ -230,7 +230,7 @@ The default value is `ultrafast`. ### Audio codec (`ffmpeg.targetAudioCodec`) {#ffmpeg.targetAudioCodec} -Which audio codec to use when the audio stream is being transcoded. Can be one of `mp3`, `aac`, `libopus`. +Which audio codec to use when the audio stream is being transcoded. Can be one of `mp3`, `aac`, `opus`. The default value is `aac`. diff --git a/docs/docs/install/config-file.md b/docs/docs/install/config-file.md index bf815521ef73..3355750603d6 100644 --- a/docs/docs/install/config-file.md +++ b/docs/docs/install/config-file.md @@ -27,7 +27,7 @@ The default configuration looks like this: "ffmpeg": { "accel": "disabled", "accelDecode": false, - "acceptedAudioCodecs": ["aac", "mp3", "libopus"], + "acceptedAudioCodecs": ["aac", "mp3", "opus"], "acceptedContainers": ["mov", "ogg", "webm"], "acceptedVideoCodecs": ["h264"], "bframes": -1, diff --git a/mobile/openapi/lib/model/audio_codec.dart b/mobile/openapi/lib/model/audio_codec.dart index 095c616995fa..be1ff0dcb9b3 100644 --- a/mobile/openapi/lib/model/audio_codec.dart +++ b/mobile/openapi/lib/model/audio_codec.dart @@ -26,6 +26,7 @@ class AudioCodec { static const mp3 = AudioCodec._(r'mp3'); static const aac = AudioCodec._(r'aac'); static const libopus = AudioCodec._(r'libopus'); + static const opus = AudioCodec._(r'opus'); static const pcmS16le = AudioCodec._(r'pcm_s16le'); /// List of all possible values in this [enum][AudioCodec]. @@ -33,6 +34,7 @@ class AudioCodec { mp3, aac, libopus, + opus, pcmS16le, ]; @@ -75,6 +77,7 @@ class AudioCodecTypeTransformer { case r'mp3': return AudioCodec.mp3; case r'aac': return AudioCodec.aac; case r'libopus': return AudioCodec.libopus; + case r'opus': return AudioCodec.opus; case r'pcm_s16le': return AudioCodec.pcmS16le; default: if (!allowNull) { diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 38e1fe8e0177..d2eb32200948 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -17260,6 +17260,7 @@ "mp3", "aac", "libopus", + "opus", "pcm_s16le" ], "type": "string" diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 1ae12cd0911e..5c8ac6dbc12a 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -7324,6 +7324,7 @@ export enum AudioCodec { Mp3 = "mp3", Aac = "aac", Libopus = "libopus", + Opus = "opus", PcmS16Le = "pcm_s16le" } export enum VideoContainer { diff --git a/server/src/config.ts b/server/src/config.ts index 2a43b5118771..e6134df477f5 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -206,7 +206,7 @@ export const defaults = Object.freeze({ targetVideoCodec: VideoCodec.H264, acceptedVideoCodecs: [VideoCodec.H264], targetAudioCodec: AudioCodec.Aac, - acceptedAudioCodecs: [AudioCodec.Aac, AudioCodec.Mp3, AudioCodec.LibOpus], + acceptedAudioCodecs: [AudioCodec.Aac, AudioCodec.Mp3, AudioCodec.Opus], acceptedContainers: [VideoContainer.Mov, VideoContainer.Ogg, VideoContainer.Webm], targetResolution: '720', maxBitrate: '0', diff --git a/server/src/constants.ts b/server/src/constants.ts index 9ea5e134b632..e24057beba3d 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -2,7 +2,7 @@ import { Duration } from 'luxon'; import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { SemVer } from 'semver'; -import { ApiTag, DatabaseExtension, ExifOrientation, VectorIndex } from 'src/enum'; +import { ApiTag, AudioCodec, DatabaseExtension, ExifOrientation, VectorIndex } from 'src/enum'; export const ErrorMessages = { InconsistentMediaLocation: @@ -201,3 +201,11 @@ export const endpointTags: Record = { [ApiTag.Workflows]: 'A workflow is a set of actions that run whenever a triggering event occurs. Workflows also can include filters to further limit execution.', }; + +export const AUDIO_ENCODER: Record = { + [AudioCodec.Aac]: 'aac', + [AudioCodec.Mp3]: 'mp3', + [AudioCodec.Libopus]: 'libopus', + [AudioCodec.Opus]: 'libopus', + [AudioCodec.PcmS16le]: 'pcm_s16le', +}; diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts index 7a0dcb6f3ad0..a214dbc46715 100644 --- a/server/src/dtos/system-config.dto.ts +++ b/server/src/dtos/system-config.dto.ts @@ -1,5 +1,5 @@ import { ApiProperty } from '@nestjs/swagger'; -import { Type } from 'class-transformer'; +import { Transform, Type } from 'class-transformer'; import { ArrayMinSize, IsInt, @@ -92,6 +92,16 @@ export class SystemConfigFFmpegDto { targetAudioCodec!: AudioCodec; @ValidateEnum({ enum: AudioCodec, name: 'AudioCodec', each: true, description: 'Accepted audio codecs' }) + @Transform(({ value }) => { + if (Array.isArray(value)) { + const libopusIndex = value.indexOf('libopus'); + if (libopusIndex !== -1) { + value[libopusIndex] = 'opus'; + } + } + + return value; + }) acceptedAudioCodecs!: AudioCodec[]; @ValidateEnum({ enum: VideoContainer, name: 'VideoContainer', each: true, description: 'Accepted containers' }) diff --git a/server/src/enum.ts b/server/src/enum.ts index 2aa9bd2aa6de..887c8fa93c5b 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -409,7 +409,9 @@ export enum VideoCodec { export enum AudioCodec { Mp3 = 'mp3', Aac = 'aac', - LibOpus = 'libopus', + /** @deprecated Use `Opus` instead */ + Libopus = 'libopus', + Opus = 'opus', PcmS16le = 'pcm_s16le', } diff --git a/server/src/schema/migrations/1772609167000-UpdateOpusCodecName.ts b/server/src/schema/migrations/1772609167000-UpdateOpusCodecName.ts new file mode 100644 index 000000000000..9fa5f7d788b2 --- /dev/null +++ b/server/src/schema/migrations/1772609167000-UpdateOpusCodecName.ts @@ -0,0 +1,65 @@ +import { Kysely, sql } from 'kysely'; + +export async function up(db: Kysely): Promise { + await sql` + UPDATE system_metadata + SET value = jsonb_set( + value, + '{ffmpeg,acceptedAudioCodecs}', + ( + SELECT jsonb_agg( + CASE + WHEN elem = 'libopus' THEN 'opus' + ELSE elem + END + ) + FROM jsonb_array_elements_text(value->'ffmpeg'->'acceptedAudioCodecs') elem + ) + ) + WHERE key = 'system-config' + AND value->'ffmpeg'->'acceptedAudioCodecs' ? 'libopus'; + `.execute(db); + + await sql` + UPDATE system_metadata + SET value = jsonb_set( + value, + '{ffmpeg,targetAudioCodec}', + '"opus"'::jsonb + ) + WHERE key = 'system-config' + AND value->'ffmpeg'->>'targetAudioCodec' = 'libopus'; + `.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql` + UPDATE system_metadata + SET value = jsonb_set( + value, + '{ffmpeg,acceptedAudioCodecs}', + ( + SELECT jsonb_agg( + CASE + WHEN elem = 'opus' THEN 'libopus' + ELSE elem + END + ) + FROM jsonb_array_elements_text(value->'ffmpeg'->'acceptedAudioCodecs') elem + ) + ) + WHERE key = 'system-config' + AND value->'ffmpeg'->'acceptedAudioCodecs' ? 'opus'; + `.execute(db); + + await sql` + UPDATE system_metadata + SET value = jsonb_set( + value, + '{ffmpeg,targetAudioCodec}', + '"libopus"'::jsonb + ) + WHERE key = 'system-config' + AND value->'ffmpeg'->>'targetAudioCodec' = 'opus'; + `.execute(db); +} diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index 12440fb2638e..cd61d7b45bd6 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -2571,6 +2571,50 @@ describe(MediaService.name, () => { expect(mocks.media.transcode).not.toHaveBeenCalled(); }); + describe('should skip transcoding for accepted audio codecs with optimal policy if video is fine', () => { + const acceptedCodecs = [ + { codec: 'aac', probeStub: probeStub.audioStreamAac }, + { codec: 'mp3', probeStub: probeStub.audioStreamMp3 }, + { codec: 'opus', probeStub: probeStub.audioStreamOpus }, + ]; + + beforeEach(() => { + mocks.systemMetadata.get.mockResolvedValue({ + ffmpeg: { + targetVideoCodec: VideoCodec.Hevc, + transcode: TranscodePolicy.Optimal, + targetResolution: '1080p', + }, + }); + }); + + it.each(acceptedCodecs)('should skip $codec', async ({ probeStub }) => { + mocks.media.probe.mockResolvedValue(probeStub); + await sut.handleVideoConversion({ id: 'video-id' }); + expect(mocks.media.transcode).not.toHaveBeenCalled(); + }); + }); + + it('should use libopus audio encoder when target audio is opus', async () => { + mocks.media.probe.mockResolvedValue(probeStub.audioStreamAac); + mocks.systemMetadata.get.mockResolvedValue({ + ffmpeg: { + targetAudioCodec: AudioCodec.Opus, + transcode: TranscodePolicy.All, + }, + }); + await sut.handleVideoConversion({ id: 'video-id' }); + expect(mocks.media.transcode).toHaveBeenCalledWith( + '/original/path.ext', + expect.any(String), + expect.objectContaining({ + inputOptions: expect.any(Array), + outputOptions: expect.arrayContaining(['-c:a libopus']), + twoPass: false, + }), + ); + }); + it('should fail if hwaccel is enabled for an unsupported codec', async () => { mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer); mocks.systemMetadata.get.mockResolvedValue({ diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index 1c93c9d7d390..b346906fc8ac 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -55,7 +55,7 @@ const updatedConfig = Object.freeze({ threads: 0, preset: 'ultrafast', targetAudioCodec: AudioCodec.Aac, - acceptedAudioCodecs: [AudioCodec.Aac, AudioCodec.Mp3, AudioCodec.LibOpus], + acceptedAudioCodecs: [AudioCodec.Aac, AudioCodec.Mp3, AudioCodec.Opus], targetResolution: '720', targetVideoCodec: VideoCodec.H264, acceptedVideoCodecs: [VideoCodec.H264], diff --git a/server/src/utils/media.ts b/server/src/utils/media.ts index b2ffb9ac8b1d..ce185305bd00 100644 --- a/server/src/utils/media.ts +++ b/server/src/utils/media.ts @@ -1,3 +1,4 @@ +import { AUDIO_ENCODER } from 'src/constants'; import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto'; import { CQMode, ToneMapping, TranscodeHardwareAcceleration, TranscodeTarget, VideoCodec } from 'src/enum'; import { @@ -117,7 +118,7 @@ export class BaseConfig implements VideoCodecSWConfig { getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) { const videoCodec = [TranscodeTarget.All, TranscodeTarget.Video].includes(target) ? this.getVideoCodec() : 'copy'; - const audioCodec = [TranscodeTarget.All, TranscodeTarget.Audio].includes(target) ? this.getAudioCodec() : 'copy'; + const audioCodec = [TranscodeTarget.All, TranscodeTarget.Audio].includes(target) ? this.getAudioEncoder() : 'copy'; const options = [ `-c:v ${videoCodec}`, @@ -305,8 +306,8 @@ export class BaseConfig implements VideoCodecSWConfig { return [options]; } - getAudioCodec(): string { - return this.config.targetAudioCodec; + getAudioEncoder(): string { + return AUDIO_ENCODER[this.config.targetAudioCodec]; } getVideoCodec(): string { diff --git a/server/test/fixtures/media.stub.ts b/server/test/fixtures/media.stub.ts index f80ad70c8f26..23617fcaf006 100644 --- a/server/test/fixtures/media.stub.ts +++ b/server/test/fixtures/media.stub.ts @@ -221,6 +221,14 @@ export const probeStub = { ...probeStubDefault, audioStreams: [{ index: 1, codecName: 'aac', bitrate: 100 }], }), + audioStreamMp3: Object.freeze({ + ...probeStubDefault, + audioStreams: [{ index: 1, codecName: 'mp3', bitrate: 100 }], + }), + audioStreamOpus: Object.freeze({ + ...probeStubDefault, + audioStreams: [{ index: 1, codecName: 'opus', bitrate: 100 }], + }), audioStreamUnknown: Object.freeze({ ...probeStubDefault, audioStreams: [ diff --git a/web/src/lib/components/admin-settings/FFmpegSettings.svelte b/web/src/lib/components/admin-settings/FFmpegSettings.svelte index 83596069f9dc..e062b616b383 100644 --- a/web/src/lib/components/admin-settings/FFmpegSettings.svelte +++ b/web/src/lib/components/admin-settings/FFmpegSettings.svelte @@ -115,7 +115,7 @@ options={[ { value: AudioCodec.Aac, text: 'AAC' }, { value: AudioCodec.Mp3, text: 'MP3' }, - { value: AudioCodec.Libopus, text: 'Opus' }, + { value: AudioCodec.Opus, text: 'Opus' }, { value: AudioCodec.PcmS16Le, text: 'PCM (16 bit)' }, ]} isEdited={!isEqual( @@ -174,7 +174,7 @@ options={[ { value: AudioCodec.Aac, text: 'aac' }, { value: AudioCodec.Mp3, text: 'mp3' }, - { value: AudioCodec.Libopus, text: 'opus' }, + { value: AudioCodec.Opus, text: 'opus' }, ]} name="acodec" isEdited={configToEdit.ffmpeg.targetAudioCodec !== config.ffmpeg.targetAudioCodec}