fix(server): opus handling as accepted audio codec in transcode policy (#26736)

* Fix opus handling as accepted audio codec in transcode policy

Fix the issue when opus is among accepted audio codecs in transcode policy
(which is default) but it still triggers transcoding because the codec name
from ffprobe (opus) does not match `libopus` literal in Immich.

Make a distinction between a codec name and encoder:
- codec name: switch to `opus` as the audio codec name. This matches what ffprobe
returns for a media file with opus audio.
- encoder: continue using the `libopus` encoder in ffmpeg.

* Add unit tests for accepted audio codecs and for libopus encoder

* Add db migration for ffmpeg.targetAudioCodec opus

* backward compatibility

* tweak

* noisy logs

* full mapping

* make check happy

* mark deprecated

* update api

* indexOf

---------

Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com>

docs/docs/administration/system-settings.md

@@ -230,7 +230,7 @@ The default value is `ultrafast`.
 
 ### Audio codec (`ffmpeg.targetAudioCodec`) {#ffmpeg.targetAudioCodec}
 
-Which audio codec to use when the audio stream is being transcoded. Can be one of `mp3`, `aac`, `libopus`.
+Which audio codec to use when the audio stream is being transcoded. Can be one of `mp3`, `aac`, `opus`.
 
 The default value is `aac`.
 

docs/docs/install/config-file.md

@@ -27,7 +27,7 @@ The default configuration looks like this:
   "ffmpeg": {
     "accel": "disabled",
     "accelDecode": false,
-    "acceptedAudioCodecs": ["aac", "mp3", "libopus"],
+    "acceptedAudioCodecs": ["aac", "mp3", "opus"],
     "acceptedContainers": ["mov", "ogg", "webm"],
     "acceptedVideoCodecs": ["h264"],
     "bframes": -1,

mobile/openapi/lib/model/audio_codec.dart

@@ -26,6 +26,7 @@ class AudioCodec {
   static const mp3 = AudioCodec._(r'mp3');
   static const aac = AudioCodec._(r'aac');
   static const libopus = AudioCodec._(r'libopus');
+  static const opus = AudioCodec._(r'opus');
   static const pcmS16le = AudioCodec._(r'pcm_s16le');
 
   /// List of all possible values in this [enum][AudioCodec].
@@ -33,6 +34,7 @@ class AudioCodec {
     mp3,
     aac,
     libopus,
+    opus,
     pcmS16le,
   ];
 
@@ -75,6 +77,7 @@ class AudioCodecTypeTransformer {
         case r'mp3': return AudioCodec.mp3;
         case r'aac': return AudioCodec.aac;
         case r'libopus': return AudioCodec.libopus;
+        case r'opus': return AudioCodec.opus;
         case r'pcm_s16le': return AudioCodec.pcmS16le;
         default:
           if (!allowNull) {

open-api/immich-openapi-specs.json

@@ -17260,6 +17260,7 @@
           "mp3",
           "aac",
           "libopus",
+          "opus",
           "pcm_s16le"
         ],
         "type": "string"

open-api/typescript-sdk/src/fetch-client.ts

@@ -7324,6 +7324,7 @@ export enum AudioCodec {
     Mp3 = "mp3",
     Aac = "aac",
     Libopus = "libopus",
+    Opus = "opus",
     PcmS16Le = "pcm_s16le"
 }
 export enum VideoContainer {

server/src/config.ts

@@ -206,7 +206,7 @@ export const defaults = Object.freeze<SystemConfig>({
     targetVideoCodec: VideoCodec.H264,
     acceptedVideoCodecs: [VideoCodec.H264],
     targetAudioCodec: AudioCodec.Aac,
-    acceptedAudioCodecs: [AudioCodec.Aac, AudioCodec.Mp3, AudioCodec.LibOpus],
+    acceptedAudioCodecs: [AudioCodec.Aac, AudioCodec.Mp3, AudioCodec.Opus],
     acceptedContainers: [VideoContainer.Mov, VideoContainer.Ogg, VideoContainer.Webm],
     targetResolution: '720',
     maxBitrate: '0',

server/src/constants.ts

@@ -2,7 +2,7 @@ import { Duration } from 'luxon';
 import { readFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { SemVer } from 'semver';
-import { ApiTag, DatabaseExtension, ExifOrientation, VectorIndex } from 'src/enum';
+import { ApiTag, AudioCodec, DatabaseExtension, ExifOrientation, VectorIndex } from 'src/enum';
 
 export const ErrorMessages = {
   InconsistentMediaLocation:
@@ -201,3 +201,11 @@ export const endpointTags: Record<ApiTag, string> = {
   [ApiTag.Workflows]:
     'A workflow is a set of actions that run whenever a triggering event occurs. Workflows also can include filters to further limit execution.',
 };
+
+export const AUDIO_ENCODER: Record<AudioCodec, string> = {
+  [AudioCodec.Aac]: 'aac',
+  [AudioCodec.Mp3]: 'mp3',
+  [AudioCodec.Libopus]: 'libopus',
+  [AudioCodec.Opus]: 'libopus',
+  [AudioCodec.PcmS16le]: 'pcm_s16le',
+};

server/src/dtos/system-config.dto.ts

@@ -1,5 +1,5 @@
 import { ApiProperty } from '@nestjs/swagger';
-import { Type } from 'class-transformer';
+import { Transform, Type } from 'class-transformer';
 import {
   ArrayMinSize,
   IsInt,
@@ -92,6 +92,16 @@ export class SystemConfigFFmpegDto {
   targetAudioCodec!: AudioCodec;
 
   @ValidateEnum({ enum: AudioCodec, name: 'AudioCodec', each: true, description: 'Accepted audio codecs' })
+  @Transform(({ value }) => {
+    if (Array.isArray(value)) {
+      const libopusIndex = value.indexOf('libopus');
+      if (libopusIndex !== -1) {
+        value[libopusIndex] = 'opus';
+      }
+    }
+
+    return value;
+  })
   acceptedAudioCodecs!: AudioCodec[];
 
   @ValidateEnum({ enum: VideoContainer, name: 'VideoContainer', each: true, description: 'Accepted containers' })

server/src/enum.ts

@@ -409,7 +409,9 @@ export enum VideoCodec {
 export enum AudioCodec {
   Mp3 = 'mp3',
   Aac = 'aac',
-  LibOpus = 'libopus',
+  /** @deprecated Use `Opus` instead */
+  Libopus = 'libopus',
+  Opus = 'opus',
   PcmS16le = 'pcm_s16le',
 }
 

server/src/schema/migrations/1772609167000-UpdateOpusCodecName.ts

@@ -0,0 +1,65 @@
+import { Kysely, sql } from 'kysely';
+
+export async function up(db: Kysely<any>): Promise<void> {
+  await sql`
+    UPDATE system_metadata
+    SET value = jsonb_set(
+      value,
+      '{ffmpeg,acceptedAudioCodecs}',
+      (
+        SELECT jsonb_agg(
+          CASE
+            WHEN elem = 'libopus' THEN 'opus'
+            ELSE elem
+          END
+        )
+        FROM jsonb_array_elements_text(value->'ffmpeg'->'acceptedAudioCodecs') elem
+      )
+    )
+    WHERE key = 'system-config'
+    AND value->'ffmpeg'->'acceptedAudioCodecs' ? 'libopus';
+  `.execute(db);
+
+  await sql`
+    UPDATE system_metadata
+    SET value = jsonb_set(
+      value,
+      '{ffmpeg,targetAudioCodec}',
+      '"opus"'::jsonb
+    )
+    WHERE key = 'system-config'
+    AND value->'ffmpeg'->>'targetAudioCodec' = 'libopus';
+  `.execute(db);
+}
+
+export async function down(db: Kysely<any>): Promise<void> {
+  await sql`
+    UPDATE system_metadata
+    SET value = jsonb_set(
+      value,
+      '{ffmpeg,acceptedAudioCodecs}',
+      (
+        SELECT jsonb_agg(
+          CASE
+            WHEN elem = 'opus' THEN 'libopus'
+            ELSE elem
+          END
+        )
+        FROM jsonb_array_elements_text(value->'ffmpeg'->'acceptedAudioCodecs') elem
+      )
+    )
+    WHERE key = 'system-config'
+    AND value->'ffmpeg'->'acceptedAudioCodecs' ? 'opus';
+  `.execute(db);
+
+  await sql`
+    UPDATE system_metadata
+    SET value = jsonb_set(
+      value,
+      '{ffmpeg,targetAudioCodec}',
+      '"libopus"'::jsonb
+    )
+    WHERE key = 'system-config'
+    AND value->'ffmpeg'->>'targetAudioCodec' = 'opus';
+  `.execute(db);
+}

server/src/services/media.service.spec.ts

@@ -2571,6 +2571,50 @@ describe(MediaService.name, () => {
       expect(mocks.media.transcode).not.toHaveBeenCalled();
     });
 
+    describe('should skip transcoding for accepted audio codecs with optimal policy if video is fine', () => {
+      const acceptedCodecs = [
+        { codec: 'aac', probeStub: probeStub.audioStreamAac },
+        { codec: 'mp3', probeStub: probeStub.audioStreamMp3 },
+        { codec: 'opus', probeStub: probeStub.audioStreamOpus },
+      ];
+
+      beforeEach(() => {
+        mocks.systemMetadata.get.mockResolvedValue({
+          ffmpeg: {
+            targetVideoCodec: VideoCodec.Hevc,
+            transcode: TranscodePolicy.Optimal,
+            targetResolution: '1080p',
+          },
+        });
+      });
+
+      it.each(acceptedCodecs)('should skip $codec', async ({ probeStub }) => {
+        mocks.media.probe.mockResolvedValue(probeStub);
+        await sut.handleVideoConversion({ id: 'video-id' });
+        expect(mocks.media.transcode).not.toHaveBeenCalled();
+      });
+    });
+
+    it('should use libopus audio encoder when target audio is opus', async () => {
+      mocks.media.probe.mockResolvedValue(probeStub.audioStreamAac);
+      mocks.systemMetadata.get.mockResolvedValue({
+        ffmpeg: {
+          targetAudioCodec: AudioCodec.Opus,
+          transcode: TranscodePolicy.All,
+        },
+      });
+      await sut.handleVideoConversion({ id: 'video-id' });
+      expect(mocks.media.transcode).toHaveBeenCalledWith(
+        '/original/path.ext',
+        expect.any(String),
+        expect.objectContaining({
+          inputOptions: expect.any(Array),
+          outputOptions: expect.arrayContaining(['-c:a libopus']),
+          twoPass: false,
+        }),
+      );
+    });
+
     it('should fail if hwaccel is enabled for an unsupported codec', async () => {
       mocks.media.probe.mockResolvedValue(probeStub.matroskaContainer);
       mocks.systemMetadata.get.mockResolvedValue({

server/src/services/system-config.service.spec.ts

@@ -55,7 +55,7 @@ const updatedConfig = Object.freeze<SystemConfig>({
     threads: 0,
     preset: 'ultrafast',
     targetAudioCodec: AudioCodec.Aac,
-    acceptedAudioCodecs: [AudioCodec.Aac, AudioCodec.Mp3, AudioCodec.LibOpus],
+    acceptedAudioCodecs: [AudioCodec.Aac, AudioCodec.Mp3, AudioCodec.Opus],
     targetResolution: '720',
     targetVideoCodec: VideoCodec.H264,
     acceptedVideoCodecs: [VideoCodec.H264],

server/src/utils/media.ts

@@ -1,3 +1,4 @@
+import { AUDIO_ENCODER } from 'src/constants';
 import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto';
 import { CQMode, ToneMapping, TranscodeHardwareAcceleration, TranscodeTarget, VideoCodec } from 'src/enum';
 import {
@@ -117,7 +118,7 @@ export class BaseConfig implements VideoCodecSWConfig {
 
   getBaseOutputOptions(target: TranscodeTarget, videoStream: VideoStreamInfo, audioStream?: AudioStreamInfo) {
     const videoCodec = [TranscodeTarget.All, TranscodeTarget.Video].includes(target) ? this.getVideoCodec() : 'copy';
-    const audioCodec = [TranscodeTarget.All, TranscodeTarget.Audio].includes(target) ? this.getAudioCodec() : 'copy';
+    const audioCodec = [TranscodeTarget.All, TranscodeTarget.Audio].includes(target) ? this.getAudioEncoder() : 'copy';
 
     const options = [
       `-c:v ${videoCodec}`,
@@ -305,8 +306,8 @@ export class BaseConfig implements VideoCodecSWConfig {
     return [options];
   }
 
-  getAudioCodec(): string {
-    return this.config.targetAudioCodec;
+  getAudioEncoder(): string {
+    return AUDIO_ENCODER[this.config.targetAudioCodec];
   }
 
   getVideoCodec(): string {

server/test/fixtures/media.stub.ts

@@ -221,6 +221,14 @@ export const probeStub = {
     ...probeStubDefault,
     audioStreams: [{ index: 1, codecName: 'aac', bitrate: 100 }],
   }),
+  audioStreamMp3: Object.freeze<VideoInfo>({
+    ...probeStubDefault,
+    audioStreams: [{ index: 1, codecName: 'mp3', bitrate: 100 }],
+  }),
+  audioStreamOpus: Object.freeze<VideoInfo>({
+    ...probeStubDefault,
+    audioStreams: [{ index: 1, codecName: 'opus', bitrate: 100 }],
+  }),
   audioStreamUnknown: Object.freeze<VideoInfo>({
     ...probeStubDefault,
     audioStreams: [

web/src/lib/components/admin-settings/FFmpegSettings.svelte

@@ -115,7 +115,7 @@
               options={[
                 { value: AudioCodec.Aac, text: 'AAC' },
                 { value: AudioCodec.Mp3, text: 'MP3' },
-                { value: AudioCodec.Libopus, text: 'Opus' },
+                { value: AudioCodec.Opus, text: 'Opus' },
                 { value: AudioCodec.PcmS16Le, text: 'PCM (16 bit)' },
               ]}
               isEdited={!isEqual(
@@ -174,7 +174,7 @@
               options={[
                 { value: AudioCodec.Aac, text: 'aac' },
                 { value: AudioCodec.Mp3, text: 'mp3' },
-                { value: AudioCodec.Libopus, text: 'opus' },
+                { value: AudioCodec.Opus, text: 'opus' },
               ]}
               name="acodec"
               isEdited={configToEdit.ffmpeg.targetAudioCodec !== config.ffmpeg.targetAudioCodec}