feat(server): track video metadata (#28023)

* track video metadata

* earlier duration check

* revert colorspace change

* duplicate constant

* formatting

* linting

* add comments

* redundant variable

* simplify tests

* use totalDuration instead of format.duration

* medium tests

* install ffmpeg

* install noble

* update test-assets commit

* make timeBase non-nullable

* linting

* use proper smallint

* add ffmpeg to mise

* simplify duration

* regenerate migration

server/src/utils/database.ts

@@ -17,11 +17,11 @@ import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres';
 import { Notice, PostgresError } from 'postgres';
 import { columns, lockableProperties, LockableProperty, Person } from 'src/database';
 import { AssetEditActionItem } from 'src/dtos/editing.dto';
-import { AssetFileType, AssetVisibility, DatabaseExtension } from 'src/enum';
+import { AssetFileType, AssetVisibility, DatabaseExtension, ExifOrientation } from 'src/enum';
 import { AssetSearchBuilderOptions } from 'src/repositories/search.repository';
 import { DB } from 'src/schema';
 import { AssetExifTable } from 'src/schema/tables/asset-exif.table';
-import { VectorExtension } from 'src/types';
+import { AudioStreamInfo, VectorExtension, VideoFormat, VideoStreamInfo } from 'src/types';
 
 export const getKyselyConfig = (connection: DatabaseConnectionParams): KyselyConfig => {
   return {
@@ -99,6 +99,65 @@ export function withExifInner<O>(qb: SelectQueryBuilder<DB, 'asset', O>) {
     .$narrowType<{ exifInfo: NotNull }>();
 }
 
+export const dummy = sql`(select 1)`.as('dummy');
+
+export function withAudioStream(eb: ExpressionBuilder<DB, 'asset_exif' | 'asset_audio'>) {
+  return jsonObjectFrom(
+    eb
+      .selectFrom(dummy)
+      .select(['asset_audio.index', 'asset_audio.codecName', 'asset_audio.profile', 'asset_audio.bitrate'])
+      .where('asset_audio.assetId', 'is not', sql.lit(null))
+      .$castTo<AudioStreamInfo | null>(),
+  );
+}
+
+export function withVideoStream(eb: ExpressionBuilder<DB, 'asset_exif' | 'asset_video'>) {
+  return jsonObjectFrom(
+    eb
+      .selectFrom(dummy)
+      .select((eb) => [
+        'asset_video.index',
+        'asset_video.codecName',
+        'asset_video.profile',
+        'asset_video.level',
+        'asset_video.bitrate',
+        'asset_exif.exifImageWidth as width',
+        'asset_exif.exifImageHeight as height',
+        'asset_video.pixelFormat',
+        'asset_video.frameCount',
+        'asset_exif.fps as frameRate',
+        'asset_video.timeBase',
+        eb
+          .case()
+          .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate90CW.toString()))
+          .then(sql.lit(-90))
+          .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate270CW.toString()))
+          .then(sql.lit(90))
+          .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate180.toString()))
+          .then(sql.lit(180))
+          .else(0)
+          .end()
+          .as('rotation'),
+        'asset_video.colorPrimaries',
+        'asset_video.colorMatrix',
+        'asset_video.colorTransfer',
+        'asset_video.dvProfile',
+        'asset_video.dvLevel',
+        'asset_video.dvBlSignalCompatibilityId',
+      ])
+      .where('asset_video.assetId', 'is not', sql.lit(null)),
+  ).$castTo<(VideoStreamInfo & { timeBase: NotNull }) | null>();
+}
+
+export function withVideoFormat(eb: ExpressionBuilder<DB, 'asset' | 'asset_video'>) {
+  return jsonObjectFrom(
+    eb
+      .selectFrom(dummy)
+      .select(['asset_video.formatName', 'asset_video.formatLongName', 'asset.duration', 'asset_video.bitrate'])
+      .where('asset_video.assetId', 'is not', sql.lit(null)),
+  ).$castTo<VideoFormat | null>();
+}
+
 export function withSmartSearch<O>(qb: SelectQueryBuilder<DB, 'asset', O>) {
   return qb
     .leftJoin('smart_search', 'asset.id', 'smart_search.assetId')
@@ -445,5 +504,3 @@ export const updateLockedColumns = <T extends Record<string, unknown> & { locked
   exif.lockedProperties = lockableProperties.filter((property) => property in exif);
   return exif;
 };
-
-export const dummy = sql`(select 1)`.as('dummy');

server/src/utils/media.ts

@@ -1,6 +1,15 @@
 import { AUDIO_ENCODER } from 'src/constants';
 import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto';
-import { CQMode, ToneMapping, TranscodeHardwareAcceleration, TranscodeTarget, VideoCodec } from 'src/enum';
+import {
+  ColorMatrix,
+  ColorPrimaries,
+  ColorTransfer,
+  CQMode,
+  ToneMapping,
+  TranscodeHardwareAcceleration,
+  TranscodeTarget,
+  VideoCodec,
+} from 'src/enum';
 import {
   AudioStreamInfo,
   BitrateDistribution,
@@ -255,7 +264,10 @@ export class BaseConfig implements VideoCodecSWConfig {
   }
 
   shouldToneMap(videoStream: VideoStreamInfo) {
-    return videoStream.isHDR && this.config.tonemap !== ToneMapping.Disabled;
+    return (
+      this.config.tonemap !== ToneMapping.Disabled &&
+      (videoStream.colorTransfer === ColorTransfer.Smpte2084 || videoStream.colorTransfer === ColorTransfer.AribStdB67)
+    );
   }
 
   getScaling(videoStream: VideoStreamInfo, mult = 2) {
@@ -409,15 +421,15 @@ export class ThumbnailConfig extends BaseConfig {
         : ['-skip_frame', 'nointra', '-sws_flags', 'accurate_rnd+full_chroma_int'];
 
     const metadataOverrides = [];
-    if (videoStream.colorPrimaries === 'reserved') {
+    if (videoStream.colorPrimaries === ColorPrimaries.Reserved) {
       metadataOverrides.push('colour_primaries=1');
     }
 
-    if (videoStream.colorSpace === 'reserved') {
+    if (videoStream.colorMatrix === ColorMatrix.Reserved) {
       metadataOverrides.push('matrix_coefficients=1');
     }
 
-    if (videoStream.colorTransfer === 'reserved') {
+    if (videoStream.colorTransfer === ColorTransfer.Reserved) {
       metadataOverrides.push('transfer_characteristics=1');
     }