diff --git a/server/src/queries/asset.job.repository.sql b/server/src/queries/asset.job.repository.sql index 06dadb9a2d..43beb28c76 100644 --- a/server/src/queries/asset.job.repository.sql +++ b/server/src/queries/asset.job.repository.sql @@ -627,9 +627,12 @@ select "asset_audio"."profile", "asset_audio"."bitrate" from - "asset_audio" + ( + select + 1 + ) as "dummy" where - "asset_audio"."assetId" = "asset"."id" + "asset_audio"."assetId" is not null ) as obj ) as "audioStream", ( @@ -695,7 +698,8 @@ select from "asset" inner join "asset_exif" on "asset"."id" = "asset_exif"."assetId" - left join "asset_video" on "asset_video"."assetId" = "asset"."id" + inner join "asset_video" on "asset_video"."assetId" = "asset"."id" + left join "asset_audio" on "asset_audio"."assetId" = "asset"."id" where "asset"."id" = $1 and "asset"."type" = 'VIDEO' diff --git a/server/src/repositories/asset-job.repository.ts b/server/src/repositories/asset-job.repository.ts index e2250c05b7..f9173bfa10 100644 --- a/server/src/repositories/asset-job.repository.ts +++ b/server/src/repositories/asset-job.repository.ts @@ -1,6 +1,6 @@ import { Injectable } from '@nestjs/common'; import { Kysely, sql } from 'kysely'; -import { jsonArrayFrom } from 'kysely/helpers/postgres'; +import { jsonArrayFrom, jsonObjectFrom } from 'kysely/helpers/postgres'; import { InjectKysely } from 'nestjs-kysely'; import { columns } from 'src/database'; import { DummyValue, GenerateSql } from 'src/decorators'; @@ -9,7 +9,7 @@ import { DB } from 'src/schema'; import { anyUuid, asUuid, - withAudioVideo, + withAudioStream, withDefaultVisibility, withEdits, withExif, @@ -17,6 +17,8 @@ import { withFaces, withFilePath, withFiles, + withVideoFormat, + withVideoStream, } from 'src/utils/database'; import { mimeTypes } from 'src/utils/mime-types'; @@ -135,7 +137,9 @@ export class AssetJobRepository { ) .select(withEdits) .$call(withExifInner) - .$call(withAudioVideo) + .leftJoin('asset_video', 'asset_video.assetId', 'asset.id') + .select((eb) => withVideoStream(eb).as('videoStream')) + .select((eb) => withVideoFormat(eb).as('format')) .where('asset.id', '=', id) .executeTakeFirst(); } @@ -336,9 +340,13 @@ export class AssetJobRepository { return this.db .selectFrom('asset') .innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId') + .innerJoin('asset_video', 'asset_video.assetId', 'asset.id') + .leftJoin('asset_audio', 'asset_audio.assetId', 'asset.id') .select(['asset.id', 'asset.ownerId', 'asset.originalPath']) .select(withFiles) - .$call((qb) => withAudioVideo(qb, true)) + .select((eb) => withAudioStream(eb).as('audioStream')) + .select((eb) => withVideoStream(eb).$notNull().as('videoStream')) + .select((eb) => withVideoFormat(eb).$notNull().as('format')) .where('asset.id', '=', id) .where('asset.type', '=', sql.lit(AssetType.Video)) .executeTakeFirst(); diff --git a/server/src/schema/migrations/1776823683957-CreateAudioVideoTables.ts b/server/src/schema/migrations/1776823683957-CreateAudioVideoTables.ts index 174c1dcb52..2e0bf36bb0 100644 --- a/server/src/schema/migrations/1776823683957-CreateAudioVideoTables.ts +++ b/server/src/schema/migrations/1776823683957-CreateAudioVideoTables.ts @@ -14,7 +14,7 @@ export async function up(db: Kysely): Promise { "assetId" uuid NOT NULL, "bitrate" integer NOT NULL, "frameCount" integer NOT NULL, - "timeBase" integer, + "timeBase" integer NOT NULL, "index" smallint NOT NULL, "profile" smallint, "level" smallint, diff --git a/server/src/schema/tables/asset-av.table.ts b/server/src/schema/tables/asset-av.table.ts index 7aa3c7e73d..e961f06f43 100644 --- a/server/src/schema/tables/asset-av.table.ts +++ b/server/src/schema/tables/asset-av.table.ts @@ -32,8 +32,8 @@ export class AssetVideoTable { @Column({ type: 'integer' }) frameCount!: number; - @Column({ type: 'integer', nullable: true }) - timeBase!: number | null; + @Column({ type: 'integer' }) + timeBase!: number; @Column({ type: smallint }) index!: number; diff --git a/server/src/services/media.service.spec.ts b/server/src/services/media.service.spec.ts index 2cca3fa4af..78d36f189a 100644 --- a/server/src/services/media.service.spec.ts +++ b/server/src/services/media.service.spec.ts @@ -1,4 +1,4 @@ -import { ShallowDehydrateObject } from 'kysely'; +import { NotNull, ShallowDehydrateObject } from 'kysely'; import { OutputInfo } from 'sharp'; import { SystemConfig } from 'src/config'; import { Exif } from 'src/database'; @@ -508,7 +508,7 @@ describe(MediaService.name, () => { expect.any(String), expect.objectContaining({ inputOptions: ['-skip_frame', 'nointra', '-sws_flags', 'accurate_rnd+full_chroma_int'], - outputOptions: [ + outputOptions: expect.arrayContaining([ '-fps_mode', 'vfr', '-frames:v', @@ -519,7 +519,7 @@ describe(MediaService.name, () => { 'verbose', '-vf', String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:1440:flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc`, - ], + ]), twoPass: false, }), ); @@ -557,7 +557,7 @@ describe(MediaService.name, () => { expect.any(String), expect.objectContaining({ inputOptions: ['-skip_frame', 'nointra', '-sws_flags', 'accurate_rnd+full_chroma_int'], - outputOptions: [ + outputOptions: expect.arrayContaining([ '-fps_mode', 'vfr', '-frames:v', @@ -567,8 +567,8 @@ describe(MediaService.name, () => { '-v', 'verbose', '-vf', - String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`, - ], + String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:250:flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`, + ]), twoPass: false, }), ); @@ -608,7 +608,7 @@ describe(MediaService.name, () => { expect.any(String), expect.objectContaining({ inputOptions: ['-skip_frame', 'nointra', '-sws_flags', 'accurate_rnd+full_chroma_int'], - outputOptions: [ + outputOptions: expect.arrayContaining([ '-fps_mode', 'vfr', '-frames:v', @@ -618,8 +618,8 @@ describe(MediaService.name, () => { '-v', 'verbose', '-vf', - String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`, - ], + String.raw`fps=12:start_time=0:eof_action=pass:round=down,thumbnail=12,select=gt(scene\,0.1)-eq(prev_selected_n\,n)+isnan(prev_selected_n)+gt(n\,20),trim=end_frame=2,reverse,scale=-2:250:flags=lanczos+accurate_rnd+full_chroma_int:out_range=pc,tonemapx=tonemap=hable:desat=0:p=bt709:t=bt709:m=bt709:r=pc:peak=100:format=yuv420p`, + ]), twoPass: false, }), ); @@ -1937,16 +1937,16 @@ describe(MediaService.name, () => { describe('handleVideoConversion', () => { let asset: ReturnType & { - videoStream: VideoStreamInfo | null; + videoStream: VideoStreamInfo & { timeBase: NotNull }; audioStream: AudioStreamInfo | null; - format: VideoFormat | null; + format: VideoFormat; }; beforeEach(() => { asset = { ...AssetFactory.create({ id: 'video-id', type: AssetType.Video, originalPath: '/original/path.ext' }), - videoStream: null, + videoStream: probeStub.videoStreamH264.videoStream, audioStream: null, - format: null, + format: probeStub.videoStreamH264.format, }; mocks.assetJob.getForVideoConversion.mockResolvedValue(asset); sut.videoInterfaces = { dri: ['renderD128'], mali: true }; diff --git a/server/src/services/metadata.service.ts b/server/src/services/metadata.service.ts index 9517f07193..5329e990aa 100644 --- a/server/src/services/metadata.service.ts +++ b/server/src/services/metadata.service.ts @@ -326,7 +326,7 @@ export class MetadataService extends BaseService { : undefined; const videoData = - format?.formatName && format?.formatLongName && video?.codecName + format?.formatName && format?.formatLongName && video?.codecName && video?.timeBase ? { assetId: asset.id, bitrate: video.bitrate, diff --git a/server/src/utils/database.ts b/server/src/utils/database.ts index 8b2cc303fb..e0298b1dcd 100644 --- a/server/src/utils/database.ts +++ b/server/src/utils/database.ts @@ -21,7 +21,7 @@ import { AssetFileType, AssetVisibility, DatabaseExtension, ExifOrientation } fr import { AssetSearchBuilderOptions } from 'src/repositories/search.repository'; import { DB } from 'src/schema'; import { AssetExifTable } from 'src/schema/tables/asset-exif.table'; -import { AudioStreamInfo, VectorExtension, VideoFormat, VideoStreamInfo } from 'src/types'; +import { AudioStreamInfo, VectorExtension, VideoFormat, VideoPacketInfo, VideoStreamInfo } from 'src/types'; export const getKyselyConfig = (connection: DatabaseConnectionParams): KyselyConfig => { return { @@ -101,83 +101,75 @@ export function withExifInner(qb: SelectQueryBuilder) { export const dummy = sql`(select 1)`.as('dummy'); -export function withAudioVideo(qb: SelectQueryBuilder, withAudio = false) { - return qb - .$if(withAudio, (qb) => - qb.select((eb) => - jsonObjectFrom( - eb - .selectFrom('asset_audio') - .select(['asset_audio.index', 'asset_audio.codecName', 'asset_audio.profile', 'asset_audio.bitrate']) - .whereRef('asset_audio.assetId', '=', 'asset.id'), - ) - .$castTo() - .as('audioStream'), - ), - ) - .leftJoin('asset_video', 'asset_video.assetId', 'asset.id') - .select((eb) => - jsonObjectFrom( +export function withAudioStream(eb: ExpressionBuilder) { + return jsonObjectFrom( + eb + .selectFrom(dummy) + .select(['asset_audio.index', 'asset_audio.codecName', 'asset_audio.profile', 'asset_audio.bitrate']) + .where('asset_audio.assetId', 'is not', sql.lit(null)) + .$castTo(), + ); +} + +export function withVideoStream(eb: ExpressionBuilder) { + return jsonObjectFrom( + eb + .selectFrom(dummy) + .select((eb) => [ + 'asset_video.index', + 'asset_video.codecName', + 'asset_video.profile', + 'asset_video.level', + 'asset_video.bitrate', + 'asset_exif.exifImageWidth as width', + 'asset_exif.exifImageHeight as height', + 'asset_video.pixelFormat', + 'asset_video.frameCount', + 'asset_exif.fps as frameRate', + 'asset_video.timeBase', eb - .selectFrom(dummy) - .where('asset_video.assetId', 'is not', sql.lit(null)) - .select((eb) => [ - 'asset_video.index', - 'asset_video.codecName', - 'asset_video.profile', - 'asset_video.level', - 'asset_video.bitrate', - 'asset_exif.exifImageWidth as width', - 'asset_exif.exifImageHeight as height', - 'asset_video.pixelFormat', - 'asset_video.frameCount', - 'asset_exif.fps as frameRate', - 'asset_video.timeBase', - eb - .case() - .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate90CW.toString())) - .then(sql.lit(-90)) - .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate270CW.toString())) - .then(sql.lit(90)) - .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate180.toString())) - .then(sql.lit(180)) - .else(0) - .end() - .as('rotation'), - 'asset_video.colorPrimaries', - 'asset_video.colorMatrix', - 'asset_video.colorTransfer', - 'asset_video.dvProfile', - 'asset_video.dvLevel', - 'asset_video.dvBlSignalCompatibilityId', - ]) - .$castTo(), - ).as('videoStream'), - ) - .select((eb) => - jsonObjectFrom( + .case() + .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate90CW.toString())) + .then(sql.lit(-90)) + .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate270CW.toString())) + .then(sql.lit(90)) + .when('asset_exif.orientation', '=', sql.lit(ExifOrientation.Rotate180.toString())) + .then(sql.lit(180)) + .else(0) + .end() + .as('rotation'), + 'asset_video.colorPrimaries', + 'asset_video.colorMatrix', + 'asset_video.colorTransfer', + 'asset_video.dvProfile', + 'asset_video.dvLevel', + 'asset_video.dvBlSignalCompatibilityId', + ]) + .where('asset_video.assetId', 'is not', sql.lit(null)), + ).$castTo<(VideoStreamInfo & { timeBase: NotNull }) | null>(); +} + +export function withVideoFormat(eb: ExpressionBuilder) { + return jsonObjectFrom( + eb + .selectFrom(dummy) + .select([ + 'asset_video.formatName', + 'asset_video.formatLongName', + // TODO: simplify after https://github.com/immich-app/immich/pull/28003 eb - .selectFrom(dummy) - .where('asset_video.assetId', 'is not', sql.lit(null)) - .select((eb) => [ - 'asset_video.formatName', - 'asset_video.formatLongName', - // TODO: simplify after https://github.com/immich-app/immich/pull/28003 - eb - .case() - .when('asset.duration', '~', sql`'^\\d{2}:\\d{2}:\\d{2}\\.\\d{3}$'`) - .then( - sql`substr(asset.duration, 1, 2)::int * 3600000 + substr(asset.duration, 4, 2)::int * 60000 + substr(asset.duration, 7, 2)::int * 1000 + substr(asset.duration, 10, 3)::int`, - ) - .else(sql.lit(0)) - .end() - .as('duration'), - 'asset_video.bitrate', - ]), - ) - .$castTo() - .as('format'), - ); + .case() + .when('asset.duration', '~', sql`'^\\d{2}:\\d{2}:\\d{2}\\.\\d{3}$'`) + .then( + sql`substr(asset.duration, 1, 2)::int * 3600000 + substr(asset.duration, 4, 2)::int * 60000 + substr(asset.duration, 7, 2)::int * 1000 + substr(asset.duration, 10, 3)::int`, + ) + .else(sql.lit(0)) + .end() + .as('duration'), + 'asset_video.bitrate', + ]) + .where('asset_video.assetId', 'is not', sql.lit(null)), + ).$castTo(); } export function withSmartSearch(qb: SelectQueryBuilder) { diff --git a/server/test/fixtures/media.stub.ts b/server/test/fixtures/media.stub.ts index 82e2684818..120fe07664 100644 --- a/server/test/fixtures/media.stub.ts +++ b/server/test/fixtures/media.stub.ts @@ -1,3 +1,4 @@ +import { NotNull } from 'kysely'; import { ColorMatrix, ColorPrimaries, ColorTransfer, DvProfile, DvSignalCompatibility } from 'src/enum'; import { AudioStreamInfo, VideoFormat, VideoInfo, VideoStreamInfo } from 'src/types'; @@ -392,9 +393,9 @@ export const videoInfoStub = { }; interface SelectedStreams { - videoStream: VideoStreamInfo | null; + videoStream: VideoStreamInfo & { timeBase: NotNull }; audioStream: AudioStreamInfo | null; - format: VideoFormat | null; + format: VideoFormat; } const toSelectedStreams = (info: VideoInfo) => ({ diff --git a/server/test/mappers.ts b/server/test/mappers.ts index 93110aa4f5..53d1d00a13 100644 --- a/server/test/mappers.ts +++ b/server/test/mappers.ts @@ -1,4 +1,4 @@ -import { Selectable, ShallowDehydrateObject } from 'kysely'; +import { NotNull, Selectable, ShallowDehydrateObject } from 'kysely'; import { MapAsset } from 'src/dtos/asset-response.dto'; import { AssetEditActionItem } from 'src/dtos/editing.dto'; import { ActivityTable } from 'src/schema/tables/activity.table'; @@ -156,7 +156,7 @@ export const getForGenerateThumbnail = (asset: ReturnType files: asset.files.map((file) => getDehydrated(file)), exifInfo: getDehydrated(asset.exifInfo), edits: asset.edits.map(({ action, parameters }) => ({ action, parameters })) as AssetEditActionItem[], - videoStream: null as VideoStreamInfo | null, + videoStream: null as (VideoStreamInfo & { timeBase: NotNull }) | null, audioStream: null as AudioStreamInfo | null, format: null as VideoFormat | null, });