From 75db82da127fdffd754f36bae4ae6a82838cf8f7 Mon Sep 17 00:00:00 2001 From: mertalev <101130780+mertalev@users.noreply.github.com> Date: Mon, 4 May 2026 09:31:45 -0400 Subject: [PATCH] hls service and controller --- open-api/immich-openapi-specs.json | 362 ++++++++++++++++++ server/src/config.ts | 6 + server/src/constants.ts | 42 +- server/src/controllers/index.ts | 2 + .../controllers/video-stream.controller.ts | 79 ++++ server/src/dtos/streaming.dto.ts | 26 ++ server/src/dtos/system-config.dto.ts | 5 + server/src/enum.ts | 6 +- server/src/repositories/event.repository.ts | 8 + server/src/repositories/media.repository.ts | 33 +- .../repositories/video-stream.repository.ts | 61 ++- .../src/repositories/websocket.repository.ts | 11 +- server/src/schema/enums.ts | 11 +- server/src/services/hls.service.spec.ts | 322 ++++++++++++++++ server/src/services/hls.service.ts | 202 ++++++++++ server/src/services/index.ts | 2 + .../services/system-config.service.spec.ts | 3 + server/src/utils/event.ts | 44 +++ server/test/fixtures/media.stub.ts | 2 +- server/tsconfig.json | 4 +- 20 files changed, 1206 insertions(+), 25 deletions(-) create mode 100644 server/src/controllers/video-stream.controller.ts create mode 100644 server/src/dtos/streaming.dto.ts create mode 100644 server/src/services/hls.service.spec.ts create mode 100644 server/src/services/hls.service.ts create mode 100644 server/src/utils/event.ts diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 6f9e886185..5e95bfd98b 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -4300,6 +4300,351 @@ "x-immich-state": "Stable" } }, + "/assets/{id}/video/stream/master.m3u8": { + "get": { + "description": "Returns an HLS master playlist with all available variants for the asset.", + "operationId": "getMasterPlaylist", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$", + "type": "string" + } + }, + { + "name": "key", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "slug", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/vnd.apple.mpegurl": { + "schema": { + "type": "string" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Get HLS master playlist", + "tags": [ + "Assets" + ], + "x-immich-history": [ + { + "version": "v3", + "state": "Added" + }, + { + "version": "v3", + "state": "Alpha" + } + ], + "x-immich-permission": "asset.view", + "x-immich-state": "Alpha" + } + }, + "/assets/{id}/video/stream/{sessionId}": { + "delete": { + "description": "Releases server resources for the streaming session.", + "operationId": "endSession", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$", + "type": "string" + } + }, + { + "name": "key", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "sessionId", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$", + "type": "string" + } + }, + { + "name": "slug", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + } + ], + "responses": { + "204": { + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "End HLS streaming session", + "tags": [ + "Assets" + ], + "x-immich-history": [ + { + "version": "v3", + "state": "Added" + }, + { + "version": "v3", + "state": "Alpha" + } + ], + "x-immich-permission": "asset.view", + "x-immich-state": "Alpha" + } + }, + "/assets/{id}/video/stream/{sessionId}/{variantIndex}/playlist.m3u8": { + "get": { + "description": "Returns an HLS media playlist for one variant of the streaming session.", + "operationId": "getMediaPlaylist", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$", + "type": "string" + } + }, + { + "name": "key", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "sessionId", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$", + "type": "string" + } + }, + { + "name": "slug", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "variantIndex", + "required": true, + "in": "path", + "schema": { + "minimum": 0, + "maximum": 9007199254740991, + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/vnd.apple.mpegurl": { + "schema": { + "type": "string" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Get HLS media playlist", + "tags": [ + "Assets" + ], + "x-immich-history": [ + { + "version": "v3", + "state": "Added" + }, + { + "version": "v3", + "state": "Alpha" + } + ], + "x-immich-permission": "asset.view", + "x-immich-state": "Alpha" + } + }, + "/assets/{id}/video/stream/{sessionId}/{variantIndex}/{filename}": { + "get": { + "description": "Streams an HLS init segment (init.mp4) or media segment (seg_N.m4s).", + "operationId": "getSegment", + "parameters": [ + { + "name": "filename", + "required": true, + "in": "path", + "schema": { + "pattern": "^(init\\.mp4|seg_\\d+\\.m4s)$", + "type": "string" + } + }, + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$", + "type": "string" + } + }, + { + "name": "key", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "sessionId", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "pattern": "^([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-4[0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12})$", + "type": "string" + } + }, + { + "name": "slug", + "required": false, + "in": "query", + "schema": { + "type": "string" + } + }, + { + "name": "variantIndex", + "required": true, + "in": "path", + "schema": { + "minimum": 0, + "maximum": 9007199254740991, + "type": "integer" + } + } + ], + "responses": { + "200": { + "content": { + "application/octet-stream": { + "schema": { + "format": "binary", + "type": "string" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "summary": "Get HLS segment or init file", + "tags": [ + "Assets" + ], + "x-immich-history": [ + { + "version": "v3", + "state": "Added" + }, + { + "version": "v3", + "state": "Alpha" + } + ], + "x-immich-permission": "asset.view", + "x-immich-state": "Alpha" + } + }, "/auth/admin-sign-up": { "post": { "description": "Create the first admin user in the system.", @@ -17897,6 +18242,7 @@ "LibrarySyncFilesQueueAll", "LibrarySyncFiles", "LibraryScanQueueAll", + "HlsSessionCleanup", "MemoryCleanup", "MemoryGenerate", "NotificationsCleanup", @@ -24095,6 +24441,9 @@ "description": "Preset", "type": "string" }, + "realtime": { + "$ref": "#/components/schemas/SystemConfigFFmpegRealtimeDto" + }, "refs": { "description": "References", "maximum": 6, @@ -24145,6 +24494,7 @@ "maxBitrate", "preferredHwDevice", "preset", + "realtime", "refs", "targetAudioCodec", "targetResolution", @@ -24157,6 +24507,18 @@ ], "type": "object" }, + "SystemConfigFFmpegRealtimeDto": { + "properties": { + "enabled": { + "description": "Enable real-time HLS transcoding (alpha)", + "type": "boolean" + } + }, + "required": [ + "enabled" + ], + "type": "object" + }, "SystemConfigFacesDto": { "properties": { "import": { diff --git a/server/src/config.ts b/server/src/config.ts index 476b0eb160..16b0e9403f 100644 --- a/server/src/config.ts +++ b/server/src/config.ts @@ -45,6 +45,9 @@ export type SystemConfig = { accel: TranscodeHardwareAcceleration; accelDecode: boolean; tonemap: ToneMapping; + realtime: { + enabled: boolean; + }; }; job: Record; logging: { @@ -224,6 +227,9 @@ export const defaults = Object.freeze({ tonemap: ToneMapping.Hable, accel: TranscodeHardwareAcceleration.Disabled, accelDecode: false, + realtime: { + enabled: false, + }, }, job: { [QueueName.BackgroundTask]: { concurrency: 5 }, diff --git a/server/src/constants.ts b/server/src/constants.ts index c771c1a995..54fce33e95 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -1,7 +1,15 @@ import { readFileSync } from 'node:fs'; import { dirname, join } from 'node:path'; import { SemVer } from 'semver'; -import { ApiTag, AudioCodec, DatabaseExtension, ExifOrientation, VectorIndex } from 'src/enum'; +import { + ApiTag, + AudioCodec, + DatabaseExtension, + ExifOrientation, + TranscodeHardwareAcceleration, + VectorIndex, + VideoCodec, +} from 'src/enum'; export const IMMICH_SERVER_START = 'Immich Server is listening'; @@ -203,3 +211,35 @@ export const AUDIO_ENCODER: Record = { [AudioCodec.Opus]: 'libopus', [AudioCodec.PcmS16le]: 'pcm_s16le', }; + +export const SUPPORTED_HWA_CODECS: Record = { + [TranscodeHardwareAcceleration.Nvenc]: [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Av1], + [TranscodeHardwareAcceleration.Qsv]: [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Vp9, VideoCodec.Av1], + [TranscodeHardwareAcceleration.Vaapi]: [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Vp9, VideoCodec.Av1], + [TranscodeHardwareAcceleration.Rkmpp]: [VideoCodec.H264, VideoCodec.Hevc], + [TranscodeHardwareAcceleration.Disabled]: [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Vp9, VideoCodec.Av1], +}; + +export const HLS_BACKPRESSURE_PAUSE_SEGMENTS = 30; +export const HLS_BACKPRESSURE_RESUME_SEGMENTS = 15; +export const HLS_INACTIVITY_SWEEP_INTERVAL_MS = 60 * 1000; +export const HLS_INACTIVITY_TIMEOUT_MS = 5 * 60 * 1000; +export const HLS_LEASE_DURATION_MS = 30 * 60 * 1000; +export const HLS_SEGMENT_DURATION = 2; +export const HLS_VARIANTS = [ + { resolution: 480, codec: VideoCodec.Av1, bitrate: 1_000_000, codecString: 'av01.0.04M.08' }, + { resolution: 480, codec: VideoCodec.Hevc, bitrate: 1_200_000, codecString: 'hvc1.1.6.L90.B0' }, + { resolution: 480, codec: VideoCodec.H264, bitrate: 2_500_000, codecString: 'avc1.64001e' }, + { resolution: 720, codec: VideoCodec.Av1, bitrate: 2_000_000, codecString: 'av01.0.08M.08' }, + { resolution: 720, codec: VideoCodec.Hevc, bitrate: 2_500_000, codecString: 'hvc1.1.6.L93.B0' }, + { resolution: 720, codec: VideoCodec.H264, bitrate: 5_000_000, codecString: 'avc1.64001f' }, + { resolution: 1080, codec: VideoCodec.Av1, bitrate: 4_000_000, codecString: 'av01.0.09M.08' }, + { resolution: 1080, codec: VideoCodec.Hevc, bitrate: 4_500_000, codecString: 'hvc1.1.6.L120.B0' }, + { resolution: 1080, codec: VideoCodec.H264, bitrate: 8_000_000, codecString: 'avc1.640028' }, + { resolution: 1440, codec: VideoCodec.Av1, bitrate: 7_000_000, codecString: 'av01.0.12M.08' }, + { resolution: 1440, codec: VideoCodec.Hevc, bitrate: 8_000_000, codecString: 'hvc1.2.4.L150.B0' }, + { resolution: 1440, codec: VideoCodec.H264, bitrate: 16_000_000, codecString: 'avc1.640032' }, +]; +export const HLS_VERSION = 6; +export const PLAYLIST_CONTENT_TYPE = 'application/vnd.apple.mpegurl'; +export const SEGMENT_FILENAME_REGEX = /^seg_(\d+)\.m4s$/; diff --git a/server/src/controllers/index.ts b/server/src/controllers/index.ts index dc3754ce24..336ea1cf91 100644 --- a/server/src/controllers/index.ts +++ b/server/src/controllers/index.ts @@ -35,6 +35,7 @@ import { TimelineController } from 'src/controllers/timeline.controller'; import { TrashController } from 'src/controllers/trash.controller'; import { UserAdminController } from 'src/controllers/user-admin.controller'; import { UserController } from 'src/controllers/user.controller'; +import { VideoStreamController } from 'src/controllers/video-stream.controller'; import { ViewController } from 'src/controllers/view.controller'; import { WorkflowController } from 'src/controllers/workflow.controller'; @@ -76,6 +77,7 @@ export const controllers = [ TrashController, UserAdminController, UserController, + VideoStreamController, ViewController, WorkflowController, ]; diff --git a/server/src/controllers/video-stream.controller.ts b/server/src/controllers/video-stream.controller.ts new file mode 100644 index 0000000000..9eeb605932 --- /dev/null +++ b/server/src/controllers/video-stream.controller.ts @@ -0,0 +1,79 @@ +import { Controller, Delete, Get, Header, HttpCode, HttpStatus, Next, Param, Res } from '@nestjs/common'; +import { ApiProduces, ApiTags } from '@nestjs/swagger'; +import { NextFunction, Response } from 'express'; +import { PLAYLIST_CONTENT_TYPE } from 'src/constants'; +import { Endpoint, HistoryBuilder } from 'src/decorators'; +import { AuthDto } from 'src/dtos/auth.dto'; +import { HlsSegmentParamDto, HlsSessionParamDto, HlsVariantParamDto } from 'src/dtos/streaming.dto'; +import { ApiTag, Permission, RouteKey } from 'src/enum'; +import { Auth, Authenticated, FileResponse } from 'src/middleware/auth.guard'; +import { LoggingRepository } from 'src/repositories/logging.repository'; +import { HlsService } from 'src/services/hls.service'; +import { sendFile } from 'src/utils/file'; +import { UUIDParamDto } from 'src/validation'; + +@ApiTags(ApiTag.Assets) +@Controller(RouteKey.Asset) +export class VideoStreamController { + constructor( + private logger: LoggingRepository, + private service: HlsService, + ) {} + + @Get(':id/video/stream/master.m3u8') + @Authenticated({ permission: Permission.AssetView, sharedLink: true }) + @Header('Cache-Control', 'no-cache') + @Header('Content-Type', PLAYLIST_CONTENT_TYPE) + @ApiProduces(PLAYLIST_CONTENT_TYPE) + @Endpoint({ + summary: 'Get HLS master playlist', + description: 'Returns an HLS master playlist with all available variants for the asset.', + history: new HistoryBuilder().added('v3').alpha('v3'), + }) + getMasterPlaylist(@Auth() auth: AuthDto, @Param() { id }: UUIDParamDto) { + return this.service.getMasterPlaylist(auth, id); + } + + @Get(':id/video/stream/:sessionId/:variantIndex/playlist.m3u8') + @Authenticated({ permission: Permission.AssetView, sharedLink: true }) + @Header('Cache-Control', 'no-cache') + @Header('Content-Type', PLAYLIST_CONTENT_TYPE) + @ApiProduces(PLAYLIST_CONTENT_TYPE) + @Endpoint({ + summary: 'Get HLS media playlist', + description: 'Returns an HLS media playlist for one variant of the streaming session.', + history: new HistoryBuilder().added('v3').alpha('v3'), + }) + getMediaPlaylist(@Auth() auth: AuthDto, @Param() { id, sessionId }: HlsVariantParamDto) { + return this.service.getMediaPlaylist(auth, id, sessionId); + } + + @Get(':id/video/stream/:sessionId/:variantIndex/:filename') + @FileResponse() + @Authenticated({ permission: Permission.AssetView, sharedLink: true }) + @Endpoint({ + summary: 'Get HLS segment or init file', + description: 'Streams an HLS init segment (init.mp4) or media segment (seg_N.m4s).', + history: new HistoryBuilder().added('v3').alpha('v3'), + }) + async getSegment( + @Auth() auth: AuthDto, + @Param() { id, sessionId, variantIndex, filename }: HlsSegmentParamDto, + @Res() res: Response, + @Next() next: NextFunction, + ) { + await sendFile(res, next, () => this.service.getSegment(auth, id, sessionId, variantIndex, filename), this.logger); + } + + @Delete(':id/video/stream/:sessionId') + @HttpCode(HttpStatus.NO_CONTENT) + @Authenticated({ permission: Permission.AssetView, sharedLink: true }) + @Endpoint({ + summary: 'End HLS streaming session', + description: 'Releases server resources for the streaming session.', + history: new HistoryBuilder().added('v3').alpha('v3'), + }) + async endSession(@Auth() auth: AuthDto, @Param() { id, sessionId }: HlsSessionParamDto) { + await this.service.endSession(auth, id, sessionId); + } +} diff --git a/server/src/dtos/streaming.dto.ts b/server/src/dtos/streaming.dto.ts new file mode 100644 index 0000000000..5270e45fc2 --- /dev/null +++ b/server/src/dtos/streaming.dto.ts @@ -0,0 +1,26 @@ +import { createZodDto } from 'nestjs-zod'; +import z from 'zod'; + +const HlsSessionParamSchema = z.object({ + id: z.uuidv4(), + sessionId: z.uuidv4(), +}); + +export class HlsSessionParamDto extends createZodDto(HlsSessionParamSchema) {} + +const HlsVariantParamSchema = z.object({ + id: z.uuidv4(), + sessionId: z.uuidv4(), + variantIndex: z.coerce.number().int().min(0), +}); + +export class HlsVariantParamDto extends createZodDto(HlsVariantParamSchema) {} + +const HlsSegmentParamSchema = z.object({ + id: z.uuidv4(), + sessionId: z.uuidv4(), + variantIndex: z.coerce.number().int().min(0), + filename: z.string().regex(/^(init\.mp4|seg_\d+\.m4s)$/, { error: 'Invalid HLS segment filename' }), +}); + +export class HlsSegmentParamDto extends createZodDto(HlsSegmentParamSchema) {} diff --git a/server/src/dtos/system-config.dto.ts b/server/src/dtos/system-config.dto.ts index 4563405093..c8d795dcad 100644 --- a/server/src/dtos/system-config.dto.ts +++ b/server/src/dtos/system-config.dto.ts @@ -83,6 +83,11 @@ const SystemConfigFFmpegSchema = z accel: TranscodeHardwareAccelerationSchema, accelDecode: configBool.describe('Accelerated decode'), tonemap: ToneMappingSchema, + realtime: z + .object({ + enabled: configBool.describe('Enable real-time HLS transcoding (alpha)'), + }) + .meta({ id: 'SystemConfigFFmpegRealtimeDto' }), }) .meta({ id: 'SystemConfigFFmpegDto' }); diff --git a/server/src/enum.ts b/server/src/enum.ts index 74f1142fec..28aca072ef 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -445,11 +445,7 @@ export enum VideoCodec { export const VideoCodecSchema = z.enum(VideoCodec).describe('Target video codec').meta({ id: 'VideoCodec' }); -export enum VideoSegmentCodec { - Av1 = 'av1', - Hevc = 'hevc', - H264 = 'h264', -} +export type VideoSegmentCodec = VideoCodec.Av1 | VideoCodec.Hevc | VideoCodec.H264; export enum AudioCodec { Mp3 = 'mp3', diff --git a/server/src/repositories/event.repository.ts b/server/src/repositories/event.repository.ts index 713828cd95..5de5cdfe29 100644 --- a/server/src/repositories/event.repository.ts +++ b/server/src/repositories/event.repository.ts @@ -92,6 +92,14 @@ type EventMap = { AuthChangePassword: [{ userId: string; currentSessionId?: string; invalidateSessions?: boolean }]; + // hls streaming events + HlsSegmentRequest: [{ sessionId: string; assetId: string; variantIndex: number; segmentIndex: number }]; + HlsSegmentResult: [{ sessionId: string; variantIndex: number; segmentIndex: number; error?: string }]; + HlsHeartbeat: [{ sessionId: string; variantIndex?: number; segmentIndex?: number }]; + HlsSessionRequest: [{ sessionId: string; assetId: string; ownerId: string }]; + HlsSessionResult: [{ sessionId: string; error?: string }]; + HlsSessionEnd: [{ sessionId: string }]; + // websocket events WebsocketConnect: [{ userId: string }]; }; diff --git a/server/src/repositories/media.repository.ts b/server/src/repositories/media.repository.ts index 088276aa17..fd6059dbc1 100644 --- a/server/src/repositories/media.repository.ts +++ b/server/src/repositories/media.repository.ts @@ -509,18 +509,43 @@ export class MediaRepository { return this.parseInt(b.bit_rate) - this.parseInt(a.bit_rate); } + /* Ported from https://code.ffmpeg.org/FFmpeg/FFmpeg/src/branch/master/fftools/ffmpeg_filter.c + * SPDX-License-Identifier: LGPL-2.1-or-later + * Copyright (c) FFmpeg authors and contributors — https://ffmpeg.org/ + * Modifications: TS port operating on probe-derived packet metadata rather than decoded AVFrames. */ private cfrOutputFrames(packets: { pts: number; duration: number }[], slotsPerTick: number) { - // Packets may be out of PTS order due to B-frames packets.sort((a, b) => a.pts - b.pts); const firstPts = packets[0].pts; let outputFrames = 0; let nextPts = 0; + const history = [0, 0, 0]; for (const pkt of packets) { - const delta = (pkt.pts - firstPts) * slotsPerTick - nextPts + pkt.duration * slotsPerTick; - const nb = delta < -1.1 ? 0 : delta > 1.1 ? Math.round(delta) : 1; + const syncIpts = (pkt.pts - firstPts) * slotsPerTick; + const duration = pkt.duration * slotsPerTick; + let delta0 = syncIpts - nextPts; + const delta = delta0 + duration; + + if (delta0 < 0 && delta > 0) { + delta0 = 0; + } + + let nb = 1; + let nbPrev = 0; + if (delta < -1.1) { + nb = 0; + } else if (delta > 1.1) { + nb = Math.round(delta); + if (delta0 > 1.1) { + nbPrev = Math.round(delta0 - 0.6); + } + } outputFrames += nb; nextPts += nb; + history[2] = history[1]; + history[1] = history[0]; + history[0] = nbPrev; } - return outputFrames; + const median = history.sort((a, b) => a - b)[1]; + return outputFrames + median; } } diff --git a/server/src/repositories/video-stream.repository.ts b/server/src/repositories/video-stream.repository.ts index e23ee4ca4c..c14179a358 100644 --- a/server/src/repositories/video-stream.repository.ts +++ b/server/src/repositories/video-stream.repository.ts @@ -8,6 +8,7 @@ import { VideoStreamSessionTable, VideoStreamVariantTable, } from 'src/schema/tables/video-stream.table'; +import { withAudioStream, withVideoFormat, withVideoPackets, withVideoStream } from 'src/utils/database'; @Injectable() export class VideoStreamRepository { @@ -27,7 +28,12 @@ export class VideoStreamRepository { @GenerateSql({ params: [DummyValue.UUID] }) getSession(id: string) { - return this.db.selectFrom('video_stream_session').selectAll().where('id', '=', id).executeTakeFirst(); + return this.db + .selectFrom('video_stream_session') + .selectAll() + .where('id', '=', id) + .where('expiresAt', '>', new Date()) + .executeTakeFirst(); } @GenerateSql({ params: [DummyValue.UUID] }) @@ -47,7 +53,12 @@ export class VideoStreamRepository { @GenerateSql() getExpiredSessions() { - return this.db.selectFrom('video_stream_session').select(['id']).where('expiresAt', '<=', new Date()).execute(); + return this.db + .selectFrom('video_stream_session') + .innerJoin('asset', 'asset.id', 'video_stream_session.assetId') + .select(['video_stream_session.id', 'asset.ownerId']) + .where('video_stream_session.expiresAt', '<=', new Date()) + .execute(); } @GenerateSql({ params: [DummyValue.UUID, DummyValue.DATE] }) @@ -59,4 +70,50 @@ export class VideoStreamRepository { async deleteSession(id: string) { await this.db.deleteFrom('video_stream_session').where('id', '=', id).execute(); } + + @GenerateSql({ params: [DummyValue.UUID] }) + async getForMasterPlaylist(id: string) { + return this.db + .selectFrom('asset') + .innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId') + .where('asset.id', '=', id) + .innerJoin('asset_video', 'asset.id', 'asset_video.assetId') + .innerJoin('asset_keyframe', 'asset.id', 'asset_keyframe.assetId') + .select((eb) => withVideoStream(eb).$notNull().as('videoStream')) + .select((eb) => withVideoPackets(eb).$notNull().as('packets')) + .executeTakeFirst(); + } + + @GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] }) + async getForMediaPlaylist(id: string, sessionId: string) { + return this.db + .selectFrom('asset') + .innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId') + .innerJoin('video_stream_session', 'asset.id', 'video_stream_session.assetId') + .where('asset.id', '=', id) + .where('video_stream_session.id', '=', sessionId) + .where('video_stream_session.expiresAt', '>', new Date()) + .innerJoin('asset_video', 'asset.id', 'asset_video.assetId') + .innerJoin('asset_keyframe', 'asset.id', 'asset_keyframe.assetId') + .select((eb) => withVideoStream(eb).$notNull().as('videoStream')) + .select((eb) => withVideoPackets(eb).$notNull().as('packets')) + .executeTakeFirst(); + } + + @GenerateSql({ params: [DummyValue.UUID] }) + async getForTranscoding(id: string) { + return this.db + .selectFrom('asset') + .innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId') + .where('asset.id', '=', id) + .leftJoin('asset_audio', 'asset.id', 'asset_audio.assetId') + .innerJoin('asset_video', 'asset.id', 'asset_video.assetId') + .innerJoin('asset_keyframe', 'asset.id', 'asset_keyframe.assetId') + .select('asset.originalPath') + .select((eb) => withAudioStream(eb).as('audioStream')) + .select((eb) => withVideoStream(eb).$notNull().as('videoStream')) + .select((eb) => withVideoFormat(eb).$notNull().as('format')) + .select((eb) => withVideoPackets(eb).$notNull().as('packets')) + .executeTakeFirst(); + } } diff --git a/server/src/repositories/websocket.repository.ts b/server/src/repositories/websocket.repository.ts index b4a0fcc00a..4f2ab07ced 100644 --- a/server/src/repositories/websocket.repository.ts +++ b/server/src/repositories/websocket.repository.ts @@ -16,7 +16,16 @@ import { AppRestartEvent, ArgsOf, EventRepository } from 'src/repositories/event import { LoggingRepository } from 'src/repositories/logging.repository'; import { handlePromiseError } from 'src/utils/misc'; -export const serverEvents = ['ConfigUpdate', 'AppRestart'] as const; +export const serverEvents = [ + 'ConfigUpdate', + 'AppRestart', + 'HlsSegmentRequest', + 'HlsSegmentResult', + 'HlsHeartbeat', + 'HlsSessionRequest', + 'HlsSessionResult', + 'HlsSessionEnd', +] as const; export type ServerEvents = (typeof serverEvents)[number]; export interface ClientEventMap { diff --git a/server/src/schema/enums.ts b/server/src/schema/enums.ts index 73f8133441..ecf559c39d 100644 --- a/server/src/schema/enums.ts +++ b/server/src/schema/enums.ts @@ -1,12 +1,5 @@ import { registerEnum } from '@immich/sql-tools'; -import { - AlbumUserRole, - AssetStatus, - AssetVisibility, - ChecksumAlgorithm, - SourceType, - VideoSegmentCodec, -} from 'src/enum'; +import { AlbumUserRole, AssetStatus, AssetVisibility, ChecksumAlgorithm, SourceType, VideoCodec } from 'src/enum'; export const album_user_role_enum = registerEnum({ name: 'album_user_role_enum', @@ -35,5 +28,5 @@ export const asset_checksum_algorithm_enum = registerEnum({ export const video_stream_variant_codec_enum = registerEnum({ name: 'video_stream_variant_codec_enum', - values: Object.values(VideoSegmentCodec), + values: [VideoCodec.Av1, VideoCodec.Hevc, VideoCodec.H264], }); diff --git a/server/src/services/hls.service.spec.ts b/server/src/services/hls.service.spec.ts new file mode 100644 index 0000000000..f6682c89c8 --- /dev/null +++ b/server/src/services/hls.service.spec.ts @@ -0,0 +1,322 @@ +import { BadRequestException, NotFoundException } from '@nestjs/common'; +import { TranscodeHardwareAcceleration } from 'src/enum'; +import { HlsService } from 'src/services/hls.service'; +import { eiffelTower, train, waterfall } from 'test/fixtures/media.stub'; +import { factory } from 'test/small.factory'; +import { newTestService, ServiceMocks } from 'test/utils'; + +// EXTINF values come from FFmpeg's playlist to enforce an exact match +const eiffelExpectedMediaPlaylist = `#EXTM3U +#EXT-X-VERSION:6 +#EXT-X-TARGETDURATION:2 +#EXT-X-MEDIA-SEQUENCE:0 +#EXT-X-PLAYLIST-TYPE:VOD +#EXT-X-MAP:URI="init.mp4" +#EXTINF:2.007222, +seg_0.m4s +#EXTINF:2.007222, +seg_1.m4s +#EXTINF:2.007222, +seg_2.m4s +#EXTINF:2.007222, +seg_3.m4s +#EXTINF:2.007222, +seg_4.m4s +#EXTINF:2.007222, +seg_5.m4s +#EXTINF:2.007222, +seg_6.m4s +#EXTINF:2.007222, +seg_7.m4s +#EXTINF:2.007222, +seg_8.m4s +#EXTINF:2.007222, +seg_9.m4s +#EXTINF:2.007222, +seg_10.m4s +#EXTINF:0.281011, +seg_11.m4s +#EXT-X-ENDLIST +`; + +const waterfallExpectedMediaPlaylist = `#EXTM3U +#EXT-X-VERSION:6 +#EXT-X-TARGETDURATION:2 +#EXT-X-MEDIA-SEQUENCE:0 +#EXT-X-PLAYLIST-TYPE:VOD +#EXT-X-MAP:URI="init.mp4" +#EXTINF:2.011405, +seg_0.m4s +#EXTINF:2.011405, +seg_1.m4s +#EXTINF:2.011405, +seg_2.m4s +#EXTINF:2.011405, +seg_3.m4s +#EXTINF:2.011405, +seg_4.m4s +#EXTINF:0.301711, +seg_5.m4s +#EXT-X-ENDLIST +`; + +const trainExpectedMediaPlaylist = `#EXTM3U +#EXT-X-VERSION:6 +#EXT-X-TARGETDURATION:2 +#EXT-X-MEDIA-SEQUENCE:0 +#EXT-X-PLAYLIST-TYPE:VOD +#EXT-X-MAP:URI="init.mp4" +#EXTINF:2.000000, +seg_0.m4s +#EXTINF:2.000000, +seg_1.m4s +#EXTINF:2.000000, +seg_2.m4s +#EXTINF:2.000000, +seg_3.m4s +#EXTINF:2.000000, +seg_4.m4s +#EXTINF:2.000000, +seg_5.m4s +#EXTINF:2.000000, +seg_6.m4s +#EXTINF:2.000000, +seg_7.m4s +#EXTINF:2.000000, +seg_8.m4s +#EXTINF:2.000000, +seg_9.m4s +#EXTINF:1.733333, +seg_10.m4s +#EXT-X-ENDLIST +`; + +const sessionId = '00000000-0000-0000-0000-000000000000'; + +const eiffelExpectedMasterDisabled = `#EXTM3U +#EXT-X-VERSION:6 +#EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/0/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/1/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/2/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=2000000,RESOLUTION=720x1280,CODECS="av01.0.08M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/3/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/4/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/5/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=4000000,RESOLUTION=1080x1920,CODECS="av01.0.09M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/6/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=4500000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/7/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/8/playlist.m3u8 +`; + +const eiffelExpectedMasterRkmpp = `#EXTM3U +#EXT-X-VERSION:6 +#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/1/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/2/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/4/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/5/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=4500000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/7/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910 +${sessionId}/8/playlist.m3u8 +`; + +const waterfallExpectedMasterDisabled = `#EXTM3U +#EXT-X-VERSION:6 +#EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/0/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/1/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/2/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=2000000,RESOLUTION=720x1280,CODECS="av01.0.08M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/3/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/4/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/5/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=4000000,RESOLUTION=1080x1920,CODECS="av01.0.09M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/6/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=4500000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/7/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/8/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=7000000,RESOLUTION=1440x2560,CODECS="av01.0.12M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/9/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1440x2560,CODECS="hvc1.2.4.L150.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/10/playlist.m3u8 +#EXT-X-STREAM-INF:BANDWIDTH=16000000,RESOLUTION=1440x2560,CODECS="avc1.640032,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830 +${sessionId}/11/playlist.m3u8 +`; + +describe(HlsService.name, () => { + let sut: HlsService; + let mocks: ServiceMocks; + + beforeEach(() => { + ({ sut, mocks } = newTestService(HlsService)); + }); + + describe('getMasterPlaylist', () => { + const auth = factory.auth(); + const assetId = 'asset-1'; + + const setup = (asset: typeof eiffelTower | typeof waterfall, accel: TranscodeHardwareAcceleration) => { + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId])); + mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { realtime: { enabled: true }, accel } }); + mocks.videoStream.getForMasterPlaylist.mockResolvedValue(asset); + mocks.crypto.randomUUID.mockReturnValue(sessionId); + mocks.websocket.serverSend.mockImplementation((event, ...rest) => { + if (event === 'HlsSessionRequest') { + const { sessionId: id } = rest[0] as { sessionId: string }; + queueMicrotask(() => sut.onSessionResult({ sessionId: id })); + } + }); + }; + + it('returns master playlist for eiffel-tower (1080p portrait, no acceleration)', async () => { + setup(eiffelTower, TranscodeHardwareAcceleration.Disabled); + await expect(sut.getMasterPlaylist(auth, assetId)).resolves.toBe(eiffelExpectedMasterDisabled); + }); + + it('returns master playlist for eiffel-tower with RKMPP (no AV1 variants)', async () => { + setup(eiffelTower, TranscodeHardwareAcceleration.Rkmpp); + await expect(sut.getMasterPlaylist(auth, assetId)).resolves.toBe(eiffelExpectedMasterRkmpp); + }); + + it('returns master playlist for waterfall (4K landscape) with no acceleration', async () => { + setup(waterfall, TranscodeHardwareAcceleration.Disabled); + await expect(sut.getMasterPlaylist(auth, assetId)).resolves.toBe(waterfallExpectedMasterDisabled); + }); + + it('throws BadRequestException when realtime transcoding is disabled', async () => { + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId])); + mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { realtime: { enabled: false } } }); + await expect(sut.getMasterPlaylist(auth, assetId)).rejects.toBeInstanceOf(BadRequestException); + }); + + it('throws NotFoundException when asset is not yet ready for streaming', async () => { + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId])); + mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { realtime: { enabled: true } } }); + await expect(sut.getMasterPlaylist(auth, assetId)).rejects.toBeInstanceOf(NotFoundException); + }); + }); + + describe('getMediaPlaylist', () => { + const auth = factory.auth(); + const assetId = 'asset-1'; + const fixtures = [ + { data: eiffelTower, playlist: eiffelExpectedMediaPlaylist }, + { data: waterfall, playlist: waterfallExpectedMediaPlaylist }, + { data: train, playlist: trainExpectedMediaPlaylist }, + ]; + + it.each(fixtures)('matches FFmpeg for $data.originalPath', async ({ data, playlist }) => { + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId])); + mocks.videoStream.getForMediaPlaylist.mockResolvedValue(data); + await expect(sut.getMediaPlaylist(auth, assetId, sessionId)).resolves.toBe(playlist); + }); + + it('throws NotFoundException when the session/asset cannot be loaded', async () => { + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId])); + await expect(sut.getMediaPlaylist(auth, assetId, sessionId)).rejects.toBeInstanceOf(NotFoundException); + }); + }); + + describe('getSegment', () => { + const auth = factory.auth(); + const assetId = 'asset-1'; + const variantIndex = 0; + + beforeEach(() => { + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId])); + mocks.videoStream.getSession.mockResolvedValue({ id: sessionId, assetId } as never); + mocks.storage.checkFileExists.mockResolvedValue(true); + }); + + it('emits HlsHeartbeat with segmentIndex 0 for the first init.mp4 request', async () => { + await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4'); + expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', { + sessionId, + variantIndex, + segmentIndex: 0, + }); + }); + + it('emits HlsHeartbeat with the parsed segment number for seg_K.m4s', async () => { + await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s'); + expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', { + sessionId, + variantIndex, + segmentIndex: 5, + }); + }); + + it('returns lastRequested + 1 for init.mp4 after a segment has been served', async () => { + await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s'); + mocks.websocket.serverSend.mockClear(); + await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4'); + expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', { + sessionId, + variantIndex, + segmentIndex: 6, + }); + }); + + it('updates lastRequested on a backward-seek segment request', async () => { + await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s'); + await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_3.m4s'); + mocks.websocket.serverSend.mockClear(); + await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4'); + expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', { + sessionId, + variantIndex, + segmentIndex: 4, + }); + }); + + it('tracks segment state per session independently', async () => { + await sut.getSegment(auth, assetId, 'session-a', variantIndex, 'seg_5.m4s'); + await sut.getSegment(auth, assetId, 'session-b', variantIndex, 'seg_2.m4s'); + mocks.websocket.serverSend.mockClear(); + await sut.getSegment(auth, assetId, 'session-a', variantIndex, 'init.mp4'); + await sut.getSegment(auth, assetId, 'session-b', variantIndex, 'init.mp4'); + expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', { + sessionId: 'session-a', + variantIndex, + segmentIndex: 6, + }); + expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', { + sessionId: 'session-b', + variantIndex, + segmentIndex: 3, + }); + }); + + it('throws NotFoundException when the session does not exist', async () => { + await expect(sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4')).rejects.toBeInstanceOf( + NotFoundException, + ); + }); + }); + + describe('endSession', () => { + it('emits HlsSessionEnd', async () => { + const auth = factory.auth(); + const assetId = 'asset-1'; + mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId])); + await sut.endSession(auth, assetId, sessionId); + expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsSessionEnd', { sessionId }); + }); + }); +}); diff --git a/server/src/services/hls.service.ts b/server/src/services/hls.service.ts new file mode 100644 index 0000000000..50b5998026 --- /dev/null +++ b/server/src/services/hls.service.ts @@ -0,0 +1,202 @@ +import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common'; +import { constants } from 'node:fs'; +import { join } from 'node:path'; +import { + HLS_SEGMENT_DURATION, + HLS_VARIANTS, + HLS_VERSION, + SEGMENT_FILENAME_REGEX, + SUPPORTED_HWA_CODECS, +} from 'src/constants'; +import { StorageCore } from 'src/cores/storage.core'; +import { OnEvent } from 'src/decorators'; +import { AuthDto } from 'src/dtos/auth.dto'; +import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto'; +import { CacheControl, ImmichWorker, Permission } from 'src/enum'; +import { ArgOf } from 'src/repositories/event.repository'; +import { BaseService } from 'src/services/base.service'; +import { VideoPacketInfo, VideoStreamInfo } from 'src/types'; +import { PendingEvents } from 'src/utils/event'; +import { ImmichFileResponse } from 'src/utils/file'; +import { getOutputSize } from 'src/utils/media'; + +type AssetWithStreamInfo = { videoStream: VideoStreamInfo & { timeBase: number }; packets: VideoPacketInfo }; + +@Injectable() +export class HlsService extends BaseService { + private pendingSegments = new PendingEvents<'HlsSegmentResult'>(15_000); + private pendingSessions = new PendingEvents<'HlsSessionResult'>(5000); + private sessions = new Map(); + + @OnEvent({ name: 'HlsSessionResult', server: true, workers: [ImmichWorker.Api] }) + onSessionResult(event: ArgOf<'HlsSessionResult'>) { + this.pendingSessions.complete(event.sessionId, event); + if (event.error) { + this.sessions.delete(event.sessionId); + this.pendingSegments.rejectByPrefix(`${event.sessionId}:`, event.error); + } + } + + @OnEvent({ name: 'HlsSessionEnd', server: true, workers: [ImmichWorker.Api] }) + onSessionEnd({ sessionId }: ArgOf<'HlsSessionEnd'>) { + this.sessions.delete(sessionId); + this.pendingSegments.rejectByPrefix(`${sessionId}:`, 'Session ended'); + } + + @OnEvent({ name: 'HlsSegmentResult', server: true, workers: [ImmichWorker.Api] }) + onSegmentResult(event: ArgOf<'HlsSegmentResult'>) { + this.pendingSegments.complete(this.getSegmentKey(event), event); + } + + async getMasterPlaylist(auth: AuthDto, assetId: string) { + await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] }); + const { ffmpeg } = await this.getConfig({ withCache: true }); + if (!ffmpeg.realtime.enabled) { + throw new BadRequestException('Real-time transcoding is not enabled'); + } + + const asset = await this.videoStreamRepository.getForMasterPlaylist(assetId); + if (!asset) { + throw new NotFoundException('Asset is not yet ready for streaming'); + } + + const sessionId = this.cryptoRepository.randomUUID(); + this.websocketRepository.serverSend('HlsSessionRequest', { sessionId, assetId, ownerId: auth.user.id }); + await this.pendingSessions.wait(sessionId); + this.sessions.set(sessionId, { lastRequestedSegment: null }); + + return this.generateMasterPlaylist(sessionId, ffmpeg, asset); + } + + async getMediaPlaylist(auth: AuthDto, assetId: string, sessionId: string) { + await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] }); + + const asset = await this.videoStreamRepository.getForMediaPlaylist(assetId, sessionId); + if (!asset) { + throw new NotFoundException('Asset not found or not yet ready for streaming'); + } + + return this.generateMediaPlaylist(asset); + } + + async getSegment(auth: AuthDto, assetId: string, sessionId: string, variantIndex: number, filename: string) { + const t0 = performance.now(); + await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] }); + const t1 = performance.now(); + + const session = await this.videoStreamRepository.getSession(sessionId); + if (!session) { + throw new NotFoundException('Session not found'); + } + const t2 = performance.now(); + + const variantDir = StorageCore.getHlsVariantFolder({ ownerId: auth.user.id, sessionId, variantIndex }); + const path = join(variantDir, filename); + const response = new ImmichFileResponse({ + path, + contentType: 'video/mp4', + cacheControl: CacheControl.PrivateWithCache, + }); + + const segmentIndex = this.getSegmentIndex(sessionId, filename); + this.websocketRepository.serverSend('HlsHeartbeat', { sessionId, variantIndex, segmentIndex }); + + if (await this.storageRepository.checkFileExists(path, constants.R_OK)) { + this.logger.log( + `[TIMING] getSegment(cached) session=${sessionId} variant=${variantIndex} file=${filename} ` + + `auth=${(t1 - t0).toFixed(1)}ms session=${(t2 - t1).toFixed(1)}ms ` + + `total=${(performance.now() - t0).toFixed(1)}ms`, + ); + return response; + } + const t3 = performance.now(); + + this.websocketRepository.serverSend('HlsSegmentRequest', { sessionId, assetId, variantIndex, segmentIndex }); + await this.pendingSegments.wait(this.getSegmentKey({ sessionId, variantIndex, segmentIndex })); + const t4 = performance.now(); + + this.logger.log( + `[TIMING] getSegment(wait) session=${sessionId} variant=${variantIndex} file=${filename} ` + + `auth=${(t1 - t0).toFixed(1)}ms session=${(t2 - t1).toFixed(1)}ms ` + + `existsCheck=${(t3 - t2).toFixed(1)}ms wait=${(t4 - t3).toFixed(1)}ms ` + + `total=${(t4 - t0).toFixed(1)}ms`, + ); + + return response; + } + + async endSession(auth: AuthDto, assetId: string, sessionId: string): Promise { + await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] }); + + this.websocketRepository.serverSend('HlsSessionEnd', { sessionId }); + } + + private generateMasterPlaylist(sessionId: string, ffmpeg: SystemConfigFFmpegDto, asset: AssetWithStreamInfo) { + const fps = ((asset.packets.packetCount * asset.videoStream.timeBase) / asset.packets.totalDuration).toFixed(3); + const sourceResolution = Math.min(asset.videoStream.height, asset.videoStream.width); + const lines = ['#EXTM3U', `#EXT-X-VERSION:${HLS_VERSION}`]; + for (let i = 0; i < HLS_VARIANTS.length; i++) { + const { resolution, bitrate, codec, codecString } = HLS_VARIANTS[i]; + if (resolution > sourceResolution || !SUPPORTED_HWA_CODECS[ffmpeg.accel].includes(codec)) { + continue; + } + const { width, height } = getOutputSize(asset.videoStream, resolution); + lines.push( + `#EXT-X-STREAM-INF:BANDWIDTH=${bitrate},RESOLUTION=${width}x${height},CODECS="${codecString},mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=${fps}`, + `${sessionId}/${i}/playlist.m3u8`, + ); + } + lines.push(''); + + if (lines.length === 3) { + throw new NotFoundException('No supported variants for this video'); + } + + return lines.join('\n'); + } + + private generateMediaPlaylist({ videoStream, packets }: AssetWithStreamInfo) { + const fps = (packets.packetCount * videoStream.timeBase) / packets.totalDuration; + const framesPerSegment = Math.ceil(HLS_SEGMENT_DURATION * fps); + const fullSegmentDuration = framesPerSegment / fps; + const segmentCount = Math.ceil(packets.outputFrames / framesPerSegment); + const lastSegmentFrames = packets.outputFrames - framesPerSegment * (segmentCount - 1); + const lastSegmentDuration = lastSegmentFrames / fps; + + const lines = [ + '#EXTM3U', + `#EXT-X-VERSION:${HLS_VERSION}`, + `#EXT-X-TARGETDURATION:${HLS_SEGMENT_DURATION}`, + '#EXT-X-MEDIA-SEQUENCE:0', + '#EXT-X-PLAYLIST-TYPE:VOD', + '#EXT-X-MAP:URI="init.mp4"', + ]; + + for (let i = 0; i < segmentCount - 1; i++) { + lines.push(`#EXTINF:${fullSegmentDuration.toFixed(6)},`, `seg_${i}.m4s`); + } + lines.push(`#EXTINF:${lastSegmentDuration.toFixed(6)},`, `seg_${segmentCount - 1}.m4s`, '#EXT-X-ENDLIST', ''); + + return lines.join('\n'); + } + + private getSegmentKey({ sessionId, variantIndex, segmentIndex }: ArgOf<'HlsSegmentResult'>) { + return `${sessionId}:${variantIndex}:${segmentIndex}`; + } + + private getSegmentIndex(sessionId: string, filename: string) { + const existing = this.sessions.get(sessionId); + let segmentIndex: number; + if (filename.endsWith('.mp4')) { + segmentIndex = (existing?.lastRequestedSegment ?? -1) + 1; + } else { + segmentIndex = Number.parseInt(SEGMENT_FILENAME_REGEX.exec(filename)![1]); + if (existing) { + existing.lastRequestedSegment = segmentIndex; + } else { + this.sessions.set(sessionId, { lastRequestedSegment: segmentIndex }); + } + } + return segmentIndex; + } +} diff --git a/server/src/services/index.ts b/server/src/services/index.ts index f4e82b13a4..6fa4883c83 100644 --- a/server/src/services/index.ts +++ b/server/src/services/index.ts @@ -11,6 +11,7 @@ import { DatabaseBackupService } from 'src/services/database-backup.service'; import { DatabaseService } from 'src/services/database.service'; import { DownloadService } from 'src/services/download.service'; import { DuplicateService } from 'src/services/duplicate.service'; +import { HlsService } from 'src/services/hls.service'; import { JobService } from 'src/services/job.service'; import { LibraryService } from 'src/services/library.service'; import { MaintenanceService } from 'src/services/maintenance.service'; @@ -60,6 +61,7 @@ export const services = [ DatabaseService, DownloadService, DuplicateService, + HlsService, JobService, LibraryService, MaintenanceService, diff --git a/server/src/services/system-config.service.spec.ts b/server/src/services/system-config.service.spec.ts index fb07eb1438..a25738c754 100644 --- a/server/src/services/system-config.service.spec.ts +++ b/server/src/services/system-config.service.spec.ts @@ -72,6 +72,9 @@ const updatedConfig = Object.freeze({ accel: TranscodeHardwareAcceleration.Disabled, accelDecode: false, tonemap: ToneMapping.Hable, + realtime: { + enabled: false, + }, }, logging: { enabled: true, diff --git a/server/src/utils/event.ts b/server/src/utils/event.ts new file mode 100644 index 0000000000..adb8c8f1df --- /dev/null +++ b/server/src/utils/event.ts @@ -0,0 +1,44 @@ +import { ArgOf, EmitEvent } from 'src/repositories/event.repository'; + +export class PendingEvents extends { error?: string } ? T : never }[EmitEvent]> { + private pending = new Map>[]; timeout: NodeJS.Timeout }>(); + + constructor(private timeoutMs: number) {} + + wait(key: string): Promise> { + const completer = Promise.withResolvers>(); + const existing = this.pending.get(key); + if (existing) { + existing.completers.push(completer); + return completer.promise; + } + + const timeout = setTimeout(() => this.complete(key, { error: 'Request timed out' }), this.timeoutMs); + this.pending.set(key, { completers: [completer], timeout }); + return completer.promise; + } + + complete(key: string, value: ArgOf | { error: string }) { + const pending = this.pending.get(key); + if (!pending) { + return; + } + clearTimeout(pending.timeout); + this.pending.delete(key); + for (const completer of pending.completers) { + if ('error' in value) { + completer.reject(new Error(value.error)); + } else { + completer.resolve(value); + } + } + } + + rejectByPrefix(prefix: string, error: string) { + for (const key of this.pending.keys()) { + if (key.startsWith(prefix)) { + this.complete(key, { error }); + } + } + } +} diff --git a/server/test/fixtures/media.stub.ts b/server/test/fixtures/media.stub.ts index f034ab873d..7edd1c2dee 100644 --- a/server/test/fixtures/media.stub.ts +++ b/server/test/fixtures/media.stub.ts @@ -597,7 +597,7 @@ export const train = { packets: { totalDuration: 12_290, packetCount: 1229, - outputFrames: 1303, + outputFrames: 1304, keyframePts: [ 0, 601, 1201, 1802, 2402, 3003, 3604, 4204, 4805, 5405, 6006, 6607, 7207, 7808, 8408, 9009, 9609, 10_210, 10_811, 11_411, 12_062, 12_703, diff --git a/server/tsconfig.json b/server/tsconfig.json index e087544f6b..c3aede3e5b 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -8,9 +8,9 @@ "experimentalDecorators": true, "allowSyntheticDefaultImports": true, "resolveJsonModule": true, - "target": "es2022", + "target": "es2024", "moduleResolution": "node16", - "lib": ["dom", "es2023"], + "lib": ["dom", "es2024"], "sourceMap": true, "outDir": "./dist", "incremental": true,