mirror of
https://github.com/immich-app/immich.git
synced 2026-05-25 09:02:31 -04:00
hls service and controller
This commit is contained in:
mertalev
@@ -45,6 +45,9 @@ export type SystemConfig = {
|
||||
accel: TranscodeHardwareAcceleration;
|
||||
accelDecode: boolean;
|
||||
tonemap: ToneMapping;
|
||||
realtime: {
|
||||
enabled: boolean;
|
||||
};
|
||||
};
|
||||
job: Record<ConcurrentQueueName, { concurrency: number }>;
|
||||
logging: {
|
||||
@@ -224,6 +227,9 @@ export const defaults = Object.freeze<SystemConfig>({
|
||||
tonemap: ToneMapping.Hable,
|
||||
accel: TranscodeHardwareAcceleration.Disabled,
|
||||
accelDecode: false,
|
||||
realtime: {
|
||||
enabled: false,
|
||||
},
|
||||
},
|
||||
job: {
|
||||
[QueueName.BackgroundTask]: { concurrency: 5 },
|
||||
|
||||
+41
-1
@@ -1,7 +1,15 @@
|
||||
import { readFileSync } from 'node:fs';
|
||||
import { dirname, join } from 'node:path';
|
||||
import { SemVer } from 'semver';
|
||||
import { ApiTag, AudioCodec, DatabaseExtension, ExifOrientation, VectorIndex } from 'src/enum';
|
||||
import {
|
||||
ApiTag,
|
||||
AudioCodec,
|
||||
DatabaseExtension,
|
||||
ExifOrientation,
|
||||
TranscodeHardwareAcceleration,
|
||||
VectorIndex,
|
||||
VideoCodec,
|
||||
} from 'src/enum';
|
||||
|
||||
export const IMMICH_SERVER_START = 'Immich Server is listening';
|
||||
|
||||
@@ -203,3 +211,35 @@ export const AUDIO_ENCODER: Record<AudioCodec, string> = {
|
||||
[AudioCodec.Opus]: 'libopus',
|
||||
[AudioCodec.PcmS16le]: 'pcm_s16le',
|
||||
};
|
||||
|
||||
export const SUPPORTED_HWA_CODECS: Record<TranscodeHardwareAcceleration, VideoCodec[]> = {
|
||||
[TranscodeHardwareAcceleration.Nvenc]: [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Av1],
|
||||
[TranscodeHardwareAcceleration.Qsv]: [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Vp9, VideoCodec.Av1],
|
||||
[TranscodeHardwareAcceleration.Vaapi]: [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Vp9, VideoCodec.Av1],
|
||||
[TranscodeHardwareAcceleration.Rkmpp]: [VideoCodec.H264, VideoCodec.Hevc],
|
||||
[TranscodeHardwareAcceleration.Disabled]: [VideoCodec.H264, VideoCodec.Hevc, VideoCodec.Vp9, VideoCodec.Av1],
|
||||
};
|
||||
|
||||
export const HLS_BACKPRESSURE_PAUSE_SEGMENTS = 30;
|
||||
export const HLS_BACKPRESSURE_RESUME_SEGMENTS = 15;
|
||||
export const HLS_INACTIVITY_SWEEP_INTERVAL_MS = 60 * 1000;
|
||||
export const HLS_INACTIVITY_TIMEOUT_MS = 5 * 60 * 1000;
|
||||
export const HLS_LEASE_DURATION_MS = 30 * 60 * 1000;
|
||||
export const HLS_SEGMENT_DURATION = 2;
|
||||
export const HLS_VARIANTS = [
|
||||
{ resolution: 480, codec: VideoCodec.Av1, bitrate: 1_000_000, codecString: 'av01.0.04M.08' },
|
||||
{ resolution: 480, codec: VideoCodec.Hevc, bitrate: 1_200_000, codecString: 'hvc1.1.6.L90.B0' },
|
||||
{ resolution: 480, codec: VideoCodec.H264, bitrate: 2_500_000, codecString: 'avc1.64001e' },
|
||||
{ resolution: 720, codec: VideoCodec.Av1, bitrate: 2_000_000, codecString: 'av01.0.08M.08' },
|
||||
{ resolution: 720, codec: VideoCodec.Hevc, bitrate: 2_500_000, codecString: 'hvc1.1.6.L93.B0' },
|
||||
{ resolution: 720, codec: VideoCodec.H264, bitrate: 5_000_000, codecString: 'avc1.64001f' },
|
||||
{ resolution: 1080, codec: VideoCodec.Av1, bitrate: 4_000_000, codecString: 'av01.0.09M.08' },
|
||||
{ resolution: 1080, codec: VideoCodec.Hevc, bitrate: 4_500_000, codecString: 'hvc1.1.6.L120.B0' },
|
||||
{ resolution: 1080, codec: VideoCodec.H264, bitrate: 8_000_000, codecString: 'avc1.640028' },
|
||||
{ resolution: 1440, codec: VideoCodec.Av1, bitrate: 7_000_000, codecString: 'av01.0.12M.08' },
|
||||
{ resolution: 1440, codec: VideoCodec.Hevc, bitrate: 8_000_000, codecString: 'hvc1.2.4.L150.B0' },
|
||||
{ resolution: 1440, codec: VideoCodec.H264, bitrate: 16_000_000, codecString: 'avc1.640032' },
|
||||
];
|
||||
export const HLS_VERSION = 6;
|
||||
export const PLAYLIST_CONTENT_TYPE = 'application/vnd.apple.mpegurl';
|
||||
export const SEGMENT_FILENAME_REGEX = /^seg_(\d+)\.m4s$/;
|
||||
|
||||
@@ -35,6 +35,7 @@ import { TimelineController } from 'src/controllers/timeline.controller';
|
||||
import { TrashController } from 'src/controllers/trash.controller';
|
||||
import { UserAdminController } from 'src/controllers/user-admin.controller';
|
||||
import { UserController } from 'src/controllers/user.controller';
|
||||
import { VideoStreamController } from 'src/controllers/video-stream.controller';
|
||||
import { ViewController } from 'src/controllers/view.controller';
|
||||
import { WorkflowController } from 'src/controllers/workflow.controller';
|
||||
|
||||
@@ -76,6 +77,7 @@ export const controllers = [
|
||||
TrashController,
|
||||
UserAdminController,
|
||||
UserController,
|
||||
VideoStreamController,
|
||||
ViewController,
|
||||
WorkflowController,
|
||||
];
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
import { Controller, Delete, Get, Header, HttpCode, HttpStatus, Next, Param, Res } from '@nestjs/common';
|
||||
import { ApiProduces, ApiTags } from '@nestjs/swagger';
|
||||
import { NextFunction, Response } from 'express';
|
||||
import { PLAYLIST_CONTENT_TYPE } from 'src/constants';
|
||||
import { Endpoint, HistoryBuilder } from 'src/decorators';
|
||||
import { AuthDto } from 'src/dtos/auth.dto';
|
||||
import { HlsSegmentParamDto, HlsSessionParamDto, HlsVariantParamDto } from 'src/dtos/streaming.dto';
|
||||
import { ApiTag, Permission, RouteKey } from 'src/enum';
|
||||
import { Auth, Authenticated, FileResponse } from 'src/middleware/auth.guard';
|
||||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
import { HlsService } from 'src/services/hls.service';
|
||||
import { sendFile } from 'src/utils/file';
|
||||
import { UUIDParamDto } from 'src/validation';
|
||||
|
||||
@ApiTags(ApiTag.Assets)
|
||||
@Controller(RouteKey.Asset)
|
||||
export class VideoStreamController {
|
||||
constructor(
|
||||
private logger: LoggingRepository,
|
||||
private service: HlsService,
|
||||
) {}
|
||||
|
||||
@Get(':id/video/stream/master.m3u8')
|
||||
@Authenticated({ permission: Permission.AssetView, sharedLink: true })
|
||||
@Header('Cache-Control', 'no-cache')
|
||||
@Header('Content-Type', PLAYLIST_CONTENT_TYPE)
|
||||
@ApiProduces(PLAYLIST_CONTENT_TYPE)
|
||||
@Endpoint({
|
||||
summary: 'Get HLS master playlist',
|
||||
description: 'Returns an HLS master playlist with all available variants for the asset.',
|
||||
history: new HistoryBuilder().added('v3').alpha('v3'),
|
||||
})
|
||||
getMasterPlaylist(@Auth() auth: AuthDto, @Param() { id }: UUIDParamDto) {
|
||||
return this.service.getMasterPlaylist(auth, id);
|
||||
}
|
||||
|
||||
@Get(':id/video/stream/:sessionId/:variantIndex/playlist.m3u8')
|
||||
@Authenticated({ permission: Permission.AssetView, sharedLink: true })
|
||||
@Header('Cache-Control', 'no-cache')
|
||||
@Header('Content-Type', PLAYLIST_CONTENT_TYPE)
|
||||
@ApiProduces(PLAYLIST_CONTENT_TYPE)
|
||||
@Endpoint({
|
||||
summary: 'Get HLS media playlist',
|
||||
description: 'Returns an HLS media playlist for one variant of the streaming session.',
|
||||
history: new HistoryBuilder().added('v3').alpha('v3'),
|
||||
})
|
||||
getMediaPlaylist(@Auth() auth: AuthDto, @Param() { id, sessionId }: HlsVariantParamDto) {
|
||||
return this.service.getMediaPlaylist(auth, id, sessionId);
|
||||
}
|
||||
|
||||
@Get(':id/video/stream/:sessionId/:variantIndex/:filename')
|
||||
@FileResponse()
|
||||
@Authenticated({ permission: Permission.AssetView, sharedLink: true })
|
||||
@Endpoint({
|
||||
summary: 'Get HLS segment or init file',
|
||||
description: 'Streams an HLS init segment (init.mp4) or media segment (seg_N.m4s).',
|
||||
history: new HistoryBuilder().added('v3').alpha('v3'),
|
||||
})
|
||||
async getSegment(
|
||||
@Auth() auth: AuthDto,
|
||||
@Param() { id, sessionId, variantIndex, filename }: HlsSegmentParamDto,
|
||||
@Res() res: Response,
|
||||
@Next() next: NextFunction,
|
||||
) {
|
||||
await sendFile(res, next, () => this.service.getSegment(auth, id, sessionId, variantIndex, filename), this.logger);
|
||||
}
|
||||
|
||||
@Delete(':id/video/stream/:sessionId')
|
||||
@HttpCode(HttpStatus.NO_CONTENT)
|
||||
@Authenticated({ permission: Permission.AssetView, sharedLink: true })
|
||||
@Endpoint({
|
||||
summary: 'End HLS streaming session',
|
||||
description: 'Releases server resources for the streaming session.',
|
||||
history: new HistoryBuilder().added('v3').alpha('v3'),
|
||||
})
|
||||
async endSession(@Auth() auth: AuthDto, @Param() { id, sessionId }: HlsSessionParamDto) {
|
||||
await this.service.endSession(auth, id, sessionId);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { createZodDto } from 'nestjs-zod';
|
||||
import z from 'zod';
|
||||
|
||||
const HlsSessionParamSchema = z.object({
|
||||
id: z.uuidv4(),
|
||||
sessionId: z.uuidv4(),
|
||||
});
|
||||
|
||||
export class HlsSessionParamDto extends createZodDto(HlsSessionParamSchema) {}
|
||||
|
||||
const HlsVariantParamSchema = z.object({
|
||||
id: z.uuidv4(),
|
||||
sessionId: z.uuidv4(),
|
||||
variantIndex: z.coerce.number().int().min(0),
|
||||
});
|
||||
|
||||
export class HlsVariantParamDto extends createZodDto(HlsVariantParamSchema) {}
|
||||
|
||||
const HlsSegmentParamSchema = z.object({
|
||||
id: z.uuidv4(),
|
||||
sessionId: z.uuidv4(),
|
||||
variantIndex: z.coerce.number().int().min(0),
|
||||
filename: z.string().regex(/^(init\.mp4|seg_\d+\.m4s)$/, { error: 'Invalid HLS segment filename' }),
|
||||
});
|
||||
|
||||
export class HlsSegmentParamDto extends createZodDto(HlsSegmentParamSchema) {}
|
||||
@@ -83,6 +83,11 @@ const SystemConfigFFmpegSchema = z
|
||||
accel: TranscodeHardwareAccelerationSchema,
|
||||
accelDecode: configBool.describe('Accelerated decode'),
|
||||
tonemap: ToneMappingSchema,
|
||||
realtime: z
|
||||
.object({
|
||||
enabled: configBool.describe('Enable real-time HLS transcoding (alpha)'),
|
||||
})
|
||||
.meta({ id: 'SystemConfigFFmpegRealtimeDto' }),
|
||||
})
|
||||
.meta({ id: 'SystemConfigFFmpegDto' });
|
||||
|
||||
|
||||
+1
-5
@@ -445,11 +445,7 @@ export enum VideoCodec {
|
||||
|
||||
export const VideoCodecSchema = z.enum(VideoCodec).describe('Target video codec').meta({ id: 'VideoCodec' });
|
||||
|
||||
export enum VideoSegmentCodec {
|
||||
Av1 = 'av1',
|
||||
Hevc = 'hevc',
|
||||
H264 = 'h264',
|
||||
}
|
||||
export type VideoSegmentCodec = VideoCodec.Av1 | VideoCodec.Hevc | VideoCodec.H264;
|
||||
|
||||
export enum AudioCodec {
|
||||
Mp3 = 'mp3',
|
||||
|
||||
@@ -92,6 +92,14 @@ type EventMap = {
|
||||
|
||||
AuthChangePassword: [{ userId: string; currentSessionId?: string; invalidateSessions?: boolean }];
|
||||
|
||||
// hls streaming events
|
||||
HlsSegmentRequest: [{ sessionId: string; assetId: string; variantIndex: number; segmentIndex: number }];
|
||||
HlsSegmentResult: [{ sessionId: string; variantIndex: number; segmentIndex: number; error?: string }];
|
||||
HlsHeartbeat: [{ sessionId: string; variantIndex?: number; segmentIndex?: number }];
|
||||
HlsSessionRequest: [{ sessionId: string; assetId: string; ownerId: string }];
|
||||
HlsSessionResult: [{ sessionId: string; error?: string }];
|
||||
HlsSessionEnd: [{ sessionId: string }];
|
||||
|
||||
// websocket events
|
||||
WebsocketConnect: [{ userId: string }];
|
||||
};
|
||||
|
||||
@@ -509,18 +509,43 @@ export class MediaRepository {
|
||||
return this.parseInt(b.bit_rate) - this.parseInt(a.bit_rate);
|
||||
}
|
||||
|
||||
/* Ported from https://code.ffmpeg.org/FFmpeg/FFmpeg/src/branch/master/fftools/ffmpeg_filter.c
|
||||
* SPDX-License-Identifier: LGPL-2.1-or-later
|
||||
* Copyright (c) FFmpeg authors and contributors — https://ffmpeg.org/
|
||||
* Modifications: TS port operating on probe-derived packet metadata rather than decoded AVFrames. */
|
||||
private cfrOutputFrames(packets: { pts: number; duration: number }[], slotsPerTick: number) {
|
||||
// Packets may be out of PTS order due to B-frames
|
||||
packets.sort((a, b) => a.pts - b.pts);
|
||||
const firstPts = packets[0].pts;
|
||||
let outputFrames = 0;
|
||||
let nextPts = 0;
|
||||
const history = [0, 0, 0];
|
||||
for (const pkt of packets) {
|
||||
const delta = (pkt.pts - firstPts) * slotsPerTick - nextPts + pkt.duration * slotsPerTick;
|
||||
const nb = delta < -1.1 ? 0 : delta > 1.1 ? Math.round(delta) : 1;
|
||||
const syncIpts = (pkt.pts - firstPts) * slotsPerTick;
|
||||
const duration = pkt.duration * slotsPerTick;
|
||||
let delta0 = syncIpts - nextPts;
|
||||
const delta = delta0 + duration;
|
||||
|
||||
if (delta0 < 0 && delta > 0) {
|
||||
delta0 = 0;
|
||||
}
|
||||
|
||||
let nb = 1;
|
||||
let nbPrev = 0;
|
||||
if (delta < -1.1) {
|
||||
nb = 0;
|
||||
} else if (delta > 1.1) {
|
||||
nb = Math.round(delta);
|
||||
if (delta0 > 1.1) {
|
||||
nbPrev = Math.round(delta0 - 0.6);
|
||||
}
|
||||
}
|
||||
outputFrames += nb;
|
||||
nextPts += nb;
|
||||
history[2] = history[1];
|
||||
history[1] = history[0];
|
||||
history[0] = nbPrev;
|
||||
}
|
||||
return outputFrames;
|
||||
const median = history.sort((a, b) => a - b)[1];
|
||||
return outputFrames + median;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
VideoStreamSessionTable,
|
||||
VideoStreamVariantTable,
|
||||
} from 'src/schema/tables/video-stream.table';
|
||||
import { withAudioStream, withVideoFormat, withVideoPackets, withVideoStream } from 'src/utils/database';
|
||||
|
||||
@Injectable()
|
||||
export class VideoStreamRepository {
|
||||
@@ -27,7 +28,12 @@ export class VideoStreamRepository {
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID] })
|
||||
getSession(id: string) {
|
||||
return this.db.selectFrom('video_stream_session').selectAll().where('id', '=', id).executeTakeFirst();
|
||||
return this.db
|
||||
.selectFrom('video_stream_session')
|
||||
.selectAll()
|
||||
.where('id', '=', id)
|
||||
.where('expiresAt', '>', new Date())
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID] })
|
||||
@@ -47,7 +53,12 @@ export class VideoStreamRepository {
|
||||
|
||||
@GenerateSql()
|
||||
getExpiredSessions() {
|
||||
return this.db.selectFrom('video_stream_session').select(['id']).where('expiresAt', '<=', new Date()).execute();
|
||||
return this.db
|
||||
.selectFrom('video_stream_session')
|
||||
.innerJoin('asset', 'asset.id', 'video_stream_session.assetId')
|
||||
.select(['video_stream_session.id', 'asset.ownerId'])
|
||||
.where('video_stream_session.expiresAt', '<=', new Date())
|
||||
.execute();
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID, DummyValue.DATE] })
|
||||
@@ -59,4 +70,50 @@ export class VideoStreamRepository {
|
||||
async deleteSession(id: string) {
|
||||
await this.db.deleteFrom('video_stream_session').where('id', '=', id).execute();
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID] })
|
||||
async getForMasterPlaylist(id: string) {
|
||||
return this.db
|
||||
.selectFrom('asset')
|
||||
.innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId')
|
||||
.where('asset.id', '=', id)
|
||||
.innerJoin('asset_video', 'asset.id', 'asset_video.assetId')
|
||||
.innerJoin('asset_keyframe', 'asset.id', 'asset_keyframe.assetId')
|
||||
.select((eb) => withVideoStream(eb).$notNull().as('videoStream'))
|
||||
.select((eb) => withVideoPackets(eb).$notNull().as('packets'))
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID] })
|
||||
async getForMediaPlaylist(id: string, sessionId: string) {
|
||||
return this.db
|
||||
.selectFrom('asset')
|
||||
.innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId')
|
||||
.innerJoin('video_stream_session', 'asset.id', 'video_stream_session.assetId')
|
||||
.where('asset.id', '=', id)
|
||||
.where('video_stream_session.id', '=', sessionId)
|
||||
.where('video_stream_session.expiresAt', '>', new Date())
|
||||
.innerJoin('asset_video', 'asset.id', 'asset_video.assetId')
|
||||
.innerJoin('asset_keyframe', 'asset.id', 'asset_keyframe.assetId')
|
||||
.select((eb) => withVideoStream(eb).$notNull().as('videoStream'))
|
||||
.select((eb) => withVideoPackets(eb).$notNull().as('packets'))
|
||||
.executeTakeFirst();
|
||||
}
|
||||
|
||||
@GenerateSql({ params: [DummyValue.UUID] })
|
||||
async getForTranscoding(id: string) {
|
||||
return this.db
|
||||
.selectFrom('asset')
|
||||
.innerJoin('asset_exif', 'asset.id', 'asset_exif.assetId')
|
||||
.where('asset.id', '=', id)
|
||||
.leftJoin('asset_audio', 'asset.id', 'asset_audio.assetId')
|
||||
.innerJoin('asset_video', 'asset.id', 'asset_video.assetId')
|
||||
.innerJoin('asset_keyframe', 'asset.id', 'asset_keyframe.assetId')
|
||||
.select('asset.originalPath')
|
||||
.select((eb) => withAudioStream(eb).as('audioStream'))
|
||||
.select((eb) => withVideoStream(eb).$notNull().as('videoStream'))
|
||||
.select((eb) => withVideoFormat(eb).$notNull().as('format'))
|
||||
.select((eb) => withVideoPackets(eb).$notNull().as('packets'))
|
||||
.executeTakeFirst();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,16 @@ import { AppRestartEvent, ArgsOf, EventRepository } from 'src/repositories/event
|
||||
import { LoggingRepository } from 'src/repositories/logging.repository';
|
||||
import { handlePromiseError } from 'src/utils/misc';
|
||||
|
||||
export const serverEvents = ['ConfigUpdate', 'AppRestart'] as const;
|
||||
export const serverEvents = [
|
||||
'ConfigUpdate',
|
||||
'AppRestart',
|
||||
'HlsSegmentRequest',
|
||||
'HlsSegmentResult',
|
||||
'HlsHeartbeat',
|
||||
'HlsSessionRequest',
|
||||
'HlsSessionResult',
|
||||
'HlsSessionEnd',
|
||||
] as const;
|
||||
export type ServerEvents = (typeof serverEvents)[number];
|
||||
|
||||
export interface ClientEventMap {
|
||||
|
||||
@@ -1,12 +1,5 @@
|
||||
import { registerEnum } from '@immich/sql-tools';
|
||||
import {
|
||||
AlbumUserRole,
|
||||
AssetStatus,
|
||||
AssetVisibility,
|
||||
ChecksumAlgorithm,
|
||||
SourceType,
|
||||
VideoSegmentCodec,
|
||||
} from 'src/enum';
|
||||
import { AlbumUserRole, AssetStatus, AssetVisibility, ChecksumAlgorithm, SourceType, VideoCodec } from 'src/enum';
|
||||
|
||||
export const album_user_role_enum = registerEnum({
|
||||
name: 'album_user_role_enum',
|
||||
@@ -35,5 +28,5 @@ export const asset_checksum_algorithm_enum = registerEnum({
|
||||
|
||||
export const video_stream_variant_codec_enum = registerEnum({
|
||||
name: 'video_stream_variant_codec_enum',
|
||||
values: Object.values(VideoSegmentCodec),
|
||||
values: [VideoCodec.Av1, VideoCodec.Hevc, VideoCodec.H264],
|
||||
});
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
import { BadRequestException, NotFoundException } from '@nestjs/common';
|
||||
import { TranscodeHardwareAcceleration } from 'src/enum';
|
||||
import { HlsService } from 'src/services/hls.service';
|
||||
import { eiffelTower, train, waterfall } from 'test/fixtures/media.stub';
|
||||
import { factory } from 'test/small.factory';
|
||||
import { newTestService, ServiceMocks } from 'test/utils';
|
||||
|
||||
// EXTINF values come from FFmpeg's playlist to enforce an exact match
|
||||
const eiffelExpectedMediaPlaylist = `#EXTM3U
|
||||
#EXT-X-VERSION:6
|
||||
#EXT-X-TARGETDURATION:2
|
||||
#EXT-X-MEDIA-SEQUENCE:0
|
||||
#EXT-X-PLAYLIST-TYPE:VOD
|
||||
#EXT-X-MAP:URI="init.mp4"
|
||||
#EXTINF:2.007222,
|
||||
seg_0.m4s
|
||||
#EXTINF:2.007222,
|
||||
seg_1.m4s
|
||||
#EXTINF:2.007222,
|
||||
seg_2.m4s
|
||||
#EXTINF:2.007222,
|
||||
seg_3.m4s
|
||||
#EXTINF:2.007222,
|
||||
seg_4.m4s
|
||||
#EXTINF:2.007222,
|
||||
seg_5.m4s
|
||||
#EXTINF:2.007222,
|
||||
seg_6.m4s
|
||||
#EXTINF:2.007222,
|
||||
seg_7.m4s
|
||||
#EXTINF:2.007222,
|
||||
seg_8.m4s
|
||||
#EXTINF:2.007222,
|
||||
seg_9.m4s
|
||||
#EXTINF:2.007222,
|
||||
seg_10.m4s
|
||||
#EXTINF:0.281011,
|
||||
seg_11.m4s
|
||||
#EXT-X-ENDLIST
|
||||
`;
|
||||
|
||||
const waterfallExpectedMediaPlaylist = `#EXTM3U
|
||||
#EXT-X-VERSION:6
|
||||
#EXT-X-TARGETDURATION:2
|
||||
#EXT-X-MEDIA-SEQUENCE:0
|
||||
#EXT-X-PLAYLIST-TYPE:VOD
|
||||
#EXT-X-MAP:URI="init.mp4"
|
||||
#EXTINF:2.011405,
|
||||
seg_0.m4s
|
||||
#EXTINF:2.011405,
|
||||
seg_1.m4s
|
||||
#EXTINF:2.011405,
|
||||
seg_2.m4s
|
||||
#EXTINF:2.011405,
|
||||
seg_3.m4s
|
||||
#EXTINF:2.011405,
|
||||
seg_4.m4s
|
||||
#EXTINF:0.301711,
|
||||
seg_5.m4s
|
||||
#EXT-X-ENDLIST
|
||||
`;
|
||||
|
||||
const trainExpectedMediaPlaylist = `#EXTM3U
|
||||
#EXT-X-VERSION:6
|
||||
#EXT-X-TARGETDURATION:2
|
||||
#EXT-X-MEDIA-SEQUENCE:0
|
||||
#EXT-X-PLAYLIST-TYPE:VOD
|
||||
#EXT-X-MAP:URI="init.mp4"
|
||||
#EXTINF:2.000000,
|
||||
seg_0.m4s
|
||||
#EXTINF:2.000000,
|
||||
seg_1.m4s
|
||||
#EXTINF:2.000000,
|
||||
seg_2.m4s
|
||||
#EXTINF:2.000000,
|
||||
seg_3.m4s
|
||||
#EXTINF:2.000000,
|
||||
seg_4.m4s
|
||||
#EXTINF:2.000000,
|
||||
seg_5.m4s
|
||||
#EXTINF:2.000000,
|
||||
seg_6.m4s
|
||||
#EXTINF:2.000000,
|
||||
seg_7.m4s
|
||||
#EXTINF:2.000000,
|
||||
seg_8.m4s
|
||||
#EXTINF:2.000000,
|
||||
seg_9.m4s
|
||||
#EXTINF:1.733333,
|
||||
seg_10.m4s
|
||||
#EXT-X-ENDLIST
|
||||
`;
|
||||
|
||||
const sessionId = '00000000-0000-0000-0000-000000000000';
|
||||
|
||||
const eiffelExpectedMasterDisabled = `#EXTM3U
|
||||
#EXT-X-VERSION:6
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/0/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/1/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/2/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=2000000,RESOLUTION=720x1280,CODECS="av01.0.08M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/3/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/4/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/5/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=4000000,RESOLUTION=1080x1920,CODECS="av01.0.09M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/6/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=4500000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/7/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/8/playlist.m3u8
|
||||
`;
|
||||
|
||||
const eiffelExpectedMasterRkmpp = `#EXTM3U
|
||||
#EXT-X-VERSION:6
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/1/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/2/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/4/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/5/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=4500000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/7/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=24.910
|
||||
${sessionId}/8/playlist.m3u8
|
||||
`;
|
||||
|
||||
const waterfallExpectedMasterDisabled = `#EXTM3U
|
||||
#EXT-X-VERSION:6
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=1000000,RESOLUTION=480x852,CODECS="av01.0.04M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
|
||||
${sessionId}/0/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=1200000,RESOLUTION=480x852,CODECS="hvc1.1.6.L90.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
|
||||
${sessionId}/1/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=480x852,CODECS="avc1.64001e,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
|
||||
${sessionId}/2/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=2000000,RESOLUTION=720x1280,CODECS="av01.0.08M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
|
||||
${sessionId}/3/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=2500000,RESOLUTION=720x1280,CODECS="hvc1.1.6.L93.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
|
||||
${sessionId}/4/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=720x1280,CODECS="avc1.64001f,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
|
||||
${sessionId}/5/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=4000000,RESOLUTION=1080x1920,CODECS="av01.0.09M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
|
||||
${sessionId}/6/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=4500000,RESOLUTION=1080x1920,CODECS="hvc1.1.6.L120.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
|
||||
${sessionId}/7/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1080x1920,CODECS="avc1.640028,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
|
||||
${sessionId}/8/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=7000000,RESOLUTION=1440x2560,CODECS="av01.0.12M.08,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
|
||||
${sessionId}/9/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=8000000,RESOLUTION=1440x2560,CODECS="hvc1.2.4.L150.B0,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
|
||||
${sessionId}/10/playlist.m3u8
|
||||
#EXT-X-STREAM-INF:BANDWIDTH=16000000,RESOLUTION=1440x2560,CODECS="avc1.640032,mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=29.830
|
||||
${sessionId}/11/playlist.m3u8
|
||||
`;
|
||||
|
||||
describe(HlsService.name, () => {
|
||||
let sut: HlsService;
|
||||
let mocks: ServiceMocks;
|
||||
|
||||
beforeEach(() => {
|
||||
({ sut, mocks } = newTestService(HlsService));
|
||||
});
|
||||
|
||||
describe('getMasterPlaylist', () => {
|
||||
const auth = factory.auth();
|
||||
const assetId = 'asset-1';
|
||||
|
||||
const setup = (asset: typeof eiffelTower | typeof waterfall, accel: TranscodeHardwareAcceleration) => {
|
||||
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
|
||||
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { realtime: { enabled: true }, accel } });
|
||||
mocks.videoStream.getForMasterPlaylist.mockResolvedValue(asset);
|
||||
mocks.crypto.randomUUID.mockReturnValue(sessionId);
|
||||
mocks.websocket.serverSend.mockImplementation((event, ...rest) => {
|
||||
if (event === 'HlsSessionRequest') {
|
||||
const { sessionId: id } = rest[0] as { sessionId: string };
|
||||
queueMicrotask(() => sut.onSessionResult({ sessionId: id }));
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
it('returns master playlist for eiffel-tower (1080p portrait, no acceleration)', async () => {
|
||||
setup(eiffelTower, TranscodeHardwareAcceleration.Disabled);
|
||||
await expect(sut.getMasterPlaylist(auth, assetId)).resolves.toBe(eiffelExpectedMasterDisabled);
|
||||
});
|
||||
|
||||
it('returns master playlist for eiffel-tower with RKMPP (no AV1 variants)', async () => {
|
||||
setup(eiffelTower, TranscodeHardwareAcceleration.Rkmpp);
|
||||
await expect(sut.getMasterPlaylist(auth, assetId)).resolves.toBe(eiffelExpectedMasterRkmpp);
|
||||
});
|
||||
|
||||
it('returns master playlist for waterfall (4K landscape) with no acceleration', async () => {
|
||||
setup(waterfall, TranscodeHardwareAcceleration.Disabled);
|
||||
await expect(sut.getMasterPlaylist(auth, assetId)).resolves.toBe(waterfallExpectedMasterDisabled);
|
||||
});
|
||||
|
||||
it('throws BadRequestException when realtime transcoding is disabled', async () => {
|
||||
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
|
||||
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { realtime: { enabled: false } } });
|
||||
await expect(sut.getMasterPlaylist(auth, assetId)).rejects.toBeInstanceOf(BadRequestException);
|
||||
});
|
||||
|
||||
it('throws NotFoundException when asset is not yet ready for streaming', async () => {
|
||||
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
|
||||
mocks.systemMetadata.get.mockResolvedValue({ ffmpeg: { realtime: { enabled: true } } });
|
||||
await expect(sut.getMasterPlaylist(auth, assetId)).rejects.toBeInstanceOf(NotFoundException);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getMediaPlaylist', () => {
|
||||
const auth = factory.auth();
|
||||
const assetId = 'asset-1';
|
||||
const fixtures = [
|
||||
{ data: eiffelTower, playlist: eiffelExpectedMediaPlaylist },
|
||||
{ data: waterfall, playlist: waterfallExpectedMediaPlaylist },
|
||||
{ data: train, playlist: trainExpectedMediaPlaylist },
|
||||
];
|
||||
|
||||
it.each(fixtures)('matches FFmpeg for $data.originalPath', async ({ data, playlist }) => {
|
||||
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
|
||||
mocks.videoStream.getForMediaPlaylist.mockResolvedValue(data);
|
||||
await expect(sut.getMediaPlaylist(auth, assetId, sessionId)).resolves.toBe(playlist);
|
||||
});
|
||||
|
||||
it('throws NotFoundException when the session/asset cannot be loaded', async () => {
|
||||
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
|
||||
await expect(sut.getMediaPlaylist(auth, assetId, sessionId)).rejects.toBeInstanceOf(NotFoundException);
|
||||
});
|
||||
});
|
||||
|
||||
describe('getSegment', () => {
|
||||
const auth = factory.auth();
|
||||
const assetId = 'asset-1';
|
||||
const variantIndex = 0;
|
||||
|
||||
beforeEach(() => {
|
||||
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
|
||||
mocks.videoStream.getSession.mockResolvedValue({ id: sessionId, assetId } as never);
|
||||
mocks.storage.checkFileExists.mockResolvedValue(true);
|
||||
});
|
||||
|
||||
it('emits HlsHeartbeat with segmentIndex 0 for the first init.mp4 request', async () => {
|
||||
await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4');
|
||||
expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
|
||||
sessionId,
|
||||
variantIndex,
|
||||
segmentIndex: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it('emits HlsHeartbeat with the parsed segment number for seg_K.m4s', async () => {
|
||||
await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s');
|
||||
expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
|
||||
sessionId,
|
||||
variantIndex,
|
||||
segmentIndex: 5,
|
||||
});
|
||||
});
|
||||
|
||||
it('returns lastRequested + 1 for init.mp4 after a segment has been served', async () => {
|
||||
await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s');
|
||||
mocks.websocket.serverSend.mockClear();
|
||||
await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4');
|
||||
expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
|
||||
sessionId,
|
||||
variantIndex,
|
||||
segmentIndex: 6,
|
||||
});
|
||||
});
|
||||
|
||||
it('updates lastRequested on a backward-seek segment request', async () => {
|
||||
await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_5.m4s');
|
||||
await sut.getSegment(auth, assetId, sessionId, variantIndex, 'seg_3.m4s');
|
||||
mocks.websocket.serverSend.mockClear();
|
||||
await sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4');
|
||||
expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
|
||||
sessionId,
|
||||
variantIndex,
|
||||
segmentIndex: 4,
|
||||
});
|
||||
});
|
||||
|
||||
it('tracks segment state per session independently', async () => {
|
||||
await sut.getSegment(auth, assetId, 'session-a', variantIndex, 'seg_5.m4s');
|
||||
await sut.getSegment(auth, assetId, 'session-b', variantIndex, 'seg_2.m4s');
|
||||
mocks.websocket.serverSend.mockClear();
|
||||
await sut.getSegment(auth, assetId, 'session-a', variantIndex, 'init.mp4');
|
||||
await sut.getSegment(auth, assetId, 'session-b', variantIndex, 'init.mp4');
|
||||
expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
|
||||
sessionId: 'session-a',
|
||||
variantIndex,
|
||||
segmentIndex: 6,
|
||||
});
|
||||
expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsHeartbeat', {
|
||||
sessionId: 'session-b',
|
||||
variantIndex,
|
||||
segmentIndex: 3,
|
||||
});
|
||||
});
|
||||
|
||||
it('throws NotFoundException when the session does not exist', async () => {
|
||||
await expect(sut.getSegment(auth, assetId, sessionId, variantIndex, 'init.mp4')).rejects.toBeInstanceOf(
|
||||
NotFoundException,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('endSession', () => {
|
||||
it('emits HlsSessionEnd', async () => {
|
||||
const auth = factory.auth();
|
||||
const assetId = 'asset-1';
|
||||
mocks.access.asset.checkOwnerAccess.mockResolvedValue(new Set([assetId]));
|
||||
await sut.endSession(auth, assetId, sessionId);
|
||||
expect(mocks.websocket.serverSend).toHaveBeenCalledWith('HlsSessionEnd', { sessionId });
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,202 @@
|
||||
import { BadRequestException, Injectable, NotFoundException } from '@nestjs/common';
|
||||
import { constants } from 'node:fs';
|
||||
import { join } from 'node:path';
|
||||
import {
|
||||
HLS_SEGMENT_DURATION,
|
||||
HLS_VARIANTS,
|
||||
HLS_VERSION,
|
||||
SEGMENT_FILENAME_REGEX,
|
||||
SUPPORTED_HWA_CODECS,
|
||||
} from 'src/constants';
|
||||
import { StorageCore } from 'src/cores/storage.core';
|
||||
import { OnEvent } from 'src/decorators';
|
||||
import { AuthDto } from 'src/dtos/auth.dto';
|
||||
import { SystemConfigFFmpegDto } from 'src/dtos/system-config.dto';
|
||||
import { CacheControl, ImmichWorker, Permission } from 'src/enum';
|
||||
import { ArgOf } from 'src/repositories/event.repository';
|
||||
import { BaseService } from 'src/services/base.service';
|
||||
import { VideoPacketInfo, VideoStreamInfo } from 'src/types';
|
||||
import { PendingEvents } from 'src/utils/event';
|
||||
import { ImmichFileResponse } from 'src/utils/file';
|
||||
import { getOutputSize } from 'src/utils/media';
|
||||
|
||||
type AssetWithStreamInfo = { videoStream: VideoStreamInfo & { timeBase: number }; packets: VideoPacketInfo };
|
||||
|
||||
@Injectable()
|
||||
export class HlsService extends BaseService {
|
||||
private pendingSegments = new PendingEvents<'HlsSegmentResult'>(15_000);
|
||||
private pendingSessions = new PendingEvents<'HlsSessionResult'>(5000);
|
||||
private sessions = new Map<string, { lastRequestedSegment: number | null }>();
|
||||
|
||||
@OnEvent({ name: 'HlsSessionResult', server: true, workers: [ImmichWorker.Api] })
|
||||
onSessionResult(event: ArgOf<'HlsSessionResult'>) {
|
||||
this.pendingSessions.complete(event.sessionId, event);
|
||||
if (event.error) {
|
||||
this.sessions.delete(event.sessionId);
|
||||
this.pendingSegments.rejectByPrefix(`${event.sessionId}:`, event.error);
|
||||
}
|
||||
}
|
||||
|
||||
@OnEvent({ name: 'HlsSessionEnd', server: true, workers: [ImmichWorker.Api] })
|
||||
onSessionEnd({ sessionId }: ArgOf<'HlsSessionEnd'>) {
|
||||
this.sessions.delete(sessionId);
|
||||
this.pendingSegments.rejectByPrefix(`${sessionId}:`, 'Session ended');
|
||||
}
|
||||
|
||||
@OnEvent({ name: 'HlsSegmentResult', server: true, workers: [ImmichWorker.Api] })
|
||||
onSegmentResult(event: ArgOf<'HlsSegmentResult'>) {
|
||||
this.pendingSegments.complete(this.getSegmentKey(event), event);
|
||||
}
|
||||
|
||||
async getMasterPlaylist(auth: AuthDto, assetId: string) {
|
||||
await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] });
|
||||
const { ffmpeg } = await this.getConfig({ withCache: true });
|
||||
if (!ffmpeg.realtime.enabled) {
|
||||
throw new BadRequestException('Real-time transcoding is not enabled');
|
||||
}
|
||||
|
||||
const asset = await this.videoStreamRepository.getForMasterPlaylist(assetId);
|
||||
if (!asset) {
|
||||
throw new NotFoundException('Asset is not yet ready for streaming');
|
||||
}
|
||||
|
||||
const sessionId = this.cryptoRepository.randomUUID();
|
||||
this.websocketRepository.serverSend('HlsSessionRequest', { sessionId, assetId, ownerId: auth.user.id });
|
||||
await this.pendingSessions.wait(sessionId);
|
||||
this.sessions.set(sessionId, { lastRequestedSegment: null });
|
||||
|
||||
return this.generateMasterPlaylist(sessionId, ffmpeg, asset);
|
||||
}
|
||||
|
||||
async getMediaPlaylist(auth: AuthDto, assetId: string, sessionId: string) {
|
||||
await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] });
|
||||
|
||||
const asset = await this.videoStreamRepository.getForMediaPlaylist(assetId, sessionId);
|
||||
if (!asset) {
|
||||
throw new NotFoundException('Asset not found or not yet ready for streaming');
|
||||
}
|
||||
|
||||
return this.generateMediaPlaylist(asset);
|
||||
}
|
||||
|
||||
async getSegment(auth: AuthDto, assetId: string, sessionId: string, variantIndex: number, filename: string) {
|
||||
const t0 = performance.now();
|
||||
await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] });
|
||||
const t1 = performance.now();
|
||||
|
||||
const session = await this.videoStreamRepository.getSession(sessionId);
|
||||
if (!session) {
|
||||
throw new NotFoundException('Session not found');
|
||||
}
|
||||
const t2 = performance.now();
|
||||
|
||||
const variantDir = StorageCore.getHlsVariantFolder({ ownerId: auth.user.id, sessionId, variantIndex });
|
||||
const path = join(variantDir, filename);
|
||||
const response = new ImmichFileResponse({
|
||||
path,
|
||||
contentType: 'video/mp4',
|
||||
cacheControl: CacheControl.PrivateWithCache,
|
||||
});
|
||||
|
||||
const segmentIndex = this.getSegmentIndex(sessionId, filename);
|
||||
this.websocketRepository.serverSend('HlsHeartbeat', { sessionId, variantIndex, segmentIndex });
|
||||
|
||||
if (await this.storageRepository.checkFileExists(path, constants.R_OK)) {
|
||||
this.logger.log(
|
||||
`[TIMING] getSegment(cached) session=${sessionId} variant=${variantIndex} file=${filename} ` +
|
||||
`auth=${(t1 - t0).toFixed(1)}ms session=${(t2 - t1).toFixed(1)}ms ` +
|
||||
`total=${(performance.now() - t0).toFixed(1)}ms`,
|
||||
);
|
||||
return response;
|
||||
}
|
||||
const t3 = performance.now();
|
||||
|
||||
this.websocketRepository.serverSend('HlsSegmentRequest', { sessionId, assetId, variantIndex, segmentIndex });
|
||||
await this.pendingSegments.wait(this.getSegmentKey({ sessionId, variantIndex, segmentIndex }));
|
||||
const t4 = performance.now();
|
||||
|
||||
this.logger.log(
|
||||
`[TIMING] getSegment(wait) session=${sessionId} variant=${variantIndex} file=${filename} ` +
|
||||
`auth=${(t1 - t0).toFixed(1)}ms session=${(t2 - t1).toFixed(1)}ms ` +
|
||||
`existsCheck=${(t3 - t2).toFixed(1)}ms wait=${(t4 - t3).toFixed(1)}ms ` +
|
||||
`total=${(t4 - t0).toFixed(1)}ms`,
|
||||
);
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
async endSession(auth: AuthDto, assetId: string, sessionId: string): Promise<void> {
|
||||
await this.requireAccess({ auth, permission: Permission.AssetView, ids: [assetId] });
|
||||
|
||||
this.websocketRepository.serverSend('HlsSessionEnd', { sessionId });
|
||||
}
|
||||
|
||||
private generateMasterPlaylist(sessionId: string, ffmpeg: SystemConfigFFmpegDto, asset: AssetWithStreamInfo) {
|
||||
const fps = ((asset.packets.packetCount * asset.videoStream.timeBase) / asset.packets.totalDuration).toFixed(3);
|
||||
const sourceResolution = Math.min(asset.videoStream.height, asset.videoStream.width);
|
||||
const lines = ['#EXTM3U', `#EXT-X-VERSION:${HLS_VERSION}`];
|
||||
for (let i = 0; i < HLS_VARIANTS.length; i++) {
|
||||
const { resolution, bitrate, codec, codecString } = HLS_VARIANTS[i];
|
||||
if (resolution > sourceResolution || !SUPPORTED_HWA_CODECS[ffmpeg.accel].includes(codec)) {
|
||||
continue;
|
||||
}
|
||||
const { width, height } = getOutputSize(asset.videoStream, resolution);
|
||||
lines.push(
|
||||
`#EXT-X-STREAM-INF:BANDWIDTH=${bitrate},RESOLUTION=${width}x${height},CODECS="${codecString},mp4a.40.2",VIDEO-RANGE=SDR,FRAME-RATE=${fps}`,
|
||||
`${sessionId}/${i}/playlist.m3u8`,
|
||||
);
|
||||
}
|
||||
lines.push('');
|
||||
|
||||
if (lines.length === 3) {
|
||||
throw new NotFoundException('No supported variants for this video');
|
||||
}
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
private generateMediaPlaylist({ videoStream, packets }: AssetWithStreamInfo) {
|
||||
const fps = (packets.packetCount * videoStream.timeBase) / packets.totalDuration;
|
||||
const framesPerSegment = Math.ceil(HLS_SEGMENT_DURATION * fps);
|
||||
const fullSegmentDuration = framesPerSegment / fps;
|
||||
const segmentCount = Math.ceil(packets.outputFrames / framesPerSegment);
|
||||
const lastSegmentFrames = packets.outputFrames - framesPerSegment * (segmentCount - 1);
|
||||
const lastSegmentDuration = lastSegmentFrames / fps;
|
||||
|
||||
const lines = [
|
||||
'#EXTM3U',
|
||||
`#EXT-X-VERSION:${HLS_VERSION}`,
|
||||
`#EXT-X-TARGETDURATION:${HLS_SEGMENT_DURATION}`,
|
||||
'#EXT-X-MEDIA-SEQUENCE:0',
|
||||
'#EXT-X-PLAYLIST-TYPE:VOD',
|
||||
'#EXT-X-MAP:URI="init.mp4"',
|
||||
];
|
||||
|
||||
for (let i = 0; i < segmentCount - 1; i++) {
|
||||
lines.push(`#EXTINF:${fullSegmentDuration.toFixed(6)},`, `seg_${i}.m4s`);
|
||||
}
|
||||
lines.push(`#EXTINF:${lastSegmentDuration.toFixed(6)},`, `seg_${segmentCount - 1}.m4s`, '#EXT-X-ENDLIST', '');
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
private getSegmentKey({ sessionId, variantIndex, segmentIndex }: ArgOf<'HlsSegmentResult'>) {
|
||||
return `${sessionId}:${variantIndex}:${segmentIndex}`;
|
||||
}
|
||||
|
||||
private getSegmentIndex(sessionId: string, filename: string) {
|
||||
const existing = this.sessions.get(sessionId);
|
||||
let segmentIndex: number;
|
||||
if (filename.endsWith('.mp4')) {
|
||||
segmentIndex = (existing?.lastRequestedSegment ?? -1) + 1;
|
||||
} else {
|
||||
segmentIndex = Number.parseInt(SEGMENT_FILENAME_REGEX.exec(filename)![1]);
|
||||
if (existing) {
|
||||
existing.lastRequestedSegment = segmentIndex;
|
||||
} else {
|
||||
this.sessions.set(sessionId, { lastRequestedSegment: segmentIndex });
|
||||
}
|
||||
}
|
||||
return segmentIndex;
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import { DatabaseBackupService } from 'src/services/database-backup.service';
|
||||
import { DatabaseService } from 'src/services/database.service';
|
||||
import { DownloadService } from 'src/services/download.service';
|
||||
import { DuplicateService } from 'src/services/duplicate.service';
|
||||
import { HlsService } from 'src/services/hls.service';
|
||||
import { JobService } from 'src/services/job.service';
|
||||
import { LibraryService } from 'src/services/library.service';
|
||||
import { MaintenanceService } from 'src/services/maintenance.service';
|
||||
@@ -60,6 +61,7 @@ export const services = [
|
||||
DatabaseService,
|
||||
DownloadService,
|
||||
DuplicateService,
|
||||
HlsService,
|
||||
JobService,
|
||||
LibraryService,
|
||||
MaintenanceService,
|
||||
|
||||
@@ -72,6 +72,9 @@ const updatedConfig = Object.freeze<SystemConfig>({
|
||||
accel: TranscodeHardwareAcceleration.Disabled,
|
||||
accelDecode: false,
|
||||
tonemap: ToneMapping.Hable,
|
||||
realtime: {
|
||||
enabled: false,
|
||||
},
|
||||
},
|
||||
logging: {
|
||||
enabled: true,
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
import { ArgOf, EmitEvent } from 'src/repositories/event.repository';
|
||||
|
||||
export class PendingEvents<T extends { [T in EmitEvent]: ArgOf<T> extends { error?: string } ? T : never }[EmitEvent]> {
|
||||
private pending = new Map<string, { completers: PromiseWithResolvers<ArgOf<T>>[]; timeout: NodeJS.Timeout }>();
|
||||
|
||||
constructor(private timeoutMs: number) {}
|
||||
|
||||
wait(key: string): Promise<ArgOf<T>> {
|
||||
const completer = Promise.withResolvers<ArgOf<T>>();
|
||||
const existing = this.pending.get(key);
|
||||
if (existing) {
|
||||
existing.completers.push(completer);
|
||||
return completer.promise;
|
||||
}
|
||||
|
||||
const timeout = setTimeout(() => this.complete(key, { error: 'Request timed out' }), this.timeoutMs);
|
||||
this.pending.set(key, { completers: [completer], timeout });
|
||||
return completer.promise;
|
||||
}
|
||||
|
||||
complete(key: string, value: ArgOf<T> | { error: string }) {
|
||||
const pending = this.pending.get(key);
|
||||
if (!pending) {
|
||||
return;
|
||||
}
|
||||
clearTimeout(pending.timeout);
|
||||
this.pending.delete(key);
|
||||
for (const completer of pending.completers) {
|
||||
if ('error' in value) {
|
||||
completer.reject(new Error(value.error));
|
||||
} else {
|
||||
completer.resolve(value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rejectByPrefix(prefix: string, error: string) {
|
||||
for (const key of this.pending.keys()) {
|
||||
if (key.startsWith(prefix)) {
|
||||
this.complete(key, { error });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Vendored
+1
-1
@@ -597,7 +597,7 @@ export const train = {
|
||||
packets: {
|
||||
totalDuration: 12_290,
|
||||
packetCount: 1229,
|
||||
outputFrames: 1303,
|
||||
outputFrames: 1304,
|
||||
keyframePts: [
|
||||
0, 601, 1201, 1802, 2402, 3003, 3604, 4204, 4805, 5405, 6006, 6607, 7207, 7808, 8408, 9009, 9609, 10_210, 10_811,
|
||||
11_411, 12_062, 12_703,
|
||||
|
||||
@@ -8,9 +8,9 @@
|
||||
"experimentalDecorators": true,
|
||||
"allowSyntheticDefaultImports": true,
|
||||
"resolveJsonModule": true,
|
||||
"target": "es2022",
|
||||
"target": "es2024",
|
||||
"moduleResolution": "node16",
|
||||
"lib": ["dom", "es2023"],
|
||||
"lib": ["dom", "es2024"],
|
||||
"sourceMap": true,
|
||||
"outDir": "./dist",
|
||||
"incremental": true,
|
||||
|
||||
Reference in New Issue
Block a user