tbelway/immich
1
0
Fork 0
mirror of https://github.com/immich-app/immich.git synced 2025-10-31 10:49:11 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(server): extract motion photo android single frame (#3903)

Browse Source
This commit is contained in:
Mert 2023-08-29 05:01:42 -04:00 committed by GitHub
parent d0a06739d8
commit e510e733cd
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 163 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
server/src/domain/media/media.repository.ts
View File

@ -8,6 +8,7 @@ export interface ResizeOptions {
} }
export interface VideoStreamInfo { export interface VideoStreamInfo {
index: number;
height: number; height: number;
width: number; width: number;
rotation: number; rotation: number;
@ -18,8 +19,10 @@ export interface VideoStreamInfo {
} }
export interface AudioStreamInfo { export interface AudioStreamInfo {
index: number;
codecName?: string; codecName?: string;
codecType?: string; codecType?: string;
frameCount: number;
} }
export interface VideoFormat { export interface VideoFormat {
@ -55,7 +58,7 @@ export interface BitrateDistribution {
} }
export interface VideoCodecSWConfig { export interface VideoCodecSWConfig {
getOptions(stream: VideoStreamInfo): TranscodeOptions; getOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo): TranscodeOptions;
} }
export interface VideoCodecHWConfig extends VideoCodecSWConfig { export interface VideoCodecHWConfig extends VideoCodecSWConfig {

140
server/src/domain/media/media.service.spec.ts
View File

@ -311,8 +311,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -350,8 +350,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -374,8 +374,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -401,8 +401,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -426,8 +426,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -451,8 +451,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -476,8 +476,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -525,8 +525,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -555,8 +555,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -582,8 +582,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -611,8 +611,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec vp9', '-c:v:0 vp9',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -642,8 +642,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec vp9', '-c:v:0 vp9',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -672,8 +672,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec vp9', '-c:v:0 vp9',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -701,8 +701,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec vp9', '-c:v:0 vp9',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -729,8 +729,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -757,8 +757,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -785,8 +785,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec hevc', '-c:v:0 hevc',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -816,8 +816,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec hevc', '-c:v:0 hevc',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -876,7 +876,6 @@ describe(MediaService.name, () => {
{ {
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [ outputOptions: [
`-vcodec h264_nvenc`,
'-tune hq', '-tune hq',
'-qmin 0', '-qmin 0',
'-g 250', '-g 250',
@ -886,7 +885,8 @@ describe(MediaService.name, () => {
'-rc-lookahead 20', '-rc-lookahead 20',
'-i_qfactor 0.75', '-i_qfactor 0.75',
'-b_qfactor 1.1', '-b_qfactor 1.1',
'-acodec aac', `-c:v:0 h264_nvenc`,
'-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -916,7 +916,6 @@ describe(MediaService.name, () => {
{ {
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [ outputOptions: [
`-vcodec h264_nvenc`,
'-tune hq', '-tune hq',
'-qmin 0', '-qmin 0',
'-g 250', '-g 250',
@ -926,7 +925,8 @@ describe(MediaService.name, () => {
'-rc-lookahead 20', '-rc-lookahead 20',
'-i_qfactor 0.75', '-i_qfactor 0.75',
'-b_qfactor 1.1', '-b_qfactor 1.1',
'-acodec aac', `-c:v:0 h264_nvenc`,
'-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -952,7 +952,6 @@ describe(MediaService.name, () => {
{ {
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [ outputOptions: [
`-vcodec h264_nvenc`,
'-tune hq', '-tune hq',
'-qmin 0', '-qmin 0',
'-g 250', '-g 250',
@ -962,7 +961,8 @@ describe(MediaService.name, () => {
'-rc-lookahead 20', '-rc-lookahead 20',
'-i_qfactor 0.75', '-i_qfactor 0.75',
'-b_qfactor 1.1', '-b_qfactor 1.1',
'-acodec aac', `-c:v:0 h264_nvenc`,
'-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -989,7 +989,6 @@ describe(MediaService.name, () => {
{ {
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [ outputOptions: [
`-vcodec h264_nvenc`,
'-tune hq', '-tune hq',
'-qmin 0', '-qmin 0',
'-g 250', '-g 250',
@ -999,7 +998,8 @@ describe(MediaService.name, () => {
'-rc-lookahead 20', '-rc-lookahead 20',
'-i_qfactor 0.75', '-i_qfactor 0.75',
'-b_qfactor 1.1', '-b_qfactor 1.1',
'-acodec aac', `-c:v:0 h264_nvenc`,
'-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -1022,7 +1022,6 @@ describe(MediaService.name, () => {
{ {
inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'], inputOptions: ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'],
outputOptions: [ outputOptions: [
`-vcodec h264_nvenc`,
'-tune hq', '-tune hq',
'-qmin 0', '-qmin 0',
'-g 250', '-g 250',
@ -1032,7 +1031,8 @@ describe(MediaService.name, () => {
'-rc-lookahead 20', '-rc-lookahead 20',
'-i_qfactor 0.75', '-i_qfactor 0.75',
'-b_qfactor 1.1', '-b_qfactor 1.1',
'-acodec aac', `-c:v:0 h264_nvenc`,
'-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -1060,12 +1060,12 @@ describe(MediaService.name, () => {
{ {
inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'], inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
outputOptions: [ outputOptions: [
`-vcodec h264_qsv`,
'-g 256', '-g 256',
'-extbrc 1', '-extbrc 1',
'-refs 5', '-refs 5',
'-bf 7', '-bf 7',
'-acodec aac', `-c:v:0 h264_qsv`,
'-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -1095,12 +1095,12 @@ describe(MediaService.name, () => {
{ {
inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'], inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
outputOptions: [ outputOptions: [
`-vcodec h264_qsv`,
'-g 256', '-g 256',
'-extbrc 1', '-extbrc 1',
'-refs 5', '-refs 5',
'-bf 7', '-bf 7',
'-acodec aac', `-c:v:0 h264_qsv`,
'-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -1127,12 +1127,12 @@ describe(MediaService.name, () => {
{ {
inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'], inputOptions: ['-init_hw_device qsv=hw', '-filter_hw_device hw'],
outputOptions: [ outputOptions: [
`-vcodec vp9_qsv`,
'-g 256', '-g 256',
'-extbrc 1', '-extbrc 1',
'-refs 5', '-refs 5',
'-bf 7', '-bf 7',
'-acodec aac', `-c:v:0 vp9_qsv`,
'-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-low_power 1', '-low_power 1',
@ -1170,8 +1170,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'], inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
outputOptions: [ outputOptions: [
`-vcodec h264_vaapi`, `-c:v:0 h264_vaapi`,
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -1199,8 +1199,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'], inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
outputOptions: [ outputOptions: [
`-vcodec h264_vaapi`, `-c:v:0 h264_vaapi`,
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -1230,8 +1230,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'], inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD128', '-filter_hw_device accel'],
outputOptions: [ outputOptions: [
`-vcodec h264_vaapi`, `-c:v:0 h264_vaapi`,
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -1257,8 +1257,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/card1', '-filter_hw_device accel'], inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/card1', '-filter_hw_device accel'],
outputOptions: [ outputOptions: [
`-vcodec h264_vaapi`, `-c:v:0 h264_vaapi`,
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -1280,8 +1280,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD129', '-filter_hw_device accel'], inputOptions: ['-init_hw_device vaapi=accel:/dev/dri/renderD129', '-filter_hw_device accel'],
outputOptions: [ outputOptions: [
`-vcodec h264_vaapi`, `-c:v:0 h264_vaapi`,
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -1310,8 +1310,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -1345,8 +1345,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -1370,8 +1370,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',
@ -1395,8 +1395,8 @@ describe(MediaService.name, () => {
{ {
inputOptions: [], inputOptions: [],
outputOptions: [ outputOptions: [
'-vcodec h264', '-c:v:0 h264',
'-acodec aac', '-c:a:0 aac',
'-movflags faststart', '-movflags faststart',
'-fps_mode passthrough', '-fps_mode passthrough',
'-v verbose', '-v verbose',

23
server/src/domain/media/media.service.ts
View File

@ -73,15 +73,16 @@ export class MediaService {
this.logger.log(`Successfully generated image thumbnail ${asset.id}`); this.logger.log(`Successfully generated image thumbnail ${asset.id}`);
break; break;
case AssetType.VIDEO: case AssetType.VIDEO:
const { videoStreams } = await this.mediaRepository.probe(asset.originalPath); const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath);
const mainVideoStream = this.getMainVideoStream(videoStreams); const mainVideoStream = this.getMainStream(videoStreams);
if (!mainVideoStream) { if (!mainVideoStream) {
this.logger.error(`Could not extract thumbnail for asset ${asset.id}: no video streams found`); this.logger.error(`Could not extract thumbnail for asset ${asset.id}: no video streams found`);
return false; return false;
} }
const mainAudioStream = this.getMainStream(audioStreams);
const { ffmpeg } = await this.configCore.getConfig(); const { ffmpeg } = await this.configCore.getConfig();
const config = { ...ffmpeg, targetResolution: thumbnail.jpegSize.toString(), twoPass: false }; const config = { ...ffmpeg, targetResolution: thumbnail.jpegSize.toString(), twoPass: false };
const options = new ThumbnailConfig(config).getOptions(mainVideoStream); const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream);
await this.mediaRepository.transcode(asset.originalPath, jpegThumbnailPath, options); await this.mediaRepository.transcode(asset.originalPath, jpegThumbnailPath, options);
this.logger.log(`Successfully generated video thumbnail ${asset.id}`); this.logger.log(`Successfully generated video thumbnail ${asset.id}`);
break; break;
@ -149,8 +150,8 @@ export class MediaService {
this.storageRepository.mkdirSync(outputFolder); this.storageRepository.mkdirSync(outputFolder);
const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input); const { videoStreams, audioStreams, format } = await this.mediaRepository.probe(input);
const mainVideoStream = this.getMainVideoStream(videoStreams); const mainVideoStream = this.getMainStream(videoStreams);
const mainAudioStream = this.getMainAudioStream(audioStreams); const mainAudioStream = this.getMainStream(audioStreams);
const containerExtension = format.formatName; const containerExtension = format.formatName;
if (!mainVideoStream || !containerExtension) { if (!mainVideoStream || !containerExtension) {
return false; return false;
@ -165,7 +166,7 @@ export class MediaService {
let transcodeOptions; let transcodeOptions;
try { try {
transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream)); transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream, mainAudioStream));
} catch (err) { } catch (err) {
this.logger.error(`An error occurred while configuring transcoding options: ${err}`); this.logger.error(`An error occurred while configuring transcoding options: ${err}`);
return false; return false;
@ -176,13 +177,13 @@ export class MediaService {
await this.mediaRepository.transcode(input, output, transcodeOptions); await this.mediaRepository.transcode(input, output, transcodeOptions);
} catch (err) { } catch (err) {
this.logger.error(err); this.logger.error(err);
if (config.accel && config.accel !== TranscodeHWAccel.DISABLED) { if (config.accel !== TranscodeHWAccel.DISABLED) {
this.logger.error( this.logger.error(
`Error occurred during transcoding. Retrying with ${config.accel.toUpperCase()} acceleration disabled.`, `Error occurred during transcoding. Retrying with ${config.accel.toUpperCase()} acceleration disabled.`,
); );
} }
config.accel = TranscodeHWAccel.DISABLED; config.accel = TranscodeHWAccel.DISABLED;
transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream)); transcodeOptions = await this.getCodecConfig(config).then((c) => c.getOptions(mainVideoStream, mainAudioStream));
await this.mediaRepository.transcode(input, output, transcodeOptions); await this.mediaRepository.transcode(input, output, transcodeOptions);
} }
@ -193,14 +194,10 @@ export class MediaService {
return true; return true;
} }
private getMainVideoStream(streams: VideoStreamInfo[]): VideoStreamInfo | null { private getMainStream<T extends VideoStreamInfo | AudioStreamInfo>(streams: T[]): T {
return streams.sort((stream1, stream2) => stream2.frameCount - stream1.frameCount)[0]; return streams.sort((stream1, stream2) => stream2.frameCount - stream1.frameCount)[0];
} }
private getMainAudioStream(streams: AudioStreamInfo[]): AudioStreamInfo | null {
return streams[0];
}
private isTranscodeRequired( private isTranscodeRequired(
asset: AssetEntity, asset: AssetEntity,
videoStream: VideoStreamInfo, videoStream: VideoStreamInfo,

114
server/src/domain/media/media.util.ts
View File

@ -1,6 +1,7 @@
import { ToneMapping, TranscodeHWAccel, VideoCodec } from '@app/infra/entities'; import { ToneMapping, TranscodeHWAccel, VideoCodec } from '@app/infra/entities';
import { SystemConfigFFmpegDto } from '../system-config/dto'; import { SystemConfigFFmpegDto } from '../system-config/dto';
import { import {
AudioStreamInfo,
BitrateDistribution, BitrateDistribution,
TranscodeOptions, TranscodeOptions,
VideoCodecHWConfig, VideoCodecHWConfig,
@ -10,13 +11,13 @@ import {
class BaseConfig implements VideoCodecSWConfig { class BaseConfig implements VideoCodecSWConfig {
constructor(protected config: SystemConfigFFmpegDto) {} constructor(protected config: SystemConfigFFmpegDto) {}
getOptions(stream: VideoStreamInfo) { getOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) {
const options = { const options = {
inputOptions: this.getBaseInputOptions(), inputOptions: this.getBaseInputOptions(),
outputOptions: this.getBaseOutputOptions().concat('-v verbose'), outputOptions: this.getBaseOutputOptions(videoStream, audioStream).concat('-v verbose'),
twoPass: this.eligibleForTwoPass(), twoPass: this.eligibleForTwoPass(),
} as TranscodeOptions; } as TranscodeOptions;
const filters = this.getFilterOptions(stream); const filters = this.getFilterOptions(videoStream);
if (filters.length > 0) { if (filters.length > 0) {
options.outputOptions.push(`-vf ${filters.join(',')}`); options.outputOptions.push(`-vf ${filters.join(',')}`);
} }
@ -31,9 +32,10 @@ class BaseConfig implements VideoCodecSWConfig {
return []; return [];
} }
getBaseOutputOptions() { getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) {
return [ return [
`-acodec ${this.config.targetAudioCodec}`, `-c:v:${videoStream.index} ${this.getVideoCodec()}`,
`-c:a:${audioStream.index} ${this.getAudioCodec()}`,
// Makes a second pass moving the moov atom to the // Makes a second pass moving the moov atom to the
// beginning of the file for improved playback speed. // beginning of the file for improved playback speed.
'-movflags faststart', '-movflags faststart',
@ -41,13 +43,13 @@ class BaseConfig implements VideoCodecSWConfig {
]; ];
} }
getFilterOptions(stream: VideoStreamInfo) { getFilterOptions(videoStream: VideoStreamInfo) {
const options = []; const options = [];
if (this.shouldScale(stream)) { if (this.shouldScale(videoStream)) {
options.push(`scale=${this.getScaling(stream)}`); options.push(`scale=${this.getScaling(videoStream)}`);
} }
if (this.shouldToneMap(stream)) { if (this.shouldToneMap(videoStream)) {
options.push(...this.getToneMapping()); options.push(...this.getToneMapping());
} }
options.push('format=yuv420p'); options.push('format=yuv420p');
@ -103,34 +105,34 @@ class BaseConfig implements VideoCodecSWConfig {
return { max, target, min, unit } as BitrateDistribution; return { max, target, min, unit } as BitrateDistribution;
} }
getTargetResolution(stream: VideoStreamInfo) { getTargetResolution(videoStream: VideoStreamInfo) {
if (this.config.targetResolution === 'original') { if (this.config.targetResolution === 'original') {
return Math.min(stream.height, stream.width); return Math.min(videoStream.height, videoStream.width);
} }
return Number.parseInt(this.config.targetResolution); return Number.parseInt(this.config.targetResolution);
} }
shouldScale(stream: VideoStreamInfo) { shouldScale(videoStream: VideoStreamInfo) {
return Math.min(stream.height, stream.width) > this.getTargetResolution(stream); return Math.min(videoStream.height, videoStream.width) > this.getTargetResolution(videoStream);
} }
shouldToneMap(stream: VideoStreamInfo) { shouldToneMap(videoStream: VideoStreamInfo) {
return stream.isHDR && this.config.tonemap !== ToneMapping.DISABLED; return videoStream.isHDR && this.config.tonemap !== ToneMapping.DISABLED;
} }
getScaling(stream: VideoStreamInfo) { getScaling(videoStream: VideoStreamInfo) {
const targetResolution = this.getTargetResolution(stream); const targetResolution = this.getTargetResolution(videoStream);
const mult = this.config.accel === TranscodeHWAccel.QSV ? 1 : 2; // QSV doesn't support scaling numbers below -1 const mult = this.config.accel === TranscodeHWAccel.QSV ? 1 : 2; // QSV doesn't support scaling numbers below -1
return this.isVideoVertical(stream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`; return this.isVideoVertical(videoStream) ? `${targetResolution}:-${mult}` : `-${mult}:${targetResolution}`;
} }
isVideoRotated(stream: VideoStreamInfo) { isVideoRotated(videoStream: VideoStreamInfo) {
return Math.abs(stream.rotation) === 90; return Math.abs(videoStream.rotation) === 90;
} }
isVideoVertical(stream: VideoStreamInfo) { isVideoVertical(videoStream: VideoStreamInfo) {
return stream.height > stream.width || this.isVideoRotated(stream); return videoStream.height > videoStream.width || this.isVideoRotated(videoStream);
} }
isBitrateConstrained() { isBitrateConstrained() {
@ -171,6 +173,14 @@ class BaseConfig implements VideoCodecSWConfig {
`zscale=p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:range=pc`, `zscale=p=${colors.primaries}:t=${colors.transfer}:m=${colors.matrix}:range=pc`,
]; ];
} }
getAudioCodec(): string {
return this.config.targetAudioCodec;
}
getVideoCodec(): string {
return this.config.targetVideoCodec;
}
} }
export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig { export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
@ -202,6 +212,10 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
return -a.localeCompare(b); return -a.localeCompare(b);
}); });
} }
getVideoCodec(): string {
return `${this.config.targetVideoCodec}_${this.config.accel}`;
}
} }
export class ThumbnailConfig extends BaseConfig { export class ThumbnailConfig extends BaseConfig {
@ -217,9 +231,9 @@ export class ThumbnailConfig extends BaseConfig {
return []; return [];
} }
getScaling(stream: VideoStreamInfo) { getScaling(videoStream: VideoStreamInfo) {
let options = super.getScaling(stream); let options = super.getScaling(videoStream);
if (!this.shouldToneMap(stream)) { if (!this.shouldToneMap(videoStream)) {
options += ':out_color_matrix=bt601:out_range=pc'; options += ':out_color_matrix=bt601:out_range=pc';
} }
return options; return options;
@ -236,10 +250,6 @@ export class ThumbnailConfig extends BaseConfig {
} }
export class H264Config extends BaseConfig { export class H264Config extends BaseConfig {
getBaseOutputOptions() {
return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()];
}
getThreadOptions() { getThreadOptions() {
if (this.config.threads <= 0) { if (this.config.threads <= 0) {
return []; return [];
@ -253,10 +263,6 @@ export class H264Config extends BaseConfig {
} }
export class HEVCConfig extends BaseConfig { export class HEVCConfig extends BaseConfig {
getBaseOutputOptions() {
return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()];
}
getThreadOptions() { getThreadOptions() {
if (this.config.threads <= 0) { if (this.config.threads <= 0) {
return []; return [];
@ -270,10 +276,6 @@ export class HEVCConfig extends BaseConfig {
} }
export class VP9Config extends BaseConfig { export class VP9Config extends BaseConfig {
getBaseOutputOptions() {
return [`-vcodec ${this.config.targetVideoCodec}`, ...super.getBaseOutputOptions()];
}
getPresetOptions() { getPresetOptions() {
const speed = Math.min(this.getPresetIndex(), 5); // values over 5 require realtime mode, which is its own can of worms since it overrides -crf and -threads const speed = Math.min(this.getPresetIndex(), 5); // values over 5 require realtime mode, which is its own can of worms since it overrides -crf and -threads
if (speed >= 0) { if (speed >= 0) {
@ -309,9 +311,8 @@ export class NVENCConfig extends BaseHWConfig {
return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda']; return ['-init_hw_device cuda=cuda:0', '-filter_hw_device cuda'];
} }
getBaseOutputOptions() { getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) {
return [ return [
`-vcodec ${this.config.targetVideoCodec}_nvenc`,
// below settings recommended from https://docs.nvidia.com/video-technologies/video-codec-sdk/12.0/ffmpeg-with-nvidia-gpu/index.html#command-line-for-latency-tolerant-high-quality-transcoding // below settings recommended from https://docs.nvidia.com/video-technologies/video-codec-sdk/12.0/ffmpeg-with-nvidia-gpu/index.html#command-line-for-latency-tolerant-high-quality-transcoding
'-tune hq', '-tune hq',
'-qmin 0', '-qmin 0',
@ -322,15 +323,15 @@ export class NVENCConfig extends BaseHWConfig {
'-rc-lookahead 20', '-rc-lookahead 20',
'-i_qfactor 0.75', '-i_qfactor 0.75',
'-b_qfactor 1.1', '-b_qfactor 1.1',
...super.getBaseOutputOptions(), ...super.getBaseOutputOptions(videoStream, audioStream),
]; ];
} }
getFilterOptions(stream: VideoStreamInfo) { getFilterOptions(videoStream: VideoStreamInfo) {
const options = this.shouldToneMap(stream) ? this.getToneMapping() : []; const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
options.push('format=nv12', 'hwupload_cuda'); options.push('format=nv12', 'hwupload_cuda');
if (this.shouldScale(stream)) { if (this.shouldScale(videoStream)) {
options.push(`scale_cuda=${this.getScaling(stream)}`); options.push(`scale_cuda=${this.getScaling(videoStream)}`);
} }
return options; return options;
@ -378,15 +379,14 @@ export class QSVConfig extends BaseHWConfig {
return ['-init_hw_device qsv=hw', '-filter_hw_device hw']; return ['-init_hw_device qsv=hw', '-filter_hw_device hw'];
} }
getBaseOutputOptions() { getBaseOutputOptions(videoStream: VideoStreamInfo, audioStream: AudioStreamInfo) {
// recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md // recommended from https://github.com/intel/media-delivery/blob/master/doc/benchmarks/intel-iris-xe-max-graphics/intel-iris-xe-max-graphics.md
const options = [ const options = [
`-vcodec ${this.config.targetVideoCodec}_qsv`,
'-g 256', '-g 256',
'-extbrc 1', '-extbrc 1',
'-refs 5', '-refs 5',
'-bf 7', '-bf 7',
...super.getBaseOutputOptions(), ...super.getBaseOutputOptions(videoStream, audioStream),
]; ];
// VP9 requires enabling low power mode https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/33583803e107b6d532def0f9d949364b01b6ad5a // VP9 requires enabling low power mode https://git.ffmpeg.org/gitweb/ffmpeg.git/commit/33583803e107b6d532def0f9d949364b01b6ad5a
if (this.config.targetVideoCodec === VideoCodec.VP9) { if (this.config.targetVideoCodec === VideoCodec.VP9) {
@ -395,11 +395,11 @@ export class QSVConfig extends BaseHWConfig {
return options; return options;
} }
getFilterOptions(stream: VideoStreamInfo) { getFilterOptions(videoStream: VideoStreamInfo) {
const options = this.shouldToneMap(stream) ? this.getToneMapping() : []; const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
options.push('format=nv12', 'hwupload=extra_hw_frames=64'); options.push('format=nv12', 'hwupload=extra_hw_frames=64');
if (this.shouldScale(stream)) { if (this.shouldScale(videoStream)) {
options.push(`scale_qsv=${this.getScaling(stream)}`); options.push(`scale_qsv=${this.getScaling(videoStream)}`);
} }
return options; return options;
} }
@ -437,15 +437,11 @@ export class VAAPIConfig extends BaseHWConfig {
return [`-init_hw_device vaapi=accel:/dev/dri/${this.devices[0]}`, '-filter_hw_device accel']; return [`-init_hw_device vaapi=accel:/dev/dri/${this.devices[0]}`, '-filter_hw_device accel'];
} }
getBaseOutputOptions() { getFilterOptions(videoStream: VideoStreamInfo) {
return [`-vcodec ${this.config.targetVideoCodec}_vaapi`, ...super.getBaseOutputOptions()]; const options = this.shouldToneMap(videoStream) ? this.getToneMapping() : [];
}
getFilterOptions(stream: VideoStreamInfo) {
const options = this.shouldToneMap(stream) ? this.getToneMapping() : [];
options.push('format=nv12', 'hwupload'); options.push('format=nv12', 'hwupload');
if (this.shouldScale(stream)) { if (this.shouldScale(videoStream)) {
options.push(`scale_vaapi=${this.getScaling(stream)}`); options.push(`scale_vaapi=${this.getScaling(videoStream)}`);
} }
return options; return options;

3
server/src/infra/repositories/media.repository.ts
View File

@ -42,6 +42,7 @@ export class MediaRepository implements IMediaRepository {
videoStreams: results.streams videoStreams: results.streams
.filter((stream) => stream.codec_type === 'video') .filter((stream) => stream.codec_type === 'video')
.map((stream) => ({ .map((stream) => ({
index: stream.index,
height: stream.height || 0, height: stream.height || 0,
width: stream.width || 0, width: stream.width || 0,
codecName: stream.codec_name === 'h265' ? 'hevc' : stream.codec_name, codecName: stream.codec_name === 'h265' ? 'hevc' : stream.codec_name,
@ -53,8 +54,10 @@ export class MediaRepository implements IMediaRepository {
audioStreams: results.streams audioStreams: results.streams
.filter((stream) => stream.codec_type === 'audio') .filter((stream) => stream.codec_type === 'audio')
.map((stream) => ({ .map((stream) => ({
index: stream.index,
codecType: stream.codec_type, codecType: stream.codec_type,
codecName: stream.codec_name, codecName: stream.codec_name,
frameCount: Number.parseInt(stream.nb_frames ?? '0'),
})), })),
}; };
} }

2
server/test/e2e/setup.ts
View File

@ -1,5 +1,5 @@
import { GenericContainer } from 'testcontainers';
import { PostgreSqlContainer } from '@testcontainers/postgresql'; import { PostgreSqlContainer } from '@testcontainers/postgresql';
import { GenericContainer } from 'testcontainers';
export default async () => { export default async () => {
process.env.NODE_ENV = 'development'; process.env.NODE_ENV = 'development';
process.env.TYPESENSE_API_KEY = 'abc123'; process.env.TYPESENSE_API_KEY = 'abc123';

23
server/test/fixtures/media.stub.ts vendored
View File

@ -7,10 +7,21 @@ const probeStubDefaultFormat: VideoFormat = {
}; };
const probeStubDefaultVideoStream: VideoStreamInfo[] = [ const probeStubDefaultVideoStream: VideoStreamInfo[] = [
{ height: 1080, width: 1920, codecName: 'hevc', codecType: 'video', frameCount: 100, rotation: 0, isHDR: false }, {
index: 0,
height: 1080,
width: 1920,
codecName: 'hevc',
codecType: 'video',
frameCount: 100,
rotation: 0,
isHDR: false,
},
]; ];
const probeStubDefaultAudioStream: AudioStreamInfo[] = [{ codecName: 'aac', codecType: 'audio' }]; const probeStubDefaultAudioStream: AudioStreamInfo[] = [
{ index: 0, codecName: 'aac', codecType: 'audio', frameCount: 100 },
];
const probeStubDefault: VideoInfo = { const probeStubDefault: VideoInfo = {
format: probeStubDefaultFormat, format: probeStubDefaultFormat,
@ -25,6 +36,7 @@ export const probeStub = {
...probeStubDefault, ...probeStubDefault,
videoStreams: [ videoStreams: [
{ {
index: 0,
height: 1080, height: 1080,
width: 400, width: 400,
codecName: 'hevc', codecName: 'hevc',
@ -34,6 +46,7 @@ export const probeStub = {
isHDR: false, isHDR: false,
}, },
{ {
index: 1,
height: 1080, height: 1080,
width: 400, width: 400,
codecName: 'h7000', codecName: 'h7000',
@ -48,6 +61,7 @@ export const probeStub = {
...probeStubDefault, ...probeStubDefault,
videoStreams: [ videoStreams: [
{ {
index: 0,
height: 0, height: 0,
width: 400, width: 400,
codecName: 'hevc', codecName: 'hevc',
@ -62,6 +76,7 @@ export const probeStub = {
...probeStubDefault, ...probeStubDefault,
videoStreams: [ videoStreams: [
{ {
index: 0,
height: 2160, height: 2160,
width: 3840, width: 3840,
codecName: 'h264', codecName: 'h264',
@ -76,6 +91,7 @@ export const probeStub = {
...probeStubDefault, ...probeStubDefault,
videoStreams: [ videoStreams: [
{ {
index: 0,
height: 480, height: 480,
width: 480, width: 480,
codecName: 'h264', codecName: 'h264',
@ -90,6 +106,7 @@ export const probeStub = {
...probeStubDefault, ...probeStubDefault,
videoStreams: [ videoStreams: [
{ {
index: 0,
height: 2160, height: 2160,
width: 3840, width: 3840,
codecName: 'h264', codecName: 'h264',
@ -102,7 +119,7 @@ export const probeStub = {
}), }),
audioStreamMp3: Object.freeze<VideoInfo>({ audioStreamMp3: Object.freeze<VideoInfo>({
...probeStubDefault, ...probeStubDefault,
audioStreams: [{ codecType: 'audio', codecName: 'aac' }], audioStreams: [{ index: 0, codecType: 'audio', codecName: 'aac', frameCount: 100 }],
}), }),
matroskaContainer: Object.freeze<VideoInfo>({ matroskaContainer: Object.freeze<VideoInfo>({
...probeStubDefault, ...probeStubDefault,