From 3da081ba86940f3fcedb188b2243445d1f95c883 Mon Sep 17 00:00:00 2001 From: Dmitry Lyzo <56478732+dmitrylyzo@users.noreply.github.com> Date: Mon, 9 Sep 2024 22:16:58 +0300 Subject: [PATCH] Add audio ranking for transcoding profiles (#12546) --- MediaBrowser.Model/Dlna/StreamBuilder.cs | 100 ++++++++++++------ .../Dlna/StreamBuilderTests.cs | 3 + ...ceInfo-mp4-h264-ac3-aac-mp3-srt-2600k.json | 100 ++++++++++++++++++ 3 files changed, 172 insertions(+), 31 deletions(-) create mode 100644 tests/Jellyfin.Model.Tests/Test Data/MediaSourceInfo-mp4-h264-ac3-aac-mp3-srt-2600k.json diff --git a/MediaBrowser.Model/Dlna/StreamBuilder.cs b/MediaBrowser.Model/Dlna/StreamBuilder.cs index 490ae4e629..f68a8bca34 100644 --- a/MediaBrowser.Model/Dlna/StreamBuilder.cs +++ b/MediaBrowser.Model/Dlna/StreamBuilder.cs @@ -751,8 +751,9 @@ namespace MediaBrowser.Model.Dlna { // Can't direct play, find the transcoding profile // If we do this for direct-stream we will overwrite the info - var transcodingProfile = GetVideoTranscodeProfile(item, options, videoStream, audioStream, candidateAudioStreams, subtitleStream, playlistItem); - if (transcodingProfile is not null) + var (transcodingProfile, playMethod) = GetVideoTranscodeProfile(item, options, videoStream, audioStream, candidateAudioStreams, subtitleStream, playlistItem); + + if (transcodingProfile is not null && playMethod.HasValue) { SetStreamInfoOptionsFromTranscodingProfile(item, playlistItem, transcodingProfile); @@ -790,7 +791,7 @@ namespace MediaBrowser.Model.Dlna return playlistItem; } - private TranscodingProfile? GetVideoTranscodeProfile( + private (TranscodingProfile? Profile, PlayMethod? PlayMethod) GetVideoTranscodeProfile( MediaSourceInfo item, MediaOptions options, MediaStream? videoStream, @@ -801,7 +802,7 @@ namespace MediaBrowser.Model.Dlna { if (!(item.SupportsTranscoding || item.SupportsDirectStream)) { - return null; + return (null, null); } var transcodingProfiles = options.Profile.TranscodingProfiles @@ -812,41 +813,78 @@ namespace MediaBrowser.Model.Dlna transcodingProfiles = transcodingProfiles.Where(i => string.Equals(i.Container, "ts", StringComparison.OrdinalIgnoreCase)); } - if (options.AllowVideoStreamCopy) - { - // prefer direct copy profile - float videoFramerate = videoStream?.ReferenceFrameRate ?? 0; - TransportStreamTimestamp? timestamp = videoStream is null ? TransportStreamTimestamp.None : item.Timestamp; - int? numAudioStreams = item.GetStreamCount(MediaStreamType.Audio); - int? numVideoStreams = item.GetStreamCount(MediaStreamType.Video); + var videoCodec = videoStream?.Codec; + float videoFramerate = videoStream?.ReferenceFrameRate ?? 0; + TransportStreamTimestamp? timestamp = videoStream is null ? TransportStreamTimestamp.None : item.Timestamp; + int? numAudioStreams = item.GetStreamCount(MediaStreamType.Audio); + int? numVideoStreams = item.GetStreamCount(MediaStreamType.Video); - transcodingProfiles = transcodingProfiles.ToLookup(transcodingProfile => + var audioCodec = audioStream?.Codec; + var audioProfile = audioStream?.Profile; + var audioChannels = audioStream?.Channels; + var audioBitrate = audioStream?.BitRate; + var audioSampleRate = audioStream?.SampleRate; + var audioBitDepth = audioStream?.BitDepth; + + var analyzedProfiles = transcodingProfiles + .Select(transcodingProfile => { - var videoCodecs = ContainerProfile.SplitValue(transcodingProfile.VideoCodec); + var rank = (Video: 3, Audio: 3); - if (ContainerProfile.ContainsContainer(videoCodecs, item.VideoStream?.Codec)) + var container = transcodingProfile.Container; + + if (options.AllowVideoStreamCopy) { - var videoCodec = videoStream?.Codec; - var container = transcodingProfile.Container; - var appliedVideoConditions = options.Profile.CodecProfiles - .Where(i => i.Type == CodecType.Video && - i.ContainsAnyCodec(videoCodec, container) && - i.ApplyConditions.All(applyCondition => ConditionProcessor.IsVideoConditionSatisfied(applyCondition, videoStream?.Width, videoStream?.Height, videoStream?.BitDepth, videoStream?.BitRate, videoStream?.Profile, videoStream?.VideoRangeType, videoStream?.Level, videoFramerate, videoStream?.PacketLength, timestamp, videoStream?.IsAnamorphic, videoStream?.IsInterlaced, videoStream?.RefFrames, numVideoStreams, numAudioStreams, videoStream?.CodecTag, videoStream?.IsAVC))) - .Select(i => - i.Conditions.All(condition => ConditionProcessor.IsVideoConditionSatisfied(condition, videoStream?.Width, videoStream?.Height, videoStream?.BitDepth, videoStream?.BitRate, videoStream?.Profile, videoStream?.VideoRangeType, videoStream?.Level, videoFramerate, videoStream?.PacketLength, timestamp, videoStream?.IsAnamorphic, videoStream?.IsInterlaced, videoStream?.RefFrames, numVideoStreams, numAudioStreams, videoStream?.CodecTag, videoStream?.IsAVC))); + var videoCodecs = ContainerProfile.SplitValue(transcodingProfile.VideoCodec); - // An empty appliedVideoConditions means that the codec has no conditions for the current video stream - var conditionsSatisfied = appliedVideoConditions.All(satisfied => satisfied); - return conditionsSatisfied ? 1 : 2; + if (ContainerProfile.ContainsContainer(videoCodecs, videoCodec)) + { + var appliedVideoConditions = options.Profile.CodecProfiles + .Where(i => i.Type == CodecType.Video && + i.ContainsAnyCodec(videoCodec, container) && + i.ApplyConditions.All(applyCondition => ConditionProcessor.IsVideoConditionSatisfied(applyCondition, videoStream?.Width, videoStream?.Height, videoStream?.BitDepth, videoStream?.BitRate, videoStream?.Profile, videoStream?.VideoRangeType, videoStream?.Level, videoFramerate, videoStream?.PacketLength, timestamp, videoStream?.IsAnamorphic, videoStream?.IsInterlaced, videoStream?.RefFrames, numVideoStreams, numAudioStreams, videoStream?.CodecTag, videoStream?.IsAVC))) + .Select(i => + i.Conditions.All(condition => ConditionProcessor.IsVideoConditionSatisfied(condition, videoStream?.Width, videoStream?.Height, videoStream?.BitDepth, videoStream?.BitRate, videoStream?.Profile, videoStream?.VideoRangeType, videoStream?.Level, videoFramerate, videoStream?.PacketLength, timestamp, videoStream?.IsAnamorphic, videoStream?.IsInterlaced, videoStream?.RefFrames, numVideoStreams, numAudioStreams, videoStream?.CodecTag, videoStream?.IsAVC))); + + // An empty appliedVideoConditions means that the codec has no conditions for the current video stream + var conditionsSatisfied = appliedVideoConditions.All(satisfied => satisfied); + rank.Video = conditionsSatisfied ? 1 : 2; + } } - return 3; - }) - .OrderBy(lookup => lookup.Key) - .SelectMany(lookup => lookup); - } + if (options.AllowAudioStreamCopy) + { + var audioCodecs = ContainerProfile.SplitValue(transcodingProfile.AudioCodec); - return transcodingProfiles.FirstOrDefault(); + if (ContainerProfile.ContainsContainer(audioCodecs, audioCodec)) + { + var appliedVideoConditions = options.Profile.CodecProfiles + .Where(i => i.Type == CodecType.VideoAudio && + i.ContainsAnyCodec(audioCodec, container) && + i.ApplyConditions.All(applyCondition => ConditionProcessor.IsVideoAudioConditionSatisfied(applyCondition, audioChannels, audioBitrate, audioSampleRate, audioBitDepth, audioProfile, false))) + .Select(i => + i.Conditions.All(condition => ConditionProcessor.IsVideoAudioConditionSatisfied(condition, audioChannels, audioBitrate, audioSampleRate, audioBitDepth, audioProfile, false))); + + // An empty appliedVideoConditions means that the codec has no conditions for the current audio stream + var conditionsSatisfied = appliedVideoConditions.All(satisfied => satisfied); + rank.Audio = conditionsSatisfied ? 1 : 2; + } + } + + PlayMethod playMethod = PlayMethod.Transcode; + + if (rank.Video == 1) + { + playMethod = PlayMethod.DirectStream; + } + + return (Profile: transcodingProfile, PlayMethod: playMethod, Rank: rank); + }) + .OrderBy(analysis => analysis.Rank); + + var profileMatch = analyzedProfiles.FirstOrDefault(); + + return (profileMatch.Profile, profileMatch.PlayMethod); } private void BuildStreamVideoItem( diff --git a/tests/Jellyfin.Model.Tests/Dlna/StreamBuilderTests.cs b/tests/Jellyfin.Model.Tests/Dlna/StreamBuilderTests.cs index 31ddd427cc..3429d1a5bd 100644 --- a/tests/Jellyfin.Model.Tests/Dlna/StreamBuilderTests.cs +++ b/tests/Jellyfin.Model.Tests/Dlna/StreamBuilderTests.cs @@ -309,6 +309,9 @@ namespace Jellyfin.Model.Tests [InlineData("Tizen4-4K-5.1", "mp4-h264-ac3-aac-srt-2600k", PlayMethod.DirectStream, TranscodeReason.SecondaryAudioNotSupported, "Remux")] [InlineData("Tizen4-4K-5.1", "mp4-h264-ac3-aac-aac-srt-2600k", PlayMethod.DirectStream, TranscodeReason.SecondaryAudioNotSupported, "Remux")] [InlineData("Tizen4-4K-5.1", "mp4-hevc-ac3-aac-srt-15200k", PlayMethod.DirectStream, TranscodeReason.SecondaryAudioNotSupported, "Remux")] + // TranscodeMedia + [InlineData("TranscodeMedia", "mp4-h264-ac3-aac-srt-2600k", PlayMethod.Transcode, TranscodeReason.DirectPlayError, "Remux", "HLS.mp4")] + [InlineData("TranscodeMedia", "mp4-h264-ac3-aac-mp3-srt-2600k", PlayMethod.Transcode, TranscodeReason.DirectPlayError, "Remux", "HLS.ts")] public async Task BuildVideoItemWithDirectPlayExplicitStreams(string deviceName, string mediaSource, PlayMethod? playMethod, TranscodeReason why = default, string transcodeMode = "DirectStream", string transcodeProtocol = "") { var options = await GetMediaOptions(deviceName, mediaSource); diff --git a/tests/Jellyfin.Model.Tests/Test Data/MediaSourceInfo-mp4-h264-ac3-aac-mp3-srt-2600k.json b/tests/Jellyfin.Model.Tests/Test Data/MediaSourceInfo-mp4-h264-ac3-aac-mp3-srt-2600k.json new file mode 100644 index 0000000000..2e05e70d69 --- /dev/null +++ b/tests/Jellyfin.Model.Tests/Test Data/MediaSourceInfo-mp4-h264-ac3-aac-mp3-srt-2600k.json @@ -0,0 +1,100 @@ +{ + "Id": "a766d122b58e45d9492d17af77748bf5", + "Path": "/Media/MyVideo-720p.mp4", + "Container": "mov,mp4,m4a,3gp,3g2,mj2", + "Size": 835317696, + "Name": "MyVideo-720p", + "ETag": "579a34c6d5dfb21d81539a51220b6a23", + "RunTimeTicks": 25801230336, + "SupportsTranscoding": true, + "SupportsDirectStream": true, + "SupportsDirectPlay": true, + "SupportsProbing": true, + "MediaStreams": [ + { + "Codec": "h264", + "CodecTag": "avc1", + "Language": "eng", + "TimeBase": "1/11988", + "VideoRange": "SDR", + "DisplayTitle": "720p H264 SDR", + "NalLengthSize": "0", + "BitRate": 2032876, + "BitDepth": 8, + "RefFrames": 1, + "IsDefault": true, + "Height": 720, + "Width": 1280, + "AverageFrameRate": 23.976, + "RealFrameRate": 23.976, + "Profile": "High", + "Type": 1, + "AspectRatio": "16:9", + "PixelFormat": "yuv420p", + "Level": 41 + }, + { + "Codec": "ac3", + "CodecTag": "ac-3", + "Language": "eng", + "TimeBase": "1/48000", + "DisplayTitle": "En - Dolby Digital - 5.1 - Default", + "ChannelLayout": "5.1", + "BitRate": 384000, + "Channels": 6, + "SampleRate": 48000, + "IsDefault": true, + "Index": 1, + "Score": 202 + }, + { + "Codec": "aac", + "CodecTag": "mp4a", + "Language": "eng", + "TimeBase": "1/48000", + "DisplayTitle": "En - AAC - Stereo", + "ChannelLayout": "stereo", + "BitRate": 164741, + "Channels": 2, + "SampleRate": 48000, + "IsDefault": false, + "Profile": "LC", + "Index": 2, + "Score": 203 + }, + { + "Codec": "mp3", + "Language": "eng", + "TimeBase": "1/48000", + "DisplayTitle": "En - MP3 - Stereo", + "ChannelLayout": "stereo", + "BitRate": 164741, + "Channels": 2, + "SampleRate": 48000, + "IsDefault": false, + "Index": 3, + "Score": 203 + }, + { + "Codec": "srt", + "Language": "eng", + "TimeBase": "1/1000000", + "localizedUndefined": "Undefined", + "localizedDefault": "Default", + "localizedForced": "Forced", + "DisplayTitle": "En - Default", + "BitRate": 92, + "IsDefault": true, + "Type": 2, + "Index": 4, + "Score": 6421, + "IsExternal": true, + "IsTextSubtitleStream": true, + "SupportsExternalStream": true, + "Path": "/Media/MyVideo-WEBDL-2160p.default.eng.srt" + } + ], + "Bitrate": 2590008, + "DefaultAudioStreamIndex": 1, + "DefaultSubtitleStreamIndex": 4 +}