From f31549cc0de0973e69b052718252bae6a2a45522 Mon Sep 17 00:00:00 2001 From: gnattu Date: Fri, 8 Mar 2024 23:23:24 +0800 Subject: [PATCH] fix: code clean up Co-authored-by: nyanmisaka Signed-off-by: gnattu --- .../MediaEncoding/EncodingHelper.cs | 157 ++++++++++-------- 1 file changed, 91 insertions(+), 66 deletions(-) diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index 0fc27f9f79..1ac40c7184 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -253,7 +253,7 @@ namespace MediaBrowser.Controller.MediaEncoding && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVulkanFrameSync); } - private bool IsVideoToolBoxFullSupported() + private bool IsVideoToolboxFullSupported() { return _mediaEncoder.SupportsHwaccel("videotoolbox") && _mediaEncoder.SupportsFilter("yadif_videotoolbox") @@ -4978,100 +4978,118 @@ namespace MediaBrowser.Controller.MediaEncoding return (null, null, null); } - var swFilterChain = GetSwVidFilterChain(state, options, vidEncoder); + var isMacOS = OperatingSystem.IsMacOS(); + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); + var isVtFullSupported = isMacOS && IsVideoToolboxFullSupported(); + var isVtOclSupported = isVtFullSupported && IsOpenclFullSupported(); - if (!options.EnableHardwareEncoding) + // legacy videotoolbox pipeline (disable hw filters) + if (!isVtEncoder + || !isVtOclSupported + || !_mediaEncoder.SupportsFilter("alphasrc")) { - return swFilterChain; + return GetSwVidFilterChain(state, options, vidEncoder); } - var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); - var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); - var doDeintH2645 = doDeintH264 || doDeintHevc; + // preferred videotoolbox + vt/ocl filters pipeline + return GetAppleVidFiltersPreferred(state, options, vidDecoder, vidEncoder); + } + + public (List MainFilters, List SubFilters, List OverlayFilters) GetAppleVidFiltersPreferred( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { var inW = state.VideoStream?.Width; var inH = state.VideoStream?.Height; var reqW = state.BaseRequest.Width; var reqH = state.BaseRequest.Height; var reqMaxW = state.BaseRequest.MaxWidth; var reqMaxH = state.BaseRequest.MaxHeight; - var mainFilters = new List(); + var threeDFormat = state.MediaSource.Video3DFormat; + + var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doDeintH2645 = doDeintH264 || doDeintHevc; + var doVtTonemap = IsVideoToolboxTonemapAvailable(state, options); + var doOclTonemap = !doVtTonemap && IsHwTonemapAvailable(state, options); + var doTonemap = doVtTonemap || doOclTonemap; + var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; var hasAssSubs = hasSubs - && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) - || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); - // VideoToolbox is special. It does not use a separate tone mapping filter like others. - // Instead, it performs both tone mapping and scaling in a single filter. - var useVtToneMapping = IsVideoToolboxTonemapAvailable(state, options); - // Use OpenCL tone mapping as a fallback - var useOclToneMapping = !useVtToneMapping && IsHwTonemapAvailable(state, options); - // Fallback to software filters if we are using filters not supported by hardware yet. - // OpenCL won't work without proper VT support as the hwmap interop required will not be present there - var useHardwareFilters = IsVideoToolBoxFullSupported(); + && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); - if (!useHardwareFilters) + // FIXME: scale_vt lacks of format option for the time being. + // hwdownload/hwmap to sw requires setting a format explicitly. + if (!isVtEncoder) { - return swFilterChain; + // should not happen. + return (null, null, null); } - if (!(useVtToneMapping || useOclToneMapping || hasSubs || doDeintH2645)) + /* Make main filters for video stream */ + var mainFilters = new List(); + + if (!(doTonemap || hasSubs || doDeintH2645)) { // Dummy action to return empty filters when nothing to do. return (mainFilters, mainFilters, mainFilters); } - // Override the color when doing OpenCL Tone mapping, where we are using hardware surface output. - if (useOclToneMapping) + // Color override is only required for OpenCL where hardware surface is in use + if (doOclTonemap) { - mainFilters.Add(GetOverwriteColorPropertiesParam(state, true)); + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doOclTonemap)); } - // With OpenCL tone mapping, we are using native hwmap and there's no need to specify a format for the main stream. - // However, for other cases, we have to specify the format for the main stream when we are doing subtitle burn-in. - // This is because the default upload option is not always processable with VideoToolbox. - // Most notably, yuv420p should be replaced by nv12. - else if (hasSubs) - { - var is8Bit = string.Equals("yuv420p", state.VideoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) - || string.Equals("yuvj420p", state.VideoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); - var is10Bit = string.Equals("yuv420p10le", state.VideoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); - - if (is8Bit) - { - mainFilters.Add("format=nv12"); - } - else if (is10Bit) - { - mainFilters.Add("format=p010"); - } - } - - mainFilters.Add("hwupload"); - var hwScaleFilter = GetHwScaleFilter("vt", null, inW, inH, reqW, reqH, reqMaxW, reqMaxH); - if (useVtToneMapping) - { - const string TonemapArgs = "color_matrix=bt709:color_primaries=bt709:color_transfer=bt709"; - hwScaleFilter = string.IsNullOrEmpty(hwScaleFilter) - ? "scale_vt=" + TonemapArgs - : hwScaleFilter + ":" + TonemapArgs; - } - - mainFilters.Add(hwScaleFilter); - - if (useOclToneMapping) - { - mainFilters.Add("hwmap=derive_device=opencl"); - mainFilters.Add(GetHwTonemapFilter(options, "opencl", "nv12")); - mainFilters.Add("hwmap=derive_device=videotoolbox:reverse=1"); - } + // INPUT videotoolbox/memory surface(vram/uma) + // this will pass-through automatically if in/out format matches. + mainFilters.Add("format=nv12|p010le|videotoolbox_vld"); + mainFilters.Add("hwupload=derive_device=videotoolbox"); + // hw deint if (doDeintH2645) { var deintFilter = GetHwDeinterlaceFilter(state, options, "videotoolbox"); mainFilters.Add(deintFilter); } + var hwScaleFilter = GetHwScaleFilter("vt", null, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + if (doVtTonemap) + { + const string VtTonemapArgs = "color_matrix=bt709:color_primaries=bt709:color_transfer=bt709"; + + // scale_vt can handle scaling & tonemapping in one shot, just like vpp_qsv. + hwScaleFilter = string.IsNullOrEmpty(hwScaleFilter) + ? "scale_vt=" + VtTonemapArgs + : hwScaleFilter + ":" + VtTonemapArgs; + } + + // hw scale & vt tonemap + mainFilters.Add(hwScaleFilter); + + // ocl tonemap + if (doOclTonemap) + { + // map from videotoolbox to opencl via videotoolbox-opencl interop. + mainFilters.Add("hwmap=derive_device=opencl:mode=read"); + + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + mainFilters.Add(tonemapFilter); + + // OUTPUT videotoolbox(nv12) surface(vram/uma) + // reverse-mapping via videotoolbox-opencl interop. + mainFilters.Add("hwmap=derive_device=videotoolbox:mode=write:reverse=1"); + } + + /* Make sub and overlay filters for subtitle stream */ var subFilters = new List(); var overlayFilters = new List(); @@ -6094,10 +6112,17 @@ namespace MediaBrowser.Controller.MediaEncoding // Hardware surface only make sense when interop with OpenCL // VideoToolbox's Hardware surface in ffmpeg is not only slower than hwupload, but also breaks HDR in many cases. // For example: https://trac.ffmpeg.org/ticket/10884 - var useOclToneMapping = !IsVideoToolboxTonemapAvailable(state, options) && - options.EnableTonemapping && - state.VideoStream.VideoRangeType == VideoRangeType.DOVI; - var useHwSurface = useOclToneMapping && IsVideoToolBoxFullSupported() && _mediaEncoder.SupportsFilter("alphasrc"); + var useOclToneMapping = !IsVideoToolboxTonemapAvailable(state, options) + && options.EnableTonemapping + && state.VideoStream is not null + && GetVideoColorBitDepth(state) == 10 + && state.VideoStream.VideoRange == VideoRange.HDR + && (state.VideoStream.VideoRangeType == VideoRangeType.HDR10 + || state.VideoStream.VideoRangeType == VideoRangeType.HLG + || (state.VideoStream.VideoRangeType == VideoRangeType.DOVI + && string.Equals(state.VideoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase))); + + var useHwSurface = useOclToneMapping && IsVideoToolboxFullSupported() && _mediaEncoder.SupportsFilter("alphasrc"); if (is8bitSwFormatsVt) {