From 0fc288936d10afc146780d118361f2e722768ee6 Mon Sep 17 00:00:00 2001 From: gnattu Date: Mon, 9 Dec 2024 16:17:49 +0800 Subject: [PATCH 1/3] Enable VideoToolbox AV1 decode This decoder differs from others provided by VideoToolbox in that it lacks any software fallback. To achieve consistent behavior with other VideoToolbox decoders, this PR implemented additional checking on the server to simulate the software fallback provided by VideoToolbox. The current fallback checking mechanism is a temporary solution. In the long term, it should be replaced with a more capable hardware capability checking system. --- .../MediaEncoding/EncodingHelper.cs | 8 ++ .../MediaEncoding/IMediaEncoder.cs | 6 ++ .../Encoder/ApplePlatformHelper.cs | 85 +++++++++++++++++++ .../Encoder/EncoderValidator.cs | 5 ++ .../Encoder/MediaEncoder.cs | 10 +++ 5 files changed, 114 insertions(+) create mode 100644 MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index 9399679a4f..7dea5f8ebd 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -6610,6 +6610,7 @@ namespace MediaBrowser.Controller.MediaEncoding || string.Equals("yuv420p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) || string.Equals("yuv422p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase) || string.Equals("yuv444p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var isAv1SupportedSwFormatsVt = is8_10bitSwFormatsVt || string.Equals("yuv420p12le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); // The related patches make videotoolbox hardware surface working is only available in jellyfin-ffmpeg 7.0.1 at the moment. bool useHwSurface = (_mediaEncoder.EncoderVersion >= _minFFmpegWorkingVtHwSurface) && IsVideoToolboxFullSupported(); @@ -6643,6 +6644,13 @@ namespace MediaBrowser.Controller.MediaEncoding { return GetHwaccelType(state, options, "hevc", bitDepth, useHwSurface); } + + if (string.Equals("av1", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + && isAv1SupportedSwFormatsVt + && _mediaEncoder.IsVideoToolboxAv1DecodeAvailable) + { + return GetHwaccelType(state, options, "av1", bitDepth, useHwSurface); + } } return null; diff --git a/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs b/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs index c767b4a519..a60f523408 100644 --- a/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs +++ b/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs @@ -75,6 +75,12 @@ namespace MediaBrowser.Controller.MediaEncoding /// true if the Vaapi device supports vulkan drm interop, false otherwise. bool IsVaapiDeviceSupportVulkanDrmInterop { get; } + /// + /// Gets a value indicating whether av1 decoding is available via VideoToolbox. + /// + /// true if the av1 is available via VideoToolbox, false otherwise. + bool IsVideoToolboxAv1DecodeAvailable { get; } + /// /// Whether given encoder codec is supported. /// diff --git a/MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs b/MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs new file mode 100644 index 0000000000..ea2289bd7d --- /dev/null +++ b/MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs @@ -0,0 +1,85 @@ +#pragma warning disable CA1031 + +using System; +using System.Linq; +using System.Runtime.InteropServices; +using Microsoft.Extensions.Logging; + +namespace MediaBrowser.MediaEncoding.Encoder; + +/// +/// Helper class for Apple platform specific operations. +/// +public static class ApplePlatformHelper +{ + private static readonly string[] _av1DecodeBlacklistedCpuClass = ["M1", "M2"]; + + private static string GetSysctlValue(string name) + { + IntPtr length = IntPtr.Zero; + // Get length of the value + int osStatus = SysctlByName(name, IntPtr.Zero, ref length, IntPtr.Zero, 0); + + if (osStatus != 0) + { + throw new NotSupportedException($"Failed to get sysctl value for {name} with error {osStatus}"); + } + + IntPtr buffer = Marshal.AllocHGlobal(length.ToInt32()); + try + { + osStatus = SysctlByName(name, buffer, ref length, IntPtr.Zero, 0); + if (osStatus != 0) + { + throw new NotSupportedException($"Failed to get sysctl value for {name} with error {osStatus}"); + } + + return Marshal.PtrToStringAnsi(buffer) ?? string.Empty; + } + finally + { + Marshal.FreeHGlobal(buffer); + } + } + + private static int SysctlByName(string name, IntPtr oldp, ref IntPtr oldlenp, IntPtr newp, uint newlen) + { + return NativeMethods.SysctlByName(System.Text.Encoding.ASCII.GetBytes(name), oldp, ref oldlenp, newp, newlen); + } + + /// + /// Check if the current system has hardware acceleration for AV1 decoding. + /// + /// The logger used for error logging. + /// Boolean indicates the hwaccel support. + public static bool HasAv1HardwareAccel(ILogger logger) + { + if (!RuntimeInformation.OSArchitecture.Equals(Architecture.Arm64)) + { + return false; + } + + try + { + string cpuBrandString = GetSysctlValue("machdep.cpu.brand_string"); + return !_av1DecodeBlacklistedCpuClass.Any(blacklistedCpuClass => cpuBrandString.Contains(blacklistedCpuClass, StringComparison.OrdinalIgnoreCase)); + } + catch (NotSupportedException e) + { + logger.LogError("Error getting CPU brand string: {Message}", e.Message); + } + catch (Exception e) + { + logger.LogError("Unknown error occured: {Exception}", e); + } + + return false; + } + + private static class NativeMethods + { + [DllImport("libc", EntryPoint = "sysctlbyname", SetLastError = true)] + [DefaultDllImportSearchPaths(DllImportSearchPath.SafeDirectories)] + internal static extern int SysctlByName(byte[] name, IntPtr oldp, ref IntPtr oldlenp, IntPtr newp, uint newlen); + } +} diff --git a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs index 23d9ca7ef5..776b2ab42c 100644 --- a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs +++ b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs @@ -437,6 +437,11 @@ namespace MediaBrowser.MediaEncoding.Encoder } } + public bool CheckIsVideoToolboxAv1DecodeAvailable() + { + return ApplePlatformHelper.HasAv1HardwareAccel(_logger); + } + private IEnumerable GetHwaccelTypes() { string? output = null; diff --git a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs index a34238cd68..44b38f03bc 100644 --- a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs +++ b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs @@ -83,6 +83,8 @@ namespace MediaBrowser.MediaEncoding.Encoder private bool _isVaapiDeviceSupportVulkanDrmModifier = false; private bool _isVaapiDeviceSupportVulkanDrmInterop = false; + private bool _isVideoToolboxAv1DecodeAvailable = false; + private static string[] _vulkanImageDrmFmtModifierExts = { "VK_EXT_image_drm_format_modifier", @@ -153,6 +155,8 @@ namespace MediaBrowser.MediaEncoding.Encoder /// public bool IsVaapiDeviceSupportVulkanDrmInterop => _isVaapiDeviceSupportVulkanDrmInterop; + public bool IsVideoToolboxAv1DecodeAvailable => _isVideoToolboxAv1DecodeAvailable; + [GeneratedRegex(@"[^\/\\]+?(\.[^\/\\\n.]+)?$")] private static partial Regex FfprobePathRegex(); @@ -255,6 +259,12 @@ namespace MediaBrowser.MediaEncoding.Encoder _logger.LogInformation("VAAPI device {RenderNodePath} supports Vulkan DRM interop", options.VaapiDevice); } } + + // Check if VideoToolbox supports AV1 decode + if (OperatingSystem.IsMacOS() && SupportsHwaccel("videotoolbox")) + { + _isVideoToolboxAv1DecodeAvailable = validator.CheckIsVideoToolboxAv1DecodeAvailable(); + } } _logger.LogInformation("FFmpeg: {FfmpegPath}", _ffmpegPath ?? string.Empty); From 384134fd25fec22792733ccaa9f10a23d5c98eac Mon Sep 17 00:00:00 2001 From: gnattu Date: Fri, 28 Mar 2025 21:22:00 +0800 Subject: [PATCH 2/3] Use string literal --- .../Encoder/ApplePlatformHelper.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs b/MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs index ea2289bd7d..76fc5f695f 100644 --- a/MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs +++ b/MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs @@ -14,7 +14,7 @@ public static class ApplePlatformHelper { private static readonly string[] _av1DecodeBlacklistedCpuClass = ["M1", "M2"]; - private static string GetSysctlValue(string name) + private static string GetSysctlValue(ReadOnlySpan name) { IntPtr length = IntPtr.Zero; // Get length of the value @@ -22,7 +22,7 @@ public static class ApplePlatformHelper if (osStatus != 0) { - throw new NotSupportedException($"Failed to get sysctl value for {name} with error {osStatus}"); + throw new NotSupportedException($"Failed to get sysctl value for {System.Text.Encoding.UTF8.GetString(name)} with error {osStatus}"); } IntPtr buffer = Marshal.AllocHGlobal(length.ToInt32()); @@ -31,7 +31,7 @@ public static class ApplePlatformHelper osStatus = SysctlByName(name, buffer, ref length, IntPtr.Zero, 0); if (osStatus != 0) { - throw new NotSupportedException($"Failed to get sysctl value for {name} with error {osStatus}"); + throw new NotSupportedException($"Failed to get sysctl value for {System.Text.Encoding.UTF8.GetString(name)} with error {osStatus}"); } return Marshal.PtrToStringAnsi(buffer) ?? string.Empty; @@ -42,9 +42,9 @@ public static class ApplePlatformHelper } } - private static int SysctlByName(string name, IntPtr oldp, ref IntPtr oldlenp, IntPtr newp, uint newlen) + private static int SysctlByName(ReadOnlySpan name, IntPtr oldp, ref IntPtr oldlenp, IntPtr newp, uint newlen) { - return NativeMethods.SysctlByName(System.Text.Encoding.ASCII.GetBytes(name), oldp, ref oldlenp, newp, newlen); + return NativeMethods.SysctlByName(name.ToArray(), oldp, ref oldlenp, newp, newlen); } /// @@ -61,7 +61,7 @@ public static class ApplePlatformHelper try { - string cpuBrandString = GetSysctlValue("machdep.cpu.brand_string"); + string cpuBrandString = GetSysctlValue("machdep.cpu.brand_string"u8); return !_av1DecodeBlacklistedCpuClass.Any(blacklistedCpuClass => cpuBrandString.Contains(blacklistedCpuClass, StringComparison.OrdinalIgnoreCase)); } catch (NotSupportedException e) From c69e9d8f2cac26a832f0ee09bed2809aae1872bf Mon Sep 17 00:00:00 2001 From: gnattu Date: Fri, 28 Mar 2025 21:30:39 +0800 Subject: [PATCH 3/3] Gate the macOS only functions --- MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs | 2 ++ MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs b/MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs index 76fc5f695f..a8ff58b091 100644 --- a/MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs +++ b/MediaBrowser.MediaEncoding/Encoder/ApplePlatformHelper.cs @@ -3,6 +3,7 @@ using System; using System.Linq; using System.Runtime.InteropServices; +using System.Runtime.Versioning; using Microsoft.Extensions.Logging; namespace MediaBrowser.MediaEncoding.Encoder; @@ -10,6 +11,7 @@ namespace MediaBrowser.MediaEncoding.Encoder; /// /// Helper class for Apple platform specific operations. /// +[SupportedOSPlatform("macos")] public static class ApplePlatformHelper { private static readonly string[] _av1DecodeBlacklistedCpuClass = ["M1", "M2"]; diff --git a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs index 776b2ab42c..54d0eb4b51 100644 --- a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs +++ b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Diagnostics; using System.Globalization; using System.Linq; +using System.Runtime.Versioning; using System.Text.RegularExpressions; using Microsoft.Extensions.Logging; @@ -437,6 +438,7 @@ namespace MediaBrowser.MediaEncoding.Encoder } } + [SupportedOSPlatform("macos")] public bool CheckIsVideoToolboxAv1DecodeAvailable() { return ApplePlatformHelper.HasAv1HardwareAccel(_logger);