From 87a3c5d11c9fcfa5e14598c028ac7a665906fd17 Mon Sep 17 00:00:00 2001
From: nyanmisaka <nst799610810@gmail.com>
Date: Tue, 19 Nov 2024 15:43:24 -0500
Subject: [PATCH] Backport pull request #12973 from jellyfin/release-10.10.z

Fix pixel format in HEVC RExt SDR transcoding

Original-merge: aa08d3f2bf155d55f748bff1f0a0c7f071f79ae7

Merged-by: crobibero <cody@robibe.ro>

Backported-by: Joshua M. Boniface <joshua@boniface.me>
---
 MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
index eaae34cad2..e1d0ed0a08 100644
--- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
+++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
@@ -4131,7 +4131,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             else if (isD3d11vaDecoder || isQsvDecoder)
             {
                 var isRext = IsVideoStreamHevcRext(state);
-                var twoPassVppTonemap = isRext;
+                var twoPassVppTonemap = false;
                 var doVppFullRangeOut = isMjpegEncoder
                     && _mediaEncoder.EncoderVersion >= _minFFmpegQsvVppOutRangeOption;
                 var doVppScaleModeHq = isMjpegEncoder
@@ -4140,6 +4140,12 @@ namespace MediaBrowser.Controller.MediaEncoding
                 var procampParams = string.Empty;
                 if (doVppTonemap)
                 {
+                    if (isRext)
+                    {
+                        // VPP tonemap requires p010 input
+                        twoPassVppTonemap = true;
+                    }
+
                     if (options.VppTonemappingBrightness != 0
                         && options.VppTonemappingBrightness >= -100
                         && options.VppTonemappingBrightness <= 100)