vo_gpu: simplify and correct color scale handling

The primary need for this change is the fact that the OOTF was
incorrectly scaled, due to the fact that the application of the OOTF can
itself change the required normalization peak. (Plus, an oversight in
pass_inverse_ootf meant we forgot to normalize at the end of it)

The linearize/delinearize functions still normalize the scale since it's
used in a number of places throughout gpu/video.c, but the color
management function now converts to absolute scale right away, instead
of in an awkward way inside the tone mapping branch. The OOTF functions
now work in absolute scale only.

In addition, minor changes have been made to the way normalization is
handled for tone mapping - we now divide out the dst_peak *after* peak
detection, in order to make the scale of the peak detection buffer
consistent even if the dst_peak were to (hypothetically) change
mid-stream. In theory, we could also do this for desaturation, but doing
the desaturation before tone mapping has the advantage of preserving
much more brightness than the other way around - and even mid-stream
changes are not that drastic here.

Finally, some preparation work has been done for allowing the user to
customize the `dst.sig_peak` in the future.

1 files changed, 34 insertions, 31 deletions

diff --git a/video/out/gpu/video_shaders.c b/video/out/gpu/video_shaders.c
index 23824cfdb5..cd63b37c11 100644
--- a/video/out/gpu/video_shaders.c
+++ b/video/out/gpu/video_shaders.c
@@ -334,6 +334,10 @@ static const float SLOG_A = 0.432699,
 // Linearize (expand), given a TRC as input. In essence, this is the ITU-R
 // EOTF, calculated on an idealized (reference) monitor with a white point of
 // MP_REF_WHITE and infinite contrast.
+//
+// These functions always output to a normalized scale of [0,1], for
+// convenience of the video.c code that calls it. To get the values in an
+// absolute scale, multiply the result by `mp_trc_nom_peak(trc)`
 void pass_linearize(struct gl_shader_cache *sc, enum mp_csp_trc trc)
 {
     if (trc == MP_CSP_TRC_LINEAR)
@@ -417,6 +421,8 @@ void pass_linearize(struct gl_shader_cache *sc, enum mp_csp_trc trc)
 // Delinearize (compress), given a TRC as output. This corresponds to the
 // inverse EOTF (not the OETF) in ITU-R terminology, again assuming a
 // reference monitor.
+//
+// Like pass_linearize, this functions ingests values on an normalized scale
 void pass_delinearize(struct gl_shader_cache *sc, enum mp_csp_trc trc)
 {
     if (trc == MP_CSP_TRC_LINEAR)
@@ -488,15 +494,13 @@ void pass_delinearize(struct gl_shader_cache *sc, enum mp_csp_trc trc)
 }
 
 // Apply the OOTF mapping from a given light type to display-referred light.
-// The extra peak parameter is used to scale the values before and after
-// the OOTF, and can be inferred using mp_trc_nom_peak
-void pass_ootf(struct gl_shader_cache *sc, enum mp_csp_light light, float peak)
+// Assumes absolute scale values.
+static void pass_ootf(struct gl_shader_cache *sc, enum mp_csp_light light)
 {
     if (light == MP_CSP_LIGHT_DISPLAY)
         return;
 
     GLSLF("// apply ootf\n");
-    GLSLF("color.rgb *= vec3(%f);\n", peak);
 
     switch (light)
     {
@@ -521,18 +525,15 @@ void pass_ootf(struct gl_shader_cache *sc, enum mp_csp_light light, float peak)
     default:
         abort();
     }
-
-    GLSLF("color.rgb *= vec3(1.0/%f);\n", peak);
 }
 
 // Inverse of the function pass_ootf, for completeness' sake.
-void pass_inverse_ootf(struct gl_shader_cache *sc, enum mp_csp_light light, float peak)
+static void pass_inverse_ootf(struct gl_shader_cache *sc, enum mp_csp_light light)
 {
     if (light == MP_CSP_LIGHT_DISPLAY)
         return;
 
     GLSLF("// apply inverse ootf\n");
-    GLSLF("color.rgb *= vec3(%f);\n", peak);
 
     switch (light)
     {
@@ -635,7 +636,7 @@ static void hdr_update_peak(struct gl_shader_cache *sc)
 // Tone map from a known peak brightness to the range [0,1]. If ref_peak
 // is 0, we will use peak detection instead
 static void pass_tone_map(struct gl_shader_cache *sc, bool detect_peak,
-                          float src_peak, float dst_range,
+                          float src_peak, float dst_peak,
                           enum tone_mapping algo, float param, float desat)
 {
     GLSLF("// HDR tone mapping\n");
@@ -647,18 +648,13 @@ static void pass_tone_map(struct gl_shader_cache *sc, bool detect_peak,
     GLSLF("float sig_peak = %f;\n", src_peak);
     GLSLF("float sig_avg = %f;\n", sdr_avg);
 
-    // Rescale the variables in order to bring it into a representation where
-    // 1.0 represents the dst_peak. This is because all of the tone mapping
-    // algorithms are defined in such a way that they map to the range [0.0, 1.0].
-    if (dst_range > 1.0) {
-        GLSLF("sig *= %f;\n", 1.0 / dst_range);
-        GLSLF("sig_peak *= %f;\n", 1.0 / dst_range);
-    }
-
     // Desaturate the color using a coefficient dependent on the signal
+    // Do this before peak detection in order to try and reclaim as much
+    // dynamic range as possible.
     if (desat > 0) {
+        float base = 0.18 * dst_peak;
         GLSL(float luma = dot(dst_luma, color.rgb);)
-        GLSL(float coeff = max(sig - 0.18, 1e-6) / max(sig, 1e-6););
+        GLSLF("float coeff = max(sig - %f, 1e-6) / max(sig, 1e-6);\n", base);
         GLSLF("coeff = pow(coeff, %f);\n", 10.0 / desat);
         GLSL(color.rgb = mix(color.rgb, vec3(luma), coeff);)
         GLSL(sig = mix(sig, luma, coeff);) // also make sure to update `sig`
@@ -667,6 +663,14 @@ static void pass_tone_map(struct gl_shader_cache *sc, bool detect_peak,
     if (detect_peak)
         hdr_update_peak(sc);
 
+    // Rescale the variables in order to bring it into a representation where
+    // 1.0 represents the dst_peak. This is because all of the tone mapping
+    // algorithms are defined in such a way that they map to the range [0.0, 1.0].
+    if (dst_peak > 1.0) {
+        GLSLF("sig *= %f;\n", 1.0 / dst_peak);
+        GLSLF("sig_peak *= %f;\n", 1.0 / dst_peak);
+    }
+
     GLSL(float sig_orig = sig;)
     GLSLF("float slope = min(1.0, %f / sig_avg);\n", sdr_avg);
     GLSL(sig *= slope;)
@@ -728,7 +732,7 @@ static void pass_tone_map(struct gl_shader_cache *sc, bool detect_peak,
     // Apply the computed scale factor to the color, linearly to prevent
     // discoloration
     GLSL(sig = min(sig, 1.0);)
-    GLSL(color.rgb *= sig / sig_orig;)
+    GLSL(color.rgb *= vec3(sig / sig_orig);)
 }
 
 // Map colors from one source space to another. These source spaces must be
@@ -746,10 +750,6 @@ void pass_color_map(struct gl_shader_cache *sc,
 {
     GLSLF("// color mapping\n");
 
-    // Compute the highest encodable level
-    float src_range = mp_trc_nom_peak(src.gamma),
-          dst_range = mp_trc_nom_peak(dst.gamma);
-
     // Some operations need access to the video's luma coefficients, so make
     // them available
     float rgb2xyz[3][3];
@@ -763,8 +763,7 @@ void pass_color_map(struct gl_shader_cache *sc,
     // operations needs it
     bool need_gamma = src.gamma != dst.gamma ||
                       src.primaries != dst.primaries ||
-                      src_range != dst_range ||
-                      src.sig_peak > dst_range ||
+                      src.sig_peak > dst.sig_peak ||
                       src.light != dst.light;
 
     if (need_gamma && !is_linear) {
@@ -773,8 +772,11 @@ void pass_color_map(struct gl_shader_cache *sc,
         is_linear = true;
     }
 
+    // Pre-scale the incoming values into an absolute scale
+    GLSLF("color.rgb *= vec3(%f);\n", mp_trc_nom_peak(src.gamma));
+
     if (src.light != dst.light)
-        pass_ootf(sc, src.light, src_range);
+        pass_ootf(sc, src.light);
 
     // Adapt to the right colorspace if necessary
     if (src.primaries != dst.primaries) {
@@ -791,15 +793,16 @@ void pass_color_map(struct gl_shader_cache *sc,
 
     // Tone map to prevent clipping when the source signal peak exceeds the
     // encodable range or we've reduced the gamut
-    if (src.sig_peak > dst_range) {
-        GLSLF("color.rgb *= vec3(%f);\n", src_range);
-        pass_tone_map(sc, detect_peak, src.sig_peak, dst_range, algo,
+    if (src.sig_peak > dst.sig_peak) {
+        pass_tone_map(sc, detect_peak, src.sig_peak, dst.sig_peak, algo,
                       tone_mapping_param, tone_mapping_desat);
-        GLSLF("color.rgb *= vec3(%f);\n", 1.0 / dst_range);
     }
 
     if (src.light != dst.light)
-        pass_inverse_ootf(sc, dst.light, dst_range);
+        pass_inverse_ootf(sc, dst.light);
+
+    // Post-scale the outgoing values from absolute scale to normalized
+    GLSLF("color.rgb *= vec3(%f);\n", 1.0 / mp_trc_nom_peak(dst.gamma));
 
     // Warn for remaining out-of-gamut colors is enabled
     if (gamut_warning) {