vo_opengl: tone map in linear XYZ instead of RGB

This preserves channel balance better and helps reduce discoloration due
to nonlinear tone mapping.

I wasn't sure whether to stuff this inside pass_color_manage or
pass_tone_map but decided for the former because adding the extra
mp_csp_prim would have made the signature of the latter longer than
80col, and also because the `mp_get_cms_matrix` below it basically does
the same thing anyway, so it doesn't look that out of place. Also why is
this justification longer than the actual description of the algorithm
and what it's good for?

1 files changed, 19 insertions, 1 deletions

diff --git a/video/out/opengl/video_shaders.c b/video/out/opengl/video_shaders.c
index f56c687903..26f751afb2 100644
--- a/video/out/opengl/video_shaders.c
+++ b/video/out/opengl/video_shaders.c
@@ -548,8 +548,26 @@ void pass_color_map(struct gl_shader_cache *sc,
 
     // Tone map to prevent clipping when the source signal peak exceeds the
     // encodable range
-    if (src.sig_peak > dst_range)
+    if (src.sig_peak > dst_range) {
+        // Convert to linear, relative XYZ before tone mapping to preserve
+        // channel balance better
+        struct mp_csp_primaries prim = mp_get_csp_primaries(src.primaries);
+        float rgb2xyz[3][3];
+        mp_get_rgb2xyz_matrix(prim, rgb2xyz);
+        gl_sc_uniform_mat3(sc, "rgb2xyz", true, &rgb2xyz[0][0]);
+        mp_invert_matrix3x3(rgb2xyz);
+        gl_sc_uniform_mat3(sc, "xyz2rgb", true, &rgb2xyz[0][0]);
+        // White balance, calculated from the relative XYZ coefficients of
+        // the white point. Failing to multiply in this difference causes
+        // the tone mapping process to shift the color temperature.
+        gl_sc_uniform_vec2(sc, "balance", (float[]){mp_xy_X(prim.white),
+                                                    mp_xy_Z(prim.white)});
+        GLSL(color.xyz = rgb2xyz * color.rgb;)
+        GLSL(color.xz /= balance;)
         pass_tone_map(sc, src.sig_peak / dst_range, algo, tone_mapping_param);
+        GLSL(color.xz *= balance;)
+        GLSL(color.rgb = xyz2rgb * color.xyz;)
+    }
 
     // Adapt to the right colorspace if necessary
     if (src.primaries != dst.primaries) {