vo_opengl: Simplify and clarify color correction code

This commit:

- Changes some of the #define and variable names for clarification and
  adds comments where appropriate.
- Unifies :srgb and :icc-profile, making them fit into the same step of
  the decoding process and removing the weird interactions between both
  of them.
- Makes :icc-profile take precedence over :srgb (to significantly reduce
  the number of confusing and useless special cases)
- Moves BT709 decompanding (approximate or actual) to the shader in all
  cases, making it happen before upscaling (instead of the old 0.45
  gamma function). This is the simpler and more proper way to do it.
- Enables the approx gamma function to work with :srgb as well due to
  this (since they now share the gamma expansion code).
- Renames :icc-approx-gamma to :approx-gamma since it is no longer tied
  to the ICC options or LittleCMS.
- Uses gamma 2.4 as input space for the actual 3DLUT, this is now a
  pretty arbitrary factor but I picked 2.4 mainly because a higher pure
  power value here seems to produce visually better results with wide
  gamut profiles, rather then the previous 1.95 or BT.709.
- Adds the input gamma space to the 3dlut cache header in case we change
  it more in the future, or even make it user customizable (though I
  don't see why the latter would really be necessary).
- Fixes the OSD's gamma when using :srgb, which was previously still
  using the old (0.45) approximation in all cases.
- Updates documentation on :srgb, it was still mentioning the old
  behavior from circa a year ago.

This commit should serve to both open up and make the CMS/shader code much
more accessible and less confusing/error-prone and simultaneously also
improve the performance of 3DLUTs with wide gamut color spaces.

I would liked to have made it more modular but almost all of these
changes are interdependent, save for the documentation updates.

Note: Right now, the "3DLUT takes precedence over SRGB" logic is just
coded into gl_lcms.c's compile_shaders function. Ideally, this should be
done earlier, when parsing the options (by overriding the actual
opts.srgb flag) and output a warning to the user.

Note: I'm not sure how well this works together with real-world
subtitles that may need to be color corrected as well. I'm not sure
whether :approx-gamma needs to apply to subtitles as well. I'll need to
test this on proper files later.

Note: As of now, linear light scaling is still intrinsically tied to
either :srgb or :icc-profile. It would be thinkable to have this as an
extra option, :linear-scaling or similar, that could be used with or
without the two color management options.

5 files changed, 56 insertions, 48 deletions

diff --git a/video/out/gl_lcms.c b/video/out/gl_lcms.c
index 8c30ce3719..08f89b81d9 100644
--- a/video/out/gl_lcms.c
+++ b/video/out/gl_lcms.c
@@ -73,7 +73,6 @@ const struct m_sub_options mp_icc_conf = {
         OPT_STRING("icc-profile", profile, 0),
         OPT_STRING("icc-cache", cache, 0),
         OPT_INT("icc-intent", intent, 0),
-        OPT_FLAG("icc-approx-gamma", approx, 0),
         OPT_STRING_VALIDATE("3dlut-size", size_str, 0, validate_3dlut_size_opt),
         {0}
     },
@@ -130,8 +129,11 @@ struct lut3d *mp_load_icc(struct mp_icc_opts *opts, struct mp_log *log,
         goto error_exit;
 
     char *cache_info =
-        talloc_asprintf(tmp, "intent=%d, size=%dx%dx%d, approx=%d\n",
-                        opts->intent, s_r, s_g, s_b, opts->approx);
+        // Gamma is included in the header to help uniquely identify it,
+        // because we may change the parameter in the future or make it
+        // customizable.
+        talloc_asprintf(tmp, "intent=%d, size=%dx%dx%d, gamma=2.4",
+                        opts->intent, s_r, s_g, s_b);
 
     // check cache
     if (opts->cache) {
@@ -166,22 +168,9 @@ struct lut3d *mp_load_icc(struct mp_icc_opts *opts, struct mp_log *log,
         .Blue  = {0.15, 0.06, 1.0},
     };
 
-    cmsToneCurve *tonecurve;
-    if (opts->approx) {
-        /* Apple's CMS, among other programs that rely on it, uses 1.95 as a
-           faster approximation of this curve. It's not quite correct, but the
-           option is provided for compatibility with such incorrect clips. */
-        tonecurve = cmsBuildGamma(NULL, 1.95);
-    } else {
-        /* Rec BT.709 defines the tone curve as:
-           V = 1.099 * L^0.45 - 0.099 for L >= 0.018
-           V = 4.500 * L              for L <  0.018
-
-           The 0.081 parameter comes from inserting 0.018 into the function */
-        tonecurve = cmsBuildParametricToneCurve(NULL, 4,
-            (cmsFloat64Number[5]){1/0.45, 1/1.099, 0.099/1.099, 1/4.5, 0.081});
-    }
-
+    // 2.4 is arbitrarily used as a gamma compression factor for the 3DLUT,
+    // reducing artifacts due to rounding errors on wide gamut profiles
+    cmsToneCurve *tonecurve = cmsBuildGamma(NULL, 2.4);
     cmsHPROFILE vid_profile = cmsCreateRGBProfile(&d65, &bt709prim,
                         (cmsToneCurve*[3]){tonecurve, tonecurve, tonecurve});
     cmsFreeToneCurve(tonecurve);
diff --git a/video/out/gl_lcms.h b/video/out/gl_lcms.h
index 4a8e0cfb6a..a579b78f43 100644
--- a/video/out/gl_lcms.h
+++ b/video/out/gl_lcms.h
@@ -8,7 +8,6 @@ struct mp_icc_opts {
     char *cache;
     char *size_str;
     int intent;
-    int approx;
 };
 
 struct lut3d;
diff --git a/video/out/gl_video.c b/video/out/gl_video.c
index be418be461..976fa3b486 100644
--- a/video/out/gl_video.c
+++ b/video/out/gl_video.c
@@ -306,6 +306,7 @@ const struct m_sub_options gl_video_conf = {
     .opts = (m_option_t[]) {
         OPT_FLOATRANGE("gamma", gamma, 0, 0.0, 10.0),
         OPT_FLAG("srgb", srgb, 0),
+        OPT_FLAG("approx-gamma", approx_gamma, 0),
         OPT_FLAG("npot", npot, 0),
         OPT_FLAG("pbo", pbo, 0),
         OPT_CHOICE("stereo", stereo_mode, 0,
@@ -835,10 +836,11 @@ static void compile_shaders(struct gl_video *p)
         shader_def_opt(&header, "USE_ALPHA", p->has_alpha);
 
     char *header_osd = talloc_strdup(tmp, header);
-    shader_def_opt(&header_osd, "USE_OSD_LINEAR_CONV", p->opts.srgb &&
-                                                      !p->use_lut_3d);
+    shader_def_opt(&header_osd, "USE_OSD_LINEAR_CONV", p->opts.srgb ||
+                                                       p->use_lut_3d);
     shader_def_opt(&header_osd, "USE_OSD_3DLUT", p->use_lut_3d);
-    shader_def_opt(&header_osd, "USE_OSD_SRGB", p->opts.srgb);
+    // 3DLUT overrides SRGB
+    shader_def_opt(&header_osd, "USE_OSD_SRGB", !p->use_lut_3d && p->opts.srgb);
 
     for (int n = 0; n < SUBBITMAP_COUNT; n++) {
         const char *name = osd_shaders[n];
@@ -861,14 +863,16 @@ static void compile_shaders(struct gl_video *p)
         conv_gamma *= 1.0 / 2.2;
     }
 
-    if (!p->is_linear_rgb && (p->opts.srgb || p->use_lut_3d))
-        conv_gamma *= 1.0 / 0.45;
-
     p->input_gamma = input_gamma;
     p->conv_gamma = conv_gamma;
 
-    bool convert_input_gamma = p->input_gamma != 1.0;
-    bool convert_input_to_linear = p->conv_gamma != 1.0;
+    bool use_input_gamma = p->input_gamma != 1.0;
+    bool use_conv_gamma = p->conv_gamma != 1.0;
+
+    // Linear light scaling is only enabled when either color correction
+    // option (3dlut or srgb) is enabled, otherwise scaling is done in the
+    // source space.
+    bool convert_to_linear_gamma = !p->is_linear_rgb && (p->opts.srgb || p->use_lut_3d);
 
     if (p->image_format == IMGFMT_NV12 || p->image_format == IMGFMT_NV21) {
         shader_def(&header_conv, "USE_CONV", "CONV_NV12");
@@ -881,18 +885,20 @@ static void compile_shaders(struct gl_video *p)
     shader_def_opt(&header_conv, "USE_SWAP_UV", p->image_format == IMGFMT_NV21);
     shader_def_opt(&header_conv, "USE_YGRAY", p->is_yuv && !p->is_packed_yuv
                                               && p->plane_count == 1);
-    shader_def_opt(&header_conv, "USE_INPUT_GAMMA", convert_input_gamma);
+    shader_def_opt(&header_conv, "USE_INPUT_GAMMA", use_input_gamma);
     shader_def_opt(&header_conv, "USE_COLORMATRIX", !p->is_rgb);
-    shader_def_opt(&header_conv, "USE_CONV_GAMMA", convert_input_to_linear);
+    shader_def_opt(&header_conv, "USE_CONV_GAMMA", use_conv_gamma);
+    shader_def_opt(&header_conv, "USE_LINEAR_LIGHT", convert_to_linear_gamma);
+    shader_def_opt(&header_conv, "USE_APPROX_GAMMA", p->opts.approx_gamma);
     if (p->opts.alpha_mode > 0 && p->has_alpha && p->plane_count > 3)
         shader_def(&header_conv, "USE_ALPHA_PLANE", "3");
     if (p->opts.alpha_mode == 2 && p->has_alpha)
         shader_def(&header_conv, "USE_ALPHA_BLEND", "1");
 
-    shader_def_opt(&header_final, "USE_LINEAR_CONV_INV", p->use_lut_3d);
     shader_def_opt(&header_final, "USE_GAMMA_POW", p->opts.gamma > 0);
     shader_def_opt(&header_final, "USE_3DLUT", p->use_lut_3d);
-    shader_def_opt(&header_final, "USE_SRGB", p->opts.srgb);
+    // 3DLUT overrides SRGB
+    shader_def_opt(&header_final, "USE_SRGB", p->opts.srgb && !p->use_lut_3d);
     shader_def_opt(&header_final, "USE_DITHER", p->dither_texture != 0);
     shader_def_opt(&header_final, "USE_TEMPORAL_DITHER", p->opts.temporal_dither);
 
@@ -913,8 +919,8 @@ static void compile_shaders(struct gl_video *p)
     bool use_indirect = p->opts.indirect;
 
     // Don't sample from input video textures before converting the input to
-    // linear light. (Unneeded when sRGB textures are used.)
-    if (convert_input_gamma || convert_input_to_linear)
+    // linear light.
+    if (use_input_gamma || use_conv_gamma)
         use_indirect = true;
 
     // It doesn't make sense to scale the chroma with cscale in the 1. scale
diff --git a/video/out/gl_video.h b/video/out/gl_video.h
index e997e88fb6..989ac3b008 100644
--- a/video/out/gl_video.h
+++ b/video/out/gl_video.h
@@ -33,6 +33,7 @@ struct gl_video_opts {
     int indirect;
     float gamma;
     int srgb;
+    int approx_gamma;
     int scale_sep;
     int fancy_downscaling;
     int scaler_resizes_only;
diff --git a/video/out/gl_video_shaders.glsl b/video/out/gl_video_shaders.glsl
index cd7e051dcd..15bb600e61 100644
--- a/video/out/gl_video_shaders.glsl
+++ b/video/out/gl_video_shaders.glsl
@@ -82,11 +82,15 @@ void main() {
     color = vertex_color;
 
 #ifdef USE_OSD_LINEAR_CONV
-    // If no 3dlut is being used, we need to pull up to linear light for
-    // the sRGB function. *IF* 3dlut is used, we do not.
+    // Although we are not scaling in linear light, both 3DLUT and SRGB still
+    // operate on linear light inputs so we have to convert to it before
+    // either step can be applied.
     color.rgb = bt709_expand(color.rgb);
+    // NOTE: This always applies the true BT709, maybe we need to use
+    // approx-gamma here too?
 #endif
 #ifdef USE_OSD_3DLUT
+    color.rgb = pow(color.rgb, vec3(1/2.4)); // linear -> 2.4 3DLUT space
     color = vec4(texture3D(lut_3d, color.rgb).rgb, color.a);
 #endif
 #ifdef USE_OSD_SRGB
@@ -369,35 +373,44 @@ void main() {
     color.gb = vec2(128.0/255.0);
 #endif
 #ifdef USE_INPUT_GAMMA
+    // Pre-colormatrix input gamma correction (eg. for MP_IMGFLAG_XYZ)
     color = pow(color, vec3(input_gamma));
 #endif
 #ifdef USE_COLORMATRIX
+    // Conversion from Y'CbCr or other spaces to RGB
     color = mat3(colormatrix) * color + colormatrix[3];
     color = clamp(color, 0, 1);
 #endif
 #ifdef USE_CONV_GAMMA
+    // Post-colormatrix converted gamma correction (eg. for MP_IMGFLAG_XYZ)
     color = pow(color, vec3(conv_gamma));
 #endif
-#ifdef USE_LINEAR_CONV_INV
-    // Convert from linear RGB to gamma RGB before putting it through the 3D-LUT
-    // in the final stage.
-    color = pow(color, vec3(0.45));
+#ifdef USE_LINEAR_LIGHT
+    // If we are scaling in linear light (SRGB or 3DLUT option enabled), we
+    // expand our source colors before scaling
+#ifdef USE_APPROX_GAMMA
+    // We differentiate between approximate BT.709 (gamma 1.95) ...
+    color = pow(color, vec3(1.95));
+#else
+    // ... and actual BT709 (two-part function)
+    color = bt709_expand(color);
+#endif
 #endif
+    // Image upscaling happens roughly here
 #ifdef USE_GAMMA_POW
+    // User-defined gamma correction factor (via the gamma sub-option)
     color = pow(color, inv_gamma);
 #endif
 #ifdef USE_3DLUT
+    // For the 3DLUT we are arbitrarily using 2.4 as input gamma to reduce
+    // the amount of rounding errors, so we pull up to that space first and
+    // then pass it through the 3D texture.
+    color = pow(color, vec3(1/2.4));
     color = texture3D(lut_3d, color).rgb;
 #endif
 #ifdef USE_SRGB
-    // Go from "linear" (0.45) to BT.709 to true linear to sRGB
-#ifndef USE_3DLUT
-    // Unless we are using a 3DLUT, in which case USE_LINEAR_CONV_INV
-    // already triggered earlier.
-    color = pow(color, vec3(0.45));
-#endif
-    color = bt709_expand(color.rgb);
-    color.rgb = srgb_compand(color.rgb);
+    // Compand from the linear scaling gamma to the sRGB output gamma
+    color = srgb_compand(color.rgb);
 #endif
 #ifdef USE_DITHER
     vec2 dither_pos = gl_FragCoord.xy / dither_size;