vo_opengl: get rid of approx-gamma and make it the default as per BT.1886

After finding out more about how video mastering is done in the real
world it dawned upon me why the "hack" we figured out in #534 looks so
much better.

Since mastering studios have historically been using only CRTs, the
practice adopted for backwards compatibility was to simulate CRT
responses even on modern digital monitors, a practice so ubiquitous that
the ITU-R formalized it in R-Rec BT.1886 to be precisely gamma 2.40.

As such, we finally have enough proof to get rid of the option
altogether and just always do that.

The value 1.961 is a rounded version of my experimentally obtained
approximation of the BT.709 curve, which resulted in a value of around
1.9610336. This is the closest average match to the source brightness
while preserving the nonlinear response of the BT.1886 ideal monitor.

For playback in dark environments, it's expected that the gamma shift
should be reproduced by a user controlled setting, up to a maximum of
1.224 (2.4/1.961) for a pitch black environment.

More information:
https://developer.apple.com/library/mac/technotes/tn2257/_index.html

4 files changed, 39 insertions, 66 deletions

diff --git a/DOCS/man/vo.rst b/DOCS/man/vo.rst
index 92b88dc523..bbca84ecdb 100644
--- a/DOCS/man/vo.rst
+++ b/DOCS/man/vo.rst
@@ -568,16 +568,6 @@ Available video output drivers are:
         3
             absolute colorimetric
 
-    ``approx-gamma``
-        Approximate the actual gamma function as a pure power curve of
-        1.95. A number of video editing programs and studios apparently use this
-        for mastering instead of the true curve. Most notably, anything in the
-        Apple ecosystem uses this approximation - including all programs
-        compatible with it. It's a sound idea to try enabling this flag first
-        when watching videos and shows to see if things look better that way.
-
-        This only affects the output when using either ``icc-profile`` or ``srgb``.
-
     ``3dlut-size=<r>x<g>x<b>``
         Size of the 3D LUT generated from the ICC profile in each dimension.
         Default is 128x256x64.
diff --git a/video/csputils.h b/video/csputils.h
index 8e8d18e7df..3f61721b12 100644
--- a/video/csputils.h
+++ b/video/csputils.h
@@ -69,8 +69,7 @@ enum mp_csp_prim {
 
 enum mp_csp_trc {
     MP_CSP_TRC_NONE,
-    MP_CSP_TRC_BT_2020_APPROX,
-    MP_CSP_TRC_BT_2020_EXACT,
+    MP_CSP_TRC_BT_1886,
     MP_CSP_TRC_SRGB
 };
 
diff --git a/video/out/gl_video.c b/video/out/gl_video.c
index 36a34449c9..c1673d5439 100644
--- a/video/out/gl_video.c
+++ b/video/out/gl_video.c
@@ -357,7 +357,6 @@ const struct m_sub_options gl_video_conf = {
     .opts = (const m_option_t[]) {
         OPT_FLOATRANGE("gamma", gamma, 0, 0.0, 10.0),
         OPT_FLAG("srgb", srgb, 0),
-        OPT_FLAG("approx-gamma", approx_gamma, 0),
         OPT_FLAG("npot", npot, 0),
         OPT_FLAG("pbo", pbo, 0),
         OPT_STRING_VALIDATE("lscale", scalers[0], 0, validate_scaler_opt),
@@ -406,6 +405,8 @@ const struct m_sub_options gl_video_conf = {
                     {"blend", 2})),
         OPT_FLAG("rectangle-textures", use_rectangle, 0),
         OPT_COLOR("background", background, 0),
+
+        OPT_REMOVED("approx-gamma", "this is always enabled now"),
         {0}
     },
     .size = sizeof(struct gl_video_opts),
@@ -1025,11 +1026,12 @@ static void compile_shaders(struct gl_video *p)
         input_gamma *= 2.6;
 
         // If we're using cms, we can treat it as proper linear input,
-        // otherwise we just scale back to 1.95 as a reasonable approximation.
+        // otherwise we just scale back to 2.40 to match typical displays,
+        // as a reasonable approximation.
         if (use_cms) {
             p->is_linear_rgb = true;
         } else {
-            conv_gamma *= 1.0 / 1.95;
+            conv_gamma *= 1.0 / 2.40;
         }
     }
 
@@ -1048,25 +1050,16 @@ static void compile_shaders(struct gl_video *p)
     if (!p->is_linear_rgb && use_cms) {
         // We just use the color level range to distinguish between PC
         // content like images, which are most likely sRGB, and TV content
-        // like movies, which are most likely BT.2020
+        // like movies, which are most likely BT.1886
         if (p->image_params.colorlevels == MP_CSP_LEVELS_PC && !p->hwdec_active) {
             // FIXME: I don't know if hwdec sets the color levels to PC or not,
             // but let's avoid the bug just in case.
             gamma_fun = MP_CSP_TRC_SRGB;
-        } else if (p->opts.approx_gamma) {
-            gamma_fun = MP_CSP_TRC_BT_2020_APPROX;
         } else {
-            gamma_fun = MP_CSP_TRC_BT_2020_EXACT;
+            gamma_fun = MP_CSP_TRC_BT_1886;
         }
     }
 
-    // We also need to linearize for the constant luminance system. This
-    // transformation really makes no sense with anything other than the
-    // official gamma curves, though. This overrides approx-gamma.
-    if (use_const_luma) {
-        gamma_fun = MP_CSP_TRC_BT_2020_EXACT;
-    }
-
     bool use_linear_light = gamma_fun != MP_CSP_TRC_NONE || p->is_linear_rgb;
 
     // Optionally transform to sigmoidal color space if requested, but only
@@ -1109,10 +1102,8 @@ static void compile_shaders(struct gl_video *p)
         shader_def_opt(&header, "USE_ALPHA", p->has_alpha);
 
     char *header_osd = talloc_strdup(tmp, header);
-    shader_def_opt(&header_osd, "USE_OSD_LINEAR_CONV_APPROX",
-                   use_cms && gamma_fun == MP_CSP_TRC_BT_2020_APPROX);
-    shader_def_opt(&header_osd, "USE_OSD_LINEAR_CONV_BT2020",
-                   use_cms && gamma_fun == MP_CSP_TRC_BT_2020_EXACT);
+    shader_def_opt(&header_osd, "USE_OSD_LINEAR_CONV_BT1886",
+                   use_cms && gamma_fun == MP_CSP_TRC_BT_1886);
     shader_def_opt(&header_osd, "USE_OSD_LINEAR_CONV_SRGB",
                    use_cms && gamma_fun == MP_CSP_TRC_SRGB);
     shader_def_opt(&header_osd, "USE_OSD_CMS_MATRIX", use_cms_matrix);
@@ -1147,10 +1138,8 @@ static void compile_shaders(struct gl_video *p)
     shader_def_opt(&header_conv, "USE_COLORMATRIX", !p->is_rgb);
     shader_def_opt(&header_conv, "USE_CONV_GAMMA", use_conv_gamma);
     shader_def_opt(&header_conv, "USE_CONST_LUMA", use_const_luma);
-    shader_def_opt(&header_conv, "USE_LINEAR_LIGHT_APPROX",
-                   gamma_fun == MP_CSP_TRC_BT_2020_APPROX);
-    shader_def_opt(&header_conv, "USE_LINEAR_LIGHT_BT2020",
-                   gamma_fun == MP_CSP_TRC_BT_2020_EXACT);
+    shader_def_opt(&header_conv, "USE_LINEAR_LIGHT_BT1886",
+                   gamma_fun == MP_CSP_TRC_BT_1886);
     shader_def_opt(&header_conv, "USE_LINEAR_LIGHT_SRGB",
                    gamma_fun == MP_CSP_TRC_SRGB);
     shader_def_opt(&header_conv, "USE_SIGMOID", use_sigmoid);
@@ -1165,7 +1154,6 @@ static void compile_shaders(struct gl_video *p)
     shader_def_opt(&header_final, "USE_3DLUT", p->use_lut_3d);
     // 3DLUT overrides SRGB
     shader_def_opt(&header_final, "USE_SRGB", p->opts.srgb && !p->use_lut_3d);
-    shader_def_opt(&header_final, "USE_CONST_LUMA_INV", use_const_luma && !use_cms);
     shader_def_opt(&header_final, "USE_DITHER", p->dither_texture != 0);
     shader_def_opt(&header_final, "USE_TEMPORAL_DITHER", p->opts.temporal_dither);
 
@@ -1186,8 +1174,8 @@ static void compile_shaders(struct gl_video *p)
     bool use_indirect = p->opts.indirect;
 
     // Don't sample from input video textures before converting the input to
-    // linear light.
-    if (use_input_gamma || use_conv_gamma || use_linear_light)
+    // its proper gamma.
+    if (use_input_gamma || use_conv_gamma || use_linear_light || use_const_luma)
         use_indirect = true;
 
     // It doesn't make sense to scale the chroma with cscale in the 1. scale
diff --git a/video/out/gl_video_shaders.glsl b/video/out/gl_video_shaders.glsl
index dac19cb673..3fd3f9ab75 100644
--- a/video/out/gl_video_shaders.glsl
+++ b/video/out/gl_video_shaders.glsl
@@ -110,11 +110,8 @@ void main() {
     // Although we are not scaling in linear light, both 3DLUT and SRGB still
     // operate on linear light inputs so we have to convert to it before
     // either step can be applied.
-#ifdef USE_OSD_LINEAR_CONV_APPROX
-    color.rgb = pow(color.rgb, vec3(1.95));
-#endif
-#ifdef USE_OSD_LINEAR_CONV_BT2020
-    color.rgb = bt2020_expand(color.rgb);
+#ifdef USE_OSD_LINEAR_CONV_BT1886
+    color.rgb = pow(color.rgb, vec3(1.961));
 #endif
 #ifdef USE_OSD_LINEAR_CONV_SRGB
     color.rgb = srgb_expand(color.rgb);
@@ -416,6 +413,17 @@ void main() {
     // contributions from the three different channels.
     color.br = color.br * mix(vec2(1.5816, 0.9936), vec2(1.9404, 1.7184),
                               lessThanEqual(color.br, vec2(0))) + color.gg;
+
+    // Expand channels to camera-linear light. This shader currently just
+    // assumes everything uses the BT.2020 12-bit gamma function, since the
+    // difference between 10 and 12-bit is negligible for anything other than
+    // 12-bit content.
+    color = bt2020_expand(color);
+    // Calculate the green channel from the expanded RYcB
+    // The BT.2020 specification says Yc = 0.2627*R + 0.6780*G + 0.0593*B
+    color.g = (color.g - 0.2627*color.r - 0.0593*color.b)/0.6780;
+    // Re-compand to receive the R'G'B' result, same as other systems
+    color = bt2020_compand(color);
 #endif
 #ifdef USE_COLORMATRIX
     // CONST_LUMA involves numbers outside the [0,1] range so we make sure
@@ -424,19 +432,18 @@ void main() {
     color = clamp(color, 0.0, 1.0);
 #endif
     // If we are scaling in linear light (SRGB or 3DLUT option enabled), we
-    // expand our source colors before scaling. This shader currently just
-    // assumes everything uses the BT.2020 12-bit gamma function, since the
-    // difference between this and BT.601, BT.709 and BT.2020 10-bit is well
-    // below the rounding error threshold for both 8-bit and even 10-bit
-    // content. It only makes a difference for 12-bit sources, so it should be
-    // fine to use here.
-#ifdef USE_LINEAR_LIGHT_APPROX
-    // We differentiate between approximate BT.2020 (gamma 1.95) ...
-    color = pow(color, vec3(1.95));
-#endif
-#ifdef USE_LINEAR_LIGHT_BT2020
-    // ... and actual BT.2020 (two-part function)
-    color = bt2020_expand(color);
+    // expand our source colors before scaling. We distinguish between
+    // BT.1886 (typical video files) and sRGB (typical image files).
+#ifdef USE_LINEAR_LIGHT_BT1886
+    // This calculation is derived from the BT.1886 recommendation which
+    // is itself derived from the curves of typical CRT monitors. It claims
+    // that a correct video playback environment should have a pure power
+    // curve transfer function (in contrast to the complex BT.709 function)
+    // with a gamma value of 2.40, but this includes the typical gamma boost
+    // of ~1.2 for dark viewing environments. The figure used here instead
+    // (1.961) is therefore a pure power curve but without the boost, which
+    // is a very close approximation of the true BT.709 function.
+    color = pow(color, vec3(1.961));
 #endif
 #ifdef USE_LINEAR_LIGHT_SRGB
     // This is not needed for most sRGB content since we can use GL_SRGB to
@@ -445,11 +452,6 @@ void main() {
     // to convert to linear light manually.
     color = srgb_expand(color);
 #endif
-#ifdef USE_CONST_LUMA
-    // Calculate the green channel from the expanded RYcB
-    // The BT.2020 specification says Yc = 0.2627*R + 0.6780*G + 0.0593*B
-    color.g = (color.g - 0.2627*color.r - 0.0593*color.b)/0.6780;
-#endif
 #ifdef USE_SIGMOID
     color = sig_center - log(1.0/(color * sig_scale + sig_offset) - 1.0)/sig_slope;
 #endif
@@ -485,12 +487,6 @@ void main() {
     // Adapt and compand from the linear BT2020 source to the sRGB output
     color = srgb_compand(color);
 #endif
-    // If none of these options took care of companding again (ie. CMS is
-    // disabled), we still need to re-compand const luma signals, because
-    // they always come out as linear light (and we can't simply output that).
-#ifdef USE_CONST_LUMA_INV
-    color = bt2020_compand(color);
-#endif
 #ifdef USE_DITHER
     vec2 dither_pos = gl_FragCoord.xy / dither_size;
 #ifdef USE_TEMPORAL_DITHER