vo_opengl: fix srgb for subtitles

Based on a patch by nand. This is needed, because sRGB mode changes the
video over-all gamma. This has to be done for subtitles as well.
The final srgb_compand() call in the OSD shader compensates for the fact
that in ed8fad729d04 framebuffer use was replaced with doing sRGB
conversion manually by srgb_compand().

This only affects subtitles rendered with libass. Nothing is changed for
RGB subs.

Also change the USE_ flags for OSD shaders to USE_OSD_ to make the
difference between video and OSD rendering more apparent.

2 files changed, 21 insertions, 8 deletions

diff --git a/libvo/vo_opengl.c b/libvo/vo_opengl.c
index 4000f55c56..0390955b5b 100644
--- a/libvo/vo_opengl.c
+++ b/libvo/vo_opengl.c
@@ -671,7 +671,10 @@ static void compile_shaders(struct gl_priv *p)
                                    shader_prelude);
 
     char *header_osd = talloc_strdup(tmp, header);
-    shader_def_opt(&header_osd, "USE_3DLUT", p->use_lut_3d);
+    shader_def_opt(&header_osd, "USE_OSD_LINEAR_CONV", p->use_srgb &&
+                                                      !p->use_lut_3d);
+    shader_def_opt(&header_osd, "USE_OSD_3DLUT", p->use_lut_3d);
+    shader_def_opt(&header_osd, "USE_OSD_SRGB", p->use_srgb);
 
     for (int n = 0; n < SUBBITMAP_COUNT; n++) {
         const char *name = osd_shaders[n];
diff --git a/libvo/vo_opengl_shaders.glsl b/libvo/vo_opengl_shaders.glsl
index 32804b99ff..5dd67bbd52 100644
--- a/libvo/vo_opengl_shaders.glsl
+++ b/libvo/vo_opengl_shaders.glsl
@@ -37,6 +37,12 @@
 # define in varying
 #endif
 
+vec3 srgb_compand(vec3 v)
+{
+    return mix(1.055 * pow(v, vec3(1.0/2.4)) - vec3(0.055), v * 12.92,
+               lessThanEqual(v, vec3(0.0031308)));
+}
+
 #!section vertex_all
 
 #if __VERSION__ < 130
@@ -61,9 +67,19 @@ void main() {
 #endif
     gl_Position = vec4(position, 1);
     color = vertex_color;
-#ifdef USE_3DLUT
+
+#ifdef USE_OSD_LINEAR_CONV
+    // If no 3dlut is being used, we need to pull up to linear light for
+    // the sRGB function. *IF* 3dlut is used, we do not.
+    color.rgb = pow(color.rgb, vec3(2.2));
+#endif
+#ifdef USE_OSD_3DLUT
     color = vec4(texture3D(lut_3d, color.rgb).rgb, color.a);
 #endif
+#ifdef USE_OSD_SRGB
+    color.rgb = srgb_compand(color.rgb);
+#endif
+
     texcoord = vertex_texcoord;
 }
 
@@ -292,12 +308,6 @@ vec4 sample_sharpen5(sampler2D tex, vec2 texsize, vec2 texcoord) {
     return p + t * filter_param1;
 }
 
-vec3 srgb_compand(vec3 v)
-{
-    return mix(1.055 * pow(v, vec3(1.0/2.4)) - vec3(0.055), v * 12.92,
-               lessThanEqual(v, vec3(0.0031308)));
-}
-
 void main() {
 #ifdef USE_PLANAR
     vec3 color = vec3(SAMPLE_L(textures[0], textures_size[0], texcoord).r,