vo_opengl: fix alpha values written to the framebuffer

When blending OSD and subtitles onto the video, we write bogus alpha
values. This doesn't normally matter, because these values are normally
unused and discarded. But at least on Wayland, the alpha values are used
by the compositor and leads to transparent windows even with opaque
video on places where the OSD happens to use transparency.

(Also see github issue #338.)

Until now, the alpha basically contained garbage. The source factor
GL_SRC_ALPHA meant that alpha was multiplied with itself. Use GL_ONE
instead (which is why we have to use glBlendFuncSeparate()). This should
give correct results, even with video that has alpha. (Or at least it's
something close to correct, I haven't thought too hard how the
compositor will blend it, and in fact I couldn't manage to test it.)

If glBlendFuncSeparate() is not available, fall back to glBlendFunc(),
which does the same as the code did before this commit. Technically, we
support GL 1.1, but glBlendFuncSeparate is 1.4, and I guess we should
try not to crash if vo_opengl_old runs on a system with GL 1.1 drivers
only.

3 files changed, 16 insertions, 5 deletions

diff --git a/video/out/gl_common.c b/video/out/gl_common.c
index 0477bb39c9..8be109f6e3 100644
--- a/video/out/gl_common.c
+++ b/video/out/gl_common.c
@@ -263,6 +263,8 @@ struct gl_functions gl_functions[] = {
             DEF_FN(UniformMatrix2fv),
             DEF_FN(UniformMatrix3fv),
             DEF_FN(TexImage3D),
+            // Added in OpenGL 1.4, but vo_opengl_old doesn't need it
+            DEF_FN(BlendFuncSeparate),
             {0},
         },
     },
diff --git a/video/out/gl_common.h b/video/out/gl_common.h
index 8d7d5a8252..6339b573b0 100644
--- a/video/out/gl_common.h
+++ b/video/out/gl_common.h
@@ -249,6 +249,7 @@ struct GL {
     void (GLAPIENTRY *DrawBuffer)(GLenum);
     void (GLAPIENTRY *DepthMask)(GLboolean);
     void (GLAPIENTRY *BlendFunc)(GLenum, GLenum);
+    void (GLAPIENTRY *BlendFuncSeparate)(GLenum, GLenum, GLenum, GLenum);
     void (GLAPIENTRY *Flush)(void);
     void (GLAPIENTRY *Finish)(void);
     void (GLAPIENTRY *PixelStorei)(GLenum, GLint);
diff --git a/video/out/gl_osd.c b/video/out/gl_osd.c
index 547a8b6789..99b64176ae 100644
--- a/video/out/gl_osd.c
+++ b/video/out/gl_osd.c
@@ -29,10 +29,12 @@ struct osd_fmt_entry {
     GLenum type;
 };
 
-// glBlendFunc() arguments
-static const int blend_factors[SUBBITMAP_COUNT][2] = {
-    [SUBBITMAP_LIBASS] = {GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA},
-    [SUBBITMAP_RGBA] =   {GL_ONE,       GL_ONE_MINUS_SRC_ALPHA},
+// glBlendFuncSeparate() arguments
+static const int blend_factors[SUBBITMAP_COUNT][4] = {
+    [SUBBITMAP_LIBASS] = {GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA,
+                          GL_ONE,       GL_ONE_MINUS_SRC_ALPHA},
+    [SUBBITMAP_RGBA] =   {GL_ONE,       GL_ONE_MINUS_SRC_ALPHA,
+                          GL_ONE,       GL_ONE_MINUS_SRC_ALPHA},
 };
 
 static const struct osd_fmt_entry osd_to_gl3_formats[SUBBITMAP_COUNT] = {
@@ -230,7 +232,13 @@ void mpgl_osd_set_gl_state(struct mpgl_osd *ctx, struct mpgl_osd_part *p)
 
     gl->BindTexture(GL_TEXTURE_2D, p->texture);
     gl->Enable(GL_BLEND);
-    gl->BlendFunc(blend_factors[p->format][0], blend_factors[p->format][1]);
+
+    const int *factors = &blend_factors[p->format][0];
+    if (gl->BlendFuncSeparate) {
+        gl->BlendFuncSeparate(factors[0], factors[1], factors[2], factors[3]);
+    } else {
+        gl->BlendFunc(factors[0], factors[1]);
+    }
 }
 
 void mpgl_osd_unset_gl_state(struct mpgl_osd *ctx, struct mpgl_osd_part *p)