vo_opengl: add alpha output

Allows playing video with alpha information on X11, as long as the video
contains alpha and the window manager does compositing. See vo.rst.

Whether a window can be transparent is decided by the choice of the X
Visual used for window creation. Unfortunately, there's no direct way to
request such a Visual through the GLX or the X API, and use of the
XRender extension is required to find out whether a Visual implies a
framebuffer with alpha used by XRender (see for example [1]). Instead of
depending on the XRender wrapper library (which would require annoying
configure checks, even though XRender is virtually always supported),
use a simple heuristics to find out whether a Visual has alpha. Since
getting it wrong just means an optional feature will not work as
expected, we consider this ok.

[1] http://stackoverflow.com/questions/4052940/how-to-make-an-opengl-
rendering-context-with-transparent-background/9215724#9215724

1 files changed, 16 insertions, 4 deletions

diff --git a/video/out/gl_video_shaders.glsl b/video/out/gl_video_shaders.glsl
index 51f18de377..ee7baa7f6c 100644
--- a/video/out/gl_video_shaders.glsl
+++ b/video/out/gl_video_shaders.glsl
@@ -112,8 +112,8 @@ void main() {
 }
 
 #!section frag_video
-uniform sampler2D textures[3];
-uniform vec2 textures_size[3];
+uniform sampler2D textures[4];
+uniform vec2 textures_size[4];
 uniform sampler1D lut_c_1d;
 uniform sampler1D lut_l_1d;
 uniform sampler2D lut_c_2d;
@@ -323,11 +323,19 @@ void main() {
     vec3 color = vec3(SAMPLE_L(textures[0], textures_size[0], texcoord).r,
                       SAMPLE_C(textures[1], textures_size[1], texcoord).r,
                       SAMPLE_C(textures[2], textures_size[2], texcoord).r);
+    float alpha = 1.0;
 #elif USE_CONV == CONV_NV12
     vec3 color = vec3(SAMPLE_L(textures[0], textures_size[0], texcoord).r,
                       SAMPLE_C(textures[1], textures_size[1], texcoord).rg);
+    float alpha = 1.0;
 #else
-    vec3 color = SAMPLE_L(textures[0], textures_size[0], texcoord).rgb;
+    vec4 acolor = SAMPLE_L(textures[0], textures_size[0], texcoord);
+    vec3 color = acolor.rgb;
+    float alpha = acolor.a;
+#endif
+#ifdef USE_ALPHA_PLANE
+    alpha = SAMPLE_L(textures[USE_ALPHA_PLANE],
+                     textures_size[USE_ALPHA_PLANE], texcoord).r;
 #endif
 #ifdef USE_GBRP
     color.gbr = color;
@@ -364,5 +372,9 @@ void main() {
     float dither_value = texture(dither, gl_FragCoord.xy / dither_size).r;
     color = floor(color * dither_multiply + dither_value ) / dither_quantization;
 #endif
-    out_color = vec4(color, 1);
+#ifdef USE_ALPHA
+    out_color = vec4(color, alpha);
+#else
+    out_color = vec4(color, 1.0);
+#endif
 }