vaapi: fight with Intel's broken video decoding GL interop

Use texture-from-pixmap instead of vaapi's "native" GLX support.
Apparently the latter is unused by other projects. Possibly it's broken
due that, and Intel's inability to provide anything non-broken in
relation to video.

The new code basically uses the X11 output method on a in-memory pixmap,
and maps this pixmap as texture using standard GLX mechanisms. This
requires a lot of X11 and GLX boilerplate, so the code grows. (I don't
know why libva's GLX interop doesn't just do the same under the hood,
instead of bothering the world with their broken/unmaintained "old"
method, whatever it did. I suspect that Intel programmers are just
genuine sadists.)

This change was suggested in issue #1765.

The old GLX support is removed, as it's redundant and broken anyway.

One remaining issue is that the first vaPutSurface() call fails with an
unknown error. It returns -1, which is pretty strange, because vaapi
error codes are normally positive. It happened with the old GLX code
too, but does not happen with vo_vaapi. I couldn't find out why.

2 files changed, 82 insertions, 25 deletions

diff --git a/video/out/gl_hwdec_vaglx.c b/video/out/gl_hwdec_vaglx.c
index c9c0535804..86ec3b5183 100644
--- a/video/out/gl_hwdec_vaglx.c
+++ b/video/out/gl_hwdec_vaglx.c
@@ -18,10 +18,10 @@
  */
 
 #include <stddef.h>
+#include <string.h>
 #include <assert.h>
 
 #include <GL/glx.h>
-#include <va/va_glx.h>
 
 #include "x11_common.h"
 #include "gl_hwdec.h"
@@ -31,23 +31,29 @@ struct priv {
     struct mp_log *log;
     struct mp_vaapi_ctx *ctx;
     VADisplay *display;
+    Display *xdisplay;
     GLuint gl_texture;
-    void *vaglx_surface;
+    GLXFBConfig fbc;
+    Pixmap pixmap;
+    GLXPixmap glxpixmap;
+    void (*glXBindTexImage)(Display *dpy, GLXDrawable draw, int buffer, int *a);
+    void (*glXReleaseTexImage)(Display *dpy, GLXDrawable draw, int buffer);
 };
 
 static void destroy_texture(struct gl_hwdec *hw)
 {
     struct priv *p = hw->priv;
     GL *gl = hw->gl;
-    VAStatus status;
 
-    if (p->vaglx_surface) {
-        va_lock(p->ctx);
-        status = vaDestroySurfaceGLX(p->display, p->vaglx_surface);
-        va_unlock(p->ctx);
-        CHECK_VA_STATUS(p, "vaDestroySurfaceGLX()");
-        p->vaglx_surface = NULL;
+    if (p->glxpixmap) {
+        p->glXReleaseTexImage(p->xdisplay, p->glxpixmap, GLX_FRONT_EXT);
+        glXDestroyPixmap(p->xdisplay, p->glxpixmap);
     }
+    p->glxpixmap = 0;
+
+    if (p->pixmap)
+        XFreePixmap(p->xdisplay, p->pixmap);
+    p->pixmap = 0;
 
     gl->DeleteTextures(1, &p->gl_texture);
     p->gl_texture = 0;
@@ -67,10 +73,21 @@ static int create(struct gl_hwdec *hw)
     Display *x11disp = glXGetCurrentDisplay();
     if (!x11disp)
         return -1;
+    int x11scr = DefaultScreen(x11disp);
     struct priv *p = talloc_zero(hw, struct priv);
     hw->priv = p;
     p->log = hw->log;
-    p->display = vaGetDisplayGLX(x11disp);
+    p->xdisplay = x11disp;
+    const char *glxext = glXQueryExtensionsString(x11disp, x11scr);
+    if (!glxext || !strstr(glxext, "GLX_EXT_texture_from_pixmap"))
+        return -1;
+    p->glXBindTexImage =
+        (void*)glXGetProcAddressARB((void*)"glXBindTexImageEXT");
+    p->glXReleaseTexImage =
+        (void*)glXGetProcAddressARB((void*)"glXReleaseTexImageEXT");
+    if (!p->glXBindTexImage || !p->glXReleaseTexImage)
+        return -1;
+    p->display = vaGetDisplay(x11disp);
     if (!p->display)
         return -1;
     p->ctx = va_initialize(p->display, p->log);
@@ -82,6 +99,32 @@ static int create(struct gl_hwdec *hw)
         destroy(hw);
         return -1;
     }
+
+    int attribs[] = {
+        GLX_BIND_TO_TEXTURE_RGBA_EXT, True,
+        GLX_DRAWABLE_TYPE, GLX_PIXMAP_BIT,
+        GLX_BIND_TO_TEXTURE_TARGETS_EXT, GLX_TEXTURE_2D_BIT_EXT,
+        GLX_Y_INVERTED_EXT, True,
+        GLX_DOUBLEBUFFER, False,
+        GLX_RED_SIZE, 8,
+        GLX_GREEN_SIZE, 8,
+        GLX_BLUE_SIZE, 8,
+        GLX_ALPHA_SIZE, 0,
+        None
+    };
+
+    int fbcount;
+    GLXFBConfig *fbc = glXChooseFBConfig(x11disp, x11scr, attribs, &fbcount);
+    if (fbcount)
+        p->fbc = fbc[0];
+    if (fbc)
+        XFree(fbc);
+    if (!fbcount) {
+        MP_VERBOSE(p, "No texture-from-pixmap support.\n");
+        destroy(hw);
+        return -1;
+    }
+
     hw->hwctx = &p->ctx->hwctx;
     hw->converted_imgfmt = IMGFMT_RGB0;
     return 0;
@@ -91,7 +134,6 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 {
     struct priv *p = hw->priv;
     GL *gl = hw->gl;
-    VAStatus status;
 
     destroy_texture(hw);
 
@@ -103,15 +145,29 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
     gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
     gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
     gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
-    gl->TexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, params->w, params->h, 0,
-                   GL_RGBA, GL_UNSIGNED_BYTE, NULL);
     gl->BindTexture(GL_TEXTURE_2D, 0);
 
-    va_lock(p->ctx);
-    status = vaCreateSurfaceGLX(p->display, GL_TEXTURE_2D,
-                                p->gl_texture, &p->vaglx_surface);
-    va_unlock(p->ctx);
-    return CHECK_VA_STATUS(p, "vaCreateSurfaceGLX()") ? 0 : -1;
+    p->pixmap = XCreatePixmap(p->xdisplay,
+                        RootWindow(p->xdisplay, DefaultScreen(p->xdisplay)),
+                        params->w, params->h, 24);
+    if (!p->pixmap) {
+        MP_FATAL(hw, "could not create pixmap\n");
+        return -1;
+    }
+
+    int attribs[] = {
+        GLX_TEXTURE_TARGET_EXT, GLX_TEXTURE_2D_EXT,
+        GLX_TEXTURE_FORMAT_EXT, GLX_TEXTURE_FORMAT_RGB_EXT,
+        GLX_MIPMAP_TEXTURE_EXT, False,
+        None,
+    };
+    p->glxpixmap = glXCreatePixmap(p->xdisplay, p->fbc, p->pixmap, attribs);
+
+    gl->BindTexture(GL_TEXTURE_2D, p->gl_texture);
+    p->glXBindTexImage(p->xdisplay, p->glxpixmap, GLX_FRONT_EXT, NULL);
+    gl->BindTexture(GL_TEXTURE_2D, 0);
+
+    return 0;
 }
 
 static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
@@ -120,16 +176,17 @@ static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
     struct priv *p = hw->priv;
     VAStatus status;
 
-    if (!p->vaglx_surface)
+    if (!p->pixmap)
         return -1;
 
     va_lock(p->ctx);
-    status = vaCopySurfaceGLX(p->display, p->vaglx_surface,
-                              va_surface_id(hw_image),
-                              va_get_colorspace_flag(hw_image->params.colorspace));
+    status = vaPutSurface(p->display, va_surface_id(hw_image), p->pixmap,
+                          0, 0, hw_image->w, hw_image->h,
+                          0, 0, hw_image->w, hw_image->h,
+                          NULL, 0,
+                          va_get_colorspace_flag(hw_image->params.colorspace));
+    CHECK_VA_STATUS(p, "vaPutSurface()");
     va_unlock(p->ctx);
-    if (!CHECK_VA_STATUS(p, "vaCopySurfaceGLX()"))
-        return -1;
 
     out_textures[0] = p->gl_texture;
     return 0;
diff --git a/wscript b/wscript
index bba7e3996d..4cf92d977d 100644
--- a/wscript
+++ b/wscript
@@ -626,7 +626,7 @@ video_output_features = [
         'name': '--vaapi-glx',
         'desc': 'VAAPI GLX',
         'deps': [ 'vaapi', 'gl-x11' ],
-        'func': check_pkg_config('libva-glx', '>= 0.32.0'),
+        'func': check_true,
     }, {
         'name': '--caca',
         'desc': 'CACA',