vo_opengl: add support for VA-API OpenGL interop

VA-API's OpenGL/GLX interop is pretty bad and perhaps slow (renders a
X11 pixmap into a FBO, and has to go over X11, probably involves one or
more copies), and this code serves more as an example, rather than for
serious use. On the other hand, this might be work much better than
vo_vaapi, even if slightly slower.

4 files changed, 147 insertions, 0 deletions

diff --git a/video/decode/vaapi.c b/video/decode/vaapi.c
index bb2a6c1049..4603a3c5e1 100644
--- a/video/decode/vaapi.c
+++ b/video/decode/vaapi.c
@@ -421,6 +421,7 @@ static int init(struct lavc_ctx *ctx)
 static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
                  const char *decoder)
 {
+    hwdec_request_api(info, "vaapi");
     if (!info || !info->vaapi_ctx)
         return HWDEC_ERR_NO_CTX;
     if (!hwdec_check_codec_support(decoder, profiles))
diff --git a/video/out/gl_common.c b/video/out/gl_common.c
index 9482f9de55..3216b0037e 100644
--- a/video/out/gl_common.c
+++ b/video/out/gl_common.c
@@ -1009,6 +1009,11 @@ void mp_log_source(struct mp_log *log, int lev, const char *src)
     }
 }
 
+extern const struct gl_hwdec_driver gl_hwdec_vaglx;
+
 const struct gl_hwdec_driver *mpgl_hwdec_drivers[] = {
+#if CONFIG_VAAPI_GLX
+    &gl_hwdec_vaglx,
+#endif
     NULL
 };
diff --git a/video/out/gl_common.h b/video/out/gl_common.h
index d8d07de06a..8d318a7c1f 100644
--- a/video/out/gl_common.h
+++ b/video/out/gl_common.h
@@ -185,6 +185,7 @@ struct gl_hwdec_driver {
     // Test whether the given IMGFMT_ is supported.
     bool (*query_format)(int imgfmt);
     // Create the hwdec device. It must fill in hw->info, if applicable.
+    // This also must set hw->converted_imgfmt.
     int (*create)(struct gl_hwdec *hw);
     // Prepare for rendering video. (E.g. create textures.)
     // Called on initialization, and every time the video size changes.
diff --git a/video/out/gl_hwdec_vaglx.c b/video/out/gl_hwdec_vaglx.c
new file mode 100644
index 0000000000..9e1030fe49
--- /dev/null
+++ b/video/out/gl_hwdec_vaglx.c
@@ -0,0 +1,140 @@
+/*
+ * This file is part of mpv.
+ *
+ * Parts based on the MPlayer VA-API patch (see vo_vaapi.c).
+ *
+ * mpv is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stddef.h>
+#include <assert.h>
+
+#include <GL/glx.h>
+#include <va/va_glx.h>
+
+#include "x11_common.h"
+#include "gl_common.h"
+#include "video/vaapi.h"
+#include "video/decode/dec_video.h"
+
+struct priv {
+    struct mp_vaapi_ctx *ctx;
+    VADisplay *display;
+    GLuint gl_texture;
+    void *vaglx_surface;
+};
+
+static bool query_format(int imgfmt)
+{
+    return imgfmt == IMGFMT_VAAPI;
+}
+
+static void destroy_texture(struct gl_hwdec *hw)
+{
+    struct priv *p = hw->priv;
+    VAStatus status;
+
+    if (p->vaglx_surface) {
+        status = vaDestroySurfaceGLX(p->display, p->vaglx_surface);
+        check_va_status(status, "vaDestroySurfaceGLX()");
+        p->vaglx_surface = NULL;
+    }
+
+    glDeleteTextures(1, &p->gl_texture);
+    p->gl_texture = 0;
+}
+
+static void destroy(struct gl_hwdec *hw)
+{
+    struct priv *p = hw->priv;
+    destroy_texture(hw);
+    va_destroy(p->ctx);
+}
+
+static int create(struct gl_hwdec *hw)
+{
+    if (hw->info->vaapi_ctx)
+        return -1;
+    if (!hw->mpgl->vo->x11 || !glXGetCurrentContext())
+        return -1;
+    struct priv *p = talloc_zero(hw, struct priv);
+    hw->priv = p;
+    p->display = vaGetDisplayGLX(hw->mpgl->vo->x11->display);
+    if (!p->display)
+        return -1;
+    p->ctx = va_initialize(p->display);
+    if (!p->ctx) {
+        vaTerminate(p->display);
+        return -1;
+    }
+    hw->info->vaapi_ctx = p->ctx;
+    hw->converted_imgfmt = IMGFMT_RGBA;
+    return 0;
+}
+
+static int reinit(struct gl_hwdec *hw, int w, int h)
+{
+    struct priv *p = hw->priv;
+    GL *gl = hw->mpgl->gl;
+    VAStatus status;
+
+    destroy_texture(hw);
+
+    gl->GenTextures(1, &p->gl_texture);
+    gl->BindTexture(GL_TEXTURE_2D, p->gl_texture);
+    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    gl->TexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, w, h, 0,
+                   GL_RGBA, GL_UNSIGNED_BYTE, NULL);
+    gl->BindTexture(GL_TEXTURE_2D, 0);
+
+    status = vaCreateSurfaceGLX(p->display, GL_TEXTURE_2D,
+                                p->gl_texture, &p->vaglx_surface);
+    return check_va_status(status, "vaCreateSurfaceGLX()") ? 0 : -1;
+}
+
+static int load_image(struct gl_hwdec *hw, struct mp_image *hw_image,
+                      GLuint *out_textures)
+{
+    struct priv *p = hw->priv;
+    VAStatus status;
+
+    if (!p->vaglx_surface)
+        return -1;
+
+    status = vaCopySurfaceGLX(p->display, p->vaglx_surface,
+                              va_surface_id_in_mp_image(hw_image),
+                              va_get_colorspace_flag(hw_image->colorspace));
+    if (!check_va_status(status, "vaCopySurfaceGLX()"))
+        return -1;
+
+    out_textures[0] = p->gl_texture;
+    return 0;
+}
+
+static void unload_image(struct gl_hwdec *hw)
+{
+}
+
+const struct gl_hwdec_driver gl_hwdec_vaglx = {
+    .api_name = "vaapi",
+    .query_format = query_format,
+    .create = create,
+    .reinit = reinit,
+    .load_image = load_image,
+    .unload_image = unload_image,
+    .destroy = destroy,
+};