vaapi: remove dependency on X11

There are at least 2 ways of using VAAPI without X11 (Wayland, DRM).
Remove the X11 requirement from the decoder part and the EGL interop.
This will be used by a following commit, which adds Wayland support.

The worst about this is the decoder part, which includes a bad hack for
using the decoder without any VO interop (also known as "vaapi-copy"
mode). Separate the X11 parts so that they're self-contained. For the
EGL interop code we do something similar (it's kept slightly simpler,
because it essentially only has to translate between our silly
MPGetNativeDisplay abstraction and the vaGetDisplay...() call).

5 files changed, 84 insertions, 21 deletions

diff --git a/video/decode/vaapi.c b/video/decode/vaapi.c
index f88e44661e..3fa0fd4759 100644
--- a/video/decode/vaapi.c
+++ b/video/decode/vaapi.c
@@ -25,7 +25,7 @@
 #include <libavcodec/vaapi.h>
 #include <libavutil/common.h>
 
-#include <X11/Xlib.h>
+#include "config.h"
 
 #include "lavc.h"
 #include "common/common.h"
@@ -53,7 +53,9 @@ struct priv {
     struct mp_log *log;
     struct mp_vaapi_ctx *ctx;
     VADisplay display;
-    Display *x11_display;
+
+    const struct va_native_display *native_display_fns;
+    void *native_display;
 
     // libavcodec shared struct
     struct vaapi_context *va_context;
@@ -65,6 +67,47 @@ struct priv {
     struct mp_image_pool *sw_pool;
 };
 
+struct va_native_display {
+    void (*create)(struct priv *p);
+    void (*destroy)(struct priv *p);
+};
+
+static const struct va_native_display disp_x11;
+
+static const struct va_native_display *const native_displays[] = {
+#if HAVE_VAAPI_X11
+    &disp_x11,
+#endif
+    NULL
+};
+
+#if HAVE_VAAPI_X11
+#include <X11/Xlib.h>
+#include <va/va_x11.h>
+
+static void x11_destroy(struct priv *p)
+{
+    if (p->native_display)
+        XCloseDisplay(p->native_display);
+    p->native_display = NULL;
+}
+
+static void x11_create(struct priv *p)
+{
+    p->native_display = XOpenDisplay(NULL);
+    if (!p->native_display)
+        return;
+    p->display = vaGetDisplay(p->native_display);
+    if (!p->display)
+        x11_destroy(p);
+}
+
+static const struct va_native_display disp_x11 = {
+    .create = x11_create,
+    .destroy = x11_destroy,
+};
+#endif
+
 #define HAS_HEVC VA_CHECK_VERSION(0, 38, 0)
 
 #define PE(av_codec_id, ff_profile, vdp_profile)                \
@@ -295,9 +338,8 @@ static void destroy_va_dummy_ctx(struct priv *p)
     va_destroy(p->ctx);
     p->ctx = NULL;
     p->display = NULL;
-    if (p->x11_display)
-        XCloseDisplay(p->x11_display);
-    p->x11_display = NULL;
+    if (p->native_display_fns)
+        p->native_display_fns->destroy(p);
 }
 
 // Creates a "private" VADisplay, disconnected from the VO. We just create a
@@ -305,15 +347,18 @@ static void destroy_va_dummy_ctx(struct priv *p)
 // connection along with struct mp_hwdec_info, if we wanted.)
 static bool create_va_dummy_ctx(struct priv *p)
 {
-    p->x11_display = XOpenDisplay(NULL);
-    if (!p->x11_display)
-        goto destroy_ctx;
-    VADisplay *display = vaGetDisplay(p->x11_display);
-    if (!display)
+    for (int n = 0; native_displays[n]; n++) {
+        native_displays[n]->create(p);
+        if (p->display) {
+            p->native_display_fns = native_displays[n];
+            break;
+        }
+    }
+    if (!p->display)
         goto destroy_ctx;
-    p->ctx = va_initialize(display, p->log, true);
+    p->ctx = va_initialize(p->display, p->log, true);
     if (!p->ctx) {
-        vaTerminate(display);
+        vaTerminate(p->display);
         goto destroy_ctx;
     }
     return true;
@@ -334,7 +379,7 @@ static void uninit(struct lavc_ctx *ctx)
     talloc_free(p->pool);
     p->pool = NULL;
 
-    if (p->x11_display)
+    if (p->native_display_fns)
         destroy_va_dummy_ctx(p);
 
     talloc_free(p);
diff --git a/video/out/opengl/hwdec.c b/video/out/opengl/hwdec.c
index d93dc99397..7665d0c056 100644
--- a/video/out/opengl/hwdec.c
+++ b/video/out/opengl/hwdec.c
@@ -37,7 +37,7 @@ extern const struct gl_hwdec_driver gl_hwdec_vdpau;
 extern const struct gl_hwdec_driver gl_hwdec_dxva2;
 
 static const struct gl_hwdec_driver *const mpgl_hwdec_drivers[] = {
-#if HAVE_VAAPI_X_EGL
+#if HAVE_VAAPI_EGL
     &gl_hwdec_vaegl,
 #endif
 #if HAVE_VAAPI_GLX
diff --git a/video/out/opengl/hwdec_vaegl.c b/video/out/opengl/hwdec_vaegl.c
index b3d5cabecf..90c7478d94 100644
--- a/video/out/opengl/hwdec_vaegl.c
+++ b/video/out/opengl/hwdec_vaegl.c
@@ -26,7 +26,6 @@
 
 #include <va/va_drmcommon.h>
 
-#include "video/out/x11_common.h"
 #include "hwdec.h"
 #include "video/vaapi.h"
 #include "video/img_fourcc.h"
@@ -47,6 +46,29 @@ typedef void *EGLImageKHR;
 #define EGL_DMA_BUF_PLANE0_PITCH_EXT      0x3274
 #endif
 
+#if HAVE_VAAPI_X11
+#include <va/va_x11.h>
+
+static VADisplay *create_x11_va_display(GL *gl)
+{
+    Display *x11 = gl->MPGetNativeDisplay("x11");
+    return x11 ? vaGetDisplay(x11) : NULL;
+}
+#endif
+
+static VADisplay *create_native_va_display(GL *gl)
+{
+    if (!gl->MPGetNativeDisplay)
+        return NULL;
+    VADisplay *display = NULL;
+#if HAVE_VAAPI_X11
+    display = create_x11_va_display(gl);
+    if (display)
+        return display;
+#endif
+    return display;
+}
+
 struct priv {
     struct mp_log *log;
     struct mp_vaapi_ctx *ctx;
@@ -152,10 +174,6 @@ static int create(struct gl_hwdec *hw)
     if (!eglGetCurrentDisplay())
         return -1;
 
-    p->xdisplay =
-        hw->gl->MPGetNativeDisplay ? hw->gl->MPGetNativeDisplay("x11") : NULL;
-    if (!p->xdisplay)
-        return -1;
     if (!strstr(gl->extensions, "EXT_image_dma_buf_import") ||
         !strstr(gl->extensions, "EGL_KHR_image_base") ||
         !strstr(gl->extensions, "GL_OES_EGL_image") ||
@@ -173,7 +191,7 @@ static int create(struct gl_hwdec *hw)
         !p->EGLImageTargetTexture2DOES)
         return -1;
 
-    p->display = vaGetDisplay(p->xdisplay);
+    p->display = create_native_va_display(gl);
     if (!p->display)
         return -1;
 
diff --git a/video/out/opengl/hwdec_vaglx.c b/video/out/opengl/hwdec_vaglx.c
index 6ad269ae3d..34e8ee937e 100644
--- a/video/out/opengl/hwdec_vaglx.c
+++ b/video/out/opengl/hwdec_vaglx.c
@@ -22,6 +22,7 @@
 #include <assert.h>
 
 #include <GL/glx.h>
+#include <va/va_x11.h>
 
 #include "video/out/x11_common.h"
 #include "hwdec.h"
diff --git a/video/vaapi.h b/video/vaapi.h
index 5f27e782b4..3f4a02b2bb 100644
--- a/video/vaapi.h
+++ b/video/vaapi.h
@@ -22,7 +22,6 @@
 #include <inttypes.h>
 #include <pthread.h>
 #include <va/va.h>
-#include <va/va_x11.h>
 
 #ifndef VA_FOURCC_I420
 #define VA_FOURCC_I420 VA_FOURCC('I', '4', '2', '0')