vaapi: remove dependency on X11

There are at least 2 ways of using VAAPI without X11 (Wayland, DRM).
Remove the X11 requirement from the decoder part and the EGL interop.
This will be used by a following commit, which adds Wayland support.

The worst about this is the decoder part, which includes a bad hack for
using the decoder without any VO interop (also known as "vaapi-copy"
mode). Separate the X11 parts so that they're self-contained. For the
EGL interop code we do something similar (it's kept slightly simpler,
because it essentially only has to translate between our silly
MPGetNativeDisplay abstraction and the vaGetDisplay...() call).

1 files changed, 24 insertions, 6 deletions

diff --git a/video/out/opengl/hwdec_vaegl.c b/video/out/opengl/hwdec_vaegl.c
index b3d5cabecf..90c7478d94 100644
--- a/video/out/opengl/hwdec_vaegl.c
+++ b/video/out/opengl/hwdec_vaegl.c
@@ -26,7 +26,6 @@
 
 #include <va/va_drmcommon.h>
 
-#include "video/out/x11_common.h"
 #include "hwdec.h"
 #include "video/vaapi.h"
 #include "video/img_fourcc.h"
@@ -47,6 +46,29 @@ typedef void *EGLImageKHR;
 #define EGL_DMA_BUF_PLANE0_PITCH_EXT      0x3274
 #endif
 
+#if HAVE_VAAPI_X11
+#include <va/va_x11.h>
+
+static VADisplay *create_x11_va_display(GL *gl)
+{
+    Display *x11 = gl->MPGetNativeDisplay("x11");
+    return x11 ? vaGetDisplay(x11) : NULL;
+}
+#endif
+
+static VADisplay *create_native_va_display(GL *gl)
+{
+    if (!gl->MPGetNativeDisplay)
+        return NULL;
+    VADisplay *display = NULL;
+#if HAVE_VAAPI_X11
+    display = create_x11_va_display(gl);
+    if (display)
+        return display;
+#endif
+    return display;
+}
+
 struct priv {
     struct mp_log *log;
     struct mp_vaapi_ctx *ctx;
@@ -152,10 +174,6 @@ static int create(struct gl_hwdec *hw)
     if (!eglGetCurrentDisplay())
         return -1;
 
-    p->xdisplay =
-        hw->gl->MPGetNativeDisplay ? hw->gl->MPGetNativeDisplay("x11") : NULL;
-    if (!p->xdisplay)
-        return -1;
     if (!strstr(gl->extensions, "EXT_image_dma_buf_import") ||
         !strstr(gl->extensions, "EGL_KHR_image_base") ||
         !strstr(gl->extensions, "GL_OES_EGL_image") ||
@@ -173,7 +191,7 @@ static int create(struct gl_hwdec *hw)
         !p->EGLImageTargetTexture2DOES)
         return -1;
 
-    p->display = vaGetDisplay(p->xdisplay);
+    p->display = create_native_va_display(gl);
     if (!p->display)
         return -1;