vaapi: move standalone display creation code to common code

Preparation for the following commits.

3 files changed, 129 insertions, 136 deletions

diff --git a/video/decode/vaapi_old.c b/video/decode/vaapi_old.c
index c095868816..9c0907d0d4 100644
--- a/video/decode/vaapi_old.c
+++ b/video/decode/vaapi_old.c
@@ -52,11 +52,9 @@
 struct priv {
     struct mp_log *log;
     struct mp_vaapi_ctx *ctx;
+    bool own_ctx;
     VADisplay display;
 
-    const struct va_native_display *native_display_fns;
-    void *native_display;
-
     // libavcodec shared struct
     struct vaapi_context *va_context;
     struct vaapi_context va_context_storage;
@@ -67,98 +65,6 @@ struct priv {
     struct mp_image_pool *sw_pool;
 };
 
-struct va_native_display {
-    void (*create)(struct priv *p);
-    void (*destroy)(struct priv *p);
-};
-
-#if HAVE_VAAPI_X11
-#include <X11/Xlib.h>
-#include <va/va_x11.h>
-
-static void x11_destroy(struct priv *p)
-{
-    if (p->native_display)
-        XCloseDisplay(p->native_display);
-    p->native_display = NULL;
-}
-
-static void x11_create(struct priv *p)
-{
-    p->native_display = XOpenDisplay(NULL);
-    if (!p->native_display)
-        return;
-    p->display = vaGetDisplay(p->native_display);
-    if (!p->display)
-        x11_destroy(p);
-}
-
-static const struct va_native_display disp_x11 = {
-    .create = x11_create,
-    .destroy = x11_destroy,
-};
-#endif
-
-#if HAVE_VAAPI_DRM
-#include <unistd.h>
-#include <fcntl.h>
-#include <va/va_drm.h>
-
-struct va_native_display_drm {
-    int drm_fd;
-};
-
-static void drm_destroy(struct priv *p)
-{
-    struct va_native_display_drm *native_display = p->native_display;
-    if (native_display) {
-        if (native_display->drm_fd >= 0)
-            close(native_display->drm_fd);
-        talloc_free(native_display);
-        p->native_display = NULL;
-    }
-}
-
-static void drm_create(struct priv *p)
-{
-    static const char *drm_device_paths[] = {
-        "/dev/dri/renderD128",
-        "/dev/dri/card0",
-        NULL
-    };
-
-    for (int i = 0; drm_device_paths[i]; i++) {
-        int drm_fd = open(drm_device_paths[i], O_RDWR);
-        if (drm_fd < 0)
-            continue;
-
-        struct va_native_display_drm *native_display = talloc_ptrtype(NULL, native_display);
-        native_display->drm_fd = drm_fd;
-        p->native_display = native_display;
-        p->display = vaGetDisplayDRM(drm_fd);
-        if (p->display)
-            return;
-
-        drm_destroy(p);
-    }
-}
-
-static const struct va_native_display disp_drm = {
-    .create = drm_create,
-    .destroy = drm_destroy,
-};
-#endif
-
-static const struct va_native_display *const native_displays[] = {
-#if HAVE_VAAPI_DRM
-    &disp_drm,
-#endif
-#if HAVE_VAAPI_X11
-    &disp_x11,
-#endif
-    NULL
-};
-
 #define HAS_HEVC VA_CHECK_VERSION(0, 38, 0)
 #define HAS_VP9 (VA_CHECK_VERSION(0, 38, 1) && defined(FF_PROFILE_VP9_0))
 
@@ -397,40 +303,6 @@ static struct mp_image *update_format(struct lavc_ctx *ctx, struct mp_image *img
     return img;
 }
 
-static void destroy_va_dummy_ctx(struct priv *p)
-{
-    va_destroy(p->ctx);
-    p->ctx = NULL;
-    p->display = NULL;
-    if (p->native_display_fns)
-        p->native_display_fns->destroy(p);
-}
-
-// Creates a "private" VADisplay, disconnected from the VO. We just create a
-// new X connection, because that's simpler. (We could also pass the X
-// connection along with struct mp_hwdec_devices, if we wanted.)
-static bool create_va_dummy_ctx(struct priv *p)
-{
-    for (int n = 0; native_displays[n]; n++) {
-        native_displays[n]->create(p);
-        if (p->display) {
-            p->native_display_fns = native_displays[n];
-            break;
-        }
-    }
-    if (!p->display)
-        goto destroy_ctx;
-    p->ctx = va_initialize(p->display, p->log, true);
-    if (!p->ctx) {
-        vaTerminate(p->display);
-        goto destroy_ctx;
-    }
-    return true;
-destroy_ctx:
-    destroy_va_dummy_ctx(p);
-    return false;
-}
-
 static void uninit(struct lavc_ctx *ctx)
 {
     struct priv *p = ctx->hwdec_priv;
@@ -443,8 +315,8 @@ static void uninit(struct lavc_ctx *ctx)
     talloc_free(p->pool);
     p->pool = NULL;
 
-    if (p->native_display_fns)
-        destroy_va_dummy_ctx(p);
+    if (p->own_ctx)
+        va_destroy(p->ctx);
 
     talloc_free(p);
     ctx->hwdec_priv = NULL;
@@ -462,11 +334,12 @@ static int init(struct lavc_ctx *ctx, bool direct)
     if (direct) {
         p->ctx = hwdec_devices_get(ctx->hwdec_devs, HWDEC_VAAPI)->ctx;
     } else {
-        create_va_dummy_ctx(p);
+        p->ctx = va_create_standalone(ctx->log, false);
         if (!p->ctx) {
             talloc_free(p);
             return -1;
         }
+        p->own_ctx = true;
     }
 
     p->display = p->ctx->display;
@@ -502,11 +375,11 @@ static int probe(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
 static int probe_copy(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                       const char *codec)
 {
-    struct priv dummy = {mp_null_log};
-    if (!create_va_dummy_ctx(&dummy))
+    struct mp_vaapi_ctx *dummy = va_create_standalone(ctx->log, true);
+    if (!dummy)
         return HWDEC_ERR_NO_CTX;
-    bool emulated = va_guess_if_emulated(dummy.ctx);
-    destroy_va_dummy_ctx(&dummy);
+    bool emulated = va_guess_if_emulated(dummy);
+    va_destroy(dummy);
     if (!hwdec_check_codec_support(codec, profiles))
         return HWDEC_ERR_NO_CODEC;
     if (emulated)
diff --git a/video/vaapi.c b/video/vaapi.c
index 3db82d21fc..3545c32837 100644
--- a/video/vaapi.c
+++ b/video/vaapi.c
@@ -199,6 +199,9 @@ void va_destroy(struct mp_vaapi_ctx *ctx)
         if (ctx->display)
             vaTerminate(ctx->display);
 
+        if (ctx->destroy_native_ctx)
+            ctx->destroy_native_ctx(ctx->native_ctx);
+
         pthread_mutex_lock(&va_log_mutex);
         for (int n = 0; n < num_va_mpv_clients; n++) {
             if (va_mpv_clients[n] == ctx) {
@@ -610,3 +613,115 @@ bool va_guess_if_emulated(struct mp_vaapi_ctx *ctx)
     va_unlock(ctx);
     return s && strstr(s, "VDPAU backend");
 }
+
+struct va_native_display {
+    void (*create)(VADisplay **out_display, void **out_native_ctx);
+    void (*destroy)(void *native_ctx);
+};
+
+#if HAVE_VAAPI_X11
+#include <X11/Xlib.h>
+#include <va/va_x11.h>
+
+static void x11_destroy(void *native_ctx)
+{
+    XCloseDisplay(native_ctx);
+}
+
+static void x11_create(VADisplay **out_display, void **out_native_ctx)
+{
+    void *native_display = XOpenDisplay(NULL);
+    if (!native_display)
+        return;
+    *out_display = vaGetDisplay(native_display);
+    if (*out_display) {
+        *out_native_ctx = native_display;
+    } else {
+        XCloseDisplay(native_display);
+    }
+}
+
+static const struct va_native_display disp_x11 = {
+    .create = x11_create,
+    .destroy = x11_destroy,
+};
+#endif
+
+#if HAVE_VAAPI_DRM
+#include <unistd.h>
+#include <fcntl.h>
+#include <va/va_drm.h>
+
+struct va_native_display_drm {
+    int drm_fd;
+};
+
+static void drm_destroy(void *native_ctx)
+{
+    struct va_native_display_drm *ctx = native_ctx;
+    close(ctx->drm_fd);
+    talloc_free(ctx);
+}
+
+static void drm_create(VADisplay **out_display, void **out_native_ctx)
+{
+    static const char *drm_device_paths[] = {
+        "/dev/dri/renderD128",
+        "/dev/dri/card0",
+        NULL
+    };
+
+    for (int i = 0; drm_device_paths[i]; i++) {
+        int drm_fd = open(drm_device_paths[i], O_RDWR);
+        if (drm_fd < 0)
+            continue;
+
+        struct va_native_display_drm *ctx = talloc_ptrtype(NULL, ctx);
+        ctx->drm_fd = drm_fd;
+        *out_display = vaGetDisplayDRM(drm_fd);
+        if (out_display) {
+            *out_native_ctx = ctx;
+            return;
+        }
+
+        close(drm_fd);
+        talloc_free(ctx);
+    }
+}
+
+static const struct va_native_display disp_drm = {
+    .create = drm_create,
+    .destroy = drm_destroy,
+};
+#endif
+
+static const struct va_native_display *const native_displays[] = {
+#if HAVE_VAAPI_DRM
+    &disp_drm,
+#endif
+#if HAVE_VAAPI_X11
+    &disp_x11,
+#endif
+    NULL
+};
+
+struct mp_vaapi_ctx *va_create_standalone(struct mp_log *plog, bool probing)
+{
+    for (int n = 0; native_displays[n]; n++) {
+        VADisplay *display = NULL;
+        void *native_ctx = NULL;
+        native_displays[n]->create(&display, &native_ctx);
+        if (display) {
+            struct mp_vaapi_ctx *ctx = va_initialize(display, plog, probing);
+            if (!ctx) {
+                vaTerminate(display);
+                native_displays[n]->destroy(native_ctx);
+                return NULL;
+            }
+            ctx->native_ctx = native_ctx;
+            ctx->destroy_native_ctx = native_displays[n]->destroy;
+            return ctx;
+        }
+    }
+    return NULL;
+}
diff --git a/video/vaapi.h b/video/vaapi.h
index 3f0d1dca37..add3216308 100644
--- a/video/vaapi.h
+++ b/video/vaapi.h
@@ -36,6 +36,9 @@ struct mp_vaapi_ctx {
     struct va_image_formats *image_formats;
     bool gpu_memcpy_message;
     pthread_mutex_t lock;
+    // Internal, for va_create_standalone()
+    void *native_ctx;
+    void (*destroy_native_ctx)(void *native_ctx);
 };
 
 bool check_va_status(struct mp_log *log, VAStatus status, const char *msg);
@@ -73,4 +76,6 @@ void va_surface_init_subformat(struct mp_image *mpi);
 
 bool va_guess_if_emulated(struct mp_vaapi_ctx *ctx);
 
+struct mp_vaapi_ctx *va_create_standalone(struct mp_log *plog, bool probing);
+
 #endif