video: refactor hw device creation for hwdec copy modes

Lots of shit code for nothing. We probably could just use libavutil's
code for all of this. But for now go with this, since it tends to
prevent stupid terminal messages during probing (libavutil has no
mechanism to selectively suppress errors specifically during probing).

Ignores the "emulated" API flag (for avoiding vaapi/vdpau wrappers), but
it doesn't matter that much for -copy anyway.

1 files changed, 62 insertions, 57 deletions

diff --git a/video/vdpau.c b/video/vdpau.c
index 4aa1a86597..1785702fdb 100644
--- a/video/vdpau.c
+++ b/video/vdpau.c
@@ -337,53 +337,88 @@ static void recheck_preemption(struct mp_hwdec_ctx *hwctx)
     mp_vdpau_handle_preemption(ctx, NULL);
 }
 
-static bool open_lavu_vdpau_device(struct mp_vdpau_ctx *ctx)
+static void free_device_ref(struct AVHWDeviceContext *hwctx)
 {
-    ctx->av_device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VDPAU);
-    if (!ctx->av_device_ref)
-        return false;
+    struct mp_vdpau_ctx *ctx = hwctx->user_opaque;
 
-    AVHWDeviceContext *hwctx = (void *)ctx->av_device_ref->data;
-    AVVDPAUDeviceContext *vdctx = hwctx->hwctx;
+    struct vdp_functions *vdp = &ctx->vdp;
+    VdpStatus vdp_st;
 
-    vdctx->device = ctx->vdp_device;
-    vdctx->get_proc_address = ctx->get_proc_address;
+    for (int i = 0; i < MAX_VIDEO_SURFACES; i++) {
+        // can't hold references past context lifetime
+        assert(!ctx->video_surfaces[i].in_use);
+        if (ctx->video_surfaces[i].surface != VDP_INVALID_HANDLE) {
+            vdp_st = vdp->video_surface_destroy(ctx->video_surfaces[i].surface);
+            CHECK_VDP_WARNING(ctx, "Error when calling vdp_video_surface_destroy");
+        }
+        if (ctx->video_surfaces[i].osurface != VDP_INVALID_HANDLE) {
+            vdp_st = vdp->output_surface_destroy(ctx->video_surfaces[i].osurface);
+            CHECK_VDP_WARNING(ctx, "Error when calling vdp_output_surface_destroy");
+        }
+    }
+
+    if (ctx->preemption_obj != VDP_INVALID_HANDLE) {
+        vdp_st = vdp->output_surface_destroy(ctx->preemption_obj);
+        CHECK_VDP_WARNING(ctx, "Error when calling vdp_output_surface_destroy");
+    }
 
-    if (av_hwdevice_ctx_init(ctx->av_device_ref) < 0)
-        av_buffer_unref(&ctx->av_device_ref);
+    if (vdp->device_destroy && ctx->vdp_device != VDP_INVALID_HANDLE) {
+        vdp_st = vdp->device_destroy(ctx->vdp_device);
+        CHECK_VDP_WARNING(ctx, "Error when calling vdp_device_destroy");
+    }
 
-    ctx->hwctx.av_device_ref = ctx->av_device_ref;
+    if (ctx->close_display)
+        XCloseDisplay(ctx->x11);
 
-    return !!ctx->av_device_ref;
+    pthread_mutex_destroy(&ctx->pool_lock);
+    pthread_mutex_destroy(&ctx->preempt_lock);
+    talloc_free(ctx);
 }
 
 struct mp_vdpau_ctx *mp_vdpau_create_device_x11(struct mp_log *log, Display *x11,
                                                 bool probing)
 {
+    AVBufferRef *avref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VDPAU);
+    if (!avref)
+        return NULL;
+
+    AVHWDeviceContext *hwctx = (void *)avref->data;
+    AVVDPAUDeviceContext *vdctx = hwctx->hwctx;
+
     struct mp_vdpau_ctx *ctx = talloc_ptrtype(NULL, ctx);
     *ctx = (struct mp_vdpau_ctx) {
         .log = log,
         .x11 = x11,
         .preemption_counter = 1,
+        .av_device_ref = avref,
         .hwctx = {
             .type = HWDEC_VDPAU,
             .ctx = ctx,
             .restore_device = recheck_preemption,
+            .av_device_ref = avref,
         },
     };
     mpthread_mutex_init_recursive(&ctx->preempt_lock);
     pthread_mutex_init(&ctx->pool_lock, NULL);
 
+    hwctx->free = free_device_ref;
+    hwctx->user_opaque = ctx;
+
     mark_vdpau_objects_uninitialized(ctx);
 
     if (win_x11_init_vdpau_procs(ctx, probing) < 0) {
         mp_vdpau_destroy(ctx);
         return NULL;
     }
-    if (!open_lavu_vdpau_device(ctx)) {
+
+    vdctx->device = ctx->vdp_device;
+    vdctx->get_proc_address = ctx->get_proc_address;
+
+    if (av_hwdevice_ctx_init(ctx->av_device_ref) < 0) {
         mp_vdpau_destroy(ctx);
         return NULL;
     }
+
     return ctx;
 }
 
@@ -392,37 +427,8 @@ void mp_vdpau_destroy(struct mp_vdpau_ctx *ctx)
     if (!ctx)
         return;
 
-    struct vdp_functions *vdp = &ctx->vdp;
-    VdpStatus vdp_st;
-
-    for (int i = 0; i < MAX_VIDEO_SURFACES; i++) {
-        // can't hold references past context lifetime
-        assert(!ctx->video_surfaces[i].in_use);
-        if (ctx->video_surfaces[i].surface != VDP_INVALID_HANDLE) {
-            vdp_st = vdp->video_surface_destroy(ctx->video_surfaces[i].surface);
-            CHECK_VDP_WARNING(ctx, "Error when calling vdp_video_surface_destroy");
-        }
-        if (ctx->video_surfaces[i].osurface != VDP_INVALID_HANDLE) {
-            vdp_st = vdp->output_surface_destroy(ctx->video_surfaces[i].osurface);
-            CHECK_VDP_WARNING(ctx, "Error when calling vdp_output_surface_destroy");
-        }
-    }
-
-    av_buffer_unref(&ctx->av_device_ref);
-
-    if (ctx->preemption_obj != VDP_INVALID_HANDLE) {
-        vdp_st = vdp->output_surface_destroy(ctx->preemption_obj);
-        CHECK_VDP_WARNING(ctx, "Error when calling vdp_output_surface_destroy");
-    }
-
-    if (vdp->device_destroy && ctx->vdp_device != VDP_INVALID_HANDLE) {
-        vdp_st = vdp->device_destroy(ctx->vdp_device);
-        CHECK_VDP_WARNING(ctx, "Error when calling vdp_device_destroy");
-    }
-
-    pthread_mutex_destroy(&ctx->pool_lock);
-    pthread_mutex_destroy(&ctx->preempt_lock);
-    talloc_free(ctx);
+    AVBufferRef *ref = ctx->av_device_ref;
+    av_buffer_unref(&ref); // frees ctx as well
 }
 
 bool mp_vdpau_get_format(int imgfmt, VdpChromaType *out_chroma_type,
@@ -531,16 +537,8 @@ bool mp_vdpau_guess_if_emulated(struct mp_vdpau_ctx *ctx)
     return vdp_st == VDP_STATUS_OK && info && strstr(info, "VAAPI");
 }
 
-static void vdpau_destroy_standalone(struct mp_hwdec_ctx *ctx)
-{
-    struct mp_vdpau_ctx *vdp = ctx->ctx;
-    Display *display = vdp->x11;
-    mp_vdpau_destroy(vdp);
-    XCloseDisplay(display);
-}
-
-struct mp_hwdec_ctx *vdpau_create_standalone(struct mpv_global *global,
-                                             struct mp_log *plog, bool probing)
+static struct AVBufferRef *vdpau_create_standalone(struct mpv_global *global,
+        struct mp_log *log, struct hwcontext_create_dev_params *params)
 {
     XInitThreads();
 
@@ -548,13 +546,20 @@ struct mp_hwdec_ctx *vdpau_create_standalone(struct mpv_global *global,
     if (!display)
         return NULL;
 
-    struct mp_vdpau_ctx *vdp = mp_vdpau_create_device_x11(plog, display, probing);
+    struct mp_vdpau_ctx *vdp =
+        mp_vdpau_create_device_x11(log, display, params->probing);
     if (!vdp) {
         XCloseDisplay(display);
         return NULL;
     }
 
     vdp->hwctx.emulated = mp_vdpau_guess_if_emulated(vdp);
-    vdp->hwctx.destroy = vdpau_destroy_standalone;
-    return &vdp->hwctx;
+    vdp->close_display = true;
+    mp_warn(log, "idk\n");
+    return vdp->hwctx.av_device_ref;
 }
+
+const struct hwcontext_fns hwcontext_fns_vdpau = {
+    .av_hwdevice_type = AV_HWDEVICE_TYPE_VDPAU,
+    .create_dev = vdpau_create_standalone,
+};