vdpau: crappy hack to allow initializing hw decoding after preemption

If vo_opengl is used, and vo_opengl already created the vdpau interop
(for whatever reasons), and then preemption happens, and then you try to
enable hw decoding, it failed. The reason was that preemption recovery
is not run at any point before libavcodec accesses the vdpau device.

The actual impact was that with libmpv + opengl-cb use, hardware
decoding was permanently broken after display mode switching (something
that caused the display to get preempted at least with older drivers).
With mpv CLI, you can for example enable hw decoding during playback,
then disable it, VT switch to console, switch back to X, and try to
enable hw decoding again.

This is mostly because libav* does not deal with preemption, and NVIDIA
driver preemption behavior being horrible garbage. In addition to being
misdesigned API, the preemption callback is not called before you try to
access vdpau API, and then only with _some_ accesses.

In summary, the preemption callback was never called, neither before nor
after libavcodec tried to init the decoder. So we have to get
mp_vdpau_handle_preemption() called before libavcodec accesses it. This
in turn will do a dummy API access which usually triggers the preemption
callback immediately (with NVIDIA's drivers).

In addition, we have to update the AVHWDeviceContext's device. In theory
it could change (in practice it usually seems to use handle "0").
Creating a new device would cause chaos, as we don't have a concept of
switching the device context on the fly. So we simply update it
directly. I'm fairly sure this violates the libav* API, but it's the
best we can do.

3 files changed, 21 insertions, 0 deletions

diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index fe4409968a..9c1a2a0317 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -603,6 +603,8 @@ static void init_avctx(struct dec_video *vd, const char *decoder,
             ctx->hwdec_dev = hwdec_create_dev(vd, ctx->hwdec, false);
             if (!ctx->hwdec_dev)
                 goto error;
+            if (ctx->hwdec_dev->restore_device)
+                ctx->hwdec_dev->restore_device(ctx->hwdec_dev);
             if (!ctx->hwdec->set_hwframes) {
 #if HAVE_VDPAU_HWACCEL
                 avctx->hw_device_ctx = av_buffer_ref(ctx->hwdec_dev->av_device_ref);
diff --git a/video/hwdec.h b/video/hwdec.h
index 49fd9895c0..c0ef9b76b6 100644
--- a/video/hwdec.h
+++ b/video/hwdec.h
@@ -66,6 +66,9 @@ struct mp_hwdec_ctx {
                                        struct mp_image *mpi,
                                        struct mp_image_pool *swpool);
 
+    // Optional. Crap for vdpau. Makes sure preemption recovery is run if needed.
+    void (*restore_device)(struct mp_hwdec_ctx *ctx);
+
     // Optional. Do not set for VO-bound devices.
     void (*destroy)(struct mp_hwdec_ctx *ctx);
 };
diff --git a/video/vdpau.c b/video/vdpau.c
index fa79e9bc67..fabd47c96b 100644
--- a/video/vdpau.c
+++ b/video/vdpau.c
@@ -188,6 +188,14 @@ static int win_x11_init_vdpau_procs(struct mp_vdpau_ctx *ctx, bool probing)
     ctx->vdp = vdp;
     ctx->get_proc_address = get_proc_address;
 
+    if (ctx->av_device_ref) {
+        AVHWDeviceContext *hwctx = (void *)ctx->av_device_ref->data;
+        AVVDPAUDeviceContext *vdctx = hwctx->hwctx;
+
+        vdctx->device = ctx->vdp_device;
+        vdctx->get_proc_address = ctx->get_proc_address;
+    }
+
     vdp_st = vdp.output_surface_create(ctx->vdp_device, VDP_RGBA_FORMAT_B8G8R8A8,
                                        1, 1, &ctx->preemption_obj);
     if (vdp_st != VDP_STATUS_OK) {
@@ -405,6 +413,13 @@ struct mp_image *mp_vdpau_get_video_surface(struct mp_vdpau_ctx *ctx,
     return mp_vdpau_get_surface(ctx, chroma, 0, false, w, h);
 }
 
+static void recheck_preemption(struct mp_hwdec_ctx *hwctx)
+{
+    struct mp_vdpau_ctx *ctx = hwctx->ctx;
+
+    mp_vdpau_handle_preemption(ctx, NULL);
+}
+
 static bool open_lavu_vdpau_device(struct mp_vdpau_ctx *ctx)
 {
     ctx->av_device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VDPAU);
@@ -437,6 +452,7 @@ struct mp_vdpau_ctx *mp_vdpau_create_device_x11(struct mp_log *log, Display *x11
             .type = HWDEC_VDPAU,
             .ctx = ctx,
             .download_image = download_image,
+            .restore_device = recheck_preemption,
         },
         .getimg_surface = VDP_INVALID_HANDLE,
     };