vdpau: force driver to report preemption early

Another fix for the crazy and insane nvidia preemption behavior.

This time, the situation is that we are using vo_opengl with vdpau
interop, and that vdpau got preempted in the background while mpv was
sitting idly. This can be e.g. reproduced by using:

   --force-window=immediate --idle --hwdec=vdpau

and switching VTs. Then after switching back, load a video file.

This will not let mp_vdpau_handle_preemption() perform preemption
recovery, simply because it will do so only once vdp_decoder_create()
has been called. There are some other API calls which trigger
preemption, but many don't.

Due to the way the libavcodec API works, vdp_decoder_create() is way too
late. It does so when get_format returns. It notices creating the
decoder fails, and continues calling get_format without the vdpau
format. We could perhaps force it to reinit again (by adding a call to
vdpau.c, that checks for preemption, and sets hwdec_request_reinit), but
this seems too much of a mess.

Solve it by calling API in mp_vdpau_handle_preemption() that empirically
does trigger preemption: output_surface_put_bits_native(). This call is
useless, and in fact should be doing nothing (empty update VdpRect).
There's the slight chance that in theory it will slow down operation,
but in practice it's bound to be harmless. It's the likely cheapest and
simplest API call I've found that can trigger the fallback this way.
(The driver is closed source, so it was up to trial & error.)

Also, when initializing decoding, allow initial preemption recovery,
which is needed to pass the test mention above.

3 files changed, 21 insertions, 3 deletions

diff --git a/video/decode/vdpau.c b/video/decode/vdpau.c
index 72e050b9f0..c69465094b 100644
--- a/video/decode/vdpau.c
+++ b/video/decode/vdpau.c
@@ -79,9 +79,7 @@ static int init(struct lavc_ctx *ctx)
     };
     ctx->hwdec_priv = p;
 
-    if (mp_vdpau_handle_preemption(p->mpvdp, &p->preemption_counter) < 1)
-        return -1;
-
+    mp_vdpau_handle_preemption(p->mpvdp, &p->preemption_counter);
     return 0;
 }
 
diff --git a/video/vdpau.c b/video/vdpau.c
index 9db161426c..9dfbc2bc6e 100644
--- a/video/vdpau.c
+++ b/video/vdpau.c
@@ -102,6 +102,7 @@ static void mark_vdpau_objects_uninitialized(struct mp_vdpau_ctx *ctx)
         ctx->video_surfaces[i].allocated = false;
     }
     ctx->vdp_device = VDP_INVALID_HANDLE;
+    ctx->preemption_obj = VDP_INVALID_HANDLE;
 }
 
 static void preemption_callback(VdpDevice device, void *context)
@@ -167,6 +168,14 @@ static int win_x11_init_vdpau_procs(struct mp_vdpau_ctx *ctx, bool probing)
     ctx->vdp = vdp;
     ctx->get_proc_address = get_proc_address;
 
+    vdp_st = vdp.output_surface_create(ctx->vdp_device, VDP_RGBA_FORMAT_B8G8R8A8,
+                                       1, 1, &ctx->preemption_obj);
+    if (vdp_st != VDP_STATUS_OK) {
+        MP_ERR(ctx, "Could not create dummy object: %s",
+               vdp.get_error_string(vdp_st));
+        return -1;
+    }
+
     vdp.preemption_callback_register(ctx->vdp_device, preemption_callback, ctx);
     return 0;
 }
@@ -211,6 +220,11 @@ int mp_vdpau_handle_preemption(struct mp_vdpau_ctx *ctx, uint64_t *counter)
     int r = 1;
     pthread_mutex_lock(&ctx->preempt_lock);
 
+    const void *p[4] = {&(uint32_t){0}};
+    uint32_t stride[4] = {4};
+    VdpRect rc = {0};
+    ctx->vdp.output_surface_put_bits_native(ctx->preemption_obj, p, stride, &rc);
+
     // First time init
     if (counter && !*counter)
         *counter = ctx->preemption_counter;
@@ -425,6 +439,11 @@ void mp_vdpau_destroy(struct mp_vdpau_ctx *ctx)
         CHECK_VDP_WARNING(ctx, "Error when calling vdp_output_surface_destroy");
     }
 
+    if (ctx->preemption_obj != VDP_INVALID_HANDLE) {
+        vdp_st = vdp->output_surface_destroy(ctx->preemption_obj);
+        CHECK_VDP_WARNING(ctx, "Error when calling vdp_output_surface_destroy");
+    }
+
     if (vdp->device_destroy && ctx->vdp_device != VDP_INVALID_HANDLE) {
         vdp_st = vdp->device_destroy(ctx->vdp_device);
         CHECK_VDP_WARNING(ctx, "Error when calling vdp_device_destroy");
diff --git a/video/vdpau.h b/video/vdpau.h
index 2e9269a420..db73a87dd7 100644
--- a/video/vdpau.h
+++ b/video/vdpau.h
@@ -59,6 +59,7 @@ struct mp_vdpau_ctx {
     uint64_t preemption_counter;        // incremented after _restoring_
     bool preemption_user_notified;
     double last_preemption_retry_fail;
+    VdpOutputSurface preemption_obj;    // dummy for reliable preempt. check
 
     // Surface pool
     pthread_mutex_t pool_lock;