vo_vdpau, vo_opengl: handle vdpau preemption differently

Use the newly provided mp_vdpau_handle_preemption() function, instead of
accessing mp_vdpau_ctx fields directly. Will probably make multithreaded
access to the vdpau context easier.

Mostly unrelated to the actual changes, I've noticed that using hw
decoding with vo_opengl sometimes leads to segfaults inside of nvidia's
libGL when doing the following:

1. use hw decoding + vo_opengl
2. switch to console (will preempt on nvidia systems)
3. switch back to X (mpv will recover, switches to sw decoding)
4. enable hw decoding again
5. exit mpv

Then it segfaults when mpv finally calls exit(). I'll just blame nvidia,
although it seems likely that something in the gl_hwdec_vdpau.c
preemption handling triggers corner cases in nvidia's code.

4 files changed, 29 insertions, 64 deletions

diff --git a/video/out/gl_hwdec_vdpau.c b/video/out/gl_hwdec_vdpau.c
index a49e24b70a..db67679e4e 100644
--- a/video/out/gl_hwdec_vdpau.c
+++ b/video/out/gl_hwdec_vdpau.c
@@ -48,28 +48,6 @@ static void mark_vdpau_objects_uninitialized(struct gl_hwdec *hw)
     p->mixer->video_mixer = VDP_INVALID_HANDLE;
 }
 
-static int handle_preemption(struct gl_hwdec *hw)
-{
-    struct priv *p = hw->priv;
-
-    if (!mp_vdpau_status_ok(p->ctx)) {
-        mark_vdpau_objects_uninitialized(hw);
-        return -1;
-    }
-
-    if (p->preemption_counter == p->ctx->preemption_counter)
-        return 0;
-
-    mark_vdpau_objects_uninitialized(hw);
-
-    p->preemption_counter = p->ctx->preemption_counter;
-
-    if (reinit(hw, &p->image_params) < 0)
-        return -1;
-
-    return 1;
-}
-
 static void destroy_objects(struct gl_hwdec *hw)
 {
     struct priv *p = hw->priv;
@@ -125,7 +103,8 @@ static int create(struct gl_hwdec *hw)
     p->ctx = mp_vdpau_create_device_x11(hw->log, hw->mpgl->vo->x11);
     if (!p->ctx)
         return -1;
-    p->preemption_counter = p->ctx->preemption_counter;
+    if (mp_vdpau_handle_preemption(p->ctx, &p->preemption_counter) < 1)
+        return -1;
     p->vdp_surface = VDP_INVALID_HANDLE;
     p->mixer = mp_vdpau_mixer_create(p->ctx, hw->log);
     hw->info->vdpau_ctx = p->ctx;
@@ -144,7 +123,7 @@ static int reinit(struct gl_hwdec *hw, const struct mp_image_params *params)
 
     p->image_params = *params;
 
-    if (!mp_vdpau_status_ok(p->ctx))
+    if (mp_vdpau_handle_preemption(p->ctx, &p->preemption_counter) < 1)
         return -1;
 
     gl->VDPAUInitNV(BRAINDEATH(p->ctx->vdp_device), p->ctx->get_proc_address);
@@ -183,8 +162,14 @@ static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
 
     assert(hw_image && hw_image->imgfmt == IMGFMT_VDPAU);
 
-    if (handle_preemption(hw) < 0)
-        return -1;
+    int pe = mp_vdpau_handle_preemption(p->ctx, &p->preemption_counter);
+    if (pe < 1) {
+        mark_vdpau_objects_uninitialized(hw);
+        if (pe < 0)
+            return -1;
+        if (reinit(hw, &p->image_params) < 0)
+            return -1;
+    }
 
     if (!p->vdpgl_surface)
         return -1;
diff --git a/video/out/vo_vdpau.c b/video/out/vo_vdpau.c
index d43f7a552e..0ce9f888f1 100644
--- a/video/out/vo_vdpau.c
+++ b/video/out/vo_vdpau.c
@@ -307,18 +307,12 @@ static int win_x11_init_vdpau_flip_queue(struct vo *vo)
      * try to reinit after preemption while the user is still switched
      * from X to a virtual terminal (creating the vdp_device initially
      * succeeds, as does creating the flip_target above). This is
-     * probably not guaranteed behavior, but we'll assume it as a simple
-     * way to reduce warnings while trying to recover from preemption.
+     * probably not guaranteed behavior.
      */
     if (vc->flip_queue == VDP_INVALID_HANDLE) {
         vdp_st = vdp->presentation_queue_create(vc->vdp_device, vc->flip_target,
                                                 &vc->flip_queue);
-        if (vc->mpvdp->is_preempted && vdp_st != VDP_STATUS_OK) {
-            MP_DBG(vo, "Failed to create flip queue while preempted: %s\n",
-                   vdp->get_error_string(vdp_st));
-            return -1;
-        } else
-            CHECK_VDP_ERROR(vo, "Error when calling vdp_presentation_queue_create");
+        CHECK_VDP_ERROR(vo, "Error when calling vdp_presentation_queue_create");
     }
 
     if (vc->colorkey.a > 0) {
@@ -468,32 +462,25 @@ static void mark_vdpau_objects_uninitialized(struct vo *vo)
     vc->output_surface_width = vc->output_surface_height = -1;
 }
 
-static int handle_preemption(struct vo *vo)
+static bool check_preemption(struct vo *vo)
 {
     struct vdpctx *vc = vo->priv;
 
-    if (!mp_vdpau_status_ok(vc->mpvdp)) {
+    int r = mp_vdpau_handle_preemption(vc->mpvdp, &vc->preemption_counter);
+    if (r < 1) {
         mark_vdpau_objects_uninitialized(vo);
-        return -1;
+        if (r < 0)
+            return false;
+        vc->vdp_device = vc->mpvdp->vdp_device;
+        if (initialize_vdpau_objects(vo) < 0)
+            return false;
     }
-
-    if (vc->preemption_counter == vc->mpvdp->preemption_counter)
-        return 0;
-
-    mark_vdpau_objects_uninitialized(vo);
-
-    vc->preemption_counter = vc->mpvdp->preemption_counter;
-    vc->vdp_device = vc->mpvdp->vdp_device;
-
-    if (initialize_vdpau_objects(vo) < 0)
-        return -1;
-
-    return 1;
+    return true;
 }
 
 static bool status_ok(struct vo *vo)
 {
-    return vo->config_ok && handle_preemption(vo) >= 0;
+    return vo->config_ok && check_preemption(vo);
 }
 
 /*
@@ -504,7 +491,7 @@ static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
 {
     struct vdpctx *vc = vo->priv;
 
-    if (handle_preemption(vo) < 0)
+    if (!check_preemption(vo))
         return -1;
 
     vc->flip = flags & VOFLAG_FLIPPING;
@@ -777,7 +764,7 @@ static void flip_page_timed(struct vo *vo, int64_t pts_us, int duration)
     VdpStatus vdp_st;
     uint32_t vsync_interval = vc->vsync_interval;
 
-    if (handle_preemption(vo) < 0)
+    if (!check_preemption(vo))
         return;
 
     if (duration > INT_MAX / 1000)
@@ -926,7 +913,7 @@ static struct mp_image *filter_image(struct vo *vo, struct mp_image *mpi)
     struct mp_image *reserved_mpi = NULL;
     VdpStatus vdp_st;
 
-    handle_preemption(vo);
+    check_preemption(vo);
 
     if (vc->rgb_mode) {
         reserved_mpi = get_rgb_surface(vo);
@@ -1099,7 +1086,8 @@ static int preinit(struct vo *vo)
     // allocated
     mark_vdpau_objects_uninitialized(vo);
 
-    vc->preemption_counter = vc->mpvdp->preemption_counter;
+    mp_vdpau_handle_preemption(vc->mpvdp, &vc->preemption_counter);
+
     vc->vdp_device = vc->mpvdp->vdp_device;
     vc->vdp = &vc->mpvdp->vdp;
 
@@ -1145,7 +1133,7 @@ static int control(struct vo *vo, uint32_t request, void *data)
 {
     struct vdpctx *vc = vo->priv;
 
-    handle_preemption(vo);
+    check_preemption(vo);
 
     switch (request) {
     case VOCTRL_PAUSE:
diff --git a/video/vdpau.c b/video/vdpau.c
index 2e21c80a56..b26d0ed27b 100644
--- a/video/vdpau.c
+++ b/video/vdpau.c
@@ -120,13 +120,6 @@ static int handle_preemption(struct mp_vdpau_ctx *ctx)
     return 1;
 }
 
-// Check whether vdpau initialization and preemption status is ok and we can
-// proceed normally.
-bool mp_vdpau_status_ok(struct mp_vdpau_ctx *ctx)
-{
-    return handle_preemption(ctx) >= 0;
-}
-
 // Check whether vdpau display preemption happened. The caller provides a
 // preemption counter, which contains the logical timestamp of the last
 // preemption handled by the caller. The counter can be 0 for init.
diff --git a/video/vdpau.h b/video/vdpau.h
index d7bc5221a4..38e679db25 100644
--- a/video/vdpau.h
+++ b/video/vdpau.h
@@ -61,7 +61,6 @@ struct mp_vdpau_ctx *mp_vdpau_create_device_x11(struct mp_log *log,
                                                 struct vo_x11_state *x11);
 void mp_vdpau_destroy(struct mp_vdpau_ctx *ctx);
 
-bool mp_vdpau_status_ok(struct mp_vdpau_ctx *ctx);
 int mp_vdpau_handle_preemption(struct mp_vdpau_ctx *ctx, uint64_t *counter);
 
 struct mp_image *mp_vdpau_get_video_surface(struct mp_vdpau_ctx *ctx,