vf_vapoursynth: fail gracefully if filter init requests frames

Some VS filters will requests frames from their parent filters while
they're initialized. Thy do this in a blocking manner, and
initialization will not succeed until the frame request is satisfied.
This deadlocked mpv, because we can feed frames to the filter only after
initialization is finished.

Return an error instead of deadlocking.

Note that we (probably) can handle frames being requested during init
fine, as long as the requests don't block initialization. But we can
distinguish this situation, and a simple test seems to indicate VS
usually doesn't do this.

See #1168.

1 files changed, 10 insertions, 0 deletions

diff --git a/video/filter/vf_vapoursynth.c b/video/filter/vf_vapoursynth.c
index 09357af5b0..5d228caa50 100644
--- a/video/filter/vf_vapoursynth.c
+++ b/video/filter/vf_vapoursynth.c
@@ -60,6 +60,7 @@ struct vf_priv_s {
     int max_requests;           // upper bound for requested[] array
     bool failed;                // frame callback returned with an error
     bool shutdown;              // ask node to return
+    bool initializing;          // filters are being built
     bool in_node_active;        // node might still be called
 
     // --- options
@@ -380,6 +381,10 @@ static const VSFrameRef *VS_CC infiltGetFrame(int frameno, int activationReason,
             p->vsapi->setFilterError("EOF or filter reinit/uninit", frameCtx);
             break;
         }
+        if (p->initializing) {
+            p->vsapi->setFilterError("Frame requested during init", frameCtx);
+            break;
+        }
         if (frameno < p->in_frameno) {
             char msg[180];
             snprintf(msg, sizeof(msg),
@@ -456,6 +461,7 @@ static void destroy_vs(struct vf_instance *vf)
 
     // Wait until our frame callbacks return.
     pthread_mutex_lock(&p->lock);
+    p->initializing = false;
     p->shutdown = true;
     pthread_cond_broadcast(&p->wakeup);
     while (num_requested(p))
@@ -506,6 +512,7 @@ static int reinit_vs(struct vf_instance *vf)
     destroy_vs(vf);
 
     MP_DBG(vf, "initializing...\n");
+    p->initializing = true;
 
     // First load an empty script to get a VSScript, so that we get the vsapi
     // and vscore.
@@ -557,6 +564,9 @@ static int reinit_vs(struct vf_instance *vf)
         goto error;
     }
 
+    pthread_mutex_lock(&p->lock);
+    p->initializing = false;
+    pthread_mutex_unlock(&p->lock);
     MP_DBG(vf, "initialized.\n");
     res = 0;
 error: