From 4f63a812de6ed8a76dde2a3739513933748ac2af Mon Sep 17 00:00:00 2001
From: Rudolf Polzer <divVerent@xonotic.org>
Date: Wed, 12 Nov 2014 12:16:07 +0100
Subject: ao_lavc, vo_lavc: Fix crashes in case of multiple init attempts.

When initialization failed, vo_lavc may cause an irrecoverable state in
the ffmpeg-related structs. Therefore, we reject additional
initialization attempts at least until we know a better way to clean up
the mess.

ao_lavc currently cannot be initialized more than once, yet it's good to
do consistent changes there as well.

Also, clean up uninit-after-failure handling to be less spammy.
---
 audio/out/ao_lavc.c |  8 ++++++++
 video/out/vo_lavc.c | 22 ++++++++++++++++------
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/audio/out/ao_lavc.c b/audio/out/ao_lavc.c
index 357c34763d..9f2179e8f7 100644
--- a/audio/out/ao_lavc.c
+++ b/audio/out/ao_lavc.c
@@ -56,6 +56,8 @@ struct priv {
 
     AVRational worst_time_base;
     int worst_time_base_is_stream;
+
+    bool shutdown;
 };
 
 static void select_format(struct ao *ao, AVCodec *codec)
@@ -174,6 +176,7 @@ static int init(struct ao *ao)
 
 fail:
     pthread_mutex_unlock(&ao->encode_lavc_ctx->lock);
+    ac->shutdown = true;
     return -1;
 }
 
@@ -184,6 +187,9 @@ static void uninit(struct ao *ao)
     struct priv *ac = ao->priv;
     struct encode_lavc_context *ectx = ao->encode_lavc_ctx;
 
+    if (!ac || ac->shutdown)
+        return;
+
     pthread_mutex_lock(&ectx->lock);
 
     if (!encode_lavc_start(ectx)) {
@@ -201,6 +207,8 @@ static void uninit(struct ao *ao)
     }
 
     pthread_mutex_unlock(&ectx->lock);
+
+    ac->shutdown = true;
 }
 
 // return: how many bytes can be played without blocking
diff --git a/video/out/vo_lavc.c b/video/out/vo_lavc.c
index d5cf263ce2..9bb3ebd866 100644
--- a/video/out/vo_lavc.c
+++ b/video/out/vo_lavc.c
@@ -56,6 +56,8 @@ struct priv {
     int worst_time_base_is_stream;
 
     struct mp_image_params real_colorspace;
+
+    bool shutdown;
 };
 
 static int preinit(struct vo *vo)
@@ -75,7 +77,7 @@ static void draw_image_unlocked(struct vo *vo, mp_image_t *mpi);
 static void uninit(struct vo *vo)
 {
     struct priv *vc = vo->priv;
-    if (!vc)
+    if (!vc || vc->shutdown)
         return;
 
     pthread_mutex_lock(&vo->encode_lavc_ctx->lock);
@@ -86,6 +88,8 @@ static void uninit(struct vo *vo)
     mp_image_unrefp(&vc->lastimg);
 
     pthread_mutex_unlock(&vo->encode_lavc_ctx->lock);
+
+    vc->shutdown = true;
 }
 
 static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
@@ -97,7 +101,7 @@ static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
     uint32_t width = params->w;
     uint32_t height = params->h;
 
-    if (!vc)
+    if (!vc || vc->shutdown)
         return -1;
 
     pthread_mutex_lock(&vo->encode_lavc_ctx->lock);
@@ -132,6 +136,14 @@ static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
         goto error;
     }
 
+    // When we get here, this must be the first call to reconfigure(). Thus, we
+    // can rely on no existing data in vc having been allocated yet.
+    // Reason:
+    // - Second calls after reconfigure() already failed once fail (due to the
+    //   vc->shutdown check above).
+    // - Second calls after reconfigure() already succeeded once return early
+    //   (due to the vc->stream check above).
+
     vc->lastipts = AV_NOPTS_VALUE;
     vc->lastframeipts = AV_NOPTS_VALUE;
     vc->lastencodedipts = AV_NOPTS_VALUE;
@@ -169,15 +181,13 @@ static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
 
     vc->buffer = talloc_size(vc, vc->buffer_size);
 
-    mp_image_unrefp(&vc->lastimg);
-
 done:
     pthread_mutex_unlock(&vo->encode_lavc_ctx->lock);
     return 0;
 
 error:
     pthread_mutex_unlock(&vo->encode_lavc_ctx->lock);
-    uninit(vo);
+    vc->shutdown = true;
     return -1;
 }
 
@@ -288,7 +298,7 @@ static void draw_image_unlocked(struct vo *vo, mp_image_t *mpi)
 
     double pts = mpi ? mpi->pts : MP_NOPTS_VALUE;
 
-    if (!vc)
+    if (!vc || vc->shutdown)
         goto done;
     if (!encode_lavc_start(ectx)) {
         MP_WARN(vo, "NOTE: skipped initial video frame (probably because audio is not there yet)\n");
-- 
cgit v1.2.3