ad_lavc, vd_lavc: return full error codes to shared decoder loop

ad_lavc and vd_lavc use the lavc_process() helper to translate the
FFmpeg push/pull API to the internal filter API (which completely
mismatch, even though I'm responsible for both, just fucking kill me).

This interface was "slightly" too tight. It returned only a bool
indicating "progress", which was not enough to handle some cases (see
following commit).

While we're at it, move all state into a struct. This is only a single
bool, but we get the chance to add more if needed.

This fixes mpv falling asleep if decoding returns an error during
draining. If decoding fails when we already sent EOF, the state machine
stopped making progress. This left mpv just sitting around and doing
nothing.

A test case can be created with: echo $RANDOM >> image.png

This makes libavformat read a proper packet plus a packet of garbage.
libavcodec will decode a frame, and then return an error code. The
lavc_process() wrapper could not deal with this, because there was no
way to differentiate between "retry" and "send new packet". Normally, it
would send a new packet, so decoding would make progress anyway. If
there was "progress", we couldn't just retry, because it'd retry
forever.

This is made worse by the fact that it tries to decode at least two
frames before starting display, meaning it will "sit around and do
nothing" before the picture is displayed.

Change it so that on error return, "receiving" a frame is retried. This
will make it return the EOF, so everything works properly.

This is a high-risk change, because all these funny bullshit exceptions
for hardware decoding are in the way, and I didn't retest them. For
example, if hardware decoding is enabled, it keeps a list of packets,
that are fed into the decoder again if hardware decoding fails, and a
software fallback is performed. Another case of horrifying accidental
complexity.

Fixes: #6618

1 files changed, 10 insertions, 14 deletions

diff --git a/audio/decode/ad_lavc.c b/audio/decode/ad_lavc.c
index fbd6ad32ac..a420021f4a 100644
--- a/audio/decode/ad_lavc.c
+++ b/audio/decode/ad_lavc.c
@@ -48,7 +48,7 @@ struct priv {
     bool preroll_done;
     double next_pts;
     AVRational codec_timebase;
-    bool eof_returned;
+    struct lavc_state state;
 
     struct mp_decoder public;
 };
@@ -159,10 +159,10 @@ static void reset(struct mp_filter *da)
     ctx->trim_samples = 0;
     ctx->preroll_done = false;
     ctx->next_pts = MP_NOPTS_VALUE;
-    ctx->eof_returned = false;
+    ctx->state = (struct lavc_state){0};
 }
 
-static bool send_packet(struct mp_filter *da, struct demux_packet *mpkt)
+static int send_packet(struct mp_filter *da, struct demux_packet *mpkt)
 {
     struct priv *priv = da->priv;
     AVCodecContext *avctx = priv->avctx;
@@ -177,16 +177,12 @@ static bool send_packet(struct mp_filter *da, struct demux_packet *mpkt)
     mp_set_av_packet(&pkt, mpkt, &priv->codec_timebase);
 
     int ret = avcodec_send_packet(avctx, mpkt ? &pkt : NULL);
-
-    if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF)
-        return false;
-
     if (ret < 0)
         MP_ERR(da, "Error decoding audio.\n");
-    return true;
+    return ret;
 }
 
-static bool receive_frame(struct mp_filter *da, struct mp_frame *out)
+static int receive_frame(struct mp_filter *da, struct mp_frame *out)
 {
     struct priv *priv = da->priv;
     AVCodecContext *avctx = priv->avctx;
@@ -198,7 +194,7 @@ static bool receive_frame(struct mp_filter *da, struct mp_frame *out)
         // over in case we get new packets at some point in the future.
         // (Dont' reset the filter itself, we want to keep other state.)
         avcodec_flush_buffers(priv->avctx);
-        return false;
+        return ret;
     } else if (ret < 0 && ret != AVERROR(EAGAIN)) {
         MP_ERR(da, "Error decoding audio.\n");
     }
@@ -209,14 +205,14 @@ static bool receive_frame(struct mp_filter *da, struct mp_frame *out)
 #endif
 
     if (!priv->avframe->buf[0])
-        return true;
+        return ret;
 
     double out_pts = mp_pts_from_av(priv->avframe->pts, &priv->codec_timebase);
 
     struct mp_aframe *mpframe = mp_aframe_from_avframe(priv->avframe);
     if (!mpframe) {
         MP_ERR(da, "Converting libavcodec frame to mpv frame failed.\n");
-        return true;
+        return ret;
     }
 
     if (priv->force_channel_map.num)
@@ -264,14 +260,14 @@ static bool receive_frame(struct mp_filter *da, struct mp_frame *out)
 
     av_frame_unref(priv->avframe);
 
-    return true;
+    return ret;
 }
 
 static void process(struct mp_filter *ad)
 {
     struct priv *priv = ad->priv;
 
-    lavc_process(ad, &priv->eof_returned, send_packet, receive_frame);
+    lavc_process(ad, &priv->state, send_packet, receive_frame);
 }
 
 static const struct mp_filter_info ad_lavc_filter = {