ad_lavc, vd_lavc: return full error codes to shared decoder loop

ad_lavc and vd_lavc use the lavc_process() helper to translate the
FFmpeg push/pull API to the internal filter API (which completely
mismatch, even though I'm responsible for both, just fucking kill me).

This interface was "slightly" too tight. It returned only a bool
indicating "progress", which was not enough to handle some cases (see
following commit).

While we're at it, move all state into a struct. This is only a single
bool, but we get the chance to add more if needed.

This fixes mpv falling asleep if decoding returns an error during
draining. If decoding fails when we already sent EOF, the state machine
stopped making progress. This left mpv just sitting around and doing
nothing.

A test case can be created with: echo $RANDOM >> image.png

This makes libavformat read a proper packet plus a packet of garbage.
libavcodec will decode a frame, and then return an error code. The
lavc_process() wrapper could not deal with this, because there was no
way to differentiate between "retry" and "send new packet". Normally, it
would send a new packet, so decoding would make progress anyway. If
there was "progress", we couldn't just retry, because it'd retry
forever.

This is made worse by the fact that it tries to decode at least two
frames before starting display, meaning it will "sit around and do
nothing" before the picture is displayed.

Change it so that on error return, "receiving" a frame is retried. This
will make it return the EOF, so everything works properly.

This is a high-risk change, because all these funny bullshit exceptions
for hardware decoding are in the way, and I didn't retest them. For
example, if hardware decoding is enabled, it keeps a list of packets,
that are fed into the decoder again if hardware decoding fails, and a
software fallback is performed. Another case of horrifying accidental
complexity.

Fixes: #6618

2 files changed, 23 insertions, 14 deletions

diff --git a/filters/f_decoder_wrapper.c b/filters/f_decoder_wrapper.c
index f18247abaf..4347917152 100644
--- a/filters/f_decoder_wrapper.c
+++ b/filters/f_decoder_wrapper.c
@@ -22,6 +22,7 @@
 #include <assert.h>
 
 #include <libavutil/buffer.h>
+#include <libavutil/common.h>
 #include <libavutil/rational.h>
 
 #include "config.h"
@@ -814,22 +815,23 @@ error:
     return NULL;
 }
 
-void lavc_process(struct mp_filter *f, bool *eof_flag,
-                  bool (*send)(struct mp_filter *f, struct demux_packet *pkt),
-                  bool (*receive)(struct mp_filter *f, struct mp_frame *res))
+void lavc_process(struct mp_filter *f, struct lavc_state *state,
+                  int (*send)(struct mp_filter *f, struct demux_packet *pkt),
+                  int (*receive)(struct mp_filter *f, struct mp_frame *res))
 {
     if (!mp_pin_in_needs_data(f->ppins[1]))
         return;
 
     struct mp_frame frame = {0};
-    if (!receive(f, &frame)) {
-        if (!*eof_flag)
+    int ret_recv = receive(f, &frame);
+    if (ret_recv == AVERROR_EOF) {
+        if (!state->eof_returned)
             mp_pin_in_write(f->ppins[1], MP_EOF_FRAME);
-        *eof_flag = true;
+        state->eof_returned = true;
     } else if (frame.type) {
-        *eof_flag = false;
+        state->eof_returned = false;
         mp_pin_in_write(f->ppins[1], frame);
-    } else {
+    } else if (ret_recv == AVERROR(EAGAIN)) {
         // Need to feed a packet.
         frame = mp_pin_out_read(f->ppins[0]);
         struct demux_packet *pkt = NULL;
@@ -843,7 +845,8 @@ void lavc_process(struct mp_filter *f, bool *eof_flag,
             }
             return;
         }
-        if (!send(f, pkt)) {
+        int ret_send = send(f, pkt);
+        if (ret_send == AVERROR(EAGAIN)) {
             // Should never happen, but can happen with broken decoders.
             MP_WARN(f, "could not consume packet\n");
             mp_pin_out_unread(f->ppins[0], frame);
@@ -852,5 +855,8 @@ void lavc_process(struct mp_filter *f, bool *eof_flag,
         }
         talloc_free(pkt);
         mp_filter_internal_mark_progress(f);
+    } else {
+        // Just try again.
+        mp_filter_internal_mark_progress(f);
     }
 }
diff --git a/filters/f_decoder_wrapper.h b/filters/f_decoder_wrapper.h
index b69c0c7680..28d9b5cb7b 100644
--- a/filters/f_decoder_wrapper.h
+++ b/filters/f_decoder_wrapper.h
@@ -109,11 +109,14 @@ extern const struct mp_decoder_fns vd_lavc;
 extern const struct mp_decoder_fns ad_lavc;
 extern const struct mp_decoder_fns ad_spdif;
 
-// Convenience wrapper for lavc based decoders. eof_flag must be set to false
-// on init and resets.
-void lavc_process(struct mp_filter *f, bool *eof_flag,
-                  bool (*send)(struct mp_filter *f, struct demux_packet *pkt),
-                  bool (*receive)(struct mp_filter *f, struct mp_frame *res));
+// Convenience wrapper for lavc based decoders. Treat lavc_state as private;
+// init to all-0 on init and resets.
+struct lavc_state {
+    bool eof_returned;
+};
+void lavc_process(struct mp_filter *f, struct lavc_state *state,
+                  int (*send)(struct mp_filter *f, struct demux_packet *pkt),
+                  int (*receive)(struct mp_filter *f, struct mp_frame *res));
 
 // ad_spdif.c
 struct mp_decoder_list *select_spdif_codec(const char *codec, const char *pref);