From 1bb726dedce9ab0cd096bd524e869cf1899697cf Mon Sep 17 00:00:00 2001
From: wm4 <wm4@nowhere>
Date: Sat, 2 Nov 2019 22:37:14 +0100
Subject: vd_lavc: simplify fallback handling for full stream hw decoder

Shovel the code around to make the data flow slightly simpler (?). At
least there's only one send_packet function now. The old code had the
problem that send_packet() could be called even if there were queued
packets; due to sending the queued packets in the receive_frame
function, this should not happen anymore (the code checking for this
case in send_packet should normally never be called).

Untested with actual full stream hw decoders (none available here); I
created a test case by making hwaccel decoding fail.
---
 filters/f_decoder_wrapper.c |  2 +-
 video/decode/vd_lavc.c      | 38 ++++++++++++++++++--------------------
 2 files changed, 19 insertions(+), 21 deletions(-)

diff --git a/filters/f_decoder_wrapper.c b/filters/f_decoder_wrapper.c
index 846a0912ed..0462bf2424 100644
--- a/filters/f_decoder_wrapper.c
+++ b/filters/f_decoder_wrapper.c
@@ -856,7 +856,7 @@ void lavc_process(struct mp_filter *f, struct lavc_state *state,
         talloc_free(pkt);
         mp_filter_internal_mark_progress(f);
     } else {
-        // Decoding error? Just try again.
+        // Decoding error, or hwdec fallback recovery. Just try again.
         mp_filter_internal_mark_progress(f);
     }
 }
diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index 42b7b959cb..014b81b3b1 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -968,11 +968,14 @@ static void handle_err(struct mp_filter *vd)
     }
 }
 
-static int do_send_packet(struct mp_filter *vd, struct demux_packet *pkt)
+static int send_packet(struct mp_filter *vd, struct demux_packet *pkt)
 {
     vd_ffmpeg_ctx *ctx = vd->priv;
     AVCodecContext *avctx = ctx->avctx;
 
+    if (ctx->num_requeue_packets && ctx->requeue_packets[0] != pkt)
+        return AVERROR(EAGAIN); // cannot consume the packet
+
     if (!prepare_decoding(vd))
         return AVERROR_UNKNOWN;
 
@@ -996,28 +999,17 @@ static int do_send_packet(struct mp_filter *vd, struct demux_packet *pkt)
     return ret;
 }
 
-static int send_queued(struct mp_filter *vd)
+static void send_queued_packet(struct mp_filter *vd)
 {
     vd_ffmpeg_ctx *ctx = vd->priv;
 
-    while (ctx->num_requeue_packets) {
-        int ret = do_send_packet(vd, ctx->requeue_packets[0]);
-        if (ret < 0)
-            return ret;
+    assert(ctx->num_requeue_packets);
+    assert(!ctx->hw_probing);
+
+    if (send_packet(vd, ctx->requeue_packets[0]) != AVERROR(EAGAIN)) {
         talloc_free(ctx->requeue_packets[0]);
         MP_TARRAY_REMOVE_AT(ctx->requeue_packets, ctx->num_requeue_packets, 0);
     }
-
-    return 0;
-}
-
-static int send_packet(struct mp_filter *vd, struct demux_packet *pkt)
-{
-    int ret = send_queued(vd);
-    if (ret < 0)
-        return false;
-
-    return do_send_packet(vd, pkt);
 }
 
 // Returns whether decoder is still active (!EOF state).
@@ -1027,7 +1019,11 @@ static int decode_frame(struct mp_filter *vd)
     AVCodecContext *avctx = ctx->avctx;
 
     if (!prepare_decoding(vd))
-        return AVERROR(EAGAIN);
+        return AVERROR_UNKNOWN;
+
+    // Re-send old packets (typically after a hwdec fallback during init).
+    if (ctx->num_requeue_packets)
+        send_queued_packet(vd);
 
     int ret = avcodec_receive_frame(avctx, ctx->pic);
     if (ret == AVERROR_EOF) {
@@ -1085,10 +1081,12 @@ static int receive_frame(struct mp_filter *vd, struct mp_frame *out_frame)
         ctx->requeue_packets = pkts;
         ctx->num_requeue_packets = num_pkts;
 
-        send_queued(vd);
-        ret = decode_frame(vd);
+        return 0; // force retry
     }
 
+    if (ret == AVERROR(EAGAIN) && ctx->num_requeue_packets)
+        return 0; // force retry, so send_queued_packet() gets called
+
     if (!ctx->num_delay_queue)
         return ret;
 
-- 
cgit v1.2.3