video: make decoder EOF reporting explicit

This is simpler and more robust, especially for the hwdec fallback case.
The most annoying issue is that C doesn't support multiple return values
(or sum types), so the decode call gets all awkward.

The hwdec fallback case does not need to try to produce some output
after the fallback anymore. Instead, it can use the normal "replay"
code path.

We invert the "eof" bool that vd_lavc.c used internally. The
receive_frame decoder API returns the inverse of EOF, because
returning "true" from the decode function if EOF was reached
feels awkward.

3 files changed, 38 insertions, 51 deletions

diff --git a/video/decode/dec_video.c b/video/decode/dec_video.c
index 79b706d1fd..273b80c3c9 100644
--- a/video/decode/dec_video.c
+++ b/video/decode/dec_video.c
@@ -272,19 +272,22 @@ static bool send_packet(struct dec_video *d_video, struct demux_packet *packet)
     return res;
 }
 
-static struct mp_image *receive_frame(struct dec_video *d_video)
+static bool receive_frame(struct dec_video *d_video, struct mp_image **out_image)
 {
     struct MPOpts *opts = d_video->opts;
+    struct mp_image *mpi = NULL;
+
+    assert(!*out_image);
 
     MP_STATS(d_video, "start decode video");
 
-    struct mp_image *mpi = d_video->vd_driver->receive_frame(d_video);
+    bool progress = d_video->vd_driver->receive_frame(d_video, &mpi);
 
     MP_STATS(d_video, "end decode video");
 
-    // Error, discarded frame, dropped frame, or initial codec delay.
+    // Error, EOF, discarded frame, dropped frame, or initial codec delay.
     if (!mpi)
-        return NULL;
+        return progress;
 
     if (opts->field_dominance == 0) {
         mpi->fields |= MP_IMGFIELD_TOP_FIRST | MP_IMGFIELD_INTERLACED;
@@ -356,7 +359,8 @@ static struct mp_image *receive_frame(struct dec_video *d_video)
         mpi->pts -= MPMAX(delay, 0) / d_video->fps;
     }
 
-    return mpi;
+    *out_image = mpi;
+    return true;
 }
 
 void video_reset_params(struct dec_video *d_video)
@@ -403,9 +407,6 @@ void video_work(struct dec_video *d_video)
         d_video->packet = NULL;
     }
 
-    bool had_input_packet = !!d_video->packet;
-    bool had_packet = had_input_packet || d_video->new_segment;
-
     double start_pts = d_video->start_pts;
     if (d_video->start != MP_NOPTS_VALUE && (start_pts == MP_NOPTS_VALUE ||
                                              d_video->start > start_pts))
@@ -426,19 +427,18 @@ void video_work(struct dec_video *d_video)
         d_video->packet = NULL;
     }
 
-    d_video->current_mpi = receive_frame(d_video);
+    bool progress = receive_frame(d_video, &d_video->current_mpi);
 
     d_video->current_state = DATA_OK;
-    if (!d_video->current_mpi) {
+    if (!progress) {
         d_video->current_state = DATA_EOF;
-        if (had_packet) {
-            if (framedrop_type == 1)
-                d_video->dropped_frames += 1;
-            d_video->current_state = DATA_AGAIN;
-        }
+    } else if (!d_video->current_mpi) {
+        if (framedrop_type == 1)
+            d_video->dropped_frames += 1;
+        d_video->current_state = DATA_AGAIN;
     }
 
-    bool segment_ended = !d_video->current_mpi && !had_input_packet;
+    bool segment_ended = d_video->current_state == DATA_EOF;
 
     if (d_video->current_mpi && d_video->current_mpi->pts != MP_NOPTS_VALUE) {
         double vpts = d_video->current_mpi->pts;
diff --git a/video/decode/vd.h b/video/decode/vd.h
index 4702b77860..3897eedc31 100644
--- a/video/decode/vd.h
+++ b/video/decode/vd.h
@@ -35,7 +35,9 @@ typedef struct vd_functions
     int (*control)(struct dec_video *vd, int cmd, void *arg);
     // Return whether or not the packet has been consumed.
     bool (*send_packet)(struct dec_video *vd, struct demux_packet *pkt);
-    struct mp_image *(*receive_frame)(struct dec_video *vd);
+    // Return whether decoding is still going on (false if EOF was reached).
+    // Never returns false & *out_image set, but can return true with no image.
+    bool (*receive_frame)(struct dec_video *vd, struct mp_image **out_image);
 } vd_functions_t;
 
 // NULL terminated array of all drivers
diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index 9b65912fe4..a8c0c41f0c 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -755,15 +755,18 @@ static int get_buffer2_hwdec(AVCodecContext *avctx, AVFrame *pic, int flags)
     return 0;
 }
 
-static struct mp_image *read_output(struct dec_video *vd, bool eof)
+static bool read_output(struct dec_video *vd, bool progress,
+                        struct mp_image **out_image)
 {
     vd_ffmpeg_ctx *ctx = vd->priv;
 
+    assert(!*out_image);
+
     if (!ctx->num_delay_queue)
-        return NULL;
+        return progress;
 
-    if (ctx->num_delay_queue <= ctx->max_delay_queue && !eof)
-        return NULL;
+    if (ctx->num_delay_queue <= ctx->max_delay_queue && progress)
+        return true;
 
     struct mp_image *res = ctx->delay_queue[0];
     MP_TARRAY_REMOVE_AT(ctx->delay_queue, ctx->num_delay_queue, 0);
@@ -773,7 +776,7 @@ static struct mp_image *read_output(struct dec_video *vd, bool eof)
 
     res = res ? mp_img_swap_to_native(res) : NULL;
     if (!res)
-        return NULL;
+        return progress;
 
     if (!ctx->hwdec_notified && vd->opts->hwdec_api != HWDEC_NONE) {
         if (ctx->hwdec) {
@@ -792,7 +795,8 @@ static struct mp_image *read_output(struct dec_video *vd, bool eof)
         ctx->hw_probing = false;
     }
 
-    return res;
+    *out_image = res;
+    return true;
 }
 
 static bool prepare_decoding(struct dec_video *vd)
@@ -878,14 +882,14 @@ static bool send_packet(struct dec_video *vd, struct demux_packet *pkt)
     return do_send_packet(vd, pkt);
 }
 
-// Returns EOF state.
+// Returns whether decoder is still active (!EOF state).
 static bool decode_frame(struct dec_video *vd)
 {
     vd_ffmpeg_ctx *ctx = vd->priv;
     AVCodecContext *avctx = ctx->avctx;
 
     if (!prepare_decoding(vd))
-        return false;
+        return true;
 
     hwdec_lock(ctx);
     int ret = avcodec_receive_frame(avctx, ctx->pic);
@@ -895,13 +899,13 @@ static bool decode_frame(struct dec_video *vd)
         // If flushing was initialized earlier and has ended now, make it start
         // over in case we get new packets at some point in the future.
         reset_avctx(vd);
-        return true;
+        return false;
     } else if (ret < 0 && ret != AVERROR(EAGAIN)) {
         handle_err(vd);
     }
 
     if (!ctx->pic->buf[0])
-        return false;
+        return true;
 
     ctx->hwdec_fail_count = 0;
 
@@ -917,7 +921,7 @@ static bool decode_frame(struct dec_video *vd)
     struct mp_image *mpi = mp_image_from_av_frame(ctx->pic);
     if (!mpi) {
         av_frame_unref(ctx->pic);
-        return false;
+        return true;
     }
     assert(mpi->planes[0] || mpi->planes[3]);
     mpi->pts = mp_pts_from_av(ctx->pic->pts, &ctx->codec_timebase);
@@ -935,14 +939,14 @@ static bool decode_frame(struct dec_video *vd)
     av_frame_unref(ctx->pic);
 
     MP_TARRAY_APPEND(ctx, ctx->delay_queue, ctx->num_delay_queue, mpi);
-    return false;
+    return true;
 }
 
-static struct mp_image *receive_frame(struct dec_video *vd)
+static bool receive_frame(struct dec_video *vd, struct mp_image **out_image)
 {
     vd_ffmpeg_ctx *ctx = vd->priv;
 
-    bool eof = decode_frame(vd);
+    bool progress = decode_frame(vd);
 
     if (ctx->hwdec_failed) {
         // Failed hardware decoding? Try again in software.
@@ -953,30 +957,11 @@ static struct mp_image *receive_frame(struct dec_video *vd)
 
         force_fallback(vd);
 
-        struct mp_image *img = NULL;
-
-        while (num_pkts > 0) {
-            if (send_packet(vd, pkts[0])) {
-                talloc_free(pkts[0]);
-                MP_TARRAY_REMOVE_AT(pkts, num_pkts, 0);
-            }
-            if (decode_frame(vd)) {
-                eof = true;
-                break;
-            }
-            img = read_output(vd, eof);
-            if (img)
-                break;
-        }
-
         ctx->requeue_packets = pkts;
         ctx->num_requeue_packets = num_pkts;
-
-        if (img)
-            return img;
     }
 
-    return read_output(vd, eof);
+    return read_output(vd, progress, out_image);
 }
 
 static int control(struct dec_video *vd, int cmd, void *arg)