video: pass future frames to VO

Now the VO can request a number of future frames with the last parameter
of vo_set_queue_params(). This will be helpful to fix the interpolation
code.

Note that the first frame (after playback start or seeking) will usually
not have any future frames (to make seeking fast). Near the end of the
file, the number of future frames will become lower as well.

7 files changed, 142 insertions, 44 deletions

diff --git a/player/core.h b/player/core.h
index 81cf613abf..b3e8018b12 100644
--- a/player/core.h
+++ b/player/core.h
@@ -26,6 +26,7 @@
 #include "options/options.h"
 #include "sub/osd.h"
 #include "demux/timeline.h"
+#include "video/out/vo.h"
 
 // definitions used internally by the core player code
 
@@ -227,7 +228,8 @@ typedef struct MPContext {
 
     struct vo *video_out;
     // next_frame[0] is the next frame, next_frame[1] the one after that.
-    struct mp_image *next_frame[2];
+    struct mp_image *next_frames[2 + VO_MAX_FUTURE_FRAMES];
+    int num_next_frames;
     struct mp_image *saved_frame;   // for hrseek_lastframe
 
     enum playback_status video_status, audio_status;
diff --git a/player/video.c b/player/video.c
index 42c184a1b3..da93203b7a 100644
--- a/player/video.c
+++ b/player/video.c
@@ -195,8 +195,9 @@ void reset_video_state(struct MPContext *mpctx)
     if (mpctx->video_out)
         vo_seek_reset(mpctx->video_out);
 
-    mp_image_unrefp(&mpctx->next_frame[0]);
-    mp_image_unrefp(&mpctx->next_frame[1]);
+    for (int n = 0; n < mpctx->num_next_frames; n++)
+        mp_image_unrefp(&mpctx->next_frames[n]);
+    mpctx->num_next_frames = 0;
     mp_image_unrefp(&mpctx->saved_frame);
 
     mpctx->delay = 0;
@@ -541,19 +542,15 @@ static void adjust_sync(struct MPContext *mpctx, double v_pts, double frame_time
     mpctx->total_avsync_change += change;
 }
 
-// Move the frame in next_frame[1] to next_frame[0]. This makes the frame
-// "known" to the playback logic. A frame in next_frame[0] is either "known" or
-// NULL, so the moving must always be done by this function.
-static void shift_new_frame(struct MPContext *mpctx)
+// Make the frame at position 0 "known" to the playback logic. This must happen
+// only once for each frame, so this function has to be called carefully.
+// Generally, if position 0 gets a new frame, this must be called.
+static void handle_new_frame(struct MPContext *mpctx)
 {
-    if (mpctx->next_frame[0] || !mpctx->next_frame[1])
-        return;
-
-    mpctx->next_frame[0] = mpctx->next_frame[1];
-    mpctx->next_frame[1] = NULL;
+    assert(mpctx->num_next_frames >= 1);
 
     double frame_time = 0;
-    double pts = mpctx->next_frame[0]->pts;
+    double pts = mpctx->next_frames[0]->pts;
     if (mpctx->video_pts != MP_NOPTS_VALUE) {
         frame_time = pts - mpctx->video_pts;
         double tolerance = 15;
@@ -585,33 +582,45 @@ static void shift_new_frame(struct MPContext *mpctx)
     MP_TRACE(mpctx, "frametime=%5.3f\n", frame_time);
 }
 
+static int get_req_frames(struct MPContext *mpctx, bool eof)
+{
+    struct MPOpts *opts = mpctx->opts;
+
+    // On EOF, drain all frames.
+    // On the first frame, output a new frame as quickly as possible.
+    if (eof || mpctx->video_pts == MP_NOPTS_VALUE)
+        return 1;
+
+    int req = 1 + vo_get_num_future_frames(mpctx->video_out);
+    if (opts->frame_dropping & 1)
+        req = MPMAX(req, 2);
+    return req;
+}
+
 // Whether it's fine to call add_new_frame() now.
 static bool needs_new_frame(struct MPContext *mpctx)
 {
-    return !mpctx->next_frame[1];
+    return mpctx->num_next_frames < get_req_frames(mpctx, false);
 }
 
-// Queue a frame to mpctx->next_frame[]. Call only if needs_new_frame() signals ok.
+// Queue a frame to mpctx->next_frames[]. Call only if needs_new_frame() signals ok.
 static void add_new_frame(struct MPContext *mpctx, struct mp_image *frame)
 {
     assert(needs_new_frame(mpctx));
     assert(frame);
-    mpctx->next_frame[1] = frame;
-    shift_new_frame(mpctx);
+    mpctx->next_frames[mpctx->num_next_frames++] = frame;
+    if (mpctx->num_next_frames == 1)
+        handle_new_frame(mpctx);
 }
 
 // Enough video filtered already to push one frame to the VO?
 // Set eof to true if no new frames are to be expected.
 static bool have_new_frame(struct MPContext *mpctx, bool eof)
 {
-    bool need_2nd = !!(mpctx->opts->frame_dropping & 1) // we need the duration
-        && mpctx->video_pts != MP_NOPTS_VALUE   // ...except for the 1st frame
-        && !eof;    // on EOF, drain the remaining frames
-
-    return mpctx->next_frame[0] && (!need_2nd || mpctx->next_frame[1]);
+    return mpctx->num_next_frames >= get_req_frames(mpctx, eof);
 }
 
-// Fill mpctx->next_frame[] with a newly filtered or decoded image.
+// Fill mpctx->next_frames[] with a newly filtered or decoded image.
 // returns VD_* code
 static int video_output_image(struct MPContext *mpctx, double endpts)
 {
@@ -620,13 +629,15 @@ static int video_output_image(struct MPContext *mpctx, double endpts)
     if (mpctx->d_video->header->attached_picture) {
         if (vo_has_frame(mpctx->video_out))
             return VD_EOF;
-        if (mpctx->next_frame[0])
+        if (mpctx->num_next_frames >= 1)
             return VD_NEW_FRAME;
         int r = video_decode_and_filter(mpctx);
         video_filter(mpctx, true); // force EOF filtering (avoid decoding more)
-        mpctx->next_frame[0] = vf_read_output_frame(mpctx->d_video->vfilter);
-        if (mpctx->next_frame[0])
-            mpctx->next_frame[0]->pts = MP_NOPTS_VALUE;
+        mpctx->next_frames[0] = vf_read_output_frame(mpctx->d_video->vfilter);
+        if (mpctx->next_frames[0]) {
+            mpctx->next_frames[0]->pts = MP_NOPTS_VALUE;
+            mpctx->num_next_frames = 1;
+        }
         return r <= 0 ? VD_EOF : VD_PROGRESS;
     }
 
@@ -802,7 +813,7 @@ void write_video(struct MPContext *mpctx, double endpts)
     }
 
     // Filter output is different from VO input?
-    struct mp_image_params p = mpctx->next_frame[0]->params;
+    struct mp_image_params p = mpctx->next_frames[0]->params;
     if (!vo->params || !mp_image_params_equal(&p, vo->params)) {
         // Changing config deletes the current frame; wait until it's finished.
         if (vo_still_displaying(vo))
@@ -839,8 +850,11 @@ void write_video(struct MPContext *mpctx, double endpts)
 
     int64_t duration = -1;
     double diff = -1;
-    double vpts0 = mpctx->next_frame[0] ? mpctx->next_frame[0]->pts : MP_NOPTS_VALUE;
-    double vpts1 = mpctx->next_frame[1] ? mpctx->next_frame[1]->pts : MP_NOPTS_VALUE;
+    assert(mpctx->num_next_frames >= 1);
+    double vpts0 = mpctx->next_frames[0]->pts;
+    double vpts1 = MP_NOPTS_VALUE;
+    if (mpctx->num_next_frames >= 2)
+        vpts1 = mpctx->next_frames[1]->pts;
     if (vpts0 != MP_NOPTS_VALUE && vpts1 != MP_NOPTS_VALUE)
         diff = vpts1 - vpts0;
     if (diff < 0 && mpctx->d_video->fps > 0)
@@ -855,7 +869,7 @@ void write_video(struct MPContext *mpctx, double endpts)
         duration = MPCLAMP(diff, 0, 10) * 1e6;
     }
 
-    mpctx->video_pts = mpctx->next_frame[0]->pts;
+    mpctx->video_pts = mpctx->next_frames[0]->pts;
     mpctx->last_vo_pts = mpctx->video_pts;
     mpctx->playback_pts = mpctx->video_pts;
 
@@ -865,10 +879,22 @@ void write_video(struct MPContext *mpctx, double endpts)
     update_osd_msg(mpctx);
     update_subtitles(mpctx);
 
-    vo_queue_frame(vo, mpctx->next_frame[0], pts, duration);
-    mpctx->next_frame[0] = NULL;
+    assert(mpctx->num_next_frames >= 1);
+    struct mp_image *frames[VO_MAX_FUTURE_FRAMES + 2] = {0};
+    frames[0] = mpctx->next_frames[0];
+    for (int n = 0; n < mpctx->num_next_frames - 1; n++)
+        mpctx->next_frames[n] = mpctx->next_frames[n + 1];
+    mpctx->num_next_frames -= 1;
+    for (int n = 0; n < mpctx->num_next_frames && n < VO_MAX_FUTURE_FRAMES; n++) {
+        frames[n + 1] = mp_image_new_ref(mpctx->next_frames[n]);
+        if (!frames[n + 1])
+            break; // OOM
+    }
+    vo_queue_frame(vo, frames, pts, duration);
 
-    shift_new_frame(mpctx);
+    // The frames were shifted down; "initialize" the new first entry.
+    if (mpctx->num_next_frames >= 1)
+        handle_new_frame(mpctx);
 
     mpctx->shown_vframes++;
     if (mpctx->video_status < STATUS_PLAYING) {
diff --git a/video/out/vo.c b/video/out/vo.c
index 954a9a4c67..68af6a38c8 100644
--- a/video/out/vo.c
+++ b/video/out/vo.c
@@ -146,6 +146,9 @@ struct vo_internal {
 
     bool rendering;                 // true if an image is being rendered
     struct mp_image *frame_queued;  // the image that should be rendered
+    struct mp_image *future_frames[VO_MAX_FUTURE_FRAMES];
+    int num_future_frames;
+    int req_future_frames;          // VO's requested value of num_future_frames
     int64_t frame_pts;              // realtime of intended display
     int64_t frame_duration;         // realtime frame duration (for framedrop)
 
@@ -410,6 +413,23 @@ int vo_control(struct vo *vo, uint32_t request, void *data)
 }
 
 // must be called locked
+// transfers ownership of frames[] items to the VO
+static void set_future_frames(struct vo *vo, struct mp_image **frames)
+{
+    struct vo_internal *in = vo->in;
+    for (int n = 0; n < in->num_future_frames; n++)
+        talloc_free(in->future_frames[n]);
+    in->num_future_frames = 0;
+    for (int n = 0; frames && frames[n]; n++) {
+        if (n < in->req_future_frames) {
+            in->future_frames[in->num_future_frames++] = frames[n];
+        } else {
+            talloc_free(frames[n]);
+        }
+    }
+}
+
+// must be called locked
 static void forget_frames(struct vo *vo)
 {
     struct vo_internal *in = vo->in;
@@ -417,6 +437,7 @@ static void forget_frames(struct vo *vo)
     in->hasframe_rendered = false;
     in->drop_count = 0;
     mp_image_unrefp(&in->frame_queued);
+    set_future_frames(vo, NULL);
     // don't unref current_frame; we always want to be able to redraw it
 }
 
@@ -530,18 +551,22 @@ bool vo_is_ready_for_frame(struct vo *vo, int64_t next_pts)
 
 // Direct the VO thread to put the currently queued image on the screen.
 // vo_is_ready_for_frame() must have returned true before this call.
-// Ownership of the image is handed to the vo.
-void vo_queue_frame(struct vo *vo, struct mp_image *image,
+// images[0] is the frame to draw, images[n+1] are future frames (NULL
+// terminated). Ownership of all the images is handed to the vo.
+void vo_queue_frame(struct vo *vo, struct mp_image **images,
                     int64_t pts_us, int64_t duration)
 {
     struct vo_internal *in = vo->in;
     pthread_mutex_lock(&in->lock);
+    struct mp_image *image = images[0];
+    assert(image);
     assert(vo->config_ok && !in->frame_queued);
     in->hasframe = true;
     in->frame_queued = image;
     in->frame_pts = pts_us;
     in->frame_duration = duration;
     in->wakeup_pts = in->vsync_timed ? 0 : in->frame_pts + MPMAX(duration, 0);
+    set_future_frames(vo, images + 1);
     wakeup_locked(vo);
     pthread_mutex_unlock(&in->lock);
 }
@@ -674,6 +699,13 @@ static bool render_frame(struct vo *vo)
     } else {
         in->rendering = true;
         in->hasframe_rendered = true;
+        int num_future_frames = in->num_future_frames;
+        in->num_future_frames = 0;
+        struct mp_image *future_frames[VO_MAX_FUTURE_FRAMES];
+        for (int n = 0; n < num_future_frames; n++) {
+            future_frames[n] = in->future_frames[n];
+            in->future_frames[n] = NULL;
+        }
         pthread_mutex_unlock(&in->lock);
         mp_input_wakeup(vo->input_ctx); // core can queue new video now
 
@@ -684,6 +716,9 @@ static bool render_frame(struct vo *vo)
                 .pts        = pts,
                 .next_vsync = next_vsync,
                 .prev_vsync = prev_vsync,
+                .frame = img,
+                .num_future_frames = num_future_frames,
+                .future_frames = future_frames,
             };
             vo->driver->draw_image_timed(vo, img, &t);
         } else {
@@ -714,6 +749,8 @@ static bool render_frame(struct vo *vo)
         pthread_mutex_lock(&in->lock);
         in->dropped_frame = drop;
         in->rendering = false;
+        for (int n = 0; n < num_future_frames; n++)
+            talloc_free(future_frames[n]);
     }
 
     if (in->dropped_frame) {
@@ -947,14 +984,27 @@ const char *vo_get_window_title(struct vo *vo)
 // flip_page[_timed] will be called offset_us microseconds too early.
 // (For vo_vdpau, which does its own timing.)
 // Setting vsync_timed to true redraws as fast as possible.
+// num_future_frames set the requested number of future frames in
+// struct frame_timing.
 // (For vo_opengl smoothmotion.)
-void vo_set_flip_queue_params(struct vo *vo, int64_t offset_us, bool vsync_timed)
+void vo_set_queue_params(struct vo *vo, int64_t offset_us, bool vsync_timed,
+                         int num_future_frames)
 {
     struct vo_internal *in = vo->in;
     pthread_mutex_lock(&in->lock);
     in->flip_queue_offset = offset_us;
     in->vsync_timed = vsync_timed;
+    in->req_future_frames = MPMIN(num_future_frames, VO_MAX_FUTURE_FRAMES);
+    pthread_mutex_unlock(&in->lock);
+}
+
+int vo_get_num_future_frames(struct vo *vo)
+{
+    struct vo_internal *in = vo->in;
+    pthread_mutex_lock(&in->lock);
+    int res = in->req_future_frames;
     pthread_mutex_unlock(&in->lock);
+    return res;
 }
 
 // to be called from the VO thread only
diff --git a/video/out/vo.h b/video/out/vo.h
index 167c08a69c..51c7816920 100644
--- a/video/out/vo.h
+++ b/video/out/vo.h
@@ -140,6 +140,8 @@ struct voctrl_get_equalizer_args {
 // VO does framedrop itself (vo_vdpau). Untimed/encoding VOs never drop.
 #define VO_CAP_FRAMEDROP 2
 
+#define VO_MAX_FUTURE_FRAMES 10
+
 struct vo;
 struct osd_state;
 struct mp_image;
@@ -153,9 +155,26 @@ struct vo_extra {
 };
 
 struct frame_timing {
+    // If > 0, realtime when frame should be shown, in mp_time_us() units.
     int64_t pts;
+    // Realtime of estimated previous and next vsync events.
     int64_t next_vsync;
     int64_t prev_vsync;
+    // The current frame to be drawn. NULL means redraw previous frame
+    // (e.g. repeated frames).
+    // (Equivalent to the mp_image parameter of draw_image_timed, until the
+    // parameter is removed.)
+    struct mp_image *frame;
+    // List of future images, starting with the next one. This does not
+    // care about repeated frames - it simply contains the next real frames.
+    // vo_set_queue_params() sets how many frames this should include, though
+    // the actual number can be lower.
+    // future_frames[0] is the next frame.
+    // Note that this has frames only when a new real frame is pushed. Redraw
+    // calls or repeated frames do not include this.
+    // Ownership of the frames belongs to the caller.
+    int num_future_frames;
+    struct mp_image **future_frames;
 };
 
 struct vo_driver {
@@ -302,7 +321,7 @@ int vo_reconfig(struct vo *vo, struct mp_image_params *p, int flags);
 
 int vo_control(struct vo *vo, uint32_t request, void *data);
 bool vo_is_ready_for_frame(struct vo *vo, int64_t next_pts);
-void vo_queue_frame(struct vo *vo, struct mp_image *image,
+void vo_queue_frame(struct vo *vo, struct mp_image **images,
                     int64_t pts_us, int64_t duration);
 void vo_wait_frame(struct vo *vo);
 bool vo_still_displaying(struct vo *vo);
@@ -318,8 +337,9 @@ void vo_query_formats(struct vo *vo, uint8_t *list);
 void vo_event(struct vo *vo, int event);
 int vo_query_and_reset_events(struct vo *vo, int events);
 struct mp_image *vo_get_current_frame(struct vo *vo);
-
-void vo_set_flip_queue_params(struct vo *vo, int64_t offset_us, bool vsync_timed);
+void vo_set_queue_params(struct vo *vo, int64_t offset_us, bool vsync_timed,
+                         int num_future_frames);
+int vo_get_num_future_frames(struct vo *vo);
 int64_t vo_get_vsync_interval(struct vo *vo);
 double vo_get_display_fps(struct vo *vo);
 
diff --git a/video/out/vo_opengl.c b/video/out/vo_opengl.c
index 5b08c21768..981e73ffcb 100644
--- a/video/out/vo_opengl.c
+++ b/video/out/vo_opengl.c
@@ -304,7 +304,7 @@ static bool reparse_cmdline(struct gl_priv *p, char *args)
     if (r >= 0) {
         int queue = 0;
         gl_video_set_options(p->renderer, opts->renderer_opts, &queue);
-        vo_set_flip_queue_params(p->vo, queue, opts->renderer_opts->interpolation);
+        vo_set_queue_params(p->vo, queue, opts->renderer_opts->interpolation, 1);
         p->vo->want_redraw = true;
     }
 
@@ -443,7 +443,7 @@ static int preinit(struct vo *vo)
                               p->glctx->depth_b);
     int queue = 0;
     gl_video_set_options(p->renderer, p->renderer_opts, &queue);
-    vo_set_flip_queue_params(p->vo, queue, p->renderer_opts->interpolation);
+    vo_set_queue_params(p->vo, queue, p->renderer_opts->interpolation, 0);
 
     p->cms = gl_lcms_init(p, vo->log, vo->global);
     if (!p->cms)
diff --git a/video/out/vo_opengl_cb.c b/video/out/vo_opengl_cb.c
index b3dc5ca84f..efc2991ba7 100644
--- a/video/out/vo_opengl_cb.c
+++ b/video/out/vo_opengl_cb.c
@@ -335,7 +335,7 @@ int mpv_opengl_cb_draw(mpv_opengl_cb_context *ctx, int fbo, int vp_w, int vp_h)
             ctx->vsync_timed = opts->renderer_opts->interpolation;
             if (ctx->vsync_timed)
                 queue += 0.050 * 1e6; // disable video timing
-            vo_set_flip_queue_params(vo, queue, false);
+            vo_set_queue_params(vo, queue, false, 0);
             ctx->gl->debug_context = opts->use_gl_debug;
             gl_video_set_debug(ctx->renderer, opts->use_gl_debug);
             frame_queue_shrink(ctx, opts->frame_queue_size);
diff --git a/video/out/vo_vdpau.c b/video/out/vo_vdpau.c
index 26ff5454dc..f326a62262 100644
--- a/video/out/vo_vdpau.c
+++ b/video/out/vo_vdpau.c
@@ -249,7 +249,7 @@ static void resize(struct vo *vo)
     vc->flip_offset_us = vo->opts->fullscreen ?
                          1000LL * vc->flip_offset_fs :
                          1000LL * vc->flip_offset_window;
-    vo_set_flip_queue_params(vo, vc->flip_offset_us, false);
+    vo_set_queue_params(vo, vc->flip_offset_us, false, 0);
 
     if (vc->output_surface_w < vo->dwidth || vc->output_surface_h < vo->dheight) {
         vc->output_surface_w = s_size(max_w, vc->output_surface_w, vo->dwidth);