video: pass future frames to VO

Now the VO can request a number of future frames with the last parameter
of vo_set_queue_params(). This will be helpful to fix the interpolation
code.

Note that the first frame (after playback start or seeking) will usually
not have any future frames (to make seeking fast). Near the end of the
file, the number of future frames will become lower as well.

5 files changed, 80 insertions, 10 deletions

diff --git a/video/out/vo.c b/video/out/vo.c
index 954a9a4c67..68af6a38c8 100644
--- a/video/out/vo.c
+++ b/video/out/vo.c
@@ -146,6 +146,9 @@ struct vo_internal {
 
     bool rendering;                 // true if an image is being rendered
     struct mp_image *frame_queued;  // the image that should be rendered
+    struct mp_image *future_frames[VO_MAX_FUTURE_FRAMES];
+    int num_future_frames;
+    int req_future_frames;          // VO's requested value of num_future_frames
     int64_t frame_pts;              // realtime of intended display
     int64_t frame_duration;         // realtime frame duration (for framedrop)
 
@@ -410,6 +413,23 @@ int vo_control(struct vo *vo, uint32_t request, void *data)
 }
 
 // must be called locked
+// transfers ownership of frames[] items to the VO
+static void set_future_frames(struct vo *vo, struct mp_image **frames)
+{
+    struct vo_internal *in = vo->in;
+    for (int n = 0; n < in->num_future_frames; n++)
+        talloc_free(in->future_frames[n]);
+    in->num_future_frames = 0;
+    for (int n = 0; frames && frames[n]; n++) {
+        if (n < in->req_future_frames) {
+            in->future_frames[in->num_future_frames++] = frames[n];
+        } else {
+            talloc_free(frames[n]);
+        }
+    }
+}
+
+// must be called locked
 static void forget_frames(struct vo *vo)
 {
     struct vo_internal *in = vo->in;
@@ -417,6 +437,7 @@ static void forget_frames(struct vo *vo)
     in->hasframe_rendered = false;
     in->drop_count = 0;
     mp_image_unrefp(&in->frame_queued);
+    set_future_frames(vo, NULL);
     // don't unref current_frame; we always want to be able to redraw it
 }
 
@@ -530,18 +551,22 @@ bool vo_is_ready_for_frame(struct vo *vo, int64_t next_pts)
 
 // Direct the VO thread to put the currently queued image on the screen.
 // vo_is_ready_for_frame() must have returned true before this call.
-// Ownership of the image is handed to the vo.
-void vo_queue_frame(struct vo *vo, struct mp_image *image,
+// images[0] is the frame to draw, images[n+1] are future frames (NULL
+// terminated). Ownership of all the images is handed to the vo.
+void vo_queue_frame(struct vo *vo, struct mp_image **images,
                     int64_t pts_us, int64_t duration)
 {
     struct vo_internal *in = vo->in;
     pthread_mutex_lock(&in->lock);
+    struct mp_image *image = images[0];
+    assert(image);
     assert(vo->config_ok && !in->frame_queued);
     in->hasframe = true;
     in->frame_queued = image;
     in->frame_pts = pts_us;
     in->frame_duration = duration;
     in->wakeup_pts = in->vsync_timed ? 0 : in->frame_pts + MPMAX(duration, 0);
+    set_future_frames(vo, images + 1);
     wakeup_locked(vo);
     pthread_mutex_unlock(&in->lock);
 }
@@ -674,6 +699,13 @@ static bool render_frame(struct vo *vo)
     } else {
         in->rendering = true;
         in->hasframe_rendered = true;
+        int num_future_frames = in->num_future_frames;
+        in->num_future_frames = 0;
+        struct mp_image *future_frames[VO_MAX_FUTURE_FRAMES];
+        for (int n = 0; n < num_future_frames; n++) {
+            future_frames[n] = in->future_frames[n];
+            in->future_frames[n] = NULL;
+        }
         pthread_mutex_unlock(&in->lock);
         mp_input_wakeup(vo->input_ctx); // core can queue new video now
 
@@ -684,6 +716,9 @@ static bool render_frame(struct vo *vo)
                 .pts        = pts,
                 .next_vsync = next_vsync,
                 .prev_vsync = prev_vsync,
+                .frame = img,
+                .num_future_frames = num_future_frames,
+                .future_frames = future_frames,
             };
             vo->driver->draw_image_timed(vo, img, &t);
         } else {
@@ -714,6 +749,8 @@ static bool render_frame(struct vo *vo)
         pthread_mutex_lock(&in->lock);
         in->dropped_frame = drop;
         in->rendering = false;
+        for (int n = 0; n < num_future_frames; n++)
+            talloc_free(future_frames[n]);
     }
 
     if (in->dropped_frame) {
@@ -947,14 +984,27 @@ const char *vo_get_window_title(struct vo *vo)
 // flip_page[_timed] will be called offset_us microseconds too early.
 // (For vo_vdpau, which does its own timing.)
 // Setting vsync_timed to true redraws as fast as possible.
+// num_future_frames set the requested number of future frames in
+// struct frame_timing.
 // (For vo_opengl smoothmotion.)
-void vo_set_flip_queue_params(struct vo *vo, int64_t offset_us, bool vsync_timed)
+void vo_set_queue_params(struct vo *vo, int64_t offset_us, bool vsync_timed,
+                         int num_future_frames)
 {
     struct vo_internal *in = vo->in;
     pthread_mutex_lock(&in->lock);
     in->flip_queue_offset = offset_us;
     in->vsync_timed = vsync_timed;
+    in->req_future_frames = MPMIN(num_future_frames, VO_MAX_FUTURE_FRAMES);
+    pthread_mutex_unlock(&in->lock);
+}
+
+int vo_get_num_future_frames(struct vo *vo)
+{
+    struct vo_internal *in = vo->in;
+    pthread_mutex_lock(&in->lock);
+    int res = in->req_future_frames;
     pthread_mutex_unlock(&in->lock);
+    return res;
 }
 
 // to be called from the VO thread only
diff --git a/video/out/vo.h b/video/out/vo.h
index 167c08a69c..51c7816920 100644
--- a/video/out/vo.h
+++ b/video/out/vo.h
@@ -140,6 +140,8 @@ struct voctrl_get_equalizer_args {
 // VO does framedrop itself (vo_vdpau). Untimed/encoding VOs never drop.
 #define VO_CAP_FRAMEDROP 2
 
+#define VO_MAX_FUTURE_FRAMES 10
+
 struct vo;
 struct osd_state;
 struct mp_image;
@@ -153,9 +155,26 @@ struct vo_extra {
 };
 
 struct frame_timing {
+    // If > 0, realtime when frame should be shown, in mp_time_us() units.
     int64_t pts;
+    // Realtime of estimated previous and next vsync events.
     int64_t next_vsync;
     int64_t prev_vsync;
+    // The current frame to be drawn. NULL means redraw previous frame
+    // (e.g. repeated frames).
+    // (Equivalent to the mp_image parameter of draw_image_timed, until the
+    // parameter is removed.)
+    struct mp_image *frame;
+    // List of future images, starting with the next one. This does not
+    // care about repeated frames - it simply contains the next real frames.
+    // vo_set_queue_params() sets how many frames this should include, though
+    // the actual number can be lower.
+    // future_frames[0] is the next frame.
+    // Note that this has frames only when a new real frame is pushed. Redraw
+    // calls or repeated frames do not include this.
+    // Ownership of the frames belongs to the caller.
+    int num_future_frames;
+    struct mp_image **future_frames;
 };
 
 struct vo_driver {
@@ -302,7 +321,7 @@ int vo_reconfig(struct vo *vo, struct mp_image_params *p, int flags);
 
 int vo_control(struct vo *vo, uint32_t request, void *data);
 bool vo_is_ready_for_frame(struct vo *vo, int64_t next_pts);
-void vo_queue_frame(struct vo *vo, struct mp_image *image,
+void vo_queue_frame(struct vo *vo, struct mp_image **images,
                     int64_t pts_us, int64_t duration);
 void vo_wait_frame(struct vo *vo);
 bool vo_still_displaying(struct vo *vo);
@@ -318,8 +337,9 @@ void vo_query_formats(struct vo *vo, uint8_t *list);
 void vo_event(struct vo *vo, int event);
 int vo_query_and_reset_events(struct vo *vo, int events);
 struct mp_image *vo_get_current_frame(struct vo *vo);
-
-void vo_set_flip_queue_params(struct vo *vo, int64_t offset_us, bool vsync_timed);
+void vo_set_queue_params(struct vo *vo, int64_t offset_us, bool vsync_timed,
+                         int num_future_frames);
+int vo_get_num_future_frames(struct vo *vo);
 int64_t vo_get_vsync_interval(struct vo *vo);
 double vo_get_display_fps(struct vo *vo);
 
diff --git a/video/out/vo_opengl.c b/video/out/vo_opengl.c
index 5b08c21768..981e73ffcb 100644
--- a/video/out/vo_opengl.c
+++ b/video/out/vo_opengl.c
@@ -304,7 +304,7 @@ static bool reparse_cmdline(struct gl_priv *p, char *args)
     if (r >= 0) {
         int queue = 0;
         gl_video_set_options(p->renderer, opts->renderer_opts, &queue);
-        vo_set_flip_queue_params(p->vo, queue, opts->renderer_opts->interpolation);
+        vo_set_queue_params(p->vo, queue, opts->renderer_opts->interpolation, 1);
         p->vo->want_redraw = true;
     }
 
@@ -443,7 +443,7 @@ static int preinit(struct vo *vo)
                               p->glctx->depth_b);
     int queue = 0;
     gl_video_set_options(p->renderer, p->renderer_opts, &queue);
-    vo_set_flip_queue_params(p->vo, queue, p->renderer_opts->interpolation);
+    vo_set_queue_params(p->vo, queue, p->renderer_opts->interpolation, 0);
 
     p->cms = gl_lcms_init(p, vo->log, vo->global);
     if (!p->cms)
diff --git a/video/out/vo_opengl_cb.c b/video/out/vo_opengl_cb.c
index b3dc5ca84f..efc2991ba7 100644
--- a/video/out/vo_opengl_cb.c
+++ b/video/out/vo_opengl_cb.c
@@ -335,7 +335,7 @@ int mpv_opengl_cb_draw(mpv_opengl_cb_context *ctx, int fbo, int vp_w, int vp_h)
             ctx->vsync_timed = opts->renderer_opts->interpolation;
             if (ctx->vsync_timed)
                 queue += 0.050 * 1e6; // disable video timing
-            vo_set_flip_queue_params(vo, queue, false);
+            vo_set_queue_params(vo, queue, false, 0);
             ctx->gl->debug_context = opts->use_gl_debug;
             gl_video_set_debug(ctx->renderer, opts->use_gl_debug);
             frame_queue_shrink(ctx, opts->frame_queue_size);
diff --git a/video/out/vo_vdpau.c b/video/out/vo_vdpau.c
index 26ff5454dc..f326a62262 100644
--- a/video/out/vo_vdpau.c
+++ b/video/out/vo_vdpau.c
@@ -249,7 +249,7 @@ static void resize(struct vo *vo)
     vc->flip_offset_us = vo->opts->fullscreen ?
                          1000LL * vc->flip_offset_fs :
                          1000LL * vc->flip_offset_window;
-    vo_set_flip_queue_params(vo, vc->flip_offset_us, false);
+    vo_set_queue_params(vo, vc->flip_offset_us, false, 0);
 
     if (vc->output_surface_w < vo->dwidth || vc->output_surface_h < vo->dheight) {
         vc->output_surface_w = s_size(max_w, vc->output_surface_w, vo->dwidth);