core/VO: Allow VO drivers to add/modify frames

Add interfaces to allow VO drivers to add or remove frames from the
video stream and to alter timestamps. Currently this functionality
only works with in correct-pts mode. Use the new functionality in
vo_vdpau to properly support frame-adding deinterlace modes.

Frames added by the VDPAU deinterlacing code are now properly timed.
Before every second frame was always shown immediately (probably next
monitor refresh) after the previous one, even if you were watching
things in slow motion, and framestepping didn't stop at them at all.
When seeking the deinterlace algorithm is no longer fed a mix of
frames from old and new positions.

As a side effect of the changes a problem with resize events was also
fixed. Resizing calls video_to_output_surface() to render the frame at
the new resolution, but before this function also changed the list of
history frames, so resizing could give an image different from the
original one, and also corrupt next frames due to them seeing the
wrong history. Now the function has no such side effects. There are
more resize-related problems though that will be fixed in a later
commit.

The deint_mpi[] list of reserved frames is increased from 2 to 3
entries for reasons related to the above. Having 2 entries is enough
when you initially get a new frame in draw_image() because then you'll
have those two entries plus the new one for a total of 3 (the code
relied on the oldest mpi implicitly staying reserved for the duration
of the call even after usage count was decreased). However if you want
to be able to reproduce the rendering outside draw_image(), relying on
the explicitly reserved list only, then it needs to store 3 entries.

3 files changed, 178 insertions, 72 deletions

diff --git a/libvo/video_out.c b/libvo/video_out.c
index 9224212b35..b6506406c3 100644
--- a/libvo/video_out.c
+++ b/libvo/video_out.c
@@ -22,6 +22,7 @@
 #include <stdlib.h>
 #include <string.h>
 #include <assert.h>
+#include <stdbool.h>
 
 #include <unistd.h>
 //#include <sys/mman.h>
@@ -278,6 +279,35 @@ int vo_control(struct vo *vo, uint32_t request, void *data)
     return vo->driver->control(vo, request, data);
 }
 
+// Return -1 if driver appears not to support a draw_image interface,
+// 0 otherwise (whether the driver actually drew something or not).
+int vo_draw_image(struct vo *vo, struct mp_image *mpi, double pts)
+{
+    if (!vo->config_ok)
+        return 0;
+    if (vo->driver->buffer_frames) {
+        vo->driver->draw_image(vo, mpi, pts);
+        return 0;
+    }
+    vo->frame_loaded = true;
+    vo->next_pts = pts;
+    if (vo_control(vo, VOCTRL_DRAW_IMAGE, mpi) == VO_NOTIMPL)
+        return -1;
+    return 0;
+}
+
+int vo_get_buffered_frame(struct vo *vo, bool eof)
+{
+    if (!vo->config_ok)
+        return -1;
+    if (vo->frame_loaded)
+        return 0;
+    if (!vo->driver->buffer_frames)
+        return -1;
+    vo->driver->get_buffered_frame(vo, eof);
+    return vo->frame_loaded ? 0 : -1;
+}
+
 int vo_draw_frame(struct vo *vo, uint8_t *src[])
 {
     assert(!vo->driver->is_new);
@@ -302,6 +332,8 @@ void vo_flip_page(struct vo *vo)
 {
     if (!vo->config_ok)
         return;
+    vo->frame_loaded = false;
+    vo->next_pts = (-1LL<<63); // MP_NOPTS_VALUE
     vo->driver->flip_page(vo);
 }
 
@@ -312,6 +344,14 @@ void vo_check_events(struct vo *vo)
     vo->driver->check_events(vo);
 }
 
+void vo_seek_reset(struct vo *vo)
+{
+    if (!vo->config_ok)
+        return;
+    vo_control(vo, VOCTRL_RESET, NULL);
+    vo->frame_loaded = false;
+}
+
 void vo_destroy(struct vo *vo)
 {
     vo->driver->uninit(vo);
diff --git a/libvo/video_out.h b/libvo/video_out.h
index a911bfdd03..4ddeaa827e 100644
--- a/libvo/video_out.h
+++ b/libvo/video_out.h
@@ -129,10 +129,14 @@ typedef struct vo_info_s
 
 struct vo;
 struct osd_state;
+struct mp_image;
 
 struct vo_driver {
     // Driver uses new API
     bool is_new;
+    // Driver buffers or adds (deinterlace) frames and will keep track
+    // of pts values itself
+    bool buffer_frames;
 
     // This is set if the driver is not new and contains pointers to
     // old-API functions to be used instead of the ones below.
@@ -164,6 +168,16 @@ struct vo_driver {
      */
     int (*control)(struct vo *vo, uint32_t request, void *data);
 
+    void (*draw_image)(struct vo *vo, struct mp_image *mpi, double pts);
+
+    /*
+     * Get extra frames from the VO, such as those added by VDPAU
+     * deinterlace. Preparing the next such frame if any could be done
+     * automatically by the VO after a previous flip_page(), but having
+     * it as a separate step seems to allow making code more robust.
+     */
+    void (*get_buffered_frame)(struct vo *vo, bool eof);
+
     /*
      * Draw a planar YUV slice to the buffer:
      * params:
@@ -214,6 +228,10 @@ struct vo_old_functions {
 struct vo {
     int config_ok;  // Last config call was successful?
     int config_count;  // Total number of successful config calls
+
+    bool frame_loaded;  // Is there a next frame the VO could flip to?
+    double next_pts;    // pts value of the next frame if any
+
     const struct vo_driver *driver;
     void *priv;
     struct MPOpts *opts;
@@ -251,11 +269,14 @@ int vo_config(struct vo *vo, uint32_t width, uint32_t height,
 void list_video_out(void);
 
 int vo_control(struct vo *vo, uint32_t request, void *data);
+int vo_draw_image(struct vo *vo, struct mp_image *mpi, double pts);
+int vo_get_buffered_frame(struct vo *vo, bool eof);
 int vo_draw_frame(struct vo *vo, uint8_t *src[]);
 int vo_draw_slice(struct vo *vo, uint8_t *src[], int stride[], int w, int h, int x, int y);
 void vo_draw_osd(struct vo *vo, struct osd_state *osd);
 void vo_flip_page(struct vo *vo);
 void vo_check_events(struct vo *vo);
+void vo_seek_reset(struct vo *vo);
 void vo_destroy(struct vo *vo);
 
 
diff --git a/libvo/vo_vdpau.c b/libvo/vo_vdpau.c
index dcd2a7092e..bf760750cc 100644
--- a/libvo/vo_vdpau.c
+++ b/libvo/vo_vdpau.c
@@ -28,6 +28,7 @@
  */
 
 #include <stdio.h>
+#include <stdlib.h>
 #include <dlfcn.h>
 #include <stdint.h>
 #include <stdbool.h>
@@ -110,14 +111,15 @@ struct vdpctx {
 #define osd_surface vc->output_surfaces[NUM_OUTPUT_SURFACES]
     VdpOutputSurface                   output_surfaces[NUM_OUTPUT_SURFACES + 1];
     VdpVideoSurface                    deint_surfaces[3];
-    mp_image_t                        *deint_mpi[2];
+    double                             deint_pts[3];
+    int                                deint_queue_pos;
+    mp_image_t                        *deint_mpi[3];
     int                                output_surface_width, output_surface_height;
 
     VdpVideoMixer                      video_mixer;
     int                                deint;
     int                                deint_type;
     int                                deint_counter;
-    int                                deint_buffer_past_frames;
     int                                pullup;
     float                              denoise;
     float                              sharpen;
@@ -133,7 +135,6 @@ struct vdpctx {
 
     struct vdpau_render_state          surface_render[MAX_VIDEO_SURFACES];
     int                                surface_num;
-    int                                vid_surface_num;
     uint32_t                           vid_width, vid_height;
     uint32_t                           image_format;
     VdpChromaType                      vdp_chroma_type;
@@ -176,14 +177,6 @@ struct vdpctx {
 };
 
 
-static void push_deint_surface(struct vo *vo, VdpVideoSurface surface)
-{
-    struct vdpctx *vc = vo->priv;
-    vc->deint_surfaces[2] = vc->deint_surfaces[1];
-    vc->deint_surfaces[1] = vc->deint_surfaces[0];
-    vc->deint_surfaces[0] = surface;
-}
-
 static void flip_page(struct vo *vo);
 static void video_to_output_surface(struct vo *vo)
 {
@@ -191,41 +184,81 @@ static void video_to_output_surface(struct vo *vo)
     struct vdp_functions *vdp = vc->vdp;
     VdpTime dummy;
     VdpStatus vdp_st;
-    int i;
-    if (vc->vid_surface_num < 0)
+    if (vc->deint_queue_pos < 0)
         return;
 
-    if (vc->deint < 2 || vc->deint_surfaces[0] == VDP_INVALID_HANDLE)
-        push_deint_surface(vo, vc->surface_render[vc->vid_surface_num].surface);
+    int field = VDP_VIDEO_MIXER_PICTURE_STRUCTURE_FRAME;
+    unsigned int dp = vc->deint_queue_pos;
+    // dp==0 means last field of latest frame, 1 earlier field of latest frame,
+    // 2 last field of previous frame and so on
+    if (vc->deint) {
+        field = vc->top_field_first ^ (dp & 1) ?
+            VDP_VIDEO_MIXER_PICTURE_STRUCTURE_BOTTOM_FIELD:
+            VDP_VIDEO_MIXER_PICTURE_STRUCTURE_TOP_FIELD;
+    }
+    VdpVideoSurface *q = vc->deint_surfaces;
+    const VdpVideoSurface *past_fields = (const VdpVideoSurface []){
+        q[(dp+1)/2], q[(dp+2)/2]};
+    const VdpVideoSurface *future_fields = (const VdpVideoSurface []){
+        q[(dp-1)/2]};
+    VdpOutputSurface output_surface = vc->output_surfaces[vc->surface_num];
+    vdp_st = vdp->presentation_queue_block_until_surface_idle(vc->flip_queue,
+                                                              output_surface,
+                                                              &dummy);
+    CHECK_ST_WARNING("Error when calling "
+                     "vdp_presentation_queue_block_until_surface_idle");
+
+    vdp_st = vdp->video_mixer_render(vc->video_mixer, VDP_INVALID_HANDLE,
+                                     0, field, 2, past_fields,
+                                     vc->deint_surfaces[dp/2], 1, future_fields,
+                                     &vc->src_rect_vid, output_surface,
+                                     NULL, &vc->out_rect_vid, 0, NULL);
+    CHECK_ST_WARNING("Error when calling vdp_video_mixer_render");
+}
 
-    for (i = 0; i <= (vc->deint > 1); i++) {
-        int field = VDP_VIDEO_MIXER_PICTURE_STRUCTURE_FRAME;
-        VdpOutputSurface output_surface;
-        if (i) {
-            // draw_eosd()
-            // draw_osd()
-            flip_page(vo);
-        }
-        if (vc->deint)
-            field = (vc->top_field_first == i) ^ (vc->deint > 1) ?
-                VDP_VIDEO_MIXER_PICTURE_STRUCTURE_BOTTOM_FIELD:
-                VDP_VIDEO_MIXER_PICTURE_STRUCTURE_TOP_FIELD;
-        output_surface = vc->output_surfaces[vc->surface_num];
-        vdp_st = vdp->
-            presentation_queue_block_until_surface_idle(vc->flip_queue,
-                                                        output_surface,
-                                                        &dummy);
-        CHECK_ST_WARNING("Error when calling "
-                         "vdp_presentation_queue_block_until_surface_idle");
-
-        vdp_st = vdp->video_mixer_render(vc->video_mixer, VDP_INVALID_HANDLE,
-                                         0, field, 2, vc->deint_surfaces + 1,
-                                         vc->deint_surfaces[0], 1,
-                                         &vc->surface_render[vc->vid_surface_num].surface,
-                                         &vc->src_rect_vid, output_surface,
-                                         NULL, &vc->out_rect_vid, 0, NULL);
-        CHECK_ST_WARNING("Error when calling vdp_video_mixer_render");
-        push_deint_surface(vo, vc->surface_render[vc->vid_surface_num].surface);
+static void add_new_video_surface(struct vo *vo, VdpVideoSurface surface,
+                                  struct mp_image *reserved_mpi, double pts)
+{
+    struct vdpctx *vc = vo->priv;
+
+    if (reserved_mpi)
+        reserved_mpi->usage_count++;
+    if (vc->deint_mpi[2])
+        vc->deint_mpi[2]->usage_count--;
+
+    for (int i = 2; i > 0; i--) {
+        vc->deint_mpi[i] = vc->deint_mpi[i - 1];
+        vc->deint_surfaces[i] = vc->deint_surfaces[i - 1];
+        vc->deint_pts[i] = vc->deint_pts[i - 1];
+    }
+    vc->deint_mpi[0] = reserved_mpi;
+    vc->deint_surfaces[0] = surface;
+    vc->deint_pts[0] = pts;
+
+    vo->frame_loaded = true;
+    vo->next_pts = pts;
+    if (vc->deint >= 2 && vc->deint_queue_pos >= 0) {
+        vc->deint_queue_pos = 2;
+        double diff = vc->deint_pts[0] - vc->deint_pts[1];
+        if (diff > 0 && diff < 0.5)
+            vo->next_pts = (vc->deint_pts[0] + vc->deint_pts[1]) / 2;
+        else
+            vo->next_pts = vc->deint_pts[1];
+    } else
+        vc->deint_queue_pos = 1;
+    video_to_output_surface(vo);
+}
+
+static void forget_frames(struct vo *vo)
+{
+    struct vdpctx *vc = vo->priv;
+
+    vc->deint_queue_pos = -1;
+    for (int i = 0; i < 3; i++) {
+        vc->deint_surfaces[i] = VDP_INVALID_HANDLE;
+        if (vc->deint_mpi[i])
+            vc->deint_mpi[i]->usage_count--;
+        vc->deint_mpi[i] = NULL;
     }
 }
 
@@ -563,7 +596,7 @@ int initialize_vdpau_objects(struct vo *vo)
                                           &vc->eosd_surface.max_height);
     CHECK_ST_WARNING("Query to get max EOSD surface size failed");
     vc->surface_num = 0;
-    vc->vid_surface_num = -1;
+    forget_frames(vo);
     resize(vo);
     return 0;
 }
@@ -575,8 +608,7 @@ static void mark_vdpau_objects_uninitialized(struct vo *vo)
     vc->decoder = VDP_INVALID_HANDLE;
     for (int i = 0; i < MAX_VIDEO_SURFACES; i++)
         vc->surface_render[i].surface = VDP_INVALID_HANDLE;
-    for (int i = 0; i < 3; i++)
-        vc->deint_surfaces[i] = VDP_INVALID_HANDLE;
+    forget_frames(vo);
     vc->video_mixer = VDP_INVALID_HANDLE;
     vc->flip_queue = VDP_INVALID_HANDLE;
     vc->flip_target = VDP_INVALID_HANDLE;
@@ -1033,10 +1065,6 @@ static void flip_page(struct vo *vo)
     if (handle_preemption(vo) < 0)
         return;
 
-    mp_msg(MSGT_VO, MSGL_DBG2, "\nFLIP_PAGE VID:%u -> OUT:%u\n",
-           vc->surface_render[vc->vid_surface_num].surface,
-           vc->output_surfaces[vc->surface_num]);
-
     vdp_st =
         vdp->presentation_queue_display(vc->flip_queue,
                                         vc->output_surfaces[vc->surface_num],
@@ -1095,27 +1123,26 @@ static struct vdpau_render_state *get_surface(struct vo *vo, int number)
     return &vc->surface_render[number];
 }
 
-static uint32_t draw_image(struct vo *vo, mp_image_t *mpi)
+static void draw_image(struct vo *vo, mp_image_t *mpi, double pts)
 {
     struct vdpctx *vc = vo->priv;
     struct vdp_functions *vdp = vc->vdp;
+    struct mp_image *reserved_mpi = NULL;
+    struct vdpau_render_state *rndr;
+
+    if (vc->is_preempted) {
+        vo->frame_loaded = true;
+        return;
+    }
 
     if (IMGFMT_IS_VDPAU(vc->image_format)) {
-        struct vdpau_render_state *rndr = mpi->priv;
-        vc->vid_surface_num = rndr - vc->surface_render;
-        if (vc->deint_buffer_past_frames) {
-            mpi->usage_count++;
-            if (vc->deint_mpi[1])
-                vc->deint_mpi[1]->usage_count--;
-            vc->deint_mpi[1] = vc->deint_mpi[0];
-            vc->deint_mpi[0] = mpi;
-        }
+        rndr = mpi->priv;
+        reserved_mpi = mpi;
     } else if (!(mpi->flags & MP_IMGFLAG_DRAW_CALLBACK)) {
         VdpStatus vdp_st;
         void *destdata[3] = {mpi->planes[0], mpi->planes[2], mpi->planes[1]};
-        struct vdpau_render_state *rndr = get_surface(vo, vc->deint_counter);
+        rndr = get_surface(vo, vc->deint_counter);
         vc->deint_counter = (vc->deint_counter + 1) % 3;
-        vc->vid_surface_num = rndr - vc->surface_render;
         if (vc->image_format == IMGFMT_NV12)
             destdata[1] = destdata[2];
         vdp_st =
@@ -1123,16 +1150,33 @@ static uint32_t draw_image(struct vo *vo, mp_image_t *mpi)
                                                 vc->vdp_pixel_format,
                                                 (const void *const*)destdata,
                                                 mpi->stride); // pitch
-        CHECK_ST_ERROR("Error when calling "
+        CHECK_ST_WARNING("Error when calling "
                        "vdp_video_surface_put_bits_y_cb_cr");
-    }
+    } else
+        // We don't support slice callbacks so this shouldn't occur -
+        // I think the flags test above in pointless, but I'm adding
+        // this instead of removing it just in case.
+        abort();
     if (mpi->fields & MP_IMGFIELD_ORDERED)
         vc->top_field_first = !!(mpi->fields & MP_IMGFIELD_TOP_FIRST);
     else
         vc->top_field_first = 1;
 
+    add_new_video_surface(vo, rndr->surface, mpi, pts);
+
+    return;
+}
+
+static void get_buffered_frame(struct vo *vo, bool eof)
+{
+    struct vdpctx *vc = vo->priv;
+
+    if (vc->deint_queue_pos < 2)
+        return;
+    vc->deint_queue_pos = 1;
     video_to_output_surface(vo);
-    return VO_TRUE;
+    vo->next_pts = vc->deint_pts[0];
+    vo->frame_loaded = true;
 }
 
 static uint32_t get_image(struct vo *vo, mp_image_t *mpi)
@@ -1287,8 +1331,6 @@ static int preinit(struct vo *vo, const char *arg)
     }
     if (vc->deint)
         vc->deint_type = vc->deint;
-    if (vc->deint > 1)
-        vc->deint_buffer_past_frames = 1;
 
     char *vdpaulibrary = "libvdpau.so.1";
     char *vdpau_device_create = "vdp_device_create_x11";
@@ -1408,7 +1450,6 @@ static int control(struct vo *vo, uint32_t request, void *data)
                                                           1, features,
                                                           feature_enables);
             CHECK_ST_WARNING("Error changing deinterlacing settings");
-            vc->deint_buffer_past_frames = 1;
         }
         return VO_TRUE;
     case VOCTRL_PAUSE:
@@ -1420,9 +1461,7 @@ static int control(struct vo *vo, uint32_t request, void *data)
     case VOCTRL_GET_IMAGE:
         return get_image(vo, data);
     case VOCTRL_DRAW_IMAGE:
-        if (vc->is_preempted)
-            return true;
-        return draw_image(vo, data);
+        abort(); // draw_image() should get called directly
     case VOCTRL_BORDER:
         vo_x11_border(vo);
         resize(vo);
@@ -1476,12 +1515,16 @@ static int control(struct vo *vo, uint32_t request, void *data)
         draw_osd(vo, data);
         flip_page(vo);
         return true;
+    case VOCTRL_RESET:
+        forget_frames(vo);
+        return true;
     }
     return VO_NOTIMPL;
 }
 
 const struct vo_driver video_out_vdpau = {
-    .is_new = 1,
+    .is_new = true,
+    .buffer_frames = true,
     .info = &(const struct vo_info_s){
         "VDPAU with X11",
         "vdpau",
@@ -1491,6 +1534,8 @@ const struct vo_driver video_out_vdpau = {
     .preinit = preinit,
     .config = config,
     .control = control,
+    .draw_image = draw_image,
+    .get_buffered_frame = get_buffered_frame,
     .draw_slice = draw_slice,
     .draw_osd = draw_osd,
     .flip_page = flip_page,