video/filter: change filter API, use refcounting, remove filter DR

Change the entire filter API to use reference counted images instead
of vf_get_image().

Remove filter "direct rendering". This was useful for vf_expand and (in
rare cases) vf_sub: DR allowed these filters to pass a cropped image to
the filters before them. Then, on filtering, the image was "uncropped",
so that black bars could be added around the image without copying. This
means that in some cases, vf_expand will be slower (-vf gradfun,expand
for example).

Note that another form of DR used for in-place filters has been replaced
by simpler logic. Instead of trying to do DR, filters can check if the
image is writeable (with mp_image_is_writeable()), and do true in-place
if that's the case. This affects filters like vf_gradfun and vf_sub.

Everything has to support strides now. If something doesn't, making a
copy of the image data is required.

4 files changed, 59 insertions, 45 deletions

diff --git a/video/out/vo.c b/video/out/vo.c
index 18e000e3ab..9a75ccb789 100644
--- a/video/out/vo.c
+++ b/video/out/vo.c
@@ -38,6 +38,7 @@
 #include "core/mp_fifo.h"
 #include "core/m_config.h"
 #include "core/mp_msg.h"
+#include "video/mp_image.h"
 #include "video/vfcap.h"
 #include "sub/sub.h"
 
@@ -159,28 +160,20 @@ int vo_control(struct vo *vo, uint32_t request, void *data)
     return vo->driver->control(vo, request, data);
 }
 
-static void draw_image_pts(struct vo *vo, struct mp_image *mpi, double pts)
-{
-    if (vo->driver->draw_image_pts) {
-        vo->driver->draw_image_pts(vo, mpi, pts);
-    } else {
-        vo->driver->draw_image(vo, mpi);
-    }
-}
-
 // Return -1 if driver appears not to support a draw_image interface,
 // 0 otherwise (whether the driver actually drew something or not).
-int vo_draw_image(struct vo *vo, struct mp_image *mpi, double pts)
+int vo_draw_image(struct vo *vo, struct mp_image *mpi)
 {
     if (!vo->config_ok)
         return 0;
     if (vo->driver->buffer_frames) {
-        draw_image_pts(vo, mpi, pts);
+        vo->driver->draw_image(vo, mpi);
         return 0;
     }
     vo->frame_loaded = true;
-    vo->next_pts = pts;
-    vo->waiting_mpi = mpi;
+    vo->next_pts = mpi->pts;
+    assert(!vo->waiting_mpi);
+    vo->waiting_mpi = mp_image_new_ref(mpi);
     return 0;
 }
 
@@ -211,6 +204,7 @@ void vo_skip_frame(struct vo *vo)
 {
     vo_control(vo, VOCTRL_SKIPFRAME, NULL);
     vo->frame_loaded = false;
+    mp_image_unrefp(&vo->waiting_mpi);
 }
 
 void vo_new_frame_imminent(struct vo *vo)
@@ -218,8 +212,11 @@ void vo_new_frame_imminent(struct vo *vo)
     if (vo->driver->buffer_frames)
         vo_control(vo, VOCTRL_NEWFRAME, NULL);
     else {
-        draw_image_pts(vo, vo->waiting_mpi, vo->next_pts);
-        vo->waiting_mpi = NULL;
+        assert(vo->frame_loaded);
+        assert(vo->waiting_mpi);
+        assert(vo->waiting_mpi->pts == vo->next_pts);
+        vo->driver->draw_image(vo, vo->waiting_mpi);
+        mp_image_unrefp(&vo->waiting_mpi);
     }
 }
 
@@ -262,6 +259,7 @@ void vo_seek_reset(struct vo *vo)
     vo_control(vo, VOCTRL_RESET, NULL);
     vo->frame_loaded = false;
     vo->hasframe = false;
+    mp_image_unrefp(&vo->waiting_mpi);
 }
 
 void vo_destroy(struct vo *vo)
@@ -269,6 +267,7 @@ void vo_destroy(struct vo *vo)
     if (vo->registered_fd != -1)
         mp_input_rm_key_fd(vo->input_ctx, vo->registered_fd);
     vo->driver->uninit(vo);
+    talloc_free(vo->waiting_mpi);
     talloc_free(vo);
 }
 
diff --git a/video/out/vo.h b/video/out/vo.h
index 7b929f985d..e883293326 100644
--- a/video/out/vo.h
+++ b/video/out/vo.h
@@ -53,7 +53,7 @@ enum mp_voctrl {
 
     /* for vdpau hardware decoding */
     VOCTRL_HWDEC_DECODER_RENDER,        // pointer to hw state
-    VOCTRL_HWDEC_GET_SURFACE,           // struct mp_image
+    VOCTRL_HWDEC_ALLOC_SURFACE,         // struct mp_image**
 
     VOCTRL_NEWFRAME,
     VOCTRL_SKIPFRAME,
@@ -285,7 +285,7 @@ int vo_config(struct vo *vo, uint32_t width, uint32_t height,
 void list_video_out(void);
 
 int vo_control(struct vo *vo, uint32_t request, void *data);
-int vo_draw_image(struct vo *vo, struct mp_image *mpi, double pts);
+int vo_draw_image(struct vo *vo, struct mp_image *mpi);
 int vo_redraw_frame(struct vo *vo);
 int vo_get_buffered_frame(struct vo *vo, bool eof);
 void vo_skip_frame(struct vo *vo);
diff --git a/video/out/vo_lavc.c b/video/out/vo_lavc.c
index 6ae06fffec..7d846da92d 100644
--- a/video/out/vo_lavc.c
+++ b/video/out/vo_lavc.c
@@ -74,7 +74,7 @@ static int preinit(struct vo *vo, const char *arg)
     return 0;
 }
 
-static void draw_image(struct vo *vo, mp_image_t *mpi, double pts);
+static void draw_image(struct vo *vo, mp_image_t *mpi);
 static void uninit(struct vo *vo)
 {
     struct priv *vc = vo->priv;
@@ -82,7 +82,7 @@ static void uninit(struct vo *vo)
         return;
 
     if (vc->lastipts >= 0 && vc->stream)
-        draw_image(vo, NULL, MP_NOPTS_VALUE);
+        draw_image(vo, NULL);
 
     if (vc->lastimg) {
         // palette hack
@@ -284,7 +284,7 @@ static int encode_video(struct vo *vo, AVFrame *frame, AVPacket *packet)
     }
 }
 
-static void draw_image(struct vo *vo, mp_image_t *mpi, double pts)
+static void draw_image(struct vo *vo, mp_image_t *mpi)
 {
     struct priv *vc = vo->priv;
     struct encode_lavc_context *ectx = vo->encode_lavc_ctx;
@@ -294,6 +294,8 @@ static void draw_image(struct vo *vo, mp_image_t *mpi, double pts)
     int64_t frameipts;
     double nextpts;
 
+    double pts = mpi ? mpi->pts : MP_NOPTS_VALUE;
+
     if (!vc)
         return;
     if (!encode_lavc_start(ectx)) {
@@ -540,7 +542,7 @@ const struct vo_driver video_out_lavc = {
     .control = control,
     .uninit = uninit,
     .check_events = check_events,
-    .draw_image_pts = draw_image,
+    .draw_image = draw_image,
     .draw_osd = draw_osd,
     .flip_page_timed = flip_page_timed,
 };
diff --git a/video/out/vo_vdpau.c b/video/out/vo_vdpau.c
index 661072b814..48080ba186 100644
--- a/video/out/vo_vdpau.c
+++ b/video/out/vo_vdpau.c
@@ -140,7 +140,8 @@ struct vdpctx {
     struct mp_osd_res                  osd_rect;
 
     struct vdpau_render_state          surface_render[MAX_VIDEO_SURFACES];
-    int                                surface_num;
+    bool                               surface_in_use[MAX_VIDEO_SURFACES];
+    int                                surface_num; // indexes output_surfaces
     int                                query_surface_num;
     VdpTime                            recent_vsync_time;
     float                              user_fps;
@@ -332,15 +333,12 @@ static void add_new_video_surface(struct vo *vo, VdpVideoSurface surface,
     struct vdpctx *vc = vo->priv;
     struct buffered_video_surface *bv = vc->buffered_video;
 
-    if (reserved_mpi)
-        reserved_mpi->usage_count++;
-    if (bv[NUM_BUFFERED_VIDEO - 1].mpi)
-        bv[NUM_BUFFERED_VIDEO - 1].mpi->usage_count--;
+    mp_image_unrefp(&bv[NUM_BUFFERED_VIDEO - 1].mpi);
 
     for (int i = NUM_BUFFERED_VIDEO - 1; i > 0; i--)
         bv[i] = bv[i - 1];
     bv[0] = (struct buffered_video_surface){
-        .mpi = reserved_mpi,
+        .mpi = reserved_mpi ? mp_image_new_ref(reserved_mpi) : NULL,
         .surface = surface,
         .pts = pts,
     };
@@ -358,8 +356,7 @@ static void forget_frames(struct vo *vo)
     vc->dropped_frame = false;
     for (int i = 0; i < NUM_BUFFERED_VIDEO; i++) {
         struct buffered_video_surface *p = vc->buffered_video + i;
-        if (p->mpi)
-            p->mpi->usage_count--;
+        mp_image_unrefp(&p->mpi);
         *p = (struct buffered_video_surface){
             .surface = VDP_INVALID_HANDLE,
         };
@@ -1299,7 +1296,7 @@ static struct vdpau_render_state *get_surface(struct vo *vo, int number)
     return &vc->surface_render[number];
 }
 
-static void draw_image(struct vo *vo, mp_image_t *mpi, double pts)
+static void draw_image(struct vo *vo, mp_image_t *mpi)
 {
     struct vdpctx *vc = vo->priv;
     struct vdp_functions *vdp = vc->vdp;
@@ -1329,7 +1326,7 @@ static void draw_image(struct vo *vo, mp_image_t *mpi, double pts)
     else
         vc->top_field_first = 1;
 
-    add_new_video_surface(vo, rndr->surface, reserved_mpi, pts);
+    add_new_video_surface(vo, rndr->surface, reserved_mpi, mpi->pts);
 
     return;
 }
@@ -1392,23 +1389,38 @@ static struct mp_image *get_window_screenshot(struct vo *vo)
     return image;
 }
 
-static uint32_t get_decoder_surface(struct vo *vo, mp_image_t *mpi)
+static void release_decoder_surface(void *ptr)
+{
+    bool *in_use_ptr = ptr;
+    *in_use_ptr = false;
+}
+
+static struct mp_image *get_decoder_surface(struct vo *vo)
 {
     struct vdpctx *vc = vo->priv;
-    struct vdpau_render_state *rndr;
 
     if (!IMGFMT_IS_VDPAU(vc->image_format))
-        return VO_FALSE;
+        return NULL;
 
-    rndr = get_surface(vo, mpi->number);
-    if (!rndr) {
-        mp_msg(MSGT_VO, MSGL_ERR, "[vdpau] no surfaces available in "
-               "get_decoder_surface\n");
-        // TODO: this probably breaks things forever, provide a dummy buffer?
-        return VO_FALSE;
+    for (int n = 0; n < MAX_VIDEO_SURFACES; n++) {
+        if (!vc->surface_in_use[n]) {
+            vc->surface_in_use[n] = true;
+            struct mp_image *res =
+                mp_image_new_custom_ref(&(struct mp_image){0},
+                                        &vc->surface_in_use[n],
+                                        release_decoder_surface);
+            mp_image_setfmt(res, vc->image_format);
+            mp_image_set_size(res, vc->vid_width, vc->vid_height);
+            struct vdpau_render_state *rndr = get_surface(vo, n);
+            res->planes[0] = (void *)rndr;
+            return res;
+        }
     }
-    mpi->planes[0] = (void *)rndr;
-    return VO_TRUE;
+
+    mp_msg(MSGT_VO, MSGL_ERR, "[vdpau] no surfaces available in "
+           "get_decoder_surface\n");
+    // TODO: this probably breaks things forever, provide a dummy buffer?
+    return NULL;
 }
 
 static int query_format(struct vo *vo, uint32_t format)
@@ -1587,8 +1599,9 @@ static int control(struct vo *vo, uint32_t request, void *data)
         if (vc->dropped_frame)
             vo->want_redraw = true;
         return true;
-    case VOCTRL_HWDEC_GET_SURFACE:
-        return get_decoder_surface(vo, data);
+    case VOCTRL_HWDEC_ALLOC_SURFACE:
+        *(struct mp_image **)data = get_decoder_surface(vo);
+        return true;
     case VOCTRL_HWDEC_DECODER_RENDER:
         return decoder_render(vo, data);
     case VOCTRL_BORDER:
@@ -1672,7 +1685,7 @@ const struct vo_driver video_out_vdpau = {
     .query_format = query_format,
     .config = config,
     .control = control,
-    .draw_image_pts = draw_image,
+    .draw_image = draw_image,
     .get_buffered_frame = set_next_frame_info,
     .draw_osd = draw_osd,
     .flip_page_timed = flip_page_timed,