video/filter: change filter API, use refcounting, remove filter DR

Change the entire filter API to use reference counted images instead
of vf_get_image().

Remove filter "direct rendering". This was useful for vf_expand and (in
rare cases) vf_sub: DR allowed these filters to pass a cropped image to
the filters before them. Then, on filtering, the image was "uncropped",
so that black bars could be added around the image without copying. This
means that in some cases, vf_expand will be slower (-vf gradfun,expand
for example).

Note that another form of DR used for in-place filters has been replaced
by simpler logic. Instead of trying to do DR, filters can check if the
image is writeable (with mp_image_is_writeable()), and do true in-place
if that's the case. This affects filters like vf_gradfun and vf_sub.

Everything has to support strides now. If something doesn't, making a
copy of the image data is required.

5 files changed, 24 insertions, 59 deletions

diff --git a/video/decode/dec_video.c b/video/decode/dec_video.c
index 9ec5e3db65..8730703216 100644
--- a/video/decode/dec_video.c
+++ b/video/decode/dec_video.c
@@ -104,7 +104,6 @@ int get_video_colors(sh_video_t *sh_video, const char *item, int *value)
 void get_detected_video_colorspace(struct sh_video *sh, struct mp_csp_details *csp)
 {
     struct MPOpts *opts = sh->opts;
-    struct vf_instance *vf = sh->vfilter;
 
     csp->format = opts->requested_colorspace;
     csp->levels_in = opts->requested_input_range;
@@ -113,7 +112,7 @@ void get_detected_video_colorspace(struct sh_video *sh, struct mp_csp_details *c
     if (csp->format == MP_CSP_AUTO)
         csp->format = sh->colorspace;
     if (csp->format == MP_CSP_AUTO)
-        csp->format = mp_csp_guess_colorspace(vf->w, vf->h);
+        csp->format = mp_csp_guess_colorspace(sh->disp_w, sh->disp_h);
 
     if (csp->levels_in == MP_CSP_LEVELS_AUTO)
         csp->levels_in = sh->color_range;
@@ -400,8 +399,10 @@ void *decode_video(sh_video_t *sh_video, struct demux_packet *packet,
     }
 #endif
 
-    if (!mpi || drop_frame)
+    if (!mpi || drop_frame) {
+        talloc_free(mpi);
         return NULL;            // error / skipped frame
+    }
 
     if (field_dominance == 0)
         mpi->fields |= MP_IMGFIELD_TOP_FIRST;
@@ -432,11 +433,3 @@ void *decode_video(sh_video_t *sh_video, struct demux_packet *packet,
         sh_video->num_sorted_pts_problems++;
     return mpi;
 }
-
-int filter_video(sh_video_t *sh_video, void *frame, double pts)
-{
-    mp_image_t *mpi = frame;
-    vf_instance_t *vf = sh_video->vfilter;
-    // apply video filters and call the leaf vo/ve
-    return vf->put_image(vf, mpi, pts);
-}
diff --git a/video/decode/dec_video.h b/video/decode/dec_video.h
index 9eb90a5d5a..c7c535c68e 100644
--- a/video/decode/dec_video.h
+++ b/video/decode/dec_video.h
@@ -33,7 +33,6 @@ struct demux_packet;
 void *decode_video(sh_video_t *sh_video, struct demux_packet *packet,
                    unsigned char *start, int in_size, int drop_frame,
                    double pts);
-int filter_video(sh_video_t *sh_video, void *frame, double pts);
 
 int get_video_quality_max(sh_video_t *sh_video);
 
diff --git a/video/decode/lavc.h b/video/decode/lavc.h
index c4d24aad94..a355f61310 100644
--- a/video/decode/lavc.h
+++ b/video/decode/lavc.h
@@ -8,13 +8,9 @@
 #include "demux/stheader.h"
 #include "video/mp_image.h"
 
-#define MAX_NUM_MPI 50
-
 typedef struct ffmpeg_ctx {
     AVCodecContext *avctx;
     AVFrame *pic;
-    struct mp_image *last_mpi;
-    struct mp_image hwdec_mpi[MAX_NUM_MPI];
     struct hwdec *hwdec;
     enum PixelFormat pix_fmt;
     int do_hw_dr1, do_dr1;
@@ -28,6 +24,7 @@ typedef struct ffmpeg_ctx {
     int rawvideo_fmt;
     AVCodec *software_fallback;
     struct FramePool *dr1_buffer_pool;
+    struct mp_image_pool *non_dr1_pool;
 } vd_ffmpeg_ctx;
 
 int mp_codec_get_buffer(AVCodecContext *s, AVFrame *frame);
diff --git a/video/decode/vd.c b/video/decode/vd.c
index e030cf4bd0..e3cb70ad1b 100644
--- a/video/decode/vd.c
+++ b/video/decode/vd.c
@@ -190,9 +190,6 @@ int mpcodecs_config_vo(sh_video_t *sh, int w, int h, unsigned int out_fmt)
            "VO Config (%dx%d->%dx%d,flags=%d,0x%X)\n", sh->disp_w,
            sh->disp_h, screen_size_x, screen_size_y, vocfg_flags, out_fmt);
 
-    vf->w = sh->disp_w;
-    vf->h = sh->disp_h;
-
     if (vf_config_wrapper
         (vf, sh->disp_w, sh->disp_h, screen_size_x, screen_size_y, vocfg_flags,
          out_fmt) == 0) {
diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index 3ee8cc7932..7fd7437a3b 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -41,6 +41,7 @@
 
 #include "vd.h"
 #include "video/img_format.h"
+#include "video/mp_image_pool.h"
 #include "video/filter/vf.h"
 #include "demux/stheader.h"
 #include "demux/demux_packet.h"
@@ -244,6 +245,7 @@ static int init(sh_video_t *sh)
 
     ctx = sh->context = talloc_zero(NULL, vd_ffmpeg_ctx);
     ctx->rawvideo_fmt = PIX_FMT_NONE;
+    ctx->non_dr1_pool = talloc_steal(ctx, mp_image_pool_new(16));
 
     if (sh->codec->dll) {
         lavc_codec = avcodec_find_decoder_by_name(sh->codec->dll);
@@ -478,7 +480,6 @@ static void uninit_avctx(sh_video_t *sh)
     av_freep(&avctx);
     avcodec_free_frame(&ctx->pic);
 
-    mp_image_unrefp(&ctx->last_mpi);
     mp_buffer_pool_free(&ctx->dr1_buffer_pool);
 }
 
@@ -518,7 +519,7 @@ static int init_vo(sh_video_t *sh)
             width != sh->disp_w || height != sh->disp_h ||
             avctx->pix_fmt != ctx->pix_fmt || !ctx->vo_initialized)
     {
-        mp_image_unrefp(&ctx->last_mpi);
+        mp_image_pool_clear(ctx->non_dr1_pool);
         ctx->vo_initialized = 0;
         mp_msg(MSGT_DECVIDEO, MSGL_V, "[ffmpeg] aspect_ratio: %f\n", aspect);
 
@@ -579,22 +580,6 @@ static void draw_slice_hwdec(struct AVCodecContext *s,
     vf->control(vf, VFCTRL_HWDEC_DECODER_RENDER, state_ptr);
 }
 
-static struct mp_image *get_image_hwdec(vd_ffmpeg_ctx *ctx)
-{
-    for (int n = 0; n < MAX_NUM_MPI; n++) {
-        struct mp_image *cur = &ctx->hwdec_mpi[n];
-        if (cur->usage_count == 0) {
-            *cur = (struct mp_image) {
-                .number = n,
-                .imgfmt = ctx->best_csp,
-                .usage_count = 1,
-            };
-            return cur;
-        }
-    }
-    return NULL;
-}
-
 static int get_buffer_hwdec(AVCodecContext *avctx, AVFrame *pic)
 {
     sh_video_t *sh = avctx->opaque;
@@ -620,12 +605,12 @@ static int get_buffer_hwdec(AVCodecContext *avctx, AVFrame *pic)
 
     assert(IMGFMT_IS_HWACCEL(ctx->best_csp));
 
-    struct mp_image *mpi = get_image_hwdec(ctx);
-    if (!mpi)
-        return -1;
+    struct mp_image *mpi = NULL;
 
     struct vf_instance *vf = sh->vfilter;
-    vf->control(vf, VFCTRL_HWDEC_GET_SURFACE, mpi);
+    vf->control(vf, VFCTRL_HWDEC_ALLOC_SURFACE, &mpi);
+    if (!mpi)
+        return -1;
 
     for (int i = 0; i < 4; i++)
         pic->data[i] = mpi->planes[i];
@@ -647,9 +632,8 @@ static void release_buffer_hwdec(AVCodecContext *avctx, AVFrame *pic)
 
     assert(pic->type == FF_BUFFER_TYPE_USER);
     assert(mpi);
-    assert(mpi->usage_count > 0);
 
-    mpi->usage_count--;
+    talloc_free(mpi);
 
     for (int i = 0; i < 4; i++)
         pic->data[i] = NULL;
@@ -726,13 +710,14 @@ static int decode(struct sh_video *sh, struct demux_packet *packet, void *data,
         return -1;
 
     struct mp_image *mpi = NULL;
-    if (ctx->do_hw_dr1 && pic->opaque)
+    if (ctx->do_hw_dr1 && pic->opaque) {
         mpi = pic->opaque; // reordered frame
+        assert(mpi);
+        mpi = mp_image_new_ref(mpi);
+    }
 
     if (!mpi) {
         struct mp_image new = {0};
-        new.type = MP_IMGTYPE_EXPORT;
-        new.flags = MP_IMGFLAG_PRESERVE;
         mp_image_set_size(&new, avctx->width, avctx->height);
         mp_image_setfmt(&new, ctx->best_csp);
         for (int i = 0; i < 4; i++) {
@@ -741,23 +726,17 @@ static int decode(struct sh_video *sh, struct demux_packet *packet, void *data,
         }
         if (ctx->do_dr1 && pic->opaque) {
             struct FrameBuffer *fb = pic->opaque;
-            mp_image_unrefp(&ctx->last_mpi);
-            mp_buffer_ref(fb); // reference for last_mpi
-            ctx->last_mpi = mp_image_new_external_ref(&new, fb, fb_ref,
-                                                      fb_unref, fb_is_unique);
+            mp_buffer_ref(fb); // initial reference for mpi
+            mpi = mp_image_new_external_ref(&new, fb, fb_ref, fb_unref,
+                                            fb_is_unique);
         } else {
-            if (!ctx->last_mpi)
-                ctx->last_mpi = mp_image_alloc(ctx->best_csp, new.w, new.h);
-            mp_image_make_writeable(&ctx->last_mpi);
-            assert(ctx->last_mpi->w == new.w && ctx->last_mpi->h == new.h);
-            assert(ctx->last_mpi->imgfmt == new.imgfmt);
-            mp_image_copy(ctx->last_mpi, &new);
+            mpi = mp_image_pool_get(ctx->non_dr1_pool, new.imgfmt,
+                                    new.w, new.h);
+            mp_image_copy(mpi, &new);
         }
-        mpi = ctx->last_mpi;
     }
 
-    if (!mpi->planes[0])
-        return 0; // ?
+    assert(mpi->planes[0]);
 
     assert(mpi->imgfmt == pixfmt2imgfmt(avctx->pix_fmt));