video/filter: change filter API, use refcounting, remove filter DR

Change the entire filter API to use reference counted images instead
of vf_get_image().

Remove filter "direct rendering". This was useful for vf_expand and (in
rare cases) vf_sub: DR allowed these filters to pass a cropped image to
the filters before them. Then, on filtering, the image was "uncropped",
so that black bars could be added around the image without copying. This
means that in some cases, vf_expand will be slower (-vf gradfun,expand
for example).

Note that another form of DR used for in-place filters has been replaced
by simpler logic. Instead of trying to do DR, filters can check if the
image is writeable (with mp_image_is_writeable()), and do true in-place
if that's the case. This affects filters like vf_gradfun and vf_sub.

Everything has to support strides now. If something doesn't, making a
copy of the image data is required.

1 files changed, 91 insertions, 243 deletions

diff --git a/video/filter/vf.c b/video/filter/vf.c
index 417d5be1a3..4da167641a 100644
--- a/video/filter/vf.c
+++ b/video/filter/vf.c
@@ -33,6 +33,7 @@
 
 #include "video/img_format.h"
 #include "video/mp_image.h"
+#include "video/mp_image_pool.h"
 #include "vf.h"
 
 #include "video/memcpy_pic.h"
@@ -177,182 +178,21 @@ void vf_mpi_clear(mp_image_t *mpi, int x0, int y0, int w, int h)
     }
 }
 
-mp_image_t *vf_get_image(vf_instance_t *vf, unsigned int outfmt,
-                         int mp_imgtype, int mp_imgflag, int w, int h)
+// Get a new image for filter output, with size and pixel format according to
+// the last vf_config call.
+struct mp_image *vf_alloc_out_image(struct vf_instance *vf)
 {
-    mp_image_t *mpi = NULL;
-    int w2;
-    int number = mp_imgtype >> 16;
-
-    assert(w == -1 || w >= vf->w);
-    assert(h == -1 || h >= vf->h);
-    assert(vf->w > 0);
-    assert(vf->h > 0);
-
-    if (w == -1)
-        w = vf->w;
-    if (h == -1)
-        h = vf->h;
-
-    w2 = (mp_imgflag & MP_IMGFLAG_ACCEPT_ALIGNED_STRIDE) ? FFALIGN(w, 32) : w;
-
-    if (vf->put_image == vf_next_put_image) {
-        // passthru mode, if the filter uses the fallback/default put_image()
-        mpi = vf_get_image(vf->next,outfmt,mp_imgtype,mp_imgflag,w,h);
-        mpi->usage_count++;
-        return mpi;
-    }
-
-    // Note: we should call libvo first to check if it supports direct rendering
-    // and if not, then fallback to software buffers:
-    switch (mp_imgtype & 0xff) {
-    case MP_IMGTYPE_EXPORT:
-        if (!vf->imgctx.export_images[0])
-            vf->imgctx.export_images[0] = new_mp_image(w2, h);
-        mpi = vf->imgctx.export_images[0];
-        break;
-    case MP_IMGTYPE_STATIC:
-        if (!vf->imgctx.static_images[0])
-            vf->imgctx.static_images[0] = new_mp_image(w2, h);
-        mpi = vf->imgctx.static_images[0];
-        break;
-    case MP_IMGTYPE_TEMP:
-        if (!vf->imgctx.temp_images[0])
-            vf->imgctx.temp_images[0] = new_mp_image(w2, h);
-        mpi = vf->imgctx.temp_images[0];
-        break;
-    case MP_IMGTYPE_IPB:
-        if (!(mp_imgflag & MP_IMGFLAG_READABLE)) { // B frame:
-            if (!vf->imgctx.temp_images[0])
-                vf->imgctx.temp_images[0] = new_mp_image(w2, h);
-            mpi = vf->imgctx.temp_images[0];
-            break;
-        }
-    case MP_IMGTYPE_IP:
-        if (!vf->imgctx.static_images[vf->imgctx.static_idx])
-            vf->imgctx.static_images[vf->imgctx.static_idx] = new_mp_image(w2, h);
-        mpi = vf->imgctx.static_images[vf->imgctx.static_idx];
-        vf->imgctx.static_idx ^= 1;
-        break;
-    case MP_IMGTYPE_NUMBERED:
-        if (number == -1) {
-            int i;
-            for (i = 0; i < NUM_NUMBERED_MPI; i++)
-                if (!vf->imgctx.numbered_images[i] ||
-                        !vf->imgctx.numbered_images[i]->usage_count)
-                    break;
-            number = i;
-        }
-        if (number < 0 || number >= NUM_NUMBERED_MPI)
-            return NULL;
-        if (!vf->imgctx.numbered_images[number])
-            vf->imgctx.numbered_images[number] = new_mp_image(w2, h);
-        mpi = vf->imgctx.numbered_images[number];
-        mpi->number = number;
-        break;
-    }
-    if (mpi) {
-        int missing_palette = !(mpi->flags & MP_IMGFLAG_RGB_PALETTE) &&
-                              (mp_imgflag & MP_IMGFLAG_RGB_PALETTE);
-        mpi->type = mp_imgtype;
-        mpi->w = vf->w;
-        mpi->h = vf->h;
-        // keep buffer allocation status & color flags only:
-        mpi->flags &= MP_IMGFLAG_ALLOCATED | MP_IMGFLAG_TYPE_DISPLAYED |
-                      MP_IMGFLAGMASK_COLORS;
-        // accept restrictions, palette flags only:
-        mpi->flags |= mp_imgflag & (MP_IMGFLAGMASK_RESTRICTIONS |
-                                    MP_IMGFLAG_RGB_PALETTE);
-        if (mpi->width != w2 || mpi->height != h || missing_palette) {
-            if (mpi->flags & MP_IMGFLAG_ALLOCATED) {
-                if (mpi->width < w2 || mpi->height < h || missing_palette) {
-                    // need to re-allocate buffer memory:
-                    av_free(mpi->planes[0]);
-                    if (mpi->flags & MP_IMGFLAG_RGB_PALETTE)
-                        av_free(mpi->planes[1]);
-                    for (int n = 0; n < MP_MAX_PLANES; n++)
-                        mpi->planes[n] = NULL;
-                    mpi->flags &= ~MP_IMGFLAG_ALLOCATED;
-                    mp_msg(MSGT_VFILTER, MSGL_V,
-                           "vf.c: have to REALLOCATE buffer memory :(\n");
-                }
-            }
-            mpi->width = w2;
-            mpi->chroma_width = (w2 + (1 << mpi->chroma_x_shift) - 1) >>
-                                                     mpi->chroma_x_shift;
-            mpi->height = h;
-            mpi->chroma_height = (h + (1 << mpi->chroma_y_shift) - 1) >>
-                                                     mpi->chroma_y_shift;
-        }
-        if (!mpi->bpp)
-            mp_image_setfmt(mpi, outfmt);
-        if (!(mpi->flags & MP_IMGFLAG_ALLOCATED) &&
-                mpi->type > MP_IMGTYPE_EXPORT) {
-            // check libvo first!
-            if (vf->get_image)
-                vf->get_image(vf, mpi);
-
-            if (!(mpi->flags & MP_IMGFLAG_DIRECT)) {
-                // non-direct and not yet allocated image. allocate it!
-                if (!mpi->bpp) { // no way we can allocate this
-                    mp_msg(MSGT_DECVIDEO, MSGL_FATAL,
-                           "vf_get_image: Tried to allocate a format that "
-                           "can not be allocated!\n");
-                    return NULL;
-                }
-
-                // check if codec prefer aligned stride:
-                if (mp_imgflag & MP_IMGFLAG_PREFER_ALIGNED_STRIDE) {
-                    int align = (mpi->flags & MP_IMGFLAG_PLANAR &&
-                                 mpi->flags & MP_IMGFLAG_YUV) ?
-                                (16 << mpi->chroma_x_shift) - 1 : 32; // OK?
-                    w2 = FFALIGN(w, align);
-                    if (mpi->width != w2) {
-                        // we have to change width... check if we CAN co it:
-                        int flags = vf->query_format(vf, outfmt);
-                        // should not fail
-                        if (!(flags & (VFCAP_CSP_SUPPORTED |
-                                       VFCAP_CSP_SUPPORTED_BY_HW)))
-                            mp_msg(MSGT_DECVIDEO, MSGL_WARN,
-                                   "??? vf_get_image{vf->query_format(outfmt)} "
-                                   "failed!\n");
-                        if (flags & VFCAP_ACCEPT_STRIDE) {
-                            mpi->width = w2;
-                            mpi->chroma_width =
-                                (w2 + (1 << mpi->chroma_x_shift) - 1) >>
-                                mpi->chroma_x_shift;
-                        }
-                    }
-                }
+    assert(vf->fmt_out.configured);
+    return mp_image_pool_get(vf->out_pool, vf->fmt_out.fmt,
+                             vf->fmt_out.w, vf->fmt_out.h);
+}
 
-                mp_image_alloc_planes(mpi);
-                vf_mpi_clear(mpi, 0, 0, mpi->width, mpi->height);
-            }
-        }
-        if (!(mpi->flags & MP_IMGFLAG_TYPE_DISPLAYED)) {
-            mp_msg(MSGT_DECVIDEO, MSGL_V,
-                   "*** [%s] %s mp_image_t, %dx%dx%dbpp %s %s, %d bytes\n",
-                   vf->info->name,
-                   (mpi->type == MP_IMGTYPE_EXPORT) ? "Exporting" :
-                   ((mpi->flags & MP_IMGFLAG_DIRECT) ?
-                            "Direct Rendering" : "Allocating"),
-                   mpi->width, mpi->height, mpi->bpp,
-                   (mpi->flags & MP_IMGFLAG_YUV) ? "YUV" :
-                           ((mpi->flags & MP_IMGFLAG_SWAPPED) ? "BGR" : "RGB"),
-                   (mpi->flags & MP_IMGFLAG_PLANAR) ? "planar" : "packed",
-                   mpi->bpp * mpi->width * mpi->height / 8);
-            mp_msg(MSGT_DECVIDEO, MSGL_DBG2, "(imgfmt: %x, planes: %p,%p,%p "
-                   "strides: %d,%d,%d, chroma: %dx%d, shift: h:%d,v:%d)\n",
-                   mpi->imgfmt, mpi->planes[0], mpi->planes[1], mpi->planes[2],
-                   mpi->stride[0], mpi->stride[1], mpi->stride[2],
-                   mpi->chroma_width, mpi->chroma_height,
-                   mpi->chroma_x_shift, mpi->chroma_y_shift);
-            mpi->flags |= MP_IMGFLAG_TYPE_DISPLAYED;
-        }
-        mpi->qscale = NULL;
-        mpi->usage_count++;
-    }
-    return mpi;
+void vf_make_out_image_writeable(struct vf_instance *vf, struct mp_image *img)
+{
+    assert(vf->fmt_out.configured);
+    assert(vf->fmt_out.fmt == img->imgfmt);
+    assert(vf->fmt_out.w == img->w && vf->fmt_out.h == img->h);
+    mp_image_pool_make_writeable(vf->out_pool, img);
 }
 
 //============================================================================
@@ -362,6 +202,14 @@ static int vf_default_query_format(struct vf_instance *vf, unsigned int fmt)
     return vf_next_query_format(vf, fmt);
 }
 
+
+static struct mp_image *vf_default_filter(struct vf_instance *vf,
+                                          struct mp_image *mpi)
+{
+    assert(!vf->filter_ext);
+    return mpi;
+}
+
 struct vf_instance *vf_open_plugin_noerr(struct MPOpts *opts,
                                          const vf_info_t *const *filter_list,
                                          vf_instance_t *next, const char *name,
@@ -378,16 +226,15 @@ struct vf_instance *vf_open_plugin_noerr(struct MPOpts *opts,
         if (!strcmp(filter_list[i]->name, name))
             break;
     }
-    vf = calloc(1, sizeof *vf);
+    vf = talloc_zero(NULL, struct vf_instance);
     vf->opts = opts;
     vf->info = filter_list[i];
     vf->next = next;
     vf->config = vf_next_config;
     vf->control = vf_next_control;
     vf->query_format = vf_default_query_format;
-    vf->put_image = vf_next_put_image;
-    vf->default_caps = VFCAP_ACCEPT_STRIDE;
-    vf->default_reqs = 0;
+    vf->filter = vf_default_filter;
+    vf->out_pool = talloc_steal(vf, mp_image_pool_new(16));
     if (vf->info->opts) { // vf_vo get some special argument
         const m_struct_t *st = vf->info->opts;
         void *vf_priv = m_struct_alloc(st);
@@ -404,7 +251,7 @@ struct vf_instance *vf_open_plugin_noerr(struct MPOpts *opts,
     *retcode = vf->info->vf_open(vf, (char *)args);
     if (*retcode > 0)
         return vf;
-    free(vf);
+    talloc_free(vf);
     return NULL;
 }
 
@@ -530,6 +377,7 @@ void vf_clone_mpi_attributes(mp_image_t *dst, mp_image_t *src)
     dst->pict_type = src->pict_type;
     dst->fields = src->fields;
     dst->qscale_type = src->qscale_type;
+    dst->pts = src->pts;
     if (dst->width == src->width && dst->height == src->height) {
         dst->qstride = src->qstride;
         dst->qscale = src->qscale;
@@ -542,60 +390,84 @@ void vf_clone_mpi_attributes(mp_image_t *dst, mp_image_t *src)
     }
 }
 
-void vf_queue_frame(vf_instance_t *vf, int (*func)(vf_instance_t *))
+// Used by filters to add a filtered frame to the output queue.
+// Ownership of img is transferred from caller to the filter chain.
+void vf_add_output_frame(struct vf_instance *vf, struct mp_image *img)
 {
-    vf->continue_buffered_image = func;
+    if (img)
+        MP_TARRAY_APPEND(vf, vf->out_queued, vf->num_out_queued, img);
 }
 
-// Output the next buffered image (if any) from the filter chain.
-// The queue could be kept as a simple stack/list instead avoiding the
-// looping here, but there's currently no good context variable where
-// that could be stored so this was easier to implement.
+static struct mp_image *vf_dequeue_output_frame(struct vf_instance *vf)
+{
+    struct mp_image *res = NULL;
+    if (vf->num_out_queued) {
+        res = vf->out_queued[0];
+        MP_TARRAY_REMOVE_AT(vf->out_queued, vf->num_out_queued, 0);
+    }
+    return res;
+}
+
+// Input a frame into the filter chain.
+// Return >= 0 on success, < 0 on failure (even if output frames were produced)
+int vf_filter_frame(struct vf_instance *vf, struct mp_image *img)
+{
+    assert(vf->fmt_in.configured);
+    assert(img->w == vf->fmt_in.w && img->h == vf->fmt_in.h);
+    assert(img->imgfmt == vf->fmt_in.fmt);
+
+    if (vf->filter_ext) {
+        return vf->filter_ext(vf, img);
+    } else {
+        vf_add_output_frame(vf, vf->filter(vf, img));
+        return 0;
+    }
+}
 
-int vf_output_queued_frame(vf_instance_t *vf)
+// Output the next queued image (if any) from the full filter chain.
+struct mp_image *vf_chain_output_queued_frame(struct vf_instance *vf)
 {
     while (1) {
-        int ret;
-        vf_instance_t *current;
-        vf_instance_t *last = NULL;
-        int (*tmp)(vf_instance_t *);
-        for (current = vf; current; current = current->next)
-            if (current->continue_buffered_image)
-                last = current;
+        struct vf_instance *last = NULL;
+        for (struct vf_instance * cur = vf; cur; cur = cur->next) {
+            if (cur->num_out_queued)
+                last = cur;
+        }
         if (!last)
-            return 0;
-        tmp = last->continue_buffered_image;
-        last->continue_buffered_image = NULL;
-        ret = tmp(last);
-        if (ret)
-            return ret;
+            return NULL;
+        struct mp_image *img = vf_dequeue_output_frame(last);
+        if (!last->next)
+            return img;
+        vf_filter_frame(last->next, img);
     }
 }
 
+static void vf_forget_frames(struct vf_instance *vf)
+{
+    for (int n = 0; n < vf->num_out_queued; n++)
+        talloc_free(vf->out_queued[n]);
+    vf->num_out_queued = 0;
+}
 
-/**
- * \brief Video config() function wrapper
- *
- * Blocks config() calls with different size or format for filters
- * with VFCAP_CONSTANT
- *
- * First call is redirected to vf->config.
- *
- * In following calls, it verifies that the configuration parameters
- * are unchanged, and returns either success or error.
- *
- */
 int vf_config_wrapper(struct vf_instance *vf,
                       int width, int height, int d_width, int d_height,
                       unsigned int flags, unsigned int outfmt)
 {
-    vf->fmt.have_configured = 1;
-    vf->fmt.orig_height = height;
-    vf->fmt.orig_width = width;
-    vf->fmt.orig_fmt = outfmt;
+    vf_forget_frames(vf);
+    mp_image_pool_clear(vf->out_pool);
+
+    vf->fmt_in = vf->fmt_out = (struct vf_format){0};
+
     int r = vf->config(vf, width, height, d_width, d_height, flags, outfmt);
-    if (!r)
-        vf->fmt.have_configured = 0;
+    if (r) {
+        vf->fmt_in = (struct vf_format) {
+            .configured = 1,
+            .w = width,
+            .h = height,
+            .fmt = outfmt,
+        };
+        vf->fmt_out = vf->next ? vf->next->fmt_in : (struct vf_format){0};
+    }
     return r;
 }
 
@@ -604,7 +476,6 @@ int vf_next_config(struct vf_instance *vf,
                    unsigned int voflags, unsigned int outfmt)
 {
     struct MPOpts *opts = vf->opts;
-    int miss;
     int flags = vf->next->query_format(vf->next, outfmt);
     if (!flags) {
         // hmm. colorspace mismatch!!!
@@ -623,19 +494,6 @@ int vf_next_config(struct vf_instance *vf,
             return 0; // FAIL
         }
     }
-    mp_msg(MSGT_VFILTER, MSGL_V, "REQ: flags=0x%X  req=0x%X  \n",
-           flags, vf->default_reqs);
-    miss = vf->default_reqs - (flags & vf->default_reqs);
-    if (miss & VFCAP_ACCEPT_STRIDE) {
-        // vf requires stride support but vf->next doesn't support it!
-        // let's insert the 'expand' filter, it does the job for us:
-        vf_instance_t *vf2 = vf_open_filter(opts, vf->next, "expand", NULL);
-        if (!vf2)
-            return 0;      // shouldn't happen!
-        vf->next = vf2;
-    }
-    vf->next->w = width;
-    vf->next->h = height;
     return vf_config_wrapper(vf->next, width, height, d_width, d_height,
                              voflags, outfmt);
 }
@@ -653,11 +511,6 @@ int vf_next_query_format(struct vf_instance *vf, unsigned int fmt)
     return flags;
 }
 
-int vf_next_put_image(struct vf_instance *vf, mp_image_t *mpi, double pts)
-{
-    return vf->next->put_image(vf->next, mpi, pts);
-}
-
 //============================================================================
 
 vf_instance_t *append_filters(vf_instance_t *last,
@@ -688,13 +541,8 @@ void vf_uninit_filter(vf_instance_t *vf)
 {
     if (vf->uninit)
         vf->uninit(vf);
-    free_mp_image(vf->imgctx.static_images[0]);
-    free_mp_image(vf->imgctx.static_images[1]);
-    free_mp_image(vf->imgctx.temp_images[0]);
-    free_mp_image(vf->imgctx.export_images[0]);
-    for (int i = 0; i < NUM_NUMBERED_MPI; i++)
-        free_mp_image(vf->imgctx.numbered_images[i]);
-    free(vf);
+    vf_forget_frames(vf);
+    talloc_free(vf);
 }
 
 void vf_uninit_filter_chain(vf_instance_t *vf)