video: support filtering hardware frames via libavfilter

Requires a bunch of hacks:
- we access AVFilterLink.hw_frames_ctx. This is not a public API in
  FFmpeg and Libav. Newer FFmpeg provides an accessor
  (av_buffersink_get_hw_frames_ctx), but it's not available in Libav or
  the current FFmpeg release or Libav. We need this value after filter
  graph creation, so We have no choice but to access this.
  One alternative is making filter creation and format negotiation
  fully lazy (i.e. delay it and do it as filters are output), but this
  would be a huge change.
  So for now, we knowingly violate FFmpeg's and Libav's ABI and API
  constraints because they don't provide anything better.
  On newer FFmpeg, we use the (quite ugly) accessor, though.
- mp_image_params doesn't (and can't) have a field for the frames
  context AVBufferRef. So we pass it via vf_set_proto_frame(), and even
  more hacks.
- if a filter needs a hw context, but we haven't created one yet
  (because normally we create them lazily), it will fail at init.
- we allow any hw format now, although this could go horrible wrong.

Why all this effort? We could move hw deinterlacing filters etc. to
FFmpeg, which is a very worthy goal.

1 files changed, 17 insertions, 0 deletions

diff --git a/video/filter/vf.c b/video/filter/vf.c
index 41fbe9b208..94e6760603 100644
--- a/video/filter/vf.c
+++ b/video/filter/vf.c
@@ -20,6 +20,7 @@
 #include <string.h>
 #include <assert.h>
 #include <sys/types.h>
+#include <libavutil/buffer.h>
 #include <libavutil/common.h>
 #include <libavutil/mem.h>
 
@@ -644,14 +645,20 @@ int vf_reconfig(struct vf_chain *c, const struct mp_image_params *params)
 
     uint8_t unused[IMGFMT_END - IMGFMT_START];
     update_formats(c, c->first, unused);
+    AVBufferRef *hwfctx = c->in_hwframes_ref;
     struct vf_instance *failing = NULL;
     for (struct vf_instance *vf = c->first; vf; vf = vf->next) {
+        av_buffer_unref(&vf->in_hwframes_ref);
+        av_buffer_unref(&vf->out_hwframes_ref);
+        vf->in_hwframes_ref = hwfctx ? av_buffer_ref(hwfctx) : NULL;
+        vf->out_hwframes_ref = hwfctx ? av_buffer_ref(hwfctx) : NULL;
         r = vf_reconfig_wrapper(vf, &cur);
         if (r < 0) {
             failing = vf;
             break;
         }
         cur = vf->fmt_out;
+        hwfctx = vf->out_hwframes_ref;
     }
     c->output_params = cur;
     c->initialized = r < 0 ? -1 : 1;
@@ -665,6 +672,13 @@ int vf_reconfig(struct vf_chain *c, const struct mp_image_params *params)
     return r;
 }
 
+// Hack to get mp_image.hwctx before vf_reconfig()
+void vf_set_proto_frame(struct vf_chain *c, struct mp_image *img)
+{
+    av_buffer_unref(&c->in_hwframes_ref);
+    c->in_hwframes_ref = img && img->hwctx ? av_buffer_ref(img->hwctx) : NULL;
+}
+
 struct vf_instance *vf_find_by_label(struct vf_chain *c, const char *label)
 {
     struct vf_instance *vf = c->first;
@@ -678,6 +692,8 @@ struct vf_instance *vf_find_by_label(struct vf_chain *c, const char *label)
 
 static void vf_uninit_filter(vf_instance_t *vf)
 {
+    av_buffer_unref(&vf->in_hwframes_ref);
+    av_buffer_unref(&vf->out_hwframes_ref);
     if (vf->uninit)
         vf->uninit(vf);
     vf_forget_frames(vf);
@@ -729,6 +745,7 @@ void vf_destroy(struct vf_chain *c)
 {
     if (!c)
         return;
+    av_buffer_unref(&c->in_hwframes_ref);
     while (c->first) {
         vf_instance_t *vf = c->first;
         c->first = vf->next;