video: support filtering hardware frames via libavfilter

Requires a bunch of hacks:
- we access AVFilterLink.hw_frames_ctx. This is not a public API in
  FFmpeg and Libav. Newer FFmpeg provides an accessor
  (av_buffersink_get_hw_frames_ctx), but it's not available in Libav or
  the current FFmpeg release or Libav. We need this value after filter
  graph creation, so We have no choice but to access this.
  One alternative is making filter creation and format negotiation
  fully lazy (i.e. delay it and do it as filters are output), but this
  would be a huge change.
  So for now, we knowingly violate FFmpeg's and Libav's ABI and API
  constraints because they don't provide anything better.
  On newer FFmpeg, we use the (quite ugly) accessor, though.
- mp_image_params doesn't (and can't) have a field for the frames
  context AVBufferRef. So we pass it via vf_set_proto_frame(), and even
  more hacks.
- if a filter needs a hw context, but we haven't created one yet
  (because normally we create them lazily), it will fail at init.
- we allow any hw format now, although this could go horrible wrong.

Why all this effort? We could move hw deinterlacing filters etc. to
FFmpeg, which is a very worthy goal.

4 files changed, 50 insertions, 6 deletions

diff --git a/player/video.c b/player/video.c
index 78d10fd107..79828aa6ce 100644
--- a/player/video.c
+++ b/player/video.c
@@ -701,6 +701,7 @@ static int video_decode_and_filter(struct MPContext *mpctx)
 
     if (vo_c->input_mpi) {
         vo_c->input_format = vo_c->input_mpi->params;
+        vf_set_proto_frame(vo_c->vf, vo_c->input_mpi);
 
         if (vo_c->is_coverart && !vo_c->cached_coverart)
             vo_c->cached_coverart = mp_image_new_ref(vo_c->input_mpi);
diff --git a/video/filter/vf.c b/video/filter/vf.c
index 41fbe9b208..94e6760603 100644
--- a/video/filter/vf.c
+++ b/video/filter/vf.c
@@ -20,6 +20,7 @@
 #include <string.h>
 #include <assert.h>
 #include <sys/types.h>
+#include <libavutil/buffer.h>
 #include <libavutil/common.h>
 #include <libavutil/mem.h>
 
@@ -644,14 +645,20 @@ int vf_reconfig(struct vf_chain *c, const struct mp_image_params *params)
 
     uint8_t unused[IMGFMT_END - IMGFMT_START];
     update_formats(c, c->first, unused);
+    AVBufferRef *hwfctx = c->in_hwframes_ref;
     struct vf_instance *failing = NULL;
     for (struct vf_instance *vf = c->first; vf; vf = vf->next) {
+        av_buffer_unref(&vf->in_hwframes_ref);
+        av_buffer_unref(&vf->out_hwframes_ref);
+        vf->in_hwframes_ref = hwfctx ? av_buffer_ref(hwfctx) : NULL;
+        vf->out_hwframes_ref = hwfctx ? av_buffer_ref(hwfctx) : NULL;
         r = vf_reconfig_wrapper(vf, &cur);
         if (r < 0) {
             failing = vf;
             break;
         }
         cur = vf->fmt_out;
+        hwfctx = vf->out_hwframes_ref;
     }
     c->output_params = cur;
     c->initialized = r < 0 ? -1 : 1;
@@ -665,6 +672,13 @@ int vf_reconfig(struct vf_chain *c, const struct mp_image_params *params)
     return r;
 }
 
+// Hack to get mp_image.hwctx before vf_reconfig()
+void vf_set_proto_frame(struct vf_chain *c, struct mp_image *img)
+{
+    av_buffer_unref(&c->in_hwframes_ref);
+    c->in_hwframes_ref = img && img->hwctx ? av_buffer_ref(img->hwctx) : NULL;
+}
+
 struct vf_instance *vf_find_by_label(struct vf_chain *c, const char *label)
 {
     struct vf_instance *vf = c->first;
@@ -678,6 +692,8 @@ struct vf_instance *vf_find_by_label(struct vf_chain *c, const char *label)
 
 static void vf_uninit_filter(vf_instance_t *vf)
 {
+    av_buffer_unref(&vf->in_hwframes_ref);
+    av_buffer_unref(&vf->out_hwframes_ref);
     if (vf->uninit)
         vf->uninit(vf);
     vf_forget_frames(vf);
@@ -729,6 +745,7 @@ void vf_destroy(struct vf_chain *c)
 {
     if (!c)
         return;
+    av_buffer_unref(&c->in_hwframes_ref);
     while (c->first) {
         vf_instance_t *vf = c->first;
         c->first = vf->next;
diff --git a/video/filter/vf.h b/video/filter/vf.h
index 901ccead95..241af02081 100644
--- a/video/filter/vf.h
+++ b/video/filter/vf.h
@@ -90,6 +90,9 @@ typedef struct vf_instance {
 
     struct mp_image_params fmt_in, fmt_out;
 
+    // This is a dirty hack.
+    struct AVBufferRef *in_hwframes_ref, *out_hwframes_ref;
+
     struct mp_image_pool *out_pool;
     struct vf_priv_s *priv;
     struct mp_log *log;
@@ -123,6 +126,9 @@ struct vf_chain {
     struct mpv_global *global;
     struct mp_hwdec_devices *hwdec_devs;
 
+    // This is a dirty hack.
+    struct AVBufferRef *in_hwframes_ref;
+
     // Call when the filter chain wants new processing (for filters with
     // asynchronous behavior) - must be immutable once filters are created,
     // since they are supposed to call it from foreign threads.
@@ -150,6 +156,7 @@ enum vf_ctrl {
 
 struct vf_chain *vf_new(struct mpv_global *global);
 void vf_destroy(struct vf_chain *c);
+void vf_set_proto_frame(struct vf_chain *c, struct mp_image *img);
 int vf_reconfig(struct vf_chain *c, const struct mp_image_params *params);
 int vf_control_any(struct vf_chain *c, int cmd, void *arg);
 int vf_control_by_label(struct vf_chain *c, int cmd, void *arg, bstr label);
diff --git a/video/filter/vf_lavfi.c b/video/filter/vf_lavfi.c
index 291ddd315b..2f4bc92ecc 100644
--- a/video/filter/vf_lavfi.c
+++ b/video/filter/vf_lavfi.c
@@ -26,6 +26,7 @@
 #include <assert.h>
 
 #include <libavutil/avstring.h>
+#include <libavutil/hwcontext.h>
 #include <libavutil/mem.h>
 #include <libavutil/mathematics.h>
 #include <libavutil/rational.h>
@@ -44,6 +45,7 @@
 #include "options/m_option.h"
 #include "common/tags.h"
 
+#include "video/hwdec.h"
 #include "video/img_format.h"
 #include "video/mp_image.h"
 #include "video/sws_utils.h"
@@ -148,6 +150,8 @@ static bool recreate_graph(struct vf_instance *vf, struct mp_image_params *fmt)
     in_params->height = fmt->h;
     in_params->sample_aspect_ratio.num = fmt->p_w;
     in_params->sample_aspect_ratio.den = fmt->p_h;
+    // Assume it's ignored for non-hwaccel formats.
+    in_params->hw_frames_ctx = vf->in_hwframes_ref;
 
     ret = av_buffersrc_parameters_set(in, in_params);
     av_free(in_params);
@@ -170,6 +174,13 @@ static bool recreate_graph(struct vf_instance *vf, struct mp_image_params *fmt)
     if (graph_parse(graph, p->cfg_graph, inputs, outputs, NULL) < 0)
         goto error;
 
+    struct mp_hwdec_ctx *hwdec = hwdec_devices_get_first(vf->hwdec_devs);
+    for (int n = 0; n < graph->nb_filters; n++) {
+        AVFilterContext *filter = graph->filters[n];
+        if (hwdec && hwdec->av_device_ref)
+            filter->hw_device_ctx = av_buffer_ref(hwdec->av_device_ref);
+    }
+
     if (avfilter_graph_config(graph, NULL) < 0)
         goto error;
 
@@ -226,17 +237,25 @@ static int reconfig(struct vf_instance *vf, struct mp_image_params *in,
     out->p_w = l_out->sample_aspect_ratio.num;
     out->p_h = l_out->sample_aspect_ratio.den;
     out->imgfmt = pixfmt2imgfmt(l_out->format);
+    av_buffer_unref(&vf->out_hwframes_ref);
+#if LIBAVFILTER_VERSION_INT >= AV_VERSION_INT(6, 69, 100) && \
+    LIBAVFILTER_VERSION_MICRO >= 100
+    AVBufferRef *hw_frames_ctx = av_buffersink_get_hw_frames_ctx(p->out);
+#else
+    AVBufferRef *hw_frames_ctx = l_out->hw_frames_ctx;
+#endif
+    if (hw_frames_ctx) {
+        AVHWFramesContext *fctx = (void *)hw_frames_ctx->data;
+        out->hw_subfmt = pixfmt2imgfmt(fctx->sw_format);
+        vf->out_hwframes_ref = av_buffer_ref(hw_frames_ctx);
+    }
     return 0;
 }
 
 static int query_format(struct vf_instance *vf, unsigned int fmt)
 {
-    // We accept all sws-convertable formats as inputs. Output formats are
-    // handled in config(). The current public libavfilter API doesn't really
-    // allow us to do anything more sophisticated.
-    // This breaks with filters which accept input pixel formats not
-    // supported by libswscale.
-    return !!mp_sws_supported_format(fmt);
+    // Format negotiation is not possible with libavfilter.
+    return 1;
 }
 
 static AVFrame *mp_to_av(struct vf_instance *vf, struct mp_image *img)