vf_vavpp: use libavutil hw frames API for frame pool and upload

Another step to get rid of the legacy crap in vaapi.c. (Most is still
kept, because it's in use by vo_vaapi.c.)

1 files changed, 83 insertions, 25 deletions

diff --git a/video/filter/vf_vavpp.c b/video/filter/vf_vavpp.c
index 4b225aa466..32d650dc77 100644
--- a/video/filter/vf_vavpp.c
+++ b/video/filter/vf_vavpp.c
@@ -20,10 +20,13 @@
 #include <va/va.h>
 #include <va/va_vpp.h>
 
+#include <libavutil/hwcontext.h>
+
 #include "config.h"
 #include "options/options.h"
 #include "vf.h"
 #include "refqueue.h"
+#include "video/fmt-conversion.h"
 #include "video/vaapi.h"
 #include "video/hwdec.h"
 #include "video/mp_image_pool.h"
@@ -64,8 +67,9 @@ struct vf_priv_s {
     VADisplay display;
     struct mp_vaapi_ctx *va;
     struct pipeline pipe;
-    struct mp_image_pool *pool;
-    int current_rt_format;
+    AVBufferRef *hw_pool;
+    int *in_formats;
+    int num_in_formats;
 
     struct mp_refqueue *queue;
 };
@@ -156,6 +160,28 @@ nodeint:
     mp_refqueue_set_mode(p->queue, 0);
 }
 
+static struct mp_image *alloc_out(struct vf_instance *vf)
+{
+    struct vf_priv_s *p = vf->priv;
+
+    AVFrame *av_frame = av_frame_alloc();
+    if (!av_frame)
+        abort();
+    if (av_hwframe_get_buffer(p->hw_pool, av_frame, 0) < 0) {
+        MP_ERR(vf, "Failed to allocate frame from hw pool.\n");
+        av_frame_free(&av_frame);
+        return NULL;
+    }
+    struct mp_image *img = mp_image_from_av_frame(av_frame);
+    av_frame_free(&av_frame);
+    if (!img) {
+        MP_ERR(vf, "Unknown error.\n");
+        return NULL;
+    }
+    mp_image_set_size(img, vf->fmt_in.w, vf->fmt_in.h);
+    return img;
+}
+
 static struct mp_image *render(struct vf_instance *vf)
 {
     struct vf_priv_s *p = vf->priv;
@@ -167,15 +193,13 @@ static struct mp_image *render(struct vf_instance *vf)
     VABufferID buffer = VA_INVALID_ID;
 
     VASurfaceID in_id = va_surface_id(in);
-    if (!p->pipe.filters || in_id == VA_INVALID_ID)
+    if (!p->pipe.filters || in_id == VA_INVALID_ID || !p->hw_pool)
         goto cleanup;
 
-    int r_w, r_h;
-    va_surface_get_uncropped_size(in, &r_w, &r_h);
-    img = mp_image_pool_get(p->pool, IMGFMT_VAAPI, r_w, r_h);
+    img = alloc_out(vf);
     if (!img)
         goto cleanup;
-    mp_image_set_size(img, in->w, in->h);
+
     mp_image_copy_attributes(img, in);
 
     unsigned int flags = va_get_colorspace_flag(p->params.color.space);
@@ -264,11 +288,10 @@ cleanup:
 
 static struct mp_image *upload(struct vf_instance *vf, struct mp_image *in)
 {
-    struct vf_priv_s *p = vf->priv;
-    struct mp_image *out = mp_image_pool_get(p->pool, IMGFMT_VAAPI, in->w, in->h);
+    struct mp_image *out = alloc_out(vf);
     if (!out)
         return NULL;
-    if (va_surface_upload(out, in) < 0) {
+    if (!mp_image_hw_upload(out, in)) {
         talloc_free(out);
         return NULL;
     }
@@ -323,23 +346,39 @@ static int reconfig(struct vf_instance *vf, struct mp_image_params *in,
     struct vf_priv_s *p = vf->priv;
 
     flush_frames(vf);
-    talloc_free(p->pool);
-    p->pool = NULL;
+    av_buffer_unref(&p->hw_pool);
 
     p->params = *in;
+    *out = *in;
 
-    p->current_rt_format = VA_RT_FORMAT_YUV420;
-    p->pool = mp_image_pool_new(20);
-    va_pool_set_allocator(p->pool, p->va, p->current_rt_format);
+    int src_w = in->w;
+    int src_h = in->h;
 
-    struct mp_image *probe = mp_image_pool_get(p->pool, IMGFMT_VAAPI, in->w, in->h);
-    if (!probe)
+    if (in->imgfmt == IMGFMT_VAAPI) {
+        if (!vf->in_hwframes_ref)
+            return -1;
+        AVHWFramesContext *hw_frames = (void *)vf->in_hwframes_ref->data;
+        // VAAPI requires the full surface size to match for input and output.
+        src_w = hw_frames->width;
+        src_h = hw_frames->height;
+    } else {
+        out->imgfmt = IMGFMT_VAAPI;
+        out->hw_subfmt = IMGFMT_NV12;
+    }
+
+    p->hw_pool = av_hwframe_ctx_alloc(p->va->av_device_ref);
+    if (!p->hw_pool)
         return -1;
-    va_surface_init_subformat(probe);
-    *out = *in;
-    out->imgfmt = probe->params.imgfmt;
-    out->hw_subfmt = probe->params.hw_subfmt;
-    talloc_free(probe);
+    AVHWFramesContext *hw_frames = (void *)p->hw_pool->data;
+    hw_frames->format = AV_PIX_FMT_VAAPI;
+    hw_frames->sw_format = imgfmt2pixfmt(out->hw_subfmt);
+    hw_frames->width = src_w;
+    hw_frames->height = src_h;
+    if (av_hwframe_ctx_init(p->hw_pool) < 0) {
+        MP_ERR(vf, "Failed to initialize libavutil vaapi frames pool.\n");
+        av_buffer_unref(&p->hw_pool);
+        return -1;
+    }
 
     return 0;
 }
@@ -353,7 +392,7 @@ static void uninit(struct vf_instance *vf)
         vaDestroyContext(p->display, p->context);
     if (p->config != VA_INVALID_ID)
         vaDestroyConfig(p->display, p->config);
-    talloc_free(p->pool);
+    av_buffer_unref(&p->hw_pool);
     flush_frames(vf);
     mp_refqueue_free(p->queue);
 }
@@ -361,7 +400,12 @@ static void uninit(struct vf_instance *vf)
 static int query_format(struct vf_instance *vf, unsigned int imgfmt)
 {
     struct vf_priv_s *p = vf->priv;
-    if (imgfmt == IMGFMT_VAAPI || va_image_format_from_imgfmt(p->va, imgfmt))
+
+    bool supported = false;
+    for (int n = 0; n < p->num_in_formats; n++)
+        supported |= imgfmt == p->in_formats[n];
+
+    if (imgfmt == IMGFMT_VAAPI || supported)
         return vf_next_query_format(vf, IMGFMT_VAAPI);
     return 0;
 }
@@ -469,9 +513,23 @@ static int vf_open(vf_instance_t *vf)
     p->queue = mp_refqueue_alloc();
 
     p->va = hwdec_devices_load(vf->hwdec_devs, HWDEC_VAAPI);
-    if (!p->va)
+    if (!p->va || !p->va->av_device_ref) {
+        uninit(vf);
         return 0;
+    }
     p->display = p->va->display;
+
+    AVBufferRef *device_ref = (void *)p->va->av_device_ref;
+    AVHWFramesConstraints *constraints =
+        av_hwdevice_get_hwframe_constraints(device_ref, NULL);
+    const enum AVPixelFormat *fmts = constraints->valid_sw_formats;
+    for (int n = 0; fmts && fmts[n] != AV_PIX_FMT_NONE; n++) {
+        int mpfmt = pixfmt2imgfmt(fmts[n]);
+        if (mpfmt)
+            MP_TARRAY_APPEND(p, p->in_formats, p->num_in_formats, mpfmt);
+    }
+    av_hwframe_constraints_free(&constraints);
+
     if (initialize(vf))
         return true;
     uninit(vf);