vd_lavc: move most vaapi hwaccel setup code to generic code

Now hw_vaapi.c contains only the device setup, which could probably also
be abstracted.

3 files changed, 95 insertions, 48 deletions

diff --git a/video/decode/hw_vaapi.c b/video/decode/hw_vaapi.c
index 99c23f48c4..81854207e2 100644
--- a/video/decode/hw_vaapi.c
+++ b/video/decode/hw_vaapi.c
@@ -34,49 +34,11 @@
 #include "video/hwdec.h"
 #include "video/filter/vf.h"
 
-#define ADDITIONAL_SURFACES (HWDEC_EXTRA_SURFACES + HWDEC_DELAY_QUEUE_COUNT)
-
 struct priv {
     struct mp_log *log;
     struct mp_vaapi_ctx *ctx;
-    struct mp_hwdec_ctx *hwdev;
 };
 
-static int init_decoder(struct lavc_ctx *ctx, int w, int h)
-{
-    struct priv *p = ctx->hwdec_priv;
-    // libavcodec has no way yet to communicate the exact surface format needed
-    // for the frame pool, or the required minimum size of the frame pool.
-    // Hopefully, this weakness in the libavcodec API will be fixed in the
-    // future.
-    // For the pixel format, we try to second-guess from what the libavcodec
-    // software decoder would require (sw_pix_fmt). It could break and require
-    // adjustment if new VAAPI surface formats are added.
-    int sw_format = ctx->avctx->sw_pix_fmt == AV_PIX_FMT_YUV420P10 ?
-                    AV_PIX_FMT_P010 : AV_PIX_FMT_NV12;
-
-    // The video output might not support all formats.
-    // Note that supported_formats==NULL means any are accepted.
-    if (p->hwdev && p->hwdev->supported_formats) {
-        int mp_format = pixfmt2imgfmt(sw_format);
-        bool found = false;
-        for (int n = 0; p->hwdev->supported_formats[n]; n++) {
-            if (p->hwdev->supported_formats[n] == mp_format) {
-                found = true;
-                break;
-            }
-        }
-        if (!found) {
-            MP_WARN(ctx, "Surface format %s not supported for direct rendering.\n",
-                    mp_imgfmt_to_name(mp_format));
-            return -1;
-        }
-    }
-
-    return hwdec_setup_hw_frames_ctx(ctx, p->ctx->av_device_ref, sw_format,
-                                hwdec_get_max_refs(ctx) + ADDITIONAL_SURFACES);
-}
-
 static void uninit(struct lavc_ctx *ctx)
 {
     struct priv *p = ctx->hwdec_priv;
@@ -84,11 +46,11 @@ static void uninit(struct lavc_ctx *ctx)
     if (!p)
         return;
 
-    if (!p->hwdev)
-        va_destroy(p->ctx);
+    va_destroy(p->ctx);
 
     talloc_free(p);
     ctx->hwdec_priv = NULL;
+    ctx->hwdec_dev = NULL;
 }
 
 static int init(struct lavc_ctx *ctx, bool direct)
@@ -99,19 +61,19 @@ static int init(struct lavc_ctx *ctx, bool direct)
     };
 
     if (direct) {
-        p->hwdev = hwdec_devices_get(ctx->hwdec_devs, HWDEC_VAAPI);
-        p->ctx = p->hwdev->ctx;
+        ctx->hwdec_dev = hwdec_devices_get(ctx->hwdec_devs, HWDEC_VAAPI);
     } else {
         p->ctx = va_create_standalone(ctx->log, false);
         if (!p->ctx) {
             talloc_free(p);
             return -1;
         }
+        ctx->hwdec_dev = &p->ctx->hwctx;
     }
 
     ctx->hwdec_priv = p;
 
-    if (!p->ctx->av_device_ref)
+    if (!ctx->hwdec_dev->av_device_ref)
         return -1;
 
     return 0;
@@ -151,21 +113,30 @@ static int init_copy(struct lavc_ctx *ctx)
 const struct vd_lavc_hwdec mp_vd_lavc_vaapi = {
     .type = HWDEC_VAAPI,
     .image_format = IMGFMT_VAAPI,
-    .volatile_context = true,
     .probe = probe,
     .init = init_direct,
     .uninit = uninit,
-    .init_decoder = init_decoder,
+    .generic_hwaccel = true,
+    .static_pool = true,
+    .pixfmt_map = (const enum AVPixelFormat[][2]) {
+        {AV_PIX_FMT_YUV420P10, AV_PIX_FMT_P010},
+        {AV_PIX_FMT_YUV420P,   AV_PIX_FMT_NV12},
+        {AV_PIX_FMT_NONE}
+    },
 };
 
 const struct vd_lavc_hwdec mp_vd_lavc_vaapi_copy = {
     .type = HWDEC_VAAPI_COPY,
     .copying = true,
     .image_format = IMGFMT_VAAPI,
-    .volatile_context = true,
     .probe = probe_copy,
     .init = init_copy,
     .uninit = uninit,
-    .init_decoder = init_decoder,
-    .delay_queue = HWDEC_DELAY_QUEUE_COUNT,
+    .generic_hwaccel = true,
+    .static_pool = true,
+    .pixfmt_map = (const enum AVPixelFormat[][2]) {
+        {AV_PIX_FMT_YUV420P10, AV_PIX_FMT_P010},
+        {AV_PIX_FMT_YUV420P,   AV_PIX_FMT_NV12},
+        {AV_PIX_FMT_NONE}
+    },
 };
diff --git a/video/decode/lavc.h b/video/decode/lavc.h
index 46bf85a6e5..b41d56ce2f 100644
--- a/video/decode/lavc.h
+++ b/video/decode/lavc.h
@@ -55,6 +55,9 @@ typedef struct lavc_ctx {
     // For free use by hwdec implementation
     void *hwdec_priv;
 
+    // Set by generic hwaccels.
+    struct mp_hwdec_ctx *hwdec_dev;
+
     int hwdec_fmt;
     int hwdec_w;
     int hwdec_h;
@@ -98,6 +101,16 @@ struct vd_lavc_hwdec {
     // with hwdec_find_decoder.
     // Intuitively, this will force the corresponding wrapper decoder.
     const char *lavc_suffix;
+    // Generic hwaccels set AVCodecContext.hw_frames_ctx in get_format().
+    // pixfmt_map must be non-NULL.
+    // struct lavc_ctx.hwdec_dev must be set at runtime (in init).
+    bool generic_hwaccel;
+    // Array of pixfmt pairs. The first pixfmt is the AVCodecContext.sw_pix_fmt,
+    // the second the required AVHWFramesContext.sw_format.
+    const enum AVPixelFormat (*pixfmt_map)[2];
+    // The generic hwaccel has a fixed pool size. Enough surfaces need to be
+    // preallocated before decoding begins. If false, pool size is left to 0.
+    bool static_pool;
 };
 
 enum {
diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index ed5e0ad851..77b8bf439d 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -677,6 +677,64 @@ int hwdec_setup_hw_frames_ctx(struct lavc_ctx *ctx, AVBufferRef *device_ctx,
     return ctx->avctx->hw_frames_ctx ? 0 : -1;
 }
 
+static int init_generic_hwaccel(struct dec_video *vd)
+{
+    struct lavc_ctx *ctx = vd->priv;
+    struct vd_lavc_hwdec *hwdec = ctx->hwdec;
+
+    if (!ctx->hwdec_dev)
+        return -1;
+
+    // libavcodec has no way yet to communicate the exact surface format needed
+    // for the frame pool, or the required minimum size of the frame pool.
+    // Hopefully, this weakness in the libavcodec API will be fixed in the
+    // future.
+    // For the pixel format, we try to second-guess from what the libavcodec
+    // software decoder would require (sw_pix_fmt). It could break and require
+    // adjustment if new hwaccel surface formats are added.
+    enum AVPixelFormat av_sw_format = AV_PIX_FMT_NONE;
+    for (int n = 0; hwdec->pixfmt_map[n][0] != AV_PIX_FMT_NONE; n++) {
+        if (ctx->avctx->sw_pix_fmt == hwdec->pixfmt_map[n][0]) {
+            av_sw_format = hwdec->pixfmt_map[n][1];
+            break;
+        }
+    }
+
+    if (av_sw_format == AV_PIX_FMT_NONE) {
+        MP_VERBOSE(ctx, "Unsupported hw decoding format: %s\n",
+                   mp_imgfmt_to_name(pixfmt2imgfmt(ctx->avctx->sw_pix_fmt)));
+        return -1;
+    }
+
+    // The video output might not support all formats.
+    // Note that supported_formats==NULL means any are accepted.
+    int *render_formats = ctx->hwdec_dev->supported_formats;
+    if (render_formats) {
+        int mp_format = pixfmt2imgfmt(av_sw_format);
+        bool found = false;
+        for (int n = 0; render_formats[n]; n++) {
+            if (render_formats[n] == mp_format) {
+                found = true;
+                break;
+            }
+        }
+        if (!found) {
+            MP_WARN(ctx, "Surface format %s not supported for direct rendering.\n",
+                    mp_imgfmt_to_name(mp_format));
+            return -1;
+        }
+    }
+
+    int pool_size = 0;
+    if (hwdec->static_pool)
+        pool_size = hwdec_get_max_refs(ctx) + HWDEC_EXTRA_SURFACES;
+
+    ctx->hwdec_fmt = hwdec->image_format;
+
+    return hwdec_setup_hw_frames_ctx(ctx, ctx->hwdec_dev->av_device_ref,
+                                     av_sw_format, pool_size);
+}
+
 static enum AVPixelFormat get_format_hwdec(struct AVCodecContext *avctx,
                                            const enum AVPixelFormat *fmt)
 {
@@ -699,6 +757,11 @@ static enum AVPixelFormat get_format_hwdec(struct AVCodecContext *avctx,
 
     for (int i = 0; fmt[i] != AV_PIX_FMT_NONE; i++) {
         if (ctx->hwdec->image_format == pixfmt2imgfmt(fmt[i])) {
+            if (ctx->hwdec->generic_hwaccel) {
+                if (init_generic_hwaccel(vd) < 0)
+                    break;
+                return fmt[i];
+            }
             // There could be more reasons for a change, and it's possible
             // that we miss some. (Might also depend on the hwaccel type.)
             bool change =