vo_gpu: hwdec_vaegl: refactor format probing

No functional changes, just preparation for the next commit. Split the
probing into multiple functions. Prepare for the yet unused possibility
to pass AVVAAPIHWConfig to probing. try_format_pixfmt() now assumes it
can be called multiple times with the same format, so it filters the
format.

The format probing is now something like O(n^2) for n formats, but n
will most likely remain something under 50 or so.

1 files changed, 64 insertions, 40 deletions

diff --git a/video/out/hwdec/hwdec_vaapi.c b/video/out/hwdec/hwdec_vaapi.c
index ee15214884..35164ca541 100644
--- a/video/out/hwdec/hwdec_vaapi.c
+++ b/video/out/hwdec/hwdec_vaapi.c
@@ -25,7 +25,11 @@
 
 #include "config.h"
 
+#include "video/out/gpu/hwdec.h"
 #include "video/out/hwdec/hwdec_vaapi.h"
+#include "video/fmt-conversion.h"
+#include "video/mp_image_pool.h"
+#include "video/vaapi.h"
 
 #if HAVE_VAAPI_DRM
 #include "libmpv/render_gl.h"
@@ -325,7 +329,7 @@ err:
     return -1;
 }
 
-static bool try_format(struct ra_hwdec *hw, struct mp_image *surface)
+static bool try_format_map(struct ra_hwdec *hw, struct mp_image *surface)
 {
     bool ok = false;
     struct ra_hwdec_mapper *mapper = ra_hwdec_mapper_create(hw, &surface->params);
@@ -335,59 +339,79 @@ static bool try_format(struct ra_hwdec *hw, struct mp_image *surface)
     return ok;
 }
 
-static void determine_working_formats(struct ra_hwdec *hw)
+static void try_format_pixfmt(struct ra_hwdec *hw, enum AVPixelFormat pixfmt)
 {
     struct priv_owner *p = hw->priv;
+
+    int mp_fmt = pixfmt2imgfmt(pixfmt);
+    if (!mp_fmt)
+        return;
+
     int num_formats = 0;
-    int *formats = NULL;
+    for (int n = 0; p->formats && p->formats[n]; n++) {
+        if (p->formats[n] == mp_fmt)
+            return; // already added
+        num_formats += 1;
+    }
 
-    p->probing_formats = true;
+    AVBufferRef *fref = NULL;
+    struct mp_image *s = NULL;
+    AVFrame *frame = NULL;
+    fref = av_hwframe_ctx_alloc(p->ctx->av_device_ref);
+    if (!fref)
+        goto err;
+    AVHWFramesContext *fctx = (void *)fref->data;
+    fctx->format = AV_PIX_FMT_VAAPI;
+    fctx->sw_format = pixfmt;
+    fctx->width = 128;
+    fctx->height = 128;
+    if (av_hwframe_ctx_init(fref) < 0)
+        goto err;
+    frame = av_frame_alloc();
+    if (!frame)
+        goto err;
+    if (av_hwframe_get_buffer(fref, frame, 0) < 0)
+        goto err;
+    s = mp_image_from_av_frame(frame);
+    if (!s || !mp_image_params_valid(&s->params))
+        goto err;
+    if (try_format_map(hw, s)) {
+        MP_TARRAY_APPEND(p, p->formats, num_formats, mp_fmt);
+        MP_TARRAY_APPEND(p, p->formats, num_formats, 0); // terminate it
+    }
+err:
+    talloc_free(s);
+    av_frame_free(&frame);
+    av_buffer_unref(&fref);
+}
 
+static void try_format_config(struct ra_hwdec *hw, AVVAAPIHWConfig *hwconfig)
+{
+    struct priv_owner *p = hw->priv;
     AVHWFramesConstraints *fc =
-            av_hwdevice_get_hwframe_constraints(p->ctx->av_device_ref, NULL);
+            av_hwdevice_get_hwframe_constraints(p->ctx->av_device_ref, hwconfig);
     if (!fc) {
         MP_WARN(hw, "failed to retrieve libavutil frame constraints\n");
-        goto done;
-    }
-    for (int n = 0; fc->valid_sw_formats[n] != AV_PIX_FMT_NONE; n++) {
-        AVBufferRef *fref = NULL;
-        struct mp_image *s = NULL;
-        AVFrame *frame = NULL;
-        fref = av_hwframe_ctx_alloc(p->ctx->av_device_ref);
-        if (!fref)
-            goto err;
-        AVHWFramesContext *fctx = (void *)fref->data;
-        fctx->format = AV_PIX_FMT_VAAPI;
-        fctx->sw_format = fc->valid_sw_formats[n];
-        fctx->width = 128;
-        fctx->height = 128;
-        if (av_hwframe_ctx_init(fref) < 0)
-            goto err;
-        frame = av_frame_alloc();
-        if (!frame)
-            goto err;
-        if (av_hwframe_get_buffer(fref, frame, 0) < 0)
-            goto err;
-        s = mp_image_from_av_frame(frame);
-        if (!s || !mp_image_params_valid(&s->params))
-            goto err;
-        if (try_format(hw, s))
-            MP_TARRAY_APPEND(p, formats, num_formats, s->params.hw_subfmt);
-    err:
-        talloc_free(s);
-        av_frame_free(&frame);
-        av_buffer_unref(&fref);
+        return;
     }
+    for (int n = 0; fc->valid_sw_formats[n] != AV_PIX_FMT_NONE; n++)
+        try_format_pixfmt(hw, fc->valid_sw_formats[n]);
     av_hwframe_constraints_free(&fc);
+}
+
+static void determine_working_formats(struct ra_hwdec *hw)
+{
+    struct priv_owner *p = hw->priv;
+
+    p->probing_formats = true;
+
+    try_format_config(hw, NULL);
 
-done:
-    MP_TARRAY_APPEND(p, formats, num_formats, 0); // terminate it
-    p->formats = formats;
     p->probing_formats = false;
 
     MP_VERBOSE(hw, "Supported formats:\n");
-    for (int n = 0; formats[n]; n++)
-        MP_VERBOSE(hw, " %s\n", mp_imgfmt_to_name(formats[n]));
+    for (int n = 0; p->formats && p->formats[n]; n++)
+        MP_VERBOSE(hw, " %s\n", mp_imgfmt_to_name(p->formats[n]));
 }
 
 const struct ra_hwdec_driver ra_hwdec_vaegl = {