From 7555e2a42afbff37fd0f226442f07ca862325af8 Mon Sep 17 00:00:00 2001
From: wm4 <wm4@nowhere>
Date: Sat, 11 Jan 2020 22:41:38 +0100
Subject: f_hwtransfer: whitelist vaapi formats that actually appear to work

(Oh yes, we could have skipped all the complexity, and hardcoded the
cases that work in the first place. This wouldn't be an issue if FFmpeg
or libva exported correct information. Also possible that FFmpeg's
filter chain does not allow to do this correctly in the first place,
since filters expose next to no meta information about what hw formats
etc. they need.)

Note that uploading yuv420p to a nv12 vaapi surface actually works, but
the blacklist excludes it. So this might get a bit slower. I'm not
bothering with this case because it's rarely needed, and the blacklist
specification would become a bit more complex if you had to specify
sw/upload format pairs.

Fixes: #7350
---
 filters/f_hwtransfer.c | 53 +++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 50 insertions(+), 3 deletions(-)

diff --git a/filters/f_hwtransfer.c b/filters/f_hwtransfer.c
index 51ffbadbb9..f75e2dab28 100644
--- a/filters/f_hwtransfer.c
+++ b/filters/f_hwtransfer.c
@@ -22,6 +22,25 @@ struct priv {
     struct mp_hwupload public;
 };
 
+struct ffmpeg_and_other_bugs {
+    int imgfmt;                             // hw format
+    const int *const whitelist_formats;     // if non-NULL, allow only these
+                                            // sw formats
+    bool force_same_upload_fmt;             // force upload fmt == sw fmt
+};
+
+// This garbage is so complex and buggy. Hardcoding knowledge makes it work,
+// trying to use the dynamic information returned by the API does not. So fuck
+// this shit, I'll just whitelist the cases that work, what the fuck.
+static const struct ffmpeg_and_other_bugs shitlist[] = {
+    {
+        .imgfmt = IMGFMT_VAAPI,
+        .whitelist_formats = (const int[]){IMGFMT_NV12, IMGFMT_P010, 0},
+        .force_same_upload_fmt = true,
+    },
+    {0}
+};
+
 static bool update_format_decision(struct priv *p, int input_fmt)
 {
     struct mp_hwupload *u = &p->public;
@@ -223,6 +242,14 @@ static bool probe_formats(struct mp_hwupload *u, int hw_imgfmt)
     // supported formats. This should be relatively cheap as we don't create
     // any real frames (although some backends do for probing info).
 
+    const struct ffmpeg_and_other_bugs *bugs = NULL;
+    for (int n = 0; shitlist[n].imgfmt; n++) {
+        if (shitlist[n].imgfmt == hw_imgfmt) {
+            bugs = &shitlist[n];
+            break;
+        }
+    }
+
     for (int n = 0; cstr->valid_sw_formats &&
                     cstr->valid_sw_formats[n] != AV_PIX_FMT_NONE; n++)
     {
@@ -230,12 +257,28 @@ static bool probe_formats(struct mp_hwupload *u, int hw_imgfmt)
         if (!imgfmt)
             continue;
 
-        MP_VERBOSE(u->f, "looking at format %s\n", mp_imgfmt_to_name(imgfmt));
+        MP_VERBOSE(u->f, "looking at format %s/%s\n",
+                   mp_imgfmt_to_name(hw_imgfmt),
+                   mp_imgfmt_to_name(imgfmt));
+
+        if (bugs && bugs->whitelist_formats) {
+            bool found = false;
+            for (int i = 0; bugs->whitelist_formats[i]; i++) {
+                if (bugs->whitelist_formats[i] == imgfmt) {
+                    found = true;
+                    break;
+                }
+            }
+            if (!found) {
+                MP_VERBOSE(u->f, "... skipping blacklisted format\n");
+                continue;
+            }
+        }
 
         // Creates an AVHWFramesContexts with the given parameters.
         AVBufferRef *frames = NULL;
         if (!mp_update_av_hw_frames_pool(&frames, ctx->av_device_ref,
-                                        hw_imgfmt, imgfmt, 128, 128))
+                                         hw_imgfmt, imgfmt, 128, 128))
         {
             MP_WARN(u->f, "failed to allocate pool\n");
             continue;
@@ -256,7 +299,11 @@ static bool probe_formats(struct mp_hwupload *u, int hw_imgfmt)
                 int fmt = pixfmt2imgfmt(fmts[i]);
                 if (!fmt)
                     continue;
-                MP_VERBOSE(u->f, "supports %s\n", mp_imgfmt_to_name(fmt));
+                MP_VERBOSE(u->f, "  supports %s\n", mp_imgfmt_to_name(fmt));
+                if (bugs && bugs->force_same_upload_fmt && imgfmt != fmt) {
+                    MP_VERBOSE(u->f, "  ... skipping blacklisted format\n");
+                    continue;
+                }
                 if (vo_supports(ctx, hw_imgfmt, fmt))
                     MP_TARRAY_APPEND(p, u->upload_fmts, u->num_upload_fmts, fmt);
             }
-- 
cgit v1.2.3