f_autoconvert: f_hwtransfer: support HW -> HW uploads

Historically, HW -> HW uploads did not exist, so the current code
assumes they will never happen. But as part of introducing Vulkan
support into ffmpeg, we added HW -> HW support to enable transfers
between Vulkan and CUDA.

Today, that means you can use the lavfi hwupload filter with the
correct configuration (and previous changes in this series) but it
would be more convenient to enable HW -> HW in the format filter so
that the transfers can be done more intuitively:

```
--vf=format=fmt=cuda
```

and

```
--vf=format=fmt=vulkan
```

Most of the work here is skipping logic that is specific to SW -> HW
uploads doing format conversion. There is no ability to do inline
conversion when moving between HW formats, so the format must be
mutually understood to begin with.

Additional work needs to be done to enable transfers between VAAPI
and Vulkan which uses mapping, rather than uploads. I'll tackle that
in the next change.

1 files changed, 50 insertions, 15 deletions

diff --git a/filters/f_hwtransfer.c b/filters/f_hwtransfer.c
index 8f7cd4a5a7..649d28497c 100644
--- a/filters/f_hwtransfer.c
+++ b/filters/f_hwtransfer.c
@@ -58,12 +58,18 @@ static const struct ffmpeg_and_other_bugs shitlist[] = {
     {0}
 };
 
-static bool select_format(struct priv *p, int input_fmt, int *out_sw_fmt,
-                          int *out_upload_fmt)
+static bool select_format(struct priv *p, int input_fmt,
+                          int *out_sw_fmt, int *out_upload_fmt)
 {
     if (!input_fmt)
         return false;
 
+    // If the input format is a hw format, then we shouldn't be doing this
+    // format selection here at all.
+    if (IMGFMT_IS_HWACCEL(input_fmt)) {
+        return false;
+    }
+
     // First find the closest sw fmt. Some hwdec APIs return crazy lists of
     // "supported" formats, which then are not supported or crash (???), so
     // the this is a good way to avoid problems.
@@ -104,7 +110,7 @@ int mp_hwupload_find_upload_format(struct mp_hwupload *u, int imgfmt)
 
     int sw = 0, up = 0;
     select_format(p, imgfmt, &sw, &up);
-    return up;
+    return sw;
 }
 
 static void process(struct mp_filter *f)
@@ -125,8 +131,16 @@ static void process(struct mp_filter *f)
     }
     struct mp_image *src = frame.data;
 
-    // As documented, just pass though HW frames.
-    if (IMGFMT_IS_HWACCEL(src->imgfmt)) {
+    /*
+     * Just pass though HW frames in the same format. This shouldn't normally
+     * occur as the upload filter will not be inserted when the formats already
+     * match.
+     *
+     * Technically, we could have frames from different device contexts,
+     * which would require an explicit transfer, but mpv doesn't let you
+     * create that configuration.
+     */
+    if (src->imgfmt == p->hw_imgfmt) {
         mp_pin_in_write(f->ppins[1], frame);
         return;
     }
@@ -137,16 +151,23 @@ static void process(struct mp_filter *f)
     }
 
     if (src->imgfmt != p->last_input_fmt) {
-        if (!select_format(p, src->imgfmt, &p->last_sw_fmt, &p->last_upload_fmt))
-        {
-            MP_ERR(f, "no hw upload format found\n");
-            goto error;
-        }
-        if (src->imgfmt != p->last_upload_fmt) {
-            // Should not fail; if it does, mp_hwupload_find_upload_format()
-            // does not return the src->imgfmt format.
-            MP_ERR(f, "input format not an upload format\n");
-            goto error;
+        if (IMGFMT_IS_HWACCEL(src->imgfmt)) {
+            // Because there cannot be any conversion of the sw format when the
+            // input is a hw format, just pick the source sw format.
+            p->last_sw_fmt = src->params.hw_subfmt;
+        } else {
+            if (!select_format(p, src->imgfmt,
+                               &p->last_sw_fmt, &p->last_upload_fmt))
+            {
+                MP_ERR(f, "no hw upload format found\n");
+                goto error;
+            }
+            if (src->imgfmt != p->last_upload_fmt) {
+                // Should not fail; if it does, mp_hwupload_find_upload_format()
+                // does not return the src->imgfmt format.
+                MP_ERR(f, "input format not an upload format\n");
+                goto error;
+            }
         }
         p->last_input_fmt = src->imgfmt;
         MP_INFO(f, "upload %s -> %s[%s]\n",
@@ -228,6 +249,12 @@ static bool probe_formats(struct mp_hwupload *u, int hw_imgfmt)
     struct mp_hwdec_ctx *ctx = NULL;
     AVHWFramesConstraints *cstr = NULL;
 
+    struct hwdec_imgfmt_request params = {
+        .imgfmt = hw_imgfmt,
+        .probing = true,
+    };
+    hwdec_devices_request_for_img_fmt(info->hwdec_devs, &params);
+
     for (int n = 0; ; n++) {
         struct mp_hwdec_ctx *cur = hwdec_devices_get_n(info->hwdec_devs, n);
         if (!cur)
@@ -294,6 +321,14 @@ static bool probe_formats(struct mp_hwupload *u, int hw_imgfmt)
             }
         }
 
+        if (IMGFMT_IS_HWACCEL(imgfmt)) {
+            // If the enumerated format is a hardware format, we don't need to
+            // do any further probing. It will be supported.
+            MP_VERBOSE(u->f, "  supports %s (a hardware format)\n",
+                       mp_imgfmt_to_name(imgfmt));
+            continue;
+        }
+
         // Creates an AVHWFramesContexts with the given parameters.
         AVBufferRef *frames = NULL;
         if (!mp_update_av_hw_frames_pool(&frames, ctx->av_device_ref,