f_autoconvert: f_hwtransfer: support HW -> HW uploads

Historically, HW -> HW uploads did not exist, so the current code
assumes they will never happen. But as part of introducing Vulkan
support into ffmpeg, we added HW -> HW support to enable transfers
between Vulkan and CUDA.

Today, that means you can use the lavfi hwupload filter with the
correct configuration (and previous changes in this series) but it
would be more convenient to enable HW -> HW in the format filter so
that the transfers can be done more intuitively:

```
--vf=format=fmt=cuda
```

and

```
--vf=format=fmt=vulkan
```

Most of the work here is skipping logic that is specific to SW -> HW
uploads doing format conversion. There is no ability to do inline
conversion when moving between HW formats, so the format must be
mutually understood to begin with.

Additional work needs to be done to enable transfers between VAAPI
and Vulkan which uses mapping, rather than uploads. I'll tackle that
in the next change.

3 files changed, 70 insertions, 20 deletions

diff --git a/filters/f_autoconvert.c b/filters/f_autoconvert.c
index 7452a13ae8..a94e3f8cf5 100644
--- a/filters/f_autoconvert.c
+++ b/filters/f_autoconvert.c
@@ -191,21 +191,31 @@ static bool build_image_converter(struct mp_autoconvert *c, struct mp_log *log,
         dst_have_sw |= !is_hw;
     }
 
-    // Source is sw, all targets are hw -> try to upload.
-    bool sw_to_hw = imgfmt_is_sw && dst_all_hw;
     // Source is hw, some targets are sw -> try to download.
     bool hw_to_sw = !imgfmt_is_sw && dst_have_sw;
 
-    if (sw_to_hw && num_fmts > 0) {
+    if (dst_all_hw && num_fmts > 0) {
         // We can probably use this! Very lazy and very approximate.
         struct mp_hwupload *upload = mp_hwupload_create(conv, fmts[0]);
         if (upload) {
+            int sw_fmt = imgfmt_is_sw ? img->imgfmt : img->params.hw_subfmt;
+
             mp_info(log, "HW-uploading to %s\n", mp_imgfmt_to_name(fmts[0]));
             filters[2] = upload->f;
-            hwupload_fmt = mp_hwupload_find_upload_format(upload, img->imgfmt);
+            hwupload_fmt = mp_hwupload_find_upload_format(upload, sw_fmt);
             fmts = &hwupload_fmt;
             num_fmts = hwupload_fmt ? 1 : 0;
             hw_to_sw = false;
+
+            // We cannot do format conversions when transferring between
+            // two hardware devices, so fail immediately if that would be
+            // required.
+            if (!imgfmt_is_sw && hwupload_fmt != sw_fmt) {
+                mp_err(log, "Format %s is not supported by %s\n",
+                       mp_imgfmt_to_name(sw_fmt),
+                       mp_imgfmt_to_name(p->imgfmts[0]));
+                goto fail;
+            }
         }
     }
 
@@ -235,6 +245,11 @@ static bool build_image_converter(struct mp_autoconvert *c, struct mp_log *log,
         force_sws_params |= !mp_image_params_equal(&imgpar, &p->imgparams);
         need_sws |= force_sws_params;
     }
+    if (!imgfmt_is_sw && dst_all_hw) {
+        // This is a hw -> hw upload, so the sw format must already be
+        // mutually understood. No conversion can be done.
+        need_sws = false;
+    }
 
     if (need_sws) {
         // Create a new conversion filter.
diff --git a/filters/f_hwtransfer.c b/filters/f_hwtransfer.c
index 8f7cd4a5a7..649d28497c 100644
--- a/filters/f_hwtransfer.c
+++ b/filters/f_hwtransfer.c
@@ -58,12 +58,18 @@ static const struct ffmpeg_and_other_bugs shitlist[] = {
     {0}
 };
 
-static bool select_format(struct priv *p, int input_fmt, int *out_sw_fmt,
-                          int *out_upload_fmt)
+static bool select_format(struct priv *p, int input_fmt,
+                          int *out_sw_fmt, int *out_upload_fmt)
 {
     if (!input_fmt)
         return false;
 
+    // If the input format is a hw format, then we shouldn't be doing this
+    // format selection here at all.
+    if (IMGFMT_IS_HWACCEL(input_fmt)) {
+        return false;
+    }
+
     // First find the closest sw fmt. Some hwdec APIs return crazy lists of
     // "supported" formats, which then are not supported or crash (???), so
     // the this is a good way to avoid problems.
@@ -104,7 +110,7 @@ int mp_hwupload_find_upload_format(struct mp_hwupload *u, int imgfmt)
 
     int sw = 0, up = 0;
     select_format(p, imgfmt, &sw, &up);
-    return up;
+    return sw;
 }
 
 static void process(struct mp_filter *f)
@@ -125,8 +131,16 @@ static void process(struct mp_filter *f)
     }
     struct mp_image *src = frame.data;
 
-    // As documented, just pass though HW frames.
-    if (IMGFMT_IS_HWACCEL(src->imgfmt)) {
+    /*
+     * Just pass though HW frames in the same format. This shouldn't normally
+     * occur as the upload filter will not be inserted when the formats already
+     * match.
+     *
+     * Technically, we could have frames from different device contexts,
+     * which would require an explicit transfer, but mpv doesn't let you
+     * create that configuration.
+     */
+    if (src->imgfmt == p->hw_imgfmt) {
         mp_pin_in_write(f->ppins[1], frame);
         return;
     }
@@ -137,16 +151,23 @@ static void process(struct mp_filter *f)
     }
 
     if (src->imgfmt != p->last_input_fmt) {
-        if (!select_format(p, src->imgfmt, &p->last_sw_fmt, &p->last_upload_fmt))
-        {
-            MP_ERR(f, "no hw upload format found\n");
-            goto error;
-        }
-        if (src->imgfmt != p->last_upload_fmt) {
-            // Should not fail; if it does, mp_hwupload_find_upload_format()
-            // does not return the src->imgfmt format.
-            MP_ERR(f, "input format not an upload format\n");
-            goto error;
+        if (IMGFMT_IS_HWACCEL(src->imgfmt)) {
+            // Because there cannot be any conversion of the sw format when the
+            // input is a hw format, just pick the source sw format.
+            p->last_sw_fmt = src->params.hw_subfmt;
+        } else {
+            if (!select_format(p, src->imgfmt,
+                               &p->last_sw_fmt, &p->last_upload_fmt))
+            {
+                MP_ERR(f, "no hw upload format found\n");
+                goto error;
+            }
+            if (src->imgfmt != p->last_upload_fmt) {
+                // Should not fail; if it does, mp_hwupload_find_upload_format()
+                // does not return the src->imgfmt format.
+                MP_ERR(f, "input format not an upload format\n");
+                goto error;
+            }
         }
         p->last_input_fmt = src->imgfmt;
         MP_INFO(f, "upload %s -> %s[%s]\n",
@@ -228,6 +249,12 @@ static bool probe_formats(struct mp_hwupload *u, int hw_imgfmt)
     struct mp_hwdec_ctx *ctx = NULL;
     AVHWFramesConstraints *cstr = NULL;
 
+    struct hwdec_imgfmt_request params = {
+        .imgfmt = hw_imgfmt,
+        .probing = true,
+    };
+    hwdec_devices_request_for_img_fmt(info->hwdec_devs, &params);
+
     for (int n = 0; ; n++) {
         struct mp_hwdec_ctx *cur = hwdec_devices_get_n(info->hwdec_devs, n);
         if (!cur)
@@ -294,6 +321,14 @@ static bool probe_formats(struct mp_hwupload *u, int hw_imgfmt)
             }
         }
 
+        if (IMGFMT_IS_HWACCEL(imgfmt)) {
+            // If the enumerated format is a hardware format, we don't need to
+            // do any further probing. It will be supported.
+            MP_VERBOSE(u->f, "  supports %s (a hardware format)\n",
+                       mp_imgfmt_to_name(imgfmt));
+            continue;
+        }
+
         // Creates an AVHWFramesContexts with the given parameters.
         AVBufferRef *frames = NULL;
         if (!mp_update_av_hw_frames_pool(&frames, ctx->av_device_ref,
diff --git a/video/mp_image_pool.c b/video/mp_image_pool.c
index 8aa584b33e..2a4a8940c0 100644
--- a/video/mp_image_pool.c
+++ b/video/mp_image_pool.c
@@ -325,7 +325,7 @@ bool mp_image_hw_upload(struct mp_image *hw_img, struct mp_image *src)
     if (hw_img->w != src->w || hw_img->h != src->h)
         return false;
 
-    if (!hw_img->hwctx || src->hwctx)
+    if (!hw_img->hwctx)
         return false;
 
     bool ok = false;