f_hwtransfer: disable vulkan multiplane images when uploading from cuda

Although we can support vulkan multiplane images, cuda lacks any such
support, and so cannot natively import such images for interop. It's
possible that we can do separate exports for each plane in the image
and have it work, but for now, we can selectively disable multiplane
when we know that we'll be consuming cuda frames.

As a reminder, even though cuda is the frame source, interop is one way
so the vulkan images have to be imported to cuda before we copy the
frame contents over.

This logic here is slightly more complex than I'd like but you can't
just set the flag blindly, as it will cause hwframes ctx creation to
fail if the format is packed or if it's planar rgb. Oh well.

3 files changed, 24 insertions, 3 deletions

diff --git a/video/filter/vf_vavpp.c b/video/filter/vf_vavpp.c
index 5b30c46163..52be148111 100644
--- a/video/filter/vf_vavpp.c
+++ b/video/filter/vf_vavpp.c
@@ -164,7 +164,8 @@ static struct mp_image *alloc_out(struct mp_filter *vf)
     int src_h = hw_frames->height;
 
     if (!mp_update_av_hw_frames_pool(&p->hw_pool, p->av_device_ref,
-                                     IMGFMT_VAAPI, IMGFMT_NV12, src_w, src_h))
+                                     IMGFMT_VAAPI, IMGFMT_NV12, src_w, src_h,
+                                     false))
     {
         MP_ERR(vf, "Failed to create hw pool.\n");
         return NULL;
diff --git a/video/mp_image_pool.c b/video/mp_image_pool.c
index 2f0259146a..8f982b414c 100644
--- a/video/mp_image_pool.c
+++ b/video/mp_image_pool.c
@@ -15,6 +15,8 @@
  * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "config.h"
+
 #include <stddef.h>
 #include <stdbool.h>
 #include <pthread.h>
@@ -22,7 +24,11 @@
 
 #include <libavutil/buffer.h>
 #include <libavutil/hwcontext.h>
+#if HAVE_VULKAN_INTEROP
+#include <libavutil/hwcontext_vulkan.h>
+#endif
 #include <libavutil/mem.h>
+#include <libavutil/pixdesc.h>
 
 #include "mpv_talloc.h"
 
@@ -354,7 +360,8 @@ done:
 
 bool mp_update_av_hw_frames_pool(struct AVBufferRef **hw_frames_ctx,
                                  struct AVBufferRef *hw_device_ctx,
-                                 int imgfmt, int sw_imgfmt, int w, int h)
+                                 int imgfmt, int sw_imgfmt, int w, int h,
+                                 bool disable_multiplane)
 {
     enum AVPixelFormat format = imgfmt2pixfmt(imgfmt);
     enum AVPixelFormat sw_format = imgfmt2pixfmt(sw_imgfmt);
@@ -385,6 +392,18 @@ bool mp_update_av_hw_frames_pool(struct AVBufferRef **hw_frames_ctx,
         hw_frames->sw_format = sw_format;
         hw_frames->width = w;
         hw_frames->height = h;
+
+#if HAVE_VULKAN_INTEROP
+        if (format == AV_PIX_FMT_VULKAN && disable_multiplane) {
+            const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sw_format);
+            if ((desc->flags & AV_PIX_FMT_FLAG_PLANAR) &&
+                !(desc->flags & AV_PIX_FMT_FLAG_RGB)) {
+                AVVulkanFramesContext *vk_frames = hw_frames->hwctx;
+                vk_frames->flags = AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE;
+            }
+        }
+#endif
+
         if (av_hwframe_ctx_init(*hw_frames_ctx) < 0) {
             av_buffer_unref(hw_frames_ctx);
             return false;
diff --git a/video/mp_image_pool.h b/video/mp_image_pool.h
index ab9065e6e9..8cb2a5f995 100644
--- a/video/mp_image_pool.h
+++ b/video/mp_image_pool.h
@@ -36,7 +36,8 @@ bool mp_image_hw_upload(struct mp_image *hw_img, struct mp_image *src);
 struct AVBufferRef;
 bool mp_update_av_hw_frames_pool(struct AVBufferRef **hw_frames_ctx,
                                  struct AVBufferRef *hw_device_ctx,
-                                 int imgfmt, int sw_imgfmt, int w, int h);
+                                 int imgfmt, int sw_imgfmt, int w, int h,
+                                 bool disable_multiplane);
 
 struct mp_image *mp_av_pool_image_hw_upload(struct AVBufferRef *hw_frames_ctx,
                                             struct mp_image *src);