f_hwtransfer: mp_image_pool: support HW -> HW mapping

Certain combinations of hardware formats require the use of hwmap to
transfer frames between the formats, rather than hwupload, which will
fail if attempted.

To keep the usage of vf_format for HW -> HW transfers as intuitive as
possible, we should detect these cases and do the map operation instead
of uploading.

For now, the relevant cases are moving between VAAPI and Vulkan, and
VAAPI and DRM Prime, in both directions. I have introduced the IMGFMT
entry for Vulkan here so that I can put in the complete mapping table.
It's actually not useless, as you can map to Vulkan, use a Vulkan
filter and then map back to VAAPI for display output.

4 files changed, 31 insertions, 0 deletions

diff --git a/video/fmt-conversion.c b/video/fmt-conversion.c
index 5194221d50..d200a5db31 100644
--- a/video/fmt-conversion.c
+++ b/video/fmt-conversion.c
@@ -67,6 +67,7 @@ static const struct {
     {IMGFMT_CUDA, AV_PIX_FMT_CUDA},
     {IMGFMT_P010, AV_PIX_FMT_P010},
     {IMGFMT_DRMPRIME, AV_PIX_FMT_DRM_PRIME},
+    {IMGFMT_VULKAN, AV_PIX_FMT_VULKAN},
 
     {0, AV_PIX_FMT_NONE}
 };
diff --git a/video/img_format.h b/video/img_format.h
index ebdbd66e29..7559c8c172 100644
--- a/video/img_format.h
+++ b/video/img_format.h
@@ -319,6 +319,7 @@ enum mp_imgfmt {
     IMGFMT_VDPAU_OUTPUT,    // VdpOutputSurface
     IMGFMT_VAAPI,
     IMGFMT_VIDEOTOOLBOX,    // CVPixelBufferRef
+    IMGFMT_VULKAN,          // VKImage
 
     // Generic pass-through of AV_PIX_FMT_*. Used for formats which don't have
     // a corresponding IMGFMT_ value.
diff --git a/video/mp_image_pool.c b/video/mp_image_pool.c
index 2a4a8940c0..d80c65d3b5 100644
--- a/video/mp_image_pool.c
+++ b/video/mp_image_pool.c
@@ -426,3 +426,30 @@ struct mp_image *mp_av_pool_image_hw_upload(struct AVBufferRef *hw_frames_ctx,
     mp_image_copy_attributes(dst, src);
     return dst;
 }
+
+struct mp_image *mp_av_pool_image_hw_map(struct AVBufferRef *hw_frames_ctx,
+                                         struct mp_image *src)
+{
+    AVFrame *dst_frame = av_frame_alloc();
+    if (!dst_frame)
+        return NULL;
+
+    dst_frame->format = ((AVHWFramesContext*)hw_frames_ctx->data)->format;
+    dst_frame->hw_frames_ctx = av_buffer_ref(hw_frames_ctx);
+
+    AVFrame *src_frame = mp_image_to_av_frame(src);
+    if (av_hwframe_map(dst_frame, src_frame, 0) < 0) {
+        av_frame_free(&src_frame);
+        av_frame_free(&dst_frame);
+        return NULL;
+    }
+    av_frame_free(&src_frame);
+
+    struct mp_image *dst = mp_image_from_av_frame(dst_frame);
+    av_frame_free(&dst_frame);
+    if (!dst)
+        return NULL;
+
+    mp_image_copy_attributes(dst, src);
+    return dst;
+}
diff --git a/video/mp_image_pool.h b/video/mp_image_pool.h
index 3c7e6221d1..ab9065e6e9 100644
--- a/video/mp_image_pool.h
+++ b/video/mp_image_pool.h
@@ -41,4 +41,6 @@ bool mp_update_av_hw_frames_pool(struct AVBufferRef **hw_frames_ctx,
 struct mp_image *mp_av_pool_image_hw_upload(struct AVBufferRef *hw_frames_ctx,
                                             struct mp_image *src);
 
+struct mp_image *mp_av_pool_image_hw_map(struct AVBufferRef *hw_frames_ctx,
+                                         struct mp_image *src);
 #endif