vo: hwdec: do hwdec interop lookup by image format

It turns out that it's generally more useful to look up hwdecs by image
format, rather than device type. In the situations where we need to
find one, we generally know the image format we're dealing with. Doing
this avoids us having to create mappings from image format to device
type.

The most significant part of this change is filling in the image format
for the various hw interops. There is a hw_imgfmt field today today, but
only a couple of the interops fill it in, and that seems to be because
we've never actually used this piece of metadata before. Well, now we
have a good use for it.

18 files changed, 33 insertions, 53 deletions

diff --git a/filters/f_hwtransfer.c b/filters/f_hwtransfer.c
index e46a6070e9..8f7cd4a5a7 100644
--- a/filters/f_hwtransfer.c
+++ b/filters/f_hwtransfer.c
@@ -198,10 +198,10 @@ static const struct mp_filter_info hwupload_filter = {
 // So filter out all not explicitly supported formats.
 static bool vo_supports(struct mp_hwdec_ctx *ctx, int hw_fmt, int sw_fmt)
 {
-    if (!ctx->hw_imgfmt)
-        return true; // if unset, all formats are allowed
     if (ctx->hw_imgfmt != hw_fmt)
         return false;
+    if (!ctx->supported_formats)
+        return true; // if unset, all formats are allowed
 
     for (int i = 0; ctx->supported_formats &&  ctx->supported_formats[i]; i++) {
         if (ctx->supported_formats[i] == sw_fmt)
diff --git a/filters/filter.c b/filters/filter.c
index d4a0dd6a0b..2522209b49 100644
--- a/filters/filter.c
+++ b/filters/filter.c
@@ -685,32 +685,19 @@ struct mp_stream_info *mp_filter_find_stream_info(struct mp_filter *f)
     return NULL;
 }
 
-struct AVBufferRef *mp_filter_load_hwdec_device(struct mp_filter *f, int avtype)
+struct AVBufferRef *mp_filter_load_hwdec_device(struct mp_filter *f, int imgfmt)
 {
     struct mp_stream_info *info = mp_filter_find_stream_info(f);
     if (!info || !info->hwdec_devs)
         return NULL;
 
-    int imgfmt = IMGFMT_NONE;
-    switch (avtype) {
-    case AV_HWDEVICE_TYPE_VAAPI:
-        imgfmt = IMGFMT_VAAPI;
-        break;
-    case AV_HWDEVICE_TYPE_VDPAU:
-        imgfmt = IMGFMT_VDPAU;
-        break;
-    default:
-        MP_WARN(f,
-               "Unrecognised HW Device type requested. Loading all devices\n");
-    }
-
     struct hwdec_imgfmt_request params = {
         .imgfmt = imgfmt,
         .probing = false,
     };
     hwdec_devices_request_for_img_fmt(info->hwdec_devs, &params);
 
-    return hwdec_devices_get_lavc(info->hwdec_devs, avtype);
+    return hwdec_devices_get_imgfmt(info->hwdec_devs, imgfmt);
 }
 
 static void filter_wakeup(struct mp_filter *f, bool mark_only)
diff --git a/filters/filter.h b/filters/filter.h
index 19aafc6f3d..52797eb433 100644
--- a/filters/filter.h
+++ b/filters/filter.h
@@ -409,7 +409,7 @@ struct mp_stream_info {
 struct mp_stream_info *mp_filter_find_stream_info(struct mp_filter *f);
 
 struct AVBufferRef;
-struct AVBufferRef *mp_filter_load_hwdec_device(struct mp_filter *f, int avtype);
+struct AVBufferRef *mp_filter_load_hwdec_device(struct mp_filter *f, int imgfmt);
 
 // Perform filtering. This runs until the filter graph is blocked (due to
 // missing external input or unread output). It returns whether any outside
diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index ce067d0ff4..4f38b669ec 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -435,12 +435,16 @@ static AVBufferRef *hwdec_create_dev(struct mp_filter *vd,
             return ref;
         }
     } else if (ctx->hwdec_devs) {
+        int imgfmt = pixfmt2imgfmt(hwdec->pix_fmt);
         struct hwdec_imgfmt_request params = {
-            .imgfmt = pixfmt2imgfmt(hwdec->pix_fmt),
+            .imgfmt = imgfmt,
             .probing = autoprobe,
         };
         hwdec_devices_request_for_img_fmt(ctx->hwdec_devs, &params);
-        return hwdec_devices_get_lavc(ctx->hwdec_devs, hwdec->lavc_device);
+
+        const struct mp_hwdec_ctx *hw_ctx =
+            hwdec_devices_get_by_imgfmt(ctx->hwdec_devs, imgfmt);
+        return hw_ctx ? av_buffer_ref(hw_ctx->av_device_ref) : NULL;
     }
 
     return NULL;
diff --git a/video/filter/vf_d3d11vpp.c b/video/filter/vf_d3d11vpp.c
index f36197c412..df029417b9 100644
--- a/video/filter/vf_d3d11vpp.c
+++ b/video/filter/vf_d3d11vpp.c
@@ -435,7 +435,7 @@ static struct mp_filter *vf_d3d11vpp_create(struct mp_filter *parent,
     hwdec_devices_request_for_img_fmt(info->hwdec_devs, &params);
 
     struct mp_hwdec_ctx *hwctx =
-        hwdec_devices_get_by_lavc(info->hwdec_devs, AV_HWDEVICE_TYPE_D3D11VA);
+        hwdec_devices_get_by_imgfmt(info->hwdec_devs, IMGFMT_D3D11);
     if (!hwctx || !hwctx->av_device_ref)
         goto fail;
     AVHWDeviceContext *avhwctx = (void *)hwctx->av_device_ref->data;
diff --git a/video/filter/vf_vavpp.c b/video/filter/vf_vavpp.c
index ad2ecc04dc..d84c393e29 100644
--- a/video/filter/vf_vavpp.c
+++ b/video/filter/vf_vavpp.c
@@ -448,7 +448,7 @@ static struct mp_filter *vf_vavpp_create(struct mp_filter *parent, void *options
 
     p->queue = mp_refqueue_alloc(f);
 
-    p->av_device_ref = mp_filter_load_hwdec_device(f, AV_HWDEVICE_TYPE_VAAPI);
+    p->av_device_ref = mp_filter_load_hwdec_device(f, IMGFMT_VAAPI);
     if (!p->av_device_ref)
         goto error;
 
diff --git a/video/filter/vf_vdpaupp.c b/video/filter/vf_vdpaupp.c
index 2f485bdbad..6de8671e39 100644
--- a/video/filter/vf_vdpaupp.c
+++ b/video/filter/vf_vdpaupp.c
@@ -136,7 +136,7 @@ static struct mp_filter *vf_vdpaupp_create(struct mp_filter *parent, void *optio
 
     p->queue = mp_refqueue_alloc(f);
 
-    AVBufferRef *ref = mp_filter_load_hwdec_device(f, AV_HWDEVICE_TYPE_VDPAU);
+    AVBufferRef *ref = mp_filter_load_hwdec_device(f, IMGFMT_VDPAU);
     if (!ref)
         goto error;
     p->ctx = mp_vdpau_get_ctx_from_av(ref);
diff --git a/video/hwdec.c b/video/hwdec.c
index 46ddff516c..1b15c95c64 100644
--- a/video/hwdec.c
+++ b/video/hwdec.c
@@ -35,34 +35,22 @@ void hwdec_devices_destroy(struct mp_hwdec_devices *devs)
     talloc_free(devs);
 }
 
-struct mp_hwdec_ctx *hwdec_devices_get_by_lavc(struct mp_hwdec_devices *devs,
-                                               int av_hwdevice_type)
+struct mp_hwdec_ctx *hwdec_devices_get_by_imgfmt(struct mp_hwdec_devices *devs,
+                                                 int hw_imgfmt)
 {
     struct mp_hwdec_ctx *res = NULL;
     pthread_mutex_lock(&devs->lock);
     for (int n = 0; n < devs->num_hwctxs; n++) {
         struct mp_hwdec_ctx *dev = devs->hwctxs[n];
-        if (dev->av_device_ref) {
-            AVHWDeviceContext *hwctx = (void *)dev->av_device_ref->data;
-            if (hwctx->type == av_hwdevice_type) {
-                res = dev;
-                break;
-            }
+        if (dev->hw_imgfmt == hw_imgfmt) {
+            res = dev;
+            break;
         }
     }
     pthread_mutex_unlock(&devs->lock);
     return res;
 }
 
-struct AVBufferRef *hwdec_devices_get_lavc(struct mp_hwdec_devices *devs,
-                                           int av_hwdevice_type)
-{
-    struct mp_hwdec_ctx *ctx = hwdec_devices_get_by_lavc(devs, av_hwdevice_type);
-    if (!ctx)
-        return NULL;
-    return av_buffer_ref(ctx->av_device_ref);
-}
-
 struct mp_hwdec_ctx *hwdec_devices_get_first(struct mp_hwdec_devices *devs)
 {
     return hwdec_devices_get_n(devs, 0);
diff --git a/video/hwdec.h b/video/hwdec.h
index 3ceb5a6897..f3f2df06f2 100644
--- a/video/hwdec.h
+++ b/video/hwdec.h
@@ -13,9 +13,10 @@ struct mp_hwdec_ctx {
     // libavutil-wrapped context, if available.
     struct AVBufferRef *av_device_ref; // AVHWDeviceContext*
 
-    // List of IMGFMT_s, terminated with 0. NULL if N/A.
+    // List of allowed IMGFMT_s, terminated with 0.
+    // If NULL, all software formats are considered to be supported.
     const int *supported_formats;
-    // HW format for which above hw_subfmts are valid.
+    // HW format used by the hwdec
     int hw_imgfmt;
 };
 
@@ -25,17 +26,8 @@ struct mp_hwdec_devices;
 struct mp_hwdec_devices *hwdec_devices_create(void);
 void hwdec_devices_destroy(struct mp_hwdec_devices *devs);
 
-// Return the device context for the given API type. Returns NULL if none
-// available. Logically, the returned pointer remains valid until VO
-// uninitialization is started (all users of it must be uninitialized before).
-// hwdec_devices_request() may be used before this to lazily load devices.
-// Contains a wrapped AVHWDeviceContext.
-// Beware that this creates a _new_ reference.
-struct AVBufferRef *hwdec_devices_get_lavc(struct mp_hwdec_devices *devs,
-                                           int av_hwdevice_type);
-
-struct mp_hwdec_ctx *hwdec_devices_get_by_lavc(struct mp_hwdec_devices *devs,
-                                               int av_hwdevice_type);
+struct mp_hwdec_ctx *hwdec_devices_get_by_imgfmt(struct mp_hwdec_devices *devs,
+                                                 int hw_imgfmt);
 
 // For code which still strictly assumes there is 1 (or none) device.
 struct mp_hwdec_ctx *hwdec_devices_get_first(struct mp_hwdec_devices *devs);
diff --git a/video/out/d3d11/hwdec_dxva2dxgi.c b/video/out/d3d11/hwdec_dxva2dxgi.c
index 8dbb1cf65d..8f8433166c 100644
--- a/video/out/d3d11/hwdec_dxva2dxgi.c
+++ b/video/out/d3d11/hwdec_dxva2dxgi.c
@@ -134,6 +134,7 @@ static int init(struct ra_hwdec *hw)
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
         .av_device_ref = d3d9_wrap_device_ref((IDirect3DDevice9 *)p->dev9),
+        .hw_imgfmt = IMGFMT_DXVA2,
     };
     hwdec_devices_add(hw->devs, &p->hwctx);
 
diff --git a/video/out/hwdec/hwdec_cuda.c b/video/out/hwdec/hwdec_cuda.c
index 910d6a4078..47428bd75f 100644
--- a/video/out/hwdec/hwdec_cuda.c
+++ b/video/out/hwdec/hwdec_cuda.c
@@ -122,6 +122,7 @@ static int cuda_init(struct ra_hwdec *hw)
     p->hwctx = (struct mp_hwdec_ctx) {
         .driver_name = hw->driver->name,
         .av_device_ref = hw_device_ctx,
+        .hw_imgfmt = IMGFMT_CUDA,
     };
     hwdec_devices_add(hw->devs, &p->hwctx);
     return 0;
diff --git a/video/out/opengl/hwdec_drmprime_drm.c b/video/out/opengl/hwdec_drmprime_drm.c
index 87b04c350f..76afd3ff88 100644
--- a/video/out/opengl/hwdec_drmprime_drm.c
+++ b/video/out/opengl/hwdec_drmprime_drm.c
@@ -298,6 +298,7 @@ static int init(struct ra_hwdec *hw)
 
     p->hwctx = (struct mp_hwdec_ctx) {
         .driver_name = hw->driver->name,
+        .hw_imgfmt = IMGFMT_DRMPRIME,
     };
     if (!av_hwdevice_ctx_create(&p->hwctx.av_device_ref, AV_HWDEVICE_TYPE_DRM,
                                 drmGetDeviceNameFromFd2(p->ctx->fd), NULL, 0)) {
diff --git a/video/out/opengl/hwdec_dxva2egl.c b/video/out/opengl/hwdec_dxva2egl.c
index 24f5c93b36..008af9842d 100644
--- a/video/out/opengl/hwdec_dxva2egl.c
+++ b/video/out/opengl/hwdec_dxva2egl.c
@@ -181,6 +181,7 @@ static int init(struct ra_hwdec *hw)
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
         .av_device_ref = d3d9_wrap_device_ref((IDirect3DDevice9 *)p->device9ex),
+        .hw_imgfmt = IMGFMT_DXVA2,
     };
     hwdec_devices_add(hw->devs, &p->hwctx);
 
diff --git a/video/out/opengl/hwdec_dxva2gldx.c b/video/out/opengl/hwdec_dxva2gldx.c
index bbf76b09b4..d92dd75d10 100644
--- a/video/out/opengl/hwdec_dxva2gldx.c
+++ b/video/out/opengl/hwdec_dxva2gldx.c
@@ -80,6 +80,7 @@ static int init(struct ra_hwdec *hw)
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
         .av_device_ref = d3d9_wrap_device_ref((IDirect3DDevice9 *)p->device),
+        .hw_imgfmt = IMGFMT_DXVA2,
     };
     hwdec_devices_add(hw->devs, &p->hwctx);
     return 0;
diff --git a/video/out/opengl/hwdec_ios.m b/video/out/opengl/hwdec_ios.m
index a16a09fb71..a2ca0310f2 100644
--- a/video/out/opengl/hwdec_ios.m
+++ b/video/out/opengl/hwdec_ios.m
@@ -70,6 +70,7 @@ static int init(struct ra_hwdec *hw)
 
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
+        .hw_imgfmt = IMGFMT_VIDEOTOOLBOX,
     };
 
     av_hwdevice_ctx_create(&p->hwctx.av_device_ref, AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
diff --git a/video/out/opengl/hwdec_osx.c b/video/out/opengl/hwdec_osx.c
index ca7a0049c2..86e7402267 100644
--- a/video/out/opengl/hwdec_osx.c
+++ b/video/out/opengl/hwdec_osx.c
@@ -71,6 +71,7 @@ static int init(struct ra_hwdec *hw)
 
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
+        .hw_imgfmt = IMGFMT_VIDEOTOOLBOX,
     };
 
     av_hwdevice_ctx_create(&p->hwctx.av_device_ref, AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
diff --git a/video/out/opengl/hwdec_vdpau.c b/video/out/opengl/hwdec_vdpau.c
index df58905f4d..7ce2e0630b 100644
--- a/video/out/opengl/hwdec_vdpau.c
+++ b/video/out/opengl/hwdec_vdpau.c
@@ -61,6 +61,7 @@ static int init(struct ra_hwdec *hw)
     if (hw->probing && mp_vdpau_guess_if_emulated(p->ctx))
         return -1;
     p->ctx->hwctx.driver_name = hw->driver->name;
+    p->ctx->hwctx.hw_imgfmt = IMGFMT_VDPAU;
     hwdec_devices_add(hw->devs, &p->ctx->hwctx);
     return 0;
 }
diff --git a/video/out/vo_mediacodec_embed.c b/video/out/vo_mediacodec_embed.c
index dc2974918b..aef7e1a23d 100644
--- a/video/out/vo_mediacodec_embed.c
+++ b/video/out/vo_mediacodec_embed.c
@@ -52,6 +52,7 @@ static int preinit(struct vo *vo)
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = "mediacodec_embed",
         .av_device_ref = create_mediacodec_device_ref(vo),
+        .hw_imgfmt = IMGFMT_MEDIACODEC,
     };
     hwdec_devices_add(vo->hwdec_devs, &p->hwctx);
     return 0;