vo: hwdec: do hwdec interop lookup by image format

It turns out that it's generally more useful to look up hwdecs by image
format, rather than device type. In the situations where we need to
find one, we generally know the image format we're dealing with. Doing
this avoids us having to create mappings from image format to device
type.

The most significant part of this change is filling in the image format
for the various hw interops. There is a hw_imgfmt field today today, but
only a couple of the interops fill it in, and that seems to be because
we've never actually used this piece of metadata before. Well, now we
have a good use for it.

15 files changed, 28 insertions, 35 deletions

diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index ce067d0ff4..4f38b669ec 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -435,12 +435,16 @@ static AVBufferRef *hwdec_create_dev(struct mp_filter *vd,
             return ref;
         }
     } else if (ctx->hwdec_devs) {
+        int imgfmt = pixfmt2imgfmt(hwdec->pix_fmt);
         struct hwdec_imgfmt_request params = {
-            .imgfmt = pixfmt2imgfmt(hwdec->pix_fmt),
+            .imgfmt = imgfmt,
             .probing = autoprobe,
         };
         hwdec_devices_request_for_img_fmt(ctx->hwdec_devs, &params);
-        return hwdec_devices_get_lavc(ctx->hwdec_devs, hwdec->lavc_device);
+
+        const struct mp_hwdec_ctx *hw_ctx =
+            hwdec_devices_get_by_imgfmt(ctx->hwdec_devs, imgfmt);
+        return hw_ctx ? av_buffer_ref(hw_ctx->av_device_ref) : NULL;
     }
 
     return NULL;
diff --git a/video/filter/vf_d3d11vpp.c b/video/filter/vf_d3d11vpp.c
index f36197c412..df029417b9 100644
--- a/video/filter/vf_d3d11vpp.c
+++ b/video/filter/vf_d3d11vpp.c
@@ -435,7 +435,7 @@ static struct mp_filter *vf_d3d11vpp_create(struct mp_filter *parent,
     hwdec_devices_request_for_img_fmt(info->hwdec_devs, &params);
 
     struct mp_hwdec_ctx *hwctx =
-        hwdec_devices_get_by_lavc(info->hwdec_devs, AV_HWDEVICE_TYPE_D3D11VA);
+        hwdec_devices_get_by_imgfmt(info->hwdec_devs, IMGFMT_D3D11);
     if (!hwctx || !hwctx->av_device_ref)
         goto fail;
     AVHWDeviceContext *avhwctx = (void *)hwctx->av_device_ref->data;
diff --git a/video/filter/vf_vavpp.c b/video/filter/vf_vavpp.c
index ad2ecc04dc..d84c393e29 100644
--- a/video/filter/vf_vavpp.c
+++ b/video/filter/vf_vavpp.c
@@ -448,7 +448,7 @@ static struct mp_filter *vf_vavpp_create(struct mp_filter *parent, void *options
 
     p->queue = mp_refqueue_alloc(f);
 
-    p->av_device_ref = mp_filter_load_hwdec_device(f, AV_HWDEVICE_TYPE_VAAPI);
+    p->av_device_ref = mp_filter_load_hwdec_device(f, IMGFMT_VAAPI);
     if (!p->av_device_ref)
         goto error;
 
diff --git a/video/filter/vf_vdpaupp.c b/video/filter/vf_vdpaupp.c
index 2f485bdbad..6de8671e39 100644
--- a/video/filter/vf_vdpaupp.c
+++ b/video/filter/vf_vdpaupp.c
@@ -136,7 +136,7 @@ static struct mp_filter *vf_vdpaupp_create(struct mp_filter *parent, void *optio
 
     p->queue = mp_refqueue_alloc(f);
 
-    AVBufferRef *ref = mp_filter_load_hwdec_device(f, AV_HWDEVICE_TYPE_VDPAU);
+    AVBufferRef *ref = mp_filter_load_hwdec_device(f, IMGFMT_VDPAU);
     if (!ref)
         goto error;
     p->ctx = mp_vdpau_get_ctx_from_av(ref);
diff --git a/video/hwdec.c b/video/hwdec.c
index 46ddff516c..1b15c95c64 100644
--- a/video/hwdec.c
+++ b/video/hwdec.c
@@ -35,34 +35,22 @@ void hwdec_devices_destroy(struct mp_hwdec_devices *devs)
     talloc_free(devs);
 }
 
-struct mp_hwdec_ctx *hwdec_devices_get_by_lavc(struct mp_hwdec_devices *devs,
-                                               int av_hwdevice_type)
+struct mp_hwdec_ctx *hwdec_devices_get_by_imgfmt(struct mp_hwdec_devices *devs,
+                                                 int hw_imgfmt)
 {
     struct mp_hwdec_ctx *res = NULL;
     pthread_mutex_lock(&devs->lock);
     for (int n = 0; n < devs->num_hwctxs; n++) {
         struct mp_hwdec_ctx *dev = devs->hwctxs[n];
-        if (dev->av_device_ref) {
-            AVHWDeviceContext *hwctx = (void *)dev->av_device_ref->data;
-            if (hwctx->type == av_hwdevice_type) {
-                res = dev;
-                break;
-            }
+        if (dev->hw_imgfmt == hw_imgfmt) {
+            res = dev;
+            break;
         }
     }
     pthread_mutex_unlock(&devs->lock);
     return res;
 }
 
-struct AVBufferRef *hwdec_devices_get_lavc(struct mp_hwdec_devices *devs,
-                                           int av_hwdevice_type)
-{
-    struct mp_hwdec_ctx *ctx = hwdec_devices_get_by_lavc(devs, av_hwdevice_type);
-    if (!ctx)
-        return NULL;
-    return av_buffer_ref(ctx->av_device_ref);
-}
-
 struct mp_hwdec_ctx *hwdec_devices_get_first(struct mp_hwdec_devices *devs)
 {
     return hwdec_devices_get_n(devs, 0);
diff --git a/video/hwdec.h b/video/hwdec.h
index 3ceb5a6897..f3f2df06f2 100644
--- a/video/hwdec.h
+++ b/video/hwdec.h
@@ -13,9 +13,10 @@ struct mp_hwdec_ctx {
     // libavutil-wrapped context, if available.
     struct AVBufferRef *av_device_ref; // AVHWDeviceContext*
 
-    // List of IMGFMT_s, terminated with 0. NULL if N/A.
+    // List of allowed IMGFMT_s, terminated with 0.
+    // If NULL, all software formats are considered to be supported.
     const int *supported_formats;
-    // HW format for which above hw_subfmts are valid.
+    // HW format used by the hwdec
     int hw_imgfmt;
 };
 
@@ -25,17 +26,8 @@ struct mp_hwdec_devices;
 struct mp_hwdec_devices *hwdec_devices_create(void);
 void hwdec_devices_destroy(struct mp_hwdec_devices *devs);
 
-// Return the device context for the given API type. Returns NULL if none
-// available. Logically, the returned pointer remains valid until VO
-// uninitialization is started (all users of it must be uninitialized before).
-// hwdec_devices_request() may be used before this to lazily load devices.
-// Contains a wrapped AVHWDeviceContext.
-// Beware that this creates a _new_ reference.
-struct AVBufferRef *hwdec_devices_get_lavc(struct mp_hwdec_devices *devs,
-                                           int av_hwdevice_type);
-
-struct mp_hwdec_ctx *hwdec_devices_get_by_lavc(struct mp_hwdec_devices *devs,
-                                               int av_hwdevice_type);
+struct mp_hwdec_ctx *hwdec_devices_get_by_imgfmt(struct mp_hwdec_devices *devs,
+                                                 int hw_imgfmt);
 
 // For code which still strictly assumes there is 1 (or none) device.
 struct mp_hwdec_ctx *hwdec_devices_get_first(struct mp_hwdec_devices *devs);
diff --git a/video/out/d3d11/hwdec_dxva2dxgi.c b/video/out/d3d11/hwdec_dxva2dxgi.c
index 8dbb1cf65d..8f8433166c 100644
--- a/video/out/d3d11/hwdec_dxva2dxgi.c
+++ b/video/out/d3d11/hwdec_dxva2dxgi.c
@@ -134,6 +134,7 @@ static int init(struct ra_hwdec *hw)
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
         .av_device_ref = d3d9_wrap_device_ref((IDirect3DDevice9 *)p->dev9),
+        .hw_imgfmt = IMGFMT_DXVA2,
     };
     hwdec_devices_add(hw->devs, &p->hwctx);
 
diff --git a/video/out/hwdec/hwdec_cuda.c b/video/out/hwdec/hwdec_cuda.c
index 910d6a4078..47428bd75f 100644
--- a/video/out/hwdec/hwdec_cuda.c
+++ b/video/out/hwdec/hwdec_cuda.c
@@ -122,6 +122,7 @@ static int cuda_init(struct ra_hwdec *hw)
     p->hwctx = (struct mp_hwdec_ctx) {
         .driver_name = hw->driver->name,
         .av_device_ref = hw_device_ctx,
+        .hw_imgfmt = IMGFMT_CUDA,
     };
     hwdec_devices_add(hw->devs, &p->hwctx);
     return 0;
diff --git a/video/out/opengl/hwdec_drmprime_drm.c b/video/out/opengl/hwdec_drmprime_drm.c
index 87b04c350f..76afd3ff88 100644
--- a/video/out/opengl/hwdec_drmprime_drm.c
+++ b/video/out/opengl/hwdec_drmprime_drm.c
@@ -298,6 +298,7 @@ static int init(struct ra_hwdec *hw)
 
     p->hwctx = (struct mp_hwdec_ctx) {
         .driver_name = hw->driver->name,
+        .hw_imgfmt = IMGFMT_DRMPRIME,
     };
     if (!av_hwdevice_ctx_create(&p->hwctx.av_device_ref, AV_HWDEVICE_TYPE_DRM,
                                 drmGetDeviceNameFromFd2(p->ctx->fd), NULL, 0)) {
diff --git a/video/out/opengl/hwdec_dxva2egl.c b/video/out/opengl/hwdec_dxva2egl.c
index 24f5c93b36..008af9842d 100644
--- a/video/out/opengl/hwdec_dxva2egl.c
+++ b/video/out/opengl/hwdec_dxva2egl.c
@@ -181,6 +181,7 @@ static int init(struct ra_hwdec *hw)
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
         .av_device_ref = d3d9_wrap_device_ref((IDirect3DDevice9 *)p->device9ex),
+        .hw_imgfmt = IMGFMT_DXVA2,
     };
     hwdec_devices_add(hw->devs, &p->hwctx);
 
diff --git a/video/out/opengl/hwdec_dxva2gldx.c b/video/out/opengl/hwdec_dxva2gldx.c
index bbf76b09b4..d92dd75d10 100644
--- a/video/out/opengl/hwdec_dxva2gldx.c
+++ b/video/out/opengl/hwdec_dxva2gldx.c
@@ -80,6 +80,7 @@ static int init(struct ra_hwdec *hw)
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
         .av_device_ref = d3d9_wrap_device_ref((IDirect3DDevice9 *)p->device),
+        .hw_imgfmt = IMGFMT_DXVA2,
     };
     hwdec_devices_add(hw->devs, &p->hwctx);
     return 0;
diff --git a/video/out/opengl/hwdec_ios.m b/video/out/opengl/hwdec_ios.m
index a16a09fb71..a2ca0310f2 100644
--- a/video/out/opengl/hwdec_ios.m
+++ b/video/out/opengl/hwdec_ios.m
@@ -70,6 +70,7 @@ static int init(struct ra_hwdec *hw)
 
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
+        .hw_imgfmt = IMGFMT_VIDEOTOOLBOX,
     };
 
     av_hwdevice_ctx_create(&p->hwctx.av_device_ref, AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
diff --git a/video/out/opengl/hwdec_osx.c b/video/out/opengl/hwdec_osx.c
index ca7a0049c2..86e7402267 100644
--- a/video/out/opengl/hwdec_osx.c
+++ b/video/out/opengl/hwdec_osx.c
@@ -71,6 +71,7 @@ static int init(struct ra_hwdec *hw)
 
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
+        .hw_imgfmt = IMGFMT_VIDEOTOOLBOX,
     };
 
     av_hwdevice_ctx_create(&p->hwctx.av_device_ref, AV_HWDEVICE_TYPE_VIDEOTOOLBOX,
diff --git a/video/out/opengl/hwdec_vdpau.c b/video/out/opengl/hwdec_vdpau.c
index df58905f4d..7ce2e0630b 100644
--- a/video/out/opengl/hwdec_vdpau.c
+++ b/video/out/opengl/hwdec_vdpau.c
@@ -61,6 +61,7 @@ static int init(struct ra_hwdec *hw)
     if (hw->probing && mp_vdpau_guess_if_emulated(p->ctx))
         return -1;
     p->ctx->hwctx.driver_name = hw->driver->name;
+    p->ctx->hwctx.hw_imgfmt = IMGFMT_VDPAU;
     hwdec_devices_add(hw->devs, &p->ctx->hwctx);
     return 0;
 }
diff --git a/video/out/vo_mediacodec_embed.c b/video/out/vo_mediacodec_embed.c
index dc2974918b..aef7e1a23d 100644
--- a/video/out/vo_mediacodec_embed.c
+++ b/video/out/vo_mediacodec_embed.c
@@ -52,6 +52,7 @@ static int preinit(struct vo *vo)
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = "mediacodec_embed",
         .av_device_ref = create_mediacodec_device_ref(vo),
+        .hw_imgfmt = IMGFMT_MEDIACODEC,
     };
     hwdec_devices_add(vo->hwdec_devs, &p->hwctx);
     return 0;