d3d11va, dxva2: simplify decoder selection

In particular, this moves the depth test to common code.

Should be functionally equivalent, except that for DXVA2, the
IDirectXVideoDecoderService_GetDecoderRenderTargets API is called
more often potentially.

4 files changed, 58 insertions, 67 deletions

diff --git a/video/decode/d3d.c b/video/decode/d3d.c
index 7a47ad2aa7..e55ca22073 100644
--- a/video/decode/d3d.c
+++ b/video/decode/d3d.c
@@ -132,7 +132,9 @@ static bool mode_supported(const struct d3dva_mode *mode,
 
 struct d3d_decoder_fmt d3d_select_decoder_mode(
     struct lavc_ctx *s, const GUID *device_guids, UINT n_guids,
-    DWORD (*get_dxfmt_cb)(struct lavc_ctx *s, const GUID *guid, int depth))
+    const struct d3d_decoded_format *formats, int n_formats,
+    bool (*test_fmt_cb)(struct lavc_ctx *s, const GUID *guid,
+                        const struct d3d_decoded_format *fmt))
 {
     struct d3d_decoder_fmt fmt = {
         .guid          = &GUID_NULL,
@@ -146,7 +148,6 @@ struct d3d_decoder_fmt d3d_select_decoder_mode(
         return fmt;
 
     int depth = IMGFMT_RGB_DEPTH(sw_img_fmt);
-    int mpfmt_decoded = depth <= 8 ? IMGFMT_NV12 : IMGFMT_P010;
 
     for (int i = 0; i < MP_ARRAY_SIZE(d3dva_modes); i++) {
         const struct d3dva_mode *mode = &d3dva_modes[i];
@@ -154,12 +155,24 @@ struct d3d_decoder_fmt d3d_select_decoder_mode(
             profile_compatible(mode, s->avctx->profile) &&
             mode_supported(mode, device_guids, n_guids)) {
 
-            DWORD dxfmt_decoded = get_dxfmt_cb(s, mode->guid, depth);
-            if (dxfmt_decoded) {
-                fmt.guid          = mode->guid;
-                fmt.mpfmt_decoded = mpfmt_decoded;
-                fmt.dxfmt_decoded = dxfmt_decoded;
-                return fmt;
+            for (int n = 0; n < n_formats; n++) {
+                const struct d3d_decoded_format *format = &formats[n];
+
+                if (depth <= format->depth && test_fmt_cb(s, mode->guid, format))
+                {
+                    MP_VERBOSE(s, "Selecting %s ",
+                               d3d_decoder_guid_to_desc(mode->guid));
+                    if (format->dxfmt >= (1 << 16)) {
+                        MP_VERBOSE(s, "%s\n", mp_tag_str(format->dxfmt));
+                    } else {
+                        MP_VERBOSE(s, "%d\n", (int)format->dxfmt);
+                    }
+
+                    fmt.guid          = mode->guid;
+                    fmt.mpfmt_decoded = format->mpfmt;
+                    fmt.dxfmt_decoded = format->dxfmt;
+                    return fmt;
+                }
             }
         }
     }
diff --git a/video/decode/d3d.h b/video/decode/d3d.h
index bbd6bdfd2a..21b930528a 100644
--- a/video/decode/d3d.h
+++ b/video/decode/d3d.h
@@ -24,6 +24,13 @@
 struct mp_image;
 struct lavc_ctx;
 
+struct d3d_decoded_format {
+    DWORD       dxfmt;  // D3DFORMAT or DXGI_FORMAT
+    const char *name;   // informational string repr. of dxfmt_decoded
+    int         depth;  // significant bits (not full size)
+    int         mpfmt;  // IMGFMT_ with compatible memory layout and semantics
+};
+
 struct d3d_decoder_fmt {
     const GUID *guid;
     int   mpfmt_decoded;
@@ -31,9 +38,12 @@ struct d3d_decoder_fmt {
 };
 
 int d3d_probe_codec(const char *codec);
+
 struct d3d_decoder_fmt d3d_select_decoder_mode(
     struct lavc_ctx *s, const GUID *device_guids, UINT n_guids,
-    DWORD (*get_dxfmt_cb)(struct lavc_ctx *s, const GUID *guid, int depth));
+    const struct d3d_decoded_format *formats, int n_formats,
+    bool (*test_fmt_cb)(struct lavc_ctx *s, const GUID *guid,
+                        const struct d3d_decoded_format *fmt));
 
 char *d3d_decoder_guid_to_desc_buf(char *buf, size_t buf_size,
                                    const GUID *mode_guid);
diff --git a/video/decode/d3d11va.c b/video/decode/d3d11va.c
index f20031ff31..c792aafa01 100644
--- a/video/decode/d3d11va.c
+++ b/video/decode/d3d11va.c
@@ -179,27 +179,21 @@ static struct mp_image *d3d11va_retrieve_image(struct lavc_ctx *s,
     return sw_img;
 }
 
-struct d3d11_format {
-    DXGI_FORMAT format;
-    const char *name;
-    int         depth;
-};
-
 #define DFMT(name) MP_CONCAT(DXGI_FORMAT_, name), # name
-static const struct d3d11_format d3d11_formats[] = {
-    {DFMT(NV12),  8},
-    {DFMT(P010), 10},
-    {DFMT(P016), 16},
+static const struct d3d_decoded_format d3d11_formats[] = {
+    {DFMT(NV12),  8, IMGFMT_NV12},
+    {DFMT(P010), 10, IMGFMT_P010},
+    {DFMT(P016), 16, IMGFMT_P010},
 };
 #undef DFMT
 
-static BOOL d3d11_format_supported(struct lavc_ctx *s, const GUID *guid,
-                                   const struct d3d11_format *format)
+static bool d3d11_format_supported(struct lavc_ctx *s, const GUID *guid,
+                                   const struct d3d_decoded_format *format)
 {
     struct priv *p = s->hwdec_priv;
     BOOL is_supported = FALSE;
     HRESULT hr = ID3D11VideoDevice_CheckVideoDecoderFormat(
-        p->video_dev, guid, format->format, &is_supported);
+        p->video_dev, guid, format->dxfmt, &is_supported);
     if (FAILED(hr)) {
         MP_ERR(p, "Check decoder output format %s for decoder %s: %s\n",
                format->name, d3d_decoder_guid_to_desc(guid),
@@ -213,29 +207,13 @@ static void dump_decoder_info(struct lavc_ctx *s, const GUID *guid)
     struct priv *p = s->hwdec_priv;
     char fmts[256] = {0};
     for (int i = 0; i < MP_ARRAY_SIZE(d3d11_formats); i++) {
-        const struct d3d11_format *format = &d3d11_formats[i];
+        const struct d3d_decoded_format *format = &d3d11_formats[i];
         if (d3d11_format_supported(s, guid, format))
             mp_snprintf_cat(fmts, sizeof(fmts), " %s", format->name);
     }
     MP_VERBOSE(p, "%s %s\n", d3d_decoder_guid_to_desc(guid), fmts);
 }
 
-static DWORD get_dxfmt_cb(struct lavc_ctx *s, const GUID *guid, int depth)
-{
-    struct priv *p = s->hwdec_priv;
-    for (int i = 0; i < MP_ARRAY_SIZE(d3d11_formats); i++) {
-        const struct d3d11_format *format = &d3d11_formats[i];
-        if (depth <= format->depth &&
-            d3d11_format_supported(s, guid, format)) {
-            MP_VERBOSE(p, "Selecting %s %s\n",
-                       d3d_decoder_guid_to_desc(guid),
-                       format->name);
-            return format->format;
-        }
-    }
-    return 0;
-}
-
 static void d3d11va_destroy_decoder(void *arg)
 {
     struct d3d11va_decoder *decoder = arg;
@@ -270,7 +248,9 @@ static int d3d11va_init_decoder(struct lavc_ctx *s, int w, int h)
     }
 
     struct d3d_decoder_fmt fmt =
-        d3d_select_decoder_mode(s, device_guids, n_guids, get_dxfmt_cb);
+        d3d_select_decoder_mode(s, device_guids, n_guids,
+                                d3d11_formats, MP_ARRAY_SIZE(d3d11_formats),
+                                d3d11_format_supported);
     if (fmt.mpfmt_decoded == IMGFMT_NONE) {
         MP_ERR(p, "Failed to find a suitable decoder\n");
         goto done;
diff --git a/video/decode/dxva2.c b/video/decode/dxva2.c
index e6499f96f2..0fd4acafba 100644
--- a/video/decode/dxva2.c
+++ b/video/decode/dxva2.c
@@ -108,15 +108,10 @@ static struct mp_image *dxva2_retrieve_image(struct lavc_ctx *s,
     return sw_img;
 }
 
-struct d3d9_format {
-    D3DFORMAT format;
-    int       depth;
-};
-
-static const struct d3d9_format d3d9_formats[] = {
-    {MKTAG('N','V','1','2'),  8},
-    {MKTAG('P','0','1','0'), 10},
-    {MKTAG('P','0','1','6'), 16},
+static const struct d3d_decoded_format d3d9_formats[] = {
+    {MKTAG('N','V','1','2'), "NV12", 8,  IMGFMT_NV12},
+    {MKTAG('P','0','1','0'), "P010", 10, IMGFMT_P010},
+    {MKTAG('P','0','1','6'), "P016", 16, IMGFMT_P010},
 };
 
 static void dump_decoder_info(struct lavc_ctx *s,
@@ -148,9 +143,10 @@ static void dump_decoder_info(struct lavc_ctx *s,
     }
 }
 
-static DWORD get_dxfmt_cb(struct lavc_ctx *s, const GUID *guid, int depth)
+static bool dxva2_format_supported(struct lavc_ctx *s, const GUID *guid,
+                                   const struct d3d_decoded_format *format)
 {
-    DWORD ret = 0;
+    bool ret = false;
     struct priv *p = s->hwdec_priv;
     D3DFORMAT *formats = NULL;
     UINT     n_formats = 0;
@@ -162,22 +158,12 @@ static DWORD get_dxfmt_cb(struct lavc_ctx *s, const GUID *guid, int depth)
         return 0;
     }
 
-    for (int i = 0; i < MP_ARRAY_SIZE(d3d9_formats); i++) {
-        const struct d3d9_format *d3d9_fmt = &d3d9_formats[i];
-        if (d3d9_fmt->depth < depth)
-            continue;
-
-        for (UINT j = 0; j < n_formats; j++) {
-            if (formats[i] == d3d9_fmt->format) {
-                ret = formats[i];
-                MP_VERBOSE(p, "Selecting %s %s\n",
-                           d3d_decoder_guid_to_desc(guid),
-                           mp_tag_str(ret));
-                goto done;
-            }
-        }
+    for (int i = 0; i < n_formats; i++) {
+        ret = formats[i] == format->dxfmt;
+        if (ret)
+            break;
     }
-done:
+
     CoTaskMemFree(formats);
     return ret;
 }
@@ -207,7 +193,9 @@ static int dxva2_init_decoder(struct lavc_ctx *s, int w, int h)
     dump_decoder_info(s, device_guids, n_guids);
 
     struct d3d_decoder_fmt fmt =
-        d3d_select_decoder_mode(s, device_guids, n_guids, get_dxfmt_cb);
+        d3d_select_decoder_mode(s, device_guids, n_guids,
+                                d3d9_formats, MP_ARRAY_SIZE(d3d9_formats),
+                                dxva2_format_supported);
     CoTaskMemFree(device_guids);
     if (fmt.mpfmt_decoded == IMGFMT_NONE) {
         MP_ERR(p, "Failed to find a suitable decoder\n");