video: refactor how VO exports hwdec device handles

The main change is with video/hwdec.h. mp_hwdec_info is made opaque (and
renamed to mp_hwdec_devices). Its accessors are mainly thread-safe (or
documented where not), which makes the whole thing saner and cleaner. In
particular, thread-safety rules become less subtle and more obvious.

The new internal API makes it easier to support multiple OpenGL interop
backends. (Although this is not done yet, and it's not clear whether it
ever will.)

This also removes all the API-specific fields from mp_hwdec_ctx and
replaces them with a "ctx" field. For d3d in particular, we drop the
mp_d3d_ctx struct completely, and pass the interfaces directly.

Remove the emulation checks from vaapi.c and vdpau.c; they are
pointless, and the checks that matter are done on the VO layer.

The d3d hardware decoders might slightly change behavior: dxva2-copy
will not use the VO device anymore if the VO supports proper interop.
This pretty much assumes that any in such cases the VO will not use any
form of exclusive mode, which makes using the VO device in copy mode
unnecessary.

This is a big refactor. Some things may be untested and could be broken.

8 files changed, 40 insertions, 61 deletions

diff --git a/video/decode/d3d11va.c b/video/decode/d3d11va.c
index 246a35ec9a..1626626767 100644
--- a/video/decode/d3d11va.c
+++ b/video/decode/d3d11va.c
@@ -26,7 +26,6 @@
 #include "video/mp_image_pool.h"
 #include "video/hwdec.h"
 
-#include "video/d3d.h"
 #include "d3d.h"
 
 #define ADDITIONAL_SURFACES (4 + HWDEC_DELAY_QUEUE_COUNT)
@@ -492,9 +491,7 @@ static int d3d11va_init(struct lavc_ctx *s)
         p->sw_pool = talloc_steal(p, mp_image_pool_new(17));
     }
 
-    if (s->hwdec_info && s->hwdec_info->hwctx && s->hwdec_info->hwctx->d3d_ctx)
-        p->device = s->hwdec_info->hwctx->d3d_ctx->d3d11_device;
-
+    p->device = hwdec_devices_load(s->hwdec_devs, s->hwdec->type);
     if (p->device) {
         ID3D11Device_AddRef(p->device);
         ID3D11Device_GetImmediateContext(p->device, &p->device_ctx);
@@ -539,15 +536,12 @@ fail:
     return -1;
 }
 
-static int d3d11va_probe(struct vd_lavc_hwdec *hwdec,
-                         struct mp_hwdec_info *info,
+static int d3d11va_probe(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                          const char *codec)
 {
-    hwdec_request_api(info, "d3d11va");
     // d3d11va-copy can do without external context; dxva2 requires it.
     if (hwdec->type != HWDEC_D3D11VA_COPY) {
-        if (!info || !info->hwctx || !info->hwctx->d3d_ctx ||
-            !info->hwctx->d3d_ctx->d3d11_device)
+        if (!hwdec_devices_load(ctx->hwdec_devs, HWDEC_D3D11VA))
             return HWDEC_ERR_NO_CTX;
     }
     return d3d_probe_codec(codec);
diff --git a/video/decode/dec_video.h b/video/decode/dec_video.h
index f4646a97d0..1030973e1c 100644
--- a/video/decode/dec_video.h
+++ b/video/decode/dec_video.h
@@ -32,7 +32,7 @@ struct dec_video {
     struct mpv_global *global;
     struct MPOpts *opts;
     const struct vd_functions *vd_driver;
-    struct mp_hwdec_info *hwdec_info; // video output hwdec handles
+    struct mp_hwdec_devices *hwdec_devs; // video output hwdec handles
     struct sh_stream *header;
     struct mp_codec_params *codec;
 
diff --git a/video/decode/dxva2.c b/video/decode/dxva2.c
index 70623c8fb4..5d3afda86c 100644
--- a/video/decode/dxva2.c
+++ b/video/decode/dxva2.c
@@ -32,7 +32,6 @@
 #include "video/mp_image_pool.h"
 #include "video/hwdec.h"
 
-#include "video/d3d.h"
 #include "video/dxva2.h"
 #include "d3d.h"
 
@@ -406,9 +405,7 @@ static int dxva2_init(struct lavc_ctx *s)
         p->sw_pool = talloc_steal(p, mp_image_pool_new(17));
     }
 
-    if (s->hwdec_info && s->hwdec_info->hwctx && s->hwdec_info->hwctx->d3d_ctx)
-        p->device = s->hwdec_info->hwctx->d3d_ctx->d3d9_device;
-
+    p->device = hwdec_devices_load(s->hwdec_devs, s->hwdec->type);
     if (p->device) {
         IDirect3D9_AddRef(p->device);
         MP_VERBOSE(p, "Using VO-supplied device %p.\n", p->device);
@@ -477,16 +474,15 @@ fail:
     return -1;
 }
 
-static int dxva2_probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+static int dxva2_probe(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                        const char *codec)
 {
-    hwdec_request_api(info, "dxva2");
     // dxva2-copy can do without external context; dxva2 requires it.
-    if (hwdec->type != HWDEC_DXVA2_COPY) {
-        if (!info || !info->hwctx || !info->hwctx->d3d_ctx ||
-            info->hwctx->type == HWDEC_DXVA2_COPY ||
-            !info->hwctx->d3d_ctx->d3d9_device)
+    if (hwdec->type == HWDEC_DXVA2) {
+        if (!hwdec_devices_load(ctx->hwdec_devs, HWDEC_DXVA2))
             return HWDEC_ERR_NO_CTX;
+    } else {
+        hwdec_devices_load(ctx->hwdec_devs, HWDEC_DXVA2_COPY);
     }
     return d3d_probe_codec(codec);
 }
diff --git a/video/decode/lavc.h b/video/decode/lavc.h
index 73243e16c4..dbefe79ad9 100644
--- a/video/decode/lavc.h
+++ b/video/decode/lavc.h
@@ -30,7 +30,7 @@ typedef struct lavc_ctx {
     int max_delay_queue;
 
     // From VO
-    struct mp_hwdec_info *hwdec_info;
+    struct mp_hwdec_devices *hwdec_devs;
 
     // For free use by hwdec implementation
     void *hwdec_priv;
@@ -54,7 +54,7 @@ struct vd_lavc_hwdec {
     // efficiency by not blocking on the hardware pipeline by reading back
     // immediately after decoding.
     int delay_queue;
-    int (*probe)(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+    int (*probe)(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                  const char *codec);
     int (*init)(struct lavc_ctx *ctx);
     int (*init_decoder)(struct lavc_ctx *ctx, int w, int h);
diff --git a/video/decode/vaapi.c b/video/decode/vaapi.c
index 169564d6c4..4b098a8804 100644
--- a/video/decode/vaapi.c
+++ b/video/decode/vaapi.c
@@ -357,7 +357,7 @@ static void destroy_va_dummy_ctx(struct priv *p)
 
 // Creates a "private" VADisplay, disconnected from the VO. We just create a
 // new X connection, because that's simpler. (We could also pass the X
-// connection along with struct mp_hwdec_info, if we wanted.)
+// connection along with struct mp_hwdec_devices, if we wanted.)
 static bool create_va_dummy_ctx(struct priv *p)
 {
     for (int n = 0; native_displays[n]; n++) {
@@ -399,21 +399,23 @@ static void uninit(struct lavc_ctx *ctx)
     ctx->hwdec_priv = NULL;
 }
 
-static int init_with_vactx(struct lavc_ctx *ctx, struct mp_vaapi_ctx *vactx)
+static int init(struct lavc_ctx *ctx, bool direct)
 {
     struct priv *p = talloc_ptrtype(NULL, p);
     *p = (struct priv) {
         .log = mp_log_new(p, ctx->log, "vaapi"),
-        .ctx = vactx,
         .va_context = &p->va_context_storage,
         .rt_format = VA_RT_FORMAT_YUV420
     };
 
-    if (!p->ctx)
+    if (direct) {
+        p->ctx = hwdec_devices_get(ctx->hwdec_devs, HWDEC_VAAPI)->ctx;
+    } else {
         create_va_dummy_ctx(p);
-    if (!p->ctx) {
-        talloc_free(p);
-        return -1;
+        if (!p->ctx) {
+            talloc_free(p);
+            return -1;
+        }
     }
 
     p->display = p->ctx->display;
@@ -431,25 +433,22 @@ static int init_with_vactx(struct lavc_ctx *ctx, struct mp_vaapi_ctx *vactx)
     return 0;
 }
 
-static int init(struct lavc_ctx *ctx)
+static int init_direct(struct lavc_ctx *ctx)
 {
-    return init_with_vactx(ctx, ctx->hwdec_info->hwctx->vaapi_ctx);
+    return init(ctx, true);
 }
 
-static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+static int probe(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                  const char *codec)
 {
-    hwdec_request_api(info, "vaapi");
-    if (!info || !info->hwctx || !info->hwctx->vaapi_ctx)
+    if (!hwdec_devices_load(ctx->hwdec_devs, HWDEC_VAAPI))
         return HWDEC_ERR_NO_CTX;
     if (!hwdec_check_codec_support(codec, profiles))
         return HWDEC_ERR_NO_CODEC;
-    if (va_guess_if_emulated(info->hwctx->vaapi_ctx))
-        return HWDEC_ERR_EMULATED;
     return 0;
 }
 
-static int probe_copy(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+static int probe_copy(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                       const char *codec)
 {
     struct priv dummy = {mp_null_log};
@@ -466,7 +465,7 @@ static int probe_copy(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
 
 static int init_copy(struct lavc_ctx *ctx)
 {
-    return init_with_vactx(ctx, NULL);
+    return init(ctx, false);
 }
 
 static struct mp_image *copy_image(struct lavc_ctx *ctx, struct mp_image *img)
@@ -497,7 +496,7 @@ const struct vd_lavc_hwdec mp_vd_lavc_vaapi = {
     .type = HWDEC_VAAPI,
     .image_format = IMGFMT_VAAPI,
     .probe = probe,
-    .init = init,
+    .init = init_direct,
     .uninit = uninit,
     .init_decoder = init_decoder,
     .allocate_image = allocate_image,
diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index ff97e3446c..bc70b7d348 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -280,18 +280,13 @@ static bool hwdec_is_wrapper(struct vd_lavc_hwdec *hwdec, const char *decoder)
     return bstr_endswith0(bstr0(decoder), hwdec->lavc_suffix);
 }
 
-void hwdec_request_api(struct mp_hwdec_info *info, const char *api_name)
-{
-    if (info && info->load_api)
-        info->load_api(info, api_name);
-}
-
-static int hwdec_probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+static int hwdec_probe(struct dec_video *vd, struct vd_lavc_hwdec *hwdec,
                        const char *codec)
 {
+    vd_ffmpeg_ctx *ctx = vd->priv;
     int r = 0;
     if (hwdec->probe)
-        r = hwdec->probe(hwdec, info, codec);
+        r = hwdec->probe(ctx, hwdec, codec);
     if (r >= 0) {
         if (hwdec->lavc_suffix && !hwdec_find_decoder(codec, hwdec->lavc_suffix))
             return HWDEC_ERR_NO_CODEC;
@@ -309,7 +304,7 @@ static struct vd_lavc_hwdec *probe_hwdec(struct dec_video *vd, bool autoprobe,
         MP_VERBOSE(vd, "Requested hardware decoder not compiled.\n");
         return NULL;
     }
-    int r = hwdec_probe(hwdec, vd->hwdec_info, codec);
+    int r = hwdec_probe(vd, hwdec, codec);
     if (r == HWDEC_ERR_EMULATED) {
         if (autoprobe)
             return NULL;
@@ -412,6 +407,7 @@ static int init(struct dec_video *vd, const char *decoder)
     ctx->log = vd->log;
     ctx->opts = vd->opts;
     ctx->decoder = talloc_strdup(ctx, decoder);
+    ctx->hwdec_devs = vd->hwdec_devs;
 
     reinit(vd);
 
@@ -441,8 +437,6 @@ static void init_avctx(struct dec_video *vd, const char *decoder,
     if (!lavc_codec)
         return;
 
-    ctx->hwdec_info = vd->hwdec_info;
-
     ctx->codec_timebase = (AVRational){0};
     if (strstr(decoder, "_mmal") || strstr(decoder, "_mediacodec"))
         ctx->codec_timebase = (AVRational){1, 1000000};
diff --git a/video/decode/vdpau.c b/video/decode/vdpau.c
index 313fabff76..2aba10c13b 100644
--- a/video/decode/vdpau.c
+++ b/video/decode/vdpau.c
@@ -75,7 +75,7 @@ static int init(struct lavc_ctx *ctx)
     struct priv *p = talloc_ptrtype(NULL, p);
     *p = (struct priv) {
         .log = mp_log_new(p, ctx->log, "vdpau"),
-        .mpvdp = ctx->hwdec_info->hwctx->vdpau_ctx,
+        .mpvdp = hwdec_devices_get(ctx->hwdec_devs, HWDEC_VDPAU)->ctx,
     };
     ctx->hwdec_priv = p;
 
@@ -83,14 +83,11 @@ static int init(struct lavc_ctx *ctx)
     return 0;
 }
 
-static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+static int probe(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                  const char *codec)
 {
-    hwdec_request_api(info, "vdpau");
-    if (!info || !info->hwctx || !info->hwctx->vdpau_ctx)
+    if (!hwdec_devices_load(ctx->hwdec_devs, HWDEC_VDPAU))
         return HWDEC_ERR_NO_CTX;
-    if (mp_vdpau_guess_if_emulated(info->hwctx->vdpau_ctx))
-        return HWDEC_ERR_EMULATED;
     return 0;
 }
 
diff --git a/video/decode/videotoolbox.c b/video/decode/videotoolbox.c
index 2d2f5f735c..c69d5e89e6 100644
--- a/video/decode/videotoolbox.c
+++ b/video/decode/videotoolbox.c
@@ -27,11 +27,10 @@
 #include "config.h"
 
 
-static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+static int probe(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                  const char *codec)
 {
-    hwdec_request_api(info, "videotoolbox");
-    if (!info || !info->hwctx || !info->hwctx->get_vt_fmt)
+    if (!hwdec_devices_load(ctx->hwdec_devs, HWDEC_VIDEOTOOLBOX))
         return HWDEC_ERR_NO_CTX;
     switch (mp_codec_to_av_codec_id(codec)) {
     case AV_CODEC_ID_H264:
@@ -89,8 +88,8 @@ static int init_decoder(struct lavc_ctx *ctx, int w, int h)
 
     AVVideotoolboxContext *vtctx = av_videotoolbox_alloc_context();
 
-    struct mp_hwdec_ctx *hwctx = ctx->hwdec_info->hwctx;
-    vtctx->cv_pix_fmt_type = hwctx->get_vt_fmt(hwctx);
+    struct mp_vt_ctx *vt = hwdec_devices_load(ctx->hwdec_devs, HWDEC_VIDEOTOOLBOX);
+    vtctx->cv_pix_fmt_type = vt->get_vt_fmt(vt);
 
     int err = av_videotoolbox_default_init2(ctx->avctx, vtctx);
     if (err < 0) {