video: refactor how VO exports hwdec device handles

The main change is with video/hwdec.h. mp_hwdec_info is made opaque (and
renamed to mp_hwdec_devices). Its accessors are mainly thread-safe (or
documented where not), which makes the whole thing saner and cleaner. In
particular, thread-safety rules become less subtle and more obvious.

The new internal API makes it easier to support multiple OpenGL interop
backends. (Although this is not done yet, and it's not clear whether it
ever will.)

This also removes all the API-specific fields from mp_hwdec_ctx and
replaces them with a "ctx" field. For d3d in particular, we drop the
mp_d3d_ctx struct completely, and pass the interfaces directly.

Remove the emulation checks from vaapi.c and vdpau.c; they are
pointless, and the checks that matter are done on the VO layer.

The d3d hardware decoders might slightly change behavior: dxva2-copy
will not use the VO device anymore if the VO supports proper interop.
This pretty much assumes that any in such cases the VO will not use any
form of exclusive mode, which makes using the VO device in copy mode
unnecessary.

This is a big refactor. Some things may be untested and could be broken.

1 files changed, 52 insertions, 25 deletions

diff --git a/video/hwdec.h b/video/hwdec.h
index 94667774e7..48ec6a2a21 100644
--- a/video/hwdec.h
+++ b/video/hwdec.h
@@ -25,16 +25,17 @@ enum hwdec_type {
 extern const struct m_opt_choice_alternatives mp_hwdec_names[];
 
 struct mp_hwdec_ctx {
-    enum hwdec_type type;
+    enum hwdec_type type; // (never HWDEC_NONE or HWDEC_AUTO)
     const char *driver_name; // NULL if unknown/not loaded
 
-    void *priv; // for free use by hwdec implementation
-
-    // API-specific, not needed by all backends.
-    struct mp_vdpau_ctx *vdpau_ctx;
-    struct mp_vaapi_ctx *vaapi_ctx;
-    struct mp_d3d_ctx *d3d_ctx;
-    uint32_t (*get_vt_fmt)(struct mp_hwdec_ctx *ctx);
+    // This is never NULL. Its meaning depends on the .type field:
+    //  HWDEC_VDPAU:            struct mp_vaapi_ctx*
+    //  HWDEC_VIDEOTOOLBOX:     struct mp_vt_ctx*
+    //  HWDEC_VAAPI:            struct mp_vaapi_ctx*
+    //  HWDEC_D3D11VA:          ID3D11Device*
+    //  HWDEC_DXVA2:            IDirect3DDevice9*
+    //  HWDEC_DXVA2_COPY:       IDirect3DDevice9*
+    void *ctx;
 
     // Optional.
     // Allocates a software image from the pool, downloads the hw image from
@@ -46,24 +47,50 @@ struct mp_hwdec_ctx {
                                        struct mp_image_pool *swpool);
 };
 
-// Used to communicate hardware decoder API handles from VO to video decoder.
-// The VO can set the context pointer for supported APIs.
-struct mp_hwdec_info {
-    // (Since currently only 1 hwdec API is loaded at a time, this pointer
-    // simply maps to the loaded one.)
-    struct mp_hwdec_ctx *hwctx;
-
-    // Can be used to lazily load a requested API.
-    // api_name is e.g. "vdpau" (like the fields above, without "_ctx")
-    // Can be NULL, is idempotent, caller checks hwctx fields for success/access.
-    // Due to threading, the callback is the only code that is allowed to
-    // change fields in this struct after initialization.
-    void (*load_api)(struct mp_hwdec_info *info, const char *api_name);
-    void *load_api_ctx;
+struct mp_vt_ctx {
+    void *priv;
+    uint32_t (*get_vt_fmt)(struct mp_vt_ctx *ctx);
 };
 
-// Trivial helper to call info->load_api().
-// Implemented in vd_lavc.c.
-void hwdec_request_api(struct mp_hwdec_info *info, const char *api_name);
+// Used to communicate hardware decoder device handles from VO to video decoder.
+struct mp_hwdec_devices;
+
+struct mp_hwdec_devices *hwdec_devices_create(void);
+void hwdec_devices_destroy(struct mp_hwdec_devices *devs);
+
+// Return the device context for the given API type. Returns NULL if none
+// available. Logically, the returned pointer remains valid until VO
+// uninitialization is started (all users of it must be uninitialized before).
+// hwdec_devices_request() may be used before this to lazily load devices.
+struct mp_hwdec_ctx *hwdec_devices_get(struct mp_hwdec_devices *devs,
+                                       enum hwdec_type type);
+
+// For code which still strictly assumes there is 1 (or none) device.
+struct mp_hwdec_ctx *hwdec_devices_get_first(struct mp_hwdec_devices *devs);
+
+// Add this to the list of internal devices. Adding the same pointer twice must
+// be avoided.
+void hwdec_devices_add(struct mp_hwdec_devices *devs, struct mp_hwdec_ctx *ctx);
+
+// Remove this from the list of internal devices. Idempotent/ignores entries
+// not added yet.
+void hwdec_devices_remove(struct mp_hwdec_devices *devs, struct mp_hwdec_ctx *ctx);
+
+// Can be used to enable lazy loading of an API with hwdec_devices_request().
+// If used at all, this must be set/unset during initialization/uninitialization,
+// as concurrent use with hwdec_devices_request() is a race condition.
+void hwdec_devices_set_loader(struct mp_hwdec_devices *devs,
+    void (*load_api)(void *ctx, enum hwdec_type type), void *load_api_ctx);
+
+// Cause VO to lazily load the requested device, and will block until this is
+// done (even if not available).
+void hwdec_devices_request(struct mp_hwdec_devices *devs, enum hwdec_type type);
+
+// Convenience function:
+// - return NULL if devs==NULL
+// - call hwdec_devices_request(devs, type)
+// - call hwdec_devices_get(devs, type)
+// - then return the mp_hwdec_ctx.ctx field
+void *hwdec_devices_load(struct mp_hwdec_devices *devs, enum hwdec_type type);
 
 #endif