vo_opengl: do not use vaapi-over-GLX

This backend is selected if vaapi is available, but vaapi-over-EGL is
not. This causes various issues around the forced RGB conversion, which
is done with fixed, usually incorrect parameters.

It seems the existing auto probing check is too weak, and doesn't really
prevent it from getting loaded. Fix this by adding a flag to not ever
load this during auto probing.

I'm still not deleting it, because it's useful for testing on nvidia
machines.

See #4555.

3 files changed, 4 insertions, 5 deletions

diff --git a/video/out/opengl/hwdec.c b/video/out/opengl/hwdec.c
index 346493671f..c9256c9c5b 100644
--- a/video/out/opengl/hwdec.c
+++ b/video/out/opengl/hwdec.c
@@ -101,7 +101,7 @@ struct gl_hwdec *gl_hwdec_load_api(struct mp_log *log, GL *gl,
     bool is_auto = HWDEC_IS_AUTO(api);
     for (int n = 0; mpgl_hwdec_drivers[n]; n++) {
         const struct gl_hwdec_driver *drv = mpgl_hwdec_drivers[n];
-        if (is_auto || api == drv->api) {
+        if ((is_auto || api == drv->api) && !drv->testing_only) {
             struct gl_hwdec *r = load_hwdec_driver(log, gl, g, devs, drv, is_auto);
             if (r)
                 return r;
diff --git a/video/out/opengl/hwdec.h b/video/out/opengl/hwdec.h
index 92875d915e..c97219d15b 100644
--- a/video/out/opengl/hwdec.h
+++ b/video/out/opengl/hwdec.h
@@ -42,6 +42,8 @@ struct gl_hwdec_driver {
     // The hardware surface IMGFMT_ that must be passed to map_image later.
     // If the test_format callback is set, this field is ignored!
     int imgfmt;
+    // Dosn't load this unless requested by name.
+    bool testing_only;
     // Create the hwdec device. It must add it to hw->devs, if applicable.
     int (*create)(struct gl_hwdec *hw);
     // Prepare for rendering video. (E.g. create textures.)
diff --git a/video/out/opengl/hwdec_vaglx.c b/video/out/opengl/hwdec_vaglx.c
index 6248a64434..fb75b48d06 100644
--- a/video/out/opengl/hwdec_vaglx.c
+++ b/video/out/opengl/hwdec_vaglx.c
@@ -74,10 +74,6 @@ static int create(struct gl_hwdec *hw)
     Display *x11disp = glXGetCurrentDisplay();
     if (!x11disp)
         return -1;
-    if (hw->probing) {
-        MP_VERBOSE(hw, "Not using this by default.\n");
-        return -1;
-    }
     int x11scr = DefaultScreen(x11disp);
     struct priv *p = talloc_zero(hw, struct priv);
     hw->priv = p;
@@ -205,6 +201,7 @@ const struct gl_hwdec_driver gl_hwdec_vaglx = {
     .name = "vaapi-glx",
     .api = HWDEC_VAAPI,
     .imgfmt = IMGFMT_VAAPI,
+    .testing_only = true,
     .create = create,
     .reinit = reinit,
     .map_frame = map_frame,