x11: separate window creation and configuration

This gets rid of an old hack, VOFLAG_HIDDEN. Although handling of it has
been sane for a while, it used to cause much pain, and is still
unintuitive and weird even today.

The main reason for this hack is that OpenGL selects a X11 Visual for
you, and you're supposed to use this Visual when creating the X window
for the OpenGL context. Which means the X window can't be created early
in the common X11 init code, but the OpenGL code needs to do something
before that. API-wise you need separate functions for X11 init and X11
window creation. The VOFLAG_HIDDEN hack conflated window creation and
the entrypoint for resizing on video resolution change into one
function, vo_x11_config_vo_window(). This required all platform backends
to handle this flag, even if they didn't need this mechanism.

Wayland still uses this for minor reasons (alpha support?), so the
wayland backend must be changed before the flag can be entirely removed.

2 files changed, 8 insertions, 5 deletions

diff --git a/video/out/opengl/x11.c b/video/out/opengl/x11.c
index eb7072dc4f..dfe86d8d1a 100644
--- a/video/out/opengl/x11.c
+++ b/video/out/opengl/x11.c
@@ -240,7 +240,8 @@ static bool config_window_x11(struct MPGLContext *ctx, int flags)
     glXGetFBConfigAttrib(vo->x11->display, fbc, GLX_GREEN_SIZE, &ctx->depth_g);
     glXGetFBConfigAttrib(vo->x11->display, fbc, GLX_BLUE_SIZE, &ctx->depth_b);
 
-    vo_x11_config_vo_window(vo, glx_ctx->vinfo, flags | VOFLAG_HIDDEN, "gl");
+    if (!vo_x11_create_vo_window(vo, glx_ctx->vinfo, "gl"))
+        return false;
 
     bool success = false;
     if (!(flags & VOFLAG_GLES)) {
@@ -265,8 +266,7 @@ static int glx_init(struct MPGLContext *ctx, int vo_flags)
 
 static int glx_reconfig(struct MPGLContext *ctx, int flags)
 {
-    struct glx_context *glx_ctx = ctx->priv;
-    vo_x11_config_vo_window(ctx->vo, glx_ctx->vinfo, flags, "gl");
+    vo_x11_config_vo_window(ctx->vo);
     return 0;
 }
 
diff --git a/video/out/opengl/x11egl.c b/video/out/opengl/x11egl.c
index 13901860f8..2b660f2904 100644
--- a/video/out/opengl/x11egl.c
+++ b/video/out/opengl/x11egl.c
@@ -142,7 +142,10 @@ static int mpegl_init(struct MPGLContext *ctx, int flags)
         goto uninit;
     }
 
-    vo_x11_config_vo_window(vo, vi, flags | VOFLAG_HIDDEN, "gl");
+    if (!vo_x11_create_vo_window(vo, vi, "gl")) {
+        XFree(vi);
+        goto uninit;
+    }
 
     XFree(vi);
 
@@ -166,7 +169,7 @@ uninit:
 
 static int mpegl_reconfig(struct MPGLContext *ctx, int flags)
 {
-    vo_x11_config_vo_window(ctx->vo, NULL, flags, "gl");
+    vo_x11_config_vo_window(ctx->vo);
     return 0;
 }