x11: separate window creation and configuration

This gets rid of an old hack, VOFLAG_HIDDEN. Although handling of it has
been sane for a while, it used to cause much pain, and is still
unintuitive and weird even today.

The main reason for this hack is that OpenGL selects a X11 Visual for
you, and you're supposed to use this Visual when creating the X window
for the OpenGL context. Which means the X window can't be created early
in the common X11 init code, but the OpenGL code needs to do something
before that. API-wise you need separate functions for X11 init and X11
window creation. The VOFLAG_HIDDEN hack conflated window creation and
the entrypoint for resizing on video resolution change into one
function, vo_x11_config_vo_window(). This required all platform backends
to handle this flag, even if they didn't need this mechanism.

Wayland still uses this for minor reasons (alpha support?), so the
wayland backend must be changed before the flag can be entirely removed.

8 files changed, 56 insertions, 36 deletions

diff --git a/video/out/opengl/x11.c b/video/out/opengl/x11.c
index eb7072dc4f..dfe86d8d1a 100644
--- a/video/out/opengl/x11.c
+++ b/video/out/opengl/x11.c
@@ -240,7 +240,8 @@ static bool config_window_x11(struct MPGLContext *ctx, int flags)
     glXGetFBConfigAttrib(vo->x11->display, fbc, GLX_GREEN_SIZE, &ctx->depth_g);
     glXGetFBConfigAttrib(vo->x11->display, fbc, GLX_BLUE_SIZE, &ctx->depth_b);
 
-    vo_x11_config_vo_window(vo, glx_ctx->vinfo, flags | VOFLAG_HIDDEN, "gl");
+    if (!vo_x11_create_vo_window(vo, glx_ctx->vinfo, "gl"))
+        return false;
 
     bool success = false;
     if (!(flags & VOFLAG_GLES)) {
@@ -265,8 +266,7 @@ static int glx_init(struct MPGLContext *ctx, int vo_flags)
 
 static int glx_reconfig(struct MPGLContext *ctx, int flags)
 {
-    struct glx_context *glx_ctx = ctx->priv;
-    vo_x11_config_vo_window(ctx->vo, glx_ctx->vinfo, flags, "gl");
+    vo_x11_config_vo_window(ctx->vo);
     return 0;
 }
 
diff --git a/video/out/opengl/x11egl.c b/video/out/opengl/x11egl.c
index 13901860f8..2b660f2904 100644
--- a/video/out/opengl/x11egl.c
+++ b/video/out/opengl/x11egl.c
@@ -142,7 +142,10 @@ static int mpegl_init(struct MPGLContext *ctx, int flags)
         goto uninit;
     }
 
-    vo_x11_config_vo_window(vo, vi, flags | VOFLAG_HIDDEN, "gl");
+    if (!vo_x11_create_vo_window(vo, vi, "gl")) {
+        XFree(vi);
+        goto uninit;
+    }
 
     XFree(vi);
 
@@ -166,7 +169,7 @@ uninit:
 
 static int mpegl_reconfig(struct MPGLContext *ctx, int flags)
 {
-    vo_x11_config_vo_window(ctx->vo, NULL, flags, "gl");
+    vo_x11_config_vo_window(ctx->vo);
     return 0;
 }
 
diff --git a/video/out/vo_vaapi.c b/video/out/vo_vaapi.c
index d5b035e350..413e1b7cb4 100644
--- a/video/out/vo_vaapi.c
+++ b/video/out/vo_vaapi.c
@@ -158,7 +158,7 @@ static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
 
     free_video_specific(p);
 
-    vo_x11_config_vo_window(vo, NULL, flags, "vaapi");
+    vo_x11_config_vo_window(vo);
 
     if (params->imgfmt != IMGFMT_VAAPI) {
         if (!alloc_swdec_surfaces(p, params->w, params->h, params->imgfmt))
@@ -577,6 +577,9 @@ static int preinit(struct vo *vo)
     if (!vo_x11_init(vo))
         goto fail;
 
+    if (!vo_x11_create_vo_window(vo, NULL, "vaapi"))
+        goto fail;
+
     p->display = vaGetDisplay(vo->x11->display);
     if (!p->display)
         goto fail;
diff --git a/video/out/vo_vdpau.c b/video/out/vo_vdpau.c
index 5eaa23cc77..f8830c8b00 100644
--- a/video/out/vo_vdpau.c
+++ b/video/out/vo_vdpau.c
@@ -524,7 +524,7 @@ static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
 
     free_video_specific(vo);
 
-    vo_x11_config_vo_window(vo, NULL, flags, "vdpau");
+    vo_x11_config_vo_window(vo);
 
     if (initialize_vdpau_objects(vo) < 0)
         return -1;
@@ -1040,6 +1040,11 @@ static int preinit(struct vo *vo)
     if (!vo_x11_init(vo))
         return -1;
 
+    if (!vo_x11_create_vo_window(vo, NULL, "vdpau")) {
+        vo_x11_uninit(vo);
+        return -1;
+    }
+
     vc->mpvdp = mp_vdpau_create_device_x11(vo->log, vo->x11->display, false);
     if (!vc->mpvdp) {
         vo_x11_uninit(vo);
diff --git a/video/out/vo_x11.c b/video/out/vo_x11.c
index 4347f8b99e..af8a5f7133 100644
--- a/video/out/vo_x11.c
+++ b/video/out/vo_x11.c
@@ -196,7 +196,7 @@ static int reconfig(struct vo *vo, struct mp_image_params *fmt, int flags)
 
     p->sws->src = *fmt;
 
-    vo_x11_config_vo_window(vo, NULL, flags, "x11");
+    vo_x11_config_vo_window(vo);
 
     if (!resize(vo))
         return -1;
@@ -402,7 +402,9 @@ static int preinit(struct vo *vo)
 
     MP_VERBOSE(vo, "selected visual: %d\n", (int)p->vinfo.visualid);
 
-    vo_x11_config_vo_window(vo, &p->vinfo, VOFLAG_HIDDEN, "x11");
+    if (!vo_x11_create_vo_window(vo, &p->vinfo, "x11"))
+        goto error;
+
     p->gc = XCreateGC(x11->display, x11->window, 0, NULL);
     return 0;
 
diff --git a/video/out/vo_xv.c b/video/out/vo_xv.c
index c2f9c9da76..cf2d92473b 100644
--- a/video/out/vo_xv.c
+++ b/video/out/vo_xv.c
@@ -489,7 +489,7 @@ static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
     if (!ctx->xv_format)
         return -1;
 
-    vo_x11_config_vo_window(vo, NULL, flags, "xv");
+    vo_x11_config_vo_window(vo);
 
     if (!ctx->f_gc && !ctx->vo_gc) {
         ctx->f_gc = XCreateGC(x11->display, x11->window, 0, 0);
@@ -761,6 +761,9 @@ static int preinit(struct vo *vo)
     if (!vo_x11_init(vo))
         return -1;
 
+    if (!vo_x11_create_vo_window(vo, NULL, "xv"))
+        goto error;
+
     struct vo_x11_state *x11 = vo->x11;
 
     /* check for Xvideo extension */
diff --git a/video/out/x11_common.c b/video/out/x11_common.c
index d077957c4e..0aa49a4257 100644
--- a/video/out/x11_common.c
+++ b/video/out/x11_common.c
@@ -1477,28 +1477,13 @@ static void wait_until_mapped(struct vo *vo)
     }
 }
 
-/* Create and setup a window suitable for display
- * vis: Visual to use for creating the window (NULL for default)
- * x, y: position of window (might be ignored)
- * width, height: size of window
- * flags: flags for window creation (VOFLAG_*)
- * classname: name to use for the X11 classhint
- *
- * If the window already exists, it just moves and resizes it.
- */
-void vo_x11_config_vo_window(struct vo *vo, XVisualInfo *vis, int flags,
+// Create the X11 window. There is only 1, and it must be created before
+// vo_x11_config_vo_window() is called. vis can be NULL for default.
+bool vo_x11_create_vo_window(struct vo *vo, XVisualInfo *vis,
                              const char *classname)
 {
-    struct mp_vo_opts *opts = vo->opts;
     struct vo_x11_state *x11 = vo->x11;
-
-    vo_x11_update_screeninfo(vo);
-
-    struct vo_win_geometry geo;
-    vo_calc_window_geometry(vo, &x11->screenrc, &geo);
-    vo_apply_window_geometry(vo, &geo);
-
-    struct mp_rect rc = geo.win;
+    assert(!x11->window);
 
     if (x11->parent) {
         if (x11->parent == x11->rootwin) {
@@ -1508,18 +1493,36 @@ void vo_x11_config_vo_window(struct vo *vo, XVisualInfo *vis, int flags,
         } else {
             XSelectInput(x11->display, x11->parent, StructureNotifyMask);
         }
-        vo_x11_update_geometry(vo);
-        rc = (struct mp_rect){0, 0, RC_W(x11->winrc), RC_H(x11->winrc)};
     }
     if (x11->window == None) {
-        vo_x11_create_window(vo, vis, rc);
+        vo_x11_create_window(vo, vis, (struct mp_rect){.x1 = 320, .y1 = 200 });
         vo_x11_classhint(vo, x11->window, classname);
         x11->window_hidden = true;
-        x11->winrc = geo.win;
     }
 
-    if (flags & VOFLAG_HIDDEN)
-        return;
+    return !!x11->window;
+}
+
+// Resize the window (e.g. new file, or video resolution change)
+void vo_x11_config_vo_window(struct vo *vo)
+{
+    struct mp_vo_opts *opts = vo->opts;
+    struct vo_x11_state *x11 = vo->x11;
+
+    assert(x11->window);
+
+    vo_x11_update_screeninfo(vo);
+
+    struct vo_win_geometry geo;
+    vo_calc_window_geometry(vo, &x11->screenrc, &geo);
+    vo_apply_window_geometry(vo, &geo);
+
+    struct mp_rect rc = geo.win;
+
+    if (x11->parent) {
+        vo_x11_update_geometry(vo);
+        rc = (struct mp_rect){0, 0, RC_W(x11->winrc), RC_H(x11->winrc)};
+    }
 
     bool reset_size = x11->old_dw != RC_W(rc) || x11->old_dh != RC_H(rc);
     x11->old_dw = RC_W(rc);
diff --git a/video/out/x11_common.h b/video/out/x11_common.h
index 50892cf26a..a25b8e2cca 100644
--- a/video/out/x11_common.h
+++ b/video/out/x11_common.h
@@ -125,8 +125,9 @@ int vo_x11_init(struct vo *vo);
 void vo_x11_uninit(struct vo *vo);
 int vo_x11_check_events(struct vo *vo);
 bool vo_x11_screen_is_composited(struct vo *vo);
-void vo_x11_config_vo_window(struct vo *vo, XVisualInfo *vis, int flags,
+bool vo_x11_create_vo_window(struct vo *vo, XVisualInfo *vis,
                              const char *classname);
+void vo_x11_config_vo_window(struct vo *vo);
 int vo_x11_control(struct vo *vo, int *events, int request, void *arg);
 
 #endif /* MPLAYER_X11_COMMON_H */