wayland: correct window-scale behavior

The way the window-scale option is supposed to behave wasn't really ever
documented until 25cfc81. It turns out that different OS/WMs may do
slightly different things. For wayland, we need to fix two things.
First, only return the wl->window_size in GET_UNFS (aka don't return the
fullscreen size). For SET_UNFS, we need to change the behavior for the
maximized window case. If the window is maximized, first attempt to
unmaximize it and send the wayland event. If the compositor allows this
to happen, then go ahead and set the new dimensions and resize it. In
the case that the attempt to unmaximize is not successful, then don't
attempt the resize and just save the window size dimensions for later to
be used when the user unmaximizes the window.

1 files changed, 10 insertions, 3 deletions

diff --git a/video/out/wayland_common.c b/video/out/wayland_common.c
index 2bc9cf3297..d472a68ed3 100644
--- a/video/out/wayland_common.c
+++ b/video/out/wayland_common.c
@@ -1599,8 +1599,8 @@ int vo_wayland_control(struct vo *vo, int *events, int request, void *arg)
     }
     case VOCTRL_GET_UNFS_WINDOW_SIZE: {
         int *s = arg;
-        s[0] = mp_rect_w(wl->geometry) * wl->scaling;
-        s[1] = mp_rect_h(wl->geometry) * wl->scaling;
+        s[0] = mp_rect_w(wl->window_size) * wl->scaling;
+        s[1] = mp_rect_h(wl->window_size) * wl->scaling;
         return VO_TRUE;
     }
     case VOCTRL_SET_UNFS_WINDOW_SIZE: {
@@ -1609,7 +1609,14 @@ int vo_wayland_control(struct vo *vo, int *events, int request, void *arg)
         wl->window_size.y0 = 0;
         wl->window_size.x1 = s[0] / wl->scaling;
         wl->window_size.y1 = s[1] / wl->scaling;
-        if (!wl->vo_opts->fullscreen && !wl->vo_opts->window_maximized) {
+        if (!wl->vo_opts->fullscreen) {
+            if (wl->vo_opts->window_maximized) {
+                xdg_toplevel_unset_maximized(wl->xdg_toplevel);
+                wl_display_dispatch_pending(wl->display);
+                /* Make sure the compositor let us unmaximize */
+                if (wl->vo_opts->window_maximized)
+                    return VO_TRUE;
+            }
             wl->geometry = wl->window_size;
             wl->pending_vo_events |= VO_EVENT_RESIZE;
         }