vo_dmabuf_wayland: scale smarter in hidpi situations

Previously, all scaling was forced to 1 with this vo and all coordinates
were calculated as if the scale was 1. This works since viewport relies
on the compositor completely for scaling so it doesn't really matter if
we don't draw directly to the correct size since the compositor will
just scale the rest for us. This does have some downsides however since
the OSD text might not be drawn at the actual resolution of the final
size of the video.

We can instead handle this by getting rid of the dmabuf_wayland specific
scaling logic and using the same values as everything else. In the
resize in vo_dmabuf_wayland, we just need to adjust the viewport
destination calls so they go to the wayland local coordinates and not
the physical ones. This should ensure that vo_dmabuf_wayland directly
goes to the desired size and the compositor doesn't need to operate on
it after the fact.

3 files changed, 12 insertions, 13 deletions

diff --git a/video/out/vo_dmabuf_wayland.c b/video/out/vo_dmabuf_wayland.c
index 3b8190e18e..a241ef1763 100644
--- a/video/out/vo_dmabuf_wayland.c
+++ b/video/out/vo_dmabuf_wayland.c
@@ -493,7 +493,7 @@ static void set_viewport_source(struct vo *vo, struct mp_rect src)
     if (p->force_window)
         return;
 
-    if (wl->video_viewport && !mp_rect_equals(&p->src, &src)) {
+    if (!mp_rect_equals(&p->src, &src)) {
         wp_viewport_set_source(wl->video_viewport, src.x0 << 8,
                                src.y0 << 8, mp_rect_w(src) << 8,
                                mp_rect_h(src) << 8);
@@ -528,15 +528,18 @@ static void resize(struct vo *vo)
     vo_get_src_dst_rects(vo, &src, &dst, &p->screen_osd_res);
     int window_w = p->screen_osd_res.ml + p->screen_osd_res.mr + mp_rect_w(dst);
     int window_h = p->screen_osd_res.mt + p->screen_osd_res.mb + mp_rect_h(dst);
-    wp_viewport_set_destination(wl->viewport, window_w, window_h);
+    wp_viewport_set_destination(wl->viewport, lround(window_w / wl->scaling),
+                                lround(window_h / wl->scaling));
 
     //now we restore pan for video viewport calculation
     vo->opts->pan_x = vo_opts->pan_x;
     vo->opts->pan_y = vo_opts->pan_y;
     vo_get_src_dst_rects(vo, &src, &dst, &p->screen_osd_res);
-    wp_viewport_set_destination(wl->video_viewport, mp_rect_w(dst), mp_rect_h(dst));
+    wp_viewport_set_destination(wl->video_viewport, lround(mp_rect_w(dst) / wl->scaling),
+                                                    lround(mp_rect_h(dst) / wl->scaling));
     wl_subsurface_set_position(wl->video_subsurface, dst.x0, dst.y0);
-    wp_viewport_set_destination(wl->osd_viewport, vo->dwidth, vo->dheight);
+    wp_viewport_set_destination(wl->osd_viewport, lround(vo->dwidth / wl->scaling),
+                                                  lround(vo->dheight / wl->scaling));
     wl_subsurface_set_position(wl->osd_subsurface, 0 - dst.x0, 0 - dst.y0);
     set_viewport_source(vo, src);
 }
diff --git a/video/out/wayland_common.c b/video/out/wayland_common.c
index de52006317..2afdd2aa5f 100644
--- a/video/out/wayland_common.c
+++ b/video/out/wayland_common.c
@@ -836,8 +836,7 @@ static void surface_handle_preferred_buffer_scale(void *data,
     if (wl->fractional_scale_manager)
         return;
 
-    // dmabuf_wayland is always wl->scaling = 1
-    wl->scaling = !wl->using_dmabuf_wayland ? scale : 1;
+    wl->scaling = scale;
     MP_VERBOSE(wl, "Obtained preferred scale, %f, from the compositor.\n",
                wl->scaling);
     wl->pending_vo_events |= VO_EVENT_DPI;
@@ -1070,8 +1069,7 @@ static void preferred_scale(void *data,
     struct vo_wayland_state *wl = data;
     double old_scale = wl->scaling;
 
-    // dmabuf_wayland is always wl->scaling = 1
-    wl->scaling = !wl->using_dmabuf_wayland ? (double)scale / 120 : 1;
+    wl->scaling = (double)scale / 120;
     MP_VERBOSE(wl, "Obtained preferred scale, %f, from the compositor.\n",
                wl->scaling);
     wl->pending_vo_events |= VO_EVENT_DPI;
@@ -1845,9 +1843,8 @@ static void set_surface_scaling(struct vo_wayland_state *wl)
     if (wl->fractional_scale_manager)
         return;
 
-    // dmabuf_wayland is always wl->scaling = 1
     double old_scale = wl->scaling;
-    wl->scaling = !wl->using_dmabuf_wayland ? wl->current_output->scale : 1;
+    wl->scaling = wl->current_output->scale;
     rescale_geometry(wl, old_scale);
 }
 
@@ -2208,7 +2205,7 @@ bool vo_wayland_init(struct vo *vo)
         .vo_opts_cache = m_config_cache_alloc(wl, vo->global, &vo_sub_opts),
     };
     wl->vo_opts = wl->vo_opts_cache->opts;
-    wl->using_dmabuf_wayland = !strcmp(wl->vo->driver->name, "dmabuf-wayland");
+    bool using_dmabuf_wayland = !strcmp(wl->vo->driver->name, "dmabuf-wayland");
 
     wl_list_init(&wl->output_list);
 
@@ -2330,7 +2327,7 @@ bool vo_wayland_init(struct vo *vo)
     update_app_id(wl);
     mp_make_wakeup_pipe(wl->wakeup_pipe);
 
-    wl->callback_surface = wl->using_dmabuf_wayland ? wl->video_surface : wl->surface;
+    wl->callback_surface = using_dmabuf_wayland ? wl->video_surface : wl->surface;
     wl->frame_callback = wl_surface_frame(wl->callback_surface);
     wl_callback_add_listener(wl->frame_callback, &frame_listener, wl);
     wl_surface_commit(wl->surface);
diff --git a/video/out/wayland_common.h b/video/out/wayland_common.h
index b6f951995b..25e973dcfd 100644
--- a/video/out/wayland_common.h
+++ b/video/out/wayland_common.h
@@ -112,7 +112,6 @@ struct vo_wayland_state {
     struct zwp_linux_dmabuf_feedback_v1 *dmabuf_feedback;
     wayland_format *format_map;
     uint32_t format_size;
-    bool using_dmabuf_wayland;
 
     /* presentation-time */
     struct wp_presentation  *presentation;