vo: different hack for VOs which need to mangle mouse input

Follow up on commit 760548da. Mouse handling is a bit confusing, because
there are at least 3 coordinate systems associated with it, and it
should be cleaned up. But that is hard, so just apply a hack which gets
the currently-annoying issue (VO backends needing access to the VO) out
of the way.

7 files changed, 47 insertions, 40 deletions

diff --git a/input/input.c b/input/input.c
index c035187126..97943e05b3 100644
--- a/input/input.c
+++ b/input/input.c
@@ -159,6 +159,9 @@ struct input_ctx {
     // Unlike mouse_x/y, this can be used to resolve mouse click bindings.
     int mouse_vo_x, mouse_vo_y;
 
+    bool mouse_mangle, mouse_src_mangle;
+    struct mp_rect mouse_src, mouse_dst;
+
     bool test;
 
     bool default_bindings;
@@ -738,11 +741,37 @@ void mp_input_put_axis(struct input_ctx *ictx, int direction, double value)
     input_unlock(ictx);
 }
 
+void mp_input_set_mouse_transform(struct input_ctx *ictx, struct mp_rect *dst,
+                                  struct mp_rect *src)
+{
+    input_lock(ictx);
+    ictx->mouse_mangle = dst || src;
+    if (ictx->mouse_mangle) {
+        ictx->mouse_dst = *dst;
+        ictx->mouse_src_mangle = !!src;
+        if (ictx->mouse_src_mangle)
+            ictx->mouse_src = *src;
+    }
+    input_unlock(ictx);
+}
+
 void mp_input_set_mouse_pos(struct input_ctx *ictx, int x, int y)
 {
     input_lock(ictx);
     MP_DBG(ictx, "mouse move %d/%d\n", x, y);
 
+    if (ictx->mouse_mangle) {
+        struct mp_rect *src = &ictx->mouse_src;
+        struct mp_rect *dst = &ictx->mouse_dst;
+        x = MPCLAMP(x, dst->x0, dst->x1) - dst->x0;
+        y = MPCLAMP(y, dst->y0, dst->y1) - dst->y0;
+        if (ictx->mouse_src_mangle) {
+            x = x * 1.0 / (dst->x1 - dst->x0) * (src->x1 - src->x0) + src->x0;
+            y = y * 1.0 / (dst->y1 - dst->y0) * (src->y1 - src->y0) + src->y0;
+        }
+        MP_DBG(ictx, "-> %d/%d\n", x, y);
+    }
+
     ictx->mouse_event_counter++;
     ictx->mouse_vo_x = x;
     ictx->mouse_vo_y = y;
diff --git a/input/input.h b/input/input.h
index 2dc7b4b366..2c572fa788 100644
--- a/input/input.h
+++ b/input/input.h
@@ -133,6 +133,13 @@ void mp_input_set_mouse_pos(struct input_ctx *ictx, int x, int y);
 
 void mp_input_get_mouse_pos(struct input_ctx *ictx, int *x, int *y);
 
+/* Make mp_input_set_mouse_pos() mangle the mouse coordinates. Hack for certain
+ * VOs. dst=NULL, src=NULL reset it. src can be NULL.
+ */
+struct mp_rect;
+void mp_input_set_mouse_transform(struct input_ctx *ictx, struct mp_rect *dst,
+                                  struct mp_rect *src);
+
 // As for the cmd one you usually don't need this function.
 void mp_input_rm_key_fd(struct input_ctx *ictx, int fd);
 
diff --git a/video/out/vo.c b/video/out/vo.c
index f7dafa2735..db6a97eb26 100644
--- a/video/out/vo.c
+++ b/video/out/vo.c
@@ -177,6 +177,7 @@ static struct vo *vo_create(struct mpv_global *global,
         .max_video_queue = 1,
     };
     talloc_steal(vo, log);
+    mp_input_set_mouse_transform(vo->input_ctx, NULL, NULL);
     if (vo->driver->encode != !!vo->encode_lavc_ctx)
         goto error;
     struct m_config *config = m_config_from_obj_desc(vo, vo->log, &desc);
@@ -415,10 +416,7 @@ void vo_mouse_movement(struct vo *vo, int posx, int posy)
 {
     if (!vo->opts->enable_mouse_movements)
         return;
-    float p[2] = {posx, posy};
-    if (vo->driver->caps & VO_CAP_EVIL_OSD)
-        vo_control(vo, VOCTRL_WINDOW_TO_OSD_COORDS, p);
-    mp_input_set_mouse_pos(vo->input_ctx, p[0], p[1]);
+    mp_input_set_mouse_pos(vo->input_ctx, posx, posy);
 }
 
 /**
diff --git a/video/out/vo.h b/video/out/vo.h
index 2dade4c9d1..cc1fd6c23a 100644
--- a/video/out/vo.h
+++ b/video/out/vo.h
@@ -71,7 +71,6 @@ enum mp_voctrl {
     VOCTRL_SET_DEINTERLACE,
     VOCTRL_GET_DEINTERLACE,
 
-    VOCTRL_WINDOW_TO_OSD_COORDS,        // float[2] (x/y)
     VOCTRL_GET_WINDOW_SIZE,             // int[2] (w/h)
     VOCTRL_SET_WINDOW_SIZE,             // int[2] (w/h)
 
@@ -134,8 +133,6 @@ struct voctrl_screenshot_args {
 
 // VO does handle mp_image_params.rotate in 90 degree steps
 #define VO_CAP_ROTATE90 1
-// Requires VOCTRL_WINDOW_TO_OSD_COORDS to map mouse coords. to OSD coords.
-#define VO_CAP_EVIL_OSD 2
 
 #define VO_MAX_QUEUE 5
 
diff --git a/video/out/vo_wayland.c b/video/out/vo_wayland.c
index e0cf143fa0..2abb2b534f 100644
--- a/video/out/vo_wayland.c
+++ b/video/out/vo_wayland.c
@@ -40,6 +40,7 @@
 #include "sub/img_convert.h"
 
 #include "common/msg.h"
+#include "input/input.h"
 
 #include "wayland_common.h"
 #include "wayland-version.h"
@@ -501,6 +502,8 @@ static bool resize(struct priv *p)
     p->dst_w = p->dst.x1 - p->dst.x0;
     p->dst_h = p->dst.y1 - p->dst.y0;
 
+    mp_input_set_mouse_transform(p->vo->input_ctx, &p->dst, NULL);
+
     MP_DBG(wl, "resizing %dx%d -> %dx%d\n", wl->window.width,
                                             wl->window.height,
                                             p->dst_w,
@@ -898,15 +901,6 @@ static int control(struct vo *vo, uint32_t request, void *data)
         return VO_TRUE;
     case VOCTRL_REDRAW_FRAME:
         return redraw_frame(p);
-    case VOCTRL_WINDOW_TO_OSD_COORDS:
-    {
-        // OSD is rendered into the scaled image
-        float *c = data;
-        struct mp_rect *dst = &p->dst;
-        c[0] = av_clipf(c[0], dst->x0, dst->x1) - dst->x0;
-        c[1] = av_clipf(c[1], dst->y0, dst->y1) - dst->y0;
-        return VO_TRUE;
-    }
     case VOCTRL_SCREENSHOT:
     {
         struct voctrl_screenshot_args *args = data;
@@ -928,7 +922,6 @@ static int control(struct vo *vo, uint32_t request, void *data)
 const struct vo_driver video_out_wayland = {
     .description = "Wayland SHM video output",
     .name = "wayland",
-    .caps = VO_CAP_EVIL_OSD,
     .priv_size = sizeof(struct priv),
     .preinit = preinit,
     .query_format = query_format,
diff --git a/video/out/vo_x11.c b/video/out/vo_x11.c
index 6cd2220c2b..f80bc0227f 100644
--- a/video/out/vo_x11.c
+++ b/video/out/vo_x11.c
@@ -24,7 +24,6 @@
 #include <sys/types.h>
 
 #include <libswscale/swscale.h>
-#include <libavutil/common.h>
 
 #include "config.h"
 #include "vo.h"
@@ -53,11 +52,10 @@
 #include "video/fmt-conversion.h"
 
 #include "common/msg.h"
+#include "input/input.h"
 #include "options/options.h"
 #include "osdep/timer.h"
 
-extern int sws_flags;
-
 struct priv {
     struct vo *vo;
 
@@ -340,6 +338,8 @@ static bool resize(struct vo *vo)
     p->osd.mr = FFMIN(0, p->osd.mr);
     p->osd.ml = FFMIN(0, p->osd.ml);
 
+    mp_input_set_mouse_transform(vo->input_ctx, &p->dst, NULL);
+
     p->image_width = (p->dst_w + 7) & (~7);
     p->image_height = p->dst_h;
 
@@ -622,14 +622,6 @@ static int control(struct vo *vo, uint32_t request, void *data)
     case VOCTRL_REDRAW_FRAME:
         draw_image(vo, p->original_image);
         return true;
-    case VOCTRL_WINDOW_TO_OSD_COORDS: {
-        // OSD is rendered into the scaled image
-        float *c = data;
-        struct mp_rect *dst = &p->dst;
-        c[0] = av_clipf(c[0], dst->x0, dst->x1) - dst->x0;
-        c[1] = av_clipf(c[1], dst->y0, dst->y1) - dst->y0;
-        return VO_TRUE;
-    }
     case VOCTRL_SCREENSHOT: {
         struct voctrl_screenshot_args *args = data;
         args->out_image = get_screenshot(vo);
@@ -647,7 +639,6 @@ static int control(struct vo *vo, uint32_t request, void *data)
 const struct vo_driver video_out_x11 = {
     .description = "X11 ( XImage/Shm )",
     .name = "x11",
-    .caps = VO_CAP_EVIL_OSD,
     .priv_size = sizeof(struct priv),
     .options = (const struct m_option []){{0}},
     .preinit = preinit,
diff --git a/video/out/vo_xv.c b/video/out/vo_xv.c
index e8f0fcd490..1ea16558ac 100644
--- a/video/out/vo_xv.c
+++ b/video/out/vo_xv.c
@@ -57,6 +57,7 @@
 #include "sub/draw_bmp.h"
 #include "video/csputils.h"
 #include "options/m_option.h"
+#include "input/input.h"
 #include "osdep/timer.h"
 
 #define CK_METHOD_NONE       0 // no colorkey drawing
@@ -412,6 +413,8 @@ static void resize(struct vo *vo)
     xv_draw_colorkey(vo, &ctx->dst_rect);
     read_xv_csp(vo);
 
+    mp_input_set_mouse_transform(vo->input_ctx, &ctx->dst_rect, &ctx->src_rect);
+
     vo->want_redraw = true;
 }
 
@@ -843,16 +846,6 @@ static int control(struct vo *vo, uint32_t request, void *data)
         args->out_image = get_screenshot(vo);
         return true;
     }
-    case VOCTRL_WINDOW_TO_OSD_COORDS: {
-        float *c = data;
-        struct mp_rect *src = &ctx->src_rect;
-        struct mp_rect *dst = &ctx->dst_rect;
-        c[0] = av_clipf(c[0], dst->x0, dst->x1) - dst->x0;
-        c[1] = av_clipf(c[1], dst->y0, dst->y1) - dst->y0;
-        c[0] = c[0] / (dst->x1 - dst->x0) * (src->x1 - src->x0) + src->x0;
-        c[1] = c[1] / (dst->y1 - dst->y0) * (src->y1 - src->y0) + src->y0;
-        return VO_TRUE;
-    }
     }
     int events = 0;
     int r = vo_x11_control(vo, &events, request, data);
@@ -866,7 +859,6 @@ static int control(struct vo *vo, uint32_t request, void *data)
 const struct vo_driver video_out_xv = {
     .description = "X11/Xv",
     .name = "xv",
-    .caps = VO_CAP_EVIL_OSD,
     .preinit = preinit,
     .query_format = query_format,
     .reconfig = reconfig,