cocoa: refactor mouse events and cursor visibility

we reported some unnecessary mouse movements and not all mouse enter
and leave events. that lead to wrongly reported activity on hover areas
like on the OSC or comparable lua scripts. sometimes menu items were
shown that shouldn't be shown or they didn't vanish because of the
missing mouse leave event.

this incorporates @torque's fix for mouse leave events that weren't
triggered during a transition, like going to fullscreen. the
tracking area was updated but the mouse never left that area because
it was never over it.

besides some known cursor visibility bugs the aforementioned changes
also revealed some other bugs that weren't reproducible before because
of the missbehavior.
known issues, in some cases the cursor doesn't show or hide properly.
for example when switching spaces, switching Apps via CMD+Tab or a
system notification. former two could be fixed while keeping our current
blank cursor approach. though the notification case couldn't. there is
no event or similar to detect a notification and the cursor visibility
couldn't be recovered in any way.

new issues, i noticed that our event view isn't initialised yet when the
first VOCTRL_SET_CURSOR_VISIBILITY event gets dispatched, which depends
on the event view to be initialised. so the mouse cursor couldn't be
hidden when mpv was opened and the cursor was within the window bounds.
this wasn't noticeable before because of various bugs and unwanted
behavior that have been fixed with this. now, in case the event view
isn't ready yet, we set the visibility at a later point when the event
view is ready and a helper flag is set.

Fixes #1817 #3856 #4147

4 files changed, 41 insertions, 39 deletions

diff --git a/video/out/cocoa/events_view.h b/video/out/cocoa/events_view.h
index 6ad51cc133..91a6289be8 100644
--- a/video/out/cocoa/events_view.h
+++ b/video/out/cocoa/events_view.h
@@ -21,5 +21,4 @@
 @interface MpvEventsView : NSView <NSDraggingDestination>
 @property(nonatomic, retain) MpvCocoaAdapter *adapter;
 - (BOOL)canHideCursor;
-- (void)signalMousePosition;
 @end
diff --git a/video/out/cocoa/events_view.m b/video/out/cocoa/events_view.m
index f76ca0d617..d91c68046b 100644
--- a/video/out/cocoa/events_view.m
+++ b/video/out/cocoa/events_view.m
@@ -74,6 +74,9 @@
                                      userInfo:nil] autorelease];
 
     [self addTrackingArea:self.tracker];
+
+    if (![self containsMouseLocation])
+        [self.adapter putKey:MP_KEY_MOUSE_LEAVE withModifiers:0];
 }
 
 - (NSPoint)mouseLocation
@@ -148,8 +151,6 @@
 
     if (self.clearing)
         return;
-
-    [self signalMousePosition];
 }
 
 - (NSPoint)convertPointToPixels:(NSPoint)point
@@ -162,14 +163,6 @@
     return point;
 }
 
-- (void)signalMousePosition
-{
-    NSPoint p = [self convertPointToPixels:[self mouseLocation]];
-    p.x = MIN(MAX(p.x, 0), self.bounds.size.width-1);
-    p.y = MIN(MAX(p.y, 0), self.bounds.size.height-1);
-    [self.adapter signalMouseMovement:p];
-}
-
 - (void)signalMouseMovement:(NSEvent *)event
 {
     NSPoint p = [self convertPointToPixels:[event locationInWindow]];
diff --git a/video/out/cocoa/mpvadapter.h b/video/out/cocoa/mpvadapter.h
index e547708e17..65832aeae5 100644
--- a/video/out/cocoa/mpvadapter.h
+++ b/video/out/cocoa/mpvadapter.h
@@ -28,7 +28,6 @@
 - (void)handleFilesArray:(NSArray *)files;
 - (void)didChangeWindowedScreenProfile:(NSNotification *)notification;
 - (void)performAsyncResize:(NSSize)size;
-- (void)didChangeMousePosition;
 
 - (BOOL)isInFullScreenMode;
 - (BOOL)keyboardEnabled;
diff --git a/video/out/cocoa_common.m b/video/out/cocoa_common.m
index 6d0e5c62c0..9b4087395d 100644
--- a/video/out/cocoa_common.m
+++ b/video/out/cocoa_common.m
@@ -74,6 +74,10 @@ struct vo_cocoa_state {
     NSInteger window_level;
     int fullscreen;
 
+    bool cursor_visibility;
+    bool cursor_visibility_wanted;
+    bool cursor_needs_set;
+
     bool embedded; // wether we are embedding in another GUI
 
     IOPMAssertionID power_mgmt_assertion;
@@ -108,8 +112,6 @@ struct vo_cocoa_state {
                                         // render frames
     int frame_w, frame_h;               // dimensions of the frame rendered
 
-    NSCursor *blankCursor;
-
     char *window_title;
 };
 
@@ -357,13 +359,10 @@ void vo_cocoa_init(struct vo *vo)
         .power_mgmt_assertion = kIOPMNullAssertionID,
         .log = mp_log_new(s, vo->log, "cocoa"),
         .embedded = vo->opts->WinID >= 0,
+        .cursor_visibility = true,
+        .cursor_visibility_wanted = true,
         .fullscreen = 0,
     };
-    if (!s->embedded) {
-        NSImage* blankImage = [[NSImage alloc] initWithSize:NSMakeSize(1, 1)];
-        s->blankCursor = [[NSCursor alloc] initWithImage:blankImage hotSpot:NSZeroPoint];
-        [blankImage release];
-    }
     pthread_mutex_init(&s->lock, NULL);
     pthread_cond_init(&s->wakeup, NULL);
     pthread_mutex_init(&s->sync_lock, NULL);
@@ -379,6 +378,22 @@ void vo_cocoa_init(struct vo *vo)
     }
 }
 
+static void vo_cocoa_update_cursor(struct vo *vo, bool forceVisible)
+{
+    struct vo_cocoa_state *s = vo->cocoa;
+
+    if (s->embedded)
+        return;
+
+    if ((forceVisible || s->cursor_visibility_wanted) && !s->cursor_visibility) {
+        [NSCursor unhide];
+        s->cursor_visibility = YES;
+    } else if (!s->cursor_visibility_wanted && s->cursor_visibility) {
+        [NSCursor hide];
+        s->cursor_visibility = NO;
+    }
+}
+
 static int vo_cocoa_set_cursor_visibility(struct vo *vo, bool *visible)
 {
     struct vo_cocoa_state *s = vo->cocoa;
@@ -386,15 +401,15 @@ static int vo_cocoa_set_cursor_visibility(struct vo *vo, bool *visible)
     if (s->embedded)
         return VO_NOTIMPL;
 
-    MpvEventsView *v = (MpvEventsView *) s->view;
-
-    if (*visible) {
-        [[NSCursor arrowCursor] set];
-    } else if ([v canHideCursor] && s->blankCursor) {
-        [s->blankCursor set];
+    if (s->view) {
+        MpvEventsView *v = (MpvEventsView *) s->view;
+        s->cursor_visibility_wanted = !(!*visible && [v canHideCursor]);
+        vo_cocoa_update_cursor(vo, false);
     } else {
-        *visible = true;
+        s->cursor_visibility_wanted = *visible;
+        s->cursor_needs_set = true;
     }
+    *visible = s->cursor_visibility;
 
     return VO_TRUE;
 }
@@ -413,6 +428,7 @@ void vo_cocoa_uninit(struct vo *vo)
     run_on_main_thread(vo, ^{
         // if using --wid + libmpv there's no window to release
         if (s->window) {
+            vo_cocoa_update_cursor(vo, true);
             [s->window setDelegate:nil];
             [s->window close];
         }
@@ -435,9 +451,6 @@ void vo_cocoa_uninit(struct vo *vo)
         [s->view removeFromSuperview];
         [s->view release];
 
-        if (!s->embedded)
-            [s->blankCursor release];
-
         pthread_cond_destroy(&s->sync_wakeup);
         pthread_mutex_destroy(&s->sync_lock);
         pthread_cond_destroy(&s->wakeup);
@@ -571,8 +584,6 @@ static void create_ui(struct vo *vo, struct mp_rect *win, int geo_flags)
     view.adapter = adapter;
     s->view = view;
     [parent addSubview:s->view];
-    // update the cursor position now that the view has been added.
-    [view signalMousePosition];
     s->adapter = adapter;
 
     cocoa_register_menu_item_action(MPM_H_SIZE,   @selector(halfSize));
@@ -1033,20 +1044,20 @@ int vo_cocoa_control(struct vo *vo, int *events, int request, void *arg)
     flag_events(self.vout, VO_EVENT_ICC_PROFILE_CHANGED);
 }
 
-- (void)didChangeMousePosition
-{
-    struct vo_cocoa_state *s = self.vout->cocoa;
-    [(MpvEventsView *)s->view signalMousePosition];
-}
-
 - (void)windowDidResignKey:(NSNotification *)notification
 {
-    [self didChangeMousePosition];
+    vo_cocoa_update_cursor(self.vout, true);
 }
 
 - (void)windowDidBecomeKey:(NSNotification *)notification
 {
-    [self didChangeMousePosition];
+    struct vo_cocoa_state *s = self.vout->cocoa;
+    if (s->cursor_needs_set) {
+        vo_cocoa_set_cursor_visibility(self.vout, &s->cursor_visibility_wanted);
+        s->cursor_needs_set = false;
+    } else {
+        vo_cocoa_update_cursor(self.vout, false);
+    }
 }
 
 - (void)windowDidMiniaturize:(NSNotification *)notification