cocoa: fullscreen refactoring

this replaces the old fullscreen with the native
macOS fullscreen. additional the
--fs-black-out-screens was removed since the new
API doesn't support it in a way the old one did.
it can possibly be re-added if done manually.

Fixes #2857 #3272 #1352 #2062 #3864

4 files changed, 163 insertions, 89 deletions

diff --git a/video/out/cocoa/events_view.h b/video/out/cocoa/events_view.h
index 3429563b4d..6ad51cc133 100644
--- a/video/out/cocoa/events_view.h
+++ b/video/out/cocoa/events_view.h
@@ -20,8 +20,6 @@
 
 @interface MpvEventsView : NSView <NSDraggingDestination>
 @property(nonatomic, retain) MpvCocoaAdapter *adapter;
-- (void)setFullScreen:(BOOL)willBeFullscreen;
-- (void)clear;
 - (BOOL)canHideCursor;
 - (void)signalMousePosition;
 @end
diff --git a/video/out/cocoa/events_view.m b/video/out/cocoa/events_view.m
index 5de4a99eec..53d1da86ff 100644
--- a/video/out/cocoa/events_view.m
+++ b/video/out/cocoa/events_view.m
@@ -28,8 +28,6 @@
 @property(nonatomic, assign) BOOL clearing;
 @property(nonatomic, assign) BOOL hasMouseDown;
 @property(nonatomic, retain) NSTrackingArea *tracker;
-- (BOOL)hasDock:(NSScreen*)screen;
-- (BOOL)hasMenubar:(NSScreen*)screen;
 - (int)mpvButtonNumber:(NSEvent*)event;
 - (void)mouseDownEvent:(NSEvent *)event;
 - (void)mouseUpEvent:(NSEvent *)event;
@@ -51,54 +49,6 @@
     return self;
 }
 
-- (void)setFullScreen:(BOOL)willBeFullscreen
-{
-    if (willBeFullscreen && ![self isInFullScreenMode]) {
-        NSApplicationPresentationOptions popts =
-            NSApplicationPresentationDefault;
-
-        if ([self hasMenubar:[self.adapter fsScreen]])
-            // Cocoa raises an exception when autohiding the menubar but
-            // not the dock. They probably got bored while programming the
-            // multi screen support and took some shortcuts (tested on 10.8).
-            popts |= NSApplicationPresentationAutoHideMenuBar |
-                     NSApplicationPresentationAutoHideDock;
-
-        if ([self hasDock:[self.adapter fsScreen]])
-            popts |= NSApplicationPresentationAutoHideDock;
-
-        NSDictionary *fsopts = @{
-            NSFullScreenModeAllScreens : @([self.adapter fsModeAllScreens]),
-            NSFullScreenModeApplicationPresentationOptions : @(popts)
-        };
-
-        // The original "windowed" window will stay around since sending a
-        // view fullscreen wraps it in another window. This is noticeable when
-        // sending the View fullscreen to another screen. Make it go away
-        // manually.
-        [self.window orderOut:self];
-
-        [self enterFullScreenMode:[self.adapter fsScreen]
-                                  withOptions:fsopts];
-    }
-
-    if (!willBeFullscreen && [self isInFullScreenMode]) {
-        [self exitFullScreenModeWithOptions:nil];
-
-        // Show the "windowed" window again.
-        [self.window makeKeyAndOrderFront:self];
-        [self.window makeFirstResponder:self];
-    }
-}
-
-- (void)clear
-{
-    if ([self isInFullScreenMode]) {
-        self.clearing = YES;
-        [self exitFullScreenModeWithOptions:nil];
-    }
-}
-
 // mpv uses flipped coordinates, because X11 uses those. So let's just use them
 // as well without having to do any coordinate conversion of mouse positions.
 - (BOOL)isFlipped { return YES; }
@@ -379,27 +329,6 @@
     return NO;
 }
 
-- (BOOL)hasDock:(NSScreen*)screen
-{
-    NSRect vF = [screen visibleFrame];
-    NSRect f  = [screen frame];
-    return
-        // The visible frame's width is smaller: dock is on left or right end
-        // of this method's receiver.
-        vF.size.width < f.size.width ||
-        // The visible frame's veritical origin is bigger: dock is
-        // on the bottom of this method's receiver.
-        vF.origin.y > f.origin.y;
-
-}
-
-- (BOOL)hasMenubar:(NSScreen*)screen
-{
-    NSRect vF = [screen visibleFrame];
-    NSRect f  = [screen frame];
-    return f.size.height + f.origin.y > vF.size.height + vF.origin.y;
-}
-
 - (int)mpvButtonNumber:(NSEvent*)event
 {
     int buttonNumber = [event buttonNumber];
diff --git a/video/out/cocoa/mpvadapter.h b/video/out/cocoa/mpvadapter.h
index 5b87e89d04..e547708e17 100644
--- a/video/out/cocoa/mpvadapter.h
+++ b/video/out/cocoa/mpvadapter.h
@@ -26,14 +26,14 @@
 - (void)putAxis:(int)mpkey delta:(float)delta;
 - (void)putCommand:(char*)cmd;
 - (void)handleFilesArray:(NSArray *)files;
-- (void)didChangeWindowedScreenProfile:(NSScreen *)screen;
+- (void)didChangeWindowedScreenProfile:(NSNotification *)notification;
 - (void)performAsyncResize:(NSSize)size;
 - (void)didChangeMousePosition;
 
 - (BOOL)isInFullScreenMode;
 - (BOOL)keyboardEnabled;
 - (BOOL)mouseEnabled;
-- (NSScreen *)fsScreen;
-- (BOOL)fsModeAllScreens;
+
+- (NSScreen *)getTargetScreen;
 @property(nonatomic, assign) struct vo *vout;
 @end
diff --git a/video/out/cocoa/window.m b/video/out/cocoa/window.m
index d89e296b40..6d63263cb4 100644
--- a/video/out/cocoa/window.m
+++ b/video/out/cocoa/window.m
@@ -26,15 +26,25 @@
 #include "window.h"
 
 @interface MpvVideoWindow()
+@property(nonatomic, retain) NSScreen *targetScreen;
+@property(nonatomic, retain) NSScreen *previousScreen;
+@property(nonatomic, retain) NSScreen *currentScreen;
+
 - (NSRect)frameRect:(NSRect)frameRect forCenteredContentSize:(NSSize)newSize;
 - (void)setCenteredContentSize:(NSSize)newSize;
 @end
 
 @implementation MpvVideoWindow {
     NSSize _queued_video_size;
+    NSRect _unfs_content_frame;
+    NSRect _unfs_screen_frame;
+    int _is_animating;
 }
 
 @synthesize adapter = _adapter;
+@synthesize targetScreen = _target_screen;
+@synthesize previousScreen = _previous_screen;
+@synthesize currentScreen = _current_screen;
 - (id)initWithContentRect:(NSRect)content_rect
                 styleMask:(NSUInteger)style_mask
                   backing:(NSBackingStoreType)buffering_type
@@ -46,10 +56,96 @@
                                     defer:flag]) {
         [self setBackgroundColor:[NSColor blackColor]];
         [self setMinSize:NSMakeSize(50,50)];
+        [self setCollectionBehavior: NSWindowCollectionBehaviorFullScreenPrimary];
+
+        self.targetScreen = [self screen];
+        self.currentScreen = [self screen];
+        _is_animating = 0;
+        _unfs_content_frame = [self convertRectToScreen:[[self contentView] frame]];
+        _unfs_screen_frame = [[self screen] frame];
     }
     return self;
 }
 
+- (void)toggleFullScreen:(id)sender
+{
+    if (_is_animating)
+        return;
+
+    _is_animating = 1;
+
+    self.targetScreen = [self.adapter getTargetScreen];
+    if(![self targetScreen] && ![self previousScreen]) {
+        self.targetScreen = [self screen];
+    } else if (![self targetScreen]) {
+        self.targetScreen = self.previousScreen;
+        self.previousScreen = nil;
+    } else {
+        self.previousScreen = [self screen];
+    }
+
+    if (![self.adapter isInFullScreenMode]) {
+        _unfs_content_frame = [self convertRectToScreen:[[self contentView] frame]];
+        _unfs_screen_frame = [[self screen] frame];
+    }
+
+    //move window to target screen when going to fullscreen
+    if (![self.adapter isInFullScreenMode] && ![[self targetScreen] isEqual:[self screen]]) {
+        [self setFrame:[self calculateWindowPositionForScreen:[self targetScreen]] display:YES];
+    }
+
+    [super toggleFullScreen:sender];
+
+    if (![self.adapter isInFullScreenMode]) {
+        [self setStyleMask:([self styleMask] | NSWindowStyleMaskFullScreen)];
+        NSRect frame = [[self targetScreen] frame];
+        [self setFrame:frame display:YES];
+    } else {
+        [self setStyleMask:([self styleMask] & ~NSWindowStyleMaskFullScreen)];
+        NSRect frame = [self calculateWindowPositionForScreen:[self targetScreen]];
+        [self setFrame:frame display:YES];
+        [self setContentAspectRatio:_unfs_content_frame.size];
+        [self setCenteredContentSize:_unfs_content_frame.size];
+    }
+}
+
+- (NSArray *)customWindowsToEnterFullScreenForWindow:(NSWindow *)window
+{
+    return [NSArray arrayWithObject:window];
+}
+
+- (NSArray*)customWindowsToExitFullScreenForWindow:(NSWindow*)window
+{
+    return [NSArray arrayWithObject:window];
+}
+
+// we still need to keep those around or it will use the standard animation
+- (void)window:(NSWindow *)window startCustomAnimationToEnterFullScreenWithDuration:(NSTimeInterval)duration {}
+
+- (void)window:(NSWindow *)window startCustomAnimationToExitFullScreenWithDuration:(NSTimeInterval)duration {}
+
+- (void)windowDidEnterFullScreen:(NSNotification *)notification
+{
+    _is_animating = 0;
+    [self.adapter windowDidEnterFullScreen:notification];
+}
+
+- (void)windowDidExitFullScreen:(NSNotification *)notification
+{
+    _is_animating = 0;
+    [self.adapter windowDidExitFullScreen:notification];
+}
+
+- (void)windowDidFailToEnterFullScreen:(NSWindow *)window
+{
+    _is_animating = 0;
+}
+
+- (void)windowDidFailToExitFullScreen:(NSWindow *)window
+{
+    _is_animating = 0;
+}
+
 - (void)windowDidChangeBackingProperties:(NSNotification *)notification
 {
     // XXX: we maybe only need expose for this
@@ -58,12 +154,18 @@
 
 - (void)windowDidChangeScreen:(NSNotification *)notification
 {
+    //this event doesn't exclusively trigger on screen change
+    //examples: screen reconfigure, toggling fullscreen
+    if (!_is_animating && ![[self currentScreen] isEqual:[self screen]]) {
+        self.previousScreen = [self screen];
+    }
+    self.currentScreen = [self screen];
     [self.adapter windowDidChangeScreen:notification];
 }
 
 - (void)windowDidChangeScreenProfile:(NSNotification *)notification
 {
-    [self.adapter didChangeWindowedScreenProfile:[self screen]];
+    [self.adapter didChangeWindowedScreenProfile:notification];
 }
 
 - (void)windowDidResignKey:(NSNotification *)notification
@@ -125,8 +227,41 @@
            animate:NO];
 }
 
+- (NSRect)calculateWindowPositionForScreen:(NSScreen *)screen
+{
+    NSRect frame = [self frameRectForContentRect:_unfs_content_frame];
+    NSRect targetFrame = [screen frame];
+
+    CGFloat x_per = (_unfs_screen_frame.size.width - frame.size.width);
+    CGFloat y_per = (_unfs_screen_frame.size.height - frame.size.height);
+    if (x_per > 0) x_per = (frame.origin.x - _unfs_screen_frame.origin.x)/x_per;
+    if (y_per > 0) y_per = (frame.origin.y - _unfs_screen_frame.origin.y)/y_per;
+
+    frame.origin.x = targetFrame.origin.x +
+        (targetFrame.size.width - frame.size.width)*x_per;
+    frame.origin.y = targetFrame.origin.y +
+        (targetFrame.size.height - frame.size.height)*y_per;
+
+    //screen bounds right and left
+    if (frame.origin.x + frame.size.width > targetFrame.origin.x + targetFrame.size.width)
+        frame.origin.x = targetFrame.origin.x + targetFrame.size.width - frame.size.width;
+    if (frame.origin.x < targetFrame.origin.x)
+        frame.origin.x = targetFrame.origin.x;
+
+    //screen bounds top and bottom
+    if (frame.origin.y + frame.size.height > targetFrame.origin.y + targetFrame.size.height)
+        frame.origin.y = targetFrame.origin.y + targetFrame.size.height - frame.size.height;
+    if (frame.origin.y < targetFrame.origin.y)
+        frame.origin.y = targetFrame.origin.y;
+
+    return frame;
+}
+
 - (NSRect)constrainFrameRect:(NSRect)nf toScreen:(NSScreen *)screen
 {
+    if (_is_animating)
+        screen = [self targetScreen];
+
     NSRect of  = [self frame];
     NSRect vf  = [screen ?: self.screen ?: [NSScreen mainScreen] visibleFrame];
     NSRect ncf = [self contentRectForFrameRect:nf];
@@ -136,7 +271,7 @@
     if (NSMaxY(nf) > NSMaxY(vf))
         nf.origin.y = NSMaxY(vf) - NSHeight(nf);
 
-    // Prevent the window's titlebar from exiting the screen on the top edge.
+    // Prevent the window's titlebar from exiting the screen on the bottom edge.
     if (NSMaxY(ncf) < NSMinY(vf))
         nf.origin.y = NSMinY(vf) + NSMinY(ncf) - NSMaxY(ncf);
 
@@ -157,30 +292,42 @@
     return nf;
 }
 
+- (void)windowWillStartLiveResize:(NSNotification *)notification
+{
+    [self.adapter windowWillStartLiveResize:notification];
+}
+
 - (void)windowDidEndLiveResize:(NSNotification *)notification
 {
+    [self.adapter windowDidEndLiveResize:notification];
     [self setFrame:[self constrainFrameRect:self.frame toScreen:self.screen]
            display:NO];
 }
 
+- (void)updateWindowFrame:(NSSize)newSize
+{
+    _unfs_content_frame = [self frameRect:_unfs_content_frame forCenteredContentSize:newSize];
+}
+
 - (void)tryDequeueSize {
     if (_queued_video_size.width <= 0.0 || _queued_video_size.height <= 0.0)
         return;
 
-    // XXX find a way to kill this state
-    if (![self.adapter isInFullScreenMode]) {
-        [self setContentAspectRatio:_queued_video_size];
-        [self setCenteredContentSize:_queued_video_size];
-        _queued_video_size = NSZeroSize;
-    }
+    [self setContentAspectRatio:_queued_video_size];
+    [self setCenteredContentSize:_queued_video_size];
+    _queued_video_size = NSZeroSize;
 }
 
-- (void)queueNewVideoSize:(NSSize)new_size
+- (void)queueNewVideoSize:(NSSize)newSize
 {
-    if (NSEqualSizes(_queued_video_size, new_size))
-        return;
-    _queued_video_size = new_size;
-    [self tryDequeueSize];
+    if ([self.adapter isInFullScreenMode]) {
+        [self updateWindowFrame:newSize];
+    } else {
+        if (NSEqualSizes(_queued_video_size, newSize))
+            return;
+        _queued_video_size = newSize;
+        [self tryDequeueSize];
+    }
 }
 
 - (void)windowDidBecomeMain:(NSNotification *)notification {