mac: fix traditional fullscreen on macOS 11

the fullscreen style mask is not supported on macOS 11 anymore outside
of the native fullscreen animation. this can lead to a none working fs
or in the worst case a crash.

to fix this we will simulate a fullscreen window with a borderless
window with the size of the whole screen, though only on macOS 11.

Fixes #8490

2 files changed, 20 insertions, 4 deletions

diff --git a/video/out/cocoa_cb_common.swift b/video/out/cocoa_cb_common.swift
index 8dd76f4bc5..dd0738f7e3 100644
--- a/video/out/cocoa_cb_common.swift
+++ b/video/out/cocoa_cb_common.swift
@@ -132,11 +132,11 @@ class CocoaCB: Common {
     }
 
     override func windowSetToFullScreen() {
-        layer?.update()
+        layer?.update(force: true)
     }
 
     override func windowSetToWindow() {
-        layer?.update()
+        layer?.update(force: true)
     }
 
     override func windowDidUpdateFrame() {
diff --git a/video/out/mac/window.swift b/video/out/mac/window.swift
index 251bc4050d..8b6c5f0518 100644
--- a/video/out/mac/window.swift
+++ b/video/out/mac/window.swift
@@ -30,6 +30,7 @@ class Window: NSWindow, NSWindowDelegate {
     var isInFullscreen: Bool = false
     var isAnimating: Bool = false
     var isMoving: Bool = false
+    var previousStyleMask: NSWindow.StyleMask = [.titled, .closable, .miniaturizable, .resizable]
 
     var unfsContentFramePixel: NSRect { get { return convertToBacking(unfsContentFrame ?? NSRect(x: 0, y: 0, width: 160, height: 90)) } }
     var framePixel: NSRect { get { return convertToBacking(frame) } }
@@ -60,6 +61,7 @@ class Window: NSWindow, NSWindowDelegate {
         set {
             let responder = firstResponder
             let windowTitle = title
+            previousStyleMask = super.styleMask
             super.styleMask = newValue
             makeFirstResponder(responder)
             title = windowTitle
@@ -228,7 +230,14 @@ class Window: NSWindow, NSWindowDelegate {
 
     func setToFullScreen() {
         guard let targetFrame = targetScreen?.frame else { return }
-        styleMask.insert(.fullScreen)
+
+        if #available(macOS 11.0, *) {
+            styleMask = .borderless
+            common.titleBar?.hide(0.0)
+        } else {
+            styleMask.insert(.fullScreen)
+        }
+
         NSApp.presentationOptions = [.autoHideMenuBar, .autoHideDock]
         setFrame(targetFrame, display: true)
         endAnimation()
@@ -239,10 +248,17 @@ class Window: NSWindow, NSWindowDelegate {
 
     func setToWindow() {
         guard let tScreen = targetScreen else { return }
+
+        if #available(macOS 11.0, *) {
+            styleMask = previousStyleMask
+            common.titleBar?.hide(0.0)
+        } else {
+            styleMask.remove(.fullScreen)
+        }
+
         let newFrame = calculateWindowPosition(for: tScreen, withoutBounds: targetScreen == screen)
         NSApp.presentationOptions = []
         setFrame(newFrame, display: true)
-        styleMask.remove(.fullScreen)
         endAnimation()
         isInFullscreen = false
         mpv?.setOption(fullscreen: isInFullscreen)