cocoa-cb: initial implementation via opengl-cb API

this is meant to replace the old and not properly working vo_gpu/opengl
cocoa backend in the future. the problems are various shortcomings of
Apple's opengl implementation and buggy behaviour in certain
circumstances that couldn't be properly worked around. there are also
certain regressions on newer macOS versions from 10.11 onwards.

- awful opengl performance with a none layer backed context
- huge amount of dropped frames with an early context flush
- flickering of system elements like the dock or volume indicator
- double buffering not properly working with a none layer backed context
- bad performance in fullscreen because of system optimisations

all the problems were caused by using a normal opengl context, that
seems somewhat abandoned by apple, and are fixed by using a layer backed
opengl context instead. problems that couldn't be fixed could be
properly worked around.

this has all features our old backend has sans the wid embedding,
the possibility to disable the automatic GPU switching and taking
screenshots of the window content. the first was deemed unnecessary by
me for now, since i just use the libmpv API that others can use anyway.
second is technically not possible atm because we have to pre-allocate
our opengl context at a time the config isn't read yet, so we can't get
the needed property. third one is a bit tricky because of deadlocking
and it needed to be in sync, hopefully i can work around that in the
future.

this also has at least one additional feature or eye-candy. a properly
working fullscreen animation with the native fs. also since this is a
direct port of the old backend of the parts that could be used, though
with adaptions and improvements, this looks a lot cleaner and easier to
understand.

some credit goes to @pigoz for the initial swift build support which
i could improve upon.

Fixes: #5478, #5393, #5152, #5151, #4615, #4476, #3978, #3746, #3739,
#2392, #2217

1 files changed, 485 insertions, 0 deletions

diff --git a/video/out/cocoa_cb_common.swift b/video/out/cocoa_cb_common.swift
new file mode 100644
index 0000000000..9a6da82ecf
--- /dev/null
+++ b/video/out/cocoa_cb_common.swift
@@ -0,0 +1,485 @@
+/*
+ * This file is part of mpv.
+ *
+ * mpv is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+import Cocoa
+import IOKit.pwr_mgt
+
+class CocoaCB: NSObject {
+
+    var mpv: MPVHelper!
+    var window: Window!
+    var view: EventsView!
+    var layer: VideoLayer!
+    var link: CVDisplayLink?
+
+    var cursorHidden: Bool = false
+    var cursorVisibilityWanted: Bool = true
+    var isShuttingDown: Bool = false
+
+    enum State {
+        case uninit
+        case needsInit
+        case `init`
+    }
+    var backendState: State = .uninit
+
+    let eventsLock = NSLock()
+    var events: Int = 0
+
+    var lightSensor: io_connect_t = 0
+    var lastLmu: UInt64 = 0
+    var lightSensorIOPort: IONotificationPortRef?
+    var displaySleepAssertion: IOPMAssertionID = IOPMAssertionID(0)
+
+    let queue: DispatchQueue = DispatchQueue(label: "io.mpv.queue")
+
+    override init() {
+        super.init()
+        window = Window(cocoaCB: self)
+
+        view = EventsView(frame: window.contentView!.bounds, cocoaCB: self)
+        window.contentView!.addSubview(view)
+
+        layer = VideoLayer(cocoaCB: self)
+        view.layer = layer
+        view.wantsLayer = true
+    }
+
+    func setMpvHandle(_ ctx: OpaquePointer) {
+        mpv = MPVHelper(ctx)
+        layer.setUpGLCB()
+    }
+
+    func preinit() {
+        if backendState == .uninit {
+            backendState = .needsInit
+            DispatchQueue.main.async {
+                self.updateICCProfile()
+            }
+            startDisplayLink()
+        } else {
+            layer.setVideo(true)
+        }
+    }
+
+    func uninit() {
+        layer.setVideo(false)
+        window.orderOut(nil)
+    }
+
+    func reconfig() {
+        if backendState == .needsInit {
+            initBackend()
+        } else {
+            updateWindowSize()
+            layer.neededFlips += 1
+        }
+    }
+
+    func initBackend() {
+        NSApp.setActivationPolicy(.regular)
+
+        let targetScreen = getTargetScreen(forFullscreen: false) ?? NSScreen.main()
+        let wr = getWindowGeometry(forScreen: targetScreen!, videoOut: mpv.mpctx!.pointee.video_out)
+        let win = Window(contentRect: wr, styleMask: window.styleMask,
+                              screen: targetScreen, cocoaCB: self)
+        win.title = window.title
+        win.setOnTop(mpv.getBoolProperty("ontop"))
+        win.setBorder(mpv.getBoolProperty("border"))
+        win.keepAspect = mpv.getBoolProperty("keepaspect-window")
+        window.close()
+        window = win
+        window.contentView!.addSubview(view)
+        view.frame = window.contentView!.frame
+
+        setAppIcon()
+        window.isRestorable = false
+        window.makeMain()
+        window.makeKeyAndOrderFront(nil)
+        NSApp.activate(ignoringOtherApps: true)
+        layer.setVideo(true)
+
+        if self.mpv.getBoolProperty("fullscreen") {
+            window.toggleFullScreen(nil)
+        } else {
+            window.isMovableByWindowBackground = true
+        }
+
+        initLightSensor()
+        addDisplayReconfigureObserver()
+        backendState = .init
+    }
+
+    func updateWindowSize() {
+        if layer.hasVideo {
+            let targetScreen = getTargetScreen(forFullscreen: false) ?? NSScreen.main()
+            let wr = getWindowGeometry(forScreen: targetScreen!, videoOut: mpv.mpctx!.pointee.video_out)
+            if !window.isVisible {
+                window.makeKeyAndOrderFront(nil)
+            }
+            window.updateSize(wr.size)
+        }
+    }
+
+    func setAppIcon() {
+        if let app = NSApp as? Application {
+            NSApp.applicationIconImage = app.getMPVIcon()
+        }
+    }
+
+    func startDisplayLink() {
+        let displayId = UInt32(window.screen!.deviceDescription["NSScreenNumber"] as! Int)
+        CVDisplayLinkCreateWithActiveCGDisplays(&link)
+        CVDisplayLinkSetCurrentCGDisplay(link!, displayId)
+        CVDisplayLinkSetOutputHandler(link!) { link, now, out, inFlags, outFlags -> CVReturn in
+            self.layer.reportFlip()
+            return kCVReturnSuccess
+        }
+        CVDisplayLinkStart(link!)
+    }
+
+    func stopDisplaylink() {
+        if link != nil && CVDisplayLinkIsRunning(link!) {
+            CVDisplayLinkStop(link!)
+        }
+    }
+
+    func updateDisplaylink() {
+        let displayId = UInt32(window.screen!.deviceDescription["NSScreenNumber"] as! Int)
+        CVDisplayLinkSetCurrentCGDisplay(link!, displayId)
+
+        queue.asyncAfter(deadline: DispatchTime.now() + 0.1) {
+            self.flagEvents(VO_EVENT_WIN_STATE)
+        }
+    }
+
+    func currentFps() -> Double {
+        var actualFps = CVDisplayLinkGetActualOutputVideoRefreshPeriod(link!)
+        let nominalData = CVDisplayLinkGetNominalOutputVideoRefreshPeriod(link!)
+
+        if (nominalData.flags & Int32(CVTimeFlags.isIndefinite.rawValue)) < 1 {
+            let nominalFps = Double(nominalData.timeScale) / Double(nominalData.timeValue)
+
+            if actualFps > 0 {
+                actualFps = 1/actualFps
+            }
+
+            if fabs(actualFps - nominalFps) > 0.1 {
+                mpv.sendVerbose("Falling back to nominal display refresh rate: \(nominalFps)")
+                return nominalFps
+            } else {
+                return actualFps
+            }
+        }
+        mpv.sendWarning("Falling back to standard display refresh rate: 60Hz")
+        return 60.0
+    }
+
+    func enableDisplaySleep() {
+        IOPMAssertionRelease(displaySleepAssertion)
+        displaySleepAssertion = IOPMAssertionID(0)
+    }
+
+    func disableDisplaySleep() {
+        if displaySleepAssertion != IOPMAssertionID(0) { return }
+        IOPMAssertionCreateWithName(
+            kIOPMAssertionTypePreventUserIdleDisplaySleep as CFString,
+            IOPMAssertionLevel(kIOPMAssertionLevelOn),
+            "io.mpv.video_playing_back" as CFString,
+            &displaySleepAssertion)
+    }
+
+    func updateCusorVisibility() {
+        setCursorVisiblility(cursorVisibilityWanted)
+    }
+
+    func setCursorVisiblility(_ visible: Bool) {
+        let visibility = visible ? true : !view.canHideCursor()
+
+        if visibility && cursorHidden {
+            NSCursor.unhide()
+            cursorHidden = false;
+        } else if !visibility && !cursorHidden {
+            NSCursor.hide()
+            cursorHidden = true
+        }
+    }
+
+    func updateICCProfile() {
+        if mpv.getBoolProperty("icc-profile-auto") {
+            mpv.setGLCBICCProfile(window.screen!.colorSpace!)
+        }
+        layer.colorspace = window.screen!.colorSpace!.cgColorSpace!
+    }
+
+    func lmuToLux(_ v: UInt64) -> Int {
+        // the polinomial approximation for apple lmu value -> lux was empirically
+        // derived by firefox developers (Apple provides no documentation).
+        // https://bugzilla.mozilla.org/show_bug.cgi?id=793728
+        let power_c4 = 1 / pow(10, 27)
+        let power_c3 = 1 / pow(10, 19)
+        let power_c2 = 1 / pow(10, 12)
+        let power_c1 = 1 / pow(10, 5)
+
+        let term4 = -3.0 * power_c4 * pow(Decimal(v), 4)
+        let term3 = 2.6 * power_c3 * pow(Decimal(v), 3)
+        let term2 = -3.4 * power_c2 * pow(Decimal(v), 2)
+        let term1 = 3.9 * power_c1 * Decimal(v)
+
+        let lux = Int(ceil( Double((term4 + term3 + term2 + term1 - 0.19) as NSNumber)))
+        return Int(lux > 0 ? lux : 0)
+    }
+
+    var lightSensorCallback: IOServiceInterestCallback = { (ctx, service, messageType, messageArgument) -> Void in
+        let ccb: CocoaCB = MPVHelper.bridge(ptr: ctx!)
+
+        var outputs: UInt32 = 2
+        var values: [UInt64] = [0, 0]
+
+        var kr = IOConnectCallMethod(ccb.lightSensor, 0, nil, 0, nil, 0, &values, &outputs, nil, nil)
+        if kr == KERN_SUCCESS {
+            var mean = (values[0] + values[1]) / 2
+            if ccb.lastLmu != mean {
+                ccb.lastLmu = mean
+                ccb.mpv.setGLCBLux(ccb.lmuToLux(ccb.lastLmu))
+            }
+        }
+    }
+
+    func initLightSensor() {
+        let srv = IOServiceGetMatchingService(kIOMasterPortDefault, IOServiceMatching("AppleLMUController"))
+        if srv == IO_OBJECT_NULL {
+            mpv.sendVerbose("Can't find an ambient light sensor")
+            return
+        }
+
+        lightSensorIOPort = IONotificationPortCreate(kIOMasterPortDefault)
+        IONotificationPortSetDispatchQueue(lightSensorIOPort, queue)
+        var n = io_object_t()
+        IOServiceAddInterestNotification(lightSensorIOPort, srv, kIOGeneralInterest, lightSensorCallback, MPVHelper.bridge(obj: self), &n)
+        let kr = IOServiceOpen(srv, mach_task_self_, 0, &lightSensor)
+        IOObjectRelease(srv)
+
+        if kr != KERN_SUCCESS {
+            mpv.sendVerbose("Can't start ambient light sensor connection")
+            return
+        }
+        lightSensorCallback(MPVHelper.bridge(obj: self), 0, 0, nil)
+    }
+
+    func uninitLightSensor() {
+        if lightSensorIOPort != nil {
+            IONotificationPortDestroy(lightSensorIOPort)
+            IOObjectRelease(lightSensor)
+        }
+    }
+
+    var reconfigureCallback: CGDisplayReconfigurationCallBack = { (display, flags, userInfo) in
+        if flags.contains(.setModeFlag) {
+            let ccb: CocoaCB = MPVHelper.bridge(ptr: userInfo!)
+            let displayID = (ccb.window.screen!.deviceDescription["NSScreenNumber"] as! NSNumber).intValue
+            if UInt32(displayID) == display {
+                ccb.mpv.sendVerbose("Detected display mode change, updating screen refresh rate\n");
+                ccb.flagEvents(VO_EVENT_WIN_STATE)
+            }
+        }
+    }
+
+    func addDisplayReconfigureObserver() {
+        CGDisplayRegisterReconfigurationCallback(reconfigureCallback, MPVHelper.bridge(obj: self))
+    }
+
+    func removeDisplayReconfigureObserver() {
+        CGDisplayRemoveReconfigurationCallback(reconfigureCallback, MPVHelper.bridge(obj: self))
+    }
+
+    func getTargetScreen(forFullscreen fs: Bool) -> NSScreen? {
+        let screenID = fs ? mpv.getStringProperty("fs-screen") ?? "current":
+                            mpv.getStringProperty("screen") ?? "current"
+
+        switch screenID {
+        case "current", "default", "all":
+            return getScreenBy(id: -1)
+        default:
+            return getScreenBy(id: Int(screenID)!)
+        }
+    }
+
+    func getScreenBy(id screenID: Int) -> NSScreen? {
+        let screens = NSScreen.screens()
+        if screenID >= screens!.count {
+            mpv.sendInfo("Screen ID \(screenID) does not exist, falling back to current device")
+            return nil
+        } else if screenID < 0 {
+            return nil
+        }
+        return screens![screenID]
+    }
+
+    func getWindowGeometry(forScreen targetScreen: NSScreen,
+                           videoOut vo: UnsafeMutablePointer<vo>) -> NSRect {
+        let r = targetScreen.convertRectToBacking(targetScreen.frame)
+        var screenRC: mp_rect = mp_rect(x0: Int32(0),
+                                        y0: Int32(0),
+                                        x1: Int32(r.size.width),
+                                        y1: Int32(r.size.height))
+
+        var geo: vo_win_geometry = vo_win_geometry()
+        vo_calc_window_geometry2(vo, &screenRC, Double(targetScreen.backingScaleFactor), &geo)
+
+        // flip y coordinates
+        geo.win.y1 = Int32(r.size.height) - geo.win.y1
+        geo.win.y0 = Int32(r.size.height) - geo.win.y0
+
+        let wr = NSMakeRect(CGFloat(geo.win.x0), CGFloat(geo.win.y1),
+                            CGFloat(geo.win.x1 - geo.win.x0),
+                            CGFloat(geo.win.y0 - geo.win.y1))
+        return targetScreen.convertRectFromBacking(wr)
+    }
+
+    func flagEvents(_ ev: Int) {
+        eventsLock.lock()
+        events |= ev
+        eventsLock.unlock()
+    }
+
+    func checkEvents() -> Int {
+        eventsLock.lock()
+        let ev = events
+        events = 0
+        eventsLock.unlock()
+        return ev
+    }
+
+    var controlCallback: mpv_opengl_cb_control_fn = { ( ctx, events, request, data ) -> Int32 in
+        let ccb: CocoaCB = MPVHelper.bridge(ptr: ctx!)
+
+        switch mp_voctrl(request) {
+        case VOCTRL_CHECK_EVENTS:
+            events!.pointee = Int32(ccb.checkEvents())
+            return VO_TRUE
+        case VOCTRL_FULLSCREEN:
+            DispatchQueue.main.async {
+                ccb.window.toggleFullScreen(nil)
+            }
+            return VO_TRUE
+        case VOCTRL_GET_FULLSCREEN:
+            let fsData = data!.assumingMemoryBound(to: Int32.self)
+            fsData.pointee = ccb.window.isInFullscreen ? 1 : 0
+            return VO_TRUE
+        case VOCTRL_GET_DISPLAY_FPS:
+            let fps = data!.assumingMemoryBound(to: CDouble.self)
+            fps.pointee = ccb.currentFps()
+            return VO_TRUE
+        case VOCTRL_RESTORE_SCREENSAVER:
+            ccb.enableDisplaySleep()
+            return VO_TRUE
+        case VOCTRL_KILL_SCREENSAVER:
+            ccb.disableDisplaySleep()
+            return VO_TRUE
+        case VOCTRL_SET_CURSOR_VISIBILITY:
+            ccb.cursorVisibilityWanted = data!.assumingMemoryBound(to: CBool.self).pointee
+            DispatchQueue.main.async {
+                ccb.setCursorVisiblility(ccb.cursorVisibilityWanted)
+            }
+            return VO_TRUE
+        case VOCTRL_SET_UNFS_WINDOW_SIZE:
+            let sizeData = data!.assumingMemoryBound(to: Int32.self)
+            let size = UnsafeBufferPointer(start: sizeData, count: 2)
+            var rect = NSMakeRect(0, 0, CGFloat(size[0]), CGFloat(size[1]))
+            DispatchQueue.main.async {
+                if !ccb.mpv.getBoolProperty("hidpi-window-scale") {
+                    rect = ccb.window.currentScreen!.convertRectFromBacking(rect)
+                }
+                ccb.window.updateSize(rect.size)
+            }
+            return VO_TRUE
+        case VOCTRL_GET_WIN_STATE:
+            let minimized = data!.assumingMemoryBound(to: Int32.self)
+            minimized.pointee = ccb.window.isMiniaturized ? VO_WIN_STATE_MINIMIZED : Int32(0)
+            return VO_TRUE
+        case VOCTRL_UPDATE_WINDOW_TITLE:
+            let titleData = data!.assumingMemoryBound(to: Int8.self)
+            let title = String(cString: titleData)
+            DispatchQueue.main.async {
+                ccb.window.title = String(cString: titleData)
+            }
+            return VO_TRUE
+        case VOCTRL_PREINIT:
+            ccb.preinit()
+            return VO_TRUE
+        case VOCTRL_UNINIT:
+            DispatchQueue.main.async {
+                ccb.uninit()
+            }
+            return VO_TRUE
+        case VOCTRL_RECONFIG:
+            DispatchQueue.main.async {
+                ccb.reconfig()
+            }
+            return VO_TRUE
+        default:
+            return VO_NOTIMPL
+        }
+    }
+
+    func processEvent(_ event: UnsafePointer<mpv_event>) {
+        switch event.pointee.event_id {
+        case MPV_EVENT_SHUTDOWN:
+            if window.isAnimating {
+                isShuttingDown = true
+                return
+            }
+            setCursorVisiblility(true)
+            stopDisplaylink()
+            uninitLightSensor()
+            removeDisplayReconfigureObserver()
+            mpv.deinitGLCB()
+            mpv.deinitMPV()
+        case MPV_EVENT_PROPERTY_CHANGE:
+            if backendState == .init {
+                handlePropertyChange(event)
+            }
+        default:
+            break
+        }
+    }
+
+    func handlePropertyChange(_ event: UnsafePointer<mpv_event>) {
+        let pData = OpaquePointer(event.pointee.data)
+        guard let property = UnsafePointer<mpv_event_property>(pData)?.pointee else {
+            return
+        }
+
+        switch String(cString: property.name) {
+        case "border":
+            if let data = MPVHelper.mpvFlagToBool(property.data) {
+                window.setBorder(data)
+            }
+        case "ontop":
+            if let data = MPVHelper.mpvFlagToBool(property.data) {
+                window.setOnTop(data)
+            }
+        case "keepaspect-window":
+            if let data = MPVHelper.mpvFlagToBool(property.data) {
+                window.keepAspect = data
+            }
+        default:
+            break
+        }
+    }
+}