vo_opengl: add smoothmotion frame blending

SmoothMotion is a way to time and blend frames made popular by MadVR. It's
intended behaviour is to remove stuttering caused by mismatches between the
display refresh rate and the video fps, while preserving the video's original
artistic qualities (no soap opera effect). It's supposed to make 24fps video
playback on 60hz monitors as close as possible to a 24hz monitor.

Instead of drawing a frame once once it's pts has passed the vsync time, we
redraw at the display refresh rate, and if we detect the vsync is between two
frames we interpolated them (depending on their position relative to the vsync).
We actually interpolate as few frames as possible to avoid a blur effect as
much as possible. For example, if we were to play back a 1fps video on a 60hz
monitor, we would blend at most on 1 vsync for each frame (while the other 59
vsyncs would be rendered as is).

Frame interpolation is always done before scaling and in linear light when
possible (an ICC profile is used, or :srgb is used).

1 files changed, 46 insertions, 6 deletions

diff --git a/video/out/vo.c b/video/out/vo.c
index 0d135c2842..943e99d246 100644
--- a/video/out/vo.c
+++ b/video/out/vo.c
@@ -127,6 +127,8 @@ struct vo_internal {
     bool want_redraw;               // redraw request from VO to player
     bool send_reset;                // send VOCTRL_RESET
     bool paused;
+    bool vsync_timed;               // the VO redraws itself as fast as possible
+                                    // at every vsync
     int queued_events;
 
     int64_t flip_queue_offset; // queue flip events at most this much in advance
@@ -144,6 +146,7 @@ struct vo_internal {
 
     // --- The following fields can be accessed from the VO thread only
     int64_t vsync_interval;
+    int64_t vsync_interval_approx;
     int64_t last_flip;
     char *window_title;
 };
@@ -486,7 +489,7 @@ bool vo_is_ready_for_frame(struct vo *vo, int64_t next_pts)
         if (next_pts > now)
             r = false;
         if (!in->wakeup_pts || next_pts < in->wakeup_pts) {
-            in->wakeup_pts = next_pts;
+            in->wakeup_pts = in->vsync_timed ? 0 : next_pts;
             wakeup_locked(vo);
         }
     }
@@ -507,7 +510,7 @@ void vo_queue_frame(struct vo *vo, struct mp_image *image,
     in->frame_queued = image;
     in->frame_pts = pts_us;
     in->frame_duration = duration;
-    in->wakeup_pts = in->frame_pts + MPMAX(duration, 0);
+    in->wakeup_pts = in->vsync_timed ? 0 : in->frame_pts + MPMAX(duration, 0);
     wakeup_locked(vo);
     pthread_mutex_unlock(&in->lock);
 }
@@ -545,7 +548,13 @@ static bool render_frame(struct vo *vo)
     int64_t pts = in->frame_pts;
     int64_t duration = in->frame_duration;
     struct mp_image *img = in->frame_queued;
-    if (!img) {
+
+    if (!img && (!in->vsync_timed || in->paused || pts <= 0)) {
+        pthread_mutex_unlock(&in->lock);
+        return false;
+    }
+
+    if (in->vsync_timed && !in->hasframe) {
         pthread_mutex_unlock(&in->lock);
         return false;
     }
@@ -556,7 +565,8 @@ static bool render_frame(struct vo *vo)
     in->frame_queued = NULL;
 
     // The next time a flip (probably) happens.
-    int64_t next_vsync = prev_sync(vo, mp_time_us()) + in->vsync_interval;
+    int64_t prev_vsync = prev_sync(vo, mp_time_us());
+    int64_t next_vsync = prev_vsync + in->vsync_interval;
     int64_t end_time = pts + duration;
 
     if (!in->hasframe_rendered)
@@ -568,6 +578,18 @@ static bool render_frame(struct vo *vo)
     // Even if we're hopelessly behind, rather degrade to 10 FPS playback,
     // instead of just freezing the display forever.
     in->dropped_frame &= mp_time_us() - in->last_flip < 100 * 1000;
+    in->dropped_frame &= in->vsync_timed && !!img;
+
+    if (in->vsync_timed && !img && in->hasframe_rendered &&
+        prev_vsync > pts + in->vsync_interval_approx) {
+        // we are very late with the frame and using vsync timing: probably
+        // no new frames are coming in. This must be done whether or not
+        // framedrop is enabled.
+        in->dropped_frame = false;
+        in->rendering = false;
+        pthread_mutex_unlock(&in->lock);
+        return false;
+    }
 
     if (in->dropped_frame) {
         in->dropped_image = img;
@@ -578,9 +600,21 @@ static bool render_frame(struct vo *vo)
 
         MP_STATS(vo, "start video");
 
-        vo->driver->draw_image(vo, img);
+        if (in->vsync_timed) {
+            struct frame_timing t = (struct frame_timing) {
+                .pts        = pts,
+                .next_vsync = next_vsync,
+            };
+            vo->driver->draw_image_timed(vo, img, &t);
+        } else {
+            vo->driver->draw_image(vo, img);
+        }
 
-        int64_t target = pts - in->flip_queue_offset;
+        int64_t target = !in->vsync_timed ?
+                         pts - in->flip_queue_offset :
+                         // this is a heuristic that wakes the thread up some
+                         // time before the next vsync
+                         next_vsync - MPMIN(in->vsync_interval / 3, 4e3);
         while (1) {
             int64_t now = mp_time_us();
             if (target <= now)
@@ -594,6 +628,8 @@ static bool render_frame(struct vo *vo)
         else
             vo->driver->flip_page(vo);
 
+        int64_t prev_flip = in->last_flip;
+
         in->last_flip = -1;
 
         vo->driver->control(vo, VOCTRL_GET_RECENT_FLIP_TIME, &in->last_flip);
@@ -601,10 +637,13 @@ static bool render_frame(struct vo *vo)
         if (in->last_flip < 0)
             in->last_flip = mp_time_us();
 
+        in->vsync_interval_approx = in->last_flip - prev_flip;
+
         long phase = in->last_flip % in->vsync_interval;
         MP_DBG(vo, "phase: %ld\n", phase);
         MP_STATS(vo, "value %ld phase", phase);
 
+        MP_STATS(vo, "display");
         MP_STATS(vo, "end video");
 
         pthread_mutex_lock(&in->lock);
@@ -669,6 +708,7 @@ static void *vo_thread(void *ptr)
     mpthread_set_name("vo");
 
     int r = vo->driver->preinit(vo) ? -1 : 0;
+    vo->driver->control(vo, VOCTRL_GET_VSYNC_TIMED, &in->vsync_timed);
     mp_rendezvous(vo, r); // init barrier
     if (r < 0)
         return NULL;