video: move display and timing to a separate thread

The VO is run inside its own thread. It also does most of video timing.
The playloop hands the image data and a realtime timestamp to the VO,
and the VO does the rest.

In particular, this allows the playloop to do other things, instead of
blocking for video redraw. But if anything accesses the VO during video
timing, it will block.

This also fixes vo_sdl.c event handling; but that is only a side-effect,
since reimplementing the broken way would require more effort.

Also drop --softsleep. In theory, this option helps if the kernel's
sleeping mechanism is too inaccurate for video timing. In practice, I
haven't ever encountered a situation where it helps, and it just burns
CPU cycles. On the other hand it's probably actively harmful, because
it prevents the libavcodec decoder threads from doing real work.

Side note:

Originally, I intended that multiple frames can be queued to the VO. But
this is not done, due to problems with OSD and other certain features.
OSD in particular is simply designed in a way that it can be neither
timed nor copied, so you do have to render it into the video frame
before you can draw the next frame. (Subtitles have no such restriction.
sd_lavc was even updated to fix this.) It seems the right solution to
queuing multiple VO frames is rendering on VO-backed framebuffers, like
vo_vdpau.c does. This requires VO driver support, and is out of scope
of this commit.

As consequence, the VO has a queue size of 1. The existing video queue
is just needed to compute frame duration, and will be moved out in the
next commit.

5 files changed, 32 insertions, 120 deletions

diff --git a/player/core.h b/player/core.h
index 61d78215b6..1a396cf25f 100644
--- a/player/core.h
+++ b/player/core.h
@@ -257,10 +257,6 @@ typedef struct MPContext {
     double delay;
     // AV sync: time until next frame should be shown
     double time_frame;
-    // How long the last vo flip() call took. Used to adjust timing with
-    // the goal of making flip() calls finish (rather than start) at the
-    // specified time.
-    double last_vo_flip_duration;
     // Display duration (as "intended") of the last flipped frame.
     double last_frame_duration;
     // Set to true some time after a new frame has been shown, and it turns out
diff --git a/player/misc.c b/player/misc.c
index 3ae4a8ab9a..a5f21ba3a5 100644
--- a/player/misc.c
+++ b/player/misc.c
@@ -148,10 +148,8 @@ void update_window_title(struct MPContext *mpctx, bool force)
         talloc_free(mpctx->last_window_title);
         mpctx->last_window_title = talloc_steal(mpctx, title);
 
-        if (mpctx->video_out) {
-            mpctx->video_out->window_title = talloc_strdup(mpctx->video_out, title);
+        if (mpctx->video_out)
             vo_control(mpctx->video_out, VOCTRL_UPDATE_WINDOW_TITLE, title);
-        }
 
         if (mpctx->ao) {
             ao_control(mpctx->ao, AOCONTROL_UPDATE_STREAM_TITLE, title);
diff --git a/player/playloop.c b/player/playloop.c
index 361b9284af..b96ba2caa2 100644
--- a/player/playloop.c
+++ b/player/playloop.c
@@ -516,8 +516,7 @@ static void handle_osd_redraw(struct MPContext *mpctx)
     // Don't redraw immediately during a seek (makes it significantly slower).
     if (mp_time_sec() - mpctx->start_timestamp < 0.1)
         return;
-    bool want_redraw = vo_get_want_redraw(mpctx->video_out) |
-                       osd_query_and_reset_want_redraw(mpctx->osd);
+    bool want_redraw = osd_query_and_reset_want_redraw(mpctx->osd);
     if (!want_redraw)
         return;
     vo_redraw(mpctx->video_out);
@@ -806,22 +805,6 @@ static void handle_dummy_ticks(struct MPContext *mpctx)
     }
 }
 
-static double get_wakeup_period(struct MPContext *mpctx)
-{
-    double sleeptime = 100.0; // infinite for all practical purposes
-
-#if !HAVE_POSIX_SELECT
-    // No proper file descriptor event handling; keep waking up to poll input
-    sleeptime = MPMIN(sleeptime, 0.02);
-#endif
-
-    if (mpctx->video_out)
-        if (mpctx->video_out->wakeup_period > 0)
-            sleeptime = MPMIN(sleeptime, mpctx->video_out->wakeup_period);
-
-    return sleeptime;
-}
-
 void run_playloop(struct MPContext *mpctx)
 {
     struct MPOpts *opts = mpctx->opts;
@@ -851,9 +834,6 @@ void run_playloop(struct MPContext *mpctx)
             endpts = end;
     }
 
-    if (mpctx->video_out)
-        vo_check_events(mpctx->video_out);
-
     handle_cursor_autohide(mpctx);
     handle_heartbeat_cmd(mpctx);
 
@@ -943,7 +923,7 @@ void run_playloop(struct MPContext *mpctx)
         mp_input_get_cmd(mpctx->input, mpctx->sleeptime * 1000, true);
         MP_STATS(mpctx, "end sleep");
     }
-    mpctx->sleeptime = get_wakeup_period(mpctx);
+    mpctx->sleeptime = 100.0; // infinite for all practical purposes
 
     handle_pause_on_low_cache(mpctx);
 
@@ -982,13 +962,11 @@ void idle_loop(struct MPContext *mpctx)
             uninit |= INITIALIZED_VO;
         uninit_player(mpctx, uninit);
         handle_force_window(mpctx, false);
-        if (mpctx->video_out)
-            vo_check_events(mpctx->video_out);
         update_osd_msg(mpctx);
         handle_osd_redraw(mpctx);
         mp_cmd_t *cmd = mp_input_get_cmd(mpctx->input, mpctx->sleeptime * 1000,
                                          false);
-        mpctx->sleeptime = get_wakeup_period(mpctx);
+        mpctx->sleeptime = 100.0;
         if (cmd)
             run_command(mpctx, cmd);
         mp_cmd_free(cmd);
diff --git a/player/screenshot.c b/player/screenshot.c
index 1ee58c2196..63bf29fbd7 100644
--- a/player/screenshot.c
+++ b/player/screenshot.c
@@ -336,8 +336,10 @@ static struct mp_image *screenshot_get(struct MPContext *mpctx, int mode)
         if (mpctx->d_video && mpctx->d_video->vfilter)
             vf_control_any(mpctx->d_video->vfilter, VFCTRL_SCREENSHOT, &args);
 
-        if (!args.out_image)
+        if (!args.out_image) {
+            vo_wait_frame(mpctx->video_out); // important for each-frame mode
             vo_control(mpctx->video_out, VOCTRL_SCREENSHOT, &args);
+        }
 
         image = args.out_image;
         if (image) {
diff --git a/player/video.c b/player/video.c
index 5b2748c894..1ee8ec1203 100644
--- a/player/video.c
+++ b/player/video.c
@@ -613,25 +613,6 @@ static int update_video(struct MPContext *mpctx, double endpts, bool reconfig_ok
     return VD_NEW_FRAME;
 }
 
-static double timing_sleep(struct MPContext *mpctx, double time_frame)
-{
-    // assume kernel HZ=100 for softsleep, works with larger HZ but with
-    // unnecessarily high CPU usage
-    struct MPOpts *opts = mpctx->opts;
-    double margin = opts->softsleep ? 0.011 : 0;
-    while (time_frame > margin) {
-        mp_sleep_us(1000000 * (time_frame - margin));
-        time_frame -= get_relative_time(mpctx);
-    }
-    if (opts->softsleep) {
-        if (time_frame < 0)
-            MP_WARN(mpctx, "Warning! Softsleep underflow!\n");
-        while (time_frame > 0)
-            time_frame -= get_relative_time(mpctx);  // burn the CPU
-    }
-    return time_frame;
-}
-
 static void update_avsync(struct MPContext *mpctx)
 {
     if (mpctx->audio_status != STATUS_PLAYING ||
@@ -673,7 +654,6 @@ static void adjust_sync(struct MPContext *mpctx, double frame_time)
     double v_pts = mpctx->video_next_pts;
     double av_delay = a_pts - v_pts;
     // Try to sync vo_flip() so it will *finish* at given time
-    av_delay += mpctx->last_vo_flip_duration;
     av_delay += mpctx->audio_delay;   // This much pts difference is desired
 
     double change = av_delay * 0.1;
@@ -772,7 +752,6 @@ void write_video(struct MPContext *mpctx, double endpts)
         return;
 
     mpctx->time_frame -= get_relative_time(mpctx);
-    double audio_pts = playing_audio_pts(mpctx);
     if (!mpctx->sync_audio_to_video || mpctx->video_status < STATUS_READY) {
         mpctx->time_frame = 0;
     } else if (mpctx->audio_status == STATUS_PLAYING &&
@@ -806,24 +785,32 @@ void write_video(struct MPContext *mpctx, double endpts)
          * If untimed is set always output frames immediately
          * without sleeping.
          */
-        if (mpctx->time_frame < -0.2 || opts->untimed || vo->untimed)
+        if (mpctx->time_frame < -0.2 || opts->untimed || vo->driver->untimed)
             mpctx->time_frame = 0;
     }
 
-    double vsleep = mpctx->time_frame - vo->flip_queue_offset;
-    if (vsleep > 0.050) {
-        mpctx->sleeptime = MPMIN(mpctx->sleeptime, vsleep - 0.040);
-        return;
-    }
-    mpctx->sleeptime = 0;
-    mpctx->playing_last_frame = false;
-
     // last frame case
+    // TODO: should be _after_ wait
+    mpctx->playing_last_frame = false;
     if (r != VD_NEW_FRAME)
         return;
 
-    //=================== FLIP PAGE (VIDEO BLT): ======================
+    double time_frame = MPMAX(mpctx->time_frame, -1);
+    int64_t pts = mp_time_us() + (int64_t)(time_frame * 1e6);
 
+    if (!vo_is_ready_for_frame(vo, pts))
+        return; // wait until VO wakes us up to get more frames
+
+    int64_t duration = -1;
+    double vpts0 = vo_get_next_pts(vo, 0);
+    double vpts1 = vo_get_next_pts(vo, 1);
+    if (vpts0 != MP_NOPTS_VALUE && vpts1 != MP_NOPTS_VALUE) {
+        // expected A/V sync correction is ignored
+        double diff = (vpts1 - vpts0) / opts->playback_speed;
+        if (mpctx->time_frame < 0)
+            diff += mpctx->time_frame;
+        duration = MPCLAMP(diff, 0, 10) * 1e6;
+    }
 
     mpctx->video_pts = mpctx->video_next_pts;
     mpctx->last_vo_pts = mpctx->video_pts;
@@ -832,66 +819,17 @@ void write_video(struct MPContext *mpctx, double endpts)
     update_subtitles(mpctx);
     update_osd_msg(mpctx);
 
-    MP_STATS(mpctx, "vo draw frame");
-
-    vo_new_frame_imminent(vo);
-
-    MP_STATS(mpctx, "vo sleep");
+    vo_queue_frame(vo, pts, duration);
 
+    // For print_status - VO call finishing early is OK for sync
     mpctx->time_frame -= get_relative_time(mpctx);
-    mpctx->time_frame -= vo->flip_queue_offset;
-    if (mpctx->time_frame > 0.001)
-        mpctx->time_frame = timing_sleep(mpctx, mpctx->time_frame);
-    mpctx->time_frame += vo->flip_queue_offset;
-
-    int64_t t2 = mp_time_us();
-    /* Playing with playback speed it's possible to get pathological
-     * cases with mpctx->time_frame negative enough to cause an
-     * overflow in pts_us calculation, thus the MPMAX. */
-    double time_frame = MPMAX(mpctx->time_frame, -1);
-    int64_t pts_us = mpctx->last_time + time_frame * 1e6;
-    int duration = -1;
-    double pts2 = vo_get_next_pts(vo, 0); // this is the next frame PTS
-    if (mpctx->video_pts != MP_NOPTS_VALUE && pts2 == MP_NOPTS_VALUE) {
-        // Make up a frame duration. Using the frame rate is not a good
-        // choice, since the frame rate could be unset/broken/random.
-        float fps = mpctx->d_video->fps;
-        double frame_duration = fps > 0 ? 1.0 / fps : 0;
-        pts2 = mpctx->video_pts + MPCLAMP(frame_duration, 0.0, 5.0);
-    }
-    if (pts2 != MP_NOPTS_VALUE) {
-        // expected A/V sync correction is ignored
-        double diff = (pts2 - mpctx->video_pts);
-        diff /= opts->playback_speed;
-        if (mpctx->time_frame < 0)
-            diff += mpctx->time_frame;
-        if (diff < 0)
-            diff = 0;
-        if (diff > 10)
-            diff = 10;
-        duration = diff * 1e6;
-        mpctx->last_frame_duration = diff;
-    }
-    if (mpctx->video_status != STATUS_PLAYING)
-        duration = -1;
-
-    MP_STATS(mpctx, "start flip");
-    vo_flip_page(vo, pts_us | 1, duration);
-    MP_STATS(mpctx, "end flip");
-
-    if (audio_pts != MP_NOPTS_VALUE)
-        MP_STATS(mpctx, "value %f ptsdiff", mpctx->video_pts - audio_pts);
-
-    mpctx->last_vo_flip_duration = (mp_time_us() - t2) * 0.000001;
-    if (vo->driver->flip_page_timed) {
-        // No need to adjust sync based on flip speed
-        mpctx->last_vo_flip_duration = 0;
-        // For print_status - VO call finishing early is OK for sync
-        mpctx->time_frame -= get_relative_time(mpctx);
-    }
+
     mpctx->shown_vframes++;
-    if (mpctx->video_status < STATUS_PLAYING)
+    if (mpctx->video_status < STATUS_PLAYING) {
         mpctx->video_status = STATUS_READY;
+        // After a seek, make sure to wait until the first frame is visible.
+        vo_wait_frame(vo);
+    }
     update_avsync(mpctx);
     screenshot_flip(mpctx);