demux, player: fix playback of sparse video streams (w/ still images)

Fixes several issues playing back mpegts with video streams marked
as having "still images". For example, see this video which has
frames only every 6s: https://s3.amazonaws.com/tmm1/music-choice.ts

Changes include:
- start playback right away, without waiting for first video frame
- do not consider the sparse video stream in demuxer underrun detection
- do not require multiple video frames for the VO
- use audio as the master stream for demuxer metadata events
- use audio stream for playback time

Signed-off-by: Aman Gupta <aman@tmm1.net>

3 files changed, 26 insertions, 3 deletions

diff --git a/player/core.h b/player/core.h
index bc6cf28ff0..71c39dcaa5 100644
--- a/player/core.h
+++ b/player/core.h
@@ -180,6 +180,8 @@ struct vo_chain {
     // - video consists of a single picture, which should be shown only once
     // - do not sync audio to video in any way
     bool is_coverart;
+    // - video consists of sparse still images
+    bool is_sparse;
 };
 
 // Like vo_chain, for audio.
diff --git a/player/playloop.c b/player/playloop.c
index 8845b58cef..f5c1fde0ef 100644
--- a/player/playloop.c
+++ b/player/playloop.c
@@ -950,7 +950,9 @@ static void handle_dummy_ticks(struct MPContext *mpctx)
 // Update current playback time.
 static void handle_playback_time(struct MPContext *mpctx)
 {
-    if (mpctx->vo_chain && !mpctx->vo_chain->is_coverart &&
+    if (mpctx->vo_chain &&
+        !mpctx->vo_chain->is_coverart &&
+        !mpctx->vo_chain->is_sparse &&
         mpctx->video_status >= STATUS_PLAYING &&
         mpctx->video_status < STATUS_EOF)
     {
@@ -986,6 +988,13 @@ static void handle_playback_restart(struct MPContext *mpctx)
 {
     struct MPOpts *opts = mpctx->opts;
 
+    // Do not wait for video stream if it only has sparse frames.
+    if (mpctx->vo_chain &&
+        mpctx->vo_chain->is_sparse &&
+        mpctx->video_status < STATUS_READY) {
+        mpctx->video_status = STATUS_READY;
+    }
+
     if (mpctx->audio_status < STATUS_READY ||
         mpctx->video_status < STATUS_READY)
         return;
@@ -1008,7 +1017,9 @@ static void handle_playback_restart(struct MPContext *mpctx)
         }
 
         // Video needed, but not started yet -> wait.
-        if (mpctx->vo_chain && !mpctx->vo_chain->is_coverart &&
+        if (mpctx->vo_chain &&
+            !mpctx->vo_chain->is_coverart &&
+            !mpctx->vo_chain->is_sparse &&
             mpctx->video_status <= STATUS_READY)
             return;
 
diff --git a/player/video.c b/player/video.c
index 17dff84984..fde92851a1 100644
--- a/player/video.c
+++ b/player/video.c
@@ -256,6 +256,7 @@ void reinit_video_chain_src(struct MPContext *mpctx, struct track *track)
         vo_c->dec_src = track->dec->f->pins[0];
         vo_c->filter->container_fps = track->dec->fps;
         vo_c->is_coverart = !!track->stream->attached_picture;
+        vo_c->is_sparse = track->stream->still_image;
 
         track->vo_c = vo_c;
         vo_c->track = track;
@@ -365,9 +366,12 @@ static void handle_new_frame(struct MPContext *mpctx)
 
     double frame_time = 0;
     double pts = mpctx->next_frames[0]->pts;
+    bool is_sparse = mpctx->vo_chain && mpctx->vo_chain->is_sparse;
+
     if (mpctx->video_pts != MP_NOPTS_VALUE) {
         frame_time = pts - mpctx->video_pts;
-        double tolerance = mpctx->demuxer->ts_resets_possible ? 5 : 1e4;
+        double tolerance = mpctx->demuxer->ts_resets_possible &&
+                           !is_sparse ? 5 : 1e4;
         if (frame_time <= 0 || frame_time >= tolerance) {
             // Assume a discontinuity.
             MP_WARN(mpctx, "Invalid video timestamp: %f -> %f\n",
@@ -403,6 +407,9 @@ static int get_req_frames(struct MPContext *mpctx, bool eof)
     if (mpctx->video_out->driver->caps & VO_CAP_NORETAIN)
         return 1;
 
+    if (mpctx->vo_chain && mpctx->vo_chain->is_sparse)
+        return 1;
+
     if (mpctx->opts->untimed || mpctx->video_out->driver->untimed)
         return 1;
 
@@ -594,6 +601,9 @@ static void update_av_diff(struct MPContext *mpctx, double offset)
         mpctx->video_status != STATUS_PLAYING)
         return;
 
+    if (mpctx->vo_chain && mpctx->vo_chain->is_sparse)
+        return;
+
     double a_pos = playing_audio_pts(mpctx);
     if (a_pos != MP_NOPTS_VALUE && mpctx->video_pts != MP_NOPTS_VALUE) {
         mpctx->last_av_difference = a_pos - mpctx->video_pts