player/sub: avoid wasteful subtitle redraws

This only affects two special cases: printing subtitles to the terminal
and printing subtitles on a still picture. Previously, mpv was very dumb
here and spammed this logic on every single loop. For terminal
subtitles, this isn't as big of a deal, but for the image case this is
pretty bad. The entire VO constantly redrew even when there was no need
to which can be very expensive depending on user settings.

Instead, let's rework sub_read_packets so that it also tells us whether
or not the subtitle packets update in some way in addition to telling us
whether or not to read more. Since we cache all packets thanks to the
previous commit, we can leverage this information to make a guess
whether or not the current subtitle packet is supposed to be visible on
the screen. Because the redraw now only happens when it is needed, the
mp_set_timeout_hack can be removed.

8 files changed, 115 insertions, 32 deletions

diff --git a/demux/packet.h b/demux/packet.h
index cd1183d417..8e8d20c402 100644
--- a/demux/packet.h
+++ b/demux/packet.h
@@ -58,6 +58,9 @@ typedef struct demux_packet {
     struct mp_codec_params *codec;  // set to non-NULL iff segmented is set
     double start, end;              // set to non-NOPTS iff segmented is set
 
+    // subtitles only
+    double sub_duration;
+
     // private
     struct demux_packet *next;
     struct AVPacket *avpacket;   // keep the buffer allocation and sidedata
diff --git a/player/core.h b/player/core.h
index 8a49585cdf..9d9c4e8307 100644
--- a/player/core.h
+++ b/player/core.h
@@ -402,6 +402,9 @@ typedef struct MPContext {
     int last_chapter_seek;
     bool last_chapter_flag;
 
+    /* Heuristic for potentially redrawing subs. */
+    bool redraw_subs;
+
     bool paused;            // internal pause state
     bool playback_active;   // not paused, restarting, loading, unloading
     bool in_playloop;
@@ -621,6 +624,7 @@ void mp_load_builtin_scripts(struct MPContext *mpctx);
 int64_t mp_load_user_script(struct MPContext *mpctx, const char *fname);
 
 // sub.c
+void redraw_subs(struct MPContext *mpctx);
 void reset_subtitle_state(struct MPContext *mpctx);
 void reinit_sub(struct MPContext *mpctx, struct track *track);
 void reinit_sub_all(struct MPContext *mpctx);
diff --git a/player/playloop.c b/player/playloop.c
index 3a10f6faee..eccc432ffe 100644
--- a/player/playloop.c
+++ b/player/playloop.c
@@ -419,6 +419,7 @@ static void mp_seek(MPContext *mpctx, struct seek_params seek)
     update_ab_loop_clip(mpctx);
 
     mpctx->current_seek = seek;
+    redraw_subs(mpctx);
 }
 
 // This combines consecutive seek requests.
@@ -665,6 +666,9 @@ static void handle_osd_redraw(struct MPContext *mpctx)
     if (!want_redraw)
         return;
     vo_redraw(mpctx->video_out);
+    // Even though we just redrew, it may need to be done again for certain
+    // cases of subtitles on an image.
+    redraw_subs(mpctx);
 }
 
 static void clear_underruns(struct MPContext *mpctx)
diff --git a/player/sub.c b/player/sub.c
index 5d7baef3e6..998b88d620 100644
--- a/player/sub.c
+++ b/player/sub.c
@@ -54,6 +54,19 @@ static void reset_subtitles(struct MPContext *mpctx, struct track *track)
     term_osd_set_subs(mpctx, NULL);
 }
 
+// Only matters for subs on an image.
+void redraw_subs(struct MPContext *mpctx)
+{
+    for (int n = 0; n < num_ptracks[STREAM_SUB]; n++) {
+        if (mpctx->current_track[n][STREAM_SUB] &&
+            mpctx->current_track[n][STREAM_SUB]->d_sub)
+        {
+            mpctx->redraw_subs = true;
+            break;
+        }
+    }
+}
+
 void reset_subtitle_state(struct MPContext *mpctx)
 {
     for (int n = 0; n < mpctx->num_tracks; n++)
@@ -100,33 +113,43 @@ static bool update_subtitle(struct MPContext *mpctx, double video_pts,
         sub_preload(dec_sub);
     }
 
-    if (!sub_read_packets(dec_sub, video_pts, mpctx->paused))
-        return false;
+    bool packets_read = false;
+    bool sub_updated = false;
+    sub_read_packets(dec_sub, video_pts, mpctx->paused, &packets_read, &sub_updated);
 
-    // Handle displaying subtitles on terminal; never done for secondary subs
-    if (mpctx->current_track[0][STREAM_SUB] == track && !mpctx->video_out) {
-        char *text = sub_get_text(dec_sub, video_pts, SD_TEXT_TYPE_PLAIN);
-        term_osd_set_subs(mpctx, text);
-        talloc_free(text);
-    }
+    double osd_pts = osd_get_force_video_pts(mpctx->osd);
+
+    // Check if we need to update subtitles for these special cases. Always
+    // update on discontinuities like seeking or a new file.
+    if (sub_updated || mpctx->redraw_subs || osd_pts == MP_NOPTS_VALUE) {
+        // Always force a redecode of all packets if we have a refresh.
+        if (mpctx->redraw_subs)
+            sub_redecode_cached_packets(dec_sub);
 
-    // Handle displaying subtitles on VO with no video being played. This is
-    // quite different, because normally subtitles are redrawn on new video
-    // frames, using the video frames' timestamps.
-    if (mpctx->video_out && mpctx->video_status == STATUS_EOF &&
-        (mpctx->opts->subs_rend->sub_past_video_end ||
-         !mpctx->current_track[0][STREAM_VIDEO] ||
-         mpctx->current_track[0][STREAM_VIDEO]->image)) {
-        if (osd_get_force_video_pts(mpctx->osd) != video_pts) {
-            osd_set_force_video_pts(mpctx->osd, video_pts);
-            osd_query_and_reset_want_redraw(mpctx->osd);
-            vo_redraw(mpctx->video_out);
-            // Force an arbitrary minimum FPS
-            mp_set_timeout(mpctx, 0.1);
+        // Handle displaying subtitles on terminal; never done for secondary subs
+        if (mpctx->current_track[0][STREAM_SUB] == track && !mpctx->video_out) {
+            char *text = sub_get_text(dec_sub, video_pts, SD_TEXT_TYPE_PLAIN);
+            term_osd_set_subs(mpctx, text);
+            talloc_free(text);
+        }
+
+        // Handle displaying subtitles on VO with no video being played. This is
+        // quite different, because normally subtitles are redrawn on new video
+        // frames, using the video frames' timestamps.
+        if (mpctx->video_out && mpctx->video_status == STATUS_EOF &&
+            (mpctx->opts->subs_rend->sub_past_video_end ||
+            !mpctx->current_track[0][STREAM_VIDEO] ||
+            mpctx->current_track[0][STREAM_VIDEO]->image)) {
+            if (osd_pts != video_pts) {
+                osd_set_force_video_pts(mpctx->osd, video_pts);
+                osd_query_and_reset_want_redraw(mpctx->osd);
+                vo_redraw(mpctx->video_out);
+            }
         }
     }
 
-    return true;
+    mpctx->redraw_subs = false;
+    return packets_read;
 }
 
 // Return true if the subtitles for the given PTS are ready; false if the player
diff --git a/sub/dec_sub.c b/sub/dec_sub.c
index 5221a65fde..c224a25fd6 100644
--- a/sub/dec_sub.c
+++ b/sub/dec_sub.c
@@ -20,6 +20,7 @@
 #include <string.h>
 #include <math.h>
 #include <assert.h>
+#include <limits.h>
 
 #include "demux/demux.h"
 #include "sd.h"
@@ -62,6 +63,7 @@ struct dec_sub {
     bool preload_attempted;
     double video_fps;
     double sub_speed;
+    bool sub_visible;
 
     struct mp_codec_params *codec;
     double start, end;
@@ -71,6 +73,7 @@ struct dec_sub {
 
     struct demux_packet *new_segment;
     struct demux_packet **cached_pkts;
+    int cached_pkt_pos;
     int num_cached_pkts;
 };
 
@@ -127,6 +130,7 @@ static void sub_destroy_cached_pkts(struct dec_sub *sub)
         TA_FREEP(&sub->cached_pkts[index]);
         ++index;
     }
+    sub->cached_pkt_pos = 0;
     sub->num_cached_pkts = 0;
 }
 
@@ -284,12 +288,47 @@ static bool is_new_segment(struct dec_sub *sub, struct demux_packet *p)
         (p->start != sub->start || p->end != sub->end || p->codec != sub->codec);
 }
 
-// Read packets from the demuxer stream passed to sub_create(). Return true if
-// enough packets were read, false if the player should wait until the demuxer
-// signals new packets available (and then should retry).
-bool sub_read_packets(struct dec_sub *sub, double video_pts, bool force)
+static bool is_packet_visible(struct demux_packet *p, double video_pts)
 {
-    bool r = true;
+    return p && p->pts <= video_pts && (video_pts <= p->pts + p->sub_duration ||
+           p->sub_duration < 0);
+}
+
+static bool update_pkt_cache(struct dec_sub *sub, double video_pts)
+{
+    if (!sub->cached_pkts[sub->cached_pkt_pos])
+        return false;
+
+    struct demux_packet *pkt = sub->cached_pkts[sub->cached_pkt_pos];
+    struct demux_packet *next_pkt = sub->cached_pkt_pos + 1 < sub->num_cached_pkts ?
+                                    sub->cached_pkts[sub->cached_pkt_pos + 1] : NULL;
+    if (!pkt)
+        return false;
+
+    double pts = video_pts + sub->shared_opts->sub_delay[sub->order];
+    double next_pts = next_pkt ? next_pkt->pts : INT_MAX;
+    double end_pts = pkt->sub_duration >= 0 ? pkt->pts + pkt->sub_duration : INT_MAX;
+
+    if (next_pts < pts || end_pts < pts) {
+        if (sub->cached_pkt_pos + 1 < sub->num_cached_pkts) {
+            TA_FREEP(&sub->cached_pkts[sub->cached_pkt_pos]);
+            sub->cached_pkt_pos++;
+        }
+        if (next_pts < pts)
+            return true;
+    }
+
+    return false;
+}
+
+// Read packets from the demuxer stream passed to sub_create(). Signals if
+// enough packets were read and if the subtitle state updated in anyway. If
+// packets_read is false, the player should wait until the demuxer signals new
+// packets and retry.
+void sub_read_packets(struct dec_sub *sub, double video_pts, bool force,
+                      bool *packets_read, bool *sub_updated)
+{
+    *packets_read = true;
     mp_mutex_lock(&sub->lock);
     video_pts = pts_to_subtitle(sub, video_pts);
     while (1) {
@@ -320,8 +359,8 @@ bool sub_read_packets(struct dec_sub *sub, double video_pts, bool force)
         // happen for interleaved subtitle streams, which never return "wait"
         // when reading, unless min_pts is set.
         if (st <= 0) {
-            r = st < 0 || (sub->last_pkt_pts != MP_NOPTS_VALUE &&
-                           sub->last_pkt_pts > video_pts);
+            *packets_read = st < 0 || (sub->last_pkt_pts != MP_NOPTS_VALUE &&
+                                       sub->last_pkt_pts > video_pts);
             break;
         }
 
@@ -341,8 +380,12 @@ bool sub_read_packets(struct dec_sub *sub, double video_pts, bool force)
         if (!(sub->preload_attempted && sub->sd->preload_ok))
             sub->sd->driver->decode(sub->sd, pkt);
     }
+    if (sub->cached_pkts && sub->num_cached_pkts) {
+        bool visible = is_packet_visible(sub->cached_pkts[sub->cached_pkt_pos], video_pts);
+        *sub_updated = update_pkt_cache(sub, video_pts) || sub->sub_visible != visible;
+        sub->sub_visible = visible;
+    }
     mp_mutex_unlock(&sub->lock);
-    return r;
 }
 
 // Redecode all cached packets if needed.
@@ -350,7 +393,7 @@ bool sub_read_packets(struct dec_sub *sub, double video_pts, bool force)
 void sub_redecode_cached_packets(struct dec_sub *sub)
 {
     mp_mutex_lock(&sub->lock);
-    int index = 0;
+    int index = sub->cached_pkt_pos;
     while (index < sub->num_cached_pkts) {
         sub->sd->driver->decode(sub->sd, sub->cached_pkts[index]);
         ++index;
diff --git a/sub/dec_sub.h b/sub/dec_sub.h
index 8c8d7d4de5..eb8406cb14 100644
--- a/sub/dec_sub.h
+++ b/sub/dec_sub.h
@@ -43,7 +43,8 @@ void sub_destroy(struct dec_sub *sub);
 bool sub_can_preload(struct dec_sub *sub);
 void sub_preload(struct dec_sub *sub);
 void sub_redecode_cached_packets(struct dec_sub *sub);
-bool sub_read_packets(struct dec_sub *sub, double video_pts, bool force);
+void sub_read_packets(struct dec_sub *sub, double video_pts, bool force,
+                      bool *packets_read, bool *sub_updated);
 struct sub_bitmaps *sub_get_bitmaps(struct dec_sub *sub, struct mp_osd_res dim,
                                     int format, double pts);
 char *sub_get_text(struct dec_sub *sub, double pts, enum sd_text_type type);
diff --git a/sub/sd_ass.c b/sub/sd_ass.c
index 0f817c22f4..67dfdc7e06 100644
--- a/sub/sd_ass.c
+++ b/sub/sd_ass.c
@@ -332,6 +332,9 @@ static bool check_packet_seen(struct sd *sd, int64_t pos)
 static void decode(struct sd *sd, struct demux_packet *packet)
 {
     struct sd_ass_priv *ctx = sd->priv;
+
+    packet->sub_duration = packet->duration;
+
     ASS_Track *track = ctx->ass_track;
     if (ctx->converter) {
         if (!sd->opts->sub_clear_on_seek && packet->pos >= 0 &&
diff --git a/sub/sd_lavc.c b/sub/sd_lavc.c
index 1bb1c96c2b..cf49f2d8f1 100644
--- a/sub/sd_lavc.c
+++ b/sub/sd_lavc.c
@@ -327,6 +327,8 @@ static void decode(struct sd *sd, struct demux_packet *packet)
     if (res < 0 || !got_sub)
         return;
 
+    packet->sub_duration = sub.end_display_time;
+
     if (sub.pts != AV_NOPTS_VALUE)
         pts = sub.pts / (double)AV_TIME_BASE;