audio: change playback restart and resyncing

This commit makes audio decoding non-blocking. If e.g. the network is
too slow the playloop will just go to sleep, instead of blocking until
enough data is available.

For video, this was already done with commit 7083f88c. For audio, it's
unfortunately much more complicated, because the audio decoder was used
in a blocking manner. Large changes are required to get around this.
The whole playback restart mechanism must be turned into a statemachine,
especially since it has close interactions with video restart. Lots of
video code is thus also changed.

(For the record, I don't think switching this code to threads would
make this conceptually easier: the code would still have to deal with
external input while blocked, so these in-between states do get visible
[and thus need to be handled] anyway. On the other hand, it certainly
should be possible to modularize this code a bit better.)

This will probably cause a bunch of regressions.

1 files changed, 162 insertions, 136 deletions

diff --git a/player/audio.c b/player/audio.c
index 4867cf5fcc..3a7d26daa7 100644
--- a/player/audio.c
+++ b/player/audio.c
@@ -19,6 +19,7 @@
 #include <stddef.h>
 #include <stdbool.h>
 #include <inttypes.h>
+#include <limits.h>
 #include <math.h>
 #include <assert.h>
 
@@ -103,6 +104,8 @@ void reinit_audio_chain(struct MPContext *mpctx)
 
     mp_notify(mpctx, MPV_EVENT_AUDIO_RECONFIG, NULL);
 
+    mpctx->audio_status = STATUS_SYNCING;
+
     if (!(mpctx->initialized_flags & INITIALIZED_ACODEC)) {
         mpctx->initialized_flags |= INITIALIZED_ACODEC;
         assert(!mpctx->d_audio);
@@ -117,9 +120,19 @@ void reinit_audio_chain(struct MPContext *mpctx)
     }
     assert(mpctx->d_audio);
 
+    if (!mpctx->ao_buffer)
+        mpctx->ao_buffer = mp_audio_buffer_create(mpctx);
+
     struct mp_audio in_format;
     mp_audio_buffer_get_format(mpctx->d_audio->decode_buffer, &in_format);
 
+    if (!mp_audio_config_valid(&in_format)) {
+        // We don't know the audio format yet - so configure it later as we're
+        // resyncing. fill_audio_buffers() will call this function again.
+        mpctx->sleeptime = 0;
+        return;
+    }
+
     if (mpctx->ao_decoder_fmt && (mpctx->initialized_flags & INITIALIZED_AO) &&
         !mp_audio_config_equals(mpctx->ao_decoder_fmt, &in_format) &&
         opts->gapless_audio < 0)
@@ -169,7 +182,6 @@ void reinit_audio_chain(struct MPContext *mpctx)
         struct mp_audio fmt;
         ao_get_format(ao, &fmt);
 
-        mpctx->ao_buffer = mp_audio_buffer_create(ao);
         mp_audio_buffer_reinit(mpctx->ao_buffer, &fmt);
 
         mpctx->ao_decoder_fmt = talloc(NULL, struct mp_audio);
@@ -185,7 +197,6 @@ void reinit_audio_chain(struct MPContext *mpctx)
     if (recreate_audio_filters(mpctx) < 0)
         goto init_error;
 
-    mpctx->syncing_audio = true;
     return;
 
 init_error:
@@ -206,6 +217,9 @@ double written_audio_pts(struct MPContext *mpctx)
     struct mp_audio in_format;
     mp_audio_buffer_get_format(d_audio->decode_buffer, &in_format);
 
+    if (!mp_audio_config_valid(&in_format) || !d_audio->afilter)
+        return MP_NOPTS_VALUE;;
+
     // first calculate the end pts of audio that has been output by decoder
     double a_pts = d_audio->pts;
     if (a_pts == MP_NOPTS_VALUE)
@@ -240,7 +254,7 @@ double written_audio_pts(struct MPContext *mpctx)
 double playing_audio_pts(struct MPContext *mpctx)
 {
     double pts = written_audio_pts(mpctx);
-    if (pts == MP_NOPTS_VALUE)
+    if (pts == MP_NOPTS_VALUE || !mpctx->ao)
         return pts;
     return pts - mpctx->opts->playback_speed * ao_get_delay(mpctx->ao);
 }
@@ -273,142 +287,109 @@ static int write_to_ao(struct MPContext *mpctx, struct mp_audio *data, int flags
     return 0;
 }
 
-static int write_silence_to_ao(struct MPContext *mpctx, int samples, int flags,
-                               double pts)
-{
-    struct mp_audio tmp = {0};
-    mp_audio_buffer_get_format(mpctx->ao_buffer, &tmp);
-    tmp.samples = samples;
-    char *p = talloc_size(NULL, tmp.samples * tmp.sstride);
-    for (int n = 0; n < tmp.num_planes; n++)
-        tmp.planes[n] = p;
-    mp_audio_fill_silence(&tmp, 0, tmp.samples);
-    int r = write_to_ao(mpctx, &tmp, 0, pts);
-    talloc_free(p);
-    return r;
-}
-
-static int audio_start_sync(struct MPContext *mpctx, int playsize)
+// Return the number of samples that must be skipped or prepended to reach the
+// target audio pts after a seek (for A/V sync or hr-seek).
+// Return value (*skip):
+//   >0: skip this many samples
+//   =0: don't do anything
+//   <0: prepend this many samples of silence
+// Returns false if PTS is not known yet.
+static bool get_sync_samples(struct MPContext *mpctx, int *skip)
 {
-    struct ao *ao = mpctx->ao;
     struct MPOpts *opts = mpctx->opts;
-    struct dec_audio *d_audio = mpctx->d_audio;
-    int res;
+    *skip = 0;
 
-    assert(d_audio);
+    if (mpctx->audio_status != STATUS_SYNCING)
+        return true;
 
-    struct mp_audio out_format;
-    ao_get_format(ao, &out_format);
+    struct mp_audio out_format = {0};
+    ao_get_format(mpctx->ao, &out_format);
+    double play_samplerate = out_format.rate / opts->playback_speed;
 
-    // Timing info may not be set without
-    res = audio_decode(d_audio, mpctx->ao_buffer, 1);
-    if (res < 0)
-        return res;
-
-    int samples;
-    bool did_retry = false;
-    double written_pts;
-    double real_samplerate = out_format.rate / opts->playback_speed;
-    bool hrseek = mpctx->hrseek_active;   // audio only hrseek
-    mpctx->hrseek_active = false;
-    while (1) {
-        written_pts = written_audio_pts(mpctx);
-        double ptsdiff;
-        if (hrseek)
-            ptsdiff = written_pts - mpctx->hrseek_pts;
-        else
-            ptsdiff = written_pts - mpctx->video_next_pts - mpctx->delay
-                      + mpctx->audio_delay;
-        samples = ptsdiff * real_samplerate;
-
-        // ogg demuxers give packets without timing
-        if (written_pts <= 1 && d_audio->pts == MP_NOPTS_VALUE) {
-            if (!did_retry) {
-                // Try to read more data to see packets that have pts
-                res = audio_decode(d_audio, mpctx->ao_buffer, out_format.rate);
-                if (res < 0)
-                    return res;
-                did_retry = true;
-                continue;
-            }
-            samples = 0;
-        }
+    bool is_pcm = !(out_format.format & AF_FORMAT_SPECIAL_MASK); // no spdif
+    if (!opts->initial_audio_sync || !is_pcm) {
+        mpctx->audio_status = STATUS_FILLING;
+        return true;
+    }
 
-        if (fabs(ptsdiff) > 300 || isnan(ptsdiff))   // pts reset or just broken?
-            samples = 0;
-
-        if (samples > 0)
-            break;
-
-        mpctx->syncing_audio = false;
-        int skip_samples = -samples;
-        int a = MPMIN(skip_samples, MPMAX(playsize, 2500));
-        res = audio_decode(d_audio, mpctx->ao_buffer, a);
-        if (skip_samples <= mp_audio_buffer_samples(mpctx->ao_buffer)) {
-            mp_audio_buffer_skip(mpctx->ao_buffer, skip_samples);
-            if (res < 0)
-                return res;
-            return audio_decode(d_audio, mpctx->ao_buffer, playsize);
+    double written_pts = written_audio_pts(mpctx);
+    if (written_pts == MP_NOPTS_VALUE && !mp_audio_buffer_samples(mpctx->ao_buffer))
+        return false; // no audio read yet
+
+    bool sync_to_video = mpctx->d_video && mpctx->sync_audio_to_video;
+
+    double sync_pts = MP_NOPTS_VALUE;
+    if (sync_to_video) {
+        if (mpctx->video_next_pts != MP_NOPTS_VALUE) {
+            sync_pts = mpctx->video_next_pts;
+        } else if (mpctx->video_status < STATUS_READY) {
+            return false; // wait until we know a video PTS
         }
-        mp_audio_buffer_clear(mpctx->ao_buffer);
-        if (res < 0)
-            return res;
+    } else if (mpctx->hrseek_active) {
+        sync_pts = mpctx->hrseek_pts;
     }
-    if (hrseek)
-        // Don't add silence in audio-only case even if position is too late
-        return 0;
-    if (samples >= playsize) {
-        /* This case could fall back to the one below with
-         * samples = playsize, but then silence would keep accumulating
-         * in ao_buffer if the AO accepts less data than it asks for
-         * in playsize. */
-        write_silence_to_ao(mpctx, playsize, 0,
-                            written_pts - samples / real_samplerate);
-        return AD_ASYNC_PLAY_DONE;
+    if (sync_pts == MP_NOPTS_VALUE) {
+        mpctx->audio_status = STATUS_FILLING;
+        return true; // syncing disabled
     }
-    mpctx->syncing_audio = false;
-    mp_audio_buffer_prepend_silence(mpctx->ao_buffer, samples);
-    return audio_decode(d_audio, mpctx->ao_buffer, playsize);
+
+    if (sync_to_video)
+        sync_pts += mpctx->delay - mpctx->audio_delay;
+
+    double ptsdiff = written_pts - sync_pts;
+    // Missing timestamp, or PTS reset, or just broken.
+    if (written_pts == MP_NOPTS_VALUE || fabs(ptsdiff) > 300) {
+        MP_WARN(mpctx, "Failed audio resync.\n");
+        mpctx->audio_status = STATUS_FILLING;
+        return true;
+    }
+
+    *skip = -ptsdiff * play_samplerate;
+    return true;
 }
 
 int fill_audio_out_buffers(struct MPContext *mpctx, double endpts)
 {
     struct MPOpts *opts = mpctx->opts;
-    struct ao *ao = mpctx->ao;
-    int playsize;
-    int playflags = 0;
-    bool audio_eof = false;
-    bool signal_eof = false;
-    bool partial_fill = false;
     struct dec_audio *d_audio = mpctx->d_audio;
-    struct mp_audio out_format;
-    ao_get_format(ao, &out_format);
-    // Can't adjust the start of audio with spdif pass-through.
-    bool modifiable_audio_format = !(out_format.format & AF_FORMAT_SPECIAL_MASK);
 
     assert(d_audio);
 
-    if (mpctx->paused)
-        playsize = 1;   // just initialize things (audio pts at least)
-    else
-        playsize = ao_get_space(ao);
-
-    // Coming here with hrseek_active still set means audio-only
-    if (!mpctx->d_video || !mpctx->sync_audio_to_video)
-        mpctx->syncing_audio = false;
-    if (!opts->initial_audio_sync || !modifiable_audio_format) {
-        mpctx->syncing_audio = false;
-        mpctx->hrseek_active = false;
+    if (!d_audio->afilter || !mpctx->ao) {
+        // Probe the initial audio format. Returns AD_OK (and does nothing) if
+        // the format is already known.
+        int r = initial_audio_decode(mpctx->d_audio);
+        if (r == AD_WAIT)
+            return -1; // continue later when new data is available
+        if (r != AD_OK) {
+            MP_ERR(mpctx, "Error initializing audio.\n");
+            struct track *track = mpctx->current_track[0][STREAM_AUDIO];
+            mp_deselect_track(mpctx, track);
+            return -2;
+        }
+        reinit_audio_chain(mpctx);
+        return -1; // try again next iteration
     }
 
-    int res;
-    if (mpctx->syncing_audio || mpctx->hrseek_active)
-        res = audio_start_sync(mpctx, playsize);
-    else
-        res = audio_decode(d_audio, mpctx->ao_buffer, playsize);
+    // if paused, just initialize things (audio format & pts)
+    int playsize = 1;
+    if (!mpctx->paused)
+        playsize = ao_get_space(mpctx->ao);
+
+    int skip = 0;
+    bool sync_known = get_sync_samples(mpctx, &skip);
+    if (skip > 0) {
+        playsize = MPMIN(skip + 1, MPMAX(playsize, 2500)); // buffer extra data
+    } else if (skip < 0) {
+        playsize = MPMAX(1, playsize + skip); // silence will be prepended
+    }
 
-    if (res < 0) {  // EOF, error or format change
-        if (res == AD_NEW_FMT) {
+    int status = AD_OK;
+    if (playsize > mp_audio_buffer_samples(mpctx->ao_buffer)) {
+        status = audio_decode(d_audio, mpctx->ao_buffer, playsize);
+        if (status == AD_WAIT)
+            return -1;
+        if (status == AD_NEW_FMT) {
             /* The format change isn't handled too gracefully. A more precise
              * implementation would require draining buffered old-format audio
              * while displaying video, then doing the output format switch.
@@ -416,16 +397,56 @@ int fill_audio_out_buffers(struct MPContext *mpctx, double endpts)
             if (mpctx->opts->gapless_audio < 1)
                 uninit_player(mpctx, INITIALIZED_AO);
             reinit_audio_chain(mpctx);
-            return -1;
-        } else if (res == AD_ASYNC_PLAY_DONE)
-            return 0;
-        else if (res == AD_EOF)
-            audio_eof = true;
+            mpctx->sleeptime = 0;
+            return -1; // retry on next iteration
+        }
+    }
+
+    bool end_sync = status != AD_OK; // (on error/EOF, start playback immediately)
+    if (skip >= 0) {
+        int max = mp_audio_buffer_samples(mpctx->ao_buffer);
+        mp_audio_buffer_skip(mpctx->ao_buffer, MPMIN(skip, max));
+        // If something is left, we definitely reached the target time.
+        end_sync |= sync_known && skip < max;
+    } else if (skip < 0) {
+        if (-skip < 1000000) { // heuristic against making the buffer too large
+            mp_audio_buffer_prepend_silence(mpctx->ao_buffer, -skip);
+        } else {
+            MP_ERR(mpctx, "Audio starts too late: sync. failed.\n");
+            ao_reset(mpctx->ao);
+        }
+        end_sync = true;
+    }
+
+    if (mpctx->audio_status == STATUS_SYNCING) {
+        if (end_sync)
+            mpctx->audio_status = STATUS_FILLING;
+        mpctx->sleeptime = 0;
+        return -1; // continue on next iteration
+    }
+
+    assert(mpctx->audio_status >= STATUS_FILLING);
+
+    // Even if we're done decoding and syncing, let video start first - this is
+    // required, because sending audio to the AO already starts playback.
+    if (mpctx->audio_status == STATUS_FILLING && mpctx->sync_audio_to_video &&
+        mpctx->video_status <= STATUS_READY)
+    {
+        mpctx->audio_status = STATUS_READY;
+        return -1;
     }
 
+    bool audio_eof = status == AD_EOF;
+    bool partial_fill = false;
+    int playflags = 0;
+
+    struct mp_audio out_format = {0};
+    ao_get_format(mpctx->ao, &out_format);
+    double play_samplerate = out_format.rate / opts->playback_speed;
+
     if (endpts != MP_NOPTS_VALUE) {
         double samples = (endpts - written_audio_pts(mpctx) - mpctx->audio_delay)
-                         * out_format.rate / opts->playback_speed;
+                         * play_samplerate;
         if (playsize > samples) {
             playsize = MPMAX(samples, 0);
             audio_eof = true;
@@ -437,17 +458,14 @@ int fill_audio_out_buffers(struct MPContext *mpctx, double endpts)
         playsize = mp_audio_buffer_samples(mpctx->ao_buffer);
         partial_fill = true;
     }
-    if (!playsize)
-        return partial_fill && audio_eof ? -2 : -partial_fill;
-
-    if (audio_eof && partial_fill) {
-        if (opts->gapless_audio) {
-            // With gapless audio, delay this to ao_uninit. There must be only
-            // 1 final chunk, and that is handled when calling ao_uninit().
-            signal_eof = true;
-        } else {
+
+    audio_eof &= partial_fill;
+
+    if (audio_eof) {
+        // With gapless audio, delay this to ao_uninit. There must be only
+        // 1 final chunk, and that is handled when calling ao_uninit().
+        if (opts->gapless_audio)
             playflags |= AOPLAY_FINAL_CHUNK;
-        }
     }
 
     if (mpctx->paused)
@@ -458,10 +476,18 @@ int fill_audio_out_buffers(struct MPContext *mpctx, double endpts)
     data.samples = MPMIN(data.samples, playsize);
     int played = write_to_ao(mpctx, &data, playflags, written_audio_pts(mpctx));
     assert(played >= 0 && played <= data.samples);
-
     mp_audio_buffer_skip(mpctx->ao_buffer, played);
 
-    return signal_eof ? -2 : -partial_fill;
+    mpctx->audio_status = STATUS_PLAYING;
+    if (audio_eof) {
+        mpctx->audio_status = STATUS_DRAINING;
+        // Wait until the AO has played all queued data. In the gapless case,
+        // we trigger EOF immediately, and let it play asynchronously.
+        if (ao_eof_reached(mpctx->ao) || opts->gapless_audio)
+            mpctx->audio_status = STATUS_EOF;
+    }
+
+    return 0;
 }
 
 // Drop data queued for output, or which the AO is currently outputting.