audio: refactor: work towards unentangling audio decoding and filtering

Similar to the video path. dec_audio.c now handles decoding only. It
also looks very similar to dec_video.c, and actually contains some of
the rewritten code from it. (A further goal might be unifying the
decoders, I guess.)

High potential for regressions.

4 files changed, 175 insertions, 53 deletions

diff --git a/player/audio.c b/player/audio.c
index 5a62a69d1a..fb1398c597 100644
--- a/player/audio.c
+++ b/player/audio.c
@@ -43,6 +43,14 @@
 #include "core.h"
 #include "command.h"
 
+enum {
+    AD_OK = 0,
+    AD_ERR = -1,
+    AD_EOF = -2,
+    AD_NEW_FMT = -3,
+    AD_WAIT = -4,
+};
+
 // Use pitch correction only for speed adjustments by the user, not minor sync
 // correction ones.
 static int get_speed_method(struct MPContext *mpctx)
@@ -55,7 +63,7 @@ static int get_speed_method(struct MPContext *mpctx)
 // return true; if filter recreation is needed, return false.
 static bool update_speed_filters(struct MPContext *mpctx)
 {
-    struct af_stream *afs = mpctx->d_audio->afilter;
+    struct af_stream *afs = mpctx->ao_chain->af;
     double speed = mpctx->audio_speed;
 
     if (afs->initialized < 1)
@@ -81,7 +89,7 @@ static bool update_speed_filters(struct MPContext *mpctx)
 // Update speed, and insert/remove filters if necessary.
 static void recreate_speed_filters(struct MPContext *mpctx)
 {
-    struct af_stream *afs = mpctx->d_audio->afilter;
+    struct af_stream *afs = mpctx->ao_chain->af;
 
     if (update_speed_filters(mpctx))
         return;
@@ -113,9 +121,9 @@ fail:
 
 static int recreate_audio_filters(struct MPContext *mpctx)
 {
-    assert(mpctx->d_audio);
+    assert(mpctx->ao_chain);
 
-    struct af_stream *afs = mpctx->d_audio->afilter;
+    struct af_stream *afs = mpctx->ao_chain->af;
     if (afs->initialized < 1 && af_init(afs) < 0)
         goto fail;
 
@@ -134,11 +142,11 @@ fail:
 
 int reinit_audio_filters(struct MPContext *mpctx)
 {
-    struct dec_audio *d_audio = mpctx->d_audio;
-    if (!d_audio)
+    struct ao_chain *ao_c = mpctx->ao_chain;
+    if (!ao_c)
         return 0;
 
-    af_uninit(mpctx->d_audio->afilter);
+    af_uninit(ao_c->af);
     return recreate_audio_filters(mpctx) < 0 ? -1 : 1;
 }
 
@@ -148,7 +156,7 @@ void update_playback_speed(struct MPContext *mpctx)
     mpctx->audio_speed = mpctx->opts->playback_speed * mpctx->speed_factor_a;
     mpctx->video_speed = mpctx->opts->playback_speed * mpctx->speed_factor_v;
 
-    if (!mpctx->d_audio || mpctx->d_audio->afilter->initialized < 1)
+    if (!mpctx->ao_chain || mpctx->ao_chain->af->initialized < 1)
         return;
 
     if (!update_speed_filters(mpctx))
@@ -161,7 +169,11 @@ void reset_audio_state(struct MPContext *mpctx)
         audio_reset_decoding(mpctx->d_audio);
     if (mpctx->ao_buffer)
         mp_audio_buffer_clear(mpctx->ao_buffer);
-    mpctx->audio_status = mpctx->d_audio ? STATUS_SYNCING : STATUS_EOF;
+    if (mpctx->ao_chain) {
+        mpctx->ao_chain->pts = MP_NOPTS_VALUE;
+        af_seek_reset(mpctx->ao_chain->af);
+    }
+    mpctx->audio_status = mpctx->ao_chain ? STATUS_SYNCING : STATUS_EOF;
     mpctx->delay = 0;
     mpctx->audio_drop_throttle = 0;
     mpctx->audio_stat_start = 0;
@@ -185,12 +197,15 @@ void uninit_audio_out(struct MPContext *mpctx)
 
 void uninit_audio_chain(struct MPContext *mpctx)
 {
-    if (mpctx->d_audio) {
+    if (mpctx->ao_chain) {
         mixer_uninit_audio(mpctx->mixer);
         audio_uninit(mpctx->d_audio);
         mpctx->d_audio = NULL;
         talloc_free(mpctx->ao_buffer);
         mpctx->ao_buffer = NULL;
+        af_destroy(mpctx->ao_chain->af);
+        talloc_free(mpctx->ao_chain);
+        mpctx->ao_chain = NULL;
         mpctx->audio_status = STATUS_EOF;
         reselect_demux_streams(mpctx);
 
@@ -210,15 +225,25 @@ void reinit_audio_chain(struct MPContext *mpctx)
 
     mp_notify(mpctx, MPV_EVENT_AUDIO_RECONFIG, NULL);
 
+    struct ao_chain *ao_c = mpctx->ao_chain;
+
     if (!mpctx->d_audio) {
+        assert(!ao_c);
         mpctx->d_audio = talloc_zero(NULL, struct dec_audio);
         mpctx->d_audio->log = mp_log_new(mpctx->d_audio, mpctx->log, "!ad");
         mpctx->d_audio->global = mpctx->global;
         mpctx->d_audio->opts = opts;
         mpctx->d_audio->header = sh;
-        mpctx->d_audio->afilter = af_new(mpctx->global);
-        mpctx->d_audio->afilter->replaygain_data = sh->codec->replaygain_data;
-        mpctx->d_audio->spdif_passthrough = true;
+        ao_c = talloc_zero(NULL, struct ao_chain);
+        mpctx->ao_chain = ao_c;
+        ao_c->log = mpctx->d_audio->log;
+        ao_c->af = af_new(mpctx->global);
+        ao_c->af->replaygain_data = sh->codec->replaygain_data;
+        ao_c->spdif_passthrough = true;
+        ao_c->pts = MP_NOPTS_VALUE;
+        ao_c->ao = mpctx->ao;
+        ao_c->audio_src = mpctx->d_audio;
+        mpctx->d_audio->try_spdif = ao_c->spdif_passthrough;
         mpctx->ao_buffer = mp_audio_buffer_create(NULL);
         if (!audio_init_best_codec(mpctx->d_audio))
             goto init_error;
@@ -232,7 +257,7 @@ void reinit_audio_chain(struct MPContext *mpctx)
     }
     assert(mpctx->d_audio);
 
-    struct mp_audio in_format = mpctx->d_audio->decode_format;
+    struct mp_audio in_format = ao_c->input_format;
 
     if (!mp_audio_config_valid(&in_format)) {
         // We don't know the audio format yet - so configure it later as we're
@@ -248,7 +273,7 @@ void reinit_audio_chain(struct MPContext *mpctx)
         uninit_audio_out(mpctx);
     }
 
-    struct af_stream *afs = mpctx->d_audio->afilter;
+    struct af_stream *afs = ao_c->af;
 
     afs->output = (struct mp_audio){0};
     if (mpctx->ao) {
@@ -272,7 +297,7 @@ void reinit_audio_chain(struct MPContext *mpctx)
 
     if (!mpctx->ao) {
         bool spdif_fallback = af_fmt_is_spdif(afs->output.format) &&
-                              mpctx->d_audio->spdif_passthrough;
+                              ao_c->spdif_passthrough;
         bool ao_null_fallback = opts->ao_null_fallback && !spdif_fallback;
 
         mp_chmap_remove_useless_channels(&afs->output.channels,
@@ -282,6 +307,7 @@ void reinit_audio_chain(struct MPContext *mpctx)
         mpctx->ao = ao_init_best(mpctx->global, ao_null_fallback, mpctx->input,
                                  mpctx->encode_lavc_ctx, afs->output.rate,
                                  afs->output.format, afs->output.channels);
+        ao_c->ao = mpctx->ao;
 
         struct mp_audio fmt = {0};
         if (mpctx->ao)
@@ -293,14 +319,16 @@ void reinit_audio_chain(struct MPContext *mpctx)
                 MP_ERR(mpctx, "Passthrough format unsupported.\n");
                 ao_uninit(mpctx->ao);
                 mpctx->ao = NULL;
+                ao_c->ao = NULL;
             }
         }
 
         if (!mpctx->ao) {
             // If spdif was used, try to fallback to PCM.
             if (spdif_fallback) {
-                mpctx->d_audio->spdif_passthrough = false;
-                mpctx->d_audio->spdif_failed = true;
+                ao_c->spdif_passthrough = false;
+                ao_c->spdif_failed = true;
+                mpctx->d_audio->try_spdif = false;
                 if (!audio_init_best_codec(mpctx->d_audio))
                     goto init_error;
                 reset_audio_state(mpctx);
@@ -346,35 +374,22 @@ no_audio:
 // ao so far.
 double written_audio_pts(struct MPContext *mpctx)
 {
-    struct dec_audio *d_audio = mpctx->d_audio;
-    if (!d_audio)
+    struct ao_chain *ao_c = mpctx->ao_chain;
+    if (!ao_c)
         return MP_NOPTS_VALUE;
 
-    struct mp_audio in_format = d_audio->decode_format;
+    struct mp_audio in_format = ao_c->input_format;
 
-    if (!mp_audio_config_valid(&in_format) || d_audio->afilter->initialized < 1)
+    if (!mp_audio_config_valid(&in_format) || ao_c->af->initialized < 1)
         return MP_NOPTS_VALUE;
 
     // first calculate the end pts of audio that has been output by decoder
-    double a_pts = d_audio->pts;
+    double a_pts = ao_c->pts;
     if (a_pts == MP_NOPTS_VALUE)
         return MP_NOPTS_VALUE;
 
-    // d_audio->pts is the timestamp of the first sample of the latest frame
-    // the with a known pts that the decoder has returned. d_audio->pts_offset
-    // is the amount of samples the decoder has returned after that timestamp
-    // (includes the frame size).
-    a_pts += d_audio->pts_offset / (double)in_format.rate;
-
-    // Now a_pts hopefully holds the pts for end of audio from decoder.
-    // Subtract data in buffers between decoder and audio out.
-
-    // Decoded but not filtered
-    if (d_audio->waiting)
-        a_pts -= d_audio->waiting->samples / (double)in_format.rate;
-
     // Data buffered in audio filters, measured in seconds of "missing" output
-    double buffered_output = af_calc_delay(d_audio->afilter);
+    double buffered_output = af_calc_delay(ao_c->af);
 
     // Data that was ready for ao but was buffered because ao didn't fully
     // accept everything to internal buffers yet
@@ -498,20 +513,106 @@ static bool get_sync_samples(struct MPContext *mpctx, int *skip)
     return true;
 }
 
+
+static bool copy_output(struct af_stream *afs, struct mp_audio_buffer *outbuf,
+                        int minsamples, bool eof)
+{
+    while (mp_audio_buffer_samples(outbuf) < minsamples) {
+        if (af_output_frame(afs, eof) < 0)
+            return true; // error, stop doing stuff
+        struct mp_audio *mpa = af_read_output_frame(afs);
+        if (!mpa)
+            return false; // out of data
+        mp_audio_buffer_append(outbuf, mpa);
+        talloc_free(mpa);
+    }
+    return true;
+}
+
+static int decode_new_frame(struct ao_chain *ao_c)
+{
+    if (ao_c->input_frame)
+        return AD_OK;
+
+    int res = AUDIO_SKIP;
+    while (res == AUDIO_SKIP) {
+        audio_work(ao_c->audio_src);
+        res = audio_get_frame(ao_c->audio_src, &ao_c->input_frame);
+    }
+
+    if (ao_c->input_frame)
+        mp_audio_copy_config(&ao_c->input_format, ao_c->input_frame);
+
+    switch (res) {
+    case AUDIO_OK:      return AD_OK;
+    case AUDIO_WAIT:    return AD_WAIT;
+    case AUDIO_EOF:     return AD_EOF;
+    default:            abort();
+    }
+}
+
+/* Try to get at least minsamples decoded+filtered samples in outbuf
+ * (total length including possible existing data).
+ * Return 0 on success, or negative AD_* error code.
+ * In the former case outbuf has at least minsamples buffered on return.
+ * In case of EOF/error it might or might not be. */
+static int filter_audio(struct ao_chain *ao_c, struct mp_audio_buffer *outbuf,
+                        int minsamples)
+{
+    struct af_stream *afs = ao_c->af;
+    if (afs->initialized < 1)
+        return AD_ERR;
+
+    MP_STATS(ao_c, "start audio");
+
+    int res;
+    while (1) {
+        res = 0;
+
+        if (copy_output(afs, outbuf, minsamples, false))
+            break;
+
+        res = decode_new_frame(ao_c);
+        if (res < 0) {
+            // drain filters first (especially for true EOF case)
+            copy_output(afs, outbuf, minsamples, true);
+            break;
+        }
+
+        // On format change, make sure to drain the filter chain.
+        if (!mp_audio_config_equals(&afs->input, ao_c->input_frame)) {
+            copy_output(afs, outbuf, minsamples, true);
+            res = AD_NEW_FMT;
+            break;
+        }
+
+        struct mp_audio *mpa = ao_c->input_frame;
+        ao_c->input_frame = NULL;
+        ao_c->pts = mpa->pts + mpa->samples / (double)mpa->rate;
+        if (af_filter_frame(afs, mpa) < 0)
+            return AD_ERR;
+    }
+
+    MP_STATS(ao_c, "end audio");
+
+    return res;
+}
+
 void fill_audio_out_buffers(struct MPContext *mpctx, double endpts)
 {
     struct MPOpts *opts = mpctx->opts;
-    struct dec_audio *d_audio = mpctx->d_audio;
+    struct ao_chain *ao_c = mpctx->ao_chain;
 
     dump_audio_stats(mpctx);
 
     if (mpctx->ao && ao_query_and_reset_events(mpctx->ao, AO_EVENT_RELOAD)) {
         ao_reset(mpctx->ao);
         uninit_audio_out(mpctx);
-        if (d_audio) {
-            if (mpctx->d_audio->spdif_failed) {
-                mpctx->d_audio->spdif_failed = false;
-                mpctx->d_audio->spdif_passthrough = true;
+        if (ao_c) {
+            struct dec_audio *d_audio = ao_c->audio_src;
+            if (d_audio && ao_c->spdif_failed) {
+                ao_c->spdif_failed = false;
+                mpctx->d_audio->try_spdif = true;
                 if (!audio_init_best_codec(mpctx->d_audio)) {
                     MP_ERR(mpctx, "Error reinitializing audio.\n");
                     error_on_track(mpctx, mpctx->current_track[0][STREAM_AUDIO]);
@@ -522,13 +623,13 @@ void fill_audio_out_buffers(struct MPContext *mpctx, double endpts)
         }
     }
 
-    if (!d_audio)
+    if (!ao_c)
         return;
 
-    if (d_audio->afilter->initialized < 1 || !mpctx->ao) {
+    if (ao_c->af->initialized < 1 || !mpctx->ao) {
         // Probe the initial audio format. Returns AD_OK (and does nothing) if
         // the format is already known.
-        int r = initial_audio_decode(mpctx->d_audio);
+        int r = decode_new_frame(mpctx->ao_chain);
         if (r == AD_WAIT)
             return; // continue later when new data is available
         reinit_audio_chain(mpctx);
@@ -536,7 +637,7 @@ void fill_audio_out_buffers(struct MPContext *mpctx, double endpts)
         return; // try again next iteration
     }
 
-    if (mpctx->vo_chain && d_audio->pts_reset) {
+    if (mpctx->vo_chain && ao_c->audio_src->pts_reset) {
         MP_VERBOSE(mpctx, "Reset playback due to audio timestamp reset.\n");
         reset_playback_state(mpctx);
         mpctx->sleeptime = 0;
@@ -586,7 +687,7 @@ void fill_audio_out_buffers(struct MPContext *mpctx, double endpts)
     int status = AD_OK;
     bool working = false;
     if (playsize > mp_audio_buffer_samples(mpctx->ao_buffer)) {
-        status = audio_decode(d_audio, mpctx->ao_buffer, playsize);
+        status = filter_audio(mpctx->ao_chain, mpctx->ao_buffer, playsize);
         if (status == AD_WAIT)
             return;
         if (status == AD_NEW_FMT) {
diff --git a/player/command.c b/player/command.c
index 413349dce3..52f0b1830f 100644
--- a/player/command.c
+++ b/player/command.c
@@ -1242,9 +1242,9 @@ static int mp_property_filter_metadata(void *ctx, struct m_property *prop,
             struct vf_chain *vf = mpctx->vo_chain->vf;
             res = vf_control_by_label(vf, VFCTRL_GET_METADATA, &metadata, key);
         } else if (strcmp(type, "af") == 0) {
-            if (!(mpctx->d_audio && mpctx->d_audio->afilter))
+            if (!(mpctx->ao_chain && mpctx->ao_chain->af))
                 return M_PROPERTY_UNAVAILABLE;
-            struct af_stream *af = mpctx->d_audio->afilter;
+            struct af_stream *af = mpctx->ao_chain->af;
             res = af_control_by_label(af, AF_CONTROL_GET_METADATA, &metadata, key);
         }
         switch (res) {
@@ -1751,8 +1751,8 @@ static int mp_property_audio_params(void *ctx, struct m_property *prop,
 {
     MPContext *mpctx = ctx;
     struct mp_audio fmt = {0};
-    if (mpctx->d_audio)
-        fmt = mpctx->d_audio->decode_format;
+    if (mpctx->ao_chain)
+        fmt = mpctx->ao_chain->input_format;
     return property_audiofmt(fmt, action, arg);
 }
 
diff --git a/player/core.h b/player/core.h
index 89502d4e1b..00d1c81460 100644
--- a/player/core.h
+++ b/player/core.h
@@ -27,6 +27,7 @@
 #include "options/options.h"
 #include "sub/osd.h"
 #include "demux/timeline.h"
+#include "audio/audio.h"
 #include "video/mp_image.h"
 #include "video/out/vo.h"
 
@@ -171,6 +172,25 @@ struct vo_chain {
     struct dec_video *video_src;
 };
 
+// Like vo_chain, for audio.
+struct ao_chain {
+    struct mp_log *log;
+
+    double pts; // timestamp of first sample output by decoder
+    bool spdif_passthrough, spdif_failed;
+
+    struct af_stream *af;
+    struct ao *ao;
+
+    // 1-element input frame queue.
+    struct mp_audio *input_frame;
+
+    // Last known input_mpi format (so vf can be reinitialized any time).
+    struct mp_audio input_format;
+
+    struct dec_audio *audio_src;
+};
+
 /* Note that playback can be paused, stopped, etc. at any time. While paused,
  * playback restart is still active, because you want seeking to work even
  * if paused.
@@ -278,6 +298,7 @@ typedef struct MPContext {
     struct ao *ao;
     struct mp_audio *ao_decoder_fmt; // for weak gapless audio check
     struct mp_audio_buffer *ao_buffer;  // queued audio; passed to ao_play() later
+    struct ao_chain *ao_chain;
 
     struct vo_chain *vo_chain;
 
diff --git a/player/video.c b/player/video.c
index faa8a652bb..18a7ef53ec 100644
--- a/player/video.c
+++ b/player/video.c
@@ -889,8 +889,8 @@ static double find_best_speed(struct MPContext *mpctx, double vsync)
 
 static bool using_spdif_passthrough(struct MPContext *mpctx)
 {
-    if (mpctx->d_audio && mpctx->d_audio->afilter)
-        return !af_fmt_is_pcm(mpctx->d_audio->afilter->input.format);
+    if (mpctx->ao_chain)
+        return !af_fmt_is_pcm(mpctx->ao_chain->input_format.format);
     return false;
 }