audio/filter: fix mul/delay scale and values

Before this commit, the af_instance->mul/delay values were in bytes.
Using bytes is confusing for non-interleaved audio, so switch mul to
samples, and delay to seconds. For delay, seconds are more intuitive
than bytes or samples, because it's used for the latency calculation.
We also might want to replace the delay mechanism with real PTS
tracking inside the filter chain some time in the future, and PTS
will also require time-adjustments to be done in seconds.

For most filters, we just remove the redundant mul=1 initialization.
(Setting this used to be required, but not anymore.)

28 files changed, 23 insertions, 55 deletions

diff --git a/audio/decode/dec_audio.c b/audio/decode/dec_audio.c
index 3f92c3c4e6..ebb54cb55a 100644
--- a/audio/decode/dec_audio.c
+++ b/audio/decode/dec_audio.c
@@ -312,10 +312,9 @@ int decode_audio(sh_audio_t *sh_audio, struct mp_audio_buffer *outbuf,
     max_decode_len -= max_decode_len % unitsize;
 
     while (minsamples >= 0 && mp_audio_buffer_samples(outbuf) < minsamples) {
-        struct af_stream *afs = sh_audio->afilter;
-        int out_sstride = afs->output.sstride;
-        int declen = (minsamples - mp_audio_buffer_samples(outbuf))
-                     * out_sstride / filter_multiplier;
+        int decsamples = (minsamples - mp_audio_buffer_samples(outbuf))
+                         / filter_multiplier;
+        int declen = decsamples * sstride;
         // + some extra for possible filter buffering
         declen += unitsize << 5;
         if (huge_filter_buffer)
diff --git a/audio/filter/af.c b/audio/filter/af.c
index 182ed27d03..63013e81d9 100644
--- a/audio/filter/af.c
+++ b/audio/filter/af.c
@@ -703,12 +703,12 @@ struct mp_audio *af_play(struct af_stream *s, struct mp_audio *data)
     return data;
 }
 
-// Calculate average ratio of filter output size to input size
+// Calculate average ratio of filter output samples to input samples.
+// e.g: num_output_samples = mul * num_input_samples
 double af_calc_filter_multiplier(struct af_stream *s)
 {
     struct af_instance *af = s->first;
     double mul = 1;
-    // Iterate through all filters and calculate total multiplication factor
     do {
         mul *= af->mul;
         af = af->next;
@@ -721,11 +721,9 @@ double af_calc_filter_multiplier(struct af_stream *s)
 double af_calc_delay(struct af_stream *s)
 {
     struct af_instance *af = s->first;
-    register double delay = 0.0;
-    // Iterate through all filters
+    double delay = 0.0;
     while (af) {
         delay += af->delay;
-        delay *= af->mul;
         af = af->next;
     }
     return delay;
diff --git a/audio/filter/af.h b/audio/filter/af.h
index 36c2cf33fd..7852fa09a6 100644
--- a/audio/filter/af.h
+++ b/audio/filter/af.h
@@ -64,10 +64,11 @@ struct af_instance {
     struct mp_audio *data; // configuration and buffer for outgoing data stream
     struct af_instance *next;
     struct af_instance *prev;
-    double delay; /* Delay caused by the filter, in units of bytes read without
-                   * corresponding output */
+    double delay; /* Delay caused by the filter, in seconds of audio consumed
+                   * without corresponding output */
     double mul; /* length multiplier: how much does this instance change
-                   the length of the buffer. */
+                 * the number of samples passed though. (Ratio of input
+                 * and output, e.g. mul=4 => 1 sample becomes 4 samples) .*/
     bool auto_inserted; // inserted by af.c, such as conversion filters
 };
 
diff --git a/audio/filter/af_bs2b.c b/audio/filter/af_bs2b.c
index 73388ed5ba..769a2b4577 100644
--- a/audio/filter/af_bs2b.c
+++ b/audio/filter/af_bs2b.c
@@ -171,7 +171,6 @@ static int af_open(struct af_instance *af)
     struct af_bs2b *s = af->priv;
     af->control = control;
     af->uninit  = uninit;
-    af->mul     = 1;
 
     // NULL means failed initialization
     if (!(s->filter = bs2b_open())) {
diff --git a/audio/filter/af_center.c b/audio/filter/af_center.c
index 018fb756e6..ed482c7a6b 100644
--- a/audio/filter/af_center.c
+++ b/audio/filter/af_center.c
@@ -107,7 +107,6 @@ static int af_open(struct af_instance* af){
   af->control=control;
   af->uninit=uninit;
   af->play=play;
-  af->mul=1;
   af->setup=s=calloc(1,sizeof(af_center_t));
   if(af->setup == NULL)
     return AF_ERROR;
diff --git a/audio/filter/af_channels.c b/audio/filter/af_channels.c
index 7e05a9e234..f77807a1af 100644
--- a/audio/filter/af_channels.c
+++ b/audio/filter/af_channels.c
@@ -168,7 +168,6 @@ static int control(struct af_instance* af, int cmd, void* arg)
 
     af->data->rate   = ((struct mp_audio*)arg)->rate;
     mp_audio_set_format(af->data, ((struct mp_audio*)arg)->format);
-    af->mul          = (double)af->data->nch / ((struct mp_audio*)arg)->nch;
     mp_audio_force_interleaved_format(af->data);
     int r = af_test_output(af,(struct mp_audio*)arg);
     if (r != AF_OK)
@@ -255,7 +254,6 @@ static int af_open(struct af_instance* af){
   af->control=control;
   af->uninit=uninit;
   af->play=play;
-  af->mul=1;
   af->setup=calloc(1,sizeof(af_channels_t));
   if(af->setup == NULL)
     return AF_ERROR;
diff --git a/audio/filter/af_convert24.c b/audio/filter/af_convert24.c
index 19904cc8e7..ee8aff5afc 100644
--- a/audio/filter/af_convert24.c
+++ b/audio/filter/af_convert24.c
@@ -53,8 +53,6 @@ static int control(struct af_instance *af, int cmd, void *arg)
 
         assert(test_conversion(in->format, out->format));
 
-        af->mul = (double)out->bps / in->bps;
-
         return mp_audio_config_equals(in, &orig_in) ? AF_OK : AF_FALSE;
     }
     case AF_CONTROL_FORMAT_FMT | AF_CONTROL_SET: {
diff --git a/audio/filter/af_convertsignendian.c b/audio/filter/af_convertsignendian.c
index 18e04be21a..65fffdf487 100644
--- a/audio/filter/af_convertsignendian.c
+++ b/audio/filter/af_convertsignendian.c
@@ -120,7 +120,6 @@ static int af_open(struct af_instance *af)
 {
     af->control = control;
     af->play = play;
-    af->mul = 1;
     return AF_OK;
 }
 
diff --git a/audio/filter/af_delay.c b/audio/filter/af_delay.c
index 27b1ca2a4a..e2ef57221d 100644
--- a/audio/filter/af_delay.c
+++ b/audio/filter/af_delay.c
@@ -181,7 +181,6 @@ static int af_open(struct af_instance* af){
   af->control=control;
   af->uninit=uninit;
   af->play=play;
-  af->mul=1;
   af->setup=calloc(1,sizeof(af_delay_t));
   if(af->setup == NULL)
     return AF_ERROR;
diff --git a/audio/filter/af_drc.c b/audio/filter/af_drc.c
index c1e043ac75..17c4a12a95 100644
--- a/audio/filter/af_drc.c
+++ b/audio/filter/af_drc.c
@@ -319,7 +319,6 @@ static int af_open(struct af_instance* af){
   af->control=control;
   af->uninit=uninit;
   af->play=play;
-  af->mul=1;
   af->setup=calloc(1,sizeof(af_drc_t));
   if(af->setup == NULL)
     return AF_ERROR;
diff --git a/audio/filter/af_dummy.c b/audio/filter/af_dummy.c
index 87b75fd1b0..d1cb054413 100644
--- a/audio/filter/af_dummy.c
+++ b/audio/filter/af_dummy.c
@@ -53,7 +53,6 @@ static struct mp_audio* play(struct af_instance* af, struct mp_audio* data)
 static int af_open(struct af_instance* af){
   af->control=control;
   af->play=play;
-  af->mul=1;
   return AF_OK;
 }
 
diff --git a/audio/filter/af_equalizer.c b/audio/filter/af_equalizer.c
index 5fcfa18b35..75a489d867 100644
--- a/audio/filter/af_equalizer.c
+++ b/audio/filter/af_equalizer.c
@@ -114,7 +114,7 @@ static int control(struct af_instance* af, int cmd, void* arg)
       bp2(s->a[k],s->b[k],F[k]/((float)af->data->rate),Q);
 
     // Calculate how much this plugin adds to the overall time delay
-    af->delay = 2 * af->data->nch * af->data->bps;
+    af->delay = 2.0 / (double)af->data->rate;
 
     // Calculate gain factor to prevent clipping at output
     for(k=0;k<AF_NCH;k++)
@@ -203,7 +203,6 @@ static int af_open(struct af_instance* af){
   af->control=control;
   af->uninit=uninit;
   af->play=play;
-  af->mul=1;
   af->setup=calloc(1,sizeof(af_equalizer_t));
   if(af->setup == NULL)
     return AF_ERROR;
diff --git a/audio/filter/af_export.c b/audio/filter/af_export.c
index 0139b17d4f..5e1096f85a 100644
--- a/audio/filter/af_export.c
+++ b/audio/filter/af_export.c
@@ -256,7 +256,6 @@ static int af_open( struct af_instance* af )
   af->control = control;
   af->uninit  = uninit;
   af->play    = play;
-  af->mul=1;
   af->setup   = calloc(1, sizeof(af_export_t));
   if(af->setup == NULL)
     return AF_ERROR;
diff --git a/audio/filter/af_extrastereo.c b/audio/filter/af_extrastereo.c
index 68aeadac5e..4561b60690 100644
--- a/audio/filter/af_extrastereo.c
+++ b/audio/filter/af_extrastereo.c
@@ -128,7 +128,6 @@ static int af_open(struct af_instance* af){
   af->control=control;
   af->uninit=uninit;
   af->play=play_s16;
-  af->mul=1;
   af->setup=calloc(1,sizeof(af_extrastereo_t));
   if(af->setup == NULL)
     return AF_ERROR;
diff --git a/audio/filter/af_format.c b/audio/filter/af_format.c
index 58f311735e..5b941951cb 100644
--- a/audio/filter/af_format.c
+++ b/audio/filter/af_format.c
@@ -106,7 +106,6 @@ static int af_open(struct af_instance *af)
 {
     af->control = control;
     af->play = play;
-    af->mul = 1;
 
     force_in_params(af, af->data);
     force_out_params(af, af->data);
diff --git a/audio/filter/af_hrtf.c b/audio/filter/af_hrtf.c
index a0dbb12675..ed51351750 100644
--- a/audio/filter/af_hrtf.c
+++ b/audio/filter/af_hrtf.c
@@ -313,7 +313,6 @@ static int control(struct af_instance *af, int cmd, void* arg)
 	      mp_audio_set_channels_old(af->data, 5);
         mp_audio_set_format(af->data, AF_FORMAT_S16_NE);
 	test_output_res = af_test_output(af, (struct mp_audio*)arg);
-	af->mul = 2.0 / af->data->nch;
 	// after testing input set the real output format
         mp_audio_set_num_channels(af->data, 2);
 	s->print_flag = 1;
@@ -598,7 +597,6 @@ static int af_open(struct af_instance* af)
     af->control = control;
     af->uninit = uninit;
     af->play = play;
-    af->mul = 1;
     af->setup = calloc(1, sizeof(af_hrtf_t));
     if(af->setup == NULL)
 	return AF_ERROR;
diff --git a/audio/filter/af_karaoke.c b/audio/filter/af_karaoke.c
index 0d0ab7b2eb..8c633b136c 100644
--- a/audio/filter/af_karaoke.c
+++ b/audio/filter/af_karaoke.c
@@ -69,7 +69,6 @@ static struct mp_audio* play(struct af_instance* af, struct mp_audio* data)
 static int af_open(struct af_instance* af){
 	af->control	= control;
 	af->play	= play;
-	af->mul		= 1;
 	return AF_OK;
 }
 
diff --git a/audio/filter/af_ladspa.c b/audio/filter/af_ladspa.c
index 3a6c258a89..df88c06ab2 100644
--- a/audio/filter/af_ladspa.c
+++ b/audio/filter/af_ladspa.c
@@ -875,7 +875,6 @@ static int af_open(struct af_instance *af) {
     af->control=control;
     af->uninit=uninit;
     af->play=play;
-    af->mul=1;
 
     af->setup = calloc(1, sizeof(af_ladspa_t));
     if (af->setup == NULL) {
diff --git a/audio/filter/af_lavcac3enc.c b/audio/filter/af_lavcac3enc.c
index 6ecefa0d8f..2820e71f41 100644
--- a/audio/filter/af_lavcac3enc.c
+++ b/audio/filter/af_lavcac3enc.c
@@ -88,12 +88,13 @@ static int control(struct af_instance *af, int cmd, void *arg)
         test_output_res = af_test_output(af, data);
 
         s->pending_len = 0;
-        s->expect_len = AC3_FRAME_SIZE * data->nch * af->data->bps;
+        int expect_samples = AC3_FRAME_SIZE;
+        s->expect_len = expect_samples * data->nch * af->data->bps;
         assert(s->expect_len <= s->pending_data_size);
         if (s->add_iec61937_header)
-            af->mul = (double)AC3_FRAME_SIZE * 2 * 2 / s->expect_len;
+            af->mul = 1;
         else
-            af->mul = (double)AC3_MAX_CODED_FRAME_SIZE / s->expect_len;
+            af->mul = (double)(AC3_MAX_CODED_FRAME_SIZE / (2 * 2)) / expect_samples;
 
         mp_msg(MSGT_AFILTER, MSGL_DBG2, "af_lavcac3enc reinit: %d, %d, %f, %d.\n",
                data->nch, data->rate, af->mul, s->expect_len);
@@ -306,7 +307,6 @@ static int af_open(struct af_instance* af){
     af->control=control;
     af->uninit=uninit;
     af->play=play;
-    af->mul=1;
     af->setup=s;
 
     s->lavc_acodec = avcodec_find_encoder_by_name("ac3");
diff --git a/audio/filter/af_lavfi.c b/audio/filter/af_lavfi.c
index e77110c067..24ff8c5985 100644
--- a/audio/filter/af_lavfi.c
+++ b/audio/filter/af_lavfi.c
@@ -197,7 +197,8 @@ static int control(struct af_instance *af, int cmd, void *arg)
 
         p->timebase_out = l_out->time_base;
 
-        af->mul = (double) (out->rate * out->nch) / (in->rate * in->nch);
+        // Blatantly incorrect; we don't know what the filters do.
+        af->mul = out->rate / (double)in->rate;
 
         return mp_audio_config_equals(in, &orig_in) ? AF_OK : AF_FALSE;
     }
@@ -270,7 +271,7 @@ static struct mp_audio *play(struct af_instance *af, struct mp_audio *data)
         // Need pts past the last output sample.
         out_time += r->samples / (double)r->rate;
 
-        af->delay = (in_time - out_time) * r->rate * r->sstride;
+        af->delay = in_time - out_time;
     }
 
     *data = *r;
@@ -286,7 +287,6 @@ static int af_open(struct af_instance *af)
     af->control = control;
     af->uninit = uninit;
     af->play = play;
-    af->mul = 1;
     struct priv *priv = af->priv;
     af->data = talloc_zero(priv, struct mp_audio),
     // Removing this requires fixing AVFrame.data vs. AVFrame.extended_data
diff --git a/audio/filter/af_lavrresample.c b/audio/filter/af_lavrresample.c
index 02d58122fb..2e298a3f39 100644
--- a/audio/filter/af_lavrresample.c
+++ b/audio/filter/af_lavrresample.c
@@ -249,8 +249,7 @@ static int control(struct af_instance *af, int cmd, void *arg)
         if (af_to_avformat(out->format) == AV_SAMPLE_FMT_NONE)
             mp_audio_set_format(out, in->format);
 
-        af->mul     = (double) (out->rate * out->nch) / (in->rate * in->nch);
-        af->delay   = out->nch * s->opts.filter_size / FFMIN(af->mul, 1);
+        af->mul     = out->rate / (double)in->rate;
 
         int r = ((in->format == orig_in.format) &&
                 mp_chmap_equals(&in->channels, &orig_in.channels))
@@ -333,9 +332,7 @@ static struct mp_audio *play(struct af_instance *af, struct mp_audio *data)
 
     mp_audio_realloc_min(out, out->samples);
 
-    af->delay = out->bps * av_rescale_rnd(get_delay(s),
-                                          s->ctx.out_rate, s->ctx.in_rate,
-                                          AV_ROUND_UP);
+    af->delay = get_delay(s) / (double)s->ctx.in_rate;
 
 #if !USE_SET_CHANNEL_MAPPING
     do_reorder(in, s->reorder_in);
@@ -380,7 +377,6 @@ static int af_open(struct af_instance *af)
     af->control = control;
     af->uninit  = uninit;
     af->play    = play;
-    af->mul     = 1;
 
     if (s->opts.cutoff <= 0.0)
         s->opts.cutoff = af_resample_default_cutoff(s->opts.filter_size);
diff --git a/audio/filter/af_pan.c b/audio/filter/af_pan.c
index cc6f129542..3d8c6045d0 100644
--- a/audio/filter/af_pan.c
+++ b/audio/filter/af_pan.c
@@ -57,7 +57,6 @@ static int control(struct af_instance* af, int cmd, void* arg)
     af->data->rate   = ((struct mp_audio*)arg)->rate;
     mp_audio_set_format(af->data, AF_FORMAT_FLOAT_NE);
     set_channels(af->data, s->nch ? s->nch: ((struct mp_audio*)arg)->nch);
-    af->mul          = (double)af->data->nch / ((struct mp_audio*)arg)->nch;
 
     if((af->data->format != ((struct mp_audio*)arg)->format) ||
        (af->data->bps != ((struct mp_audio*)arg)->bps)){
@@ -191,7 +190,6 @@ static int af_open(struct af_instance* af){
   af->control=control;
   af->uninit=uninit;
   af->play=play;
-  af->mul=1;
   af->setup=calloc(1,sizeof(af_pan_t));
   if(af->setup == NULL)
     return AF_ERROR;
diff --git a/audio/filter/af_scaletempo.c b/audio/filter/af_scaletempo.c
index 456de89812..c8560af502 100644
--- a/audio/filter/af_scaletempo.c
+++ b/audio/filter/af_scaletempo.c
@@ -256,7 +256,8 @@ static struct mp_audio *play(struct af_instance *af, struct mp_audio *data)
     // This filter can have a negative delay when scale > 1:
     // output corresponding to some length of input can be decided and written
     // after receiving only a part of that input.
-    af->delay = s->bytes_queued - s->bytes_to_slide;
+    af->delay = (s->bytes_queued - s->bytes_to_slide) / s->scale
+                / af->data->sstride / af->data->rate;
 
     data->planes[0] = af->data->planes[0];
     data->samples   = (pout - (int8_t *)af->data->planes[0]) / af->data->sstride;
@@ -299,9 +300,8 @@ static int control(struct af_instance *af, int cmd, void *arg)
         s->frames_stride        = srate * s->ms_stride;
         s->bytes_stride         = s->frames_stride * bps * nch;
         s->frames_stride_scaled = s->scale * s->frames_stride;
-        int bytes_stride_scaled = s->scale * s->bytes_stride;
         s->frames_stride_error  = 0;
-        af->mul = (double)s->bytes_stride / bytes_stride_scaled;
+        af->mul = 1.0 / s->scale;
         af->delay = 0;
 
         int frames_overlap = s->frames_stride * s->percent_overlap;
@@ -462,7 +462,6 @@ static int af_open(struct af_instance *af)
     af->control   = control;
     af->uninit    = uninit;
     af->play      = play;
-    af->mul       = 1;
 
     s->speed_tempo = !!(s->speed_opt & SCALE_TEMPO);
     s->speed_pitch = !!(s->speed_opt & SCALE_PITCH);
diff --git a/audio/filter/af_sinesuppress.c b/audio/filter/af_sinesuppress.c
index 33486390b9..ef6fd7d37b 100644
--- a/audio/filter/af_sinesuppress.c
+++ b/audio/filter/af_sinesuppress.c
@@ -148,7 +148,6 @@ static int af_open(struct af_instance* af){
   af->control=control;
   af->uninit=uninit;
   af->play=play_s16;
-  af->mul=1;
   af->setup=calloc(1,sizeof(af_sinesuppress_t));
   if(af->setup == NULL)
     return AF_ERROR;
diff --git a/audio/filter/af_sub.c b/audio/filter/af_sub.c
index 561ae94be6..4fd16904c9 100644
--- a/audio/filter/af_sub.c
+++ b/audio/filter/af_sub.c
@@ -165,7 +165,6 @@ static int af_open(struct af_instance* af){
   af->control=control;
   af->uninit=uninit;
   af->play=play;
-  af->mul=1;
   af->setup=s=calloc(1,sizeof(af_sub_t));
   if(af->setup == NULL)
     return AF_ERROR;
diff --git a/audio/filter/af_surround.c b/audio/filter/af_surround.c
index 9f9d46b788..efeecdc1e3 100644
--- a/audio/filter/af_surround.c
+++ b/audio/filter/af_surround.c
@@ -243,7 +243,6 @@ static int af_open(struct af_instance* af){
   af->control=control;
   af->uninit=uninit;
   af->play=play;
-  af->mul=2;
   af->setup=calloc(1,sizeof(af_surround_t));
   if(af->setup == NULL)
     return AF_ERROR;
diff --git a/audio/filter/af_sweep.c b/audio/filter/af_sweep.c
index 5b1e9d39c1..c153d4261a 100644
--- a/audio/filter/af_sweep.c
+++ b/audio/filter/af_sweep.c
@@ -84,7 +84,6 @@ static int af_open(struct af_instance* af){
   af->control=control;
   af->uninit=uninit;
   af->play=play;
-  af->mul=1;
   af->setup=calloc(1,sizeof(af_sweept));
   return AF_OK;
 }
diff --git a/audio/filter/af_volume.c b/audio/filter/af_volume.c
index 982b3816c7..b0eef1865c 100644
--- a/audio/filter/af_volume.c
+++ b/audio/filter/af_volume.c
@@ -104,7 +104,6 @@ static int af_open(struct af_instance *af)
     struct priv *s = af->priv;
     af->control = control;
     af->play = play;
-    af->mul = 1;
     af_from_dB(1, &s->cfg_volume, &s->level, 20.0, -200.0, 60.0);
     return AF_OK;
 }