player: add complex filter graph support

See --lavfi-complex option.

This is still quite rough. There's no support for dynamic configuration
of any kind. There are probably corner cases where playback might freeze
or burn 100% CPU (due to dataflow problems when interaction with
libavfilter).

Future possible plans might include:
- freely switch tracks by providing some sort of default track graph
  label
- automatically enabling audio visualization
- automatically mix audio or stack video when multiple tracks are
  selected at once (similar to how multiple sub tracks can be selected)

11 files changed, 1121 insertions, 53 deletions

diff --git a/DOCS/man/options.rst b/DOCS/man/options.rst
index dc4d6ed3e0..97f84c6219 100644
--- a/DOCS/man/options.rst
+++ b/DOCS/man/options.rst
@@ -1396,7 +1396,7 @@ Subtitles
     .. admonition:: Warning
 
         Enabling hinting can lead to mispositioned text (in situations it's
-        supposed to match up with video background), or reduce the smoothness
+        supposed to match up video background), or reduce the smoothness
         of animations with some badly authored ASS scripts. It is recommended
         to not use this option, unless really needed.
 
@@ -3553,3 +3553,45 @@ Miscellaneous
     Force the contents of the ``media-title`` property to this value. Useful
     for scripts which want to set a title, without overriding the user's
     setting in ``--title``.
+
+``--lavfi-complex=<string>``
+    Set a "complex" libavfilter filter, which means a single filter graph can
+    take input from multiple source audio and video tracks. The graph can result
+    in a single audio or video output (or both).
+
+    Currently, the filter graph labels are used to select the participating
+    input tracks and audio/video output. The following rules apply:
+
+    - A label of the form ``aidN`` selects audio track N as input (e.g.
+      ``aid1``).
+    - A label of the form ``vidN`` selects video track N as input.
+    - A label named ``ao`` will be connected to the audio input.
+    - A label named ``vo`` will be connected to the video output.
+
+    Each label can be used only once. If you want to use e.g. an audio stream
+    for multiple filters, you need to use the ``asplit`` filter. Multiple
+    video or audio outputs are not possible, but you can use filters to merge
+    them into one.
+
+    The complex filter can not be changed yet during playback. It's also not
+    possible to change the tracks connected to the filter at runtime. Other
+    tracks, as long as they're not connected to the filter, and the
+    corresponding output is not connected to the filter, can still be freely
+    changed.
+
+    .. admonition:: Examples
+
+        - ``--lavfi-complex='[aid1] asplit [ao] [t] ; [t] aphasemeter [vo]'``
+          Play audio track 1, and visualize it as video using the ``aphasemeter``
+          filter.
+        - ``--lavfi-complex='[vid1] [vid2] vstack [vo]'``
+          Stack video track 1 and 2 and play them at the same time. Note that
+          both tracks need to have the same width, or filter initialization
+          will fail (you can add ``scale`` filters before the ``vstack`` filter
+          to fix the size).
+        - ``--lavfi-complex='[aid1] asplit [ao] [t] ; [t] aphasemeter [t2] ; [vid1] [t2] overlay [vo]'``
+          Play audio track 1, and overlay its visualization over video track 1.
+
+    See the Ffmpeg libavfilter documentation for details on the filter.
+
+
diff --git a/options/options.c b/options/options.c
index 0377854c0e..4352b6ceed 100644
--- a/options/options.c
+++ b/options/options.c
@@ -226,6 +226,8 @@ const m_option_t mp_opts[] = {
     OPT_STRINGLIST("alang", stream_lang[STREAM_AUDIO], 0),
     OPT_STRINGLIST("slang", stream_lang[STREAM_SUB], 0),
 
+    OPT_STRING("lavfi-complex", lavfi_complex, 0),
+
     OPT_CHOICE("audio-display", audio_display, 0,
                ({"no", 0}, {"attachment", 1})),
 
diff --git a/options/options.h b/options/options.h
index 6ce4be472a..488823ca15 100644
--- a/options/options.h
+++ b/options/options.h
@@ -186,6 +186,7 @@ typedef struct MPOpts {
     int ignore_path_in_watch_later_config;
     int pause;
     int keep_open;
+    char *lavfi_complex;
     int stream_id[2][STREAM_TYPE_COUNT];
     int stream_id_ff[STREAM_TYPE_COUNT];
     char **stream_lang[STREAM_TYPE_COUNT];
diff --git a/player/audio.c b/player/audio.c
index 5ecbab5ad1..18f2818fe5 100644
--- a/player/audio.c
+++ b/player/audio.c
@@ -49,6 +49,7 @@ enum {
     AD_EOF = -2,
     AD_NEW_FMT = -3,
     AD_WAIT = -4,
+    AD_NO_PROGRESS = -5,
 };
 
 // Use pitch correction only for speed adjustments by the user, not minor sync
@@ -204,6 +205,18 @@ void uninit_audio_out(struct MPContext *mpctx)
 
 static void ao_chain_uninit(struct ao_chain *ao_c)
 {
+    struct track *track = ao_c->track;
+    if (track) {
+        assert(track->ao_c == ao_c);
+        track->ao_c = NULL;
+        assert(track->d_audio == ao_c->audio_src);
+        track->d_audio = NULL;
+        audio_uninit(ao_c->audio_src);
+    }
+
+    if (ao_c->filter_src)
+        lavfi_set_connected(ao_c->filter_src, false);
+
     af_destroy(ao_c->af);
     talloc_free(ao_c->input_frame);
     talloc_free(ao_c->ao_buffer);
@@ -213,18 +226,10 @@ static void ao_chain_uninit(struct ao_chain *ao_c)
 void uninit_audio_chain(struct MPContext *mpctx)
 {
     if (mpctx->ao_chain) {
-        struct track *track = mpctx->current_track[0][STREAM_AUDIO];
-        assert(track);
-        assert(track->d_audio == mpctx->ao_chain->audio_src);
-
         mixer_uninit_audio(mpctx->mixer);
-
-        audio_uninit(track->d_audio);
-        track->d_audio = NULL;
-        mpctx->ao_chain->audio_src = NULL;
-
         ao_chain_uninit(mpctx->ao_chain);
         mpctx->ao_chain = NULL;
+
         mpctx->audio_status = STATUS_EOF;
         reselect_demux_streams(mpctx);
 
@@ -379,6 +384,9 @@ int init_audio_decoder(struct MPContext *mpctx, struct track *track)
     return 1;
 
 init_error:
+    if (track->sink)
+        lavfi_set_connected(track->sink, false);
+    track->sink = NULL;
     audio_uninit(track->d_audio);
     track->d_audio = NULL;
     error_on_track(mpctx, track);
@@ -387,14 +395,24 @@ init_error:
 
 void reinit_audio_chain(struct MPContext *mpctx)
 {
-    assert(!mpctx->ao_chain);
+    reinit_audio_chain_src(mpctx, NULL);
+}
 
-    struct track *track = mpctx->current_track[0][STREAM_AUDIO];
-    struct sh_stream *sh = track ? track->stream : NULL;
-    if (!sh) {
-        uninit_audio_out(mpctx);
-        goto no_audio;
+void reinit_audio_chain_src(struct MPContext *mpctx, struct lavfi_pad *src)
+{
+    struct track *track = NULL;
+    struct sh_stream *sh = NULL;
+    if (!src) {
+        track = mpctx->current_track[0][STREAM_AUDIO];
+        if (!track)
+            return;
+        sh = track ? track->stream : NULL;
+        if (!sh) {
+            uninit_audio_out(mpctx);
+            goto no_audio;
+        }
     }
+    assert(!mpctx->ao_chain);
 
     mp_notify(mpctx, MPV_EVENT_AUDIO_RECONFIG, NULL);
 
@@ -402,16 +420,21 @@ void reinit_audio_chain(struct MPContext *mpctx)
     mpctx->ao_chain = ao_c;
     ao_c->log = mpctx->log;
     ao_c->af = af_new(mpctx->global);
-    ao_c->af->replaygain_data = sh->codec->replaygain_data;
+    if (sh)
+        ao_c->af->replaygain_data = sh->codec->replaygain_data;
     ao_c->spdif_passthrough = true;
     ao_c->pts = MP_NOPTS_VALUE;
     ao_c->ao_buffer = mp_audio_buffer_create(NULL);
     ao_c->ao = mpctx->ao;
 
-    if (!init_audio_decoder(mpctx, track))
-        goto init_error;
-
-    ao_c->audio_src = track->d_audio;
+    ao_c->filter_src = src;
+    if (!ao_c->filter_src) {
+        ao_c->track = track;
+        track->ao_c = ao_c;
+        if (!init_audio_decoder(mpctx, track))
+            goto init_error;
+        ao_c->audio_src = track->d_audio;
+    }
 
     reset_audio_state(mpctx);
 
@@ -597,8 +620,10 @@ static int decode_new_frame(struct ao_chain *ao_c)
     if (ao_c->input_frame)
         return AD_OK;
 
-    int res = DATA_AGAIN;
-    while (res == DATA_AGAIN) {
+    int res = DATA_EOF;
+    if (ao_c->filter_src) {
+        res = lavfi_request_frame_a(ao_c->filter_src, &ao_c->input_frame);
+    } else if (ao_c->audio_src) {
         audio_work(ao_c->audio_src);
         res = audio_get_frame(ao_c->audio_src, &ao_c->input_frame);
     }
@@ -606,6 +631,7 @@ static int decode_new_frame(struct ao_chain *ao_c)
     switch (res) {
     case DATA_OK:       return AD_OK;
     case DATA_WAIT:     return AD_WAIT;
+    case DATA_AGAIN:    return AD_NO_PROGRESS;
     case DATA_EOF:      return AD_EOF;
     default:            abort();
     }
@@ -633,6 +659,8 @@ static int filter_audio(struct ao_chain *ao_c, struct mp_audio_buffer *outbuf,
             break;
 
         res = decode_new_frame(ao_c);
+        if (res == AD_NO_PROGRESS)
+            break;
         if (res < 0) {
             // drain filters first (especially for true EOF case)
             copy_output(afs, outbuf, minsamples, true);
@@ -764,6 +792,10 @@ void fill_audio_out_buffers(struct MPContext *mpctx, double endpts)
         status = filter_audio(mpctx->ao_chain, ao_c->ao_buffer, playsize);
         if (status == AD_WAIT)
             return;
+        if (status == AD_NO_PROGRESS) {
+            mpctx->sleeptime = 0;
+            return;
+        }
         if (status == AD_NEW_FMT) {
             /* The format change isn't handled too gracefully. A more precise
              * implementation would require draining buffered old-format audio
diff --git a/player/core.h b/player/core.h
index df79dd8c01..72820c5c5a 100644
--- a/player/core.h
+++ b/player/core.h
@@ -31,6 +31,8 @@
 #include "video/mp_image.h"
 #include "video/out/vo.h"
 
+#include "lavfi.h"
+
 // definitions used internally by the core player code
 
 enum stop_play_reason {
@@ -149,6 +151,11 @@ struct track {
     struct dec_video *d_video;
     struct dec_audio *d_audio;
 
+    // Where the decoded result goes to (one of them is not NULL if active)
+    struct vo_chain *vo_c;
+    struct ao_chain *ao_c;
+    struct lavfi_pad *sink;
+
     // For external subtitles, which are read fully on init. Do not attempt
     // to read packets from them.
     bool preloaded;
@@ -170,6 +177,8 @@ struct vo_chain {
     // Last known input_mpi format (so vf can be reinitialized any time).
     struct mp_image_params input_format;
 
+    struct track *track;
+    struct lavfi_pad *filter_src;
     struct dec_video *video_src;
 
     // - video consists of a single picture, which should be shown only once
@@ -195,6 +204,8 @@ struct ao_chain {
     // Last known input_mpi format (so vf can be reinitialized any time).
     struct mp_audio input_format;
 
+    struct track *track;
+    struct lavfi_pad *filter_src;
     struct dec_audio *audio_src;
 };
 
@@ -294,6 +305,8 @@ typedef struct MPContext {
     // Currently, this is used for the secondary subtitle track only.
     struct track *current_track[NUM_PTRACKS][STREAM_TYPE_COUNT];
 
+    struct lavfi *lavfi;
+
     // Uses: accessing metadata (consider ordered chapters case, where the main
     // demuxer defines metadata), or special purpose demuxers like TV.
     struct demuxer *master_demuxer;
@@ -438,6 +451,8 @@ void clear_audio_output_buffers(struct MPContext *mpctx);
 void update_playback_speed(struct MPContext *mpctx);
 void uninit_audio_out(struct MPContext *mpctx);
 void uninit_audio_chain(struct MPContext *mpctx);
+int init_audio_decoder(struct MPContext *mpctx, struct track *track);
+void reinit_audio_chain_src(struct MPContext *mpctx, struct lavfi_pad *src);
 
 // configfiles.c
 void mp_parse_cfgfiles(struct MPContext *mpctx);
@@ -558,11 +573,13 @@ int video_vf_vo_control(struct vo_chain *vo_c, int vf_cmd, void *data);
 void reset_video_state(struct MPContext *mpctx);
 int init_video_decoder(struct MPContext *mpctx, struct track *track);
 int reinit_video_chain(struct MPContext *mpctx);
+int reinit_video_chain_src(struct MPContext *mpctx, struct lavfi_pad *src);
 int reinit_video_filters(struct MPContext *mpctx);
 void write_video(struct MPContext *mpctx, double endpts);
 void mp_force_video_refresh(struct MPContext *mpctx);
 void uninit_video_out(struct MPContext *mpctx);
 void uninit_video_chain(struct MPContext *mpctx);
 double calc_average_frame_duration(struct MPContext *mpctx);
+int init_video_decoder(struct MPContext *mpctx, struct track *track);
 
 #endif /* MPLAYER_MP_CORE_H */
diff --git a/player/lavfi.c b/player/lavfi.c
new file mode 100644
index 0000000000..118a766ea2
--- /dev/null
+++ b/player/lavfi.c
@@ -0,0 +1,722 @@
+/*
+ * This file is part of mpv.
+ *
+ * mpv is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include <inttypes.h>
+#include <stdarg.h>
+#include <assert.h>
+
+#include <libavutil/avstring.h>
+#include <libavutil/mem.h>
+#include <libavutil/mathematics.h>
+#include <libavutil/rational.h>
+#include <libavutil/error.h>
+#include <libavfilter/avfilter.h>
+#include <libavfilter/buffersink.h>
+#include <libavfilter/buffersrc.h>
+
+#include "common/common.h"
+#include "common/av_common.h"
+#include "common/msg.h"
+
+#include "audio/audio.h"
+#include "video/mp_image.h"
+#include "audio/fmt-conversion.h"
+#include "video/fmt-conversion.h"
+
+#include "lavfi.h"
+
+struct lavfi {
+    struct mp_log *log;
+    char *graph_string;
+
+    AVFilterGraph *graph;
+    // Set to true once all inputs have been initialized, and the graph is
+    // linked.
+    bool initialized;
+
+    // Set if all inputs have been marked as LAVFI_WAIT (except LAVFI_EOF pads).
+    bool all_waiting;
+
+    // Graph is draining to undo previously sent EOF. (If a stream leaves EOF
+    // state, the graph needs to be recreated to "unstuck" it.)
+    bool draining_recover_eof;
+    // Graph is draining for format changes.
+    bool draining_new_format;
+
+    // Filter can't be put into a working state.
+    bool failed;
+
+    struct lavfi_pad **pads;
+    int num_pads;
+};
+
+struct lavfi_pad {
+    struct lavfi *main;
+    enum stream_type type;
+    enum lavfi_direction dir;
+    char *name; // user-given pad name
+    AVFrame *tmp_frame;
+
+    bool connected;     // if false, inputs/otuputs are considered always EOF
+
+    AVFilterContext *filter;
+    int filter_pad;
+    // buffersrc or buffersink connected to filter/filter_pad
+    AVFilterContext *buffer;
+    AVRational timebase;
+    bool buffer_is_eof; // received/sent EOF to the buffer
+
+    // 1-frame queue (used for both input and output)
+    struct mp_image *pending_v;
+    struct mp_audio *pending_a;
+
+    // -- dir==LAVFI_IN
+
+    bool input_needed;  // filter has signaled it needs new input
+    bool input_waiting; // caller notified us that it will feed after a wakeup
+    bool input_again;   // caller wants us to feed data in the next iteration
+    bool input_eof;     // caller notified us that no input will come anymore
+
+    // used to check for format changes manually
+    struct mp_image_params in_fmt_v;
+    struct mp_audio in_fmt_a;
+
+    // -- dir==LAVFI_OUT
+
+    bool output_needed; // caller has signaled it needs new output
+    bool output_eof;    // last filter output was EOF
+};
+
+static void add_pad(struct lavfi *c, enum lavfi_direction dir, AVFilterInOut *item)
+{
+    int type = -1;
+    enum AVMediaType avmt;
+    if (dir == LAVFI_IN) {
+        avmt = avfilter_pad_get_type(item->filter_ctx->input_pads, item->pad_idx);
+    } else {
+        avmt = avfilter_pad_get_type(item->filter_ctx->output_pads, item->pad_idx);
+    }
+    switch (avmt) {
+    case AVMEDIA_TYPE_VIDEO: type = STREAM_VIDEO; break;
+    case AVMEDIA_TYPE_AUDIO: type = STREAM_AUDIO; break;
+    default: abort();
+    }
+
+    if (!item->name) {
+        MP_FATAL(c, "what the shit\n");
+        return;
+    }
+
+    struct lavfi_pad *p = lavfi_find_pad(c, item->name);
+    if (p) {
+        // Graph recreation case: reassociate an existing pad.
+        if (p->dir != dir || p->type != type) {
+            MP_FATAL(c, "pad '%s' changed type or direction\n", item->name);
+            return;
+        }
+    } else {
+        p = talloc_zero(c, struct lavfi_pad);
+        p->main = c;
+        p->dir = dir;
+        p->name = talloc_strdup(p, item->name);
+        p->tmp_frame = av_frame_alloc();
+        if (!p->tmp_frame)
+            abort();
+        p->type = type;
+        MP_TARRAY_APPEND(c, c->pads, c->num_pads, p);
+    }
+    p->filter = item->filter_ctx;
+    p->filter_pad = item->pad_idx;
+}
+
+static void add_pads(struct lavfi *c, enum lavfi_direction dir, AVFilterInOut *list)
+{
+    for (; list; list = list->next)
+        add_pad(c, dir, list);
+}
+
+// Parse the user-provided filter graph, and populate the unlinked filter pads.
+static void precreate_graph(struct lavfi *c)
+{
+    assert(!c->graph);
+    c->graph = avfilter_graph_alloc();
+    if (!c->graph)
+        abort();
+    AVFilterInOut *in = NULL, *out = NULL;
+    if (avfilter_graph_parse2(c->graph, c->graph_string, &in, &out) < 0) {
+        c->graph = NULL;
+        MP_FATAL(c, "parsing the filter graph failed\n");
+        c->failed = true;
+        return;
+    }
+    add_pads(c, LAVFI_IN, in);
+    add_pads(c, LAVFI_OUT, out);
+    avfilter_inout_free(&in);
+    avfilter_inout_free(&out);
+
+    // Now check for pads which could not be reassociated.
+    for (int n = 0; n < c->num_pads; n++) {
+        struct lavfi_pad *pad = c->pads[n];
+        // ok, not much we can do
+        if (!pad->filter)
+            MP_FATAL(c, "filter pad '%s' can not be reconnected\n", pad->name);
+    }
+}
+
+static void free_graph(struct lavfi *c)
+{
+    avfilter_graph_free(&c->graph);
+    for (int n = 0; n < c->num_pads; n++) {
+        struct lavfi_pad *pad = c->pads[n];
+        pad->filter = NULL;
+        pad->filter_pad = -1;
+        pad->buffer = NULL;
+        pad->in_fmt_v = (struct mp_image_params){0};
+        pad->in_fmt_a = (struct mp_audio){0};
+        pad->buffer_is_eof = false;
+        pad->input_needed = false;
+        pad->input_waiting = false;
+        pad->input_again = false;
+        pad->input_eof = false;
+        pad->output_needed = false;
+        pad->output_eof = false;
+    }
+    c->initialized = false;
+    c->all_waiting = false;
+    c->draining_recover_eof = false;
+    c->draining_new_format = false;
+}
+
+static void drop_pad_data(struct lavfi_pad *pad)
+{
+    talloc_free(pad->pending_a);
+    pad->pending_a = NULL;
+    talloc_free(pad->pending_v);
+    pad->pending_v = NULL;
+}
+
+static void clear_data(struct lavfi *c)
+{
+    for (int n = 0; n < c->num_pads; n++)
+        drop_pad_data(c->pads[n]);
+}
+
+void lavfi_seek_reset(struct lavfi *c)
+{
+    free_graph(c);
+    clear_data(c);
+    precreate_graph(c);
+}
+
+struct lavfi *lavfi_create(struct mp_log *log, char *graph_string)
+{
+    struct lavfi *c = talloc_zero(NULL, struct lavfi);
+    c->log = log;
+    c->graph_string = graph_string;
+    precreate_graph(c);
+    return c;
+}
+
+void lavfi_destroy(struct lavfi *c)
+{
+    free_graph(c);
+    clear_data(c);
+    talloc_free(c);
+}
+
+struct lavfi_pad *lavfi_find_pad(struct lavfi *c, char *name)
+{
+    for (int n = 0; n < c->num_pads; n++) {
+        if (strcmp(c->pads[n]->name, name) == 0)
+            return c->pads[n];
+    }
+    return NULL;
+}
+
+enum lavfi_direction lavfi_pad_direction(struct lavfi_pad *pad)
+{
+    return pad->dir;
+}
+
+enum stream_type lavfi_pad_type(struct lavfi_pad *pad)
+{
+    return pad->type;
+}
+
+void lavfi_set_connected(struct lavfi_pad *pad, bool connected)
+{
+    pad->connected = connected;
+}
+
+bool lavfi_get_connected(struct lavfi_pad *pad)
+{
+    return pad->connected;
+}
+
+// Ensure to send EOF to each input pad, so the graph can be drained properly.
+static void send_global_eof(struct lavfi *c)
+{
+    for (int n = 0; n < c->num_pads; n++) {
+        struct lavfi_pad *pad = c->pads[n];
+        if (!pad->buffer || pad->dir != LAVFI_IN || pad->buffer_is_eof)
+            continue;
+
+        if (av_buffersrc_add_frame(pad->buffer, NULL) < 0)
+            MP_FATAL(c, "could not send EOF to filter\n");
+
+        pad->buffer_is_eof = true;
+    }
+}
+
+// libavfilter allows changing some parameters on the fly, but not
+// others.
+static bool is_aformat_ok(struct mp_audio *a, struct mp_audio *b)
+{
+    return mp_audio_config_equals(a, b);
+}
+static bool is_vformat_ok(struct mp_image_params *a, struct mp_image_params *b)
+{
+    return a->imgfmt == b->imgfmt &&
+           a->w == b->w && a->h && b->h &&
+           a->p_w == b->p_w && a->p_h == b->p_h;
+}
+
+static void check_format_changes(struct lavfi *c)
+{
+    // check each pad for new input format
+    for (int n = 0; n < c->num_pads; n++) {
+        struct lavfi_pad *pad = c->pads[n];
+        if (!pad->buffer || pad->dir != LAVFI_IN)
+            continue;
+
+        if (pad->type == STREAM_AUDIO && pad->pending_a && pad->in_fmt_a.format) {
+            c->draining_new_format |= !is_aformat_ok(pad->pending_a,
+                                                     &pad->in_fmt_a);
+        }
+        if (pad->type == STREAM_VIDEO && pad->pending_v && pad->in_fmt_v.imgfmt) {
+            c->draining_new_format |= !is_vformat_ok(&pad->pending_v->params,
+                                                     &pad->in_fmt_v);
+        }
+    }
+
+    if (c->initialized && c->draining_new_format)
+        send_global_eof(c);
+}
+
+// Attempt to initialize all pads. Return true if all are initialized, or
+// false if more data is needed (or on error).
+static bool init_pads(struct lavfi *c)
+{
+    if (!c->graph)
+        goto error;
+
+    for (int n = 0; n < c->num_pads; n++) {
+        struct lavfi_pad *pad = c->pads[n];
+        if (pad->buffer)
+            continue;
+
+        if (!pad->filter)
+            goto error; // can happen if pad reassociation fails
+
+        if (pad->dir == LAVFI_OUT) {
+            AVFilter *dst_filter;
+            if (pad->type == STREAM_AUDIO) {
+                dst_filter = avfilter_get_by_name("abuffersink");
+            } else if (pad->type == STREAM_VIDEO) {
+                dst_filter = avfilter_get_by_name("buffersink");
+            } else {
+                assert(0);
+            }
+
+            char name[256];
+            snprintf(name, sizeof(name), "mpv_sink_%s", pad->name);
+
+            if (avfilter_graph_create_filter(&pad->buffer, dst_filter,
+                                             name, NULL, NULL, c->graph) < 0)
+                goto error;
+
+            if (avfilter_link(pad->filter, pad->filter_pad, pad->buffer, 0) < 0)
+                goto error;
+        } else {
+            char src_args[256];
+            AVFilter *src_filter;
+
+            pad->input_eof |= !pad->connected;
+
+            if (pad->pending_a) {
+                assert(pad->type == STREAM_AUDIO);
+                mp_audio_copy_config(&pad->in_fmt_a, pad->pending_a);
+            } else if (pad->pending_v) {
+                assert(pad->type == STREAM_VIDEO);
+                pad->in_fmt_v = pad->pending_v->params;
+            } else if (pad->input_eof) {
+                // libavfilter makes this painful. Init it with a dummy config,
+                // just so we can tell it the stream is EOF.
+                if (pad->type == STREAM_AUDIO) {
+                    mp_audio_set_format(&pad->in_fmt_a, AF_FORMAT_FLOAT);
+                    mp_audio_set_num_channels(&pad->in_fmt_a, 2);
+                    pad->in_fmt_a.rate = 48000;
+                } else if (pad->type == STREAM_VIDEO) {
+                    pad->in_fmt_v = (struct mp_image_params){
+                        .imgfmt = IMGFMT_420P,
+                        .w = 64, .h = 64,
+                        .p_w = 1, .p_h = 1,
+                    };
+                }
+            } else {
+                // no input data, format unknown, can't init, wait longer.
+                pad->input_needed = true;
+                return false;
+            }
+
+            if (pad->type == STREAM_AUDIO) {
+                pad->timebase = (AVRational){1, pad->in_fmt_a.rate};
+                snprintf(src_args, sizeof(src_args),
+                    "sample_rate=%d:sample_fmt=%s:time_base=%d/%d:"
+                    "channel_layout=0x%"PRIx64, pad->in_fmt_a.rate,
+                    av_get_sample_fmt_name(af_to_avformat(pad->in_fmt_a.format)),
+                    pad->timebase.num, pad->timebase.den,
+                    mp_chmap_to_lavc(&pad->in_fmt_a.channels));
+                src_filter = avfilter_get_by_name("abuffer");
+            } else if (pad->type == STREAM_VIDEO) {
+                pad->timebase = AV_TIME_BASE_Q;
+                snprintf(src_args, sizeof(src_args), "%d:%d:%d:%d:%d:%d:%d",
+                         pad->in_fmt_v.w, pad->in_fmt_v.h,
+                         imgfmt2pixfmt(pad->in_fmt_v.imgfmt),
+                         pad->timebase.num, pad->timebase.den,
+                         pad->in_fmt_v.p_w, pad->in_fmt_v.p_h);
+                src_filter = avfilter_get_by_name("buffer");
+            } else {
+                assert(0);
+            }
+
+            char name[256];
+            snprintf(name, sizeof(name), "mpv_src_%s", pad->name);
+
+            if (avfilter_graph_create_filter(&pad->buffer, src_filter,
+                                             name, src_args, NULL, c->graph) < 0)
+                goto error;
+
+            if (avfilter_link(pad->buffer, 0, pad->filter, pad->filter_pad) < 0)
+                goto error;
+        }
+    }
+
+    return true;
+error:
+    MP_FATAL(c, "could not initialize filter pads\n");
+    c->failed = true;
+    return false;
+}
+
+// Initialize the graph if all inputs have formats set. If it's already
+// initialized, or can't be initialized yet, do nothing.
+static void init_graph(struct lavfi *c)
+{
+    assert(!c->initialized);
+
+    if (init_pads(c)) {
+        // And here the actual libavfilter initialization happens.
+        if (avfilter_graph_config(c->graph, NULL) < 0) {
+            MP_FATAL(c, "failed to configure the filter graph\n");
+            free_graph(c);
+            c->failed = true;
+            return;
+        }
+
+        for (int n = 0; n < c->num_pads; n++) {
+            struct lavfi_pad *pad = c->pads[n];
+            if (pad->dir == LAVFI_OUT)
+                pad->timebase = pad->buffer->inputs[0]->time_base;
+        }
+
+        c->initialized = true;
+    }
+}
+
+static void feed_input_pads(struct lavfi *c)
+{
+    assert(c->initialized);
+
+    for (int n = 0; n < c->num_pads; n++) {
+        struct lavfi_pad *pad = c->pads[n];
+        if (pad->dir != LAVFI_IN)
+            continue;
+
+        pad->input_needed = false;
+        pad->input_eof |= !pad->connected;
+
+        if (!av_buffersrc_get_nb_failed_requests(pad->buffer))
+            continue;
+
+        if (c->draining_recover_eof || c->draining_new_format)
+            continue;
+
+        if (pad->buffer_is_eof)
+            continue;
+
+        AVFrame *frame = NULL;
+        double pts = 0;
+        bool eof = false;
+        if (pad->pending_v) {
+            pts = pad->pending_v->pts;
+            frame = mp_image_to_av_frame_and_unref(pad->pending_v);
+            pad->pending_v = NULL;
+        } else if (pad->pending_a) {
+            pts = pad->pending_a->pts;
+            frame = mp_audio_to_avframe_and_unref(pad->pending_a);
+            pad->pending_a = NULL;
+        } else {
+            if (!pad->input_eof) {
+                pad->input_needed = true;
+                continue;
+            }
+            eof = true;
+        }
+
+        if (!frame && !eof) {
+            MP_FATAL(c, "out of memory or unsupported format\n");
+            continue;
+        }
+
+        if (frame)
+            frame->pts = mp_pts_to_av(pts, &pad->timebase);
+
+        pad->buffer_is_eof = !frame;
+
+        if (av_buffersrc_add_frame(pad->buffer, frame) < 0)
+            MP_FATAL(c, "could not pass frame to filter\n");
+        av_frame_free(&frame);
+
+        pad->input_waiting = pad->input_again = false;
+        pad->input_eof = eof;
+    }
+}
+
+static void read_output_pads(struct lavfi *c)
+{
+    assert(c->initialized);
+
+    for (int n = 0; n < c->num_pads; n++) {
+        struct lavfi_pad *pad = c->pads[n];
+
+        if (pad->dir != LAVFI_OUT)
+            continue;
+
+        // If disconnected, read and discard everything.
+        if (!pad->pending_v && !pad->pending_a && !pad->connected)
+            pad->output_needed = true;
+
+        if (!pad->output_needed)
+            continue;
+
+        assert(pad->buffer);
+        assert(!pad->pending_v && !pad->pending_a);
+
+        int r = AVERROR(EAGAIN);
+        if (!pad->buffer_is_eof)
+            r = av_buffersink_get_frame(pad->buffer, pad->tmp_frame);
+        if (r >= 0) {
+            pad->output_needed = false;
+            double pts = mp_pts_from_av(pad->tmp_frame->pts, &pad->timebase);
+            if (pad->type == STREAM_AUDIO) {
+                pad->pending_a = mp_audio_from_avframe(pad->tmp_frame);
+                if (pad->pending_a)
+                    pad->pending_a->pts = pts;
+            } else if (pad->type == STREAM_VIDEO) {
+                pad->pending_v = mp_image_from_av_frame(pad->tmp_frame);
+                if (pad->pending_v)
+                    pad->pending_v->pts = pts;
+            } else {
+                assert(0);
+            }
+            av_frame_unref(pad->tmp_frame);
+            if (!pad->pending_v && !pad->pending_a)
+                MP_ERR(c, "could not use filter output\n");
+            pad->output_eof = false;
+            if (!pad->connected)
+                drop_pad_data(pad);
+        } else if (r == AVERROR(EAGAIN)) {
+            // We expect that libavfilter will request input on one of the
+            // input pads (via av_buffersrc_get_nb_failed_requests()).
+            pad->output_eof = false;
+        } else if (r == AVERROR_EOF) {
+            pad->buffer_is_eof = true;
+            if (!c->draining_recover_eof && !c->draining_new_format)
+                pad->output_eof = true;
+        } else {
+            // Real error - ignore it.
+            MP_ERR(c, "error on filtering (%d)\n", r);
+        }
+    }
+}
+
+// Process filter input and outputs. Return if progress was made (then the
+// caller should repeat). If it returns false, the caller should go to sleep
+// (as all inputs are asleep as well and no further output can be produced).
+bool lavfi_process(struct lavfi *c)
+{
+    check_format_changes(c);
+
+    if (!c->initialized)
+        init_graph(c);
+
+    if (c->initialized) {
+        read_output_pads(c);
+        feed_input_pads(c);
+    }
+
+    bool all_