From b9f804b566c4c528714e4ec5e63675ad7ba5fefd Mon Sep 17 00:00:00 2001 From: wm4 Date: Thu, 18 Jan 2018 14:44:20 +0100 Subject: audio: rewrite filtering glue code Use the new filtering code for audio too. --- filters/f_output_chain.c | 355 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 353 insertions(+), 2 deletions(-) (limited to 'filters/f_output_chain.c') diff --git a/filters/f_output_chain.c b/filters/f_output_chain.c index d98c7ca4b3..3cbbaa2ccb 100644 --- a/filters/f_output_chain.c +++ b/filters/f_output_chain.c @@ -1,3 +1,5 @@ +#include "audio/aframe.h" +#include "audio/out/ao.h" #include "common/global.h" #include "options/m_config.h" #include "options/m_option.h" @@ -41,6 +43,22 @@ struct chain { struct mp_autoconvert *convert; struct vo *vo; + struct ao *ao; + + struct mp_frame pending_input; + + // Some chain types (MP_OUTPUT_CHAIN_AUDIO) require draining the entire + // filter chain on format changes and further complex actions: + // 0: normal filtering + // 1: input changed, flushing out remaining frames from current filters + // 2: flushing finished + // 3: sent new frame through chain for format probing + // 4: sent EOF through chain for format probing + // 5: received format probing frame; now waiting for API user to call + // mp_output_chain_set_ao(). + int format_change_phase; + // True if it's a second run trying to see if downmix can be moved up. + bool format_change_second_try; struct mp_output_chain public; }; @@ -60,13 +78,20 @@ struct mp_user_filter { char *name; bool is_output_converter; bool is_input; + bool is_channelremix; struct mp_image_params last_out_params; + struct mp_aframe *last_out_aformat; + + int64_t last_in_pts, last_out_pts; bool failed; bool error_eof_sent; }; +static void recheck_channelremix_filter(struct chain *p); +static void remove_channelremix_filter(struct chain *p); + static void update_output_caps(struct chain *p) { if (p->type != MP_OUTPUT_CHAIN_VIDEO) @@ -119,6 +144,25 @@ static bool check_out_format_change(struct mp_user_filter *u, } } + if (frame.type == MP_FRAME_AUDIO) { + struct mp_aframe *aframe = frame.data; + + if (!mp_aframe_config_equals(aframe, u->last_out_aformat)) { + MP_VERBOSE(p, "[%s] %s\n", u->name, + mp_aframe_format_str(aframe)); + mp_aframe_config_copy(u->last_out_aformat, aframe); + + if (u->is_input) { + mp_aframe_config_copy(p->public.input_aformat, aframe); + } else if (u->is_output_converter) { + mp_aframe_config_copy(p->public.output_aformat, aframe); + } + + p->public.reconfig_happened = true; + changed = true; + } + } + return changed; } @@ -137,12 +181,15 @@ static void process_user(struct mp_filter *f) MP_FATAL(p, "Cannot convert decoder/filter output to any format " "supported by the output.\n"); p->public.failed_output_conversion = true; + p->format_change_phase = 0; mp_filter_wakeup(p->f); } else { MP_ERR(p, "Disabling filter %s because it has failed.\n", name); mp_filter_reset(u->f); // clear out staled buffered data } u->failed = true; + if (p->format_change_phase) + p->format_change_phase = 2; // redo without it } if (u->failed) { @@ -159,12 +206,35 @@ static void process_user(struct mp_filter *f) return; } - mp_pin_transfer_data(u->f->pins[0], f->ppins[0]); + if (mp_pin_can_transfer_data(u->f->pins[0], f->ppins[0])) { + struct mp_frame frame = mp_pin_out_read(f->ppins[0]); + + double pts = mp_frame_get_pts(frame); + if (pts != MP_NOPTS_VALUE) + u->last_in_pts = pts; + + mp_pin_in_write(u->f->pins[0], frame); + } if (mp_pin_can_transfer_data(f->ppins[1], u->f->pins[1])) { struct mp_frame frame = mp_pin_out_read(u->f->pins[1]); - check_out_format_change(u, frame); + bool changed = check_out_format_change(u, frame); + if (p->type == MP_OUTPUT_CHAIN_AUDIO && (!p->ao || changed) && + u->is_input && !p->format_change_phase) + { + // Format changed -> block filtering, start draining current filters. + MP_VERBOSE(p, "format changed, draining filter chain\n"); + mp_frame_unref(&p->pending_input); + p->pending_input = frame; + p->format_change_phase = 1; + mp_pin_in_write(f->ppins[1], MP_EOF_FRAME); + return; + } + + double pts = mp_frame_get_pts(frame); + if (pts != MP_NOPTS_VALUE) + u->last_out_pts = pts; mp_pin_in_write(f->ppins[1], frame); } @@ -175,6 +245,7 @@ static void reset_user(struct mp_filter *f) struct mp_user_filter *u = f->priv; u->error_eof_sent = false; + u->last_in_pts = u->last_out_pts = MP_NOPTS_VALUE; } static void destroy_user(struct mp_filter *f) @@ -203,6 +274,7 @@ static struct mp_user_filter *create_wrapper_filter(struct chain *p) struct mp_user_filter *wrapper = f->priv; wrapper->wrapper = f; wrapper->p = p; + wrapper->last_out_aformat = talloc_steal(wrapper, mp_aframe_create()); mp_filter_add_pin(f, MP_PIN_IN, "in"); mp_filter_add_pin(f, MP_PIN_OUT, "out"); return wrapper; @@ -237,10 +309,100 @@ static void relink_filter_list(struct chain *p) } } +// Special logic for draining on format changes (for audio). Never used or +// initiated video. +static void process_format_change(struct mp_filter *f) +{ + struct chain *p = f->priv; + + if (mp_pin_in_needs_data(p->filters_in)) { + if (p->format_change_phase == 2) { + MP_VERBOSE(p, "probing new format\n"); + // Clear any old state. + if (!p->format_change_second_try) { + mp_autoconvert_clear(p->convert); + remove_channelremix_filter(p); + } + for (int n = 0; n < p->num_all_filters; n++) + mp_filter_reset(p->all_filters[n]->f); + // Filter a copy of the new input frame to see what comes out. + struct mp_frame frame = mp_frame_ref(p->pending_input); + if (!frame.type) + abort(); + mp_pin_in_write(p->filters_in, frame); + mp_pin_out_request_data(p->filters_out); + p->format_change_phase = 3; + } else if (p->format_change_phase == 3) { + MP_VERBOSE(p, "probing new format (drain)\n"); + mp_pin_in_write(p->filters_in, MP_EOF_FRAME); + p->format_change_phase = 4; + } + } + + if (mp_pin_can_transfer_data(f->ppins[1], p->filters_out)) { + struct mp_frame frame = mp_pin_out_read(p->filters_out); + + if (frame.type == MP_FRAME_EOF) { + // We're apparently draining for a format change, and we got EOF + // from the chain, which means we're done draining. + if (p->format_change_phase == 1) { + MP_VERBOSE(p, "done format change draining\n"); + // Then we need to start probing the new format. + p->format_change_phase = 2; + mp_pin_out_request_data(p->filters_out); + } else if (!p->public.failed_output_conversion) { + MP_ERR(p, "we didn't get an output frame? (broken filter?)\n"); + } + mp_filter_internal_mark_progress(f); + return; + } + + if (p->format_change_phase >= 2) { + // We were filtering a "test" frame to probe the format. Now + // that we have it (apparently), just discard it, and make the + // user aware of the previously grabbed format. + MP_VERBOSE(p, "got output format from probing\n"); + mp_frame_unref(&frame); + for (int n = 0; n < p->num_all_filters; n++) + mp_filter_reset(p->all_filters[n]->f); + if (p->format_change_second_try) { + p->format_change_second_try = false; + p->format_change_phase = 0; + recheck_channelremix_filter(p); + } else { + p->ao = NULL; + p->public.ao_needs_update = true; + p->format_change_phase = 5; + } + // Do something silly to ensure the f_output_chain user gets + // notified properly. + mp_filter_wakeup(f); + return; + } + + // Draining remaining data. + mp_pin_in_write(f->ppins[1], frame); + } +} + + static void process(struct mp_filter *f) { struct chain *p = f->priv; + if (p->format_change_phase) { + process_format_change(f); + return; + } + + // Send remaining input from previous format change. + if (p->pending_input.type) { + if (mp_pin_in_needs_data(p->filters_in)) { + mp_pin_in_write(p->filters_in, p->pending_input); + p->pending_input = MP_NO_FRAME; + } + } + if (mp_pin_can_transfer_data(p->filters_in, f->ppins[0])) { struct mp_frame frame = mp_pin_out_read(f->ppins[0]); @@ -268,6 +430,14 @@ static void reset(struct mp_filter *f) { struct chain *p = f->priv; + // (if format initialization was in progress, this can be repeated next time) + mp_frame_unref(&p->pending_input); + p->format_change_phase = 0; + if (p->format_change_second_try) + remove_channelremix_filter(p); + p->format_change_second_try = false; + p->public.ao_needs_update = false; + p->public.got_input_eof = false; p->public.got_output_eof = false; } @@ -322,6 +492,120 @@ void mp_output_chain_set_vo(struct mp_output_chain *c, struct vo *vo) update_output_caps(p); } +// If there are any user filters, and they don't affect the channel config, +// then move upmix/downmix to the start of the chain. +static void maybe_move_up_channelremix(struct chain *p, struct mp_chmap *final) +{ + assert(p->num_all_filters >= 2); // at least in/convert filters + struct mp_user_filter *first = p->all_filters[0]; // "in" pseudo filter + struct mp_chmap in = {0}; + mp_aframe_get_chmap(first->last_out_aformat, &in); + if (mp_chmap_is_unknown(&in)) + return; + mp_chmap_reorder_to_lavc(&in); + if (!mp_chmap_is_valid(&in) || mp_chmap_equals_reordered(&in, final)) + return; + for (int n = 0; n < p->num_all_filters; n++) { + struct mp_user_filter *u = p->all_filters[n]; + struct mp_chmap chmap = {0}; + mp_aframe_get_chmap(u->last_out_aformat, &chmap); + if (!mp_chmap_equals_reordered(&in, &chmap)) + return; // some remix going in + } + + if (!p->num_user_filters) + return; // would be a NOP + + MP_VERBOSE(p, "trying with channel remixing moved to start of chain\n"); + + struct mp_user_filter *remix = create_wrapper_filter(p); + struct mp_autoconvert *convert = mp_autoconvert_create(remix->wrapper); + if (!convert) + abort(); + mp_autoconvert_add_chmap(convert, final); + remix->name = "channelremix"; + remix->f = convert->f; + remix->is_channelremix = true; + MP_TARRAY_APPEND(p, p->pre_filters, p->num_pre_filters, remix); + relink_filter_list(p); + + // now run the scary state machine again in order to see what filters do + // with the remixed channel data and if it was a good idea + p->format_change_phase = 2; + p->format_change_second_try = true; +} + +static void remove_channelremix_filter(struct chain *p) +{ + for (int n = 0; n < p->num_pre_filters; n++) { + struct mp_user_filter *u = p->pre_filters[n]; + if (u->is_channelremix) { + MP_TARRAY_REMOVE_AT(p->pre_filters, p->num_pre_filters, n); + talloc_free(u->wrapper); + relink_filter_list(p); + break; + } + } +} + +static void recheck_channelremix_filter(struct chain *p) +{ + struct mp_chmap in = {0}; + int start = -1; + for (int n = 0; n < p->num_all_filters; n++) { + struct mp_user_filter *u = p->all_filters[n]; + if (u->is_channelremix) { + mp_aframe_get_chmap(u->last_out_aformat, &in); + start = n; + break; + } + } + + if (start < 0 || !mp_chmap_is_valid(&in)) + goto remove; + + for (int n = start; n < p->num_all_filters; n++) { + struct mp_user_filter *u = p->all_filters[n]; + struct mp_chmap chmap = {0}; + mp_aframe_get_chmap(u->last_out_aformat, &chmap); + if (!mp_chmap_equals_reordered(&in, &chmap)) + goto remove; + } + + return; +remove: + MP_VERBOSE(p, "reverting moved up channel remixing\n"); + remove_channelremix_filter(p); +} + +void mp_output_chain_set_ao(struct mp_output_chain *c, struct ao *ao) +{ + struct chain *p = c->f->priv; + + assert(p->public.ao_needs_update); // can't just call it any time + assert(p->format_change_phase == 5); + assert(!p->format_change_second_try); + + p->public.ao_needs_update = false; + p->format_change_phase = 0; + + p->ao = ao; + + int out_format = 0; + int out_rate = 0; + struct mp_chmap out_channels = {0}; + ao_get_format(p->ao, &out_rate, &out_format, &out_channels); + + mp_autoconvert_clear(p->convert); + mp_autoconvert_add_afmt(p->convert, out_format); + mp_autoconvert_add_srate(p->convert, out_rate); + mp_autoconvert_add_chmap(p->convert, &out_channels); + + maybe_move_up_channelremix(p, &out_channels); + + mp_filter_wakeup(p->f); +} + static struct mp_user_filter *find_by_label(struct chain *p, const char *label) { for (int n = 0; n < p->num_user_filters; n++) { @@ -354,6 +638,57 @@ bool mp_output_chain_command(struct mp_output_chain *c, const char *target, return mp_filter_command(f->f, cmd); } +// Set the speed on the last filter in the chain that supports it. If a filter +// supports it, reset *speed, then keep setting the speed on the other filters. +// The purpose of this is to make sure only 1 filter changes speed. +static void set_speed_any(struct mp_user_filter **filters, int num_filters, + bool resample, double *speed) +{ + for (int n = num_filters - 1; n >= 0; n--) { + assert(*speed); + struct mp_filter_command cmd = { + .type = resample ? MP_FILTER_COMMAND_SET_SPEED_RESAMPLE + : MP_FILTER_COMMAND_SET_SPEED, + .speed = *speed, + }; + if (mp_filter_command(filters[n]->f, &cmd)) + *speed = 1.0; + } +} + +void mp_output_chain_set_audio_speed(struct mp_output_chain *c, + double speed, double resample) +{ + struct chain *p = c->f->priv; + + // We always resample with the final libavresample instance. + set_speed_any(p->post_filters, p->num_post_filters, true, &resample); + + // If users have filters like "scaletempo" insert anywhere, use that, + // otherwise use the builtin ones. + set_speed_any(p->user_filters, p->num_user_filters, false, &speed); + set_speed_any(p->post_filters, p->num_post_filters, false, &speed); +} + +double mp_output_get_measured_total_delay(struct mp_output_chain *c) +{ + struct chain *p = c->f->priv; + + double delay = 0; + + for (int n = 0; n < p->num_all_filters; n++) { + struct mp_user_filter *u = p->all_filters[n]; + + if (u->last_in_pts != MP_NOPTS_VALUE && + u->last_out_pts != MP_NOPTS_VALUE) + { + delay += u->last_in_pts - u->last_out_pts; + } + } + + return delay; +} + static bool compare_filter(struct m_obj_settings *a, struct m_obj_settings *b) { if (a == b || !a || !b) @@ -509,6 +844,18 @@ static void create_video_things(struct chain *p) MP_TARRAY_APPEND(p, p->post_filters, p->num_post_filters, f); } +static void create_audio_things(struct chain *p) +{ + p->frame_type = MP_FRAME_AUDIO; + + struct mp_user_filter *f = create_wrapper_filter(p); + f->name = "userspeed"; + f->f = mp_autoaspeed_create(f->wrapper); + if (!f->f) + abort(); + MP_TARRAY_APPEND(p, p->post_filters, p->num_post_filters, f); +} + struct mp_output_chain *mp_output_chain_create(struct mp_filter *parent, enum mp_output_chain_type type) { @@ -522,6 +869,7 @@ struct mp_output_chain *mp_output_chain_create(struct mp_filter *parent, const char *log_name = NULL; switch (type) { case MP_OUTPUT_CHAIN_VIDEO: log_name = "!vf"; break; + case MP_OUTPUT_CHAIN_AUDIO: log_name = "!af"; break; } if (log_name) f->log = mp_log_new(f, parent->global->log, log_name); @@ -533,6 +881,8 @@ struct mp_output_chain *mp_output_chain_create(struct mp_filter *parent, struct mp_output_chain *c = &p->public; c->f = f; + c->input_aformat = talloc_steal(p, mp_aframe_create()); + c->output_aformat = talloc_steal(p, mp_aframe_create()); // Dummy filter for reporting and logging the input format. p->input = create_wrapper_filter(p); @@ -545,6 +895,7 @@ struct mp_output_chain *mp_output_chain_create(struct mp_filter *parent, switch (type) { case MP_OUTPUT_CHAIN_VIDEO: create_video_things(p); break; + case MP_OUTPUT_CHAIN_AUDIO: create_audio_things(p); break; } p->output = create_wrapper_filter(p); -- cgit v1.2.3