From 76276c92104c31ee936ba5c76a76072f09978c5f Mon Sep 17 00:00:00 2001 From: wm4 Date: Tue, 16 Jan 2018 11:53:44 +0100 Subject: video: rewrite filtering glue code Get rid of the old vf.c code. Replace it with a generic filtering framework, which can potentially handle more than just --vf. At least reimplementing --af with this code is planned. This changes some --vf semantics (including runtime behavior and the "vf" command). The most important ones are listed in interface-changes. vf_convert.c is renamed to f_swscale.c. It is now an internal filter that can not be inserted by the user manually. f_lavfi.c is a refactor of player/lavfi.c. The latter will be removed once --lavfi-complex is reimplemented on top of f_lavfi.c. (which is conceptually easy, but a big mess due to the data flow changes). The existing filters are all changed heavily. The data flow of the new filter framework is different. Especially EOF handling changes - EOF is now a "frame" rather than a state, and must be passed through exactly once. Another major thing is that all filters must support dynamic format changes. The filter reconfig() function goes away. (This sounds complex, but since all filters need to handle EOF draining anyway, they can use the same code, and it removes the mess with reconfig() having to predict the output format, which completely breaks with libavfilter anyway.) In addition, there is no automatic format negotiation or conversion. libavfilter's primitive and insufficient API simply doesn't allow us to do this in a reasonable way. Instead, filters can use f_autoconvert as sub-filter, and tell it which formats they support. This filter will in turn add actual conversion filters, such as f_swscale, to perform necessary format changes. vf_vapoursynth.c uses the same basic principle of operation as before, but with worryingly different details in data flow. Still appears to work. The hardware deint filters (vf_vavpp.c, vf_d3d11vpp.c, vf_vdpaupp.c) are heavily changed. Fortunately, they all used refqueue.c, which is for sharing the data flow logic (especially for managing future/past surfaces and such). It turns out it can be used to factor out most of the data flow. Some of these filters accepted software input. Instead of having ad-hoc upload code in each filter, surface upload is now delegated to f_autoconvert, which can use f_hwupload to perform this. Exporting VO capabilities is still a big mess (mp_stream_info stuff). The D3D11 code drops the redundant image formats, and all code uses the hw_subfmt (sw_format in FFmpeg) instead. Although that too seems to be a big mess for now. f_async_queue is unused. --- DOCS/interface-changes.rst | 7 + DOCS/man/vf.rst | 4 +- common/common.c | 11 + common/common.h | 2 + filters/f_auto_filters.c | 244 +++++++++ filters/f_auto_filters.h | 10 + filters/f_autoconvert.c | 288 +++++++++++ filters/f_autoconvert.h | 39 ++ filters/f_hwtransfer.c | 299 +++++++++++ filters/f_hwtransfer.h | 32 ++ filters/f_lavfi.c | 952 +++++++++++++++++++++++++++++++++++ filters/f_lavfi.h | 30 ++ filters/f_output_chain.c | 564 +++++++++++++++++++++ filters/f_output_chain.h | 59 +++ filters/f_swscale.c | 148 ++++++ filters/f_swscale.h | 25 + filters/f_utils.c | 175 +++++++ filters/f_utils.h | 72 +++ filters/filter.c | 790 +++++++++++++++++++++++++++++ filters/filter.h | 379 ++++++++++++++ filters/filter_internal.h | 144 ++++++ filters/frame.c | 179 +++++++ filters/frame.h | 55 ++ filters/user_filters.c | 119 +++++ filters/user_filters.h | 29 ++ options/m_option.h | 1 - options/options.c | 14 +- options/options.h | 7 +- player/command.c | 50 +- player/core.h | 12 +- player/loadfile.c | 3 + player/playloop.c | 5 +- player/sub.c | 3 +- player/video.c | 292 +++-------- video/d3d.c | 3 - video/filter/refqueue.c | 192 ++++++- video/filter/refqueue.h | 18 +- video/filter/vf.c | 797 ----------------------------- video/filter/vf.h | 179 ------- video/filter/vf_convert.c | 133 ----- video/filter/vf_d3d11vpp.c | 318 ++++++------ video/filter/vf_format.c | 135 ++--- video/filter/vf_lavfi.c | 517 ------------------- video/filter/vf_sub.c | 177 ++++--- video/filter/vf_vapoursynth.c | 682 ++++++++++++------------- video/filter/vf_vavpp.c | 291 +++++------ video/filter/vf_vdpaupp.c | 175 +++---- video/fmt-conversion.c | 3 +- video/hwdec.c | 25 +- video/hwdec.h | 10 +- video/img_format.c | 10 - video/img_format.h | 9 +- video/out/d3d11/hwdec_d3d11va.c | 5 +- video/out/opengl/hwdec_d3d11egl.c | 5 +- video/out/opengl/hwdec_d3d11eglrgb.c | 10 +- wscript_build.py | 14 +- 56 files changed, 5887 insertions(+), 2864 deletions(-) create mode 100644 filters/f_auto_filters.c create mode 100644 filters/f_auto_filters.h create mode 100644 filters/f_autoconvert.c create mode 100644 filters/f_autoconvert.h create mode 100644 filters/f_hwtransfer.c create mode 100644 filters/f_hwtransfer.h create mode 100644 filters/f_lavfi.c create mode 100644 filters/f_lavfi.h create mode 100644 filters/f_output_chain.c create mode 100644 filters/f_output_chain.h create mode 100644 filters/f_swscale.c create mode 100644 filters/f_swscale.h create mode 100644 filters/f_utils.c create mode 100644 filters/f_utils.h create mode 100644 filters/filter.c create mode 100644 filters/filter.h create mode 100644 filters/filter_internal.h create mode 100644 filters/frame.c create mode 100644 filters/frame.h create mode 100644 filters/user_filters.c create mode 100644 filters/user_filters.h delete mode 100644 video/filter/vf.c delete mode 100644 video/filter/vf.h delete mode 100644 video/filter/vf_convert.c delete mode 100644 video/filter/vf_lavfi.c diff --git a/DOCS/interface-changes.rst b/DOCS/interface-changes.rst index 2b36506920..2379470277 100644 --- a/DOCS/interface-changes.rst +++ b/DOCS/interface-changes.rst @@ -36,6 +36,13 @@ Interface changes - deprecate --af=lavrresample. Use the ``--audio-resample-...`` options to customize resampling, or the libavfilter ``--af=aresample`` filter. - add --osd-on-seek + - remove outfmt sub-parameter from "format" video filter (no replacement) + - some behavior changes in the video filter chain, including: + - before, using an incompatible filter with hwdec would disable hwdec; + now it disables the filter at runtime instead + - inserting an incompatible filter with hwdec at runtime would refuse + to insert the filter; now it will add it successfully, but disables + the filter slightly later --- mpv 0.28.0 --- - rename --hwdec=mediacodec option to mediacodec-copy, to reflect conventions followed by other hardware video decoding APIs diff --git a/DOCS/man/vf.rst b/DOCS/man/vf.rst index 5f953b2204..13e80014c4 100644 --- a/DOCS/man/vf.rst +++ b/DOCS/man/vf.rst @@ -110,9 +110,7 @@ Available mpv-only filters are: ```` Format name, e.g. rgb15, bgr24, 420p, etc. (default: don't change). - ```` - Format name that should be substituted for the output. If they do not - have the same bytes per pixel and chroma subsampling, it will fail. + ```` Controls the YUV to RGB color space conversion when playing video. There are various standards. Normally, BT.601 should be used for SD video, and diff --git a/common/common.c b/common/common.c index 9ed63e1783..12e8e141ec 100644 --- a/common/common.c +++ b/common/common.c @@ -302,3 +302,14 @@ char *mp_tprintf_buf(char *buf, size_t buf_size, const char *format, ...) va_end(ap); return buf; } + +char **mp_dup_str_array(void *tctx, char **s) +{ + char **r = NULL; + int num_r = 0; + for (int n = 0; s && s[n]; n++) + MP_TARRAY_APPEND(tctx, r, num_r, talloc_strdup(tctx, s[n])); + if (r) + MP_TARRAY_APPEND(tctx, r, num_r, NULL); + return r; +} diff --git a/common/common.h b/common/common.h index 8dd02026f6..224a6e023a 100644 --- a/common/common.h +++ b/common/common.h @@ -112,4 +112,6 @@ char *mp_tag_str_buf(char *buf, size_t buf_size, uint32_t tag); char *mp_tprintf_buf(char *buf, size_t buf_size, const char *format, ...) PRINTF_ATTRIBUTE(3, 4); +char **mp_dup_str_array(void *tctx, char **s); + #endif /* MPLAYER_MPCOMMON_H */ diff --git a/filters/f_auto_filters.c b/filters/f_auto_filters.c new file mode 100644 index 0000000000..eac6f745ca --- /dev/null +++ b/filters/f_auto_filters.c @@ -0,0 +1,244 @@ +#include + +#include "common/common.h" +#include "common/msg.h" +#include "options/m_config.h" +#include "options/options.h" +#include "video/mp_image.h" + +#include "f_auto_filters.h" +#include "f_swscale.h" +#include "f_utils.h" +#include "filter.h" +#include "filter_internal.h" +#include "user_filters.h" + +struct deint_priv { + struct mp_subfilter sub; + int prev_imgfmt; + int prev_setting; + struct m_config_cache *opts; +}; + +static void deint_process(struct mp_filter *f) +{ + struct deint_priv *p = f->priv; + + if (!mp_subfilter_read(&p->sub)) + return; + + struct mp_frame frame = p->sub.frame; + + if (mp_frame_is_signaling(frame)) { + mp_subfilter_continue(&p->sub); + return; + } + + if (frame.type != MP_FRAME_VIDEO) { + MP_ERR(f, "video input required!\n"); + mp_filter_internal_mark_failed(f); + return; + } + + m_config_cache_update(p->opts); + struct filter_opts *opts = p->opts->opts; + + if (!opts->deinterlace) + mp_subfilter_destroy(&p->sub); + + struct mp_image *img = frame.data; + + if (img->imgfmt == p->prev_imgfmt && p->prev_setting == opts->deinterlace) { + mp_subfilter_continue(&p->sub); + return; + } + + if (!mp_subfilter_drain_destroy(&p->sub)) + return; + + assert(!p->sub.filter); + + p->prev_imgfmt = img->imgfmt; + p->prev_setting = opts->deinterlace; + if (!p->prev_setting) { + mp_subfilter_continue(&p->sub); + return; + } + + if (img->imgfmt == IMGFMT_VDPAU) { + char *args[] = {"deint", "yes", NULL}; + p->sub.filter = + mp_create_user_filter(f, MP_OUTPUT_CHAIN_VIDEO, "vdpaupp", args); + } else if (img->imgfmt == IMGFMT_VAAPI) { + p->sub.filter = + mp_create_user_filter(f, MP_OUTPUT_CHAIN_VIDEO, "vavpp", NULL); + } else if (img->imgfmt == IMGFMT_D3D11) { + p->sub.filter = + mp_create_user_filter(f, MP_OUTPUT_CHAIN_VIDEO, "d3d11vpp", NULL); + } else if (mp_sws_supports_input(img->imgfmt)) { + char *args[] = {"mode", "send_field", "deint", "interlaced", NULL}; + p->sub.filter = + mp_create_user_filter(f, MP_OUTPUT_CHAIN_VIDEO, "yadif", args); + } else { + MP_ERR(f, "no deinterlace filter available for this format\n"); + mp_subfilter_continue(&p->sub); + return; + } + + if (!p->sub.filter) + MP_ERR(f, "creating deinterlacer failed\n"); + + mp_subfilter_continue(&p->sub); +} + +static void deint_reset(struct mp_filter *f) +{ + struct deint_priv *p = f->priv; + + mp_subfilter_reset(&p->sub); +} + +static void deint_destroy(struct mp_filter *f) +{ + struct deint_priv *p = f->priv; + + mp_subfilter_reset(&p->sub); + TA_FREEP(&p->sub.filter); +} + +static const struct mp_filter_info deint_filter = { + .name = "deint", + .priv_size = sizeof(struct deint_priv), + .process = deint_process, + .reset = deint_reset, + .destroy = deint_destroy, +}; + +struct mp_filter *mp_deint_create(struct mp_filter *parent) +{ + struct mp_filter *f = mp_filter_create(parent, &deint_filter); + if (!f) + return NULL; + + struct deint_priv *p = f->priv; + + p->sub.in = mp_filter_add_pin(f, MP_PIN_IN, "in"); + p->sub.out = mp_filter_add_pin(f, MP_PIN_OUT, "out"); + + p->opts = m_config_cache_alloc(f, f->global, &filter_conf); + + return f; +} + +struct rotate_priv { + struct mp_subfilter sub; + int prev_rotate; + int prev_imgfmt; + int target_rotate; +}; + +static void rotate_process(struct mp_filter *f) +{ + struct rotate_priv *p = f->priv; + + if (!mp_subfilter_read(&p->sub)) + return; + + struct mp_frame frame = p->sub.frame; + + if (mp_frame_is_signaling(frame)) { + mp_subfilter_continue(&p->sub); + return; + } + + if (frame.type != MP_FRAME_VIDEO) { + MP_ERR(f, "video input required!\n"); + return; + } + + struct mp_image *img = frame.data; + + if (img->params.rotate == p->prev_rotate && + img->imgfmt == p->prev_imgfmt) + { + img->params.rotate = p->target_rotate; + mp_subfilter_continue(&p->sub); + return; + } + + if (!mp_subfilter_drain_destroy(&p->sub)) + return; + + assert(!p->sub.filter); + + int rotate = p->prev_rotate = img->params.rotate; + p->target_rotate = rotate; + p->prev_imgfmt = img->imgfmt; + + struct mp_stream_info *info = mp_filter_find_stream_info(f); + if (rotate == 0 || (info && info->rotate90 && !(rotate % 90))) { + mp_subfilter_continue(&p->sub); + return; + } + + if (mp_sws_supports_input(img->imgfmt)) { + MP_ERR(f, "Video rotation with this format not supported\n"); + mp_subfilter_continue(&p->sub); + return; + } + + double angle = rotate / 360.0 * M_PI * 2; + char *args[] = {"angle", mp_tprintf(30, "%f", angle), + "ow", mp_tprintf(30, "rotw(%f)", angle), + "oh", mp_tprintf(30, "roth(%f)", angle), + NULL}; + p->sub.filter = + mp_create_user_filter(f, MP_OUTPUT_CHAIN_VIDEO, "rotate", args); + + if (p->sub.filter) { + MP_INFO(f, "Inserting rotation filter.\n"); + p->target_rotate = 0; + } else { + MP_ERR(f, "could not create rotation filter\n"); + } + + mp_subfilter_continue(&p->sub); +} + +static void rotate_reset(struct mp_filter *f) +{ + struct rotate_priv *p = f->priv; + + mp_subfilter_reset(&p->sub); +} + +static void rotate_destroy(struct mp_filter *f) +{ + struct rotate_priv *p = f->priv; + + mp_subfilter_reset(&p->sub); + TA_FREEP(&p->sub.filter); +} + +static const struct mp_filter_info rotate_filter = { + .name = "autorotate", + .priv_size = sizeof(struct rotate_priv), + .process = rotate_process, + .reset = rotate_reset, + .destroy = rotate_destroy, +}; + +struct mp_filter *mp_autorotate_create(struct mp_filter *parent) +{ + struct mp_filter *f = mp_filter_create(parent, &rotate_filter); + if (!f) + return NULL; + + struct rotate_priv *p = f->priv; + p->prev_rotate = -1; + + p->sub.in = mp_filter_add_pin(f, MP_PIN_IN, "in"); + p->sub.out = mp_filter_add_pin(f, MP_PIN_OUT, "out"); + + return f; +} diff --git a/filters/f_auto_filters.h b/filters/f_auto_filters.h new file mode 100644 index 0000000000..5f1a99f636 --- /dev/null +++ b/filters/f_auto_filters.h @@ -0,0 +1,10 @@ +#pragma once + +#include "filter.h" + +// A filter which inserts the required deinterlacing filter based on the +// hardware decode mode and the deinterlace user option. +struct mp_filter *mp_deint_create(struct mp_filter *parent); + +// Rotate according to mp_image.rotate and VO capabilities. +struct mp_filter *mp_autorotate_create(struct mp_filter *parent); diff --git a/filters/f_autoconvert.c b/filters/f_autoconvert.c new file mode 100644 index 0000000000..687a846ae5 --- /dev/null +++ b/filters/f_autoconvert.c @@ -0,0 +1,288 @@ +#include "config.h" + +#include "common/common.h" +#include "common/msg.h" +#include "video/hwdec.h" +#include "video/mp_image.h" + +#include "f_autoconvert.h" +#include "f_hwtransfer.h" +#include "f_swscale.h" +#include "f_utils.h" +#include "filter.h" +#include "filter_internal.h" + +struct priv { + struct mp_log *log; + + struct mp_subfilter sub; + + bool force_update; + + int *imgfmts; + int *subfmts; + int num_imgfmts; + + // Enable special conversion for the final stage before the VO. + bool vo_convert; + + // sws state + int in_imgfmt, in_subfmt; + + struct mp_autoconvert public; +}; + +// Dummy filter for bundling sub-conversion filters. +static const struct mp_filter_info convert_filter = { + .name = "convert", +}; + +// For hw decoding: thing which can convert between underlying surface formats. +// The filter detects the needed target format from struct mp_hwdec_ctx. +struct subfmt_conv { + int hw_imgfmt; + struct mp_filter *(*create)(struct mp_filter *parent); +}; + +static const struct subfmt_conv subfmt_converters[] = { +#if HAVE_D3D_HWACCEL + {IMGFMT_D3D11, vf_d3d11_create_outconv}, +#endif + {0} +}; + +void mp_autoconvert_clear(struct mp_autoconvert *c) +{ + struct priv *p = c->f->priv; + + p->num_imgfmts = 0; +} + +void mp_autoconvert_add_imgfmt(struct mp_autoconvert *c, int imgfmt, int subfmt) +{ + struct priv *p = c->f->priv; + + MP_TARRAY_GROW(p, p->imgfmts, p->num_imgfmts); + MP_TARRAY_GROW(p, p->subfmts, p->num_imgfmts); + + p->imgfmts[p->num_imgfmts] = imgfmt; + p->subfmts[p->num_imgfmts] = subfmt; + + p->num_imgfmts += 1; + p->force_update = true; +} + +void mp_autoconvert_add_vo_hwdec_subfmts(struct mp_autoconvert *c, + struct mp_hwdec_devices *devs) +{ + struct priv *p = c->f->priv; + assert(devs); + + int prev_format = 0; + + for (int n = 0; ; n++) { + struct mp_hwdec_ctx *ctx = hwdec_devices_get_n(devs, n); + if (!ctx) + break; + if (!ctx->hw_imgfmt || !ctx->supported_formats) + continue; + // Very hacky: don't let d3d11-egl-rgb overwrite d3d11-egl + if (ctx->hw_imgfmt == prev_format) + continue; + prev_format = ctx->hw_imgfmt; + // Stupidity: VOs export imgfmt only, so subfmt is always 0. Remove it + // to fix it up. + for (int i = 0; i < p->num_imgfmts; i++) { + if (p->imgfmts[i] != ctx->hw_imgfmt) + continue; + + int count = p->num_imgfmts; + MP_TARRAY_REMOVE_AT(p->imgfmts, count, i); + count = p->num_imgfmts; + MP_TARRAY_REMOVE_AT(p->subfmts, count, i); + p->num_imgfmts -= 1; + break; + } + for (int i = 0; ctx->supported_formats[i]; i++) + mp_autoconvert_add_imgfmt(c, ctx->hw_imgfmt, ctx->supported_formats[i]); + } + + p->vo_convert = true; +} + +static void handle_video_frame(struct mp_filter *f) +{ + struct priv *p = f->priv; + + struct mp_frame frame = p->sub.frame; + if (frame.type != MP_FRAME_VIDEO) { + MP_ERR(p, "video input required!\n"); + mp_filter_internal_mark_failed(f); + return; + } + + struct mp_image *img = frame.data; + + if (p->force_update) + p->in_imgfmt = p->in_subfmt = 0; + + if (img->imgfmt == p->in_imgfmt && img->params.hw_subfmt == p->in_subfmt) { + mp_subfilter_continue(&p->sub); + return; + } + + if (!mp_subfilter_drain_destroy(&p->sub)) { + p->in_imgfmt = p->in_subfmt = 0; + return; + } + + p->in_imgfmt = img->params.imgfmt; + p->in_subfmt = img->params.hw_subfmt; + p->force_update = false; + + bool different_subfmt = false; + + for (int n = 0; n < p->num_imgfmts; n++) { + bool samefmt = img->params.imgfmt == p->imgfmts[n]; + bool samesubffmt = img->params.hw_subfmt == p->subfmts[n]; + if (samefmt && !samesubffmt) + different_subfmt = true; + if (samefmt && (samesubffmt || !p->subfmts[n])) { + mp_subfilter_continue(&p->sub); + return; + } + } + + struct mp_stream_info *info = mp_filter_find_stream_info(f); + + struct mp_filter *conv = mp_filter_create(f, &convert_filter); + mp_filter_add_pin(conv, MP_PIN_IN, "in"); + mp_filter_add_pin(conv, MP_PIN_OUT, "out"); + + struct mp_filter *filters[2] = {0}; + bool need_sws = true; + + int *fmts = p->imgfmts; + int num_fmts = p->num_imgfmts; + + // Source is sw, all targets are hw -> try to upload. + bool sw_to_hw = !IMGFMT_IS_HWACCEL(img->imgfmt); + for (int n = 0; n < num_fmts; n++) + sw_to_hw &= IMGFMT_IS_HWACCEL(fmts[n]); + + if (sw_to_hw && num_fmts > 0) { + // We can probably use this! Very lazy and very approximate. + struct mp_hwupload *upload = mp_hwupload_create(conv, fmts[0]); + if (upload) { + MP_INFO(p, "HW-uploading to %s\n", mp_imgfmt_to_name(fmts[0])); + filters[1] = upload->f; + fmts = upload->upload_fmts; + num_fmts = upload->num_upload_fmts; + } + } else if (p->vo_convert && different_subfmt && info && info->hwdec_devs) { + for (int n = 0; subfmt_converters[n].hw_imgfmt; n++) { + if (subfmt_converters[n].hw_imgfmt == img->imgfmt) { + MP_INFO(p, "Using HW sub-conversion.\n"); + filters[1] = subfmt_converters[n].create(conv); + if (filters[1]) { + need_sws = false; + break; + } + } + } + } + + if (need_sws) { + // Create a new conversion filter. + struct mp_sws_filter *sws = mp_sws_filter_create(conv); + if (!sws) { + MP_ERR(p, "error creating conversion filter\n"); + return; + } + + int out = mp_sws_find_best_out_format(img->imgfmt, fmts, num_fmts); + if (!out) { + MP_ERR(p, "can't find video conversion for %s/%s\n", + mp_imgfmt_to_name(img->imgfmt), + mp_imgfmt_to_name(img->params.hw_subfmt)); + talloc_free(conv); + mp_filter_internal_mark_failed(f); + return; + } + + if (out == img->imgfmt) { + // Can happen if hwupload goes to same format. + talloc_free(sws->f); + } else { + sws->out_format = out; + MP_INFO(p, "Converting %s -> %s\n", mp_imgfmt_to_name(img->imgfmt), + mp_imgfmt_to_name(sws->out_format)); + filters[0] = sws->f; + } + } + + mp_chain_filters(conv->ppins[0], conv->ppins[1], filters, 2); + + p->sub.filter = conv; + mp_subfilter_continue(&p->sub); +} + +static void process(struct mp_filter *f) +{ + struct priv *p = f->priv; + + if (!mp_subfilter_read(&p->sub)) + return; + + struct mp_frame frame = p->sub.frame; + + if (!mp_frame_is_signaling(frame)) { + if (p->num_imgfmts) { + handle_video_frame(f); + return; + } + } + + mp_subfilter_continue(&p->sub); +} + +static void reset(struct mp_filter *f) +{ + struct priv *p = f->priv; + + mp_subfilter_reset(&p->sub); +} + +static void destroy(struct mp_filter *f) +{ + struct priv *p = f->priv; + + mp_subfilter_reset(&p->sub); + TA_FREEP(&p->sub.filter); +} + +static const struct mp_filter_info autoconvert_filter = { + .name = "autoconvert", + .priv_size = sizeof(struct priv), + .process = process, + .reset = reset, + .destroy = destroy, +}; + +struct mp_autoconvert *mp_autoconvert_create(struct mp_filter *parent) +{ + struct mp_filter *f = mp_filter_create(parent, &autoconvert_filter); + if (!f) + return NULL; + + mp_filter_add_pin(f, MP_PIN_IN, "in"); + mp_filter_add_pin(f, MP_PIN_OUT, "out"); + + struct priv *p = f->priv; + p->public.f = f; + p->log = f->log; + p->sub.in = f->ppins[0]; + p->sub.out = f->ppins[1]; + + return &p->public; +} diff --git a/filters/f_autoconvert.h b/filters/f_autoconvert.h new file mode 100644 index 0000000000..72af21a0df --- /dev/null +++ b/filters/f_autoconvert.h @@ -0,0 +1,39 @@ +#pragma once + +#include "filter.h" + +// A filter which automatically creates and uses a conversion filter based on +// the filter settings, or passes through data unchanged if no conversion is +// required. +struct mp_autoconvert { + // f->pins[0] is input, f->pins[1] is output + struct mp_filter *f; +}; + +// (to free this, free the filter itself, mp_autoconvert.f) +struct mp_autoconvert *mp_autoconvert_create(struct mp_filter *parent); + +// Add the imgfmt as allowed video image format, and error on non-video frames. +// Each call adds to the list of allowed formats. Before the first call, all +// formats are allowed (even non-video). +// subfmt can be used to specify underlying surface formats for hardware formats, +// otherwise must be 0. +void mp_autoconvert_add_imgfmt(struct mp_autoconvert *c, int imgfmt, int subfmt); + +// Add the formats supported by the hwdec interops (or essentially refine them), +// and trigger conversion if hw_subfmts mismatch. This is mostly a hack for +// D3D11/ANGLE (which supports NV12 only). +// Must be called mp_autoconvert_add_imgfmt(), and overrides them where formats +// collide. +struct mp_hwdec_devices; +void mp_autoconvert_add_vo_hwdec_subfmts(struct mp_autoconvert *c, + struct mp_hwdec_devices *devs); + +// Reset set of allowed formats back to initial state. (This does not flush +// any frames or remove currently active filters, although to get reasonable +// behavior, you need to readd all previously allowed formats, or reset the +// filter.) +void mp_autoconvert_clear(struct mp_autoconvert *c); + +// vf_d3d11vpp.c +struct mp_filter *vf_d3d11_create_outconv(struct mp_filter *parent); diff --git a/filters/f_hwtransfer.c b/filters/f_hwtransfer.c new file mode 100644 index 0000000000..6ffda567ae --- /dev/null +++ b/filters/f_hwtransfer.c @@ -0,0 +1,299 @@ +#include +#include +#include + +#include "video/fmt-conversion.h" +#include "video/hwdec.h" +#include "video/mp_image.h" +#include "video/mp_image_pool.h" + +#include "f_hwtransfer.h" +#include "filter_internal.h" + +struct priv { + AVBufferRef *av_device_ctx; + + AVBufferRef *hw_pool; + + int last_input_fmt; + int last_upload_fmt; + int last_sw_fmt; + + struct mp_hwupload public; +}; + +static bool update_format_decision(struct priv *p, int input_fmt) +{ + struct mp_hwupload *u = &p->public; + + if (!input_fmt) + return false; + + if (input_fmt == p->last_input_fmt) + return true; + + p->last_input_fmt = 0; + + int res = mp_imgfmt_select_best_list(u->upload_fmts, u->num_upload_fmts, + input_fmt); + + if (!res) + return false; + + // Find which sw format we should use. + // NOTE: if there are ever any hw APIs that actually do expensive + // conversions on mismatching format uploads, we should probably first look + // which sw format is preferred? + int index = -1; + for (int n = 0; n < u->num_upload_fmts; n++) { + if (u->upload_fmts[n] == res) + index = n; + } + + if (index < 0) + return false; + + for (int n = 0; n < u->num_fmts; n++) { + if (u->fmt_upload_index[n] >= index && + index < u->fmt_upload_index[n] + u->fmt_upload_num[n]) + { + p->last_input_fmt = input_fmt; + p->last_upload_fmt = u->upload_fmts[index]; + p->last_sw_fmt = u->fmts[n]; + MP_INFO(u->f, "upload %s -> %s\n", + mp_imgfmt_to_name(p->last_sw_fmt), + mp_imgfmt_to_name(p->last_input_fmt)); + return true; + } + } + + return false; +} + +int mp_hwupload_find_upload_format(struct mp_hwupload *u, int imgfmt) +{ + struct priv *p = u->f->priv; + + if (!update_format_decision(p, imgfmt)) + return 0; + return p->last_upload_fmt; +} + +static void process(struct mp_filter *f) +{ + struct priv *p = f->priv; + + if (!mp_pin_can_transfer_data(f->ppins[1], f->ppins[0])) + return; + + struct mp_frame frame = mp_pin_out_read(f->ppins[0]); + if (mp_frame_is_signaling(frame)) { + mp_pin_in_write(f->ppins[1], frame); + return; + } + if (frame.type != MP_FRAME_VIDEO) { + MP_ERR(f, "unsupported frame type\n"); + goto error; + } + struct mp_image *src = frame.data; + + // As documented, just pass though HW frames. + if (IMGFMT_IS_HWACCEL(src->imgfmt)) { + mp_pin_in_write(f->ppins[1], frame); + return; + } + + if (src->w % 2 || src->h % 2) { + MP_ERR(f, "non-mod 2 input frames unsupported\n"); + goto error; + } + + if (!update_format_decision(p, src->imgfmt)) { + MP_ERR(f, "no hw upload format found\n"); + goto error; + } + + if (!mp_update_av_hw_frames_pool(&p->hw_pool, p->av_device_ctx, + p->public.hw_imgfmt, p->last_sw_fmt, + src->w, src->h)) + { + MP_ERR(f, "failed to create frame pool\n"); + goto error; + } + + struct mp_image *dst = mp_av_pool_image_hw_upload(p->hw_pool, src); + if (!dst) + goto error; + + mp_frame_unref(&frame); + mp_pin_in_write(f->ppins[1], MAKE_FRAME(MP_FRAME_VIDEO, dst)); + + return; + +error: + mp_frame_unref(&frame); + MP_ERR(f, "failed to upload frame\n"); + mp_filter_internal_mark_failed(f); +} + +static void destroy(struct mp_filter *f) +{ + struct priv *p = f->priv; + + av_buffer_unref(&p->hw_pool); + av_buffer_unref(&p->av_device_ctx); +} + +static const struct mp_filter_info hwupload_filter = { + .name = "hwupload", + .priv_size = sizeof(struct priv), + .process = process, + .destroy = destroy, +}; + +// The VO layer might have restricted format support. It might actually +// work if this is input to a conversion filter anyway, but our format +// negotiation is too stupid and non-existent to detect this. +// So filter out all not explicitly supported formats. +static bool vo_supports(struct mp_hwdec_ctx *ctx, int hw_fmt, int sw_fmt) +{ + if (!ctx->hw_imgfmt) + return true; // if unset, all formats are allowed + if (ctx->hw_imgfmt != hw_fmt) + return false; + + for (int i = 0; ctx->supported_formats && ctx->supported_formats[i]; i++) { + if (ctx->supported_formats[i] == sw_fmt) + return true; + } + + return false; +} + +static bool probe_formats(struct mp_hwupload *u, int hw_imgfmt) +{ + struct priv *p = u->f->priv; + + u->hw_imgfmt = hw_imgfmt; + u->num_fmts = 0; + u->num_upload_fmts = 0; + + struct mp_stream_info *info = mp_filter_find_stream_info(u->f); + if (!info || !info->hwdec_devs) { + MP_ERR(u->f, "no hw context\n"); + return false; + } + + struct mp_hwdec_ctx *ctx = NULL; + AVHWFramesConstraints *cstr = NULL; + + for (int n = 0; ; n++) { + struct mp_hwdec_ctx *cur = hwdec_devices_get_n(info->hwdec_devs, n); + if (!cur) + break; + if (!cur->av_device_ref) + continue; + cstr = av_hwdevice_get_hwframe_constraints(cur->av_device_ref, NULL); + if (!cstr) + continue; + bool found = false; + for (int i = 0; cstr->valid_hw_formats && + cstr->valid_hw_formats[i] != AV_PIX_FMT_NONE; i++) + { + found |= cstr->valid_hw_formats[i] == imgfmt2pixfmt(hw_imgfmt); + } + if (found && (!cur->hw_imgfmt || cur->hw_imgfmt == hw_imgfmt)) { + ctx = cur; + break; + } + av_hwframe_constraints_free(&cstr); + } + + if (!ctx) { + MP_ERR(u->f, "no support for this hw format\n"); + return false; + } + + // Probe for supported formats. This is very roundabout, because the + // hwcontext API does not give us this information directly. We resort to + // creating temporary AVHWFramesContexts in order to retrieve the list of + // supported formats. This should be relatively cheap as we don't create + // any real frames (although some backends do for probing info). + + for (int n = 0; cstr->valid_sw_formats && + cstr->valid_sw_formats[n] != AV_PIX_FMT_NONE; n++) + { + int imgfmt = pixfmt2imgfmt(cstr->valid_sw_formats[n]); + if (!imgfmt) + continue; + + MP_VERBOSE(u->f, "looking at format %s\n", mp_imgfmt_to_name(imgfmt)); + + // Creates an AVHWFramesContexts with the given parameters. + AVBufferRef *frames = NULL; + if (!mp_update_av_hw_frames_pool(&frames, ctx->av_device_ref, + hw_imgfmt, imgfmt, 128, 128)) + { + MP_WARN(u->f, "failed to allocate pool\n"); + continue; + } + + enum AVPixelFormat *fmts; + if (av_hwframe_transfer_get_formats(frames, + AV_HWFRAME_TRANSFER_DIRECTION_TO, &fmts, 0) >= 0) + { + int index = u->num_fmts; + MP_TARRAY_APPEND(p, u->fmts, u->num_fmts, imgfmt); + MP_TARRAY_GROW(p, u->fmt_upload_index, index); + MP_TARRAY_GROW(p, u->fmt_upload_num, index); + + u->fmt_upload_index[index] = u->num_upload_fmts; + + for (int i = 0; fmts[i] != AV_PIX_FMT_NONE; i++) { + int fmt = pixfmt2imgfmt(fmts[i]); + if (!fmt) + continue; + MP_VERBOSE(u->f, "supports %s\n", mp_imgfmt_to_name(fmt)); + if (vo_supports(ctx, hw_imgfmt, fmt)) + MP_TARRAY_APPEND(p, u->upload_fmts, u->num_upload_fmts, fmt); + } + + u->fmt_upload_num[index] = + u->num_upload_fmts - u->fmt_upload_index[index]; + + av_free(fmts); + } + + av_buffer_unref(&frames); + } + + p->av_device_ctx = av_buffer_ref(ctx->av_device_ref); + if (!p->av_device_ctx) + return false; + + return u->num_upload_fmts > 0; +} + +struct mp_hwupload *mp_hwupload_create(struct mp_filter *parent, int hw_imgfmt) +{ + struct mp_filter *f = mp_filter_create(parent, &hwupload_filter); + if (!f) + return NULL; + + struct priv *p = f->priv; + struct mp_hwupload *u = &p->public; + u->f = f; + + mp_filter_add_pin(f, MP_PIN_IN, "in"); + mp_filter_add_pin(f, MP_PIN_OUT, "out"); + + if (!probe_formats(u, hw_imgfmt)) { + MP_ERR(f, "hardware format not supported\n"); + goto error; + } + + return u; +error: + talloc_free(f); + return NULL; +} diff --git a/filters/f_hwtransfer.h b/filters/f_hwtransfer.h new file mode 100644 index 0000000000..4595cb393d --- /dev/null +++ b/filters/f_hwtransfer.h @@ -0,0 +1,32 @@ +#pragma once + +#include "filter.h" + +// A filter which uploads sw frames to hw. Ignores hw frames. +struct mp_hwupload { + struct mp_filter *f; + + // Hardware wrapper format, e.g. IMGFMT_VAAPI. + int hw_imgfmt; + + // List of supported underlying surface formats. + int *fmts; + int num_fmts; + // List of supported upload image formats. May contain duplicate entries + // (which should be ignored). + int *upload_fmts; + int num_upload_fmts; + // For fmts[n], fmt_upload_index[n] gives the index of the first supported + // upload format in upload_fmts[], and fmt_upload_num[n] gives the number + // of formats at this position. + int *fmt_upload_index; + int *fmt_upload_num; +}; + +struct mp_hwupload *mp_hwupload_create(struct mp_filter *parent, int hw_imgfmt); + +// Return the best format suited for upload that is supported for a given input +// imgfmt. This returns the same as imgfmt if the format is natively supported, +// and otherwise a format that likely results in the least loss. +// Returns 0 if completely unsupported. +int mp_hwupload_find_upload_format(struct mp_hwupload *u, int imgfmt); diff --git a/filters/f_lavfi.c b/filters/f_lavfi.c new file mode 100644 index 0000000000..a97f126efb --- /dev/null +++ b/filters/f_lavfi.c @@ -0,0 +1,952 @@ +/* + * This file is part of mpv. + * + * mpv is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * mpv is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with mpv. If not, see . + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "common/common.h" +#include "common/av_common.h" +#include "common/tags.h" +#include "common/msg.h" + +#include "audio/format.h" +#include "audio/aframe.h" +#include "video/mp_image.h" +#include "audio/fmt-conversion.h" +#include "video/fmt-conversion.h" +#include "video/hwdec.h" + +#include "f_lavfi.h" +#include "filter.h" +#include "filter_internal.h" +#include "user_filters.h" + +#if LIBAVFILTER_VERSION_MICRO < 100 +#define av_buffersink_get_frame_flags(a, b, c) av_buffersink_get_frame(a, b) +#define AV_BUFFERSINK_FLAG_NO_REQUEST 0 +#endif + +struct lavfi { + struct mp_log *log; + struct mp_filter *f; + + char *graph_string; + char **graph_opts; + bool force_bidir; + enum mp_frame_type force_type; + bool direct_filter; + char **direct_filter_opts; + + AVFilterGraph *graph; + // Set to true once all inputs have been initialized, and the graph is + // linked. + bool initialized; + + // Graph is draining to either handle format changes (if input format + // changes for one pad, recreate the graph after draining all buffered + // frames), or undo previously sent EOF (libavfilter does not accept + // input anymore after sending EOF, so recreate the graph to "unstuck" it). + bool draining_recover; + + // Filter can't be put into a working state. + bool failed; + + struct lavfi_pad **in_pads; + int num_in_pads; + + struct lavfi_pad **out_pads; + int num_out_pads; + + struct lavfi_pad **all_pads; + int num_all_pads; + + AVFrame *tmp_frame; + + struct mp_lavfi public; +}; + +struct lavfi_pad { + struct lavfi *main; + enum mp_frame_type type; + enum mp_pin_dir dir; + char *name; // user-given pad name + + struct mp_pin *pin; // internal pin for this (never NULL once initialized) + int pin_index; + + AVFilterContext *filter; + int filter_pad; + // buffersrc or buffersink connected to filter/filter_pad + AVFilterContext *buffer; + AVRational timebase; + bool buffer_is_eof; // received/sent EOF to the buffer + + struct mp_tags *metadata; + + // 1-frame queue for input. + struct mp_frame pending; + + // Used to check input format changes. + struct mp_frame in_fmt; +}; + +// Free the libavfilter graph (not c), reset all state. +// Does not free pending data intentionally. +static void free_graph(struct lavfi *c) +{ + avfilter_graph_free(&c->graph); + for (int n = 0; n < c->num_all_pads; n++) { + struct lavfi_pad *pad = c->all_pads[n]; + + pad->filter = NULL; + pad->filter_pad = -1; + pad->buffer = NULL; + mp_frame_unref(&pad->in_fmt); + pad->buffer_is_eof = false; + } + c->initialized = false; + c->draining_recover = false; +} + +static void add_pad(struct lavfi *c, int dir, int index, AVFilterContext *filter, + int filter_pad, const char *name, bool first_init) +{ + if (c->failed) + return; + + enum AVMediaType avmt; + if (dir == MP_PIN_IN) { + avmt = avfilter_pad_get_type(filter->input_pads, filter_pad); + } else { + avmt = avfilter_pad_get_type(filter->output_pads, filter_pad); + } + int type; + switch (avmt) { + case AVMEDIA_TYPE_VIDEO: type = MP_FRAME_VIDEO; break; + case AVMEDIA_TYPE_AUDIO: type = MP_FRAME_AUDIO; break; + default: + MP_FATAL(c, "unknown media type\n"); + c->failed = true; + return; + } + + // For anonymous pads, just make something up. libavfilter allows duplicate + // pad names (while we don't), so we check for collisions along with normal + // duplicate pads below. + char tmp[80]; + const char *dir_string = dir == MP_PIN_IN ? "in" : "out"; + if (name) { + if (c->direct_filter) { + // libavfilter has this very unpleasant thing that filter labels + // don't have to be unique - in particular, both input and output + // are usually named "default". With direct filters, the user has + // no chance to provide better names, so do something to resolve it. + snprintf(tmp, sizeof(tmp), "%s_%s", name, dir_string); + name = tmp; + } + } else { + snprintf(tmp, sizeof(tmp), "%s%d", dir_string, index); + name = tmp; + } + + struct lavfi_pad *p = NULL; + for (int n = 0; n < c->num_all_pads; n++) { + if (strcmp(c->all_pads[n]->name, name) == 0) { + p = c->all_pads[n]; + break; + } + } + + if (p) { + // Graph recreation case: reassociate an existing pad. + if (p->filter) { + // Collision due to duplicate names. + MP_FATAL(c, "more than one pad with label '%s'\n", name); + c->failed = true; + return; + } + if (p->dir != dir || p->type != type) { + // libavfilter graph parser behavior not deterministic. + MP_FATAL(c, "pad '%s' changed type or direction\n", name); + c->failed = true; + return; + } + } else { + if (!first_init) { + MP_FATAL(c, "filter pad '%s' got added later?\n", name); + c->failed = true; + return; + } + p = talloc_zero(c, struct lavfi_pad); + p->main = c; + p->dir = dir; + p->name = talloc_strdup(p, name); + p->type = type; + p->pin_index = -1; + p->metadata = talloc_zero(p, struct mp_tags); + if (p->dir == MP_PIN_IN) + MP_TARRAY_APPEND(c, c->in_pads, c->num_in_pads, p); + if (p->dir == MP_PIN_OUT) + MP_TARRAY_APPEND(c, c->out_pads, c->num_out_pads, p); + MP_TARRAY_APPEND(c, c->all_pads, c->num_all_pads, p); + } + p->filter = filter; + p->filter_pad = filter_pad; +} + +static void add_pads(struct lavfi *c, int dir, AVFilterInOut *l, bool first_init) +{ + int index = 0; + for (; l; l = l->next) + add_pad(c, dir, index++, l->filter_ctx, l->pad_idx, l->name, first_init); +} + +static void add_pads_direct(struct lavfi *c, int dir, AVFilterContext *f, + AVFilterPad *pads, int num_pads, bool first_init) +{ + for (int n = 0; n < num_pads; n++) + add_pad(c, dir, n, f, n, avfilter_pad_get_name(pads, n), first_init); +} + +// Parse the user-provided filter graph, and populate the unlinked filter pads. +static void precreate_graph(struct lavfi *c, bool first_init) +{ + assert(!c->graph); + + c->failed = false; + + c->graph = avfilter_graph_alloc(); + if (!c->graph) + abort(); + + if (mp_set_avopts(c->log, c->graph, c->graph_opts) < 0) + goto error; + + if (c->direct_filter) { + AVFilterContext *filter = avfilter_graph_alloc_filter(c->graph, + avfilter_get_by_name(c->graph_string), "filter"); + if (!filter) { + MP_FATAL(c, "filter '%s' not found or failed to allocate\n", + c->graph_string); + goto error; + } + + if (mp_set_avopts(c->log, filter->priv, c->direct_filter_opts) < 0) + goto error; + + if (avfilter_init_str(filter, NULL) < 0) { + MP_FATAL(c, "filter failed to initialize\n"); + goto error; + } + + add_pads_direct(c, MP_PIN_IN, filter, filter->input_pads, + filter->nb_inputs, first_init); + add_pads_direct(c, MP_PIN_OUT, filter, filter->output_pads, + filter->nb_outputs, first_init); + } else { + AVFilterInOut *in = NULL, *out = NULL; + if (avfilter_graph_parse2(c->graph, c->graph_string, &in, &out) < 0) { + MP_FATAL(c, "parsing the filter graph failed\n"); + goto error; + } + add_pads(c, MP_PIN_IN, in, first_init); + add_pads(c, MP_PIN_OUT, out, first_init); + avfilter_inout_free(&in); + avfilter_inout_free(&out); + } + + for (int n = 0; n < c->num_all_pads; n++) + c->failed |= !c->all_pads[n]->filter; + + if (c->failed) + goto error; + + return; + +error: + free_graph(c); + c->failed = true; + mp_filter_internal_mark_failed(c->f); + return; +} + +// Ensure to send EOF to each input pad, so the graph can be drained properly. +static void send_global_eof(struct lavfi *c) +{ + for (int n = 0; n < c->num_in_pads; n++) { + struct lavfi_pad *pad = c->in_pads[n]; + if (!pad->buffer || pad->buffer_is_eof) + continue; + + if (av_buffersrc_add_frame(pad->buffer, NULL) < 0) + MP_FATAL(c, "could not send EOF to filter\n"); + + pad->buffer_is_eof = true; + } +} + +// libavfilter allows changing some parameters on the fly, but not +// others. +static bool is_aformat_ok(struct mp_aframe *a, struct mp_aframe *b) +{ + struct mp_chmap ca = {0}, cb = {0}; + mp_aframe_get_chmap(a, &ca); + mp_aframe_get_chmap(b, &cb); + return mp_chmap_equals(&ca, &cb) && + mp_aframe_get_rate(a) == mp_aframe_get_rate(b) && + mp_aframe_get_format(a) == mp_aframe_get_format(b); +} +static bool is_vformat_ok(struct mp_image *a, struct mp_image *b) +{ + return a->imgfmt == b->imgfmt && + a->w == b->w && a->h && b->h && + a->params.p_w == b->params.p_w && a->params.p_h == b->params.p_h; +} +static bool is_format_ok(struct mp_frame a, struct mp_frame b) +{ + if (a.type == b.type && a.type == MP_FRAME_VIDEO) + return is_vformat_ok(a.data, b.data); + if (a.type == b.type && a.type == MP_FRAME_AUDIO) + return is_aformat_ok(a.data, b.data); + return false; +} + +static void read_pad_input(struct lavfi *c, struct lavfi_pad *pad) +{ + assert(pad->dir == MP_PIN_IN); + + if (pad->pending.type || c->draining_recover) + return; + + pad->pending = mp_pin_out_read(pad->pin); + + if (pad->pending.type && pad->pending.type != MP_FRAME_EOF && + pad->pending.type != pad->type) + { + MP_FATAL(c, "unknown frame %s\n", mp_frame_type_str(pad->pending.type)); + mp_frame_unref(&pad->pending); + } + + if (mp_frame_is_data(pad->pending) && pad->in_fmt.type && + !is_format_ok(pad->pending, pad->in_fmt)) + { + if (!c->draining_recover) + MP_VERBOSE(c, "format change on %s\n", pad->name); + c->draining_recover = true; + if (c->initialized) + send_global_eof(c); + } +} + +// Attempt to initialize all pads. Return true if all are initialized, or +// false if more data is needed (or on error). +static bool init_pads(struct lavfi *c) +{ + if (!c->graph) + goto error; + + for (int n = 0; n < c->num_out_pads; n++) { + struct lavfi_pad *pad = c->out_pads[n]; + if (pad->buffer) + continue; + + const AVFilter *dst_filter = NULL; + if (pad->type == MP_FRAME_AUDIO) { + dst_filter = avfilter_get_by_name("abuffersink"); + } else if (pad->type == MP_FRAME_VIDEO) { + dst_filter = avfilter_get_by_name("buffersink"); + } else { + assert(0); + } + + if (!dst_filter) + goto error; + + char name[256]; + snprintf(name, sizeof(name), "mpv_sink_%s", pad->name); + + if (avfilter_graph_create_filter(&pad->buffer, dst_filter, + name, NULL, NULL, c->graph) < 0) + goto error; + + if (avfilter_link(pad->filter, pad->filter_pad, pad->buffer, 0) < 0) + goto error; + } + + for (int n = 0; n < c->num_in_pads; n++) { + struct lavfi_pad *pad = c->in_pads[n]; + if (pad->buffer) + continue; + + mp_frame_unref(&pad->in_fmt); + + read_pad_input(c, pad); + // no input data, format unknown, can't init, wait longer. + if (!pad->pending.type) + return false; + + if (mp_frame_is_data(pad->pending)) { + assert(pad->pending.type == pad->type); + + pad->in_fmt = mp_frame_ref(pad->pending); + if (!pad->in_fmt.type) + goto error; + + if (pad->in_fmt.type == MP_FRAME_VIDEO) + mp_image_unref_data(pad->in_fmt.data); + if (pad->in_fmt.type == MP_FRAME_AUDIO) + mp_aframe_unref_data(pad->in_fmt.data); + } + + if (pad->pending.type == MP_FRAME_EOF && !pad->in_fmt.type) { + // libavfilter makes this painful. Init it with a dummy config, + // just so we can tell it the stream is EOF. + if (pad->type == MP_FRAME_AUDIO) { + struct mp_aframe *fmt = mp_aframe_create(); + mp_aframe_set_format(fmt, AF_FORMAT_FLOAT); + mp_aframe_set_chmap(fmt, &(struct mp_chmap)MP_CHMAP_INIT_STEREO); + mp_aframe_set_rate(fmt, 48000); + pad->in_fmt = (struct mp_frame){MP_FRAME_AUDIO, fmt}; + } + if (pad->type == MP_FRAME_VIDEO) { + struct mp_image *fmt = talloc_zero(NULL, struct mp_image); + mp_image_setfmt(fmt, IMGFMT_420P); + mp_image_set_size(fmt, 64, 64); + pad->in_fmt = (struct mp_frame){MP_FRAME_VIDEO, fmt}; + } + } + + if (pad->in_fmt.type != pad->type) + goto error; + + AVBufferSrcParameters *params = av_buffersrc_parameters_alloc(); + if (!params) + goto error; + + pad->timebase = AV_TIME_BASE_Q; + + char *filter_name = NULL; + if (pad->type == MP_FRAME_AUDIO) { + struct mp_aframe *fmt = pad->in_fmt.data; + params->format = af_to_avformat(mp_aframe_get_format(fmt)); + params->sample_rate = mp_aframe_get_rate(fmt); + struct mp_chmap chmap = {0}; + mp_aframe_get_chmap(fmt, &chmap); + params->channel_layout = mp_chmap_to_lavc(&chmap); + pad->timebase = (AVRational){1, mp_aframe_get_rate(fmt)}; + filter_name = "abuffer"; + } else if (pad->type == MP_FRAME_VIDEO) { + struct mp_image *fmt = pad->in_fmt.data; + params->format = imgfmt2pixfmt(fmt->imgfmt); + params->width = fmt->w; + params->height = fmt->h; + params->sample_aspect_ratio.num = fmt->params.p_w; + params->sample_aspect_ratio.den = fmt->params.p_h; + params->hw_frames_ctx = fmt->hwctx; + filter_name = "buffer"; + } else { + assert(0); + } + + params->time_base = pad->timebase; + + const AVFilter *filter = avfilter_get_by_name(filter_name); + if (filter) { + char name[256]; + snprintf(name, sizeof(name), "mpv_src_%s", pad->name); + + pad->buffer = avfilter_graph_alloc_filter(c->graph, filter, name); + } + if (!pad->buffer) { + av_free(params); + goto error; + } + + int ret = av_buffersrc_parameters_set(pad->buffer, params); + av_free(params); + if (ret < 0) + goto error; + + if (avfilter_init_str(pad->buffer, NULL) < 0) + goto error; + + if (avfilter_link(pad->buffer, 0, pad->filter, pad->filter_pad) < 0) + goto error; + } + + return true; +error: + MP_FATAL(c, "could not initialize filter pads\n"); + c->failed = true; + mp_filter_internal_mark_failed(c->f); + return false; +} + +static void dump_graph(struct lavfi *c) +{ +#if LIBAVFILTER_VERSION_MICRO >= 100 + MP_DBG(c, "Filter graph:\n"); + char *s = avfilter_graph_dump(c->graph, NULL); + if (s) + MP_DBG(c, "%s\n", s); + av_free(s); +#endif +} + +// Initialize the graph if all inputs have formats set. If it's already +// initialized, or can't be initialized yet, do nothing. +static void init_graph(struct lavfi *c) +{ + assert(!c->initialized); + + if (!c->graph) + precreate_graph(c, false); + + if (init_pads(c)) { + struct mp_stream_info *info = mp_filter_find_stream_info(c->f); + if (info && info->hwdec_devs) { + struct mp_hwdec_ctx *hwdec = hwdec_devices_get_first(info->hwdec_devs); + for (int n = 0; n < c->graph->nb_filters; n++) { + AVFilterContext *filter = c->graph->filters[n]; + if (hwdec && hwdec->av_device_ref) + filter->hw_device_ctx = av_buffer_ref(hwdec->av_device_ref); + } + } + + // And here the actual libavfilter initialization happens. + if (avfilter_graph_config(c->graph, NULL) < 0) { + MP_FATAL(c, "failed to configure the filter graph\n"); + free_graph(c); + c->failed = true; + mp_filter_internal_mark_failed(c->f); + return; + } + + // The timebase is available after configuring. + for (int n = 0; n < c->num_out_pads; n++) { + struct lavfi_pad *pad = c->out_pads[n]; + + pad->timebase = pad->buffer->inputs[0]->time_base; + } + + c->initialized = true; + + if (!c->direct_filter) // (output uninteresting for direct filters) + dump_graph(c); + } +} + +static bool feed_input_pads(struct lavfi *c) +{ + bool progress = false; + bool was_draining = c->draining_recover; + + assert(c->initialized); + + for (int n = 0; n < c->num_in_pads; n++) { + struct lavfi_pad *pad = c->in_pads[n]; + + bool requested = true; +#if LIBAVFILTER_VERSION_MICRO >= 100 + requested = av_buffersrc_get_nb_failed_requests(pad->buffer) > 0; +#endif + + // Always request a frame after EOF so that we can know if the EOF state + // changes (e.g. for sparse streams with midstream EOF). + requested |= pad->buffer_is_eof; + + if (requested) + read_pad_input(c, pad); + + if (!pad->pending.type || c->draining_recover) + continue; + + if (pad->buffer_is_eof) { + MP_WARN(c, "eof state changed on %s\n", pad->name); + c->draining_recover = true; + send_global_eof(c); + continue; + } + + AVFrame *frame = mp_frame_to_av(pad->pending, &pad->timebase); + bool eof = pad->pending.type == MP_FRAME_EOF; + + mp_frame_unref(&pad->pending); + + if (!frame && !eof) { + MP_FATAL(c, "out of memory or unsupported format\n"); + continue; + } + + pad->buffer_is_eof = !frame; + + if (av_buffersrc_add_frame(pad->buffer, frame) < 0) + MP_FATAL(c, "could not pass frame to filter\n"); + av_frame_free(&frame); + + progress = true; + } + + if (!was_draining && c->draining_recover) + progress = true; + + return progress; +} + +static bool read_output_pads(struct lavfi *c) +{ + bool progress = false; + + assert(c->initialized); + + for (int n = 0; n < c->num_out_pads; n++) { + struct lavfi_pad *pad = c->out_pads[n]; + + if (!mp_pin_in_needs_data(pad->pin)) + continue; + + assert(pad->buffer); + + int r = AVERROR_EOF; + if (!pad->buffer_is_eof) + r = av_buffersink_get_frame_flags(pad->buffer, c->tmp_frame, 0); + if (r >= 0) { +#if LIBAVUTIL_VERSION_MICRO >= 100 + mp_tags_copy_from_av_dictionary(pad->metadata, c->tmp_frame->metadata); +#endif + struct mp_frame frame = + mp_frame_from_av(pad->type, c->tmp_frame, &pad->timebase); + av_frame_unref(c->tmp_frame); + if (frame.type) { + mp_pin_in_write(pad->pin, frame); + } else { + MP_ERR(c, "could not use filter output\n"); + mp_frame_unref(&frame); + } + progress = true; + } else if (r == AVERROR(EAGAIN)) { + // We expect that libavfilter will request input on one of the + // input pads (via av_buffersrc_get_nb_failed_requests()). + } else if (r == AVERROR_EOF) { + if (!c->draining_recover && !pad->buffer_is_eof) + mp_pin_in_write(pad->pin, MP_EOF_FRAME); + if (!pad->buffer_is_eof) + progress = true; + pad->buffer_is_eof = true; + } else { + // Real error - ignore it. + MP_ERR(c, "error on filtering (%d)\n", r); + } + } + + return progress; +} + +static void lavfi_process(struct mp_filter *f) +{ + struct lavfi *c = f->priv; + + if (!c->initialized) + init_graph(c); + + while (c->initialized) { + bool a = read_output_pads(c); + bool b = feed_input_pads(c); + if (!a && !b) + break; + } + + // Start over on format changes or EOF draining. + if (c->draining_recover) { + // Wait until all outputs got EOF. + bool all_eof = true; + for (int n = 0; n < c->num_out_pads; n++) + all_eof &= c->out_pads[n]->buffer_is_eof; + + if (all_eof) { + MP_VERBOSE(c, "recovering all eof\n"); + free_graph(c); + mp_filter_internal_mark_progress(c->f); + } + } + + if (c->failed) + mp_filter_internal_mark_failed(c->f); +} + +static void lavfi_reset(struct mp_filter *f) +{ + struct lavfi *c = f->priv; + + free_graph(c); + + for (int n = 0; n < c->num_in_pads; n++) + mp_frame_unref(&c->in_pads[n]->pending); +} + +static void lavfi_destroy(struct mp_filter *f) +{ + struct lavfi *c = f->priv; + + lavfi_reset(f); + av_frame_free(&c->tmp_frame); +} + +static bool lavfi_command(struct mp_filter *f, struct mp_filter_command *cmd) +{ + struct lavfi *c = f->priv; + + if (!c->initialized) + return false; + + switch (cmd->type) { +#if LIBAVFILTER_VERSION_MICRO >= 100 + case MP_FILTER_COMMAND_TEXT: { + return avfilter_graph_send_command(c->graph, "all", cmd->cmd, cmd->arg, + &(char){0}, 0, 0) >= 0; + } +#endif + case MP_FILTER_COMMAND_GET_META: { + // We can worry later about what it should do to multi output filters. + if (c->num_out_pads < 1) + return false; + struct mp_tags **ptags = cmd->res; + *ptags = mp_tags_dup(NULL, c->out_pads[0]->metadata); + return true; + } + default: + return false; + } +} + +static const struct mp_filter_info lavfi_filter = { + .name = "lavfi", + .priv_size = sizeof(struct lavfi), + .process = lavfi_process, + .reset = lavfi_reset, + .destroy = lavfi_destroy, + .command = lavfi_command, +}; + +static struct lavfi *lavfi_alloc(struct mp_filter *parent) +{ + struct mp_filter *f = mp_filter_create(parent, &lavfi_filter); + if (!f) + return NULL; + + struct lavfi *c = f->priv; + + c->f = f; + c->log = f->log; + c->public.f = f; + c->tmp_frame = av_frame_alloc(); + if (!c->tmp_frame) + abort(); + + return c; +} + +static struct mp_lavfi *do_init(struct lavfi *c) +{ + precreate_graph(c, true); + + if (c->failed) + goto error; + + for (int n = 0; n < c->num_in_pads + c->num_out_pads; n++) { + // First add input pins to satisfy order for bidir graph types. + struct lavfi_pad *pad = + n < c->num_in_pads ? c->in_pads[n] : c->out_pads[n - c->num_in_pads]; + + pad->pin_index = c->f->num_pins; + pad->pin = mp_filter_add_pin(c->f, pad->dir, pad->name); + + if (c->force_type && c->force_type != pad->type) { + MP_FATAL(c, "mismatching media type\n"); + goto error; + } + } + + if (c->force_bidir) { + if (c->f->num_pins != 2) { + MP_FATAL(c, "exactly 2 pads required\n"); + goto error; + } + if (mp_pin_get_dir(c->f->ppins[0]) != MP_PIN_OUT || + mp_pin_get_dir(c->f->ppins[1]) != MP_PIN_IN) + { + MP_FATAL(c, "1 input and 1 output pad required\n"); + goto error; + } + } + + return &c->public; + +error: + talloc_free(c->f); + return NULL; +} + +struct mp_lavfi *mp_lavfi_create_graph(struct mp_filter *parent, + enum mp_frame_type type, bool bidir, + char **graph_opts, const char *graph) +{ + struct lavfi *c = lavfi_alloc(parent); + if (!c) + return NULL; + + c->force_type = type; + c->force_bidir = bidir; + c->graph_opts = mp_dup_str_array(c, graph_opts); + c->graph_string = talloc_strdup(c, graph); + + return do_init(c); +} + +struct mp_lavfi *mp_lavfi_create_filter(struct mp_filter *parent, + enum mp_frame_type type, bool bidir, + char **graph_opts, + const char *filter, char **filter_opts) +{ + struct lavfi *c = lavfi_alloc(parent); + if (!c) + return NULL; + + c->force_type = type; + c->force_bidir = bidir; + c->graph_opts = mp_dup_str_array(c, graph_opts); + c->graph_string = talloc_strdup(c, filter); + c->direct_filter_opts = mp_dup_str_array(c, filter_opts); + c->direct_filter = true; + + return do_init(c); +} + +struct lavfi_user_opts { + bool is_bridge; + + char *graph; + char **avopts; + + char *filter_name; + char **filter_opts; +}; + +static struct mp_filter *vf_lavfi_create(struct mp_filter *parent, void *options) +{ + struct lavfi_user_opts *opts = options; + struct mp_lavfi *l; + if (opts->is_bridge) { + l = mp_lavfi_create_filter(parent, MP_FRAME_VIDEO, true, + opts->avopts, opts->filter_name, + opts->filter_opts); + } else { + l = mp_lavfi_create_graph(parent, MP_FRAME_VIDEO, true, + opts->avopts, opts->graph); + } + talloc_free(opts); + return l ? l->f : NULL; +} + +static bool is_single_video_only(const AVFilterPad *pads) +{ + int count = avfilter_pad_count(pads); + if (count != 1) + return false; + return avfilter_pad_get_type(pads, 0) == AVMEDIA_TYPE_VIDEO; +} + +// Does it have exactly one video input and one video output? +static bool is_usable(const AVFilter *filter) +{ + return is_single_video_only(filter->inputs) && + is_single_video_only(filter->outputs); +} + +static void print_help(struct mp_log *log) +{ + mp_info(log, "List of libavfilter filters:\n"); + for (const AVFilter *filter = avfilter_next(NULL); filter; + filter = avfilter_next(filter)) + { + if (is_usable(filter)) + mp_info(log, " %-16s %s\n", filter->name, filter->description); + } + mp_info(log, "\n" + "This lists video->video filters only. Refer to\n" + "\n" + " https://ffmpeg.org/ffmpeg-filters.html\n" + "\n" + "to see how to use each filter and what arguments each filter takes.\n" + "Also, be sure to quote the FFmpeg filter string properly, e.g.:\n" + "\n" + "