video: move VO reinit from filter chain to player

This gets rid of the vf_vo pseudo-filter. It ends the idea of MPlayer's
architecture that the VO is just a (terminating) video filter. It didn't
really work for us with respect to video timing (the "end" of the video
chain isn't really made for video timing, and making it do so would be
awkward), and now we're removing it entirely. We will be able to fix
some things, such as properly draining video on reconfiguration.

6 files changed, 90 insertions, 149 deletions

diff --git a/mpvcore/player/video.c b/mpvcore/player/video.c
index 7584f84c1c..bee23dd3d1 100644
--- a/mpvcore/player/video.c
+++ b/mpvcore/player/video.c
@@ -53,6 +53,51 @@ void update_fps(struct MPContext *mpctx)
 #endif
 }
 
+static void set_allowed_vo_formats(struct vf_chain *c, struct vo *vo)
+{
+    for (int fmt = IMGFMT_START; fmt < IMGFMT_END; fmt++) {
+        c->allowed_output_formats[fmt - IMGFMT_START] =
+            vo->driver->query_format(vo, fmt);
+    }
+}
+
+static void reconfig_video(struct MPContext *mpctx,
+                           const struct mp_image_params *params,
+                           bool probe_only)
+{
+    struct dec_video *d_video = mpctx->d_video;
+
+    d_video->decoder_output = *params;
+
+    set_allowed_vo_formats(d_video->vfilter, mpctx->video_out);
+
+    if (video_reconfig_filters(d_video, params) < 0) {
+        // Most video filters don't work with hardware decoding, so this
+        // might be the reason filter reconfig failed.
+        if (!probe_only &&
+            video_vd_control(d_video, VDCTRL_FORCE_HWDEC_FALLBACK, NULL) == CONTROL_OK)
+        {
+            // Fallback active; decoder will return software format next
+            // time. Don't abort video decoding.
+            d_video->vfilter->initialized = 0;
+        }
+        return;
+    }
+
+    if (d_video->vfilter->initialized < 1)
+        return;
+
+    struct mp_image_params p = d_video->vfilter->output_params;
+    const struct vo_driver *info = mpctx->video_out->driver;
+    mp_msg(MSGT_CPLAYER, MSGL_INFO, "VO: [%s] %dx%d => %dx%d %s\n",
+           info->name, p.w, p.h, p.d_w, p.d_h, vo_format_name(p.imgfmt));
+    mp_msg(MSGT_CPLAYER, MSGL_V, "VO: Description: %s\n", info->description);
+
+    int r = vo_reconfig(mpctx->video_out, &p, 0);
+    if (r < 0)
+        d_video->vfilter->initialized = -1;
+}
+
 static void recreate_video_filters(struct MPContext *mpctx)
 {
     struct MPOpts *opts = mpctx->opts;
@@ -63,26 +108,25 @@ static void recreate_video_filters(struct MPContext *mpctx)
     d_video->vfilter = vf_new(opts);
     d_video->vfilter->hwdec = &d_video->hwdec_info;
 
-    vf_append_filter(d_video->vfilter, "vo", NULL);
-    vf_control_any(d_video->vfilter, VFCTRL_SET_VO, mpctx->video_out);
-
     vf_append_filter_list(d_video->vfilter, opts->vf_settings);
 
     // for vf_sub
     vf_control_any(d_video->vfilter, VFCTRL_SET_OSD_OBJ, mpctx->osd);
     mpctx->osd->render_subs_in_filter
         = vf_control_any(d_video->vfilter, VFCTRL_INIT_OSD, NULL) == CONTROL_OK;
+
+    set_allowed_vo_formats(d_video->vfilter, mpctx->video_out);
 }
 
 int reinit_video_filters(struct MPContext *mpctx)
 {
     struct dec_video *d_video = mpctx->d_video;
 
-    if (!d_video)
+    if (!d_video || !d_video->decoder_output.imgfmt)
         return -2;
 
     recreate_video_filters(mpctx);
-    video_reconfig_filters(d_video, &d_video->decoder_output);
+    reconfig_video(mpctx, &d_video->decoder_output, true);
 
     return d_video->vfilter && d_video->vfilter->initialized > 0 ? 0 : -1;
 }
@@ -208,6 +252,8 @@ static bool load_next_vo_frame(struct MPContext *mpctx, bool eof)
     return false;
 }
 
+// Called after video reinit. This can be generally used to try to insert more
+// filters using the filter chain edit functionality in command.c.
 static void init_filter_params(struct MPContext *mpctx)
 {
     struct MPOpts *opts = mpctx->opts;
@@ -220,39 +266,19 @@ static void init_filter_params(struct MPContext *mpctx)
         mp_property_do("deinterlace", M_PROPERTY_SET, &opts->deinterlace, mpctx);
 }
 
-static void reconfig_video(struct MPContext *mpctx,
-                           const struct mp_image_params *params)
+static void filter_video(struct MPContext *mpctx, struct mp_image *frame)
 {
     struct dec_video *d_video = mpctx->d_video;
 
-    if (!mp_image_params_equals(&d_video->decoder_output, params) ||
+    struct mp_image_params params;
+    mp_image_params_from_image(&params, frame);
+    if (!mp_image_params_equals(&d_video->decoder_output, &params) ||
         d_video->vfilter->initialized < 1)
     {
-        d_video->decoder_output = *params;
-        if (video_reconfig_filters(d_video, params) < 0) {
-            // Most video filters don't work with hardware decoding, so this
-            // might be the reason filter reconfig failed.
-            if (video_vd_control(d_video, VDCTRL_FORCE_HWDEC_FALLBACK, NULL)
-                == CONTROL_OK)
-            {
-                // Fallback active; decoder will return software format next
-                // time. Don't abort video decoding.
-                d_video->vfilter->initialized = 0;
-            }
-            return;
-        }
+        reconfig_video(mpctx, &params, false);
         if (d_video->vfilter->initialized > 0)
             init_filter_params(mpctx);
     }
-}
-
-static void filter_video(struct MPContext *mpctx, struct mp_image *frame)
-{
-    struct dec_video *d_video = mpctx->d_video;
-
-    struct mp_image_params params;
-    mp_image_params_from_image(&params, frame);
-    reconfig_video(mpctx, &params);
 
     if (d_video->vfilter->initialized < 1) {
         talloc_free(frame);
diff --git a/old-makefile b/old-makefile
index 8aa9ffd0f2..f6108bac67 100644
--- a/old-makefile
+++ b/old-makefile
@@ -292,7 +292,6 @@ SOURCES = audio/audio.c \
           video/filter/vf_sub.c \
           video/filter/vf_swapuv.c \
           video/filter/vf_unsharp.c \
-          video/filter/vf_vo.c \
           video/filter/vf_yadif.c \
           video/out/bitmap_packer.c \
           video/out/aspect.c \
diff --git a/video/filter/vf.c b/video/filter/vf.c
index 1345826e57..69d5b29137 100644
--- a/video/filter/vf.c
+++ b/video/filter/vf.c
@@ -39,7 +39,6 @@
 
 #include "video/memcpy_pic.h"
 
-extern const vf_info_t vf_info_vo;
 extern const vf_info_t vf_info_crop;
 extern const vf_info_t vf_info_expand;
 extern const vf_info_t vf_info_pp;
@@ -75,7 +74,6 @@ static const vf_info_t *const filter_list[] = {
     &vf_info_crop,
     &vf_info_expand,
     &vf_info_scale,
-    &vf_info_vo,
     &vf_info_format,
     &vf_info_noformat,
     &vf_info_flip,
@@ -259,19 +257,17 @@ error:
 static vf_instance_t *vf_open_filter(struct vf_chain *c, const char *name,
                                      char **args)
 {
-    if (strcmp(name, "vo") != 0) {
-        int i, l = 0;
-        for (i = 0; args && args[2 * i]; i++)
-            l += 1 + strlen(args[2 * i]) + 1 + strlen(args[2 * i + 1]);
-        l += strlen(name);
-        char str[l + 1];
-        char *p = str;
-        p += sprintf(str, "%s", name);
-        for (i = 0; args && args[2 * i]; i++)
-            p += sprintf(p, " %s=%s", args[2 * i], args[2 * i + 1]);
-        mp_msg(MSGT_VFILTER, MSGL_INFO, "%s[%s]\n",
-               "Opening video filter: ", str);
-    }
+    int i, l = 0;
+    for (i = 0; args && args[2 * i]; i++)
+        l += 1 + strlen(args[2 * i]) + 1 + strlen(args[2 * i + 1]);
+    l += strlen(name);
+    char str[l + 1];
+    char *p = str;
+    p += sprintf(str, "%s", name);
+    for (i = 0; args && args[2 * i]; i++)
+        p += sprintf(p, " %s=%s", args[2 * i], args[2 * i + 1]);
+    mp_msg(MSGT_VFILTER, MSGL_INFO, "%s[%s]\n",
+           "Opening video filter: ", str);
     return vf_open(c, name, args);
 }
 
@@ -280,7 +276,7 @@ struct vf_instance *vf_append_filter(struct vf_chain *c, const char *name,
 {
     struct vf_instance *vf = vf_open_filter(c, name, args);
     if (vf) {
-        // Insert it before the last filter, which is the "vo" filter
+        // Insert it before the last filter, which is the "out" pseudo-filter
         // (But after the "in" pseudo-filter)
         struct vf_instance **pprev = &c->first->next;
         while (*pprev && (*pprev)->next)
@@ -309,9 +305,7 @@ int vf_append_filter_list(struct vf_chain *c, struct m_obj_settings *list)
 void vf_add_output_frame(struct vf_instance *vf, struct mp_image *img)
 {
     if (img) {
-        // vf_vo doesn't have output config
-        if (vf->fmt_out.imgfmt)
-            vf_fix_img_params(img, &vf->fmt_out);
+        vf_fix_img_params(img, &vf->fmt_out);
         MP_TARRAY_APPEND(vf, vf->out_queued, vf->num_out_queued, img);
     }
 }
@@ -508,6 +502,8 @@ int vf_reconfig(struct vf_chain *c, const struct mp_image_params *params)
             break;
         cur = vf->fmt_out;
     }
+    if (r >= 0)
+        c->output_params = cur;
     c->initialized = r < 0 ? -1 : 1;
     int loglevel = r < 0 ? MSGL_WARN : MSGL_V;
     if (r == -2)
@@ -544,6 +540,14 @@ static int input_query_format(struct vf_instance *vf, unsigned int fmt)
     return 0;
 }
 
+static int output_query_format(struct vf_instance *vf, unsigned int fmt)
+{
+    struct vf_chain *c = (void *)vf->priv;
+    if (fmt >= IMGFMT_START && fmt < IMGFMT_END)
+        return c->allowed_output_formats[fmt - IMGFMT_START];
+    return 0;
+}
+
 struct vf_chain *vf_new(struct MPOpts *opts)
 {
     struct vf_chain *c = talloc_ptrtype(NULL, c);
@@ -556,6 +560,13 @@ struct vf_chain *vf_new(struct MPOpts *opts)
         .info = &in,
         .query_format = input_query_format,
     };
+    static const struct vf_info out = { .name = "out" };
+    c->first->next = talloc(c, struct vf_instance);
+    *c->first->next = (struct vf_instance) {
+        .info = &out,
+        .query_format = output_query_format,
+        .priv = (void *)c,
+    };
     return c;
 }
 
diff --git a/video/filter/vf.h b/video/filter/vf.h
index a852a245f3..5892247556 100644
--- a/video/filter/vf.h
+++ b/video/filter/vf.h
@@ -95,7 +95,10 @@ typedef struct vf_instance {
 struct vf_chain {
     int initialized; // 0: no, 1: yes, -1: attempted to, but failed
 
-    struct vf_instance *first;
+    struct vf_instance *first, *last;
+
+    struct mp_image_params output_params;
+    uint8_t allowed_output_formats[IMGFMT_END - IMGFMT_START];
 
     struct MPOpts *opts;
     struct mp_hwdec_info *hwdec;
@@ -117,7 +120,6 @@ enum vf_ctrl {
     /* Hack to make the OSD state object available to vf_sub which
      * access OSD/subtitle state outside of normal OSD draw time. */
     VFCTRL_SET_OSD_OBJ,
-    VFCTRL_SET_VO,
 };
 
 struct vf_chain *vf_new(struct MPOpts *opts);
diff --git a/video/filter/vf_vo.c b/video/filter/vf_vo.c
deleted file mode 100644
index 4f90f6ab3e..0000000000
--- a/video/filter/vf_vo.c
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * This file is part of MPlayer.
- *
- * MPlayer is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * MPlayer is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with MPlayer; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <stdbool.h>
-
-#include "config.h"
-#include "mpvcore/mp_msg.h"
-#include "mpvcore/options.h"
-
-#include "video/mp_image.h"
-#include "vf.h"
-
-#include "video/out/vo.h"
-
-struct vf_priv_s {
-    struct vo *vo;
-};
-#define video_out (vf->priv->vo)
-
-static int reconfig(struct vf_instance *vf, struct mp_image_params *in,
-                    struct mp_image_params *out)
-{
-    if (!video_out)
-        return -1;
-
-    struct mp_image_params *p = in;
-    *out = *in;
-
-    if (p->w <= 0 || p->h <= 0 || p->d_w <= 0 || p->d_h <= 0) {
-        mp_msg(MSGT_CPLAYER, MSGL_ERR, "VO: invalid dimensions!\n");
-        return -1;
-    }
-
-    const struct vo_driver *info = video_out->driver;
-    mp_msg(MSGT_CPLAYER, MSGL_INFO, "VO: [%s] %dx%d => %dx%d %s\n",
-           info->name,
-           p->w, p->h, p->d_w, p->d_h,
-           vo_format_name(p->imgfmt));
-    mp_msg(MSGT_CPLAYER, MSGL_V, "VO: Description: %s\n", info->description);
-
-    return vo_reconfig(video_out, p, 0);
-}
-
-static int control(struct vf_instance *vf, int request, void *data)
-{
-    if (request == VFCTRL_SET_VO) {
-        video_out = data;
-        return CONTROL_OK;
-    }
-    return CONTROL_UNKNOWN;
-}
-
-static int query_format(struct vf_instance *vf, unsigned int fmt)
-{
-    if (!video_out)
-        return 0;
-    return video_out->driver->query_format(video_out, fmt);
-}
-
-static void uninit(struct vf_instance *vf)
-{
-}
-
-static int vf_open(vf_instance_t *vf)
-{
-    vf->reconfig = reconfig;
-    vf->control = control;
-    vf->query_format = query_format;
-    vf->uninit = uninit;
-    return 1;
-}
-
-const vf_info_t vf_info_vo = {
-    .description = "libvo wrapper",
-    .name = "vo",
-    .open = vf_open,
-    .priv_size = sizeof(struct vf_priv_s),
-};
diff --git a/wscript_build.py b/wscript_build.py
index bd1b41676d..a15c5c161a 100644
--- a/wscript_build.py
+++ b/wscript_build.py
@@ -321,7 +321,6 @@ def build(ctx):
         ( "video/filter/vf_swapuv.c" ),
         ( "video/filter/vf_unsharp.c" ),
         ( "video/filter/vf_vavpp.c",             "vaapi-vpp"),
-        ( "video/filter/vf_vo.c" ),
         ( "video/filter/vf_yadif.c" ),
         ( "video/out/aspect.c" ),
         ( "video/out/bitmap_packer.c" ),