video: remove vfcap.h

And remove all uses of the VFCAP_CSP_SUPPORTED* constants. This is
supposed to reduce conversions if many filters are used (with many
incompatible pixel formats), and also for preferring the VO's natively
supported pixel formats (as opposed to conversion).

This is worthless by now. Not only do the main VOs not use software
conversion, but also the way vf_lavfi and libavfilter work mostly break
the way the old MPlayer mechanism worked. Other important filters like
vf_vapoursynth do not support "proper" format negotation either.

Part of this was already removed with the vf_scale cleanup from today.

While I'm touching every single VO, also fix the query_format argument
(it's not a FourCC anymore).

22 files changed, 40 insertions, 112 deletions

diff --git a/common/encode_lavc.c b/common/encode_lavc.c
index 92af771d61..4cb653d511 100644
--- a/common/encode_lavc.c
+++ b/common/encode_lavc.c
@@ -26,7 +26,6 @@
 #include "common/global.h"
 #include "common/msg.h"
 #include "common/msg_control.h"
-#include "video/vfcap.h"
 #include "options/m_option.h"
 #include "options/options.h"
 #include "osdep/timer.h"
@@ -836,12 +835,12 @@ int encode_lavc_supports_pixfmt(struct encode_lavc_context *ctx,
         return 0;
 
     if (!ctx->vc->pix_fmts)
-        return VFCAP_CSP_SUPPORTED;
+        return 1;
     else {
         const enum AVPixelFormat *p;
         for (p = ctx->vc->pix_fmts; *p >= 0; ++p) {
             if (pix_fmt == *p)
-                return VFCAP_CSP_SUPPORTED;
+                return 1;
         }
     }
     return 0;
diff --git a/video/filter/vf.c b/video/filter/vf.c
index 0dd9b1f0b1..c1682c9652 100644
--- a/video/filter/vf.c
+++ b/video/filter/vf.c
@@ -580,7 +580,7 @@ static void update_formats(struct vf_chain *c, struct vf_instance *vf,
         // error messages in some cases, so we can configure all filter
         // until it fails, which will be visible in vf_print_filter_chain().
         for (int n = IMGFMT_START; n < IMGFMT_END; n++)
-            vf->last_outfmts[n - IMGFMT_START] = VFCAP_CSP_SUPPORTED;
+            vf->last_outfmts[n - IMGFMT_START] = 1;
         query_formats(fmts, vf);
     }
 }
diff --git a/video/filter/vf.h b/video/filter/vf.h
index 37ad4ca29c..07f38d680b 100644
--- a/video/filter/vf.h
+++ b/video/filter/vf.h
@@ -24,8 +24,6 @@
 #include "video/mp_image.h"
 #include "common/common.h"
 
-#include "video/vfcap.h"
-
 struct MPOpts;
 struct mpv_global;
 struct vf_instance;
diff --git a/video/filter/vf_lavfi.c b/video/filter/vf_lavfi.c
index da455f1c09..a356229a30 100644
--- a/video/filter/vf_lavfi.c
+++ b/video/filter/vf_lavfi.c
@@ -268,7 +268,7 @@ static int query_format(struct vf_instance *vf, unsigned int fmt)
     // allow us to do anything more sophisticated.
     // This breaks with filters which accept input pixel formats not
     // supported by libswscale.
-    return mp_sws_supported_format(fmt) ? VFCAP_CSP_SUPPORTED : 0;
+    return !!mp_sws_supported_format(fmt);
 }
 
 static AVFrame *mp_to_av(struct vf_instance *vf, struct mp_image *img)
diff --git a/video/filter/vf_scale.c b/video/filter/vf_scale.c
index d9506f31bc..8b513d6dcc 100644
--- a/video/filter/vf_scale.c
+++ b/video/filter/vf_scale.c
@@ -220,25 +220,11 @@ static int control(struct vf_instance *vf, int request, void *data)
     return CONTROL_UNKNOWN;
 }
 
-//===========================================================================//
-
 static int query_format(struct vf_instance *vf, unsigned int fmt)
 {
-    if (!IMGFMT_IS_HWACCEL(fmt) && imgfmt2pixfmt(fmt) != AV_PIX_FMT_NONE) {
-        if (sws_isSupportedInput(imgfmt2pixfmt(fmt)) < 1)
-            return 0;
-        int best = find_best_out(vf, fmt);
-        int flags;
-        if (!best)
-            return 0;            // no matching out-fmt
-        flags = vf_next_query_format(vf, best);
-        if (!(flags & (VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW)))
-            return 0;
-        if (fmt != best)
-            flags &= ~VFCAP_CSP_SUPPORTED_BY_HW;
-        return flags;
-    }
-    return 0;   // nomatching in-fmt
+    if (IMGFMT_IS_HWACCEL(fmt) || sws_isSupportedInput(imgfmt2pixfmt(fmt)) < 1)
+        return 0;
+    return !!find_best_out(vf, fmt);
 }
 
 static void uninit(struct vf_instance *vf)
diff --git a/video/filter/vf_vapoursynth.c b/video/filter/vf_vapoursynth.c
index ea5e86d599..61959473a5 100644
--- a/video/filter/vf_vapoursynth.c
+++ b/video/filter/vf_vapoursynth.c
@@ -654,7 +654,7 @@ static int config(struct vf_instance *vf, int width, int height,
 
 static int query_format(struct vf_instance *vf, unsigned int fmt)
 {
-    return mp_to_vs(fmt) != pfNone ? VFCAP_CSP_SUPPORTED : 0;
+    return mp_to_vs(fmt) != pfNone;
 }
 
 static int control(vf_instance_t *vf, int request, void *data)
diff --git a/video/out/vo.c b/video/out/vo.c
index 9ac35796c5..0d135c2842 100644
--- a/video/out/vo.c
+++ b/video/out/vo.c
@@ -44,7 +44,6 @@
 #include "common/msg.h"
 #include "common/global.h"
 #include "video/mp_image.h"
-#include "video/vfcap.h"
 #include "sub/osd.h"
 #include "osdep/io.h"
 #include "osdep/threads.h"
diff --git a/video/out/vo.h b/video/out/vo.h
index 5d136432e3..d943d4ad78 100644
--- a/video/out/vo.h
+++ b/video/out/vo.h
@@ -192,9 +192,9 @@ struct vo_driver {
     /*
      * Whether the given image format is supported and config() will succeed.
      * format: one of IMGFMT_*
-     * returns: 0 on not supported, otherwise a bitmask of VFCAP_* values
+     * returns: 0 on not supported, otherwise 1
      */
-    int (*query_format)(struct vo *vo, uint32_t format);
+    int (*query_format)(struct vo *vo, int format);
 
     /*
      * Initialize or reconfigure the display driver.
diff --git a/video/out/vo_caca.c b/video/out/vo_caca.c
index bcef78534c..d9e478e055 100644
--- a/video/out/vo_caca.c
+++ b/video/out/vo_caca.c
@@ -37,7 +37,6 @@
 #include "config.h"
 #include "vo.h"
 #include "video/mp_image.h"
-#include "video/vfcap.h"
 #include "video/memcpy_pic.h"
 
 #include "input/keycodes.h"
@@ -279,12 +278,9 @@ static int preinit(struct vo *vo)
     return 0;
 }
 
-static int query_format(struct vo *vo, uint32_t format)
+static int query_format(struct vo *vo, int format)
 {
-    if (format == IMGFMT_BGR24)
-        return VFCAP_CSP_SUPPORTED;
-
-    return 0;
+    return format == IMGFMT_BGR24;
 }
 
 static int control(struct vo *vo, uint32_t request, void *data)
diff --git a/video/out/vo_direct3d.c b/video/out/vo_direct3d.c
index 0ef9375e9c..293eff55fe 100644
--- a/video/out/vo_direct3d.c
+++ b/video/out/vo_direct3d.c
@@ -32,7 +32,6 @@
 #include "options/m_option.h"
 #include "talloc.h"
 #include "vo.h"
-#include "video/vfcap.h"
 #include "video/csputils.h"
 #include "video/mp_image.h"
 #include "video/img_format.h"
@@ -1127,13 +1126,13 @@ static bool init_rendering_mode(d3d_priv *priv, uint32_t fmt, bool initialize)
  *  @return 0 on failure, device capabilities (not probed
  *          currently) on success.
  */
-static int query_format(struct vo *vo, uint32_t movie_fmt)
+static int query_format(struct vo *vo, int movie_fmt)
 {
     d3d_priv *priv = vo->priv;
     if (!init_rendering_mode(priv, movie_fmt, false))
         return 0;
 
-    return VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW;
+    return 1;
 }
 
 /****************************************************************************
diff --git a/video/out/vo_image.c b/video/out/vo_image.c
index ccb8fba830..00f7bd6468 100644
--- a/video/out/vo_image.c
+++ b/video/out/vo_image.c
@@ -33,7 +33,6 @@
 #include "common/msg.h"
 #include "video/out/vo.h"
 #include "video/csputils.h"
-#include "video/vfcap.h"
 #include "video/mp_image.h"
 #include "video/fmt-conversion.h"
 #include "video/image_writer.h"
@@ -105,10 +104,10 @@ static void flip_page(struct vo *vo)
     mp_image_unrefp(&p->current);
 }
 
-static int query_format(struct vo *vo, uint32_t fmt)
+static int query_format(struct vo *vo, int fmt)
 {
     if (mp_sws_supported_format(fmt))
-        return VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW;
+        return 1;
     return 0;
 }
 
diff --git a/video/out/vo_lavc.c b/video/out/vo_lavc.c
index 9bb3ebd866..032f58c0f1 100644
--- a/video/out/vo_lavc.c
+++ b/video/out/vo_lavc.c
@@ -26,7 +26,6 @@
 #include "options/options.h"
 #include "video/fmt-conversion.h"
 #include "video/mp_image.h"
-#include "video/vfcap.h"
 #include "talloc.h"
 #include "vo.h"
 
@@ -191,7 +190,7 @@ error:
     return -1;
 }
 
-static int query_format(struct vo *vo, uint32_t format)
+static int query_format(struct vo *vo, int format)
 {
     enum AVPixelFormat pix_fmt = imgfmt2pixfmt(format);
 
@@ -201,7 +200,7 @@ static int query_format(struct vo *vo, uint32_t format)
     pthread_mutex_lock(&vo->encode_lavc_ctx->lock);
     int flags = 0;
     if (encode_lavc_supports_pixfmt(vo->encode_lavc_ctx, pix_fmt))
-        flags = VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW;
+        flags = 1;
     pthread_mutex_unlock(&vo->encode_lavc_ctx->lock);
     return flags;
 }
diff --git a/video/out/vo_null.c b/video/out/vo_null.c
index 2259e50c48..da9de61bf3 100644
--- a/video/out/vo_null.c
+++ b/video/out/vo_null.c
@@ -27,7 +27,6 @@
 #include "config.h"
 #include "common/msg.h"
 #include "vo.h"
-#include "video/vfcap.h"
 #include "video/mp_image.h"
 
 static void draw_image(struct vo *vo, mp_image_t *mpi)
@@ -39,9 +38,9 @@ static void flip_page(struct vo *vo)
 {
 }
 
-static int query_format(struct vo *vo, uint32_t format)
+static int query_format(struct vo *vo, int format)
 {
-    return VFCAP_CSP_SUPPORTED;
+    return 1;
 }
 
 static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
diff --git a/video/out/vo_opengl.c b/video/out/vo_opengl.c
index 54da31f2e3..bfb1adbec8 100644
--- a/video/out/vo_opengl.c
+++ b/video/out/vo_opengl.c
@@ -40,7 +40,6 @@
 #include "common/msg.h"
 #include "options/m_config.h"
 #include "vo.h"
-#include "video/vfcap.h"
 #include "video/mp_image.h"
 #include "sub/osd.h"
 
@@ -179,13 +178,12 @@ static void draw_image(struct vo *vo, mp_image_t *mpi)
     mpgl_unlock(p->glctx);
 }
 
-static int query_format(struct vo *vo, uint32_t format)
+static int query_format(struct vo *vo, int format)
 {
     struct gl_priv *p = vo->priv;
-    int caps = VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW;
     if (!gl_video_check_format(p->renderer, format))
         return 0;
-    return caps;
+    return 1;
 }
 
 static void video_resize_redraw_callback(struct vo *vo, int w, int h)
diff --git a/video/out/vo_opengl_cb.c b/video/out/vo_opengl_cb.c
index 4a0ba3dc54..3d5c16d8f2 100644
--- a/video/out/vo_opengl_cb.c
+++ b/video/out/vo_opengl_cb.c
@@ -17,7 +17,6 @@
 #include "options/options.h"
 #include "aspect.h"
 #include "vo.h"
-#include "video/vfcap.h"
 #include "video/mp_image.h"
 #include "sub/osd.h"
 
@@ -387,7 +386,7 @@ static void flip_page(struct vo *vo)
     pthread_mutex_unlock(&p->ctx->lock);
 }
 
-static int query_format(struct vo *vo, uint32_t format)
+static int query_format(struct vo *vo, int format)
 {
     struct vo_priv *p = vo->priv;
 
@@ -396,7 +395,7 @@ static int query_format(struct vo *vo, uint32_t format)
     if (format >= IMGFMT_START && format < IMGFMT_END)
         ok = p->ctx->imgfmt_supported[format - IMGFMT_START];
     pthread_mutex_unlock(&p->ctx->lock);
-    return ok ? VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW : 0;
+    return ok;
 }
 
 static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
diff --git a/video/out/vo_sdl.c b/video/out/vo_sdl.c
index 693a543c94..bc5ddda5c6 100644
--- a/video/out/vo_sdl.c
+++ b/video/out/vo_sdl.c
@@ -41,7 +41,6 @@
 #include "sub/osd.h"
 
 #include "video/mp_image.h"
-#include "video/vfcap.h"
 
 #include "win_state.h"
 #include "config.h"
@@ -847,16 +846,15 @@ static int preinit(struct vo *vo)
     return 0;
 }
 
-static int query_format(struct vo *vo, uint32_t format)
+static int query_format(struct vo *vo, int format)
 {
     struct priv *vc = vo->priv;
     int i, j;
-    int cap = VFCAP_CSP_SUPPORTED;
     for (i = 0; i < vc->renderer_info.num_texture_formats; ++i)
         for (j = 0; j < sizeof(formats) / sizeof(formats[0]); ++j)
             if (vc->renderer_info.texture_formats[i] == formats[j].sdl)
                 if (format == formats[j].mpv)
-                    return cap;
+                    return 1;
     return 0;
 }
 
diff --git a/video/out/vo_vaapi.c b/video/out/vo_vaapi.c
index b51337c7d9..f8edd83b4e 100644
--- a/video/out/vo_vaapi.c
+++ b/video/out/vo_vaapi.c
@@ -37,7 +37,6 @@
 #include "sub/img_convert.h"
 #include "x11_common.h"
 
-#include "video/vfcap.h"
 #include "video/mp_image.h"
 #include "video/vaapi.h"
 #include "video/hwdec.h"
@@ -175,11 +174,11 @@ static int reconfig(struct vo *vo, struct mp_image_params *params, int flags)
     return 0;
 }
 
-static int query_format(struct vo *vo, uint32_t imgfmt)
+static int query_format(struct vo *vo, int imgfmt)
 {
     struct priv *p = vo->priv;
     if (imgfmt == IMGFMT_VAAPI || va_image_format_from_imgfmt(p->va_image_formats, imgfmt))
-        return VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW;
+        return 1;
 
     return 0;
 }
diff --git a/video/out/vo_vdpau.c b/video/out/vo_vdpau.c
index e1795e321f..640e16e1a8 100644
--- a/video/out/vo_vdpau.c
+++ b/video/out/vo_vdpau.c
@@ -46,7 +46,6 @@
 #include "video/csputils.h"
 #include "sub/osd.h"
 #include "options/m_option.h"
-#include "video/vfcap.h"
 #include "video/mp_image.h"
 #include "osdep/timer.h"
 #include "bitmap_packer.h"
@@ -915,15 +914,14 @@ static struct mp_image *get_window_screenshot(struct vo *vo)
     return image;
 }
 
-static int query_format(struct vo *vo, uint32_t format)
+static int query_format(struct vo *vo, int format)
 {
     struct vdpctx *vc = vo->priv;
 
-    int flags = VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW;
     if (mp_vdpau_get_format(format, NULL, NULL))
-        return flags;
+        return 1;
     if (!vc->force_yuv && mp_vdpau_get_rgb_format(format, NULL))
-        return flags;
+        return 1;
     return 0;
 }
 
diff --git a/video/out/vo_wayland.c b/video/out/vo_wayland.c
index 168763ed5c..770142ae9d 100644
--- a/video/out/vo_wayland.c
+++ b/video/out/vo_wayland.c
@@ -25,7 +25,6 @@
 #include "config.h"
 
 #include "vo.h"
-#include "video/vfcap.h"
 #include "video/mp_image.h"
 #include "video/sws_utils.h"
 #include "video/memcpy_pic.h"
@@ -553,17 +552,17 @@ static void flip_page(struct vo *vo)
     }
 }
 
-static int query_format(struct vo *vo, uint32_t format)
+static int query_format(struct vo *vo, int format)
 {
     struct priv *p = vo->priv;
     struct supported_format *sf;
     wl_list_for_each_reverse(sf, &p->format_list, link) {
         if (sf->format.mp_format == format)
-            return VFCAP_CSP_SUPPORTED_BY_HW | VFCAP_CSP_SUPPORTED;
+            return 1;
     }
 
     if (mp_sws_supported_format(format))
-        return VFCAP_CSP_SUPPORTED;
+        return 1;
 
     return 0;
 }
diff --git a/video/out/vo_x11.c b/video/out/vo_x11.c
index 419467279f..13fa0a3c9f 100644
--- a/video/out/vo_x11.c
+++ b/video/out/vo_x11.c
@@ -29,7 +29,6 @@
 #include "vo.h"
 #include "video/csputils.h"
 #include "video/mp_image.h"
-#include "video/vfcap.h"
 #include "video/filter/vf.h"
 
 #include <X11/Xlib.h>
@@ -492,25 +491,19 @@ static void draw_image(struct vo *vo, mp_image_t *mpi)
     }
 }
 
-static int query_format(struct vo *vo, uint32_t format)
+static int query_format(struct vo *vo, int format)
 {
-    struct priv *p = vo->priv;
     MP_DBG(vo, "query_format was called: %x (%s)\n", format,
             vo_format_name(format));
     if (IMGFMT_IS_RGB(format)) {
         for (int n = 0; fmt2Xfmt[n].mpfmt; n++) {
-            if (fmt2Xfmt[n].mpfmt == format) {
-                if (IMGFMT_RGB_DEPTH(format) == p->ximage_depth) {
-                    return VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW;
-                } else {
-                    return VFCAP_CSP_SUPPORTED;
-                }
-            }
+            if (fmt2Xfmt[n].mpfmt == format)
+                return 1;
         }
     }
 
     if (sws_isSupportedInput(imgfmt2pixfmt(format)))
-        return VFCAP_CSP_SUPPORTED;
+        return 1;
     return 0;
 }
 
diff --git a/video/out/vo_xv.c b/video/out/vo_xv.c
index 07495fb89e..202ba13735 100644
--- a/video/out/vo_xv.c
+++ b/video/out/vo_xv.c
@@ -48,7 +48,6 @@
 #include "talloc.h"
 #include "common/msg.h"
 #include "vo.h"
-#include "video/vfcap.h"
 #include "video/mp_image.h"
 #include "video/img_fourcc.h"
 #include "x11_common.h"
@@ -681,17 +680,16 @@ static void draw_image(struct vo *vo, mp_image_t *mpi)
     }
 }
 
-static int query_format(struct vo *vo, uint32_t format)
+static int query_format(struct vo *vo, int format)
 {
     struct xvctx *ctx = vo->priv;
     uint32_t i;
-    int flag = VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW;
 
     int fourcc = find_xv_format(format);
     if (fourcc) {
         for (i = 0; i < ctx->formats; i++) {
             if (ctx->fo[i].id == fourcc)
-                return flag;
+                return 1;
         }
     }
     return 0;
diff --git a/video/vfcap.h b/video/vfcap.h
deleted file mode 100644
index 281a2f3c9d..0000000000
--- a/video/vfcap.h
+++ /dev/null
@@ -1,28 +0,0 @@
-/* VFCAP_* values: they are flags, returned by query_format():
- *
- * This file is part of MPlayer.
- *
- * MPlayer is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * MPlayer is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with MPlayer; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- */
-
-#ifndef MPLAYER_VFCAP_H
-#define MPLAYER_VFCAP_H
-
-// set, if the given colorspace is supported (with or without conversion)
-#define VFCAP_CSP_SUPPORTED 0x1
-// set, if the given colorspace is supported _without_ conversion
-#define VFCAP_CSP_SUPPORTED_BY_HW 0x2
-
-#endif /* MPLAYER_VFCAP_H */