vf_format: don't crash if nonsense parameters are passed

It was "by design" possible to make mpv crash if the parameters didn't
make enough sense, like "format=rgb24:yuv420p". While forcing the format
has some minor (rather questionable) use for debugging, allowing it to
crash is just stupid.

2 files changed, 25 insertions, 14 deletions

diff --git a/DOCS/man/vf.rst b/DOCS/man/vf.rst
index d79774c721..984bdda656 100644
--- a/DOCS/man/vf.rst
+++ b/DOCS/man/vf.rst
@@ -209,17 +209,8 @@ Available filters are:
     ``<fmt>``
         Format name, e.g. rgb15, bgr24, 420p, etc. (default: don't change).
     ``<outfmt>``
-        Format name that should be substituted for the output. If this is not
-        100% compatible with the ``<fmt>`` value, it will crash.
-
-        .. admonition:: Examples
-
-            ====================== =====================
-            Valid                  Invalid (will crash)
-            ====================== =====================
-            ``format=rgb24:bgr24`` ``format=rgb24:420p``
-            ``format=yuyv:uyvy``
-            ====================== =====================
+        Format name that should be substituted for the output. If they do not
+        have the same bytes per pixel and chroma subsamplimg, it will fail.
 
 ``noformat[=fmt]``
     Restricts the color space for the next filter without doing any conversion.
diff --git a/video/filter/vf_format.c b/video/filter/vf_format.c
index df629e3276..a638cb37a1 100644
--- a/video/filter/vf_format.c
+++ b/video/filter/vf_format.c
@@ -21,8 +21,8 @@
 #include <string.h>
 #include <inttypes.h>
 
-#include "config.h"
 #include "common/msg.h"
+#include "common/common.h"
 
 #include "video/img_format.h"
 #include "video/mp_image.h"
@@ -35,11 +35,32 @@ struct vf_priv_s {
     int outfmt;
 };
 
+static bool is_compatible(int fmt1, int fmt2)
+{
+    struct mp_imgfmt_desc d1 = mp_imgfmt_get_desc(fmt1);
+    struct mp_imgfmt_desc d2 = mp_imgfmt_get_desc(fmt2);
+    if (d1.num_planes < d2.num_planes)
+        return false;
+    if (!(d1.flags & MP_IMGFLAG_BYTE_ALIGNED) ||
+        !(d2.flags & MP_IMGFLAG_BYTE_ALIGNED))
+        return false;
+    for (int n = 0; n < MPMIN(d1.num_planes, d2.num_planes); n++) {
+        if (d1.bytes[n] != d2.bytes[n])
+            return false;
+        if (d1.xs[n] != d2.xs[n] || d1.ys[n] != d2.ys[n])
+            return false;
+    }
+    return true;
+}
+
 static int query_format(struct vf_instance *vf, unsigned int fmt)
 {
     if (fmt == vf->priv->fmt || !vf->priv->fmt) {
-        if (vf->priv->outfmt)
+        if (vf->priv->outfmt) {
+            if (!is_compatible(fmt, vf->priv->outfmt))
+                return 0;
             fmt = vf->priv->outfmt;
+        }
         return vf_next_query_format(vf, fmt);
     }
     return 0;
@@ -56,7 +77,6 @@ static int reconfig(struct vf_instance *vf, struct mp_image_params *in,
 
 static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi)
 {
-    // As documented in the manpage, the user can easily provoke crashes
     if (vf->priv->outfmt)
         mp_image_setfmt(mpi, vf->priv->outfmt);
     return mpi;