video: make IMGFMT_RGB0 etc. exist even if libavutil doesn't support it

These formats are helpful for distinguishing surfaces with and without
alpha. Unfortunately, Libav and older version of FFmpeg don't support
them, so code will break. Fix this by treating these formats specially
on the mpv side, mapping them to RGBA on Libav, and unseting the alpha
bit in the mp_imgfmt_desc struct.

1 files changed, 15 insertions, 17 deletions

diff --git a/video/img_format.c b/video/img_format.c
index 0316261393..5e3c382df4 100644
--- a/video/img_format.c
+++ b/video/img_format.c
@@ -157,12 +157,20 @@ const char *mp_imgfmt_to_name(unsigned int fmt)
     return NULL;
 }
 
-static struct mp_imgfmt_desc get_avutil_fmt(enum PixelFormat fmt)
+struct mp_imgfmt_desc mp_imgfmt_get_desc(int mpfmt)
 {
-    const AVPixFmtDescriptor *pd = &av_pix_fmt_descriptors[fmt];
-    int mpfmt = pixfmt2imgfmt(fmt);
-    if (!pd || !mpfmt)
+    enum PixelFormat fmt = imgfmt2pixfmt(mpfmt);
+    if (fmt == PIX_FMT_NONE) {
+        const char *name = mp_imgfmt_to_name(mpfmt);
+        if (name) {
+            mp_msg(MSGT_DECVIDEO, MSGL_V,
+                   "libavutil does not know image format '%s'\n", name);
+        }
         return (struct mp_imgfmt_desc) {0};
+    }
+
+    const AVPixFmtDescriptor *pd = &av_pix_fmt_descriptors[fmt];
+    assert(pd);
 
     struct mp_imgfmt_desc desc = {
         .id = mpfmt,
@@ -217,6 +225,9 @@ static struct mp_imgfmt_desc get_avutil_fmt(enum PixelFormat fmt)
         desc.flags |= MP_IMGFLAG_ALPHA;
 #endif
 
+    if (mpfmt >= IMGFMT_RGB0_START && mpfmt <= IMGFMT_RGB0_END)
+        desc.flags &= ~MP_IMGFLAG_ALPHA;
+
     if (desc.num_planes == pd->nb_components)
         desc.flags |= MP_IMGFLAG_PLANAR;
 
@@ -251,19 +262,6 @@ static struct mp_imgfmt_desc get_avutil_fmt(enum PixelFormat fmt)
     return desc;
 }
 
-struct mp_imgfmt_desc mp_imgfmt_get_desc(unsigned int out_fmt)
-{
-    struct mp_imgfmt_desc fmt = {0};
-    enum PixelFormat avfmt = imgfmt2pixfmt(out_fmt);
-    if (avfmt != PIX_FMT_NONE)
-        fmt = get_avutil_fmt(avfmt);
-    if (!fmt.id) {
-        mp_msg(MSGT_DECVIDEO, MSGL_V, "mp_image: unknown out_fmt: 0x%X\n",
-               out_fmt);
-    }
-    return fmt;
-}
-
 // Find a format that is MP_IMGFLAG_YUV_P with the following configuration.
 int mp_imgfmt_find_yuv_planar(int xs, int ys, int planes, int component_bits)
 {