video: make IMGFMT_RGB0 etc. exist even if libavutil doesn't support it

These formats are helpful for distinguishing surfaces with and without
alpha. Unfortunately, Libav and older version of FFmpeg don't support
them, so code will break. Fix this by treating these formats specially
on the mpv side, mapping them to RGBA on Libav, and unseting the alpha
bit in the mp_imgfmt_desc struct.

3 files changed, 36 insertions, 27 deletions

diff --git a/video/fmt-conversion.c b/video/fmt-conversion.c
index 7d37e8ca87..b908837547 100644
--- a/video/fmt-conversion.c
+++ b/video/fmt-conversion.c
@@ -159,16 +159,22 @@ static const struct {
     {IMGFMT_GBRP14_BE, PIX_FMT_GBRP14BE},
     {IMGFMT_GBRP14_LE, PIX_FMT_GBRP14LE},
 
+    {IMGFMT_RGBA64_BE,  PIX_FMT_RGBA64BE},
+    {IMGFMT_RGBA64_LE,  PIX_FMT_RGBA64LE},
+    {IMGFMT_BGRA64_BE,  PIX_FMT_BGRA64BE},
+    {IMGFMT_BGRA64_LE,  PIX_FMT_BGRA64LE},
+
     {IMGFMT_BGR0,  PIX_FMT_BGR0},
     {IMGFMT_0RGB,  PIX_FMT_0RGB},
     {IMGFMT_RGB0,  PIX_FMT_RGB0},
     {IMGFMT_0BGR,  PIX_FMT_0BGR},
     {IMGFMT_BGR0,  PIX_FMT_BGR0},
-
-    {IMGFMT_RGBA64_BE,  PIX_FMT_RGBA64BE},
-    {IMGFMT_RGBA64_LE,  PIX_FMT_RGBA64LE},
-    {IMGFMT_BGRA64_BE,  PIX_FMT_BGRA64BE},
-    {IMGFMT_BGRA64_LE,  PIX_FMT_BGRA64LE},
+#else
+    {IMGFMT_BGR0,  PIX_FMT_BGRA},
+    {IMGFMT_0RGB,  PIX_FMT_ARGB},
+    {IMGFMT_RGB0,  PIX_FMT_RGBA},
+    {IMGFMT_0BGR,  PIX_FMT_ABGR},
+    {IMGFMT_BGR0,  PIX_FMT_BGRA},
 #endif
 
 #if HAVE_AVCODEC_NEW_VDPAU_API
diff --git a/video/img_format.c b/video/img_format.c
index 0316261393..5e3c382df4 100644
--- a/video/img_format.c
+++ b/video/img_format.c
@@ -157,12 +157,20 @@ const char *mp_imgfmt_to_name(unsigned int fmt)
     return NULL;
 }
 
-static struct mp_imgfmt_desc get_avutil_fmt(enum PixelFormat fmt)
+struct mp_imgfmt_desc mp_imgfmt_get_desc(int mpfmt)
 {
-    const AVPixFmtDescriptor *pd = &av_pix_fmt_descriptors[fmt];
-    int mpfmt = pixfmt2imgfmt(fmt);
-    if (!pd || !mpfmt)
+    enum PixelFormat fmt = imgfmt2pixfmt(mpfmt);
+    if (fmt == PIX_FMT_NONE) {
+        const char *name = mp_imgfmt_to_name(mpfmt);
+        if (name) {
+            mp_msg(MSGT_DECVIDEO, MSGL_V,
+                   "libavutil does not know image format '%s'\n", name);
+        }
         return (struct mp_imgfmt_desc) {0};
+    }
+
+    const AVPixFmtDescriptor *pd = &av_pix_fmt_descriptors[fmt];
+    assert(pd);
 
     struct mp_imgfmt_desc desc = {
         .id = mpfmt,
@@ -217,6 +225,9 @@ static struct mp_imgfmt_desc get_avutil_fmt(enum PixelFormat fmt)
         desc.flags |= MP_IMGFLAG_ALPHA;
 #endif
 
+    if (mpfmt >= IMGFMT_RGB0_START && mpfmt <= IMGFMT_RGB0_END)
+        desc.flags &= ~MP_IMGFLAG_ALPHA;
+
     if (desc.num_planes == pd->nb_components)
         desc.flags |= MP_IMGFLAG_PLANAR;
 
@@ -251,19 +262,6 @@ static struct mp_imgfmt_desc get_avutil_fmt(enum PixelFormat fmt)
     return desc;
 }
 
-struct mp_imgfmt_desc mp_imgfmt_get_desc(unsigned int out_fmt)
-{
-    struct mp_imgfmt_desc fmt = {0};
-    enum PixelFormat avfmt = imgfmt2pixfmt(out_fmt);
-    if (avfmt != PIX_FMT_NONE)
-        fmt = get_avutil_fmt(avfmt);
-    if (!fmt.id) {
-        mp_msg(MSGT_DECVIDEO, MSGL_V, "mp_image: unknown out_fmt: 0x%X\n",
-               out_fmt);
-    }
-    return fmt;
-}
-
 // Find a format that is MP_IMGFLAG_YUV_P with the following configuration.
 int mp_imgfmt_find_yuv_planar(int xs, int ys, int planes, int component_bits)
 {
diff --git a/video/img_format.h b/video/img_format.h
index 4d876304a0..1cfe4465ec 100644
--- a/video/img_format.h
+++ b/video/img_format.h
@@ -78,7 +78,7 @@ struct mp_imgfmt_desc {
     int8_t ys[MP_MAX_PLANES];
 };
 
-struct mp_imgfmt_desc mp_imgfmt_get_desc(unsigned int out_fmt);
+struct mp_imgfmt_desc mp_imgfmt_get_desc(int imgfmt);
 
 enum mp_imgfmt {
     IMGFMT_NONE = 0,
@@ -176,13 +176,9 @@ enum mp_imgfmt {
 
     // Byte accessed (low address to high address)
     IMGFMT_ARGB,
-    IMGFMT_0RGB,                // "0" is a padding byte (as opposed to alpha)
     IMGFMT_BGRA,
-    IMGFMT_BGR0,
     IMGFMT_ABGR,
-    IMGFMT_0BGR,
     IMGFMT_RGBA,
-    IMGFMT_RGB0,
     IMGFMT_BGR24,               // 3 bytes per pixel
     IMGFMT_RGB24,
     IMGFMT_RGB48_LE,            // 6 bytes per pixel, uint16_t channels
@@ -192,6 +188,15 @@ enum mp_imgfmt {
     IMGFMT_BGRA64_LE,
     IMGFMT_BGRA64_BE,
 
+    // Like e.g. IMGFMT_ARGB, but has a padding byte instead of alpha
+    IMGFMT_0RGB,
+    IMGFMT_BGR0,
+    IMGFMT_0BGR,
+    IMGFMT_RGB0,
+
+    IMGFMT_RGB0_START = IMGFMT_0RGB,
+    IMGFMT_RGB0_END = IMGFMT_RGB0,
+
     // Accessed with bit-shifts (components ordered from LSB to MSB)
     IMGFMT_RGB8,                // r3 g3 b2
     IMGFMT_BGR8,