From f30c2c99d16527c045caee90c9f7b2a7a7c85c00 Mon Sep 17 00:00:00 2001
From: wm4 <wm4@nowhere>
Date: Sun, 1 Dec 2013 20:45:44 +0100
Subject: mp_image: deal with FFmpeg PSEUDOPAL braindeath

We got a crash in libavutil when encoding with Y8 (GRAY8). The reason
was that libavutil was copying an Y8 image allocated by us, and expected
a palette. This is because GRAY8 is a PSEUDOPAL format. It's not clear
what PSEUDOPAL means, and it makes literally no sense at all. However,
it does expect a palette allocated for some formats that are not
paletted, and libavutil crashed when trying to access the non-existent
palette.
---
 video/img_format.c | 5 +++++
 video/img_format.h | 7 +++++++
 video/mp_image.c   | 6 +++---
 3 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/video/img_format.c b/video/img_format.c
index b3d0c8bd91..9e396832bf 100644
--- a/video/img_format.c
+++ b/video/img_format.c
@@ -233,6 +233,11 @@ struct mp_imgfmt_desc mp_imgfmt_get_desc(int mpfmt)
             desc.bytes[p] = desc.bpp[p] / 8;
     }
 
+    // PSEUDOPAL is a complete braindeath nightmare, however it seems various
+    // parts of FFmpeg expect that it has a palette allocated.
+    if (pd->flags & (PIX_FMT_PAL | PIX_FMT_PSEUDOPAL))
+        desc.flags |= MP_IMGFLAG_PAL;
+
     if ((desc.flags & MP_IMGFLAG_YUV) && (desc.flags & MP_IMGFLAG_BYTE_ALIGNED))
     {
         bool same_depth = true;
diff --git a/video/img_format.h b/video/img_format.h
index 5abeb9c46a..e7b6fc5566 100644
--- a/video/img_format.h
+++ b/video/img_format.h
@@ -56,6 +56,13 @@
 #define MP_IMGFLAG_BE 0x4000
 // set if in native (host) endian, or endian independent
 #define MP_IMGFLAG_NE MP_SELECT_LE_BE(MP_IMGFLAG_LE, MP_IMGFLAG_BE)
+// Carries a palette in plane[1] (see IMGFMT_PAL8 for format of the palette).
+// Note that some non-paletted formats have this flag set, because FFmpeg
+// mysteriously expects some formats to carry a palette plane for no apparent
+// reason. FFmpeg developer braindeath?
+// The only real paletted format we support is IMGFMT_PAL8, so check for that
+// format directly if you want an actual paletted format.
+#define MP_IMGFLAG_PAL 0x8000
 
 // Exactly one of these bits is set in mp_imgfmt_desc.flags
 #define MP_IMGFLAG_COLOR_CLASS_MASK \
diff --git a/video/mp_image.c b/video/mp_image.c
index 242e09e2d0..b0f174d769 100644
--- a/video/mp_image.c
+++ b/video/mp_image.c
@@ -132,7 +132,7 @@ static void mp_image_alloc_planes(struct mp_image *mpi)
         mpi->stride[n] = FFALIGN(line_bytes, SWS_MIN_BYTE_ALIGN);
         plane_size[n] = mpi->stride[n] * alloc_h;
     }
-    if (mpi->imgfmt == IMGFMT_PAL8)
+    if (mpi->fmt.flags & MP_IMGFLAG_PAL)
         plane_size[1] = MP_PALETTE_SIZE;
 
     size_t sum = 0;
@@ -330,7 +330,7 @@ void mp_image_copy(struct mp_image *dst, struct mp_image *src)
         memcpy_pic(dst->planes[n], src->planes[n], line_bytes, dst->plane_h[n],
                    dst->stride[n], src->stride[n]);
     }
-    if (dst->imgfmt == IMGFMT_PAL8)
+    if (dst->fmt.flags & MP_IMGFLAG_PAL)
         memcpy(dst->planes[1], src->planes[1], MP_PALETTE_SIZE);
 }
 
@@ -349,7 +349,7 @@ void mp_image_copy_attributes(struct mp_image *dst, struct mp_image *src)
         dst->levels = src->levels;
         dst->chroma_location = src->chroma_location;
     }
-    if (dst->imgfmt == IMGFMT_PAL8 && src->imgfmt == IMGFMT_PAL8) {
+    if ((dst->fmt.flags & MP_IMGFLAG_PAL) && (src->fmt.flags & MP_IMGFLAG_PAL)) {
         if (dst->planes[1] && src->planes[1])
             memcpy(dst->planes[1], src->planes[1], MP_PALETTE_SIZE);
     }
-- 
cgit v1.2.3