mp_image: change how palette is handled

According to DOCS/OUTDATED-tech/colorspaces.txt, the following formats
are supposed to be palettized:

    IMGFMT_BGR8
    IMGFMT_RGB8,
    IMGFMT_BGR4_CHAR
    IMGFMT_RGB4_CHAR
    IMGFMT_BGR4
    IMGFMT_RGB4

Of these, only BGR8 and RGB8 are actually treated as palettized in some
way. ffmpeg has only one palettized format (AV_PIX_FMT_PAL8), and
IMGFMT_BGR8 was inconsistently mapped to packed non-palettized RGB
formats too (AV_PIX_FMT_BGR8). Moreover, vf_scale.c contained messy
hacks to generate a palette when AV_PIX_FMT_BGR8 is output. (libswscale
does not support AV_PIX_FMT_PAL8 output in the first place.)

Get rid of all of this, and introduce IMGFMT_PAL8, which directly maps
to AV_PIX_FMT_PAL8. Remove the palette creation code from vf_scale.c.
IMGFMT_BGR8 maps to AV_PIX_FMT_RGB8 (don't ask me why it's swapped),
without any palette use. Enabling it in vo_x11 or using it as vf_scale
input seems to give correct results.

6 files changed, 5 insertions, 56 deletions

diff --git a/video/filter/vf.c b/video/filter/vf.c
index b31d8b3971..1d21413787 100644
--- a/video/filter/vf.c
+++ b/video/filter/vf.c
@@ -380,6 +380,9 @@ void vf_clone_mpi_attributes(mp_image_t *dst, mp_image_t *src)
         dst->colorspace = src->colorspace;
         dst->levels = src->levels;
     }
+    if (dst->imgfmt == IMGFMT_PAL8 && src->imgfmt == IMGFMT_PAL8) {
+        memcpy(dst->planes[1], src->planes[1], MP_PALETTE_SIZE);
+    }
 }
 
 // Used by filters to add a filtered frame to the output queue.
diff --git a/video/filter/vf_dlopen.c b/video/filter/vf_dlopen.c
index 323f5231ee..81d924ca40 100644
--- a/video/filter/vf_dlopen.c
+++ b/video/filter/vf_dlopen.c
@@ -240,7 +240,7 @@ static int query_format(struct vf_instance *vf, unsigned int fmt)
 {
     if (IMGFMT_IS_HWACCEL(fmt))
         return 0;  // these can't really be filtered
-    if (fmt == IMGFMT_RGB8 || fmt == IMGFMT_BGR8)
+    if (fmt == IMGFMT_PAL8)
         return 0;  // we don't have palette support, sorry
     const char *fmtname = mp_imgfmt_to_name(fmt);
     if (!fmtname)
diff --git a/video/filter/vf_mirror.c b/video/filter/vf_mirror.c
index 10ac5f308f..b826ee46f7 100644
--- a/video/filter/vf_mirror.c
+++ b/video/filter/vf_mirror.c
@@ -102,7 +102,6 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi)
 	mirror(dmpi->planes[0],mpi->planes[0],
 	       dmpi->stride[0],mpi->stride[0],
 	       dmpi->w,dmpi->h,dmpi->bpp>>3,mpi->imgfmt);
-	dmpi->planes[1]=mpi->planes[1]; // passthrough rgb8 palette
     }
 
     talloc_free(mpi);
diff --git a/video/filter/vf_rotate.c b/video/filter/vf_rotate.c
index 1ef6bf2300..f44c874c1f 100644
--- a/video/filter/vf_rotate.c
+++ b/video/filter/vf_rotate.c
@@ -99,7 +99,6 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi)
 	rotate(dmpi->planes[0],mpi->planes[0],
 	       dmpi->stride[0],mpi->stride[0],
 	       dmpi->w,dmpi->h,dmpi->bpp>>3,vf->priv->direction);
-	dmpi->planes[1] = mpi->planes[1]; // passthrough rgb8 palette
     }
 
     talloc_free(mpi);
diff --git a/video/filter/vf_scale.c b/video/filter/vf_scale.c
index 30a185390e..a10825ee56 100644
--- a/video/filter/vf_scale.c
+++ b/video/filter/vf_scale.c
@@ -47,7 +47,6 @@ static struct vf_priv_s {
     double param[2];
     unsigned int fmt;
     struct SwsContext *ctx;
-    unsigned char* palette;
     int interlaced;
     int noup;
     int accurate_rnd;
@@ -57,9 +56,6 @@ static struct vf_priv_s {
   -1,-1,
   0,
   {SWS_PARAM_DEFAULT, SWS_PARAM_DEFAULT},
-  0,
-  NULL,
-  NULL
 };
 
 static int mp_sws_set_colorspace(struct SwsContext *sws,
@@ -162,6 +158,7 @@ static int preferred_conversions[][2] = {
     {IMGFMT_GBRP, IMGFMT_RGB24},
     {IMGFMT_GBRP, IMGFMT_BGR32},
     {IMGFMT_GBRP, IMGFMT_RGB32},
+    {IMGFMT_PAL8, IMGFMT_BGR32},
     {0, 0}
 };
 
@@ -210,7 +207,6 @@ static int config(struct vf_instance *vf,
     unsigned int best=find_best_out(vf, outfmt);
     int int_sws_flags=0;
     int round_w=0, round_h=0;
-    int i;
     SwsFilter *srcFilter, *dstFilter;
     enum PixelFormat dfmt, sfmt;
 
@@ -221,7 +217,6 @@ static int config(struct vf_instance *vf,
 	return 0;
     }
     sfmt = imgfmt2pixfmt(outfmt);
-    if (outfmt == IMGFMT_RGB8 || outfmt == IMGFMT_BGR8) sfmt = PIX_FMT_PAL8;
     dfmt = imgfmt2pixfmt(best);
 
     vf->next->query_format(vf->next,best);
@@ -316,51 +311,6 @@ static int config(struct vf_instance *vf,
     }
     vf->priv->fmt=best;
 
-    free(vf->priv->palette);
-    vf->priv->palette=NULL;
-    switch(best){
-    case IMGFMT_RGB8: {
-      /* set 332 palette for 8 bpp */
-	vf->priv->palette=malloc(4*256);
-	for(i=0; i<256; i++){
-	    vf->priv->palette[4*i+0]=4*(i>>6)*21;
-	    vf->priv->palette[4*i+1]=4*((i>>3)&7)*9;
-	    vf->priv->palette[4*i+2]=4*((i&7)&7)*9;
-            vf->priv->palette[4*i+3]=0;
-	}
-	break; }
-    case IMGFMT_BGR8: {
-      /* set 332 palette for 8 bpp */
-	vf->priv->palette=malloc(4*256);
-	for(i=0; i<256; i++){
-	    vf->priv->palette[4*i+0]=4*(i&3)*21;
-	    vf->priv->palette[4*i+1]=4*((i>>2)&7)*9;
-	    vf->priv->palette[4*i+2]=4*((i>>5)&7)*9;
-            vf->priv->palette[4*i+3]=0;
-	}
-	break; }
-    case IMGFMT_BGR4:
-    case IMGFMT_BG4B: {
-	vf->priv->palette=malloc(4*16);
-	for(i=0; i<16; i++){
-	    vf->priv->palette[4*i+0]=4*(i&1)*63;
-	    vf->priv->palette[4*i+1]=4*((i>>1)&3)*21;
-	    vf->priv->palette[4*i+2]=4*((i>>3)&1)*63;
-            vf->priv->palette[4*i+3]=0;
-	}
-	break; }
-    case IMGFMT_RGB4:
-    case IMGFMT_RG4B: {
-	vf->priv->palette=malloc(4*16);
-	for(i=0; i<16; i++){
-	    vf->priv->palette[4*i+0]=4*(i>>3)*63;
-	    vf->priv->palette[4*i+1]=4*((i>>1)&3)*21;
-	    vf->priv->palette[4*i+2]=4*((i&1)&1)*63;
-            vf->priv->palette[4*i+3]=0;
-	}
-	break; }
-    }
-
     if (!opts->screen_size_x && !opts->screen_size_y
         && !(opts->screen_size_xy >= 0.001)) {
 	// Compute new d_width and d_height, preserving aspect
@@ -566,7 +516,6 @@ static int query_format(struct vf_instance *vf, unsigned int fmt){
 
 static void uninit(struct vf_instance *vf){
     if(vf->priv->ctx) sws_freeContext(vf->priv->ctx);
-    free(vf->priv->palette);
     free(vf->priv);
 }
 
diff --git a/video/filter/vf_sub.c b/video/filter/vf_sub.c
index 10fcdbfc5b..96100801ff 100644
--- a/video/filter/vf_sub.c
+++ b/video/filter/vf_sub.c
@@ -144,7 +144,6 @@ static void prepare_image(struct vf_instance *vf, struct mp_image *dmpi,
 		   mpi->h,
                    dmpi->stride[0],
 		   mpi->stride[0]);
-        dmpi->planes[1] = mpi->planes[1];   // passthrough rgb8 palette
     }
     if (tmargin)
         blank(dmpi, 0, tmargin);