spudec: use csputils for color conversion

Just to get rid of that conversion copy&pasted from the internet.

R and G are swapped for unknown reasons. Testing various subtitles
seem to yield the same results as VLC. The sub-bitmap renderers output
the correct colors. The colorspace conversion is used without problems
for vo_gl, vo_gl3 and vo_vdpau. The problem is most likely that
apparently, the DVD palette read from the subtitle track extradata is
converted to YUV using vobsub_palette_to_yuv(), and swapped in the
process. Or in other words, the YUV colors spu->global_palette are
encoded with R and G swapped.

Add some utility definition to csputils.c/h to make converting single
color values easier.

3 files changed, 30 insertions, 10 deletions

diff --git a/libvo/csputils.c b/libvo/csputils.c
index b894a2f869..b0255a00dc 100644
--- a/libvo/csputils.c
+++ b/libvo/csputils.c
@@ -361,3 +361,17 @@ void mp_invert_yuv2rgb(float out[3][4], float in[3][4])
     out[1][3] = -(out[1][0] * m03 + out[1][1] * m13 + out[1][2] * m23);
     out[2][3] = -(out[2][0] * m03 + out[2][1] * m13 + out[2][2] * m23);
 }
+
+// Multiply the color in c with the given matrix.
+// c is {R, G, B} or {Y, U, V} (depending on input/output and matrix).
+void mp_map_color(float matrix[3][4], uint8_t c[3])
+{
+    uint8_t in[3] = {c[0], c[1], c[2]};
+    for (int i = 0; i < 3; i++) {
+        double val = matrix[i][3] * 255;
+        for (int x = 0; x < 3; x++)
+            val += matrix[i][x] * in[x];
+        int ival = lrint(val);
+        c[i] = FFMIN(FFMAX(ival, 0), 255);
+    }
+}
diff --git a/libvo/csputils.h b/libvo/csputils.h
index 0dd75b0549..e4fb32e156 100644
--- a/libvo/csputils.h
+++ b/libvo/csputils.h
@@ -73,6 +73,12 @@ struct mp_csp_params {
     int input_bits;
 };
 
+#define MP_CSP_PARAMS_DEFAULTS {                                \
+    .colorspace = MP_CSP_DETAILS_DEFAULTS,                      \
+    .brightness = 0, .contrast = 1, .hue = 0, .saturation = 1,  \
+    .rgamma = 1, .ggamma = 1, .bgamma = 1,                      \
+    .texture_bits = 8, .input_bits = 8}
+
 enum mp_csp_equalizer_param {
     MP_CSP_EQ_BRIGHTNESS,
     MP_CSP_EQ_CONTRAST,
@@ -143,4 +149,6 @@ void mp_gen_yuv2rgb_map(struct mp_csp_params *params, uint8_t *map, int size);
                                                        (minv)[c][3] * (scale))
 void mp_invert_yuv2rgb(float out[3][4], float in[3][4]);
 
+void mp_map_color(float matrix[3][4], uint8_t c[3]);
+
 #endif /* MPLAYER_CSPUTILS_H */
diff --git a/sub/spudec.c b/sub/spudec.c
index 734b8f1846..353f72de7a 100644
--- a/sub/spudec.c
+++ b/sub/spudec.c
@@ -46,6 +46,7 @@
 #include "vobsub.h"
 #include "sub.h"
 #include "mpcommon.h"
+#include "libvo/csputils.h"
 
 typedef struct spu_packet_t packet_t;
 struct spu_packet_t {
@@ -186,6 +187,9 @@ static int spudec_alloc_image(spudec_handle_t *this, int stride, int height)
 static void setup_palette(spudec_handle_t *spu, uint32_t palette[256])
 {
     memset(palette, 0, sizeof(palette));
+    struct mp_csp_params csp = MP_CSP_PARAMS_DEFAULTS;
+    float cmatrix[3][4];
+    mp_get_yuv2rgb_coeffs(&csp, cmatrix);
     for (int i = 0; i < 4; ++i) {
         int alpha = spu->alpha[i];
         // extend 4 -> 8 bit
@@ -194,16 +198,10 @@ static void setup_palette(spudec_handle_t *spu, uint32_t palette[256])
             alpha = 0;
         int color = spu->custom ? spu->cuspal[i] :
                     spu->global_palette[spu->palette[i]];
-        int y = (color >> 16) & 0xff;
-        int u = (color >> 8) & 0xff;
-        int v = color & 0xff;
-        // stolen from some site, likely incorrect
-        int b = 1.164 * (y - 16)                     + 2.018 * (u - 128);
-        int g = 1.164 * (y - 16) - 0.813 * (v - 128) - 0.391 * (u - 128);
-        int r = 1.164 * (y - 16) + 1.596 * (v - 128);
-#define CL(x) FFMAX(FFMIN((x), 255), 0)
-        palette[i] = (alpha << 24) | CL(r) | (CL(g) << 8) | (CL(b) << 16);
-#undef CL
+        uint8_t c[3] = {(color >> 16) & 0xff, (color >> 8) & 0xff, color & 0xff};
+        mp_map_color(cmatrix, c);
+        // R and G swapped, possibly due to vobsub_palette_to_yuv()
+        palette[i] = (alpha << 24) | (c[2] << 16) | (c[1] << 8) | c[0];
     }
 }