csputils: remove obscure int_bits matrix scaling

This has no reason to be there. Put the functionality into another
function instead. While we're at it, also adjust for possible accuracy
issues with high bit depth YUV (matters for rendering subtitles into
screenshots only).

5 files changed, 22 insertions, 37 deletions

diff --git a/demux/demux_disc.c b/demux/demux_disc.c
index a816a74fa8..dd2fd866a5 100644
--- a/demux/demux_disc.c
+++ b/demux/demux_disc.c
@@ -93,8 +93,6 @@ static void add_dvd_streams(demuxer_t *demuxer)
 
             // emulate the extradata
             struct mp_csp_params csp = MP_CSP_PARAMS_DEFAULTS;
-            csp.int_bits_in = 8;
-            csp.int_bits_out = 8;
             struct mp_cmat cmatrix;
             mp_get_yuv2rgb_coeffs(&csp, &cmatrix);
 
@@ -102,8 +100,9 @@ static void add_dvd_streams(demuxer_t *demuxer)
             s = talloc_asprintf_append(s, "palette: ");
             for (int i = 0; i < 16; i++) {
                 int color = info.palette[i];
-                int c[3] = {(color >> 16) & 0xff, (color >> 8) & 0xff, color & 0xff};
-                mp_map_int_color(&cmatrix, 8, c);
+                int y[3] = {(color >> 16) & 0xff, (color >> 8) & 0xff, color & 0xff};
+                int c[3];
+                mp_map_fixp_color(&cmatrix, 8, y, 8, c);
                 color = (c[2] << 16) | (c[1] << 8) | c[0];
 
                 if (i != 0)
diff --git a/sub/draw_bmp.c b/sub/draw_bmp.c
index 2093e5f9d1..0c12c7078a 100644
--- a/sub/draw_bmp.c
+++ b/sub/draw_bmp.c
@@ -290,8 +290,8 @@ static void draw_ass(struct mp_draw_sub_cache *cache, struct mp_rect bb,
     struct mp_csp_params cspar = MP_CSP_PARAMS_DEFAULTS;
     mp_csp_set_image_params(&cspar, &temp->params);
     cspar.levels_out = MP_CSP_LEVELS_PC; // RGB (libass.color)
-    cspar.int_bits_in = bits;
-    cspar.int_bits_out = 8;
+    cspar.input_bits = bits;
+    cspar.texture_bits = (bits + 7) / 8 * 8;
 
     struct mp_cmat yuv2rgb, rgb2yuv;
     bool need_conv = temp->fmt.flags & MP_IMGFLAG_YUV;
@@ -312,9 +312,10 @@ static void draw_ass(struct mp_draw_sub_cache *cache, struct mp_rect bb,
         int g = (sb->libass.color >> 16) & 0xFF;
         int b = (sb->libass.color >> 8) & 0xFF;
         int a = 255 - (sb->libass.color & 0xFF);
-        int color_yuv[3] = {r, g, b};
+        int color_yuv[3];
         if (need_conv) {
-            mp_map_int_color(&rgb2yuv, bits, color_yuv);
+            int rgb[3] = {r, g, b};
+            mp_map_fixp_color(&rgb2yuv, 8, rgb, cspar.texture_bits, color_yuv);
         } else {
             color_yuv[0] = g;
             color_yuv[1] = b;
diff --git a/sub/sd_ass.c b/sub/sd_ass.c
index 9e4da9a379..092b260677 100644
--- a/sub/sd_ass.c
+++ b/sub/sd_ass.c
@@ -667,8 +667,6 @@ static void mangle_colors(struct sd *sd, struct sub_bitmaps *parts)
     struct mp_csp_params vs_params = MP_CSP_PARAMS_DEFAULTS;
     vs_params.colorspace = csp;
     vs_params.levels_in = levels;
-    vs_params.int_bits_in = 8;
-    vs_params.int_bits_out = 8;
     struct mp_cmat vs_yuv2rgb, vs_rgb2yuv;
     mp_get_yuv2rgb_coeffs(&vs_params, &vs_yuv2rgb);
     mp_invert_yuv2rgb(&vs_rgb2yuv, &vs_yuv2rgb);
@@ -677,8 +675,6 @@ static void mangle_colors(struct sd *sd, struct sub_bitmaps *parts)
     struct mp_csp_params rgb_params = MP_CSP_PARAMS_DEFAULTS;
     rgb_params.colorspace = params.colorspace;
     rgb_params.levels_in = params.colorlevels;
-    rgb_params.int_bits_in = 8;
-    rgb_params.int_bits_out = 8;
     struct mp_cmat vs2rgb;
     mp_get_yuv2rgb_coeffs(&rgb_params, &vs2rgb);
 
@@ -689,9 +685,9 @@ static void mangle_colors(struct sd *sd, struct sub_bitmaps *parts)
         int g = (color >> 16u) & 0xff;
         int b = (color >>  8u) & 0xff;
         int a = 0xff - (color & 0xff);
-        int c[3] = {r, g, b};
-        mp_map_int_color(&vs_rgb2yuv, 8, c);
-        mp_map_int_color(&vs2rgb, 8, c);
-        sb->libass.color = MP_ASS_RGBA(c[0], c[1], c[2], a);
+        int rgb[3] = {r, g, b}, yuv[3];
+        mp_map_fixp_color(&vs_rgb2yuv, 8, rgb, 8, yuv);
+        mp_map_fixp_color(&vs2rgb, 8, yuv, 8, rgb);
+        sb->libass.color = MP_ASS_RGBA(rgb[0], rgb[1], rgb[2], a);
     }
 }
diff --git a/video/csputils.c b/video/csputils.c
index 1f0fa7d90e..71ab0513a5 100644
--- a/video/csputils.c
+++ b/video/csputils.c
@@ -710,16 +710,6 @@ void mp_get_yuv2rgb_coeffs(struct mp_csp_params *params, struct mp_cmat *m)
                   - (m->m[i][1] + m->m[i][2]) * yuvlev.cmid
                   + params->brightness;
     }
-
-    int in_bits = FFMAX(params->int_bits_in, 1);
-    int out_bits = FFMAX(params->int_bits_out, 1);
-    double in_scale = (1 << in_bits) - 1.0;
-    double out_scale = (1 << out_bits) - 1.0;
-    for (int i = 0; i < 3; i++) {
-        m->c[i] *= out_scale; // constant is 1.0
-        for (int x = 0; x < 3; x++)
-            m->m[i][x] *= out_scale / in_scale;
-    }
 }
 
 // Set colorspace related fields in p from f. Don't touch other fields.
@@ -793,16 +783,17 @@ void mp_invert_yuv2rgb(struct mp_cmat *out, struct mp_cmat *in)
 }
 
 // Multiply the color in c with the given matrix.
-// c is {R, G, B} or {Y, U, V} (depending on input/output and matrix).
-// Output is clipped to the given number of bits.
-void mp_map_int_color(struct mp_cmat *matrix, int clip_bits, int c[3])
+// i/o is {R, G, B} or {Y, U, V} (depending on input/output and matrix), using
+// a fixed point representation with the given number of bits (so for bits==8,
+// [0,255] maps to [0,1]). The output is clipped to the range as needed.
+void mp_map_fixp_color(struct mp_cmat *matrix, int ibits, int in[3],
+                                               int obits, int out[3])
 {
-    int in[3] = {c[0], c[1], c[2]};
     for (int i = 0; i < 3; i++) {
         double val = matrix->c[i];
         for (int x = 0; x < 3; x++)
-            val += matrix->m[i][x] * in[x];
-        int ival = lrint(val);
-        c[i] = av_clip(ival, 0, (1 << clip_bits) - 1);
+            val += matrix->m[i][x] * in[x] / ((1 << ibits) - 1);
+        int ival = lrint(val * ((1 << obits) - 1));
+        out[i] = av_clip(ival, 0, (1 << obits) - 1);
     }
 }
diff --git a/video/csputils.h b/video/csputils.h
index 0a9b11af9d..bc43a146a8 100644
--- a/video/csputils.h
+++ b/video/csputils.h
@@ -130,9 +130,6 @@ struct mp_csp_params {
     // texture_bits/input_bits is for rescaling fixed point input to range [0,1]
     int texture_bits;
     int input_bits;
-    // for scaling integer input and output (if 0, assume range [0,1])
-    int int_bits_in;
-    int int_bits_out;
 };
 
 #define MP_CSP_PARAMS_DEFAULTS {                                \
@@ -259,6 +256,7 @@ void mp_get_yuv2rgb_coeffs(struct mp_csp_params *params, struct mp_cmat *yuv2rgb
 
 void mp_invert_matrix3x3(float m[3][3]);
 void mp_invert_yuv2rgb(struct mp_cmat *out, struct mp_cmat *in);
-void mp_map_int_color(struct mp_cmat *matrix, int clip_bits, int c[3]);
+void mp_map_fixp_color(struct mp_cmat *matrix, int ibits, int in[3],
+                                               int obits, int out[3]);
 
 #endif /* MPLAYER_CSPUTILS_H */