csputils: remove obscure int_bits matrix scaling

This has no reason to be there. Put the functionality into another
function instead. While we're at it, also adjust for possible accuracy
issues with high bit depth YUV (matters for rendering subtitles into
screenshots only).

2 files changed, 10 insertions, 21 deletions

diff --git a/video/csputils.c b/video/csputils.c
index 1f0fa7d90e..71ab0513a5 100644
--- a/video/csputils.c
+++ b/video/csputils.c
@@ -710,16 +710,6 @@ void mp_get_yuv2rgb_coeffs(struct mp_csp_params *params, struct mp_cmat *m)
                   - (m->m[i][1] + m->m[i][2]) * yuvlev.cmid
                   + params->brightness;
     }
-
-    int in_bits = FFMAX(params->int_bits_in, 1);
-    int out_bits = FFMAX(params->int_bits_out, 1);
-    double in_scale = (1 << in_bits) - 1.0;
-    double out_scale = (1 << out_bits) - 1.0;
-    for (int i = 0; i < 3; i++) {
-        m->c[i] *= out_scale; // constant is 1.0
-        for (int x = 0; x < 3; x++)
-            m->m[i][x] *= out_scale / in_scale;
-    }
 }
 
 // Set colorspace related fields in p from f. Don't touch other fields.
@@ -793,16 +783,17 @@ void mp_invert_yuv2rgb(struct mp_cmat *out, struct mp_cmat *in)
 }
 
 // Multiply the color in c with the given matrix.
-// c is {R, G, B} or {Y, U, V} (depending on input/output and matrix).
-// Output is clipped to the given number of bits.
-void mp_map_int_color(struct mp_cmat *matrix, int clip_bits, int c[3])
+// i/o is {R, G, B} or {Y, U, V} (depending on input/output and matrix), using
+// a fixed point representation with the given number of bits (so for bits==8,
+// [0,255] maps to [0,1]). The output is clipped to the range as needed.
+void mp_map_fixp_color(struct mp_cmat *matrix, int ibits, int in[3],
+                                               int obits, int out[3])
 {
-    int in[3] = {c[0], c[1], c[2]};
     for (int i = 0; i < 3; i++) {
         double val = matrix->c[i];
         for (int x = 0; x < 3; x++)
-            val += matrix->m[i][x] * in[x];
-        int ival = lrint(val);
-        c[i] = av_clip(ival, 0, (1 << clip_bits) - 1);
+            val += matrix->m[i][x] * in[x] / ((1 << ibits) - 1);
+        int ival = lrint(val * ((1 << obits) - 1));
+        out[i] = av_clip(ival, 0, (1 << obits) - 1);
     }
 }
diff --git a/video/csputils.h b/video/csputils.h
index 0a9b11af9d..bc43a146a8 100644
--- a/video/csputils.h
+++ b/video/csputils.h
@@ -130,9 +130,6 @@ struct mp_csp_params {
     // texture_bits/input_bits is for rescaling fixed point input to range [0,1]
     int texture_bits;
     int input_bits;
-    // for scaling integer input and output (if 0, assume range [0,1])
-    int int_bits_in;
-    int int_bits_out;
 };
 
 #define MP_CSP_PARAMS_DEFAULTS {                                \
@@ -259,6 +256,7 @@ void mp_get_yuv2rgb_coeffs(struct mp_csp_params *params, struct mp_cmat *yuv2rgb
 
 void mp_invert_matrix3x3(float m[3][3]);
 void mp_invert_yuv2rgb(struct mp_cmat *out, struct mp_cmat *in);
-void mp_map_int_color(struct mp_cmat *matrix, int clip_bits, int c[3]);
+void mp_map_fixp_color(struct mp_cmat *matrix, int ibits, int in[3],
+                                               int obits, int out[3]);
 
 #endif /* MPLAYER_CSPUTILS_H */