vo_gl: improve 10-bit YUV->RGB conversion accuracy slightly

Modify the YUV->RGB conversion matrix to take into account the
difference between the same color value being x/255 in a 8-bit texture
and x*256/65535 in a 16-bit texture (actually things are stored as
x*4/65535 for 10-bit color, but that can be ignored here). This 0.4 %
difference in the shader float value could make shades of gray in
10-bit (or generally more than 8 bit) YUV produce RGB values with
green slightly higher than red/blue.

4 files changed, 16 insertions, 17 deletions

diff --git a/libvo/csputils.c b/libvo/csputils.c
index 6656e5e196..3bd6e48f67 100644
--- a/libvo/csputils.c
+++ b/libvo/csputils.c
@@ -143,10 +143,13 @@ void mp_get_yuv2rgb_coeffs(struct mp_csp_params *params, float m[3][4])
     int levels_in = params->colorspace.levels_in;
     if (levels_in <= MP_CSP_LEVELS_AUTO || levels_in >= MP_CSP_LEVELS_COUNT)
         levels_in = MP_CSP_LEVELS_TV;
+    assert(params->input_bits >= 8);
+    assert(params->texture_bits >= params->input_bits);
+    double s = (1 << params->input_bits-8) / ((1<<params->texture_bits)-1.);
     // The values below are written in 0-255 scale
     struct yuvlevels { double ymin, ymax, cmin, cmid; }
-        yuvlim =  { 16, 235, 16, 128 },
-        yuvfull = {  0, 255,  1, 128 },  // '1' to make it symmetric around 128
+        yuvlim =  { 16*s, 235*s, 16*s, 128*s },
+        yuvfull = {  0*s, 255*s,  1*s, 128*s },  // '1' for symmetry around 128
         yuvlev;
     switch (levels_in) {
     case MP_CSP_LEVELS_TV: yuvlev = yuvlim; break;
@@ -159,8 +162,8 @@ void mp_get_yuv2rgb_coeffs(struct mp_csp_params *params, float m[3][4])
     if (levels_out <= MP_CSP_LEVELS_AUTO || levels_out >= MP_CSP_LEVELS_COUNT)
         levels_out = MP_CSP_LEVELS_PC;
     struct rgblevels { double min, max; }
-        rgblim =  { 16, 235 },
-        rgbfull = {  0, 255 },
+        rgblim =  { 16/255., 235/255. },
+        rgbfull = {      0,        1  },
         rgblev;
     switch (levels_out) {
     case MP_CSP_LEVELS_TV: rgblev = rgblim; break;
@@ -176,8 +179,8 @@ void mp_get_yuv2rgb_coeffs(struct mp_csp_params *params, float m[3][4])
         m[i][COL_U] *= cmul;
         m[i][COL_V] *= cmul;
         // Set COL_C so that Y=umin,UV=cmid maps to RGB=min (black to black)
-        m[i][COL_C] = (rgblev.min - m[i][COL_Y] * yuvlev.ymin
-                       -(m[i][COL_U] + m[i][COL_V]) * yuvlev.cmid) / 255;
+        m[i][COL_C] = rgblev.min - m[i][COL_Y] * yuvlev.ymin
+                      -(m[i][COL_U] + m[i][COL_V]) * yuvlev.cmid;
     }
 
     // Brightness adds a constant to output R,G,B.
@@ -185,15 +188,8 @@ void mp_get_yuv2rgb_coeffs(struct mp_csp_params *params, float m[3][4])
     for (int i = 0; i < 3; i++) {
         m[i][COL_C] += params->brightness;
         m[i][COL_Y] *= params->contrast;
-        m[i][COL_C] += (rgblev.max-rgblev.min)/255 * (1 - params->contrast)/2;
+        m[i][COL_C] += (rgblev.max-rgblev.min) * (1 - params->contrast)/2;
     }
-
-    float depth_multiplier = params->input_shift >= 0 ?
-                             (1 << params->input_shift) :
-                             (1.0 / (1 << -params->input_shift));
-    for (int i = 0; i < 3; i++)
-        for (int j = 0; j < 3; j++)
-            m[i][j] *= depth_multiplier;
 }
 
 //! size of gamma map use to avoid slow exp function in gen_yuv2rgb_map
diff --git a/libvo/csputils.h b/libvo/csputils.h
index 3a754a8273..434be4a9a1 100644
--- a/libvo/csputils.h
+++ b/libvo/csputils.h
@@ -68,7 +68,8 @@ struct mp_csp_params {
     float rgamma;
     float ggamma;
     float bgamma;
-    int input_shift;
+    int texture_bits;
+    int input_bits;
 };
 
 enum mp_csp_equalizer_param {
diff --git a/libvo/vo_gl.c b/libvo/vo_gl.c
index e68465f009..12749426ab 100644
--- a/libvo/vo_gl.c
+++ b/libvo/vo_gl.c
@@ -220,7 +220,8 @@ static void update_yuvconv(struct vo *vo)
     mp_get_chroma_shift(p->image_format, &xs, &ys, &depth);
     params.chrom_texw = params.texw >> xs;
     params.chrom_texh = params.texh >> ys;
-    params.csp_params.input_shift = -depth & 7;
+    params.csp_params.input_bits = depth;
+    params.csp_params.texture_bits = depth+7 & ~7;
     glSetupYUVConversion(gl, &params);
     if (p->custom_prog) {
         FILE *f = fopen(p->custom_prog, "rb");
diff --git a/libvo/vo_vdpau.c b/libvo/vo_vdpau.c
index f8b70a4255..0971ae47ea 100644
--- a/libvo/vo_vdpau.c
+++ b/libvo/vo_vdpau.c
@@ -599,7 +599,8 @@ static void update_csc_matrix(struct vo *vo)
     // both are float[3][4]
     VdpCSCMatrix matrix;
 
-    struct mp_csp_params cparams = { .colorspace = vc->colorspace };
+    struct mp_csp_params cparams = {
+        .colorspace = vc->colorspace, .input_bits = 8, .texture_bits = 8 };
     mp_csp_copy_equalizer_values(&cparams, &vc->video_eq);
     mp_get_yuv2rgb_coeffs(&cparams, matrix);