csputils/vo_gl: rewrite YUV->RGB matrix generation

Rewrite the csputils.c code generating a conversion matrix for
YUV->RGB conversions (currently used by vo_gl only). Functional
differences:
- The separate "mplayer default" colorspace is removed, and BT.601 is
  used instead (the default colorspace was in fact BT.601; see below).
- The old code was missing chroma scaling. As a result the "mplayer
  default" colorspace actually mapped to BT.601, and everything else
  was buggy (I guess the other colorspaces were added with particular
  coefficient semantics, without understanding that the original
  "default colorspace" was actually BT.601 and why its coefficients
  differed from the added version).
- The old code had a bug in the equalizer hue equations.
- The old code assumed that for specifying whether input and output
  were limited-range or full-range YUV or RGB it would make sense to
  specify "no conversion" meaning full-range YUV to full-range RGB or
  limited-range YUV to limited-range RGB. This isn't true; limited-
  range YUV has different ranges for luma and chroma (16-235
  vs 16-240) which means you have to scale chroma for limited->limited
  conversions. The new code assumes limited->limited conversions for
  the levelconv parameter 2. It'd probably make sense to change the
  API later to specify the ranges of input and output separately.
- The undocumented EBU and XYZ colorspaces are removed. I doubt any
  videos use these. Also the EBU colorspace looks like it'd expect
  a different input range - at least no input would map to full RGB
  red as it was.

2 files changed, 97 insertions, 77 deletions

diff --git a/libvo/csputils.c b/libvo/csputils.c
index 7c488f77b0..831e5c774a 100644
--- a/libvo/csputils.c
+++ b/libvo/csputils.c
@@ -27,7 +27,9 @@
 
 #include <stdint.h>
 #include <math.h>
-#include "libavutil/common.h"
+#include <assert.h>
+#include <libavutil/common.h>
+
 #include "csputils.h"
 
 /**
@@ -55,92 +57,112 @@ void mp_gen_gamma_map(uint8_t *map, int size, float gamma)
     }
 }
 
+/* Fill in the Y, U, V vectors of a yuv2rgb conversion matrix
+ * based on the given luma weights of the R, G and B components (lr, lg, lb).
+ * lr+lg+lb is assumed to equal 1.
+ * This function is meant for colorspaces satisfying the following
+ * conditions (which are true for common YUV colorspaces):
+ * - The mapping from input [Y, U, V] to output [R, G, B] is linear.
+ * - Y is the vector [1, 1, 1].  (meaning input Y component maps to 1R+1G+1B)
+ * - U maps to a value with zero R and positive B ([0, x, y], y > 0;
+ *   i.e. blue and green only).
+ * - V maps to a value with zero B and positive R ([x, y, 0], x > 0;
+ *   i.e. red and green only).
+ * - U and V are orthogonal to the luma vector [lr, lg, lb].
+ * - The magnitudes of the vectors U and V are the minimal ones for which
+ *   the image of the set Y=[0...1],U=[-0.5...0.5],V=[-0.5...0.5] under the
+ *   conversion function will cover the set R=[0...1],G=[0...1],B=[0...1]
+ *   (the resulting matrix can be converted for other input/output ranges
+ *   outside this function).
+ * Under these conditions the given parameters lr, lg, lb uniquely
+ * determine the mapping of Y, U, V to R, G, B.
+ */
+static void luma_coeffs(float m[3][4], float lr, float lg, float lb)
+{
+    assert(fabs(lr+lg+lb - 1) < 1e-6);
+    m[0][0] = m[1][0] = m[2][0] = 1;
+    m[0][1] = 0;
+    m[1][1] = -2 * (1-lb) * lb/lg;
+    m[2][1] = 2 * (1-lb);
+    m[0][2] = 2 * (1-lr);
+    m[1][2] = -2 * (1-lr) * lr/lg;
+    m[2][2] = 0;
+    // Constant coefficients (m[x][3]) not set here
+}
+
 /**
  * \brief get the coefficients of the yuv -> rgb conversion matrix
  * \param params struct specifying the properties of the conversion like
  *  brightness, ...
- * \param yuv2rgb array to store coefficients into
- *
- * Note: contrast, hue and saturation will only work as expected with YUV
- * formats, not with e.g. MP_CSP_XYZ
+ * \param m array to store coefficients into
  */
-void mp_get_yuv2rgb_coeffs(struct mp_csp_params *params, float yuv2rgb[3][4])
+void mp_get_yuv2rgb_coeffs(struct mp_csp_params *params, float m[3][4])
 {
-    float depth_multiplier = params->input_shift >= 0 ?
-                             (1 << params->input_shift) :
-                             (1.0 / (1 << -params->input_shift));
-    float uvcos = params->saturation * cos(params->hue);
-    float uvsin = params->saturation * sin(params->hue);
     int format = params->format;
-    int levelconv = params->levelconv;
-    int i;
-    const float (*uv_coeffs)[3];
-    const float *level_adjust;
-    static const float yuv_level_adjust[MP_CSP_LEVELCONV_COUNT][4] = {
-        {-16 / 255.0, -128 / 255.0, -128 / 255.0, 1.164},
-        { 16 / 255.0 * 1.164, -128 / 255.0, -128 / 255.0, 1.0 / 1.164},
-        { 0, -128 / 255.0, -128 / 255.0, 1},
-    };
-    static const float xyz_level_adjust[4] = {
-        0, 0, 0, 0
-    };
-    static const float uv_coeffs_table[MP_CSP_COUNT][3][3] = {
-        [MP_CSP_DEFAULT] = {
-            {1, 0.000, 1.596},
-            {1, -0.391, -0.813},
-            {1, 2.018, 0.000}
-        },
-        [MP_CSP_BT_601] = {
-            {1, 0.000, 1.403},
-            {1, -0.344, -0.714},
-            {1, 1.773, 0.000}
-        },
-        [MP_CSP_BT_709] = {
-            {1, 0.0000, 1.5701},
-            {1, -0.1870, -0.4664},
-            {1, 1.8556, 0.0000}
-        },
-        [MP_CSP_SMPTE_240M] = {
-            {1, 0.0000, 1.5756},
-            {1, -0.2253, -0.5000},
-            {1, 1.8270, 0.0000}
-        },
-        [MP_CSP_EBU] = {
-            {1, 0.000, 1.140},
-            {1, -0.396, -0.581},
-            {1, 2.029, 0.000}
-        },
-        [MP_CSP_XYZ] = {
-            { 3.2404542, -1.5371385, -0.4985314},
-            {-0.9692660, 1.8760108, 0.0415560},
-            { 0.0556434, -0.2040259, 1.0572252}
-        },
+    if (format <= MP_CSP_DEFAULT || format >= MP_CSP_COUNT)
+        format = MP_CSP_BT_601;
+    switch (format) {
+    case MP_CSP_BT_601:     luma_coeffs(m, 0.299,  0.587,  0.114 ); break;
+    case MP_CSP_BT_709:     luma_coeffs(m, 0.2126, 0.7152, 0.0722); break;
+    case MP_CSP_SMPTE_240M: luma_coeffs(m, 0.2122, 0.7013, 0.0865); break;
+    default:
+        abort();
     };
 
-    if (format < 0 || format >= MP_CSP_COUNT)
-        format = MP_CSP_DEFAULT;
-    uv_coeffs = uv_coeffs_table[format];
+    // Hue is equivalent to rotating input [U, V] subvector around the origin.
+    // Saturation scales [U, V].
+    float huecos = params->saturation * cos(params->hue);
+    float huesin = params->saturation * sin(params->hue);
+    for (int i = 0; i < 3; i++) {
+        float u = m[i][COL_U];
+        m[i][COL_U] = huecos * u - huesin * m[i][COL_V];
+        m[i][COL_V] = huesin * u + huecos * m[i][COL_V];
+    }
+
+    int levelconv = params->levelconv;
     if (levelconv < 0 || levelconv >= MP_CSP_LEVELCONV_COUNT)
         levelconv = MP_CSP_LEVELCONV_TV_TO_PC;
-    level_adjust = yuv_level_adjust[levelconv];
-    if (format == MP_CSP_XYZ)
-        level_adjust = xyz_level_adjust;
+    // The values below are written in 0-255 scale
+    struct yuvlevels { double ymin, ymax, cmin, cmid; }
+        yuvlim =  { 16, 235, 16, 128 },
+        yuvfull = { 16, 235,  1, 128 },  // '1' to make it symmetric around 128
+        yuvlev;
+    struct rgblevels { double min, max; }
+        rgblim =  { 16, 235 },
+        rgbfull = {  0, 255 },
+        rgblev;
+    switch (levelconv) {
+    case MP_CSP_LEVELCONV_TV_TO_PC: yuvlev = yuvlim; rgblev = rgbfull; break;
+    case MP_CSP_LEVELCONV_PC_TO_TV: yuvlev = yuvfull; rgblev = rgblim; break;
+    case MP_CSP_LEVELCONV_TV_TO_TV: yuvlev = yuvlim; rgblev = rgblim; break;
+    default:
+        abort();
+    }
+    double ymul = (rgblev.max - rgblev.min) / (yuvlev.ymax - yuvlev.ymin);
+    double cmul = (rgblev.max - rgblev.min) / (yuvlev.cmid - yuvlev.cmin) / 2;
+    for (int i = 0; i < 3; i++) {
+        m[i][COL_Y] *= ymul;
+        m[i][COL_U] *= cmul;
+        m[i][COL_V] *= cmul;
+        // Set COL_C so that Y=umin,UV=cmid maps to RGB=min (black to black)
+        m[i][COL_C] = (rgblev.min - m[i][COL_Y] * yuvlev.ymin
+                       -(m[i][COL_U] + m[i][COL_V]) * yuvlev.cmid) / 255;
+    }
 
-    for (i = 0; i < 3; i++) {
-        yuv2rgb[i][COL_C]  = params->brightness;
-        yuv2rgb[i][COL_Y]  = uv_coeffs[i][COL_Y] * level_adjust[COL_C] * params->contrast;
-        yuv2rgb[i][COL_C] += level_adjust[COL_Y] * yuv2rgb[i][COL_Y];
-        yuv2rgb[i][COL_U]  = uv_coeffs[i][COL_U] * uvcos + uv_coeffs[i][COL_V] * uvsin;
-        yuv2rgb[i][COL_C] += level_adjust[COL_U] * yuv2rgb[i][COL_U];
-        yuv2rgb[i][COL_V]  = uv_coeffs[i][COL_U] * uvsin + uv_coeffs[i][COL_V] * uvcos;
-        yuv2rgb[i][COL_C] += level_adjust[COL_V] * yuv2rgb[i][COL_V];
-        // this "centers" contrast control so that e.g. a contrast of 0
-        // leads to a grey image, not a black one
-        yuv2rgb[i][COL_C] += 0.5 - params->contrast / 2.0;
-        yuv2rgb[i][COL_Y] *= depth_multiplier;
-        yuv2rgb[i][COL_U] *= depth_multiplier;
-        yuv2rgb[i][COL_V] *= depth_multiplier;
+    // Brightness adds a constant to output R,G,B.
+    // Contrast scales Y around 1/2 (not 0 in this implementation).
+    for (int i = 0; i < 3; i++) {
+        m[i][COL_C] += params->brightness;
+        m[i][COL_Y] *= params->contrast;
+        m[i][COL_C] += (rgblev.max-rgblev.min)/255 * (1 - params->contrast)/2;
     }
+
+    float depth_multiplier = params->input_shift >= 0 ?
+                             (1 << params->input_shift) :
+                             (1.0 / (1 << -params->input_shift));
+    for (int i = 0; i < 3; i++)
+        for (int j = 0; j < 3; j++)
+            m[i][j] *= depth_multiplier;
 }
 
 //! size of gamma map use to avoid slow exp function in gen_yuv2rgb_map
diff --git a/libvo/csputils.h b/libvo/csputils.h
index 64b23b3d57..93f9ca0316 100644
--- a/libvo/csputils.h
+++ b/libvo/csputils.h
@@ -31,15 +31,13 @@ enum mp_csp_standard {
     MP_CSP_BT_601,
     MP_CSP_BT_709,
     MP_CSP_SMPTE_240M,
-    MP_CSP_EBU,
-    MP_CSP_XYZ,
     MP_CSP_COUNT
 };
 
 enum mp_csp_levelconv {
     MP_CSP_LEVELCONV_TV_TO_PC,
     MP_CSP_LEVELCONV_PC_TO_TV,
-    MP_CSP_LEVELCONV_NONE,
+    MP_CSP_LEVELCONV_TV_TO_TV,
     MP_CSP_LEVELCONV_COUNT
 };