video, options: implement better YUV->RGB conversion control

Rewrite control of the colorspace and input/output level parameters
used in YUV-RGB conversions, replacing VO-specific suboptions with new
common options and adding configuration support to more cases.

Add new option --colormatrix which selects the colorspace the original
video is assumed to have in YUV->RGB conversions. The default
behavior changes from assuming BT.601 to colorspace autoselection
between BT.601 and BT.709 using a simple heuristic based on video
size. Add new options --colormatrix-input-range and
--colormatrix-output-range which select input YUV and output RGB range.
Disable the previously existing VO-specific colorspace and level
conversion suboptions in vo_gl and vo_vdpau. Remove the
"yuv_colorspace" property and replace it with one named "colormatrix"
and semantics matching the new option. Add new properties matching the
options for level conversion.

Colorspace selection is currently supported by vo_gl, vo_vdpau, vo_xv
and vf_scale, and all can change it at runtime (previously only
vo_vdpau and vo_xv could). vo_vdpau now uses the same conversion
matrix generation as vo_gl instead of libvdpau functionality; the main
functional difference is that the "contrast" equalizer control behaves
somewhat differently (it scales the Y component around 1/2 instead of
around 0, so that contrast 0 makes the image gray rather than black).
vo_xv does not support level conversion. vf_scale supports range
setting for input, but always outputs full-range RGB.

The value of the slave properties is the policy setting used for
conversions. This means they can be set to any value regardless of
whether the current VO supports that value or whether there currently
even is any video. Possibly separate properties could be added to
query the conversion actually used at the moment, if any.

Because the colorspace and level settings are now set with a single
VF/VO control call, the return value of that is no longer used to
signal whether all the settings are actually supported. Instead code
should set all the details it can support, and ignore the rest. The
core will use GET_YUV_COLORSPACE to check which colorspace details
have been set and which not. In other words, the return value for
SET_YUV_COLORSPACE only signals whether any kind of YUV colorspace
conversion handling exists at all, and VOs have to take care to return
the actual state with GET_YUV_COLORSPACE instead.

To be changed in later commits: add missing option documentation.

1 files changed, 67 insertions, 9 deletions

diff --git a/libvo/csputils.h b/libvo/csputils.h
index 93f9ca0316..3a754a8273 100644
--- a/libvo/csputils.h
+++ b/libvo/csputils.h
@@ -26,24 +26,41 @@
 
 #include <stdint.h>
 
-enum mp_csp_standard {
-    MP_CSP_DEFAULT,
+
+/* NOTE: the csp and levels AUTO values are converted to specific ones
+ * above vf/vo level. At least vf_scale relies on all valid settings being
+ * nonzero at vf/vo level.
+ */
+
+enum mp_csp {
+    MP_CSP_AUTO,
     MP_CSP_BT_601,
     MP_CSP_BT_709,
     MP_CSP_SMPTE_240M,
     MP_CSP_COUNT
 };
 
-enum mp_csp_levelconv {
-    MP_CSP_LEVELCONV_TV_TO_PC,
-    MP_CSP_LEVELCONV_PC_TO_TV,
-    MP_CSP_LEVELCONV_TV_TO_TV,
-    MP_CSP_LEVELCONV_COUNT
+// Any enum mp_csp value is a valid index (except MP_CSP_COUNT)
+extern char * const mp_csp_names[MP_CSP_COUNT];
+
+enum mp_csp_levels {
+    MP_CSP_LEVELS_AUTO,
+    MP_CSP_LEVELS_TV,
+    MP_CSP_LEVELS_PC,
+    MP_CSP_LEVELS_COUNT,
+};
+
+struct mp_csp_details {
+    enum mp_csp format;
+    enum mp_csp_levels levels_in;      // encoded video
+    enum mp_csp_levels levels_out;     // output device
 };
 
+// initializer for struct mp_csp_details that contains reasonable defaults
+#define MP_CSP_DETAILS_DEFAULTS {MP_CSP_BT_601, MP_CSP_LEVELS_TV, MP_CSP_LEVELS_PC}
+
 struct mp_csp_params {
-    enum mp_csp_standard format;
-    enum mp_csp_levelconv levelconv;
+    struct mp_csp_details colorspace;
     float brightness;
     float contrast;
     float hue;
@@ -54,6 +71,47 @@ struct mp_csp_params {
     int input_shift;
 };
 
+enum mp_csp_equalizer_param {
+    MP_CSP_EQ_BRIGHTNESS,
+    MP_CSP_EQ_CONTRAST,
+    MP_CSP_EQ_HUE,
+    MP_CSP_EQ_SATURATION,
+    MP_CSP_EQ_GAMMA,
+    MP_CSP_EQ_COUNT,
+};
+
+#define MP_CSP_EQ_CAPS_COLORMATRIX \
+    ( (1 << MP_CSP_EQ_BRIGHTNESS) \
+    | (1 << MP_CSP_EQ_CONTRAST) \
+    | (1 << MP_CSP_EQ_HUE) \
+    | (1 << MP_CSP_EQ_SATURATION) )
+
+#define MP_CSP_EQ_CAPS_GAMMA (1 << MP_CSP_EQ_GAMMA)
+
+extern char * const mp_csp_equalizer_names[MP_CSP_EQ_COUNT];
+
+// Default initialization with 0 is enough, except for the capabilities field
+struct mp_csp_equalizer {
+    // Bit field of capabilities. For example (1 << MP_CSP_EQ_HUE) means hue
+    // support is available.
+    int capabilities;
+    // Value for each property is in the range [-100, 100].
+    // 0 is default, meaning neutral or no change.
+    int values[MP_CSP_EQ_COUNT];
+};
+
+
+void mp_csp_copy_equalizer_values(struct mp_csp_params *params,
+                                  const struct mp_csp_equalizer *eq);
+
+int mp_csp_equalizer_set(struct mp_csp_equalizer *eq, const char *property,
+                         int value);
+
+int mp_csp_equalizer_get(struct mp_csp_equalizer *eq, const char *property,
+                         int *out_value);
+
+enum mp_csp mp_csp_guess_colorspace(int width, int height);
+
 void mp_gen_gamma_map(unsigned char *map, int size, float gamma);
 #define ROW_R 0
 #define ROW_G 1