video, options: implement better YUV->RGB conversion control

Rewrite control of the colorspace and input/output level parameters
used in YUV-RGB conversions, replacing VO-specific suboptions with new
common options and adding configuration support to more cases.

Add new option --colormatrix which selects the colorspace the original
video is assumed to have in YUV->RGB conversions. The default
behavior changes from assuming BT.601 to colorspace autoselection
between BT.601 and BT.709 using a simple heuristic based on video
size. Add new options --colormatrix-input-range and
--colormatrix-output-range which select input YUV and output RGB range.
Disable the previously existing VO-specific colorspace and level
conversion suboptions in vo_gl and vo_vdpau. Remove the
"yuv_colorspace" property and replace it with one named "colormatrix"
and semantics matching the new option. Add new properties matching the
options for level conversion.

Colorspace selection is currently supported by vo_gl, vo_vdpau, vo_xv
and vf_scale, and all can change it at runtime (previously only
vo_vdpau and vo_xv could). vo_vdpau now uses the same conversion
matrix generation as vo_gl instead of libvdpau functionality; the main
functional difference is that the "contrast" equalizer control behaves
somewhat differently (it scales the Y component around 1/2 instead of
around 0, so that contrast 0 makes the image gray rather than black).
vo_xv does not support level conversion. vf_scale supports range
setting for input, but always outputs full-range RGB.

The value of the slave properties is the policy setting used for
conversions. This means they can be set to any value regardless of
whether the current VO supports that value or whether there currently
even is any video. Possibly separate properties could be added to
query the conversion actually used at the moment, if any.

Because the colorspace and level settings are now set with a single
VF/VO control call, the return value of that is no longer used to
signal whether all the settings are actually supported. Instead code
should set all the details it can support, and ignore the rest. The
core will use GET_YUV_COLORSPACE to check which colorspace details
have been set and which not. In other words, the return value for
SET_YUV_COLORSPACE only signals whether any kind of YUV colorspace
conversion handling exists at all, and VOs have to take care to return
the actual state with GET_YUV_COLORSPACE instead.

To be changed in later commits: add missing option documentation.

1 files changed, 79 insertions, 0 deletions

diff --git a/libmpcodecs/vf_scale.c b/libmpcodecs/vf_scale.c
index 1a2cb28e94..8cd0a37c33 100644
--- a/libmpcodecs/vf_scale.c
+++ b/libmpcodecs/vf_scale.c
@@ -35,6 +35,8 @@
 #include "libswscale/swscale.h"
 #include "vf_scale.h"
 
+#include "libvo/csputils.h"
+
 #include "m_option.h"
 #include "m_struct.h"
 
@@ -49,6 +51,7 @@ static struct vf_priv_s {
     int interlaced;
     int noup;
     int accurate_rnd;
+    struct mp_csp_details colorspace;
 } const vf_priv_dflt = {
   -1,-1,
   0,
@@ -187,6 +190,8 @@ static int config(struct vf_instance *vf,
     SwsFilter *srcFilter, *dstFilter;
     enum PixelFormat dfmt, sfmt;
 
+    vf->priv->colorspace = (struct mp_csp_details) {0};
+
     if(!best){
 	mp_msg(MSGT_VFILTER,MSGL_WARN,"SwScale: no supported outfmt found :(\n");
 	return 0;
@@ -505,6 +510,32 @@ static int control(struct vf_instance *vf, int request, void* data){
         }
 
 	return CONTROL_TRUE;
+    case VFCTRL_SET_YUV_COLORSPACE: {
+        struct mp_csp_details colorspace = *(struct mp_csp_details *)data;
+        if (mp_sws_set_colorspace(vf->priv->ctx, &colorspace) >= 0) {
+            if (vf->priv->ctx2)
+                mp_sws_set_colorspace(vf->priv->ctx2, &colorspace);
+            vf->priv->colorspace = colorspace;
+            return 1;
+        }
+        break;
+    }
+    case VFCTRL_GET_YUV_COLORSPACE: {
+        /* This scale filter should never react to colorspace commands if it
+         * doesn't do YUV->RGB conversion. But because finding out whether this
+         * is really YUV->RGB (and not YUV->YUV or anything else) is hard,
+         * react only if the colorspace has been set explicitly before. The
+         * trick is that mp_sws_set_colorspace does not succeed for YUV->YUV
+         * and RGB->YUV conversions, which makes this code correct in "most"
+         * cases. (This would be trivial to do correctly if libswscale exposed
+         * functionality like isYUV()).
+         */
+        if (vf->priv->colorspace.format) {
+            *(struct mp_csp_details *)data = vf->priv->colorspace;
+            return CONTROL_TRUE;
+        }
+        break;
+    }
     default:
 	break;
     }
@@ -512,6 +543,54 @@ static int control(struct vf_instance *vf, int request, void* data){
     return vf_next_control(vf,request,data);
 }
 
+static const int mp_csp_to_swscale[MP_CSP_COUNT] = {
+    [MP_CSP_BT_601] = SWS_CS_ITU601,
+    [MP_CSP_BT_709] = SWS_CS_ITU709,
+    [MP_CSP_SMPTE_240M] = SWS_CS_SMPTE240M,
+};
+
+// Adjust the colorspace used for YUV->RGB conversion. On other conversions,
+// do nothing or return an error.
+// The csp argument is set to the supported values.
+// Return 0 on success and -1 on error.
+int mp_sws_set_colorspace(struct SwsContext *sws, struct mp_csp_details *csp)
+{
+    int *table, *inv_table;
+    int brightness, contrast, saturation, srcRange, dstRange;
+
+    csp->levels_out = MP_CSP_LEVELS_PC;
+
+    // NOTE: returns an error if the destination format is YUV
+    if (sws_getColorspaceDetails(sws, &inv_table, &srcRange, &table, &dstRange,
+                                 &brightness, &contrast, &saturation) == -1)
+        goto error_out;
+
+    int sws_csp = mp_csp_to_swscale[csp->format];
+    if (sws_csp == 0) {
+        // colorspace not supported, go with a reasonable default
+        csp->format = SWS_CS_ITU601;
+        sws_csp = MP_CSP_BT_601;
+    }
+
+    /* The swscale API for these is hardly documented.
+     * Apparently table/range only apply to YUV. Thus dstRange has no effect
+     * for YUV->RGB conversions, and conversions to limited-range RGB are
+     * not supported.
+     */
+    srcRange = csp->levels_in == MP_CSP_LEVELS_PC;
+    const int *new_inv_table = sws_getCoefficients(sws_csp);
+
+    if (sws_setColorspaceDetails(sws, new_inv_table, srcRange, table, dstRange,
+        brightness, contrast, saturation) == -1)
+        goto error_out;
+
+    return 0;
+
+error_out:
+    *csp = (struct mp_csp_details){0};
+    return -1;
+}
+
 //===========================================================================//
 
 //  supported Input formats: YV12, I420, IYUV, YUY2, UYVY, BGR32, BGR24, BGR16, BGR15, RGB32, RGB24, Y8, Y800