vo_gl: Support 9- and 10-bit YUV input for OpenGL VOs

git-svn-id: svn://svn.mplayerhq.hu/mplayer/trunk@33502 b3059339-0415-0410-9bf9-f77b7e298cf2

Fix clear/border color of chroma texture for 9- and 10-bit formats.
Avoids pink borders for those formats.

git-svn-id: svn://svn.mplayerhq.hu/mplayer/trunk@33504 b3059339-0415-0410-9bf9-f77b7e298cf2

5 files changed, 46 insertions, 15 deletions

diff --git a/libvo/csputils.c b/libvo/csputils.c
index 2978279398..0a58aa5a0e 100644
--- a/libvo/csputils.c
+++ b/libvo/csputils.c
@@ -62,6 +62,9 @@ void mp_gen_gamma_map(uint8_t *map, int size, float gamma) {
  * not with e.g. MP_CSP_XYZ
  */
 void mp_get_yuv2rgb_coeffs(struct mp_csp_params *params, float yuv2rgb[3][4]) {
+  float depth_multiplier = params->input_shift >= 0 ?
+                           (1 << params->input_shift) :
+                           (1.0 / (1 << -params->input_shift));
   float uvcos = params->saturation * cos(params->hue);
   float uvsin = params->saturation * sin(params->hue);
   int format = params->format;
@@ -128,6 +131,9 @@ void mp_get_yuv2rgb_coeffs(struct mp_csp_params *params, float yuv2rgb[3][4]) {
     // this "centers" contrast control so that e.g. a contrast of 0
     // leads to a grey image, not a black one
     yuv2rgb[i][COL_C] += 0.5 - params->contrast / 2.0;
+    yuv2rgb[i][COL_Y] *= depth_multiplier;
+    yuv2rgb[i][COL_U] *= depth_multiplier;
+    yuv2rgb[i][COL_V] *= depth_multiplier;
   }
 }
 
diff --git a/libvo/csputils.h b/libvo/csputils.h
index acc8f59459..07d9932279 100644
--- a/libvo/csputils.h
+++ b/libvo/csputils.h
@@ -53,6 +53,7 @@ struct mp_csp_params {
   float rgamma;
   float ggamma;
   float bgamma;
+  int input_shift;
 };
 
 void mp_gen_gamma_map(unsigned char *map, int size, float gamma);
diff --git a/libvo/gl_common.h b/libvo/gl_common.h
index fa0cffaf74..fca91d6f7d 100644
--- a/libvo/gl_common.h
+++ b/libvo/gl_common.h
@@ -353,6 +353,20 @@ int loadGPUProgram(GLenum target, char *prog);
 #define SET_YUV_CONVERSION(c)   ((c) & YUV_CONVERSION_MASK)
 #define SET_YUV_LUM_SCALER(s)   (((s) & YUV_SCALER_MASK) << YUV_LUM_SCALER_SHIFT)
 #define SET_YUV_CHROM_SCALER(s) (((s) & YUV_SCALER_MASK) << YUV_CHROM_SCALER_SHIFT)
+//! returns whether the yuv conversion supports large brightness range etc.
+static inline int glYUVLargeRange(int conv)
+{
+  switch (conv)
+  {
+  case YUV_CONVERSION_NONE:
+  case YUV_CONVERSION_COMBINERS:
+  case YUV_CONVERSION_COMBINERS_ATI:
+  case YUV_CONVERSION_FRAGMENT_LOOKUP3D:
+  case YUV_CONVERSION_TEXT_FRAGMENT:
+    return 0;
+  }
+  return 1;
+}
 /** \} */
 
 typedef struct {
diff --git a/libvo/vo_gl.c b/libvo/vo_gl.c
index bc50e245b3..f23d44ba64 100644
--- a/libvo/vo_gl.c
+++ b/libvo/vo_gl.c
@@ -247,7 +247,7 @@ static void texSize(int w, int h, int *texw, int *texh) {
 //! maximum size of custom fragment program
 #define MAX_CUSTOM_PROG_SIZE (1024 * 1024)
 static void update_yuvconv(void) {
-  int xs, ys;
+  int xs, ys, depth;
   float bri = eq_bri / 100.0;
   float cont = (eq_cont + 100) / 100.0;
   float hue = eq_hue / 100.0 * 3.1415927;
@@ -256,11 +256,12 @@ static void update_yuvconv(void) {
   float ggamma = exp(log(8.0) * eq_ggamma / 100.0);
   float bgamma = exp(log(8.0) * eq_bgamma / 100.0);
   gl_conversion_params_t params = {gl_target, yuvconvtype,
-      {colorspace, levelconv, bri, cont, hue, sat, rgamma, ggamma, bgamma},
+      {colorspace, levelconv, bri, cont, hue, sat, rgamma, ggamma, bgamma, 0},
       texture_width, texture_height, 0, 0, filter_strength};
-  mp_get_chroma_shift(image_format, &xs, &ys, NULL);
+  mp_get_chroma_shift(image_format, &xs, &ys, &depth);
   params.chrom_texw = params.texw >> xs;
   params.chrom_texh = params.texh >> ys;
+  params.csp_params.input_shift = -depth & 7;
   glSetupYUVConversion(&params);
   if (custom_prog) {
     FILE *f = fopen(custom_prog, "rb");
@@ -566,9 +567,11 @@ static int initGl(uint32_t d_width, uint32_t d_height) {
 
   if (is_yuv) {
     int i;
-    int xs, ys;
+    int xs, ys, depth;
+    int chroma_clear_val = 128;
     scale_type = get_scale_type(1);
-    mp_get_chroma_shift(image_format, &xs, &ys, NULL);
+    mp_get_chroma_shift(image_format, &xs, &ys, &depth);
+    chroma_clear_val >>= -depth & 7;
     mpglGenTextures(21, default_texs);
     default_texs[21] = 0;
     for (i = 0; i < 7; i++) {
@@ -579,12 +582,14 @@ static int initGl(uint32_t d_width, uint32_t d_height) {
     }
     mpglActiveTexture(GL_TEXTURE1);
     glCreateClearTex(gl_target, gl_texfmt, gl_format, gl_type, scale_type,
-                     texture_width >> xs, texture_height >> ys, 128);
+                     texture_width >> xs, texture_height >> ys,
+                     chroma_clear_val);
     if (mipmap_gen)
       mpglTexParameteri(gl_target, GL_GENERATE_MIPMAP, GL_TRUE);
     mpglActiveTexture(GL_TEXTURE2);
     glCreateClearTex(gl_target, gl_texfmt, gl_format, gl_type, scale_type,
-                     texture_width >> xs, texture_height >> ys, 128);
+                     texture_width >> xs, texture_height >> ys,
+                     chroma_clear_val);
     if (mipmap_gen)
       mpglTexParameteri(gl_target, GL_GENERATE_MIPMAP, GL_TRUE);
     mpglActiveTexture(GL_TEXTURE0);
@@ -1097,7 +1102,7 @@ query_format(uint32_t format)
     if (format == IMGFMT_RGB24 || format == IMGFMT_RGBA)
         return caps;
     if (use_yuv && mp_get_chroma_shift(format, NULL, NULL, &depth) &&
-        (depth == 8 || depth == 16) &&
+        (depth == 8 || depth == 16 || glYUVLargeRange(use_yuv)) &&
         (IMGFMT_IS_YUVP16_NE(format) || !IMGFMT_IS_YUVP16(format)))
         return caps;
     // HACK, otherwise we get only b&w with some filters (e.g. -vf eq)
diff --git a/libvo/vo_gl2.c b/libvo/vo_gl2.c
index 21f779866c..ac1d957136 100644
--- a/libvo/vo_gl2.c
+++ b/libvo/vo_gl2.c
@@ -273,14 +273,18 @@ static int initTextures(void)
 
       glTexEnvf (GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE);
       if (is_yuv) {
-        int xs, ys;
-        mp_get_chroma_shift(image_format, &xs, &ys, NULL);
+        int xs, ys, depth;
+        int chroma_clear_val = 128;
+        mp_get_chroma_shift(image_format, &xs, &ys, &depth);
+        chroma_clear_val >>= -depth & 7;
         mpglActiveTexture(GL_TEXTURE1);
         glCreateClearTex(GL_TEXTURE_2D, gl_internal_format, gl_bitmap_format,  gl_bitmap_type, GL_LINEAR,
-                         texture_width >> xs, texture_height >> ys, 128);
+                         texture_width >> xs, texture_height >> ys,
+                         chroma_clear_val);
         mpglActiveTexture(GL_TEXTURE2);
         glCreateClearTex(GL_TEXTURE_2D, gl_internal_format, gl_bitmap_format,  gl_bitmap_type, GL_LINEAR,
-                         texture_width >> xs, texture_height >> ys, 128);
+                         texture_width >> xs, texture_height >> ys,
+                         chroma_clear_val);
         mpglActiveTexture(GL_TEXTURE0);
       }
 
@@ -559,7 +563,7 @@ static int initGl(uint32_t d_width, uint32_t d_height)
   glDisable(GL_CULL_FACE);
   glEnable (GL_TEXTURE_2D);
   if (is_yuv) {
-    int xs, ys;
+    int xs, ys, depth;
     gl_conversion_params_t params = {GL_TEXTURE_2D, use_yuv,
           {-1, -1, 0.0, 1.0, 0.0, 1.0, 1.0, 1.0, 1.0},
           texture_width, texture_height, 0, 0, 0};
@@ -580,9 +584,10 @@ static int initGl(uint32_t d_width, uint32_t d_height)
         mpglBindProgram(GL_FRAGMENT_PROGRAM, fragprog);
         break;
     }
-    mp_get_chroma_shift(image_format, &xs, &ys, NULL);
+    mp_get_chroma_shift(image_format, &xs, &ys, &depth);
     params.chrom_texw = params.texw >> xs;
     params.chrom_texh = params.texh >> ys;
+    params.csp_params.input_shift = -depth & 7;
     glSetupYUVConversion(&params);
   }
 
@@ -804,7 +809,7 @@ query_format(uint32_t format)
 {
   int depth;
   if (use_yuv && mp_get_chroma_shift(format, NULL, NULL, &depth) &&
-      (depth == 8 || depth == 16) &&
+      (depth == 8 || depth == 16 || glYUVLargeRange(use_yuv)) &&
       (IMGFMT_IS_YUVP16_NE(format) || !IMGFMT_IS_YUVP16(format)))
     return VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW | VFCAP_OSD |
            VFCAP_HWSCALE_UP | VFCAP_HWSCALE_DOWN | VFCAP_ACCEPT_STRIDE;