vo_gl: don't accept 9/10-bit formats as input

Make mp_get_chroma_shift() simpler/more generic and add an argument
to get the per-component bit depth.
Use this to check more properly for supported formats in
gl and gl2 vos (only 8 and 16 bit are supported, 9 and 10 are not).

git-svn-id: svn://svn.mplayerhq.hu/mplayer/trunk@33452 b3059339-0415-0410-9bf9-f77b7e298cf2

8 files changed, 74 insertions, 58 deletions

diff --git a/libmpcodecs/img_format.c b/libmpcodecs/img_format.c
index 09711e5270..c4884f3eca 100644
--- a/libmpcodecs/img_format.c
+++ b/libmpcodecs/img_format.c
@@ -19,6 +19,7 @@
 #include "config.h"
 #include "img_format.h"
 #include "stdio.h"
+#include "mpbswap.h"
 
 const char *vo_format_name(int format)
 {
@@ -117,20 +118,57 @@ const char *vo_format_name(int format)
     return unknown_format;
 }
 
-int mp_get_chroma_shift(int format, int *x_shift, int *y_shift)
+int mp_get_chroma_shift(int format, int *x_shift, int *y_shift, int *component_bits)
 {
     int xs = 0, ys = 0;
     int bpp;
-    int bpp_factor = 1;
     int err = 0;
-    switch (format) {
-    case IMGFMT_420P16_LE:
-    case IMGFMT_420P16_BE:
-    case IMGFMT_420P10_LE:
-    case IMGFMT_420P10_BE:
-    case IMGFMT_420P9_LE:
-    case IMGFMT_420P9_BE:
-        bpp_factor = 2;
+    int bits = 8;
+    if ((format & 0xff0000f0) == 0x34000050)
+        format = bswap_32(format);
+    if ((format & 0xf00000ff) == 0x50000034) {
+        switch (format >> 24) {
+        case 0x50:
+            break;
+        case 0x51:
+            bits = 16;
+            break;
+        case 0x52:
+            bits = 10;
+            break;
+        case 0x53:
+            bits = 9;
+            break;
+        default:
+            err = 1;
+            break;
+        }
+        switch (format & 0x00ffffff) {
+        case 0x00343434: // 444
+            xs = 0;
+            ys = 0;
+            break;
+        case 0x00323234: // 422
+            xs = 1;
+            ys = 0;
+            break;
+        case 0x00303234: // 420
+            xs = 1;
+            ys = 1;
+            break;
+        case 0x00313134: // 411
+            xs = 2;
+            ys = 0;
+            break;
+        case 0x00303434: // 440
+            xs = 0;
+            ys = 1;
+            break;
+        default:
+            err = 1;
+            break;
+        }
+    } else switch (format) {
     case IMGFMT_420A:
     case IMGFMT_I420:
     case IMGFMT_IYUV:
@@ -143,34 +181,6 @@ int mp_get_chroma_shift(int format, int *x_shift, int *y_shift)
         xs = 2;
         ys = 2;
         break;
-    case IMGFMT_444P16_LE:
-    case IMGFMT_444P16_BE:
-    case IMGFMT_444P10_LE:
-    case IMGFMT_444P10_BE:
-    case IMGFMT_444P9_LE:
-    case IMGFMT_444P9_BE:
-        bpp_factor = 2;
-    case IMGFMT_444P:
-        xs = 0;
-        ys = 0;
-        break;
-    case IMGFMT_422P16_LE:
-    case IMGFMT_422P16_BE:
-    case IMGFMT_422P10_LE:
-    case IMGFMT_422P10_BE:
-        bpp_factor = 2;
-    case IMGFMT_422P:
-        xs = 1;
-        ys = 0;
-        break;
-    case IMGFMT_411P:
-        xs = 2;
-        ys = 0;
-        break;
-    case IMGFMT_440P:
-        xs = 0;
-        ys = 1;
-        break;
     case IMGFMT_Y8:
     case IMGFMT_Y800:
         xs = 31;
@@ -182,9 +192,10 @@ int mp_get_chroma_shift(int format, int *x_shift, int *y_shift)
     }
     if (x_shift) *x_shift = xs;
     if (y_shift) *y_shift = ys;
+    if (component_bits) *component_bits = bits;
     bpp = 8 + ((16 >> xs) >> ys);
     if (format == IMGFMT_420A)
         bpp += 8;
-    bpp *= bpp_factor;
+    bpp *= (bits + 7) >> 3;
     return err ? 0 : bpp;
 }
diff --git a/libmpcodecs/img_format.h b/libmpcodecs/img_format.h
index 52a5ffbe25..2fc9775ee1 100644
--- a/libmpcodecs/img_format.h
+++ b/libmpcodecs/img_format.h
@@ -227,8 +227,9 @@ const char *vo_format_name(int format);
 /**
  * Calculates the scale shifts for the chroma planes for planar YUV
  *
+ * \param component_bits bits per component
  * \return bits-per-pixel for format if successful (i.e. format is 3 or 4-planes planar YUV), 0 otherwise
  */
-int mp_get_chroma_shift(int format, int *x_shift, int *y_shift);
+int mp_get_chroma_shift(int format, int *x_shift, int *y_shift, int *component_bits);
 
 #endif /* MPLAYER_IMG_FORMAT_H */
diff --git a/libmpcodecs/mp_image.c b/libmpcodecs/mp_image.c
index 4d2892cf81..88702dde60 100644
--- a/libmpcodecs/mp_image.c
+++ b/libmpcodecs/mp_image.c
@@ -123,9 +123,9 @@ void mp_image_setfmt(mp_image_t* mpi,unsigned int out_fmt){
     }
     mpi->flags|=MP_IMGFLAG_YUV;
     mpi->num_planes=3;
-    if (mp_get_chroma_shift(out_fmt, NULL, NULL)) {
+    if (mp_get_chroma_shift(out_fmt, NULL, NULL, NULL)) {
         mpi->flags|=MP_IMGFLAG_PLANAR;
-        mpi->bpp = mp_get_chroma_shift(out_fmt, &mpi->chroma_x_shift, &mpi->chroma_y_shift);
+        mpi->bpp = mp_get_chroma_shift(out_fmt, &mpi->chroma_x_shift, &mpi->chroma_y_shift, NULL);
         mpi->chroma_width  = mpi->width  >> mpi->chroma_x_shift;
         mpi->chroma_height = mpi->height >> mpi->chroma_y_shift;
     }
diff --git a/libmpcodecs/vf_sab.c b/libmpcodecs/vf_sab.c
index 7f05af9249..2773ed3109 100644
--- a/libmpcodecs/vf_sab.c
+++ b/libmpcodecs/vf_sab.c
@@ -116,7 +116,7 @@ static int config(struct vf_instance *vf,
 //__asm__ volatile("emms\n\t");
 	allocStuff(&vf->priv->luma, width, height);
 
-	mp_get_chroma_shift(outfmt, &sw, &sh);
+	mp_get_chroma_shift(outfmt, &sw, &sh, NULL);
 	allocStuff(&vf->priv->chroma, width>>sw, height>>sh);
 
 	return vf_next_config(vf,width,height,d_width,d_height,flags,outfmt);
diff --git a/libmpcodecs/vf_smartblur.c b/libmpcodecs/vf_smartblur.c
index 6599c967de..7c54231e40 100644
--- a/libmpcodecs/vf_smartblur.c
+++ b/libmpcodecs/vf_smartblur.c
@@ -76,7 +76,7 @@ static int config(struct vf_instance *vf,
 
 	allocStuff(&vf->priv->luma, width, height);
 
-	mp_get_chroma_shift(outfmt, &sw, &sh);
+	mp_get_chroma_shift(outfmt, &sw, &sh, NULL);
 	allocStuff(&vf->priv->chroma, width>>sw, height>>sh);
 
 	return vf_next_config(vf,width,height,d_width,d_height,flags,outfmt);
diff --git a/libvo/gl_common.c b/libvo/gl_common.c
index ad6fc99a64..f98e6afa4c 100644
--- a/libvo/gl_common.c
+++ b/libvo/gl_common.c
@@ -254,7 +254,7 @@ int glFindFormat(uint32_t fmt, int *bpp, GLint *gl_texfmt,
   if (!gl_format) gl_format = &dummy2;
   if (!gl_type) gl_type = &dummy2;
 
-  if (mp_get_chroma_shift(fmt, NULL, NULL)) {
+  if (mp_get_chroma_shift(fmt, NULL, NULL, NULL)) {
     // reduce the possible cases a bit
     if (IMGFMT_IS_YUVP16_LE(fmt))
       fmt = IMGFMT_420P16_LE;
diff --git a/libvo/vo_gl.c b/libvo/vo_gl.c
index e663057beb..bc50e245b3 100644
--- a/libvo/vo_gl.c
+++ b/libvo/vo_gl.c
@@ -258,7 +258,7 @@ static void update_yuvconv(void) {
   gl_conversion_params_t params = {gl_target, yuvconvtype,
       {colorspace, levelconv, bri, cont, hue, sat, rgamma, ggamma, bgamma},
       texture_width, texture_height, 0, 0, filter_strength};
-  mp_get_chroma_shift(image_format, &xs, &ys);
+  mp_get_chroma_shift(image_format, &xs, &ys, NULL);
   params.chrom_texw = params.texw >> xs;
   params.chrom_texh = params.texh >> ys;
   glSetupYUVConversion(&params);
@@ -568,7 +568,7 @@ static int initGl(uint32_t d_width, uint32_t d_height) {
     int i;
     int xs, ys;
     scale_type = get_scale_type(1);
-    mp_get_chroma_shift(image_format, &xs, &ys);
+    mp_get_chroma_shift(image_format, &xs, &ys, NULL);
     mpglGenTextures(21, default_texs);
     default_texs[21] = 0;
     for (i = 0; i < 7; i++) {
@@ -670,7 +670,7 @@ config(uint32_t width, uint32_t height, uint32_t d_width, uint32_t d_height, uin
   image_height = height;
   image_width = width;
   image_format = format;
-  is_yuv = mp_get_chroma_shift(image_format, &xs, &ys) > 0;
+  is_yuv = mp_get_chroma_shift(image_format, &xs, &ys, NULL) > 0;
   is_yuv |= (xs << 8) | (ys << 16);
   glFindFormat(format, NULL, &gl_texfmt, &gl_format, &gl_type);
 
@@ -884,7 +884,7 @@ static int draw_slice(uint8_t *src[], int stride[], int w,int h,int x,int y)
               x, y, w, h, slice_height);
   if (is_yuv) {
     int xs, ys;
-    mp_get_chroma_shift(image_format, &xs, &ys);
+    mp_get_chroma_shift(image_format, &xs, &ys, NULL);
     mpglActiveTexture(GL_TEXTURE1);
     glUploadTex(gl_target, gl_format, gl_type, src[1], stride[1],
                 x >> xs, y >> ys, w >> xs, h >> ys, slice_height);
@@ -951,7 +951,7 @@ static uint32_t get_image(mp_image_t *mpi) {
   if (is_yuv) {
     // planar YUV
     int xs, ys;
-    mp_get_chroma_shift(image_format, &xs, &ys);
+    mp_get_chroma_shift(image_format, &xs, &ys, NULL);
     mpi->flags |= MP_IMGFLAG_COMMON_STRIDE | MP_IMGFLAG_COMMON_PLANE;
     mpi->stride[0] = mpi->width;
     mpi->planes[1] = mpi->planes[0] + mpi->stride[0] * mpi->height;
@@ -1009,7 +1009,7 @@ static uint32_t draw_image(mp_image_t *mpi) {
   if (force_pbo && !(mpi->flags & MP_IMGFLAG_DIRECT) && !gl_bufferptr && get_image(&mpi2) == VO_TRUE) {
     int bpp = is_yuv ? 8 : mpi->bpp;
     int xs, ys;
-    mp_get_chroma_shift(image_format, &xs, &ys);
+    mp_get_chroma_shift(image_format, &xs, &ys, NULL);
     memcpy_pic(mpi2.planes[0], mpi->planes[0], mpi->w * bpp / 8, mpi->h, mpi2.stride[0], mpi->stride[0]);
     if (is_yuv) {
       memcpy_pic(mpi2.planes[1], mpi->planes[1], mpi->w >> xs, mpi->h >> ys, mpi2.stride[1], mpi->stride[1]);
@@ -1051,7 +1051,7 @@ static uint32_t draw_image(mp_image_t *mpi) {
               mpi->x, mpi->y, w, h, slice);
   if (is_yuv) {
     int xs, ys;
-    mp_get_chroma_shift(image_format, &xs, &ys);
+    mp_get_chroma_shift(image_format, &xs, &ys, NULL);
     if ((mpi->flags & MP_IMGFLAG_DIRECT) && !(mpi->flags & MP_IMGFLAG_COMMON_PLANE)) {
       mpglBindBuffer(GL_PIXEL_UNPACK_BUFFER, gl_buffer_uv[0]);
       mpglUnmapBuffer(GL_PIXEL_UNPACK_BUFFER);
@@ -1088,6 +1088,7 @@ draw_frame(uint8_t *src[])
 static int
 query_format(uint32_t format)
 {
+    int depth;
     int caps = VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW |
                VFCAP_FLIP |
                VFCAP_HWSCALE_UP | VFCAP_HWSCALE_DOWN | VFCAP_ACCEPT_STRIDE;
@@ -1095,7 +1096,8 @@ query_format(uint32_t format)
       caps |= VFCAP_OSD | VFCAP_EOSD | (scaled_osd ? 0 : VFCAP_EOSD_UNSCALED);
     if (format == IMGFMT_RGB24 || format == IMGFMT_RGBA)
         return caps;
-    if (use_yuv && mp_get_chroma_shift(format, NULL, NULL) &&
+    if (use_yuv && mp_get_chroma_shift(format, NULL, NULL, &depth) &&
+        (depth == 8 || depth == 16) &&
         (IMGFMT_IS_YUVP16_NE(format) || !IMGFMT_IS_YUVP16(format)))
         return caps;
     // HACK, otherwise we get only b&w with some filters (e.g. -vf eq)
diff --git a/libvo/vo_gl2.c b/libvo/vo_gl2.c
index 957125069b..21f779866c 100644
--- a/libvo/vo_gl2.c
+++ b/libvo/vo_gl2.c
@@ -274,7 +274,7 @@ static int initTextures(void)
       glTexEnvf (GL_TEXTURE_ENV, GL_TEXTURE_ENV_MODE, GL_MODULATE);
       if (is_yuv) {
         int xs, ys;
-        mp_get_chroma_shift(image_format, &xs, &ys);
+        mp_get_chroma_shift(image_format, &xs, &ys, NULL);
         mpglActiveTexture(GL_TEXTURE1);
         glCreateClearTex(GL_TEXTURE_2D, gl_internal_format, gl_bitmap_format,  gl_bitmap_type, GL_LINEAR,
                          texture_width >> xs, texture_height >> ys, 128);
@@ -580,7 +580,7 @@ static int initGl(uint32_t d_width, uint32_t d_height)
         mpglBindProgram(GL_FRAGMENT_PROGRAM, fragprog);
         break;
     }
-    mp_get_chroma_shift(image_format, &xs, &ys);
+    mp_get_chroma_shift(image_format, &xs, &ys, NULL);
     params.chrom_texw = params.texw >> xs;
     params.chrom_texh = params.texh >> ys;
     glSetupYUVConversion(&params);
@@ -616,7 +616,7 @@ config(uint32_t width, uint32_t height, uint32_t d_width, uint32_t d_height, uin
   image_height = height;
   image_width = width;
   image_format = format;
-  is_yuv = mp_get_chroma_shift(image_format, &xs, &ys) > 0;
+  is_yuv = mp_get_chroma_shift(image_format, &xs, &ys, NULL) > 0;
   is_yuv |= (xs << 8) | (ys << 16);
 
 #ifdef CONFIG_GL_WIN32
@@ -741,7 +741,7 @@ static int draw_slice(uint8_t *src[], int stride[], int w,int h,int x,int y)
   struct TexSquare *texline = &texgrid[y / texture_height * texnumx];
   int subtex_y = y % texture_width;
   int xs, ys;
-  mp_get_chroma_shift(image_format, &xs, &ys);
+  mp_get_chroma_shift(image_format, &xs, &ys, NULL);
   while (rem_h > 0) {
     int rem_w = w;
     struct TexSquare *tsq = &texline[x / texture_width];
@@ -802,7 +802,9 @@ draw_frame(uint8_t *src[])
 static int
 query_format(uint32_t format)
 {
-  if (use_yuv && mp_get_chroma_shift(format, NULL, NULL) &&
+  int depth;
+  if (use_yuv && mp_get_chroma_shift(format, NULL, NULL, &depth) &&
+      (depth == 8 || depth == 16) &&
       (IMGFMT_IS_YUVP16_NE(format) || !IMGFMT_IS_YUVP16(format)))
     return VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW | VFCAP_OSD |
            VFCAP_HWSCALE_UP | VFCAP_HWSCALE_DOWN | VFCAP_ACCEPT_STRIDE;