vo_gl: don't accept 9/10-bit formats as input

Make mp_get_chroma_shift() simpler/more generic and add an argument
to get the per-component bit depth.
Use this to check more properly for supported formats in
gl and gl2 vos (only 8 and 16 bit are supported, 9 and 10 are not).

git-svn-id: svn://svn.mplayerhq.hu/mplayer/trunk@33452 b3059339-0415-0410-9bf9-f77b7e298cf2

5 files changed, 56 insertions, 44 deletions

diff --git a/libmpcodecs/img_format.c b/libmpcodecs/img_format.c
index 09711e5270..c4884f3eca 100644
--- a/libmpcodecs/img_format.c
+++ b/libmpcodecs/img_format.c
@@ -19,6 +19,7 @@
 #include "config.h"
 #include "img_format.h"
 #include "stdio.h"
+#include "mpbswap.h"
 
 const char *vo_format_name(int format)
 {
@@ -117,20 +118,57 @@ const char *vo_format_name(int format)
     return unknown_format;
 }
 
-int mp_get_chroma_shift(int format, int *x_shift, int *y_shift)
+int mp_get_chroma_shift(int format, int *x_shift, int *y_shift, int *component_bits)
 {
     int xs = 0, ys = 0;
     int bpp;
-    int bpp_factor = 1;
     int err = 0;
-    switch (format) {
-    case IMGFMT_420P16_LE:
-    case IMGFMT_420P16_BE:
-    case IMGFMT_420P10_LE:
-    case IMGFMT_420P10_BE:
-    case IMGFMT_420P9_LE:
-    case IMGFMT_420P9_BE:
-        bpp_factor = 2;
+    int bits = 8;
+    if ((format & 0xff0000f0) == 0x34000050)
+        format = bswap_32(format);
+    if ((format & 0xf00000ff) == 0x50000034) {
+        switch (format >> 24) {
+        case 0x50:
+            break;
+        case 0x51:
+            bits = 16;
+            break;
+        case 0x52:
+            bits = 10;
+            break;
+        case 0x53:
+            bits = 9;
+            break;
+        default:
+            err = 1;
+            break;
+        }
+        switch (format & 0x00ffffff) {
+        case 0x00343434: // 444
+            xs = 0;
+            ys = 0;
+            break;
+        case 0x00323234: // 422
+            xs = 1;
+            ys = 0;
+            break;
+        case 0x00303234: // 420
+            xs = 1;
+            ys = 1;
+            break;
+        case 0x00313134: // 411
+            xs = 2;
+            ys = 0;
+            break;
+        case 0x00303434: // 440
+            xs = 0;
+            ys = 1;
+            break;
+        default:
+            err = 1;
+            break;
+        }
+    } else switch (format) {
     case IMGFMT_420A:
     case IMGFMT_I420:
     case IMGFMT_IYUV:
@@ -143,34 +181,6 @@ int mp_get_chroma_shift(int format, int *x_shift, int *y_shift)
         xs = 2;
         ys = 2;
         break;
-    case IMGFMT_444P16_LE:
-    case IMGFMT_444P16_BE:
-    case IMGFMT_444P10_LE:
-    case IMGFMT_444P10_BE:
-    case IMGFMT_444P9_LE:
-    case IMGFMT_444P9_BE:
-        bpp_factor = 2;
-    case IMGFMT_444P:
-        xs = 0;
-        ys = 0;
-        break;
-    case IMGFMT_422P16_LE:
-    case IMGFMT_422P16_BE:
-    case IMGFMT_422P10_LE:
-    case IMGFMT_422P10_BE:
-        bpp_factor = 2;
-    case IMGFMT_422P:
-        xs = 1;
-        ys = 0;
-        break;
-    case IMGFMT_411P:
-        xs = 2;
-        ys = 0;
-        break;
-    case IMGFMT_440P:
-        xs = 0;
-        ys = 1;
-        break;
     case IMGFMT_Y8:
     case IMGFMT_Y800:
         xs = 31;
@@ -182,9 +192,10 @@ int mp_get_chroma_shift(int format, int *x_shift, int *y_shift)
     }
     if (x_shift) *x_shift = xs;
     if (y_shift) *y_shift = ys;
+    if (component_bits) *component_bits = bits;
     bpp = 8 + ((16 >> xs) >> ys);
     if (format == IMGFMT_420A)
         bpp += 8;
-    bpp *= bpp_factor;
+    bpp *= (bits + 7) >> 3;
     return err ? 0 : bpp;
 }
diff --git a/libmpcodecs/img_format.h b/libmpcodecs/img_format.h
index 52a5ffbe25..2fc9775ee1 100644
--- a/libmpcodecs/img_format.h
+++ b/libmpcodecs/img_format.h
@@ -227,8 +227,9 @@ const char *vo_format_name(int format);
 /**
  * Calculates the scale shifts for the chroma planes for planar YUV
  *
+ * \param component_bits bits per component
  * \return bits-per-pixel for format if successful (i.e. format is 3 or 4-planes planar YUV), 0 otherwise
  */
-int mp_get_chroma_shift(int format, int *x_shift, int *y_shift);
+int mp_get_chroma_shift(int format, int *x_shift, int *y_shift, int *component_bits);
 
 #endif /* MPLAYER_IMG_FORMAT_H */
diff --git a/libmpcodecs/mp_image.c b/libmpcodecs/mp_image.c
index 4d2892cf81..88702dde60 100644
--- a/libmpcodecs/mp_image.c
+++ b/libmpcodecs/mp_image.c
@@ -123,9 +123,9 @@ void mp_image_setfmt(mp_image_t* mpi,unsigned int out_fmt){
     }
     mpi->flags|=MP_IMGFLAG_YUV;
     mpi->num_planes=3;
-    if (mp_get_chroma_shift(out_fmt, NULL, NULL)) {
+    if (mp_get_chroma_shift(out_fmt, NULL, NULL, NULL)) {
         mpi->flags|=MP_IMGFLAG_PLANAR;
-        mpi->bpp = mp_get_chroma_shift(out_fmt, &mpi->chroma_x_shift, &mpi->chroma_y_shift);
+        mpi->bpp = mp_get_chroma_shift(out_fmt, &mpi->chroma_x_shift, &mpi->chroma_y_shift, NULL);
         mpi->chroma_width  = mpi->width  >> mpi->chroma_x_shift;
         mpi->chroma_height = mpi->height >> mpi->chroma_y_shift;
     }
diff --git a/libmpcodecs/vf_sab.c b/libmpcodecs/vf_sab.c
index 7f05af9249..2773ed3109 100644
--- a/libmpcodecs/vf_sab.c
+++ b/libmpcodecs/vf_sab.c
@@ -116,7 +116,7 @@ static int config(struct vf_instance *vf,
 //__asm__ volatile("emms\n\t");
 	allocStuff(&vf->priv->luma, width, height);
 
-	mp_get_chroma_shift(outfmt, &sw, &sh);
+	mp_get_chroma_shift(outfmt, &sw, &sh, NULL);
 	allocStuff(&vf->priv->chroma, width>>sw, height>>sh);
 
 	return vf_next_config(vf,width,height,d_width,d_height,flags,outfmt);
diff --git a/libmpcodecs/vf_smartblur.c b/libmpcodecs/vf_smartblur.c
index 6599c967de..7c54231e40 100644
--- a/libmpcodecs/vf_smartblur.c
+++ b/libmpcodecs/vf_smartblur.c
@@ -76,7 +76,7 @@ static int config(struct vf_instance *vf,
 
 	allocStuff(&vf->priv->luma, width, height);
 
-	mp_get_chroma_shift(outfmt, &sw, &sh);
+	mp_get_chroma_shift(outfmt, &sw, &sh, NULL);
 	allocStuff(&vf->priv->chroma, width>>sw, height>>sh);
 
 	return vf_next_config(vf,width,height,d_width,d_height,flags,outfmt);