video: remove img_format compat hacks

Remove the strange things the old mp_image_setfmt() code did to the
image format parameters. This includes setting chroma shift to 31 for
gray (Y8) formats and more.

Y8 + vo_opengl_old didn't actually work for unknown reasons (regression
in this branch). Fix this. The difference is that Y8 is now interpreted
as gray RGB (LUMINANCE texture) instead of involving YUV (and levels)
conversion.

Get rid of distinguishing RGB and BGR. There wasn't really any good
reason for this.

Remove mp_get_chroma_shift() and IMGFMT_IS_YUVP16*(). mp_imgfmt_desc
gives the same information and more.

5 files changed, 68 insertions, 70 deletions

diff --git a/video/out/gl_common.c b/video/out/gl_common.c
index 697b1bbbbc..c2a9c88b28 100644
--- a/video/out/gl_common.c
+++ b/video/out/gl_common.c
@@ -110,7 +110,7 @@ void glAdjustAlignment(GL *gl, int stride)
  *
  * All parameters may be NULL.
  * \param fmt MPlayer format to analyze.
- * \param bpp [OUT] bits per pixel of that format.
+ * \param dummy reserved
  * \param gl_texfmt [OUT] internal texture format that fits the
  * image format, not necessarily the best for performance.
  * \param gl_format [OUT] OpenGL format for this image format.
@@ -118,15 +118,12 @@ void glAdjustAlignment(GL *gl, int stride)
  * \return 1 if format is supported by OpenGL, 0 if not.
  * \ingroup gltexture
  */
-int glFindFormat(uint32_t fmt, int have_texture_rg, int *bpp, GLint *gl_texfmt,
+int glFindFormat(uint32_t fmt, int have_texture_rg, int *dummy, GLint *gl_texfmt,
                  GLenum *gl_format, GLenum *gl_type)
 {
     int supported = 1;
-    int dummy1;
     GLenum dummy2;
     GLint dummy3;
-    if (!bpp)
-        bpp = &dummy1;
     if (!gl_texfmt)
         gl_texfmt = &dummy3;
     if (!gl_format)
@@ -134,17 +131,15 @@ int glFindFormat(uint32_t fmt, int have_texture_rg, int *bpp, GLint *gl_texfmt,
     if (!gl_type)
         gl_type = &dummy2;
 
-    if (mp_get_chroma_shift(fmt, NULL, NULL, NULL)) {
+    struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(fmt);
+    if (desc.flags & MP_IMGFLAG_YUV_P) {
         // reduce the possible cases a bit
-        if (IMGFMT_IS_YUVP16_LE(fmt))
-            fmt = IMGFMT_420P16_LE;
-        else if (IMGFMT_IS_YUVP16_BE(fmt))
-            fmt = IMGFMT_420P16_BE;
+        if (desc.plane_bits > 8)
+            fmt = IMGFMT_420P16;
         else
             fmt = IMGFMT_420P;
     }
 
-    *bpp = IMGFMT_IS_BGR(fmt) ? IMGFMT_BGR_DEPTH(fmt) : IMGFMT_RGB_DEPTH(fmt);
     *gl_texfmt = 3;
     switch (fmt) {
     case IMGFMT_RGB48:
@@ -163,7 +158,6 @@ int glFindFormat(uint32_t fmt, int have_texture_rg, int *bpp, GLint *gl_texfmt,
     case IMGFMT_420P16:
         supported = 0; // no native YUV support
         *gl_texfmt = have_texture_rg ? GL_R16 : GL_LUMINANCE16;
-        *bpp = 16;
         *gl_format = have_texture_rg ? GL_RED : GL_LUMINANCE;
         *gl_type = GL_UNSIGNED_SHORT;
         break;
@@ -171,13 +165,11 @@ int glFindFormat(uint32_t fmt, int have_texture_rg, int *bpp, GLint *gl_texfmt,
         supported = 0; // no native YV12 support
     case IMGFMT_Y8:
         *gl_texfmt = 1;
-        *bpp = 8;
         *gl_format = GL_LUMINANCE;
         *gl_type = GL_UNSIGNED_BYTE;
         break;
     case IMGFMT_UYVY:
         *gl_texfmt = GL_YCBCR_MESA;
-        *bpp = 16;
         *gl_format = GL_YCBCR_MESA;
         *gl_type = fmt == IMGFMT_UYVY ? GL_UNSIGNED_SHORT_8_8 : GL_UNSIGNED_SHORT_8_8_REV;
         break;
diff --git a/video/out/vo_direct3d.c b/video/out/vo_direct3d.c
index 6dbf1d3d59..6b9ca7e32b 100644
--- a/video/out/vo_direct3d.c
+++ b/video/out/vo_direct3d.c
@@ -961,14 +961,12 @@ static int get_chroma_clear_val(int bit_depth)
 // this macro is supposed to work on all formats supported by 3D rendering, and
 // that produce "reasonable" output (i.e. no mixed up colors)
 #define IMGFMT_IS_ANY_RND(x) \
-    (IMGFMT_IS_BGR(x) || IMGFMT_IS_RGB(x) || IMGFMT_IS_Y(x))
+    (IMGFMT_IS_RGB(x) || IMGFMT_IS_Y(x))
 
 // pixel size in bit for any IMGFMT_IS_ANY_RND(x)==true
 // we assume that the actual pixel strides are always aligned on bytes
 static int imgfmt_any_rnd_depth(int fmt)
 {
-    if (IMGFMT_IS_BGR(fmt))
-        return IMGFMT_BGR_DEPTH(fmt);
     if (IMGFMT_IS_RGB(fmt))
         return IMGFMT_RGB_DEPTH(fmt);
     if (IMGFMT_IS_Y(fmt))
@@ -1033,9 +1031,12 @@ static D3DFORMAT check_shader_conversion(d3d_priv *priv, uint32_t fmt)
 {
     if (priv->opt_disable_shaders)
         return 0;
-    int component_bits;
-    if (!mp_get_chroma_shift(fmt, NULL, NULL, &component_bits))
+    struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(fmt);
+    if (!(desc.flags & MP_IMGFLAG_YUV_P) || !(desc.flags & MP_IMGFLAG_NE))
         return 0;
+    if (desc.num_planes != 3)
+        return 0;
+    int component_bits = desc.plane_bits;
     if (component_bits < 8 || component_bits > 16)
         return 0;
     bool is_8bit = component_bits == 8;
@@ -1116,17 +1117,15 @@ static bool init_rendering_mode(d3d_priv *priv, uint32_t fmt, bool initialize)
         } else {
             mp_msg(MSGT_VO, MSGL_V, "<vo_direct3d>Using YUV shaders.\n");
 
-            int sx, sy, component_bits;
-            mp_get_chroma_shift(priv->image_format, &sx, &sy, &component_bits);
+            struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(priv->image_format);
             priv->plane_count = 3;
             for (n = 0; n < 3; n++) {
                 planes[n].d3d_format = shader_d3dfmt;
-                planes[n].bits_per_pixel = component_bits;
-                if (n > 0) {
-                    planes[n].shift_x = sx;
-                    planes[n].shift_y = sy;
-                    planes[n].clearval = get_chroma_clear_val(component_bits);
-                }
+                planes[n].bits_per_pixel = desc.plane_bits;
+                planes[n].shift_x = desc.xs[n];
+                planes[n].shift_y = desc.ys[n];
+                if (n > 0)
+                    planes[n].clearval = get_chroma_clear_val(desc.plane_bits);
             }
             if (shader_d3dfmt != D3DFMT_A8L8) {
                 priv->pixel_shader_data = d3d_shader_yuv;
diff --git a/video/out/vo_opengl.c b/video/out/vo_opengl.c
index d0c435c59a..4cb9fb76e1 100644
--- a/video/out/vo_opengl.c
+++ b/video/out/vo_opengl.c
@@ -1558,17 +1558,19 @@ static bool init_format(int fmt, struct gl_priv *init)
     if (!init)
         init = &dummy;
 
-    mp_image_t dummy_img = {0};
-    mp_image_setfmt(&dummy_img, fmt);
+    struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(fmt);
+    if (!desc.id)
+        return false;
 
     init->image_format = fmt;
     init->component_bits = -1;
+    init->plane_bits = desc.plane_bits;
 
     // RGB/packed formats
     for (const struct fmt_entry *e = mp_to_gl_formats; e->mp_format; e++) {
         if (e->mp_format == fmt) {
             supported = true;
-            init->plane_bits = dummy_img.bpp;
+            init->plane_bits = desc.bpp[0];
             init->gl_format = e->format;
             init->gl_internal_format = e->internal_format;
             init->component_bits = e->component_bits;
@@ -1578,14 +1580,14 @@ static bool init_format(int fmt, struct gl_priv *init)
     }
 
     // YUV/planar formats
-    if (!supported && mp_get_chroma_shift(fmt, NULL, NULL, &init->plane_bits)) {
+    if (!supported && (desc.flags & MP_IMGFLAG_YUV_P)) {
         init->gl_format = GL_RED;
         init->component_bits = init->plane_bits;
         if (init->plane_bits == 8) {
             supported = true;
             init->gl_internal_format = GL_RED;
             init->gl_type = GL_UNSIGNED_BYTE;
-        } else if (IMGFMT_IS_YUVP16_NE(fmt)) {
+        } else if (init->plane_bits <= 16 && (desc.flags & MP_IMGFLAG_NE)) {
             supported = true;
             init->gl_internal_format = GL_R16;
             init->gl_type = GL_UNSIGNED_SHORT;
@@ -1605,19 +1607,19 @@ static bool init_format(int fmt, struct gl_priv *init)
         return false;
 
     init->plane_bytes = (init->plane_bits + 7) / 8;
-    init->is_yuv = dummy_img.flags & MP_IMGFLAG_YUV;
+    init->is_yuv = desc.flags & MP_IMGFLAG_YUV;
     init->is_linear_rgb = false;
 
     // NOTE: we throw away the additional alpha plane, if one exists.
-    init->plane_count = dummy_img.num_planes > 2 ? 3 : 1;
-    assert(dummy_img.num_planes >= init->plane_count);
-    assert(dummy_img.num_planes <= init->plane_count + 1);
+    init->plane_count = desc.num_planes > 2 ? 3 : 1;
+    assert(desc.num_planes >= init->plane_count);
+    assert(desc.num_planes <= init->plane_count + 1);
 
     for (int n = 0; n < init->plane_count; n++) {
         struct texplane *plane = &init->planes[n];
 
-        plane->shift_x = n > 0 ? dummy_img.chroma_x_shift : 0;
-        plane->shift_y = n > 0 ? dummy_img.chroma_y_shift : 0;
+        plane->shift_x = desc.xs[n];
+        plane->shift_y = desc.ys[n];
     }
 
     return true;
diff --git a/video/out/vo_opengl_old.c b/video/out/vo_opengl_old.c
index de38fcb067..5cfe306f8e 100644
--- a/video/out/vo_opengl_old.c
+++ b/video/out/vo_opengl_old.c
@@ -169,7 +169,6 @@ static void update_yuvconv(struct vo *vo)
     struct gl_priv *p = vo->priv;
     GL *gl = p->gl;
 
-    int xs, ys, depth;
     struct mp_csp_params cparams = { .colorspace = p->colorspace };
     mp_csp_copy_equalizer_values(&cparams, &p->video_eq);
     gl_conversion_params_t params = {
@@ -177,9 +176,10 @@ static void update_yuvconv(struct vo *vo)
         p->texture_width, p->texture_height, 0, 0, p->filter_strength,
         p->noise_strength
     };
-    mp_get_chroma_shift(p->image_format, &xs, &ys, &depth);
-    params.chrom_texw = params.texw >> xs;
-    params.chrom_texh = params.texh >> ys;
+    struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(p->image_format);
+    int depth = desc.plane_bits;
+    params.chrom_texw = params.texw >> desc.chroma_xs;
+    params.chrom_texh = params.texh >> desc.chroma_ys;
     params.csp_params.input_bits = depth;
     params.csp_params.texture_bits = depth+7 & ~7;
     glSetupYUVConversion(gl, &params);
@@ -427,10 +427,10 @@ static int initGl(struct vo *vo, uint32_t d_width, uint32_t d_height)
         gl->TexParameteri(p->target, GL_GENERATE_MIPMAP, GL_TRUE);
 
     if (p->is_yuv) {
+        struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(p->image_format);
         int i;
-        int xs, ys, depth;
+        int xs = desc.chroma_xs, ys = desc.chroma_ys, depth = desc.plane_bits;
         scale_type = get_scale_type(vo, 1);
-        mp_get_chroma_shift(p->image_format, &xs, &ys, &depth);
         int clear = get_chroma_clear_val(depth);
         gl->GenTextures(21, p->default_texs);
         p->default_texs[21] = 0;
@@ -502,12 +502,15 @@ static int config(struct vo *vo, uint32_t width, uint32_t height,
 {
     struct gl_priv *p = vo->priv;
 
-    int xs, ys;
+    struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(format);
+
     p->image_height = height;
     p->image_width = width;
     p->image_format = format;
-    p->is_yuv = mp_get_chroma_shift(p->image_format, &xs, &ys, NULL) > 0;
-    p->is_yuv |= (xs << 8) | (ys << 16);
+    p->is_yuv = !!(desc.flags & MP_IMGFLAG_YUV_P);
+    p->is_yuv |= (desc.chroma_xs << 8) | (desc.chroma_ys << 16);
+    if (format == IMGFMT_Y8)
+        p->is_yuv = 0;
     glFindFormat(format, p->have_texture_rg, NULL, &p->texfmt, &p->gl_format,
                  &p->gl_type);
 
@@ -630,9 +633,9 @@ static bool get_image(struct vo *vo, mp_image_t *mpi, int *th, bool *cplane)
     }
     if (p->is_yuv) {
         // planar YUV
-        int xs, ys, component_bits;
-        mp_get_chroma_shift(p->image_format, &xs, &ys, &component_bits);
-        int bp = (component_bits + 7) / 8;
+        struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(p->image_format);
+        int xs = desc.chroma_xs, ys = desc.chroma_ys, depth = desc.plane_bits;
+        int bp = (depth + 7) / 8;
         common_plane = true;
         mpi->stride[0] = width * bp;
         mpi->planes[1] = mpi->planes[0] + mpi->stride[0] * height;
@@ -702,11 +705,9 @@ static void draw_image(struct vo *vo, mp_image_t *mpi)
     if (p->force_pbo && !p->bufferptr
         && get_image(vo, &mpi2, &th, &common_plane))
     {
-        int bp = mpi->bpp / 8;
-        int xs, ys, component_bits;
-        mp_get_chroma_shift(p->image_format, &xs, &ys, &component_bits);
-        if (p->is_yuv)
-            bp = (component_bits + 7) / 8;
+        struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(p->image_format);
+        int bp = desc.bytes[0];
+        int xs = desc.chroma_xs, ys = desc.chroma_ys, depth = desc.plane_bits;
         memcpy_pic(mpi2.planes[0], mpi->planes[0], mpi->w * bp, mpi->h,
                    mpi2.stride[0], mpi->stride[0]);
         int uv_bytes = (mpi->w >> xs) * bp;
@@ -721,7 +722,7 @@ static void draw_image(struct vo *vo, mp_image_t *mpi)
             clear_border(vo, mpi2.planes[0], mpi->w * bp, mpi2.stride[0],
                          mpi->h, th, 0);
             if (p->is_yuv) {
-                int clear = get_chroma_clear_val(component_bits);
+                int clear = get_chroma_clear_val(depth);
                 clear_border(vo, mpi2.planes[1], uv_bytes, mpi2.stride[1],
                              mpi->h >> ys, th >> ys, clear);
                 clear_border(vo, mpi2.planes[2], uv_bytes, mpi2.stride[2],
@@ -759,8 +760,8 @@ static void draw_image(struct vo *vo, mp_image_t *mpi)
     glUploadTex(gl, p->target, p->gl_format, p->gl_type, planes[0],
                 stride[0], 0, 0, w, h, slice);
     if (p->is_yuv) {
-        int xs, ys;
-        mp_get_chroma_shift(p->image_format, &xs, &ys, NULL);
+        struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(p->image_format);
+        int xs = desc.chroma_xs, ys = desc.chroma_ys;
         if (pbo && !common_plane) {
             gl->BindBuffer(GL_PIXEL_UNPACK_BUFFER, p->buffer_uv[0]);
             gl->UnmapBuffer(GL_PIXEL_UNPACK_BUFFER);
@@ -818,16 +819,18 @@ static int query_format(struct vo *vo, uint32_t format)
 {
     struct gl_priv *p = vo->priv;
 
-    int depth;
+    struct mp_imgfmt_desc desc = mp_imgfmt_get_desc(format);
+
+    int depth = desc.plane_bits;
     int caps = VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW | VFCAP_FLIP;
     if (p->use_osd)
         caps |= VFCAP_OSD;
     if (format == IMGFMT_RGB24 || format == IMGFMT_RGBA)
         return caps;
-    if (p->use_yuv && mp_get_chroma_shift(format, NULL, NULL, &depth) &&
+    if (p->use_yuv && (desc.flags & MP_IMGFLAG_YUV_P) &&
         (depth == 8 || depth == 16 ||
          p->max_tex_component_size >= 16 && glYUVLargeRange(p->use_yuv)) &&
-        (IMGFMT_IS_YUVP16_NE(format) || !IMGFMT_IS_YUVP16(format)))
+        (depth <= 16 && (desc.flags & MP_IMGFLAG_NE)))
         return caps;
     // HACK, otherwise we get only b&w with some filters (e.g. -vf eq)
     // ideally MPlayer should be fixed instead not to use Y800 when it has the choice
diff --git a/video/out/vo_x11.c b/video/out/vo_x11.c
index 2ce28a4292..8b41a7afc3 100644
--- a/video/out/vo_x11.c
+++ b/video/out/vo_x11.c
@@ -524,15 +524,17 @@ static int query_format(struct vo *vo, uint32_t format)
     mp_msg(MSGT_VO, MSGL_DBG2,
            "vo_x11: query_format was called: %x (%s)\n", format,
            vo_format_name(format));
-    if (IMGFMT_IS_BGR(format)) {
-        if (IMGFMT_BGR_DEPTH(format) < 8)
-            return 0;
-        if (IMGFMT_BGR_DEPTH(format) == vo->x11->depthonscreen)
-            return VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW |
-                   VFCAP_OSD | VFCAP_FLIP;
-        else
-            return VFCAP_CSP_SUPPORTED | VFCAP_OSD |
-                   VFCAP_FLIP;
+    if (IMGFMT_IS_RGB(format)) {
+        for (int n = 0; fmt2Xfmt[n].mpfmt; n++) {
+            if (fmt2Xfmt[n].mpfmt == format) {
+                if (IMGFMT_RGB_DEPTH(format) == vo->x11->depthonscreen) {
+                    return VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW |
+                           VFCAP_OSD | VFCAP_FLIP;
+                } else {
+                    return VFCAP_CSP_SUPPORTED | VFCAP_OSD | VFCAP_FLIP;
+                }
+            }
+        }
     }
 
     switch (format) {