video: decouple internal pixel formats from FourCCs

mplayer's video chain traditionally used FourCCs for pixel formats. For
example, it used IMGFMT_YV12 for 4:2:0 YUV, which was defined to the
string 'YV12' interpreted as unsigned int. Additionally, it used to
encode information into the numeric values of some formats. The RGB
formats had their bit depth and endian encoded into the least
significant byte. Extended planar formats (420P10 etc.) had chroma
shift, endian, and component bit depth encoded. (This has been removed
in recent commits.)

Replace the FourCC mess with a simple enum. Remove all the redundant
formats like YV12/I420/IYUV. Replace some image format names by
something more intuitive, most importantly IMGFMT_YV12 -> IMGFMT_420P.

Add img_fourcc.h, which contains the old IDs for code that actually uses
FourCCs. Change the way demuxers, that output raw video, identify the
video format: they set either MP_FOURCC_RAWVIDEO or MP_FOURCC_IMGFMT to
request the rawvideo decoder, and sh_video->imgfmt specifies the pixel
format. Like the previous hack, this is supposed to avoid the need for
a complete codecs.cfg entry per format, or other lookup tables. (Note
that the RGB raw video FourCCs mostly rely on ffmpeg's mappings for NUT
raw video, but this is still considered better than adding a raw video
decoder - even if trivial, it would be full of annoying lookup tables.)

The TV code has not been tested.

Some corrective changes regarding endian and other image format flags
creep in.

10 files changed, 57 insertions, 41 deletions

diff --git a/video/out/gl_common.c b/video/out/gl_common.c
index b7b30702a6..697b1bbbbc 100644
--- a/video/out/gl_common.c
+++ b/video/out/gl_common.c
@@ -141,13 +141,13 @@ int glFindFormat(uint32_t fmt, int have_texture_rg, int *bpp, GLint *gl_texfmt,
         else if (IMGFMT_IS_YUVP16_BE(fmt))
             fmt = IMGFMT_420P16_BE;
         else
-            fmt = IMGFMT_YV12;
+            fmt = IMGFMT_420P;
     }
 
     *bpp = IMGFMT_IS_BGR(fmt) ? IMGFMT_BGR_DEPTH(fmt) : IMGFMT_RGB_DEPTH(fmt);
     *gl_texfmt = 3;
     switch (fmt) {
-    case IMGFMT_RGB48NE:
+    case IMGFMT_RGB48:
         *gl_format = GL_RGB;
         *gl_type = GL_UNSIGNED_SHORT;
         break;
@@ -167,9 +167,8 @@ int glFindFormat(uint32_t fmt, int have_texture_rg, int *bpp, GLint *gl_texfmt,
         *gl_format = have_texture_rg ? GL_RED : GL_LUMINANCE;
         *gl_type = GL_UNSIGNED_SHORT;
         break;
-    case IMGFMT_YV12:
+    case IMGFMT_420P:
         supported = 0; // no native YV12 support
-    case IMGFMT_Y800:
     case IMGFMT_Y8:
         *gl_texfmt = 1;
         *bpp = 8;
@@ -177,9 +176,6 @@ int glFindFormat(uint32_t fmt, int have_texture_rg, int *bpp, GLint *gl_texfmt,
         *gl_type = GL_UNSIGNED_BYTE;
         break;
     case IMGFMT_UYVY:
-    // IMGFMT_YUY2 would be more logical for the _REV format,
-    // but gives clearly swapped colors.
-    case IMGFMT_YVYU:
         *gl_texfmt = GL_YCBCR_MESA;
         *bpp = 16;
         *gl_format = GL_YCBCR_MESA;
diff --git a/video/out/vo.h b/video/out/vo.h
index eb55417f4c..73eb19c8d7 100644
--- a/video/out/vo.h
+++ b/video/out/vo.h
@@ -156,7 +156,7 @@ struct vo_driver {
 
     /*
      * Whether the given image format is supported and config() will succeed.
-     * format: fourcc of pixel format
+     * format: one of IMGFMT_*
      * returns: 0 on not supported, otherwise a bitmask of VFCAP_* values
      */
     int (*query_format)(struct vo *vo, uint32_t format);
diff --git a/video/out/vo_corevideo.m b/video/out/vo_corevideo.m
index 5e1ecf25a7..a31a2bd7b1 100644
--- a/video/out/vo_corevideo.m
+++ b/video/out/vo_corevideo.m
@@ -248,7 +248,7 @@ static int query_format(struct vo *vo, uint32_t format)
     const int flags = VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW |
                       VFCAP_OSD;
     switch (format) {
-        case IMGFMT_YUY2:
+        case IMGFMT_YUYV:
             p->pixelFormat = kYUVSPixelFormat;
             return flags;
 
@@ -346,7 +346,7 @@ static int get_image_fmt(struct vo *vo)
 {
     struct priv *p = vo->priv;
     switch (p->pixelFormat) {
-        case kYUVSPixelFormat:   return IMGFMT_YUY2;
+        case kYUVSPixelFormat:   return IMGFMT_YUYV;
         case k24RGBPixelFormat:  return IMGFMT_RGB24;
         case k32ARGBPixelFormat: return IMGFMT_ARGB;
         case k32BGRAPixelFormat: return IMGFMT_BGRA;
diff --git a/video/out/vo_direct3d.c b/video/out/vo_direct3d.c
index 51caaf2be9..6dbf1d3d59 100644
--- a/video/out/vo_direct3d.c
+++ b/video/out/vo_direct3d.c
@@ -202,12 +202,12 @@ struct fmt_entry {
 
 static const struct fmt_entry fmt_table[] = {
     // planar YUV
-    {IMGFMT_YV12,  MAKEFOURCC('Y','V','1','2')},
-    {IMGFMT_I420,  MAKEFOURCC('I','4','2','0')},
-    {IMGFMT_IYUV,  MAKEFOURCC('I','Y','U','V')},
-    {IMGFMT_YVU9,  MAKEFOURCC('Y','V','U','9')},
+    {IMGFMT_420P,  MAKEFOURCC('Y','V','1','2')},
+    {IMGFMT_420P,  MAKEFOURCC('I','4','2','0')},
+    {IMGFMT_420P,  MAKEFOURCC('I','Y','U','V')},
+    {IMGFMT_410P,  MAKEFOURCC('Y','V','U','9')},
     // packed YUV
-    {IMGFMT_YUY2,  D3DFMT_YUY2},
+    {IMGFMT_YUYV,  D3DFMT_YUY2},
     {IMGFMT_UYVY,  D3DFMT_UYVY},
     // packed RGB
     {IMGFMT_BGR32, D3DFMT_X8R8G8B8},
@@ -1460,7 +1460,7 @@ static int control(struct vo *vo, uint32_t request, void *data)
  *  @param d_height Screen (destination) height
  *  @param options  Options bitmap
  *  @param format   Movie colorspace format (using MPlayer's
- *                  defines, e.g. IMGFMT_YUY2)
+ *                  defines, e.g. IMGFMT_YUYV)
  *  @return 0 on success, VO_ERROR on failure
  */
 static int config(struct vo *vo, uint32_t width, uint32_t height,
diff --git a/video/out/vo_opengl.c b/video/out/vo_opengl.c
index 665e008178..d0c435c59a 100644
--- a/video/out/vo_opengl.c
+++ b/video/out/vo_opengl.c
@@ -237,7 +237,7 @@ struct fmt_entry {
 };
 
 static const struct fmt_entry mp_to_gl_formats[] = {
-    {IMGFMT_RGB48NE, GL_RGB16, GL_RGB,  16, GL_UNSIGNED_SHORT},
+    {IMGFMT_RGB48,   GL_RGB16, GL_RGB,  16, GL_UNSIGNED_SHORT},
     {IMGFMT_RGB24,   GL_RGB,   GL_RGB,  8,  GL_UNSIGNED_BYTE},
     {IMGFMT_RGBA,    GL_RGBA,  GL_RGBA, 8,  GL_UNSIGNED_BYTE},
     {IMGFMT_RGB15,   GL_RGBA,  GL_RGBA, 5,  GL_UNSIGNED_SHORT_1_5_5_5_REV},
diff --git a/video/out/vo_opengl_old.c b/video/out/vo_opengl_old.c
index 6605876226..de38fcb067 100644
--- a/video/out/vo_opengl_old.c
+++ b/video/out/vo_opengl_old.c
@@ -831,9 +831,9 @@ static int query_format(struct vo *vo, uint32_t format)
         return caps;
     // HACK, otherwise we get only b&w with some filters (e.g. -vf eq)
     // ideally MPlayer should be fixed instead not to use Y800 when it has the choice
-    if (!p->use_yuv && (format == IMGFMT_Y8 || format == IMGFMT_Y800))
+    if (!p->use_yuv && (format == IMGFMT_Y8))
         return 0;
-    if (!p->use_ycbcr && (format == IMGFMT_UYVY || format == IMGFMT_YVYU))
+    if (!p->use_ycbcr && (format == IMGFMT_UYVY))
         return 0;
     if (p->many_fmts &&
         glFindFormat(format, p->have_texture_rg, NULL, NULL, NULL, NULL))
diff --git a/video/out/vo_sdl.c b/video/out/vo_sdl.c
index c3ed3c6774..d760aff50c 100644
--- a/video/out/vo_sdl.c
+++ b/video/out/vo_sdl.c
@@ -54,11 +54,11 @@ struct formatmap_entry {
     int is_rgba;
 };
 const struct formatmap_entry formats[] = {
-    {SDL_PIXELFORMAT_YV12, IMGFMT_YV12, 0},
-    {SDL_PIXELFORMAT_IYUV, IMGFMT_IYUV, 0},
-    {SDL_PIXELFORMAT_YUY2, IMGFMT_YUY2, 0},
+    {SDL_PIXELFORMAT_YV12, IMGFMT_420P, 0},
+    {SDL_PIXELFORMAT_IYUV, IMGFMT_420P, 0},
+    {SDL_PIXELFORMAT_YUY2, IMGFMT_YUYV, 0},
     {SDL_PIXELFORMAT_UYVY, IMGFMT_UYVY, 0},
-    {SDL_PIXELFORMAT_YVYU, IMGFMT_YVYU, 0},
+    //{SDL_PIXELFORMAT_YVYU, IMGFMT_YVYU, 0},
 #if BYTE_ORDER == BIG_ENDIAN
     {SDL_PIXELFORMAT_RGBX8888, IMGFMT_RGBA, 0}, // has no alpha -> bad for OSD
     {SDL_PIXELFORMAT_BGRX8888, IMGFMT_BGRA, 0}, // has no alpha -> bad for OSD
@@ -172,6 +172,7 @@ struct priv {
     int renderer_index;
     SDL_RendererInfo renderer_info;
     SDL_Texture *tex;
+    int tex_swapped;
     mp_image_t texmpi;
     mp_image_t *ssmpi;
     struct mp_rect src_rect;
@@ -427,6 +428,7 @@ static int config(struct vo *vo, uint32_t width, uint32_t height,
         return -1;
     }
 
+    vc->tex_swapped = texfmt == SDL_PIXELFORMAT_YV12;
     vc->tex = SDL_CreateTexture(vc->renderer, texfmt,
                                 SDL_TEXTUREACCESS_STREAMING, width, height);
     if (!vc->tex) {
@@ -878,7 +880,7 @@ static void draw_image(struct vo *vo, mp_image_t *mpi)
         texmpi->planes[0] = pixels;
         texmpi->stride[0] = pitch;
         if (texmpi->num_planes == 3) {
-            if (texmpi->imgfmt == IMGFMT_YV12) {
+            if (vc->tex_swapped) {
                 texmpi->planes[2] =
                     ((Uint8 *) texmpi->planes[0] + texmpi->h * pitch);
                 texmpi->stride[2] = pitch / 2;
diff --git a/video/out/vo_vdpau.c b/video/out/vo_vdpau.c
index 42c8bc6a24..1d5f0ed86c 100644
--- a/video/out/vo_vdpau.c
+++ b/video/out/vo_vdpau.c
@@ -756,15 +756,13 @@ static int initialize_vdpau_objects(struct vo *vo)
 
     vc->vdp_chroma_type = VDP_CHROMA_TYPE_420;
     switch (vc->image_format) {
-    case IMGFMT_YV12:
-    case IMGFMT_I420:
-    case IMGFMT_IYUV:
+    case IMGFMT_420P:
         vc->vdp_pixel_format = VDP_YCBCR_FORMAT_YV12;
         break;
     case IMGFMT_NV12:
         vc->vdp_pixel_format = VDP_YCBCR_FORMAT_NV12;
         break;
-    case IMGFMT_YUY2:
+    case IMGFMT_YUYV:
         vc->vdp_pixel_format = VDP_YCBCR_FORMAT_YUYV;
         vc->vdp_chroma_type  = VDP_CHROMA_TYPE_422;
         break;
@@ -1428,11 +1426,9 @@ static int query_format(struct vo *vo, uint32_t format)
     int default_flags = VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW
         | VFCAP_OSD | VFCAP_FLIP;
     switch (format) {
-    case IMGFMT_YV12:
-    case IMGFMT_I420:
-    case IMGFMT_IYUV:
+    case IMGFMT_420P:
     case IMGFMT_NV12:
-    case IMGFMT_YUY2:
+    case IMGFMT_YUYV:
     case IMGFMT_UYVY:
     case IMGFMT_VDPAU_MPEG1:
     case IMGFMT_VDPAU_MPEG2:
diff --git a/video/out/vo_x11.c b/video/out/vo_x11.c
index 5a81648233..2ce28a4292 100644
--- a/video/out/vo_x11.c
+++ b/video/out/vo_x11.c
@@ -536,9 +536,7 @@ static int query_format(struct vo *vo, uint32_t format)
     }
 
     switch (format) {
-    case IMGFMT_I420:
-    case IMGFMT_IYUV:
-    case IMGFMT_YV12:
+    case IMGFMT_420P:
         return VFCAP_CSP_SUPPORTED | VFCAP_OSD;
     }
     return 0;
diff --git a/video/out/vo_xv.c b/video/out/vo_xv.c
index d5225cd7cd..dc7f805e4b 100644
--- a/video/out/vo_xv.c
+++ b/video/out/vo_xv.c
@@ -47,6 +47,7 @@
 #include "vo.h"
 #include "video/vfcap.h"
 #include "video/mp_image.h"
+#include "video/img_fourcc.h"
 #include "x11_common.h"
 #include "video/memcpy_pic.h"
 #include "sub/sub.h"
@@ -88,10 +89,31 @@ struct xvctx {
 #endif
 };
 
+struct fmt_entry {
+    int imgfmt;
+    int fourcc;
+};
+static const struct fmt_entry fmt_table[] = {
+    {IMGFMT_420P,       MP_FOURCC_YV12},
+    {IMGFMT_420P,       MP_FOURCC_I420},
+    {IMGFMT_YUYV,       MP_FOURCC_YUY2},
+    {IMGFMT_UYVY,       MP_FOURCC_UYVY},
+    {0}
+};
+
 static void allocate_xvimage(struct vo *, int);
 static void deallocate_xvimage(struct vo *vo, int foo);
 static struct mp_image get_xv_buffer(struct vo *vo, int buf_index);
 
+static int find_xv_format(int imgfmt)
+{
+    for (int n = 0; fmt_table[n].imgfmt; n++) {
+        if (fmt_table[n].imgfmt == imgfmt)
+            return fmt_table[n].fourcc;
+    }
+    return 0;
+}
+
 static void read_xv_csp(struct vo *vo)
 {
     struct xvctx *ctx = vo->priv;
@@ -158,7 +180,7 @@ static int config(struct vo *vo, uint32_t width, uint32_t height,
         mp_msg(MSGT_VO, MSGL_V, "Xvideo image format: 0x%x (%4.4s) %s\n",
                ctx->fo[i].id, (char *) &ctx->fo[i].id,
                (ctx->fo[i].format == XvPacked) ? "packed" : "planar");
-        if (ctx->fo[i].id == format)
+        if (ctx->fo[i].id == find_xv_format(format))
             ctx->xv_format = ctx->fo[i].id;
     }
     if (!ctx->xv_format)
@@ -320,7 +342,7 @@ static struct mp_image get_xv_buffer(struct vo *vo, int buf_index)
     mp_image_set_size(&img, ctx->image_width, ctx->image_height);
     mp_image_setfmt(&img, ctx->image_format);
 
-    bool swapuv = ctx->image_format == IMGFMT_YV12;
+    bool swapuv = ctx->xv_format == MP_FOURCC_YV12;
     for (int n = 0; n < img.num_planes; n++) {
         int sn = n > 0 &&  swapuv ? (n == 1 ? 2 : 1) : n;
         img.planes[n] = xv_image->data + xv_image->offsets[sn];
@@ -417,10 +439,12 @@ static int query_format(struct vo *vo, uint32_t format)
     uint32_t i;
     int flag = VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW | VFCAP_OSD;
 
-    /* check image formats */
-    for (i = 0; i < ctx->formats; i++) {
-        if (ctx->fo[i].id == format)
-            return flag;        //xv_format = fo[i].id;
+    int fourcc = find_xv_format(format);
+    if (fourcc) {
+        for (i = 0; i < ctx->formats; i++) {
+            if (ctx->fo[i].id == fourcc)
+                return flag;
+        }
     }
     return 0;
 }