vo_opengl: rely on FFmpeg pixdesc a bit more

Add something that allows is to extract the component order from various
RGBA formats. In fact, also handle YUV, GBRP, and XYZ formats with this.

It introduces a new struct mp_regular_imgfmt, that hopefully will
eventually replace struct mp_imgfmt_desc. The latter is still needed by
a lot of code though, especially generic code. Also vo_opengl still uses
the old one, so this commit is sort of incomplete.

Due to its genericness, it's also possible that this commit introduces
rendering bugs, or accepts formats it shouldn't accept.

3 files changed, 231 insertions, 80 deletions

diff --git a/video/img_format.c b/video/img_format.c
index 759f707de2..b3fb27a49f 100644
--- a/video/img_format.c
+++ b/video/img_format.c
@@ -285,6 +285,157 @@ struct mp_imgfmt_desc mp_imgfmt_get_desc(int mpfmt)
     return desc;
 }
 
+static bool validate_regular_imgfmt(const struct mp_regular_imgfmt *fmt)
+{
+    bool present[MP_NUM_COMPONENTS] = {0};
+    int n_comp = 0;
+
+    for (int n = 0; n < fmt->num_planes; n++) {
+        const struct mp_regular_imgfmt_plane *plane = &fmt->planes[n];
+        n_comp += plane->num_components;
+        if (n_comp > MP_NUM_COMPONENTS)
+            return false;
+        if (!plane->num_components)
+            return false; // no empty planes in between allowed
+
+        bool pad_only = true;
+        int chroma_luma = 0; // luma: 1, chroma: 2, both: 3
+        for (int i = 0; i < plane->num_components; i++) {
+            int comp = plane->components[i];
+            if (comp > MP_NUM_COMPONENTS)
+                return false;
+            if (comp == 0)
+                continue;
+            pad_only = false;
+            if (present[comp - 1])
+                return false; // no duplicates
+            present[comp - 1] = true;
+            chroma_luma |= (comp == 2 || comp == 3) ? 2 : 1;
+        }
+        if (pad_only)
+            return false; // no planes with only padding allowed
+        if ((fmt->chroma_w > 1 || fmt->chroma_h > 1) && chroma_luma == 3)
+            return false; // separate chroma/luma planes required
+    }
+
+    if (!(present[0] || present[3]) ||  // at least component 1 or alpha needed
+        (present[1] && !present[0]) ||  // component 2 requires component 1
+        (present[2] && !present[1]))    // component 3 requires component 2
+        return false;
+
+    return true;
+}
+
+enum mp_csp mp_imgfmt_get_forced_csp(int imgfmt)
+{
+    const AVPixFmtDescriptor *pixdesc =
+        av_pix_fmt_desc_get(imgfmt2pixfmt(imgfmt));
+
+    // FFmpeg does not provide a flag for XYZ, so this is the best we can do.
+    if (pixdesc && strncmp(pixdesc->name, "xyz", 3) == 0)
+        return MP_CSP_XYZ;
+
+    if (pixdesc && (pixdesc->flags & AV_PIX_FMT_FLAG_RGB))
+        return MP_CSP_RGB;
+
+    return MP_CSP_AUTO;
+}
+
+static bool is_native_endian(const AVPixFmtDescriptor *pixdesc)
+{
+    enum AVPixelFormat pixfmt = av_pix_fmt_desc_get_id(pixdesc);
+    enum AVPixelFormat other = av_pix_fmt_swap_endianness(pixfmt);
+    if (other == AV_PIX_FMT_NONE || other == pixfmt)
+        return true; // no endian nonsense
+    bool is_le = *(char *)&(uint32_t){1};
+    return pixdesc && (is_le != !!(pixdesc->flags & AV_PIX_FMT_FLAG_BE));
+}
+
+bool mp_get_regular_imgfmt(struct mp_regular_imgfmt *dst, int imgfmt)
+{
+    struct mp_regular_imgfmt res = {0};
+
+    const AVPixFmtDescriptor *pixdesc =
+        av_pix_fmt_desc_get(imgfmt2pixfmt(imgfmt));
+
+    if (!pixdesc || (pixdesc->flags & AV_PIX_FMT_FLAG_BITSTREAM) ||
+        (pixdesc->flags & AV_PIX_FMT_FLAG_HWACCEL) ||
+        (pixdesc->flags & AV_PIX_FMT_FLAG_PAL) ||
+        pixdesc->nb_components < 1 ||
+        pixdesc->nb_components > MP_NUM_COMPONENTS ||
+        !is_native_endian(pixdesc))
+        return false;
+
+    const AVComponentDescriptor *comp0 = &pixdesc->comp[0];
+
+    int depth = comp0->depth + comp0->shift;
+    if (depth < 1 || depth > 64)
+        return false;
+    res.component_size = (depth + 7) / 8;
+
+    for (int n = 0; n < pixdesc->nb_components; n++) {
+        const AVComponentDescriptor *comp = &pixdesc->comp[n];
+
+        if (comp->plane < 0 || comp->plane >= MP_MAX_PLANES)
+            return false;
+
+        res.num_planes = MPMAX(res.num_planes, comp->plane + 1);
+
+        // We support uniform depth only.
+        if (comp->depth != comp0->depth || comp->shift != comp0->shift)
+            return false;
+
+        // Uniform component size; even the padding must have same size.
+        int ncomp = comp->step / res.component_size;
+        if (!ncomp || ncomp * res.component_size != comp->step)
+            return false;
+
+        struct mp_regular_imgfmt_plane *plane = &res.planes[comp->plane];
+
+        if (plane->num_components && plane->num_components != ncomp)
+            return false;
+        plane->num_components = ncomp;
+
+        int pos = comp->offset / res.component_size;
+        if (pos < 0 || pos >= ncomp || ncomp > MP_NUM_COMPONENTS)
+            return false;
+        if (plane->components[pos])
+            return false;
+        plane->components[pos] = n + 1;
+    }
+
+    // Make sure alpha is always component 4.
+    if (pixdesc->nb_components == 2 && (pixdesc->flags & AV_PIX_FMT_FLAG_ALPHA)) {
+        for (int n = 0; n < res.num_planes; n++) {
+            for (int i = 0; i < res.planes[n].num_components; i++) {
+                if (res.planes[n].components[i] == 2)
+                    res.planes[n].components[i] = 4;
+            }
+        }
+    }
+
+    res.component_pad = comp0->depth - res.component_size * 8;
+    if (comp0->shift) {
+        // We support padding only on 1 side.
+        if (comp0->shift + comp0->depth != res.component_size * 8)
+            return false;
+        res.component_pad = -res.component_pad;
+    }
+
+    res.chroma_w = 1 << pixdesc->log2_chroma_w;
+    res.chroma_h = 1 << pixdesc->log2_chroma_h;
+
+    if (strncmp(pixdesc->name, "bayer_", 6) == 0)
+        return false; // it's satan himself
+
+    if (!validate_regular_imgfmt(&res))
+        return false;
+
+    *dst = res;
+    return true;
+}
+
+
 // Find a format that has the given flags set with the following configuration.
 int mp_imgfmt_find(int xs, int ys, int planes, int component_bits, int flags)
 {
@@ -352,6 +503,9 @@ int main(int argc, char **argv)
         FLAG(MP_IMGFLAG_BE, "be")
         FLAG(MP_IMGFLAG_PAL, "pal")
         FLAG(MP_IMGFLAG_HWACCEL, "hw")
+        int fcsp = mp_imgfmt_get_forced_csp(mpfmt);
+        if (fcsp)
+            printf(" fcsp=%d", fcsp);
         printf("\n");
         printf("  planes=%d, chroma=%d:%d align=%d:%d bits=%d cbits=%d\n",
                d.num_planes, d.chroma_xs, d.chroma_ys, d.align_x, d.align_y,
@@ -381,6 +535,23 @@ int main(int argc, char **argv)
             talloc_free(mpi);
             av_frame_free(&fr);
         }
+        struct mp_regular_imgfmt reg;
+        if (mp_get_regular_imgfmt(&reg, mpfmt)) {
+            printf("  Regular: %d planes, %d bytes per comp., %d bit-pad "
+                   "%dx%d chroma\n",
+                   reg.num_planes, reg.component_size, reg.component_pad,
+                   reg.chroma_w, reg.chroma_h);
+            for (int n = 0; n < reg.num_planes; n++) {
+                struct mp_regular_imgfmt_plane *plane = &reg.planes[n];
+                printf("     %d: {", n);
+                for (int i = 0; i < plane->num_components; i++) {
+                    if (i > 0)
+                        printf(", ");
+                    printf("%d", plane->components[i]);
+                }
+                printf("}\n");
+            }
+        }
     }
 }
 
diff --git a/video/img_format.h b/video/img_format.h
index 11d3fc92b8..3411bd0b2c 100644
--- a/video/img_format.h
+++ b/video/img_format.h
@@ -22,6 +22,7 @@
 
 #include "osdep/endian.h"
 #include "misc/bstr.h"
+#include "video/csputils.h"
 
 #if BYTE_ORDER == BIG_ENDIAN
 #define MP_SELECT_LE_BE(LE, BE) BE
@@ -101,6 +102,38 @@ struct mp_imgfmt_desc {
 
 struct mp_imgfmt_desc mp_imgfmt_get_desc(int imgfmt);
 
+// MP_CSP_AUTO for YUV, MP_CSP_RGB or MP_CSP_XYZ otherwise.
+// (Because IMGFMT/AV_PIX_FMT conflate format and csp for RGB and XYZ.)
+enum mp_csp mp_imgfmt_get_forced_csp(int imgfmt);
+
+#define MP_NUM_COMPONENTS 4
+
+struct mp_regular_imgfmt_plane {
+    uint8_t num_components;
+    // 1 is luminance/red/gray, 2 is green/Cb, 3 is blue/Cr, 4 is alpha.
+    // 0 is used for padding (undefined contents).
+    uint8_t components[MP_NUM_COMPONENTS];
+};
+
+// This describes pixel formats that are byte aligned, have byte aligned
+// components, native endian, etc.
+struct mp_regular_imgfmt {
+    // Size of each component in bytes.
+    uint8_t component_size;
+
+    // If >0, LSB padding, if <0, MSB padding. The padding bits are always 0.
+    // This applies: bit_depth = component_size * 8 - abs(component_pad)
+    int8_t component_pad;
+
+    uint8_t num_planes;
+    struct mp_regular_imgfmt_plane planes[MP_MAX_PLANES];
+
+    // Chroma pixel size (1x1 is 4:4:4)
+    uint8_t chroma_w, chroma_h;
+};
+
+bool mp_get_regular_imgfmt(struct mp_regular_imgfmt *dst, int imgfmt);
+
 enum mp_imgfmt {
     IMGFMT_NONE = 0,
 
diff --git a/video/out/opengl/formats.c b/video/out/opengl/formats.c
index 5c9a2ab331..bf8f0f9eb5 100644
--- a/video/out/opengl/formats.c
+++ b/video/out/opengl/formats.c
@@ -106,36 +106,6 @@ static const int special_formats[][2] = {
     {0}
 };
 
-struct packed_fmt_entry {
-    int fmt;
-    int8_t component_size;
-    int8_t components[4]; // source component - 0 means unmapped
-};
-
-// Regular packed formats, which can be mapped to GL formats by finding a
-// texture format with same component count/size, and swizzling the result.
-static const struct packed_fmt_entry mp_packed_formats[] = {
-    //                  w   R  G  B  A
-    {IMGFMT_Y8,         1, {1, 0, 0, 0}},
-    {IMGFMT_Y16,        2, {1, 0, 0, 0}},
-    {IMGFMT_YA8,        1, {1, 0, 0, 2}},
-    {IMGFMT_YA16,       2, {1, 0, 0, 2}},
-    {IMGFMT_ARGB,       1, {2, 3, 4, 1}},
-    {IMGFMT_0RGB,       1, {2, 3, 4, 0}},
-    {IMGFMT_BGRA,       1, {3, 2, 1, 4}},
-    {IMGFMT_BGR0,       1, {3, 2, 1, 0}},
-    {IMGFMT_ABGR,       1, {4, 3, 2, 1}},
-    {IMGFMT_0BGR,       1, {4, 3, 2, 0}},
-    {IMGFMT_RGBA,       1, {1, 2, 3, 4}},
-    {IMGFMT_RGB0,       1, {1, 2, 3, 0}},
-    {IMGFMT_BGR24,      1, {3, 2, 1, 0}},
-    {IMGFMT_RGB24,      1, {1, 2, 3, 0}},
-    {IMGFMT_RGB48,      2, {1, 2, 3, 0}},
-    {IMGFMT_RGBA64,     2, {1, 2, 3, 4}},
-    {IMGFMT_BGRA64,     2, {3, 2, 1, 4}},
-    {0},
-};
-
 // Return an or-ed combination of all F_ flags that apply.
 int gl_format_feature_flags(GL *gl)
 {
@@ -317,25 +287,25 @@ bool gl_format_is_regular(const struct gl_format *fmt)
     return bpp == gl_bytes_per_pixel(fmt->format, fmt->type);
 }
 
-// dest = src.<w> (always using 4 components)
-static void packed_fmt_swizzle(char w[5], const struct packed_fmt_entry *fmt)
-{
-    for (int c = 0; c < 4; c++)
-        w[c] = "rgba"[MPMAX(fmt->components[c] - 1, 0)];
-    w[4] = '\0';
-}
-
 // Like gl_find_unorm_format(), but takes bits (not bytes), and if no fixed
 // point format is available, return an unsigned integer format.
-static const struct gl_format *find_plane_format(GL *gl, int bits, int n_channels)
+static const struct gl_format *find_plane_format(GL *gl, int bytes, int n_channels)
 {
-    int bytes = (bits + 7) / 8;
     const struct gl_format *f = gl_find_unorm_format(gl, bytes, n_channels);
     if (f)
         return f;
     return gl_find_uint_format(gl, bytes, n_channels);
 }
 
+static int find_comp(uint8_t *components, int num_components, int component)
+{
+    for (int n = 0; n < num_components; n++) {
+        if (components[n] == component)
+            return n;
+    }
+    return -1;
+}
+
 // Put a mapping of imgfmt to OpenGL textures into *out. Basically it selects
 // the correct texture formats needed to represent an imgfmt in OpenGL, with
 // textures using the same memory organization as on the CPU.
@@ -356,52 +326,29 @@ bool gl_get_imgfmt_desc(GL *gl, int imgfmt, struct gl_imgfmt_desc *out)
         return false;
 
     const struct gl_format *planes[4] = {0};
-    char swizzle_tmp[5] = {0};
+    char swizzle_tmp[MP_NUM_COMPONENTS + 1] = {0};
     char *swizzle = "rgba";
-
-    // YUV/planar formats
-    if (desc.flags & (MP_IMGFLAG_YUV_P | MP_IMGFLAG_RGB_P)) {
-        int bits = desc.component_bits;
-        if ((desc.flags & MP_IMGFLAG_NE) && bits >= 8 && bits <= 16) {
-            planes[0] = find_plane_format(gl, bits, 1);
-            for (int n = 1; n < desc.num_planes; n++)
-                planes[n] = planes[0];
-            // RGB/planar
-            if (desc.flags & MP_IMGFLAG_RGB_P)
-                swizzle = "brga";
-            goto supported;
+    struct mp_regular_imgfmt regfmt = {0};
+
+    if (mp_get_regular_imgfmt(&regfmt, imgfmt)) {
+        uint8_t components[MP_NUM_COMPONENTS + 1] = {0};
+        int num_components = 0;
+        for (int n = 0; n < regfmt.num_planes; n++) {
+            struct mp_regular_imgfmt_plane *plane = &regfmt.planes[n];
+            planes[n] = find_plane_format(gl, regfmt.component_size,
+                                          plane->num_components);
+            for (int i = 0; i < plane->num_components; i++)
+                components[num_components++] = plane->components[i];
         }
-    }
-
-    // YUV/half-packed
-    if (desc.flags & MP_IMGFLAG_YUV_NV) {
-        int bits = desc.component_bits;
-        if ((desc.flags & MP_IMGFLAG_NE) && bits >= 8 && bits <= 16) {
-            planes[0] = find_plane_format(gl, bits, 1);
-            planes[1] = find_plane_format(gl, bits, 2);
-            if (desc.flags & MP_IMGFLAG_YUV_NV_SWAP)
-                swizzle = "rbga";
-            goto supported;
+        swizzle = swizzle_tmp;
+        for (int c = 0; c < 4; c++) {
+            int loc = find_comp(components, num_components, c + 1);
+            swizzle[c] = "rgba"[loc >= 0 && loc < 4 ? loc : 0];
         }
-    }
-
-    // XYZ (same organization as RGB packed, but requires conversion matrix)
-    if (imgfmt == IMGFMT_XYZ12) {
-        planes[0] = gl_find_unorm_format(gl, 2, 3);
+        swizzle[4] = '\0';
         goto supported;
     }
 
-    // Packed RGB(A) formats
-    for (const struct packed_fmt_entry *e = mp_packed_formats; e->fmt; e++) {
-        if (e->fmt == imgfmt) {
-            int n_comp = desc.bytes[0] / e->component_size;
-            planes[0] = gl_find_unorm_format(gl, e->component_size, n_comp);
-            swizzle = swizzle_tmp;
-            packed_fmt_swizzle(swizzle, e);
-            goto supported;
-        }
-    }
-
     // Special formats for which OpenGL happens to have direct support.
     planes[0] = gl_find_special_format(gl, imgfmt);
     if (planes[0]) {