vo_opengl: support all kinds of GBRP formats

Adds support for AV_PIX_FMT_GBRP9, AV_PIX_FMT_GBRP10, AV_PIX_FMT_GBRP12,
AV_PIX_FMT_GBRP14, AV_PIX_FMT_GBRP16, AV_PIX_FMT_GBRAP, and
AV_PIX_FMT_GBRAP16.

(Not that it matters, because nobody uses these anyway.)

3 files changed, 15 insertions, 13 deletions

diff --git a/video/img_format.c b/video/img_format.c
index 7ed4c722e8..c2bbf6bb01 100644
--- a/video/img_format.c
+++ b/video/img_format.c
@@ -231,15 +231,21 @@ struct mp_imgfmt_desc mp_imgfmt_get_desc(int mpfmt)
     if (pd->flags & (AV_PIX_FMT_FLAG_PAL | AV_PIX_FMT_FLAG_PSEUDOPAL))
         desc.flags |= MP_IMGFLAG_PAL;
 
-    if ((desc.flags & MP_IMGFLAG_YUV) && (desc.flags & MP_IMGFLAG_BYTE_ALIGNED))
+    if ((desc.flags & (MP_IMGFLAG_YUV | MP_IMGFLAG_RGB))
+        && (desc.flags & MP_IMGFLAG_BYTE_ALIGNED))
     {
         bool same_depth = true;
         for (int p = 0; p < desc.num_planes; p++) {
             same_depth &= planedepth[p] == planedepth[0] &&
                           desc.bpp[p] == desc.bpp[0];
         }
-        if (same_depth && pd->nb_components == desc.num_planes)
-            desc.flags |= MP_IMGFLAG_YUV_P;
+        if (same_depth && pd->nb_components == desc.num_planes) {
+            if (desc.flags & MP_IMGFLAG_YUV) {
+                desc.flags |= MP_IMGFLAG_YUV_P;
+            } else {
+                desc.flags |= MP_IMGFLAG_RGB_P;
+            }
+        }
     }
 
     for (int p = 0; p < desc.num_planes; p++) {
diff --git a/video/img_format.h b/video/img_format.h
index fefe0b82e4..cd2ebf5360 100644
--- a/video/img_format.h
+++ b/video/img_format.h
@@ -67,6 +67,8 @@
 #define MP_IMGFLAG_HWACCEL 0x10000
 // Set if the chroma resolution is lower than luma resolution. Unset for non-YUV.
 #define MP_IMGFLAG_SUBSAMPLED 0x20000
+// Like MP_IMGFLAG_YUV_P, but RGB. The planes are organized as in IMGFMT_GBRP.
+#define MP_IMGFLAG_RGB_P 0x40000
 
 // Exactly one of these bits is set in mp_imgfmt_desc.flags
 #define MP_IMGFLAG_COLOR_CLASS_MASK \
diff --git a/video/out/opengl/video.c b/video/out/opengl/video.c
index fb339992cc..baa660de51 100644
--- a/video/out/opengl/video.c
+++ b/video/out/opengl/video.c
@@ -2278,13 +2278,16 @@ static bool init_format(int fmt, struct gl_video *init)
     init->has_alpha = false;
 
     // YUV/planar formats
-    if (desc.flags & MP_IMGFLAG_YUV_P) {
+    if (desc.flags & (MP_IMGFLAG_YUV_P | MP_IMGFLAG_RGB_P)) {
         int bits = desc.component_bits;
         if ((desc.flags & MP_IMGFLAG_NE) && bits >= 8 && bits <= 16) {
             init->has_alpha = desc.num_planes > 3;
             plane_format[0] = find_tex_format(gl, (bits + 7) / 8, 1);
             for (int p = 1; p < desc.num_planes; p++)
                 plane_format[p] = plane_format[0];
+            // RGB/planar
+            if (desc.flags & MP_IMGFLAG_RGB_P)
+                snprintf(init->color_swizzle, sizeof(init->color_swizzle), "brga");
             goto supported;
         }
     }
@@ -2300,15 +2303,6 @@ static bool init_format(int fmt, struct gl_video *init)
         goto supported;
     }
 
-    // RGB/planar
-    if (fmt == IMGFMT_GBRP) {
-        snprintf(init->color_swizzle, sizeof(init->color_swizzle), "brga");
-        plane_format[0] = find_tex_format(gl, 1, 1);
-        for (int p = 1; p < desc.num_planes; p++)
-            plane_format[p] = plane_format[0];
-        goto supported;
-    }
-
     // XYZ (same organization as RGB packed, but requires conversion matrix)
     if (fmt == IMGFMT_XYZ12) {
         plane_format[0] = find_tex_format(gl, 2, 3);