vo_opengl: don't make assumptions about plane order

The renderer code doesn't list a fixed set of supported formats, but
supports anything that is described by AVPixFmtDescriptor and follows a
number of constraints.

Plane order is not included in those constraints. This means the planes
could be in random order, rather than what the vo_opengl renderer
happens to assume. For example, it assumes that the 4th plane is alpha,
even though alpha could be on any plane. Likewise it assumes that plane
0 was always luma, and planes 2/3 chroma. (In earlier iterations of
format support, this was guaranteed by MP_IMGFLAG_YUV_P, but this is not
used anymore.)

Explicitly set the plane semantics (enum plane_type) by the component
descriptors and the used colorspace. The behavior should mostly not
change, but it's less likely to break when FFmpeg adds new pixel
formats.

1 files changed, 30 insertions, 9 deletions

diff --git a/video/out/opengl/video.c b/video/out/opengl/video.c
index bdd6c50050..1bd9351b9c 100644
--- a/video/out/opengl/video.c
+++ b/video/out/opengl/video.c
@@ -669,6 +669,20 @@ static int chroma_upsize(int size, int pixel)
     return (size + pixel - 1) / pixel * pixel;
 }
 
+// If a and b are on the same plane, return what plane type should be used.
+// If a or b are none, the other type always wins.
+// Usually: LUMA/RGB/XYZ > CHROMA > ALPHA
+static enum plane_type merge_plane_types(enum plane_type a, enum plane_type b)
+{
+    if (a == PLANE_NONE)
+        return b;
+    if (b == PLANE_LUMA || b == PLANE_RGB || b == PLANE_XYZ)
+        return b;
+    if (b != PLANE_NONE && a == PLANE_ALPHA)
+        return b;
+    return a;
+}
+
 // Places a video_image's image textures + associated metadata into tex[]. The
 // number of textures is equal to p->plane_count. Any necessary plane offsets
 // are stored in off. (e.g. chroma position)
@@ -710,15 +724,22 @@ static void pass_get_img_tex(struct gl_video *p, struct video_image *vimg,
     for (int n = 0; n < p->plane_count; n++) {
         struct texplane *t = &vimg->planes[n];
 
-        enum plane_type type;
-        if (n >= 3) {
-            type = PLANE_ALPHA;
-        } else if (p->image_params.color.space == MP_CSP_RGB) {
-            type = PLANE_RGB;
-        } else if (p->image_params.color.space == MP_CSP_XYZ) {
-            type = PLANE_XYZ;
-        } else {
-            type = n == 0 ? PLANE_LUMA : PLANE_CHROMA;
+        enum plane_type type = PLANE_NONE;
+        for (int i = 0; i < 4; i++) {
+            int c = p->gl_format.components[n][i];
+            enum plane_type ctype;
+            if (c == 0) {
+                ctype = PLANE_NONE;
+            } else if (c == 4) {
+                ctype = PLANE_ALPHA;
+            } else if (p->image_params.color.space == MP_CSP_RGB) {
+                ctype = PLANE_RGB;
+            } else if (p->image_params.color.space == MP_CSP_XYZ) {
+                ctype = PLANE_XYZ;
+            } else {
+                ctype = c == 1 ? PLANE_LUMA : PLANE_CHROMA;
+            }
+            type = merge_plane_types(type, ctype);
         }
 
         tex[n] = (struct img_tex){