hwdec_vaapi_pl: support simple multi-plane image formats

This is somewhat academic for now, as we explicitly ask for separate
layers and the scenarios where multi-plane images are required also use
complex formats that cannot be decomposed after the fact, but
nevertheless it is possible for us to consume simple multi-plane
images where there is one layer with n planes instead of n layers with
one plane each.

In these cases, we just treat the planes the same as we would if they
were each in a separate layer and everything works out.

It ought to be possible to make this work for OpenGL but I couldn't
wrap my head around how to provide the right DRM fourcc when
pretending a plane is a layer by itself. So I've left that
unimplemented.

3 files changed, 31 insertions, 11 deletions

diff --git a/video/out/hwdec/hwdec_vaapi.c b/video/out/hwdec/hwdec_vaapi.c
index b96f644576..ea0fa9e746 100644
--- a/video/out/hwdec/hwdec_vaapi.c
+++ b/video/out/hwdec/hwdec_vaapi.c
@@ -249,12 +249,21 @@ static int mapper_map(struct ra_hwdec_mapper *mapper)
     CHECK_VA_STATUS(mapper, "vaSyncSurface()");
     p->surface_acquired = true;
 
-    if (p->num_planes != p->desc.num_layers) {
+    // We can handle composed formats if the total number of planes is still
+    // equal the number of planes we expect. Complex formats with auxilliary
+    // planes cannot be supported.
+
+    int num_returned_planes = 0;
+    for (int i = 0; i < p->desc.num_layers; i++) {
+        num_returned_planes += p->desc.layers[i].num_planes;
+    }
+
+    if (p->num_planes != num_returned_planes) {
         mp_msg(mapper->log, p_owner->probing_formats ? MSGL_DEBUG : MSGL_ERR,
                "Mapped surface with format '%s' has unexpected number of planes. "
-               "(%d instead of %d)\n",
+               "(%d layers and %d planes, but expected %d planes)\n",
                mp_imgfmt_to_name(mapper->src->params.hw_subfmt),
-               p->desc.num_layers, p->num_planes);
+               p->desc.num_layers, num_returned_planes, p->num_planes);
         goto err;
     }
 
diff --git a/video/out/hwdec/hwdec_vaapi_gl.c b/video/out/hwdec/hwdec_vaapi_gl.c
index 9140e47d8b..b8628ef8b7 100644
--- a/video/out/hwdec/hwdec_vaapi_gl.c
+++ b/video/out/hwdec/hwdec_vaapi_gl.c
@@ -166,6 +166,12 @@ static bool vaapi_gl_map(struct ra_hwdec_mapper *mapper, bool probing)
     GL *gl = ra_gl_get(mapper->ra);
 
     for (int n = 0; n < p_mapper->num_planes; n++) {
+        if (p_mapper->desc.layers[n].num_planes > 1) {
+            // Should never happen because we request separate layers
+            MP_ERR(mapper, "Multi-plane VA surfaces are not supported\n");
+            return false;
+        }
+
         int attribs[48] = {EGL_NONE};
         int num_attribs = 0;
 
diff --git a/video/out/hwdec/hwdec_vaapi_pl.c b/video/out/hwdec/hwdec_vaapi_pl.c
index 790150e223..b8b79f8070 100644
--- a/video/out/hwdec/hwdec_vaapi_pl.c
+++ b/video/out/hwdec/hwdec_vaapi_pl.c
@@ -32,19 +32,18 @@ static bool vaapi_pl_map(struct ra_hwdec_mapper *mapper, bool probing)
     if (!ra_get_imgfmt_desc(mapper->ra, mapper->dst_params.imgfmt, &desc))
         return false;
 
+    // The calling code validates that the total number of exported planes
+    // equals the number we expected in p->num_planes.
+    int layer = 0;
+    int layer_plane = 0;
     for (int n = 0; n < p->num_planes; n++) {
-        if (p->desc.layers[n].num_planes > 1) {
-            // Should never happen because we request separate layers
-            MP_ERR(mapper, "Multi-plane VA surfaces are not supported\n");
-            return false;
-        }
 
         const struct ra_format *format = desc.planes[n];
-        int id = p->desc.layers[n].object_index[0];
+        int id = p->desc.layers[layer].object_index[layer_plane];
         int fd = p->desc.objects[id].fd;
         uint32_t size = p->desc.objects[id].size;
-        uint32_t offset = p->desc.layers[n].offset[0];
-        uint32_t pitch = p->desc.layers[n].pitch[0];
+        uint32_t offset = p->desc.layers[layer].offset[layer_plane];
+        uint32_t pitch = p->desc.layers[layer].pitch[layer_plane];
 
         // AMD drivers do not return the size in the surface description, so we
         // need to query it manually.
@@ -98,6 +97,12 @@ static bool vaapi_pl_map(struct ra_hwdec_mapper *mapper, bool probing)
 
         MP_TRACE(mapper, "Object %d with fd %d imported as %p\n",
                 id, fd, ratex);
+
+        layer_plane++;
+        if (layer_plane == p->desc.layers[layer].num_planes) {
+            layer_plane = 0;
+            layer++;
+        }
     }
     return true;
 }