vo_dmabuf_wayland: reject hw formats not supported by compositor

10 files changed, 78 insertions, 33 deletions

diff --git a/video/out/hwdec/dmabuf_interop.h b/video/out/hwdec/dmabuf_interop.h
index 90c96399d7..9c52ad5f75 100644
--- a/video/out/hwdec/dmabuf_interop.h
+++ b/video/out/hwdec/dmabuf_interop.h
@@ -24,6 +24,7 @@
 
 struct dmabuf_interop {
     bool use_modifiers;
+    bool composed_layers;
 
     bool (*interop_init)(struct ra_hwdec_mapper *mapper,
                          const struct ra_imgfmt_desc *desc);
diff --git a/video/out/hwdec/dmabuf_interop_wl.c b/video/out/hwdec/dmabuf_interop_wl.c
index 74b421d583..eede6bb0f2 100644
--- a/video/out/hwdec/dmabuf_interop_wl.c
+++ b/video/out/hwdec/dmabuf_interop_wl.c
@@ -21,9 +21,6 @@
 static bool mapper_init(struct ra_hwdec_mapper *mapper,
                         const struct ra_imgfmt_desc *desc)
 {
-    struct dmabuf_interop_priv *p = mapper->priv;
-
-    p->num_planes = 1;
     return true;
 }
 
@@ -35,6 +32,32 @@ static bool map(struct ra_hwdec_mapper *mapper,
                 struct dmabuf_interop *dmabuf_interop,
                 bool probing)
 {
+    // 1. only validate format when composed layers is enabled (i.e. vaapi)
+    // 2. for drmprime, just return true for now, as this use case
+    // has not been tested.
+    if (!dmabuf_interop->composed_layers)
+        return true;
+
+    int layer_no = 0;
+    struct dmabuf_interop_priv *mapper_p = mapper->priv;
+    uint32_t drm_format = mapper_p->desc.layers[layer_no].format;
+
+    if (mapper_p->desc.nb_layers != 1) {
+        MP_VERBOSE(mapper, "Mapped surface has separate layers - expected composed layers.\n");
+        return false;
+    } else if (!ra_compatible_format(mapper->ra, drm_format,
+        mapper_p->desc.objects[0].format_modifier)) {
+        MP_VERBOSE(mapper, "Mapped surface with format %s; drm format '%s(%016lx)' "
+                   "is not supported by compositor.\n",
+                   mp_imgfmt_to_name(mapper->src->params.hw_subfmt),
+                   mp_tag_str(drm_format),
+                   mapper_p->desc.objects[0].format_modifier);
+        return false;
+    }
+
+    MP_VERBOSE(mapper, "Supported Wayland display format: '%s(%016lx)'\n",
+               mp_tag_str(drm_format), mapper_p->desc.objects[0].format_modifier);
+
     return true;
 }
 
@@ -48,6 +71,9 @@ bool dmabuf_interop_wl_init(const struct ra_hwdec *hw,
     if (!ra_is_wldmabuf(hw->ra))
         return false;
 
+    if (strstr(hw->driver->name, "vaapi") != NULL)
+        dmabuf_interop->composed_layers = true;
+
     dmabuf_interop->interop_init = mapper_init;
     dmabuf_interop->interop_uninit = mapper_uninit;
     dmabuf_interop->interop_map = map;
diff --git a/video/out/hwdec/hwdec_drmprime.c b/video/out/hwdec/hwdec_drmprime.c
index 6ecacbb1ce..d74f9593ba 100644
--- a/video/out/hwdec/hwdec_drmprime.c
+++ b/video/out/hwdec/hwdec_drmprime.c
@@ -244,7 +244,7 @@ static int mapper_map(struct ra_hwdec_mapper *mapper)
         num_returned_planes += p->desc.layers[i].nb_planes;
     }
 
-    if (p->num_planes != num_returned_planes) {
+    if (p->num_planes != 0 && p->num_planes != num_returned_planes) {
         MP_ERR(mapper,
                "Mapped surface with format '%s' has unexpected number of planes. "
                "(%d layers and %d planes, but expected %d planes)\n",
diff --git a/video/out/hwdec/hwdec_vaapi.c b/video/out/hwdec/hwdec_vaapi.c
index df5ad4a832..e5ea3b1289 100644
--- a/video/out/hwdec/hwdec_vaapi.c
+++ b/video/out/hwdec/hwdec_vaapi.c
@@ -259,10 +259,12 @@ static int mapper_map(struct ra_hwdec_mapper *mapper)
     VADisplay *display = p_owner->display;
     VADRMPRIMESurfaceDescriptor desc = {0};
 
+    uint32_t flags = p_owner->dmabuf_interop.composed_layers ?
+        VA_EXPORT_SURFACE_COMPOSED_LAYERS : VA_EXPORT_SURFACE_SEPARATE_LAYERS;
     status = vaExportSurfaceHandle(display, va_surface_id(mapper->src),
                                    VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
                                    VA_EXPORT_SURFACE_READ_ONLY |
-                                   VA_EXPORT_SURFACE_SEPARATE_LAYERS,
+                                   flags,
                                    &desc);
     if (!CHECK_VA_STATUS_LEVEL(mapper, "vaExportSurfaceHandle()",
                                p_owner->probing_formats ? MSGL_DEBUG : MSGL_ERR))
@@ -301,7 +303,7 @@ static int mapper_map(struct ra_hwdec_mapper *mapper)
     // We can handle composed formats if the total number of planes is still
     // equal the number of planes we expect. Complex formats with auxilliary
     // planes cannot be supported.
-    if (p->num_planes != num_returned_planes) {
+    if (p->num_planes != 0 && p->num_planes != num_returned_planes) {
         mp_msg(mapper->log, p_owner->probing_formats ? MSGL_DEBUG : MSGL_ERR,
                "Mapped surface with format '%s' has unexpected number of planes. "
                "(%d layers and %d planes, but expected %d planes)\n",
diff --git a/video/out/vo_dmabuf_wayland.c b/video/out/vo_dmabuf_wayland.c
index ca7f5941eb..da9f92f259 100644
--- a/video/out/vo_dmabuf_wayland.c
+++ b/video/out/vo_dmabuf_wayland.c
@@ -39,6 +39,7 @@
 #include "present_sync.h"
 #include "wayland_common.h"
 #include "wlbuf_pool.h"
+#include "wldmabuf/ra_wldmabuf.h"
 
 // Generated from wayland-protocols
 #include "generated/wayland/linux-dmabuf-unstable-v1.h"
@@ -101,10 +102,13 @@ static bool vaapi_dmabuf_importer(struct mp_image *src, struct wlbuf_pool_entry*
     }
     bool success = false;
     uint32_t drm_format = desc.layers[layer_no].drm_format;
-    if (!vo_wayland_supported_format(entry->vo, drm_format, desc.objects[0].drm_format_modifier)) {
-        MP_VERBOSE(entry->vo, "%s(%016lx) is not supported.\n",
-                   mp_tag_str(drm_format), desc.objects[0].drm_format_modifier);
-        goto done;
+    if (!ra_compatible_format(p->ctx->ra, drm_format, desc.objects[0].drm_format_modifier)) {
+            MP_VERBOSE(entry->vo, "Surface with format %s; drm format '%s(%016lx)' is "
+                                  "not supported by compositor.\n",
+                                   mp_imgfmt_to_name(src->params.hw_subfmt),
+                                   mp_tag_str(drm_format),
+                                   desc.objects[0].drm_format_modifier);
+            goto done;
     }
     entry->drm_format = drm_format;
     for (int plane_no = 0; plane_no < desc.layers[layer_no].num_planes; ++plane_no) {
diff --git a/video/out/wayland_common.c b/video/out/wayland_common.c
index b1631cbb20..9c415b38de 100644
--- a/video/out/wayland_common.c
+++ b/video/out/wayland_common.c
@@ -2234,24 +2234,6 @@ void vo_wayland_set_opaque_region(struct vo_wayland_state *wl, int alpha)
     }
 }
 
-bool vo_wayland_supported_format(struct vo *vo, uint32_t drm_format, uint64_t modifier)
-{
-    struct vo_wayland_state *wl = vo->wl;
-
-    const struct {
-        uint32_t format;
-        uint32_t padding;
-        uint64_t modifier;
-    } *formats = wl->format_map;
-
-    for (int i = 0; i < wl->format_size / 16; ++i) {
-        if (drm_format == formats[i].format && modifier == formats[i].modifier)
-            return true;
-    }
-
-    return false;
-}
-
 void vo_wayland_uninit(struct vo *vo)
 {
     struct vo_wayland_state *wl = vo->wl;
diff --git a/video/out/wayland_common.h b/video/out/wayland_common.h
index fa65e80ac2..5c36f6f05a 100644
--- a/video/out/wayland_common.h
+++ b/video/out/wayland_common.h
@@ -22,6 +22,12 @@
 #include "input/event.h"
 #include "vo.h"
 
+typedef struct {
+    uint32_t format;
+    uint32_t padding;
+    uint64_t modifier;
+} wayland_format;
+
 struct wayland_opts {
     int configure_bounds;
     int content_type;
@@ -96,7 +102,7 @@ struct vo_wayland_state {
     struct zwp_linux_dmabuf_v1 *dmabuf;
     /* TODO: unvoid this if required wayland protocols is bumped to 1.24+ */
     void *dmabuf_feedback;
-    void *format_map;
+    wayland_format *format_map;
     uint32_t format_size;
 
     /* presentation-time */
diff --git a/video/out/wldmabuf/context_wldmabuf.c b/video/out/wldmabuf/context_wldmabuf.c
index 472374c427..c494575010 100644
--- a/video/out/wldmabuf/context_wldmabuf.c
+++ b/video/out/wldmabuf/context_wldmabuf.c
@@ -29,7 +29,7 @@ static bool init(struct ra_ctx *ctx)
 {
     if (!vo_wayland_init(ctx->vo))
         return false;
-    ctx->ra = ra_create_wayland(ctx->log, ctx->vo->wl->display);
+    ctx->ra = ra_create_wayland(ctx->log, ctx->vo);
 
     return true;
 }
diff --git a/video/out/wldmabuf/ra_wldmabuf.c b/video/out/wldmabuf/ra_wldmabuf.c
index 8b59bd4a07..eda2cc9fe8 100644
--- a/video/out/wldmabuf/ra_wldmabuf.c
+++ b/video/out/wldmabuf/ra_wldmabuf.c
@@ -19,21 +19,44 @@
 #include "video/out/gpu/ra.h"
 #include "ra_wldmabuf.h"
 
+struct priv {
+    struct vo *vo;
+};
+
 static void destroy(struct ra *ra)
 {
+    talloc_free(ra->priv);
+}
+
+bool ra_compatible_format(struct ra* ra, uint32_t drm_format, uint64_t modifier)
+{
+    struct priv* p = ra->priv;
+    struct vo_wayland_state *wl = p->vo->wl;
+    const wayland_format *formats = wl->format_map;
+
+    for (int i = 0; i < wl->format_size / sizeof(wayland_format); i++)
+    {
+        if (drm_format == formats[i].format && modifier == formats[i].modifier)
+            return true;
+    }
+
+    return false;
 }
 
 static struct ra_fns ra_fns_wldmabuf = {
     .destroy                = destroy,
 };
 
-struct ra *ra_create_wayland(struct mp_log *log, struct wl_display *display)
+struct ra *ra_create_wayland(struct mp_log *log, struct vo* vo)
 {
     struct ra *ra =  talloc_zero(NULL, struct ra);
 
     ra->fns = &ra_fns_wldmabuf;
     ra->log = log;
-    ra_add_native_resource(ra, "wl", display);
+    ra_add_native_resource(ra, "wl", vo->wl->display);
+    ra->priv = talloc_zero(NULL, struct priv);
+    struct priv *p = ra->priv;
+    p->vo = vo;
 
     return ra;
 }
diff --git a/video/out/wldmabuf/ra_wldmabuf.h b/video/out/wldmabuf/ra_wldmabuf.h
index 231b4c3ce8..8e20173546 100644
--- a/video/out/wldmabuf/ra_wldmabuf.h
+++ b/video/out/wldmabuf/ra_wldmabuf.h
@@ -18,5 +18,6 @@
 #pragma once
 #include "video/out/wayland_common.h"
 
-struct ra *ra_create_wayland(struct mp_log *log, struct wl_display *display);
+struct ra *ra_create_wayland(struct mp_log *log, struct vo *vo);
+bool ra_compatible_format(struct ra* ra, uint32_t drm_format, uint64_t modifier);
 bool ra_is_wldmabuf(struct ra *ra);