From 49fd6caa432a89b715670ccbabff9c2d14d975b0 Mon Sep 17 00:00:00 2001 From: Aaron Boxer Date: Tue, 7 Feb 2023 21:03:35 -0500 Subject: vo_dmabuf_wayland: reject hw formats not supported by compositor --- video/out/hwdec/dmabuf_interop.h | 1 + video/out/hwdec/dmabuf_interop_wl.c | 32 +++++++++++++++++++++++++++++--- video/out/hwdec/hwdec_drmprime.c | 2 +- video/out/hwdec/hwdec_vaapi.c | 6 ++++-- video/out/vo_dmabuf_wayland.c | 12 ++++++++---- video/out/wayland_common.c | 18 ------------------ video/out/wayland_common.h | 8 +++++++- video/out/wldmabuf/context_wldmabuf.c | 2 +- video/out/wldmabuf/ra_wldmabuf.c | 27 +++++++++++++++++++++++++-- video/out/wldmabuf/ra_wldmabuf.h | 3 ++- 10 files changed, 78 insertions(+), 33 deletions(-) diff --git a/video/out/hwdec/dmabuf_interop.h b/video/out/hwdec/dmabuf_interop.h index 90c96399d7..9c52ad5f75 100644 --- a/video/out/hwdec/dmabuf_interop.h +++ b/video/out/hwdec/dmabuf_interop.h @@ -24,6 +24,7 @@ struct dmabuf_interop { bool use_modifiers; + bool composed_layers; bool (*interop_init)(struct ra_hwdec_mapper *mapper, const struct ra_imgfmt_desc *desc); diff --git a/video/out/hwdec/dmabuf_interop_wl.c b/video/out/hwdec/dmabuf_interop_wl.c index 74b421d583..eede6bb0f2 100644 --- a/video/out/hwdec/dmabuf_interop_wl.c +++ b/video/out/hwdec/dmabuf_interop_wl.c @@ -21,9 +21,6 @@ static bool mapper_init(struct ra_hwdec_mapper *mapper, const struct ra_imgfmt_desc *desc) { - struct dmabuf_interop_priv *p = mapper->priv; - - p->num_planes = 1; return true; } @@ -35,6 +32,32 @@ static bool map(struct ra_hwdec_mapper *mapper, struct dmabuf_interop *dmabuf_interop, bool probing) { + // 1. only validate format when composed layers is enabled (i.e. vaapi) + // 2. for drmprime, just return true for now, as this use case + // has not been tested. + if (!dmabuf_interop->composed_layers) + return true; + + int layer_no = 0; + struct dmabuf_interop_priv *mapper_p = mapper->priv; + uint32_t drm_format = mapper_p->desc.layers[layer_no].format; + + if (mapper_p->desc.nb_layers != 1) { + MP_VERBOSE(mapper, "Mapped surface has separate layers - expected composed layers.\n"); + return false; + } else if (!ra_compatible_format(mapper->ra, drm_format, + mapper_p->desc.objects[0].format_modifier)) { + MP_VERBOSE(mapper, "Mapped surface with format %s; drm format '%s(%016lx)' " + "is not supported by compositor.\n", + mp_imgfmt_to_name(mapper->src->params.hw_subfmt), + mp_tag_str(drm_format), + mapper_p->desc.objects[0].format_modifier); + return false; + } + + MP_VERBOSE(mapper, "Supported Wayland display format: '%s(%016lx)'\n", + mp_tag_str(drm_format), mapper_p->desc.objects[0].format_modifier); + return true; } @@ -48,6 +71,9 @@ bool dmabuf_interop_wl_init(const struct ra_hwdec *hw, if (!ra_is_wldmabuf(hw->ra)) return false; + if (strstr(hw->driver->name, "vaapi") != NULL) + dmabuf_interop->composed_layers = true; + dmabuf_interop->interop_init = mapper_init; dmabuf_interop->interop_uninit = mapper_uninit; dmabuf_interop->interop_map = map; diff --git a/video/out/hwdec/hwdec_drmprime.c b/video/out/hwdec/hwdec_drmprime.c index 6ecacbb1ce..d74f9593ba 100644 --- a/video/out/hwdec/hwdec_drmprime.c +++ b/video/out/hwdec/hwdec_drmprime.c @@ -244,7 +244,7 @@ static int mapper_map(struct ra_hwdec_mapper *mapper) num_returned_planes += p->desc.layers[i].nb_planes; } - if (p->num_planes != num_returned_planes) { + if (p->num_planes != 0 && p->num_planes != num_returned_planes) { MP_ERR(mapper, "Mapped surface with format '%s' has unexpected number of planes. " "(%d layers and %d planes, but expected %d planes)\n", diff --git a/video/out/hwdec/hwdec_vaapi.c b/video/out/hwdec/hwdec_vaapi.c index df5ad4a832..e5ea3b1289 100644 --- a/video/out/hwdec/hwdec_vaapi.c +++ b/video/out/hwdec/hwdec_vaapi.c @@ -259,10 +259,12 @@ static int mapper_map(struct ra_hwdec_mapper *mapper) VADisplay *display = p_owner->display; VADRMPRIMESurfaceDescriptor desc = {0}; + uint32_t flags = p_owner->dmabuf_interop.composed_layers ? + VA_EXPORT_SURFACE_COMPOSED_LAYERS : VA_EXPORT_SURFACE_SEPARATE_LAYERS; status = vaExportSurfaceHandle(display, va_surface_id(mapper->src), VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, VA_EXPORT_SURFACE_READ_ONLY | - VA_EXPORT_SURFACE_SEPARATE_LAYERS, + flags, &desc); if (!CHECK_VA_STATUS_LEVEL(mapper, "vaExportSurfaceHandle()", p_owner->probing_formats ? MSGL_DEBUG : MSGL_ERR)) @@ -301,7 +303,7 @@ static int mapper_map(struct ra_hwdec_mapper *mapper) // We can handle composed formats if the total number of planes is still // equal the number of planes we expect. Complex formats with auxilliary // planes cannot be supported. - if (p->num_planes != num_returned_planes) { + if (p->num_planes != 0 && p->num_planes != num_returned_planes) { mp_msg(mapper->log, p_owner->probing_formats ? MSGL_DEBUG : MSGL_ERR, "Mapped surface with format '%s' has unexpected number of planes. " "(%d layers and %d planes, but expected %d planes)\n", diff --git a/video/out/vo_dmabuf_wayland.c b/video/out/vo_dmabuf_wayland.c index ca7f5941eb..da9f92f259 100644 --- a/video/out/vo_dmabuf_wayland.c +++ b/video/out/vo_dmabuf_wayland.c @@ -39,6 +39,7 @@ #include "present_sync.h" #include "wayland_common.h" #include "wlbuf_pool.h" +#include "wldmabuf/ra_wldmabuf.h" // Generated from wayland-protocols #include "generated/wayland/linux-dmabuf-unstable-v1.h" @@ -101,10 +102,13 @@ static bool vaapi_dmabuf_importer(struct mp_image *src, struct wlbuf_pool_entry* } bool success = false; uint32_t drm_format = desc.layers[layer_no].drm_format; - if (!vo_wayland_supported_format(entry->vo, drm_format, desc.objects[0].drm_format_modifier)) { - MP_VERBOSE(entry->vo, "%s(%016lx) is not supported.\n", - mp_tag_str(drm_format), desc.objects[0].drm_format_modifier); - goto done; + if (!ra_compatible_format(p->ctx->ra, drm_format, desc.objects[0].drm_format_modifier)) { + MP_VERBOSE(entry->vo, "Surface with format %s; drm format '%s(%016lx)' is " + "not supported by compositor.\n", + mp_imgfmt_to_name(src->params.hw_subfmt), + mp_tag_str(drm_format), + desc.objects[0].drm_format_modifier); + goto done; } entry->drm_format = drm_format; for (int plane_no = 0; plane_no < desc.layers[layer_no].num_planes; ++plane_no) { diff --git a/video/out/wayland_common.c b/video/out/wayland_common.c index b1631cbb20..9c415b38de 100644 --- a/video/out/wayland_common.c +++ b/video/out/wayland_common.c @@ -2234,24 +2234,6 @@ void vo_wayland_set_opaque_region(struct vo_wayland_state *wl, int alpha) } } -bool vo_wayland_supported_format(struct vo *vo, uint32_t drm_format, uint64_t modifier) -{ - struct vo_wayland_state *wl = vo->wl; - - const struct { - uint32_t format; - uint32_t padding; - uint64_t modifier; - } *formats = wl->format_map; - - for (int i = 0; i < wl->format_size / 16; ++i) { - if (drm_format == formats[i].format && modifier == formats[i].modifier) - return true; - } - - return false; -} - void vo_wayland_uninit(struct vo *vo) { struct vo_wayland_state *wl = vo->wl; diff --git a/video/out/wayland_common.h b/video/out/wayland_common.h index fa65e80ac2..5c36f6f05a 100644 --- a/video/out/wayland_common.h +++ b/video/out/wayland_common.h @@ -22,6 +22,12 @@ #include "input/event.h" #include "vo.h" +typedef struct { + uint32_t format; + uint32_t padding; + uint64_t modifier; +} wayland_format; + struct wayland_opts { int configure_bounds; int content_type; @@ -96,7 +102,7 @@ struct vo_wayland_state { struct zwp_linux_dmabuf_v1 *dmabuf; /* TODO: unvoid this if required wayland protocols is bumped to 1.24+ */ void *dmabuf_feedback; - void *format_map; + wayland_format *format_map; uint32_t format_size; /* presentation-time */ diff --git a/video/out/wldmabuf/context_wldmabuf.c b/video/out/wldmabuf/context_wldmabuf.c index 472374c427..c494575010 100644 --- a/video/out/wldmabuf/context_wldmabuf.c +++ b/video/out/wldmabuf/context_wldmabuf.c @@ -29,7 +29,7 @@ static bool init(struct ra_ctx *ctx) { if (!vo_wayland_init(ctx->vo)) return false; - ctx->ra = ra_create_wayland(ctx->log, ctx->vo->wl->display); + ctx->ra = ra_create_wayland(ctx->log, ctx->vo); return true; } diff --git a/video/out/wldmabuf/ra_wldmabuf.c b/video/out/wldmabuf/ra_wldmabuf.c index 8b59bd4a07..eda2cc9fe8 100644 --- a/video/out/wldmabuf/ra_wldmabuf.c +++ b/video/out/wldmabuf/ra_wldmabuf.c @@ -19,21 +19,44 @@ #include "video/out/gpu/ra.h" #include "ra_wldmabuf.h" +struct priv { + struct vo *vo; +}; + static void destroy(struct ra *ra) { + talloc_free(ra->priv); +} + +bool ra_compatible_format(struct ra* ra, uint32_t drm_format, uint64_t modifier) +{ + struct priv* p = ra->priv; + struct vo_wayland_state *wl = p->vo->wl; + const wayland_format *formats = wl->format_map; + + for (int i = 0; i < wl->format_size / sizeof(wayland_format); i++) + { + if (drm_format == formats[i].format && modifier == formats[i].modifier) + return true; + } + + return false; } static struct ra_fns ra_fns_wldmabuf = { .destroy = destroy, }; -struct ra *ra_create_wayland(struct mp_log *log, struct wl_display *display) +struct ra *ra_create_wayland(struct mp_log *log, struct vo* vo) { struct ra *ra = talloc_zero(NULL, struct ra); ra->fns = &ra_fns_wldmabuf; ra->log = log; - ra_add_native_resource(ra, "wl", display); + ra_add_native_resource(ra, "wl", vo->wl->display); + ra->priv = talloc_zero(NULL, struct priv); + struct priv *p = ra->priv; + p->vo = vo; return ra; } diff --git a/video/out/wldmabuf/ra_wldmabuf.h b/video/out/wldmabuf/ra_wldmabuf.h index 231b4c3ce8..8e20173546 100644 --- a/video/out/wldmabuf/ra_wldmabuf.h +++ b/video/out/wldmabuf/ra_wldmabuf.h @@ -18,5 +18,6 @@ #pragma once #include "video/out/wayland_common.h" -struct ra *ra_create_wayland(struct mp_log *log, struct wl_display *display); +struct ra *ra_create_wayland(struct mp_log *log, struct vo *vo); +bool ra_compatible_format(struct ra* ra, uint32_t drm_format, uint64_t modifier); bool ra_is_wldmabuf(struct ra *ra); -- cgit v1.2.3