vo_dmabuf_wayland: add osd support

This adds osd support via shm buffers using a similar approach that the
normal buffers do, but it differs in a few key areas. One thing to note
is that sway and weston actually handle this extremely differently which
required all the abstractions here. In particular, weston does not cope
well with destroying the wl_buffer from shm outside of the release
handler at all (i.e. it segfaults). The workaround here is to simply
attach a NULL to the osd surface and do a surface commit before we
destroy the buffers. This is reasonable enough and seems to work well
although it's pretty weird. Sway is more straightforward although it
actually releases the osd buffer when the window goes out of sight.
Also, I found that it doesn't always release every buffer before you
close it unlike weston seems to do which is part of the reason all this
bookkeeping is required. I don't know if there's any other compositor
out there that can possibly handle vo_dmabuf_wayland right now, but
suffering through these two is good enough for now I think.

3 files changed, 230 insertions, 10 deletions

diff --git a/video/out/vo_dmabuf_wayland.c b/video/out/vo_dmabuf_wayland.c
index 92a3aaa7eb..62fdad7499 100644
--- a/video/out/vo_dmabuf_wayland.c
+++ b/video/out/vo_dmabuf_wayland.c
@@ -15,6 +15,7 @@
  * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <sys/mman.h>
 #include <unistd.h>
 #include "config.h"
 
@@ -30,6 +31,7 @@
 #include "gpu/video.h"
 #include "mpv_talloc.h"
 #include "present_sync.h"
+#include "sub/draw_bmp.h"
 #include "video/mp_image.h"
 #include "vo.h"
 #include "wayland_common.h"
@@ -67,6 +69,14 @@ struct buffer {
     uintptr_t id;
 };
 
+struct osd_buffer {
+    struct vo *vo;
+    struct wl_buffer *buffer;
+    struct wl_list link;
+    struct mp_image image;
+    size_t size;
+};
+
 struct priv {
     struct mp_log *log;
     struct mp_rect src;
@@ -78,6 +88,17 @@ struct priv {
     struct wl_shm_pool *solid_buffer_pool;
     struct wl_buffer *solid_buffer;
     struct wl_list buffer_list;
+    struct wl_list osd_buffer_list;
+
+    struct wl_shm_pool *osd_shm_pool;
+    uint8_t *osd_shm_data;
+    int osd_shm_width;
+    int osd_shm_stride;
+    int osd_shm_height;
+
+    struct osd_buffer *osd_buffer;
+    struct mp_draw_sub_cache *osd_cache;
+    struct mp_osd_res screen_osd_res;
 
     bool destroy_buffers;
     enum hwdec_type hwdec_type;
@@ -96,6 +117,21 @@ static const struct wl_buffer_listener buffer_listener = {
     buffer_handle_release,
 };
 
+static void osd_buffer_handle_release(void *data, struct wl_buffer *wl_buffer)
+{
+    struct osd_buffer *osd_buf = data;
+    wl_list_remove(&osd_buf->link);
+    if (osd_buf->buffer) {
+        wl_buffer_destroy(osd_buf->buffer);
+        osd_buf->buffer = NULL;
+    }
+    talloc_free(osd_buf);
+}
+
+static const struct wl_buffer_listener osd_buffer_listener = {
+    osd_buffer_handle_release,
+};
+
 #if HAVE_VAAPI
 static void close_file_descriptors(VADRMPRIMESurfaceDescriptor desc)
 {
@@ -295,7 +331,105 @@ static void destroy_buffers(struct vo *vo)
     }
 }
 
-static void set_viewport_source(struct vo *vo, struct mp_rect src) {
+static void destroy_osd_buffers(struct vo *vo)
+{
+    // Remove any existing buffer before we destroy them.
+    wl_surface_attach(vo->wl->osd_surface, NULL, 0, 0);
+    wl_surface_commit(vo->wl->osd_surface);
+
+    struct priv *p = vo->priv;
+    struct osd_buffer *osd_buf, *tmp;
+    wl_list_for_each_safe(osd_buf, tmp, &p->osd_buffer_list, link) {
+        wl_list_remove(&osd_buf->link);
+        munmap(osd_buf->image.planes[0], osd_buf->size);
+        if (osd_buf->buffer) {
+            wl_buffer_destroy(osd_buf->buffer);
+            osd_buf->buffer = NULL;
+        }
+    }
+}
+
+static struct osd_buffer *osd_buffer_check(struct vo *vo)
+{
+    struct priv *p = vo->priv;
+    struct osd_buffer *osd_buf;
+    wl_list_for_each(osd_buf, &p->osd_buffer_list, link) {
+        return osd_buf;
+    }
+    return NULL;
+}
+
+static struct osd_buffer *osd_buffer_create(struct vo *vo)
+{
+    struct priv *p = vo->priv;
+    struct osd_buffer *osd_buf = talloc_zero(vo, struct osd_buffer);
+
+    osd_buf->vo = vo;
+    osd_buf->size = p->osd_shm_height * p->osd_shm_stride;
+    mp_image_set_size(&osd_buf->image, p->osd_shm_width, p->osd_shm_height);
+    osd_buf->image.planes[0] = p->osd_shm_data;
+    osd_buf->image.stride[0] = p->osd_shm_stride;
+    osd_buf->buffer = wl_shm_pool_create_buffer(p->osd_shm_pool, 0,
+                                                p->osd_shm_width, p->osd_shm_height,
+                                                p->osd_shm_stride, WL_SHM_FORMAT_ARGB8888);
+
+    if (!osd_buf->buffer) {
+        talloc_free(osd_buf);
+        return NULL;
+    }
+
+    wl_list_insert(&p->osd_buffer_list, &osd_buf->link);
+    wl_buffer_add_listener(osd_buf->buffer, &osd_buffer_listener, osd_buf);
+    return osd_buf;
+}
+
+static struct osd_buffer *osd_buffer_get(struct vo *vo)
+{
+    struct osd_buffer *osd_buf = osd_buffer_check(vo);
+    if (osd_buf) {
+        return osd_buf;
+    } else {
+        return osd_buffer_create(vo);
+    }
+}
+
+static void create_shm_pool(struct vo *vo)
+{
+    struct vo_wayland_state *wl = vo->wl;
+    struct priv *p = vo->priv;
+
+    int stride = MP_ALIGN_UP(vo->dwidth * 4, 16);
+    size_t size = vo->dheight * stride;
+    int fd = vo_wayland_allocate_memfd(vo, size);
+    if (fd < 0)
+        return;
+    uint8_t *data = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+    if (data == MAP_FAILED)
+        goto error1;
+    struct wl_shm_pool *pool = wl_shm_create_pool(wl->shm, fd, size);
+    if (!pool)
+        goto error2;
+    close(fd);
+
+    destroy_osd_buffers(vo);
+
+    if (p->osd_shm_pool)
+        wl_shm_pool_destroy(p->osd_shm_pool);
+    p->osd_shm_pool = pool;
+    p->osd_shm_width = vo->dwidth;
+    p->osd_shm_height = vo->dheight;
+    p->osd_shm_stride = stride;
+    p->osd_shm_data = data;
+    return;
+
+error2:
+    munmap(data, size);
+error1:
+    close(fd);
+}
+
+static void set_viewport_source(struct vo *vo, struct mp_rect src)
+{
     struct priv *p = vo->priv;
     struct vo_wayland_state *wl = vo->wl;
 
@@ -310,40 +444,94 @@ static void set_viewport_source(struct vo *vo, struct mp_rect src) {
 static void resize(struct vo *vo)
 {
     struct vo_wayland_state *wl = vo->wl;
+    struct priv *p = vo->priv;
+
     struct mp_rect src;
     struct mp_rect dst;
-    struct mp_osd_res osd;
     struct mp_vo_opts *vo_opts = wl->vo_opts;
+
     const int width = mp_rect_w(wl->geometry);
     const int height = mp_rect_h(wl->geometry);
 
     if (width == 0 || height == 0)
         return;
-    
+
     vo_wayland_set_opaque_region(wl, false);
     vo->dwidth = width;
     vo->dheight = height;
 
+    create_shm_pool(vo);
+
     // top level viewport is calculated with pan set to zero
     vo->opts->pan_x = 0;
     vo->opts->pan_y = 0;
-    vo_get_src_dst_rects(vo, &src, &dst, &osd);
+    vo_get_src_dst_rects(vo, &src, &dst, &p->screen_osd_res);
     wp_viewport_set_destination(wl->viewport, 2 * dst.x0 + mp_rect_w(dst), 2 * dst.y0 + mp_rect_h(dst));
 
     //now we restore pan for video viewport calculation
     vo->opts->pan_x = vo_opts->pan_x;
     vo->opts->pan_y = vo_opts->pan_y;
-    vo_get_src_dst_rects(vo, &src, &dst, &osd);
+    vo_get_src_dst_rects(vo, &src, &dst, &p->screen_osd_res);
     wp_viewport_set_destination(wl->video_viewport, mp_rect_w(dst), mp_rect_h(dst));
     wl_subsurface_set_position(wl->video_subsurface, dst.x0, dst.y0);
+    wp_viewport_set_destination(wl->osd_viewport, vo->dwidth, vo->dheight);
+    wl_subsurface_set_position(wl->osd_subsurface, 0 - dst.x0, 0 - dst.y0);
     set_viewport_source(vo, src);
 }
 
+static bool draw_osd(struct vo *vo, struct mp_image *cur, double pts)
+{
+    struct priv *p = vo->priv;
+    struct mp_osd_res *res = &p->screen_osd_res;
+    bool draw = false;
+
+    struct sub_bitmap_list *sbs = osd_render(vo->osd, *res, pts, 0, mp_draw_sub_formats);
+
+    if (!sbs)
+        return draw;
+
+    struct mp_rect act_rc[1], mod_rc[64];
+    int num_act_rc = 0, num_mod_rc = 0;
+
+    if (!p->osd_cache)
+        p->osd_cache = mp_draw_sub_alloc(p, vo->global);
+
+    struct mp_image *osd = mp_draw_sub_overlay(p->osd_cache, sbs, act_rc,
+                                               MP_ARRAY_SIZE(act_rc), &num_act_rc,
+                                               mod_rc, MP_ARRAY_SIZE(mod_rc), &num_mod_rc);
+
+    if (!osd || !num_mod_rc)
+        goto done;
+
+    for (int n = 0; n < num_mod_rc; n++) {
+        struct mp_rect rc = mod_rc[n];
+
+        int rw = mp_rect_w(rc);
+        int rh = mp_rect_h(rc);
+
+        void *src = mp_image_pixel_ptr(osd, 0, rc.x0, rc.y0);
+        void *dst = cur->planes[0] + rc.x0 * 4 + rc.y0 * cur->stride[0];
+
+        // Avoid overflowing if we have this special case.
+        if (n == num_mod_rc - 1)
+            --rh;
+
+        memcpy_pic(dst, src, rw * 4, rh, cur->stride[0], osd->stride[0]);
+    }
+
+    draw = true;
+done:
+    talloc_free(sbs);
+    return draw;
+}
+
 static void draw_frame(struct vo *vo, struct vo_frame *frame)
 {
     struct priv *p = vo->priv;
     struct vo_wayland_state *wl = vo->wl;
     struct buffer *buf;
+    struct osd_buffer *osd_buf;
+    double pts = frame->current->pts;
 
     if (!vo_wayland_check_visible(vo) || !frame->current)
         return;
@@ -353,11 +541,20 @@ static void draw_frame(struct vo *vo, struct vo_frame *frame)
 
     struct mp_image *src = mp_image_new_ref(frame->current);
     buf = buffer_get(vo, src);
+    osd_buf = osd_buffer_get(vo);
 
     if (buf && buf->image) {
         wl_surface_attach(wl->video_surface, buf->buffer, 0, 0);
-        wl_surface_damage_buffer(wl->video_surface, 0, 0, buf->image->params.w,
-                                 buf->image->params.h);
+        wl_surface_damage_buffer(wl->video_surface, 0, 0, buf->image->w,
+                                 buf->image->h);
+
+        if (osd_buf && osd_buf->buffer) {
+            if (draw_osd(vo, &osd_buf->image, pts)) {
+                wl_surface_attach(wl->osd_surface, osd_buf->buffer, 0, 0);
+                wl_surface_damage_buffer(wl->osd_surface, 0, 0, osd_buf->image.w,
+                                         osd_buf->image.h);
+            }
+        }
     }
 }
 
@@ -366,13 +563,14 @@ static void flip_page(struct vo *vo)
     struct vo_wayland_state *wl = vo->wl;
 
     wl_surface_commit(wl->video_surface);
+    wl_surface_commit(wl->osd_surface);
     wl_surface_commit(wl->surface);
 
     if (!wl->opts->disable_vsync)
         vo_wayland_wait_frame(wl);
 
     if (wl->use_present)
-       present_sync_swap(wl->present);
+        present_sync_swap(wl->present);
 }
 
 static void get_vsync(struct vo *vo, struct vo_vsync_info *info)
@@ -446,6 +644,9 @@ static void uninit(struct vo *vo)
     struct priv *p = vo->priv;
 
     destroy_buffers(vo);
+    destroy_osd_buffers(vo);
+    if (p->osd_shm_pool)
+        wl_shm_pool_destroy(p->osd_shm_pool);
     if (p->solid_buffer_pool)
         wl_shm_pool_destroy(p->solid_buffer_pool);
     if (p->solid_buffer)
@@ -468,6 +669,7 @@ static int preinit(struct vo *vo)
     p->global = vo->global;
     p->ctx = ra_ctx_create_by_name(vo, "wldmabuf");
     wl_list_init(&p->buffer_list);
+    wl_list_init(&p->osd_buffer_list);
     if (!p->ctx)
        goto err;
 
diff --git a/video/out/wayland_common.c b/video/out/wayland_common.c
index 9d1e8e8784..cecacf5215 100644
--- a/video/out/wayland_common.c
+++ b/video/out/wayland_common.c
@@ -1257,10 +1257,14 @@ static void registry_handle_add(void *data, struct wl_registry *reg, uint32_t id
         wl->compositor = wl_registry_bind(reg, id, &wl_compositor_interface, ver);
         wl->surface = wl_compositor_create_surface(wl->compositor);
         wl->video_surface = wl_compositor_create_surface(wl->compositor);
-        /* never accept input events on the video surface */
+        wl->osd_surface = wl_compositor_create_surface(wl->compositor);
+
+        /* never accept input events on anything besides the main surface */
         struct wl_region *region = wl_compositor_create_region(wl->compositor);
+        wl_surface_set_input_region(wl->osd_surface, region);
         wl_surface_set_input_region(wl->video_surface, region);
         wl_region_destroy(region);
+
         wl->cursor_surface = wl_compositor_create_surface(wl->compositor);
         wl_surface_add_listener(wl->surface, &surface_listener, wl);
     }
@@ -1470,10 +1474,11 @@ static int create_viewports(struct vo_wayland_state *wl)
 {
     if (wl->viewporter) {
         wl->viewport = wp_viewporter_get_viewport(wl->viewporter, wl->surface);
+        wl->osd_viewport = wp_viewporter_get_viewport(wl->viewporter, wl->osd_surface);
         wl->video_viewport = wp_viewporter_get_viewport(wl->viewporter, wl->video_surface);
     }
 
-    if (wl->viewporter && (!wl->viewport || !wl->video_viewport)) {
+    if (wl->viewporter && (!wl->viewport || !wl->osd_viewport || !wl->video_viewport)) {
         MP_ERR(wl, "failed to create viewport interfaces!\n");
         return 1;
     }
@@ -2206,6 +2211,7 @@ bool vo_wayland_init(struct vo *vo)
         goto err;
 
     if (wl->subcompositor) {
+        wl->osd_subsurface = wl_subcompositor_get_subsurface(wl->subcompositor, wl->osd_surface, wl->video_surface);
         wl->video_subsurface = wl_subcompositor_get_subsurface(wl->subcompositor, wl->video_surface, wl->surface);
         wl_subsurface_set_desync(wl->video_subsurface);
     }
@@ -2443,6 +2449,9 @@ void vo_wayland_uninit(struct vo *vo)
     if (wl->viewport)
         wp_viewport_destroy(wl->viewport);
 
+    if (wl->osd_viewport)
+        wp_viewport_destroy(wl->osd_viewport);
+
     if (wl->video_viewport)
         wp_viewport_destroy(wl->video_viewport);
 
@@ -2466,6 +2475,12 @@ void vo_wayland_uninit(struct vo *vo)
     if (wl->surface)
         wl_surface_destroy(wl->surface);
 
+    if (wl->osd_surface)
+        wl_surface_destroy(wl->osd_surface);
+
+    if (wl->osd_subsurface)
+        wl_subsurface_destroy(wl->osd_subsurface);
+
     if (wl->video_surface)
         wl_surface_destroy(wl->video_surface);
 
diff --git a/video/out/wayland_common.h b/video/out/wayland_common.h
index 1a26c7faaa..8dd9f804b8 100644
--- a/video/out/wayland_common.h
+++ b/video/out/wayland_common.h
@@ -49,6 +49,8 @@ struct vo_wayland_state {
     struct wl_registry      *registry;
     struct wl_shm           *shm;
     struct wl_surface       *surface;
+    struct wl_surface       *osd_surface;
+    struct wl_subsurface    *osd_subsurface;
     struct wl_surface       *video_surface;
     struct wl_surface       *callback_surface;
     struct wl_subsurface    *video_subsurface;
@@ -133,6 +135,7 @@ struct vo_wayland_state {
     /* viewporter */
     struct wp_viewporter *viewporter;
     struct wp_viewport   *viewport;
+    struct wp_viewport   *osd_viewport;
     struct wp_viewport   *video_viewport;
 
     /* Input */