vo_wayland: pixel perfect buffers

1.  Separate buffer and temporary file handling from the vo to make maintenance
    and reading code easier

2.  Skip resizing as much as possible if back buffer is still busy.

3.  Detach and mark osd buffers for deletion if we want to redraw them and they
    are still busy. This could be a possible case for the video buffers as
    well. Maybe better than double buffering.

All the above steps made it possible to have resizing without any artifacts
even for subtitles. Also fixes dozen of bugs only I knew, like broken subtitles
for rgb565 buffers. I can now sleep at night again.

4 files changed, 370 insertions, 0 deletions

diff --git a/video/out/wayland/buffer.c b/video/out/wayland/buffer.c
new file mode 100644
index 0000000000..26fd3d1cc8
--- /dev/null
+++ b/video/out/wayland/buffer.c
@@ -0,0 +1,137 @@
+/*
+ * This file is part of mpv video player.
+ * Copyright © 2014 Alexander Preisinger <alexander.preisinger@gmail.com>
+ *
+ * mpv is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "buffer.h"
+#include "memfile.h"
+
+#include <unistd.h>
+#include <sys/mman.h>
+
+int8_t format_get_bytes(const format_t *fmt)
+{
+    return mp_imgfmt_get_desc(fmt->mp_format).bytes[0];
+}
+
+shm_buffer_t* shm_buffer_create(uint32_t width,
+                                uint32_t height,
+                                format_t fmt,
+                                struct wl_shm *shm,
+                                const struct wl_buffer_listener *listener)
+{
+    int8_t bytes = format_get_bytes(&fmt);
+    uint32_t stride = SHM_BUFFER_STRIDE(width, bytes);
+    uint32_t size = stride * height;
+
+    shm_buffer_t *buffer = calloc(1, sizeof(shm_buffer_t));
+    int fd = memfile_create(size);
+
+    if (fd < 0)
+        return NULL;
+
+    buffer->data = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+
+    if (buffer->data == MAP_FAILED) {
+        close(fd);
+        return NULL;
+    }
+
+    buffer->shm_pool = wl_shm_create_pool(shm, fd, size);
+    buffer->buffer = wl_shm_pool_create_buffer(buffer->shm_pool,
+                                               0, width, height, stride,
+                                               fmt.wl_format);
+
+    wl_buffer_add_listener(buffer->buffer, listener, buffer);
+
+    buffer->fd = fd;
+    buffer->height = height;
+    buffer->stride = stride;
+    buffer->format = fmt;
+    buffer->bytes = bytes;
+    buffer->pool_size = size;
+    buffer->pending_height = 0;
+    buffer->pending_width = 0;
+
+    return buffer;
+}
+
+int shm_buffer_resize(shm_buffer_t *buffer, uint32_t width, uint32_t height)
+{
+    uint32_t new_stride = SHM_BUFFER_STRIDE(width, buffer->bytes);
+    uint32_t new_size = new_stride * height;
+
+    if (!!(buffer->flags & SHM_BUFFER_BUSY)) {
+        buffer->flags |= SHM_BUFFER_RESIZE_LATER;
+        buffer->pending_width = width;
+        buffer->pending_height = height;
+        return SHM_BUFFER_BUSY;
+    }
+
+    buffer->flags &= ~SHM_BUFFER_RESIZE_LATER;
+
+    if (new_size > buffer->pool_size) {
+        munmap(buffer->data, buffer->pool_size);
+        ftruncate(buffer->fd, new_size);
+
+        buffer->data = mmap(NULL, new_size, PROT_READ | PROT_WRITE,
+                            MAP_SHARED, buffer->fd, 0);
+
+        // TODO: the buffer should be destroyed when -1 is return
+        if (buffer->data == MAP_FAILED)
+            return -1;
+
+        wl_shm_pool_resize(buffer->shm_pool, new_size);
+        buffer->pool_size = new_size;
+    }
+
+    const void *listener = wl_proxy_get_listener((struct wl_proxy*)buffer->buffer);
+
+    wl_buffer_destroy(buffer->buffer);
+    buffer->buffer = wl_shm_pool_create_buffer(buffer->shm_pool,
+                                               0, width, height, new_stride,
+                                               buffer->format.wl_format);
+
+    wl_buffer_add_listener(buffer->buffer, listener, buffer);
+
+    buffer->height = height;
+    buffer->stride = new_stride;
+
+    return 0;
+}
+
+int shm_buffer_pending_resize(shm_buffer_t *buffer)
+{
+    if (SHM_BUFFER_PENDING_RESIZE(buffer)) {
+        SHM_BUFFER_CLEAR_PNDNG_RSZ(buffer);
+        return shm_buffer_resize(buffer, buffer->pending_width, buffer->pending_height);
+    }
+    else {
+        return 0;
+    }
+}
+
+void shm_buffer_destroy(shm_buffer_t *buffer)
+{
+    if (!buffer)
+        return;
+
+    wl_buffer_destroy(buffer->buffer);
+    wl_shm_pool_destroy(buffer->shm_pool);
+    munmap(buffer->data, buffer->pool_size);
+    close(buffer->fd);
+    free(buffer);
+}
diff --git a/video/out/wayland/buffer.h b/video/out/wayland/buffer.h
new file mode 100644
index 0000000000..04e94b9d7f
--- /dev/null
+++ b/video/out/wayland/buffer.h
@@ -0,0 +1,102 @@
+/*
+ * This file is part of mpv video player.
+ * Copyright © 2014 Alexander Preisinger <alexander.preisinger@gmail.com>
+ *
+ * mpv is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef MPLAYER_WAYLAND_BUFFER_H
+#define MPLAYER_WAYLAND_BUFFER_H
+
+#include <libavutil/common.h>
+#include "video/sws_utils.h"
+#include "video/img_format.h"
+#include "video/out/wayland_common.h"
+
+#define SHM_BUFFER_STRIDE(width, bytes) \
+    FFALIGN((width) * (bytes), SWS_MIN_BYTE_ALIGN)
+
+typedef struct format {
+    enum wl_shm_format wl_format;
+    enum mp_imgfmt     mp_format;
+} format_t;
+
+int8_t format_get_bytes(const format_t *fmt);
+
+typedef enum shm_buffer_flags {
+    SHM_BUFFER_BUSY         = 1 << 0, // in use by the compositor
+    SHM_BUFFER_DIRTY        = 1 << 1, // buffer contains new content
+    SHM_BUFFER_ONESHOT      = 1 << 2, // free after release
+    SHM_BUFFER_RESIZE_LATER = 1 << 3, // free after release
+} shm_buffer_flags_t;
+
+#define SHM_BUFFER_IS_BUSY(b)        (!!((b)->flags & SHM_BUFFER_BUSY))
+#define SHM_BUFFER_IS_DIRTY(b)       (!!((b)->flags & SHM_BUFFER_DIRTY))
+#define SHM_BUFFER_IS_ONESHOT(b)     (!!((b)->flags & SHM_BUFFER_ONESHOT))
+#define SHM_BUFFER_PENDING_RESIZE(b) (!!((b)->flags & SHM_BUFFER_RESIZE_LATER))
+
+#define SHM_BUFFER_SET_BUSY(b)      (b)->flags |= SHM_BUFFER_BUSY
+#define SHM_BUFFER_SET_DIRTY(b)     (b)->flags |= SHM_BUFFER_DIRTY
+#define SHM_BUFFER_SET_ONESHOT(b)   (b)->flags |= SHM_BUFFER_ONESHOT
+#define SHM_BUFFER_SET_PNDNG_RSZ(b) (b)->flags |= SHM_BUFFER_RESIZE_LATER
+
+#define SHM_BUFFER_CLEAR_BUSY(b)      (b)->flags &= ~SHM_BUFFER_BUSY
+#define SHM_BUFFER_CLEAR_DIRTY(b)     (b)->flags &= ~SHM_BUFFER_DIRTY
+#define SHM_BUFFER_CLEAR_ONESHOT(b)   (b)->flags &= ~SHM_BUFFER_ONESHOT
+#define SHM_BUFFER_CLEAR_PNDNG_RSZ(b) (b)->flags &= ~SHM_BUFFER_RESIZE_LATER
+
+typedef struct buffer {
+    struct wl_buffer *buffer;
+
+    int flags;
+
+    uint32_t height;
+    uint32_t stride;
+    uint32_t bytes; // bytes per pixel
+    // width = stride / bytes per pixel
+    // size = stride * height
+
+    struct wl_shm_pool *shm_pool; // for growing buffers;
+
+    int fd;
+    void *data;
+    uint32_t pool_size; // size of pool and data XXX
+                        // pool_size can be far bigger than the buffer size
+
+    format_t format;
+
+    uint32_t pending_height;
+    uint32_t pending_width;
+} shm_buffer_t;
+
+shm_buffer_t* shm_buffer_create(uint32_t width,
+                                uint32_t height,
+                                format_t fmt,
+                                struct wl_shm *shm,
+                                const struct wl_buffer_listener *listener);
+
+// shm pool is only able to grow and won't shrink
+// returns 0 on success or buffer flags indicating the buffer status which
+// prevent it from resizing 
+int shm_buffer_resize(shm_buffer_t *buffer, uint32_t width, uint32_t height);
+
+// if shm_buffer_resize returns SHM_BUFFER_BUSY this function can be called
+// after the buffer is released to resize it afterwards
+// returns 0 if no pending resize flag was set and -1 on errors
+int shm_buffer_pending_resize(shm_buffer_t *buffer);
+
+// buffer is freed, don't use the buffer after calling this function on it
+void shm_buffer_destroy(shm_buffer_t *buffer);
+
+#endif /* MPLAYER_WAYLAND_BUFFER_H */
diff --git a/video/out/wayland/memfile.c b/video/out/wayland/memfile.c
new file mode 100644
index 0000000000..179abe8755
--- /dev/null
+++ b/video/out/wayland/memfile.c
@@ -0,0 +1,105 @@
+/*
+ * This file is part of mpv video player.
+ * Copyright © 2014 Alexander Preisinger <alexander.preisinger@gmail.com>
+ *
+ * mpv is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <fcntl.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <errno.h>
+#include <string.h>
+
+#include "video/out/wayland/memfile.h"
+
+/* copied from weston clients */
+static int set_cloexec_or_close(int fd)
+{
+    long flags;
+
+    if (fd == -1)
+        return -1;
+
+    if ((flags = fcntl(fd, F_GETFD)) == -1)
+        goto err;
+
+    if (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) == -1)
+        goto err;
+
+    return fd;
+
+err:
+    close(fd);
+    return -1;
+}
+
+static int create_tmpfile_cloexec(char *tmpname)
+{
+    int fd;
+
+#ifdef HAVE_MKOSTEMP
+    fd = mkostemp(tmpname, O_CLOEXEC);
+    if (fd >= 0)
+        unlink(tmpname);
+#else
+    fd = mkstemp(tmpname);
+    if (fd >= 0) {
+        fd = set_cloexec_or_close(fd);
+        unlink(tmpname);
+    }
+#endif
+
+    return fd;
+}
+
+static int os_create_anonymous_file(off_t size)
+{
+    static const char template[] = "/mpv-temp-XXXXXX";
+    const char *path;
+    char *name;
+    int fd;
+
+    path = getenv("XDG_RUNTIME_DIR");
+    if (!path) {
+        errno = ENOENT;
+        return -1;
+    }
+
+    name = malloc(strlen(path) + sizeof(template));
+    if (!name)
+        return -1;
+
+    strcpy(name, path);
+    strcat(name, template);
+
+    fd = create_tmpfile_cloexec(name);
+
+    free(name);
+
+    if (fd < 0)
+        return -1;
+
+    if (ftruncate(fd, size) < 0) {
+        close(fd);
+        return -1;
+    }
+
+    return fd;
+}
+
+int memfile_create(off_t size)
+{
+    return os_create_anonymous_file(size);
+}
diff --git a/video/out/wayland/memfile.h b/video/out/wayland/memfile.h
new file mode 100644
index 0000000000..9d59111eec
--- /dev/null
+++ b/video/out/wayland/memfile.h
@@ -0,0 +1,26 @@
+/*
+ * This file is part of mpv video player.
+ * Copyright © 2014 Alexander Preisinger <alexander.preisinger@gmail.com>
+ *
+ * mpv is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef MPLAYER_WAYLAND_MEMFILE_H
+#define MPLAYER_WAYLAND_MEMFILE_H
+
+// create file decsriptor to memory space without filesystem representation
+// truncates to size immediately
+int memfile_create(off_t size);
+
+#endif /* MPLAYER_WAYLAND_MEMFILE_H */