vo_wlshm: use memfd_create() instead of shm_open()

This syscall avoids the need to guess an unused filename in /dev/shm and
allows seals to be placed on it.  We immediately return if no fd got
returned, as there isn’t anything we can do otherwise.

Seals especially allow the compositor to drop the SIGBUS protections,
since the kernel promises the fd won’t ever shrink.

This removes support for any platform but Linux from this vo.

4 files changed, 19 insertions, 25 deletions

diff --git a/video/out/vo.c b/video/out/vo.c
index 545ff8bf0c..64ac8aa3ed 100644
--- a/video/out/vo.c
+++ b/video/out/vo.c
@@ -77,7 +77,7 @@ const struct vo_driver *const video_out_drivers[] =
 #if HAVE_DIRECT3D
     &video_out_direct3d,
 #endif
-#if HAVE_WAYLAND
+#if HAVE_WAYLAND && HAVE_MEMFD_CREATE
     &video_out_wlshm,
 #endif
 #if HAVE_XV
diff --git a/video/out/vo_wlshm.c b/video/out/vo_wlshm.c
index 1577248407..213b05b150 100644
--- a/video/out/vo_wlshm.c
+++ b/video/out/vo_wlshm.c
@@ -73,30 +73,18 @@ static void buffer_destroy(void *p)
     munmap(buf->mpi.planes[0], buf->size);
 }
 
-/* modeled after randname and mkostemps from musl */
-static int alloc_shm(size_t size)
+static int allocate_memfd(size_t size)
 {
-    struct timespec ts;
-    unsigned long r;
-    int i, fd, retries = 100;
-    char template[] = "/mpv-XXXXXX";
-
-    do {
-        clock_gettime(CLOCK_REALTIME, &ts);
-        r = ts.tv_nsec * 65537 ^ ((uintptr_t)&ts / 16 + (uintptr_t)template);
-        for (i = 5; i < sizeof(template) - 1; i++, r >>= 5)
-            template[i] = 'A' + (r & 15) + (r & 16) * 2;
-
-        fd = shm_open(template, O_RDWR | O_CREAT | O_EXCL, 0600);
-        if (fd >= 0) {
-            shm_unlink(template);
-            if (posix_fallocate(fd, 0, size) == 0)
-                return fd;
-            close(fd);
-            break;
-        }
-    } while (--retries && errno == EEXIST);
+    int fd = memfd_create("mpv", MFD_CLOEXEC | MFD_ALLOW_SEALING);
+    if (fd < 0)
+        return -1;
 
+    fcntl(fd, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_SEAL);
+
+    if (posix_fallocate(fd, 0, size) == 0)
+        return fd;
+
+    close(fd);
     return -1;
 }
 
@@ -112,7 +100,7 @@ static struct buffer *buffer_create(struct vo *vo, int width, int height)
 
     stride = MP_ALIGN_UP(width * 4, 16);
     size = height * stride;
-    fd = alloc_shm(size);
+    fd = allocate_memfd(size);
     if (fd < 0)
         goto error0;
     data = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
diff --git a/wscript b/wscript
index 5baa274645..d4d1353112 100644
--- a/wscript
+++ b/wscript
@@ -323,6 +323,12 @@ iconv support use --disable-iconv.",
         'func': check_statement('sys/vfs.h',
                                 'struct statfs fs; fstatfs(0, &fs); fs.f_namelen')
     }, {
+        'name': 'memfd_create',
+        'desc': "Linux's memfd_create()",
+        'deps': 'os-linux',
+        'func': check_statement('sys/mman.h',
+                                'memfd_create("mpv", MFD_CLOEXEC | MFD_ALLOW_SEALING)')
+    }, {
         'name': '--libsmbclient',
         'desc': 'Samba support (makes mpv GPLv3)',
         'deps': 'libdl && gpl',
diff --git a/wscript_build.py b/wscript_build.py
index 1fed29ceea..d148ac7ae9 100644
--- a/wscript_build.py
+++ b/wscript_build.py
@@ -488,7 +488,7 @@ def build(ctx):
         ( "video/out/vo_tct.c" ),
         ( "video/out/vo_vaapi.c",                "vaapi-x11 && gpl" ),
         ( "video/out/vo_vdpau.c",                "vdpau" ),
-        ( "video/out/vo_wlshm.c",                "wayland" ),
+        ( "video/out/vo_wlshm.c",                "wayland && memfd_create" ),
         ( "video/out/vo_x11.c" ,                 "x11" ),
         ( "video/out/vo_xv.c",                   "xv" ),
         ( "video/out/vulkan/context.c",          "vulkan" ),