context_drm_egl: use gbm_surface_create_with_modifiers

The GBM supporting nvidia driver doesn't support creating surfaces
without modifiers and using modifiers is more and more recommended as
the right way to do this.

Enumerating modifiers is painfully verbose, but necessary if we are to
allow the driver to pick the best possible one.

2 files changed, 94 insertions, 10 deletions

diff --git a/video/out/opengl/context_drm_egl.c b/video/out/opengl/context_drm_egl.c
index 99cc6116fb..cb98c14616 100644
--- a/video/out/opengl/context_drm_egl.c
+++ b/video/out/opengl/context_drm_egl.c
@@ -26,6 +26,7 @@
 #include <gbm.h>
 #include <EGL/egl.h>
 #include <EGL/eglext.h>
+#include <drm_fourcc.h>
 
 #include "libmpv/render_gl.h"
 #include "video/out/drm_common.h"
@@ -91,6 +92,8 @@ struct priv {
     unsigned int num_vsync_fences;
 
     uint32_t gbm_format;
+    uint64_t *gbm_modifiers;
+    unsigned int num_gbm_modifiers;
 
     bool active;
     bool waiting_for_flip;
@@ -239,12 +242,22 @@ static bool init_gbm(struct ra_ctx *ctx)
 
     MP_VERBOSE(ctx->vo, "Initializing GBM surface (%d x %d)\n",
         p->draw_surface_size.width, p->draw_surface_size.height);
-    p->gbm.surface = gbm_surface_create(
-        p->gbm.device,
-        p->draw_surface_size.width,
-        p->draw_surface_size.height,
-        p->gbm_format,
-        GBM_BO_USE_SCANOUT | GBM_BO_USE_RENDERING);
+    if (p->num_gbm_modifiers == 0) {
+        p->gbm.surface = gbm_surface_create(
+            p->gbm.device,
+            p->draw_surface_size.width,
+            p->draw_surface_size.height,
+            p->gbm_format,
+            GBM_BO_USE_SCANOUT | GBM_BO_USE_RENDERING);
+    } else {
+        p->gbm.surface = gbm_surface_create_with_modifiers(
+            p->gbm.device,
+            p->draw_surface_size.width,
+            p->draw_surface_size.height,
+            p->gbm_format,
+            p->gbm_modifiers,
+            p->num_gbm_modifiers);
+    }
     if (!p->gbm.surface) {
         MP_ERR(ctx->vo, "Failed to create GBM surface.\n");
         return false;
@@ -273,16 +286,39 @@ static void update_framebuffer_from_bo(struct ra_ctx *ctx, struct gbm_bo *bo)
     fb->fd     = p->kms->fd;
     fb->width  = gbm_bo_get_width(bo);
     fb->height = gbm_bo_get_height(bo);
-    uint32_t stride = gbm_bo_get_stride(bo);
-    uint32_t handle = gbm_bo_get_handle(bo).u32;
+    uint64_t modifier = gbm_bo_get_modifier(bo);
 
-    int ret = drmModeAddFB2(fb->fd, fb->width, fb->height,
+    int ret;
+    if (p->num_gbm_modifiers == 0 || modifier == DRM_FORMAT_MOD_INVALID) {
+        uint32_t stride = gbm_bo_get_stride(bo);
+        uint32_t handle = gbm_bo_get_handle(bo).u32;
+        ret = drmModeAddFB2(fb->fd, fb->width, fb->height,
                             p->gbm_format,
                             (uint32_t[4]){handle, 0, 0, 0},
                             (uint32_t[4]){stride, 0, 0, 0},
                             (uint32_t[4]){0, 0, 0, 0},
                             &fb->id, 0);
+    } else {
+        MP_VERBOSE(ctx, "GBM surface using modifier 0x%"PRIX64"\n", modifier);
+
+        uint32_t handles[4] = {0};
+        uint32_t strides[4] = {0};
+        uint32_t offsets[4] = {0};
+        uint64_t modifiers[4] = {0};
+
+        const int num_planes = gbm_bo_get_plane_count(bo);
+        for (int i = 0; i < num_planes; ++i) {
+            handles[i] = gbm_bo_get_handle_for_plane(bo, i).u32;
+            strides[i] = gbm_bo_get_stride_for_plane(bo, i);
+            offsets[i] = gbm_bo_get_offset(bo, i);
+            modifiers[i] = modifier;
+        }
 
+        ret = drmModeAddFB2WithModifiers(fb->fd, fb->width, fb->height,
+                                         p->gbm_format,
+                                         handles, strides, offsets, modifiers,
+                                         &fb->id, DRM_MODE_FB_MODIFIERS);
+    }
     if (ret) {
         MP_ERR(ctx->vo, "Failed to create framebuffer: %s\n", mp_strerror(errno));
     }
@@ -711,6 +747,50 @@ static bool probe_gbm_format(struct ra_ctx *ctx, uint32_t argb_format, uint32_t
     return result;
 }
 
+static bool probe_gbm_modifiers(struct ra_ctx *ctx)
+{
+    struct priv *p = ctx->priv;
+
+    if (!p->kms->atomic_context) {
+        MP_VERBOSE(ctx->vo, "Not using DRM Atomic: Not using modifiers.\n");
+        return false;
+    }
+
+    drmModePropertyBlobPtr blob =
+        drm_object_get_property_blob(p->kms->atomic_context->draw_plane, "IN_FORMATS");
+    if (!blob) {
+        MP_VERBOSE(ctx->vo, "Failed to find IN_FORMATS property\n");
+        return false;
+    }
+
+    struct drm_format_modifier_blob *data = blob->data;
+    uint32_t *fmts = (uint32_t *)((char *)data + data->formats_offset);
+    struct drm_format_modifier *mods =
+        (struct drm_format_modifier *)((char *)data + data->modifiers_offset);
+
+    for (unsigned int j = 0; j < data->count_modifiers; ++j) {
+        struct drm_format_modifier *mod = &mods[j];
+        for (uint64_t k = 0; k < 64; ++k) {
+            if (mod->formats & (1ull << k)) {
+                uint32_t fmt = fmts[k + mod->offset];
+                if (fmt == p->gbm_format) {
+                    MP_TARRAY_APPEND(p, p->gbm_modifiers,
+                                        p->num_gbm_modifiers, mod->modifier);
+                    MP_VERBOSE(ctx->vo, "Supported modifier: 0x%"PRIX64"\n",
+                                (uint64_t)mod->modifier);
+                    break;
+                }
+            }
+        }
+    }
+    drmModeFreePropertyBlob(blob);
+
+    if (p->num_gbm_modifiers == 0) {
+        MP_VERBOSE(ctx->vo, "No supported DRM modifiers found.\n");
+    }
+    return true;
+}
+
 static void drm_egl_get_vsync(struct ra_ctx *ctx, struct vo_vsync_info *info)
 {
     struct priv *p = ctx->priv;
@@ -774,6 +854,9 @@ static bool drm_egl_init(struct ra_ctx *ctx)
         return false;
     }
 
+    // It is not fatal if this fails. We'll just try without modifiers.
+    probe_gbm_modifiers(ctx);
+
     if (!init_gbm(ctx)) {
         MP_ERR(ctx->vo, "Failed to setup GBM.\n");
         return false;
@@ -800,6 +883,7 @@ static bool drm_egl_init(struct ra_ctx *ctx)
         MP_ERR(ctx, "Failed to lock GBM surface.\n");
         return false;
     }
+
     enqueue_bo(ctx, new_bo);
     update_framebuffer_from_bo(ctx, new_bo);
     if (!p->fb || !p->fb->id) {
diff --git a/wscript b/wscript
index dec82d3465..a0227e648e 100644
--- a/wscript
+++ b/wscript
@@ -496,7 +496,7 @@ video_output_features = [
         'name': '--gbm',
         'desc': 'GBM',
         'deps': 'gbm.h',
-        'func': check_pkg_config('gbm'),
+        'func': check_pkg_config('gbm', '>= 17.1.0'),
     } , {
         'name': '--wayland-scanner',
         'desc': 'wayland-scanner',