vaapi: support drm devices when running in vaapi-copy mode

When the vaapi decoder is used in copy mode, it creates a dummy
display to render to. In theory, this should support hardware
decoding on on a separate GPU that is not actually connected to
any output (like an iGPU which supports more formats than the
external GPU to which the monitor is connected).

However, before this change, only X11 displays were supported as
dummy displays. This caused some graphics drivers (namely
intel-driver) to core dump when they were not actually used as X11
module.
This change introduces support for drm libav displays, which
allows vaapi-copy to run on such cards which are not actually
rendering the X11 output.

1 files changed, 53 insertions, 0 deletions

diff --git a/video/decode/vaapi.c b/video/decode/vaapi.c
index aa8291da79..c095868816 100644
--- a/video/decode/vaapi.c
+++ b/video/decode/vaapi.c
@@ -99,7 +99,60 @@ static const struct va_native_display disp_x11 = {
 };
 #endif
 
+#if HAVE_VAAPI_DRM
+#include <unistd.h>
+#include <fcntl.h>
+#include <va/va_drm.h>
+
+struct va_native_display_drm {
+    int drm_fd;
+};
+
+static void drm_destroy(struct priv *p)
+{
+    struct va_native_display_drm *native_display = p->native_display;
+    if (native_display) {
+        if (native_display->drm_fd >= 0)
+            close(native_display->drm_fd);
+        talloc_free(native_display);
+        p->native_display = NULL;
+    }
+}
+
+static void drm_create(struct priv *p)
+{
+    static const char *drm_device_paths[] = {
+        "/dev/dri/renderD128",
+        "/dev/dri/card0",
+        NULL
+    };
+
+    for (int i = 0; drm_device_paths[i]; i++) {
+        int drm_fd = open(drm_device_paths[i], O_RDWR);
+        if (drm_fd < 0)
+            continue;
+
+        struct va_native_display_drm *native_display = talloc_ptrtype(NULL, native_display);
+        native_display->drm_fd = drm_fd;
+        p->native_display = native_display;
+        p->display = vaGetDisplayDRM(drm_fd);
+        if (p->display)
+            return;
+
+        drm_destroy(p);
+    }
+}
+
+static const struct va_native_display disp_drm = {
+    .create = drm_create,
+    .destroy = drm_destroy,
+};
+#endif
+
 static const struct va_native_display *const native_displays[] = {
+#if HAVE_VAAPI_DRM
+    &disp_drm,
+#endif
 #if HAVE_VAAPI_X11
     &disp_x11,
 #endif