vdpau: support new vdpau libavcodec decode API

The new API works like the new vaapi API, using generic hwaccel support.

One minor detail is the error message that will be printed if using
non-4:2:0 surfaces (which as far as I can tell is completely broken in
the nVidia drivers and thus not supported by mpv). The HEVC warning
(which is completely broken in the nVidia drivers but should work with
Mesa) had to be added to the generic hwaccel code.

This also trashes display preemption recovery. Fuck that. It never
really worked. If someone complains, I might attempt to add it back
somehow.

This is the 4th iteration of the libavcodec vdpau API (after the
separate decoder API, the manual hwaccel API, and the automatic vdpau
hwaccel API). Fortunately, further iterations will be generic, and not
require much vdpau-specific changes (if any at all).

3 files changed, 73 insertions, 3 deletions

diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index fd4bf59092..d51e834ce2 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -125,8 +125,6 @@ const struct m_sub_options vd_lavc_conf = {
     },
 };
 
-extern const struct vd_lavc_hwdec mp_vd_lavc_vdpau;
-extern const struct vd_lavc_hwdec mp_vd_lavc_vdpau_copy;
 extern const struct vd_lavc_hwdec mp_vd_lavc_videotoolbox;
 extern const struct vd_lavc_hwdec mp_vd_lavc_videotoolbox_copy;
 extern const struct vd_lavc_hwdec mp_vd_lavc_dxva2;
@@ -205,12 +203,43 @@ extern const struct vd_lavc_hwdec mp_vd_lavc_vaapi_copy;
 #endif
 #endif
 
+#if HAVE_VDPAU_HWACCEL
+#if HAVE_VDPAU_HWACCEL_NEW
+const struct vd_lavc_hwdec mp_vd_lavc_vdpau = {
+    .type = HWDEC_VDPAU,
+    .image_format = IMGFMT_VDPAU,
+    .generic_hwaccel = true,
+    .pixfmt_map = (const enum AVPixelFormat[][2]) {
+        {AV_PIX_FMT_YUV420P,   AV_PIX_FMT_YUV420P},
+        {AV_PIX_FMT_NONE}
+    },
+};
+
+#include "video/vdpau.h"
+
+const struct vd_lavc_hwdec mp_vd_lavc_vdpau_copy = {
+    .type = HWDEC_VDPAU_COPY,
+    .copying = true,
+    .image_format = IMGFMT_VDPAU,
+    .generic_hwaccel = true,
+    .create_dev = vdpau_create_standalone,
+    .pixfmt_map = (const enum AVPixelFormat[][2]) {
+        {AV_PIX_FMT_YUV420P,   AV_PIX_FMT_YUV420P},
+        {AV_PIX_FMT_NONE}
+    },
+};
+#else
+extern const struct vd_lavc_hwdec mp_vd_lavc_vdpau;
+extern const struct vd_lavc_hwdec mp_vd_lavc_vdpau_copy;
+#endif
+#endif
+
 static const struct vd_lavc_hwdec *const hwdec_list[] = {
 #if HAVE_RPI
     &mp_vd_lavc_rpi,
     &mp_vd_lavc_rpi_copy,
 #endif
-#if HAVE_VDPAU_HWACCEL
+#if HAVE_VDPAU_HWACCEL_OLD || HAVE_VDPAU_HWACCEL_NEW
     &mp_vd_lavc_vdpau,
     &mp_vd_lavc_vdpau_copy,
 #endif
@@ -555,6 +584,9 @@ static void init_avctx(struct dec_video *vd, const char *decoder,
 
     if (ctx->hwdec) {
         avctx->thread_count = 1;
+#if HAVE_VDPAU_HWACCEL_NEW
+        avctx->hwaccel_flags |= AV_HWACCEL_FLAG_IGNORE_LEVEL;
+#endif
         if (ctx->hwdec->image_format)
             avctx->get_format = get_format_hwdec;
         if (ctx->hwdec->allocate_image)
@@ -797,6 +829,12 @@ static int init_generic_hwaccel(struct dec_video *vd)
 
     ctx->hwdec_fmt = hwdec->image_format;
 
+    if (hwdec->image_format == IMGFMT_VDPAU &&
+        ctx->avctx->codec_id == AV_CODEC_ID_HEVC)
+    {
+        MP_WARN(ctx, "HEVC video output may be broken due to nVidia bugs.\n");
+    }
+
     return hwdec_setup_hw_frames_ctx(ctx, ctx->hwdec_dev->av_device_ref,
                                      av_sw_format, pool_size);
 }
diff --git a/video/vdpau.c b/video/vdpau.c
index f4c85a0bab..8895053249 100644
--- a/video/vdpau.c
+++ b/video/vdpau.c
@@ -605,3 +605,31 @@ bool mp_vdpau_guess_if_emulated(struct mp_vdpau_ctx *ctx)
     CHECK_VDP_WARNING(ctx, "Error when calling vdp_get_information_string");
     return vdp_st == VDP_STATUS_OK && info && strstr(info, "VAAPI");
 }
+
+static void vdpau_destroy_standalone(struct mp_hwdec_ctx *ctx)
+{
+    struct mp_vdpau_ctx *vdp = ctx->ctx;
+    Display *display = vdp->x11;
+    mp_vdpau_destroy(vdp);
+    XCloseDisplay(display);
+}
+
+struct mp_hwdec_ctx *vdpau_create_standalone(struct mpv_global *global,
+                                             struct mp_log *plog, bool probing)
+{
+    XInitThreads();
+
+    Display *display = XOpenDisplay(NULL);
+    if (!display)
+        return NULL;
+
+    struct mp_vdpau_ctx *vdp = mp_vdpau_create_device_x11(plog, display, probing);
+    if (!vdp) {
+        XCloseDisplay(display);
+        return NULL;
+    }
+
+    vdp->hwctx.emulated = mp_vdpau_guess_if_emulated(vdp);
+    vdp->hwctx.destroy = vdpau_destroy_standalone;
+    return &vdp->hwctx;
+}
diff --git a/video/vdpau.h b/video/vdpau.h
index bffe901373..b320fa525a 100644
--- a/video/vdpau.h
+++ b/video/vdpau.h
@@ -88,6 +88,10 @@ struct mp_vdpau_ctx *mp_vdpau_create_device_x11(struct mp_log *log, Display *x11
                                                 bool probing);
 void mp_vdpau_destroy(struct mp_vdpau_ctx *ctx);
 
+struct mpv_global;
+struct mp_hwdec_ctx *vdpau_create_standalone(struct mpv_global *global,
+                                             struct mp_log *plog, bool probing);
+
 int mp_vdpau_handle_preemption(struct mp_vdpau_ctx *ctx, uint64_t *counter);
 
 struct mp_image *mp_vdpau_get_video_surface(struct mp_vdpau_ctx *ctx,