vo_opengl: vaapi: probe the surface format

Probe the surface format, and check whether it's really something we
support. This also does a complete check whether the EGL interop works
at all (the only way to find this out is actually running this code).
Also, support YV12. Under some circumstances, vaapi (with Intel
drivers) can be made to use this format.

Unfortunately, the Intel drivers show some very weird behavior, which
is hopefully a bug. insane_hack() provides a very evil workaround (see
comments). A proper solution might be passing the hw format as part of
mp_image_params, but as long as hw surfaces appear to be able to change
the format on the fly, attempting this is probably not worth the extra
complexity and likely fragility. The hack allows us to pretend that
there is sane behavior for now.

1 files changed, 68 insertions, 2 deletions

diff --git a/video/out/opengl/hwdec_vaegl.c b/video/out/opengl/hwdec_vaegl.c
index 197c8beae8..74f6c0d6f8 100644
--- a/video/out/opengl/hwdec_vaegl.c
+++ b/video/out/opengl/hwdec_vaegl.c
@@ -29,6 +29,7 @@
 #include "hwdec.h"
 #include "video/vaapi.h"
 #include "video/img_fourcc.h"
+#include "video/mp_image_pool.h"
 #include "common.h"
 
 struct priv {
@@ -43,6 +44,8 @@ struct priv {
     struct mp_image *current_ref;
 };
 
+static bool test_format(struct gl_hwdec *hw);
+
 static void unref_image(struct gl_hwdec *hw)
 {
     struct priv *p = hw->priv;
@@ -90,6 +93,30 @@ static void destroy(struct gl_hwdec *hw)
     va_destroy(p->ctx);
 }
 
+// Create an empty dummy VPP. This works around a weird bug that affects the
+// VA surface format, as it is reported by vaDeriveImage(). Before a VPP
+// context or a decoder context is created, the surface format will be reported
+// as YV12. Surfaces created after context creation will report NV12 (even
+// though surface creation does not take a context as argument!). Existing
+// surfaces will change their format from YV12 to NV12 as soon as the decoder
+// renders to them! Because we want know the surface format in advance (to
+// simplify our renderer configuration logic), we hope that this hack gives
+// us reasonable behavior.
+// See: https://bugs.freedesktop.org/show_bug.cgi?id=79848
+static void insane_hack(struct gl_hwdec *hw)
+{
+    struct priv *p = hw->priv;
+    VAConfigID config;
+    if (vaCreateConfig(p->display, VAProfileNone, VAEntrypointVideoProc,
+                       NULL, 0, &config) == VA_STATUS_SUCCESS)
+    {
+        // We want to keep this until the VADisplay is destroyed. It will
+        // implicitly free the context.
+        VAContextID context;
+        vaCreateContext(p->display, config, 0, 0, 0, NULL, 0, &context);
+    }
+}
+
 static int create(struct gl_hwdec *hw)
 {
     GL *gl = hw->gl;
@@ -130,8 +157,13 @@ static int create(struct gl_hwdec *hw)
 
     MP_VERBOSE(p, "using VAAPI EGL interop\n");
 
+    insane_hack(hw);
+    if (!test_format(hw)) {
+        destroy(hw);
+        return -1;
+    }
+
     hw->hwctx = &p->ctx->hwctx;
-    hw->converted_imgfmt = IMGFMT_NV12;
     return 0;
 }
 
@@ -185,12 +217,24 @@ static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
         goto err;
 
     int mpfmt = va_fourcc_to_imgfmt(va_image->format.fourcc);
-    if (mpfmt != IMGFMT_NV12) {
+    if (mpfmt != IMGFMT_NV12 && mpfmt != IMGFMT_420P) {
         MP_FATAL(p, "unsupported VA image format %s\n",
                  VA_STR_FOURCC(va_image->format.fourcc));
         goto err;
     }
 
+    if (!hw->converted_imgfmt) {
+        MP_VERBOSE(p, "format: %s %s\n", VA_STR_FOURCC(va_image->format.fourcc),
+                   mp_imgfmt_to_name(mpfmt));
+        hw->converted_imgfmt = mpfmt;
+    }
+
+    if (hw->converted_imgfmt != mpfmt) {
+        MP_FATAL(p, "mid-stream hwdec format change (%s -> %s) not supported\n",
+                 mp_imgfmt_to_name(hw->converted_imgfmt), mp_imgfmt_to_name(mpfmt));
+        goto err;
+    }
+
     VABufferInfo buffer_info = {.mem_type = VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME};
     status = vaAcquireBufferHandle(p->display, va_image->buf, &buffer_info);
     if (!CHECK_VA_STATUS(p, "vaAcquireBufferHandle()"))
@@ -243,6 +287,28 @@ err:
     return -1;
 }
 
+static bool test_format(struct gl_hwdec *hw)
+{
+    struct priv *p = hw->priv;
+    bool ok = false;
+
+    struct mp_image_pool *alloc = mp_image_pool_new(1);
+    va_pool_set_allocator(alloc, p->ctx, VA_RT_FORMAT_YUV420);
+    struct mp_image *surface = mp_image_pool_get(alloc, IMGFMT_VAAPI, 64, 64);
+    if (surface) {
+        struct mp_image_params params = surface->params;
+        if (reinit(hw, &params) >= 0) {
+            GLuint textures[4];
+            ok = map_image(hw, surface, textures) >= 0;
+        }
+        unref_image(hw);
+    }
+    talloc_free(surface);
+    talloc_free(alloc);
+
+    return ok;
+}
+
 const struct gl_hwdec_driver gl_hwdec_vaegl = {
     .api_name = "vaapi",
     .imgfmt = IMGFMT_VAAPI,