vo_gpu: hwdec_vaapi: Improve logging when probing surface formats

Our logging here today is very poor. We don't make it clear what
formats we are probing, or even that a certain format failed in most
cases. In the case where we do log the error, we don't make it clear
which format it was that failed.

The end result is that we have no idea what the possible and final
format spaces are, which makes it very hard to debug whether things are
working correctly, or to work on supporting additional formats.

2 files changed, 25 insertions, 4 deletions

diff --git a/video/out/hwdec/hwdec_vaapi.c b/video/out/hwdec/hwdec_vaapi.c
index 01b5be476c..b96f644576 100644
--- a/video/out/hwdec/hwdec_vaapi.c
+++ b/video/out/hwdec/hwdec_vaapi.c
@@ -249,6 +249,15 @@ static int mapper_map(struct ra_hwdec_mapper *mapper)
     CHECK_VA_STATUS(mapper, "vaSyncSurface()");
     p->surface_acquired = true;
 
+    if (p->num_planes != p->desc.num_layers) {
+        mp_msg(mapper->log, p_owner->probing_formats ? MSGL_DEBUG : MSGL_ERR,
+               "Mapped surface with format '%s' has unexpected number of planes. "
+               "(%d instead of %d)\n",
+               mp_imgfmt_to_name(mapper->src->params.hw_subfmt),
+               p->desc.num_layers, p->num_planes);
+        goto err;
+    }
+
     if (!p_owner->interop_map(mapper, p_owner->probing_formats))
         goto err;
 
@@ -267,16 +276,20 @@ err:
 
 static bool try_format_map(struct ra_hwdec *hw, struct mp_image *surface)
 {
-    bool ok = false;
     struct ra_hwdec_mapper *mapper = ra_hwdec_mapper_create(hw, &surface->params);
-    if (mapper)
-        ok = ra_hwdec_mapper_map(mapper, surface) >= 0;
+    if (!mapper) {
+        MP_DBG(hw, "Failed to create mapper\n");
+        return false;
+    }
+
+    bool ok = ra_hwdec_mapper_map(mapper, surface) >= 0;
     ra_hwdec_mapper_free(&mapper);
     return ok;
 }
 
 static void try_format_pixfmt(struct ra_hwdec *hw, enum AVPixelFormat pixfmt)
 {
+    bool supported = false;
     struct priv_owner *p = hw->priv;
 
     int mp_fmt = pixfmt2imgfmt(pixfmt);
@@ -312,10 +325,15 @@ static void try_format_pixfmt(struct ra_hwdec *hw, enum AVPixelFormat pixfmt)
     if (!s || !mp_image_params_valid(&s->params))
         goto err;
     if (try_format_map(hw, s)) {
+        supported = true;
         MP_TARRAY_APPEND(p, p->formats, num_formats, mp_fmt);
         MP_TARRAY_APPEND(p, p->formats, num_formats, 0); // terminate it
     }
 err:
+    if (!supported)
+        MP_DBG(hw, "Unsupported format: %s\n",
+               mp_imgfmt_to_name(mp_fmt));
+
     talloc_free(s);
     av_frame_free(&frame);
     av_buffer_unref(&fref);
diff --git a/video/out/hwdec/hwdec_vaapi_gl.c b/video/out/hwdec/hwdec_vaapi_gl.c
index ed72701fc2..9140e47d8b 100644
--- a/video/out/hwdec/hwdec_vaapi_gl.c
+++ b/video/out/hwdec/hwdec_vaapi_gl.c
@@ -183,8 +183,11 @@ static bool vaapi_gl_map(struct ra_hwdec_mapper *mapper, bool probing)
 
         p->images[n] = p->CreateImageKHR(eglGetCurrentDisplay(),
             EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, NULL, attribs);
-        if (!p->images[n])
+        if (!p->images[n]) {
+            mp_msg(mapper->log, p_owner->probing_formats ? MSGL_DEBUG : MSGL_ERR,
+                   "Failed to import surface in EGL: %u\n", eglGetError());
             return false;
+        }
 
         gl->BindTexture(GL_TEXTURE_2D, p->gl_textures[n]);
         p->EGLImageTargetTexture2DOES(GL_TEXTURE_2D, p->images[n]);