vo_dmabuf_wayland: improve error handling exporting VA surface

1. check for all error conditions when calling vaExportSurfaceHandle
2. only clean up valid descriptors

1 files changed, 25 insertions, 23 deletions

diff --git a/video/out/vo_dmabuf_wayland.c b/video/out/vo_dmabuf_wayland.c
index fe320de884..a1a845bd9c 100644
--- a/video/out/vo_dmabuf_wayland.c
+++ b/video/out/vo_dmabuf_wayland.c
@@ -73,37 +73,39 @@ static bool vaapi_dmabuf_importer(struct mp_image *src, struct wlbuf_pool_entry*
                                   struct zwp_linux_buffer_params_v1 *params)
 {
     struct priv *p = entry->vo->priv;
-    VADRMPRIMESurfaceDescriptor desc;
-    bool dmabuf_imported = false;
+    VADRMPRIMESurfaceDescriptor desc = { 0 };
     /* composed has single layer */
     int layer_no = 0;
     VAStatus status = vaExportSurfaceHandle(p->display, entry->key, VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2,
                                             VA_EXPORT_SURFACE_COMPOSED_LAYERS | VA_EXPORT_SURFACE_READ_ONLY, &desc);
-    if (status == VA_STATUS_ERROR_INVALID_SURFACE) {
-        MP_VERBOSE(entry->vo, "VA export to composed layers not supported.\n");
-    } else if (!vo_wayland_supported_format(entry->vo, desc.layers[0].drm_format, desc.objects[0].drm_format_modifier)) {
+
+    if (!CHECK_VA_STATUS(entry->vo, "vaExportSurfaceHandle()")) {
+        /* invalid surface warning => composed layers not supported */
+        if (status == VA_STATUS_ERROR_INVALID_SURFACE)
+            MP_VERBOSE(entry->vo, "vaExportSurfaceHandle: composed layers not supported.\n");
+        return false;
+    }
+    bool success = false;
+    uint32_t drm_format = desc.layers[layer_no].drm_format;
+    if (!vo_wayland_supported_format(entry->vo, drm_format, desc.objects[0].drm_format_modifier)) {
         MP_VERBOSE(entry->vo, "%s(%016lx) is not supported.\n",
-                   mp_tag_str(desc.layers[0].drm_format), desc.objects[0].drm_format_modifier);
-    } else if (CHECK_VA_STATUS(entry->vo, "vaExportSurfaceHandle()")) {
-        entry->drm_format = desc.layers[layer_no].drm_format;
-        for (int plane_no = 0; plane_no < desc.layers[layer_no].num_planes; ++plane_no) {
-            int object = desc.layers[layer_no].object_index[plane_no];
-            uint64_t modifier = desc.objects[object].drm_format_modifier;
-            zwp_linux_buffer_params_v1_add(params, desc.objects[object].fd, plane_no, desc.layers[layer_no].offset[plane_no],
-                                           desc.layers[layer_no].pitch[plane_no], modifier >> 32, modifier & 0xffffffff);
-        }
-        dmabuf_imported = true;
+                   mp_tag_str(drm_format), desc.objects[0].drm_format_modifier);
+        goto done;
     }
-
-    /* clean up descriptor */
-    if (status != VA_STATUS_ERROR_INVALID_SURFACE) {
-        for (int i = 0; i < desc.num_objects; i++) {
-            close(desc.objects[i].fd);
-            desc.objects[i].fd = 0;
-        }
+    entry->drm_format = drm_format;
+    for (int plane_no = 0; plane_no < desc.layers[layer_no].num_planes; ++plane_no) {
+        int object = desc.layers[layer_no].object_index[plane_no];
+        uint64_t modifier = desc.objects[object].drm_format_modifier;
+        zwp_linux_buffer_params_v1_add(params, desc.objects[object].fd, plane_no, desc.layers[layer_no].offset[plane_no],
+                                       desc.layers[layer_no].pitch[plane_no], modifier >> 32, modifier & 0xffffffff);
     }
+    success = true;
+
+done:
+    for (int i = 0; i < desc.num_objects; i++)
+        close(desc.objects[i].fd);
 
-    return dmabuf_imported;
+    return success;
 }
 #endif