From 50bd2807ad4f94bcf215991566336060ebfdd249 Mon Sep 17 00:00:00 2001
From: wm4 <wm4@nowhere>
Date: Wed, 15 Jul 2015 13:27:25 +0200
Subject: vaapi: don't assume vaQueryImageFormats() returns sorted list

vaQueryImageFormats() returns a randomly ordered list - so we shouldn't
assume the first format on the list which works is the best. This
effectively switches to nv12 instead of yuv420p on some drivers.

We handle this by reusing va_to_imgfmt[], and ordering it by preference.
We hardcode that GPUs prefer nv12 pver yuv420p. In theory we could do
complicated probing (allocate dummy surface + use vaDeriveImage on it,
then retrieve the FourCC) - but all things which could break assumption
in the future are not supported yet (like 10 bit or 4:4:4), so this is
fine.
---
 video/vaapi.c | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/video/vaapi.c b/video/vaapi.c
index 6017bc4ded..ed021cfdd1 100644
--- a/video/vaapi.c
+++ b/video/vaapi.c
@@ -50,10 +50,10 @@ struct fmtentry {
 };
 
 static const struct fmtentry va_to_imgfmt[] = {
+    {VA_FOURCC_NV12, IMGFMT_NV12},
     {VA_FOURCC_YV12, IMGFMT_420P},
     {VA_FOURCC_I420, IMGFMT_420P},
     {VA_FOURCC_IYUV, IMGFMT_420P},
-    {VA_FOURCC_NV12, IMGFMT_NV12},
     {VA_FOURCC_UYVY, IMGFMT_UYVY},
     {VA_FOURCC_YUY2, IMGFMT_YUYV},
     // Note: not sure about endian issues (the mp formats are byte-addressed)
@@ -456,13 +456,18 @@ struct mp_image *va_surface_download(struct mp_image *src,
         return mpi;
 
     // We have no clue which format will work, so try them all.
-    for (int i = 0; i < ctx->image_formats->num; i++) {
-        VAImageFormat *format = &ctx->image_formats->entries[i];
-        if (va_surface_image_alloc(src, format) < 0)
-            continue;
-        mpi = try_download(src, pool);
-        if (mpi)
-            return mpi;
+    // Make sure to start with the most preferred format (nv12), to avoid
+    // slower code paths.
+    for (int n = 0; va_to_imgfmt[n].mp; n++) {
+        VAImageFormat *format =
+            va_image_format_from_imgfmt(ctx, va_to_imgfmt[n].mp);
+        if (format) {
+            if (va_surface_image_alloc(src, format) < 0)
+                continue;
+            mpi = try_download(src, pool);
+            if (mpi)
+                return mpi;
+        }
     }
 
     MP_ERR(ctx, "failed to get surface data.\n");
-- 
cgit v1.2.3