video: add vaapi decode and output support

This is based on the MPlayer VA API patches. To be exact it's based on
a very stripped down version of commit f1ad459a263f8537f6c from
git://gitorious.org/vaapi/mplayer.git.

This doesn't contain useless things like benchmarking hacks and the
demo code for GLX interop. Also, unlike in the original patch, decoding
and video output are split into separate source files (the separation
between decoding and display also makes pixel format hacks unnecessary).

On the other hand, some features not present in the original patch were
added, like screenshot support.

VA API is rather bad for actual video output. Dealing with older libva
versions or the completely broken vdpau backend doesn't help. OSD is
low quality and should be rather slow. In some cases, only either OSD
or subtitles can be shown at the same time (because OSD is drawn first,
OSD is prefered).

Also, libva can't decide whether it accepts straight or premultiplied
alpha for OSD sub-pictures: the vdpau backend seems to assume
premultiplied, while a native vaapi driver uses straight. So I picked
straight alpha. It doesn't matter much, because the blending code for
straight alpha I added to img_convert.c is probably buggy, and ASS
subtitles might be blended incorrectly.

Really good video output with VA API would probably use OpenGL and the
GL interop features, but at this point you might just use vo_opengl.
(Patches for making HW decoding with vo_opengl have a chance of being
accepted.)

Despite these issues, decoding seems to work ok. I still got tearing
on the Intel system I tested (Intel(R) Core(TM) i3-2350M). It was also
tested with the vdpau vaapi wrapper on a nvidia system; however this
was rather broken. (Fortunately, there is no reason to use mpv's VAAPI
support over native VDPAU.)

1 files changed, 73 insertions, 0 deletions

diff --git a/video/vaapi.h b/video/vaapi.h
new file mode 100644
index 0000000000..2fc149d524
--- /dev/null
+++ b/video/vaapi.h
@@ -0,0 +1,73 @@
+#ifndef MPV_VDPAU_H
+#define MPV_VDPAU_H
+
+#include <stdbool.h>
+#include <inttypes.h>
+
+#include <va/va.h>
+
+/* Compatibility glue with VA-API >= 0.31 */
+#if defined VA_CHECK_VERSION
+#if VA_CHECK_VERSION(0,31,0)
+#define vaPutImage2             vaPutImage
+#define vaAssociateSubpicture2  vaAssociateSubpicture
+#endif
+#endif
+
+/* Compatibility glue with VA-API >= 0.34 */
+#if VA_CHECK_VERSION(0,34,0)
+#include <va/va_compat.h>
+#endif
+
+/* Compatibility glue with upstream libva */
+#ifndef VA_SDS_VERSION
+#define VA_SDS_VERSION          0
+#endif
+
+/* Compatibility glue with VA-API >= 0.30 */
+#ifndef VA_INVALID_ID
+#define VA_INVALID_ID           0xffffffff
+#endif
+#ifndef VA_FOURCC
+#define VA_FOURCC(ch0, ch1, ch2, ch3)           \
+    ((uint32_t)(uint8_t)(ch0) |                 \
+     ((uint32_t)(uint8_t)(ch1) << 8) |          \
+     ((uint32_t)(uint8_t)(ch2) << 16) |         \
+     ((uint32_t)(uint8_t)(ch3) << 24 ))
+#endif
+#if defined VA_SRC_BT601 && defined VA_SRC_BT709
+# define USE_VAAPI_COLORSPACE 1
+#else
+# define USE_VAAPI_COLORSPACE 0
+#endif
+
+/* Compatibility glue with VA-API >= 0.31.1 */
+#ifndef VA_SRC_SMPTE_240
+#define VA_SRC_SMPTE_240        0x00000040
+#endif
+#if defined VA_FILTER_SCALING_MASK
+# define USE_VAAPI_SCALING 1
+#else
+# define USE_VAAPI_SCALING 0
+#endif
+
+#include "mpvcore/mp_msg.h"
+
+static inline bool check_va_status(VAStatus status, const char *msg)
+{
+    if (status != VA_STATUS_SUCCESS) {
+        mp_msg(MSGT_VO, MSGL_ERR, "[vaapi] %s: %s\n", msg, vaErrorStr(status));
+        return false;
+    }
+    return true;
+}
+
+struct mp_vaapi_ctx {
+    VADisplay display;
+    struct mp_image *(*get_surface)(struct mp_vaapi_ctx *ctx, int va_format,
+                                    int mp_format, int w, int h);
+    void (*flush)(struct mp_vaapi_ctx *ctx);
+    void *priv; // for VO
+};
+
+#endif