video: add vda decode support (with hwaccel) and direct rendering

Decoding H264 using Video Decode Acceleration used the custom 'vda_h264_dec'
decoder in FFmpeg.

The Good: This new implementation has some advantages over the previous one:

 - It works with Libav: vda_h264_dec never got into Libav since they prefer
   client applications to use the hwaccel API.

 - It is way more efficient: in my tests this implementation yields a
   reduction of CPU usage of roughly ~50% compared to using `vda_h264_dec` and
   ~65-75% compared to h264 software decoding. This is mainly because
   `vo_corevideo` was adapted to perform direct rendering of the
   `CVPixelBufferRefs` created by the Video Decode Acceleration API Framework.

The Bad:
  - `vo_corevideo` is required to use VDA decoding acceleration.
  - only works with versions of ffmpeg/libav new enough (needs reference
    refcounting). That is FFmpeg 2.0+ and Libav's git master currently.

The Ugly: VDA was hardcoded to use UYVY (2vuy) for the uploaded video texture.
One one end this makes the code simple since Apple's OpenGL implementation
actually supports this out of the box. It would be nice to support other
output image formats and choose the best format depending on the input, or at
least making it configurable. My tests indicate that CPU usage actually
increases with a 420p IMGFMT output which is not what I would have expected.

NOTE: There is a small memory leak with old versions of FFmpeg and with Libav
since the CVPixelBufferRef is not automatically released when the AVFrame is
deallocated. This can cause leaks inside libavcodec for decoded frames that
are discarded before mpv wraps them inside a refcounted mp_image (this only
happens on seeks).
For frames that enter mpv's refcounting facilities, this is not a problem
since we rewrap the CVPixelBufferRef in our mp_image that properly forwards
CVPixelBufferRetain/CvPixelBufferRelease calls to the underying
CVPixelBufferRef.

So, for FFmpeg use something more recent than `b3d63995` for Libav the patch
was posted to the dev ML in July and in review since, apparently, the proposed
fix is rather hacky.

1 files changed, 46 insertions, 0 deletions

diff --git a/configure b/configure
index 06f871b32b..95da109f4f 100755
--- a/configure
+++ b/configure
@@ -347,6 +347,7 @@ Video output:
   --enable-sdl2            enable SDL 2.0+ audio and video output [disable]
   --enable-xv              enable Xv video output [autodetect]
   --enable-vdpau           enable VDPAU acceleration [autodetect]
+  --enable-vda             enable VDA acceleration [autodetect]
   --enable-vaapi           enable VAAPI acceleration [autodetect]
   --enable-vm              enable XF86VidMode support [autodetect]
   --enable-xinerama        enable Xinerama support [autodetect]
@@ -422,6 +423,8 @@ _wayland=auto
 _xss=auto
 _xv=auto
 _vdpau=auto
+_vda=auto
+_vda_refcounting=auto
 _vaapi=auto
 _direct3d=auto
 _sdl=no
@@ -584,6 +587,8 @@ for ac_option do
   --disable-xv)         _xv=no          ;;
   --enable-vdpau)       _vdpau=yes      ;;
   --disable-vdpau)      _vdpau=no       ;;
+  --enable-vda)         _vda=yes        ;;
+  --disable-vda)        _vda=no         ;;
   --enable-vaapi)       _vaapi=yes      ;;
   --disable-vaapi)      _vaapi=no       ;;
   --enable-direct3d)    _direct3d=yes   ;;
@@ -2815,6 +2820,42 @@ fi
 echores "$libpostproc"
 
 
+if darwin ; then
+
+echocheck "VDA"
+if test "$_vda" = auto ; then
+ _vda=no
+ if test "$_avutil_has_refcounting" = "yes" ; then
+   header_check VideoDecodeAcceleration/VDADecoder.h &&
+     header_check libavcodec/vda.h && _vda=yes
+ else
+   res_comment="libavutil too old"
+ fi
+fi
+if test "$_vda" = yes ; then
+  def_vda='#define CONFIG_VDA 1'
+  libs_mplayer="$libs_mplayer -framework VideoDecodeAcceleration -framework QuartzCore -framework IOSurface"
+else
+  def_vda='#define CONFIG_VDA 0'
+fi
+echores "$_vda"
+
+echocheck "VDA libavcodec refcounting"
+_vda_refcounting=no
+if test "$_vda" = yes ; then
+  statement_check libavcodec/vda.h 'struct vda_context a = (struct vda_context) { .use_ref_buffer = 1 }' &&
+    _vda_refcounting=yes
+fi
+if test "$_vda_refcounting" = "yes" ; then
+  def_vda_refcounting='#define HAVE_VDA_LIBAVCODEC_REFCOUNTING 1'
+else
+  def_vda_refcounting='#define HAVE_VDA_LIBAVCODEC_REFCOUNTING 0'
+fi
+echores "$_vda_refcounting"
+
+fi
+
+
 echocheck "TV interface"
 if test "$_tv" = yes ; then
   def_tv='#define CONFIG_TV 1'
@@ -3110,6 +3151,8 @@ VCD = $_vcd
 VDPAU = $_vdpau
 VDPAU_DEC = $_vdpau_dec
 VDPAU_DEC_OLD = $_vdpau_dec_old
+VDA = $_vda
+VDA_REFCOUNTING = $_vda_refcounting
 VAAPI = $_vaapi
 WIN32 = $_win32
 X11 = $_x11
@@ -3120,6 +3163,7 @@ XV = $_xv
 ENCODING = $_encoding
 
 CONFIG_VDPAU    = $_vdpau
+CONFIG_VDA      = $_vda
 CONFIG_VAAPI    = $_vaapi
 CONFIG_ZLIB     = $_zlib
 
@@ -3286,6 +3330,8 @@ $def_jpeg
 $def_mng
 $def_v4l2
 $def_vdpau
+$def_vda
+$def_vda_refcounting
 $def_vaapi
 $def_vm
 $def_x11