hwdec: add VideoToolbox support

VDA is being deprecated in OS X 10.11 so this is needed to keep hwdec working.
The code needs libavcodec support which was added recently (to FFmpeg git,
libav doesn't support it).

Signed-off-by: Stefano Pigozzi <stefano.pigozzi@gmail.com>

11 files changed, 198 insertions, 12 deletions

diff --git a/options/options.c b/options/options.c
index 4f6839eede..bb19a44688 100644
--- a/options/options.c
+++ b/options/options.c
@@ -85,6 +85,7 @@ const struct m_opt_choice_alternatives mp_hwdec_names[] = {
     {"auto",        HWDEC_AUTO},
     {"vdpau",       HWDEC_VDPAU},
     {"vda",         HWDEC_VDA},
+    {"videotoolbox",HWDEC_VIDEOTOOLBOX},
     {"vaapi",       HWDEC_VAAPI},
     {"vaapi-copy",  HWDEC_VAAPI_COPY},
     {"dxva2-copy",  HWDEC_DXVA2_COPY},
diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index 1dc6e28694..b8042a007d 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -117,6 +117,7 @@ const struct m_sub_options vd_lavc_conf = {
 
 const struct vd_lavc_hwdec mp_vd_lavc_vdpau;
 const struct vd_lavc_hwdec mp_vd_lavc_vda;
+const struct vd_lavc_hwdec mp_vd_lavc_videotoolbox;
 const struct vd_lavc_hwdec mp_vd_lavc_vaapi;
 const struct vd_lavc_hwdec mp_vd_lavc_vaapi_copy;
 const struct vd_lavc_hwdec mp_vd_lavc_dxva2_copy;
@@ -129,6 +130,9 @@ static const struct vd_lavc_hwdec *const hwdec_list[] = {
 #if HAVE_VDPAU_HWACCEL
     &mp_vd_lavc_vdpau,
 #endif
+#if HAVE_VIDEOTOOLBOX_HWACCEL
+    &mp_vd_lavc_videotoolbox,
+#endif
 #if HAVE_VDA_HWACCEL
     &mp_vd_lavc_vda,
 #endif
diff --git a/video/decode/videotoolbox.c b/video/decode/videotoolbox.c
new file mode 100644
index 0000000000..c4f7c05f05
--- /dev/null
+++ b/video/decode/videotoolbox.c
@@ -0,0 +1,115 @@
+/*
+ * This file is part of mpv.
+ *
+ * Copyright (c) 2015 Sebastien Zwickert
+ *
+ * mpv is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <libavcodec/version.h>
+#include <libavcodec/videotoolbox.h>
+
+#include "common/av_common.h"
+#include "common/msg.h"
+#include "video/mp_image.h"
+#include "video/decode/lavc.h"
+#include "config.h"
+
+
+static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+                 const char *decoder)
+{
+    hwdec_request_api(info, "videotoolbox");
+    if (!info || !info->hwctx)
+        return HWDEC_ERR_NO_CTX;
+    switch (mp_codec_to_av_codec_id(decoder)) {
+    case AV_CODEC_ID_H264:
+    case AV_CODEC_ID_H263:
+    case AV_CODEC_ID_MPEG1VIDEO:
+    case AV_CODEC_ID_MPEG2VIDEO:
+    case AV_CODEC_ID_MPEG4:
+        break;
+    default:
+        return HWDEC_ERR_NO_CODEC;
+    }
+    return 0;
+}
+
+static int init(struct lavc_ctx *ctx)
+{
+    return 0;
+}
+
+struct videotoolbox_error {
+    int  code;
+    char *reason;
+};
+
+static const struct videotoolbox_error videotoolbox_errors[] = {
+    { AVERROR(ENOSYS),
+        "Hardware doesn't support accelerated decoding for this stream"
+        " or Videotoolbox decoder is not available at the moment (another"
+        " application is using it)."
+    },
+    { AVERROR(EINVAL),
+        "Invalid configuration provided to VTDecompressionSessionCreate" },
+    { AVERROR_INVALIDDATA,
+        "Generic error returned by the decoder layer. The cause can be Videotoolbox"
+        " found errors in the bitstream." },
+    { 0, NULL },
+};
+
+static void print_videotoolbox_error(struct mp_log *log, int lev, char *message,
+                            int error_code)
+{
+    for (int n = 0; videotoolbox_errors[n].code < 0; n++)
+        if (videotoolbox_errors[n].code == error_code) {
+            mp_msg(log, lev, "%s: %s (%d)\n",
+                   message, videotoolbox_errors[n].reason, error_code);
+            return;
+        }
+
+    mp_msg(log, lev, "%s: %d\n", message, error_code);
+}
+
+static int init_decoder(struct lavc_ctx *ctx, int fmt, int w, int h)
+{
+    av_videotoolbox_default_free(ctx->avctx);
+
+    AVVideotoolboxContext *vtctx = av_videotoolbox_alloc_context();
+    vtctx->cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
+    int err = av_videotoolbox_default_init2(ctx->avctx, vtctx);
+
+    if (err < 0) {
+        print_videotoolbox_error(ctx->log, MSGL_ERR, "failed to init videotoolbox decoder", err);
+        return -1;
+    }
+
+    return 0;
+}
+
+static void uninit(struct lavc_ctx *ctx)
+{
+    if (ctx->avctx)
+        av_videotoolbox_default_free(ctx->avctx);
+}
+
+const struct vd_lavc_hwdec mp_vd_lavc_videotoolbox = {
+    .type = HWDEC_VIDEOTOOLBOX,
+    .image_format = IMGFMT_VIDEOTOOLBOX,
+    .probe = probe,
+    .init = init,
+    .uninit = uninit,
+    .init_decoder = init_decoder,
+};
diff --git a/video/fmt-conversion.c b/video/fmt-conversion.c
index b280e36945..797d243002 100644
--- a/video/fmt-conversion.c
+++ b/video/fmt-conversion.c
@@ -121,6 +121,9 @@ static const struct {
 #if HAVE_VDA_HWACCEL
     {IMGFMT_VDA,   AV_PIX_FMT_VDA},
 #endif
+#if HAVE_VIDEOTOOLBOX_HWACCEL
+    {IMGFMT_VIDEOTOOLBOX,   AV_PIX_FMT_VIDEOTOOLBOX},
+#endif
     {IMGFMT_VAAPI, AV_PIX_FMT_VAAPI_VLD},
     {IMGFMT_DXVA2, AV_PIX_FMT_DXVA2_VLD},
 #if HAVE_AV_PIX_FMT_MMAL
diff --git a/video/hwdec.h b/video/hwdec.h
index c9eb8e6a95..d950a86ef2 100644
--- a/video/hwdec.h
+++ b/video/hwdec.h
@@ -11,6 +11,7 @@ enum hwdec_type {
     HWDEC_NONE = 0,
     HWDEC_VDPAU = 1,
     HWDEC_VDA = 2,
+    HWDEC_VIDEOTOOLBOX = 3,
     HWDEC_VAAPI = 4,
     HWDEC_VAAPI_COPY = 5,
     HWDEC_DXVA2_COPY = 6,
diff --git a/video/img_format.c b/video/img_format.c
index 6cf585db61..5defa6662e 100644
--- a/video/img_format.c
+++ b/video/img_format.c
@@ -36,6 +36,7 @@ static const struct mp_imgfmt_entry mp_imgfmt_list[] = {
     {"vdpau_output",    IMGFMT_VDPAU_OUTPUT},
     // FFmpeg names have an annoying "_vld" suffix
     {"vda",             IMGFMT_VDA},
+    {"videotoolbox",    IMGFMT_VIDEOTOOLBOX},
     {"vaapi",           IMGFMT_VAAPI},
     // names below this are not preferred over the FFmpeg names
     // the "none" entry makes mp_imgfmt_to_name prefer FFmpeg names
diff --git a/video/img_format.h b/video/img_format.h
index 8c79d9f95b..77d722f7c9 100644
--- a/video/img_format.h
+++ b/video/img_format.h
@@ -203,6 +203,7 @@ enum mp_imgfmt {
     IMGFMT_VDPAU,           // VdpVideoSurface
     IMGFMT_VDPAU_OUTPUT,    // VdpOutputSurface
     IMGFMT_VDA,
+    IMGFMT_VIDEOTOOLBOX,
     IMGFMT_VAAPI,
     IMGFMT_DXVA2,           // IDirect3DSurface9 (NV12)
     IMGFMT_MMAL,            // MMAL_BUFFER_HEADER_T
diff --git a/video/out/gl_hwdec.c b/video/out/gl_hwdec.c
index c32f36a699..17a7c7bb67 100644
--- a/video/out/gl_hwdec.c
+++ b/video/out/gl_hwdec.c
@@ -31,6 +31,7 @@
 
 extern const struct gl_hwdec_driver gl_hwdec_vaglx;
 extern const struct gl_hwdec_driver gl_hwdec_vda;
+extern const struct gl_hwdec_driver gl_hwdec_videotoolbox;
 extern const struct gl_hwdec_driver gl_hwdec_vdpau;
 extern const struct gl_hwdec_driver gl_hwdec_dxva2;
 
@@ -47,6 +48,9 @@ static const struct gl_hwdec_driver *const mpgl_hwdec_drivers[] = {
 #if HAVE_DXVA2_HWACCEL
     &gl_hwdec_dxva2,
 #endif
+#if HAVE_VIDEOTOOLBOX_GL
+    &gl_hwdec_videotoolbox,
+#endif
     NULL
 };
 
diff --git a/video/out/gl_hwdec_vda.c b/video/out/gl_hwdec_vda.c
index 0b7cda2c3e..a94cdbc594 100644
--- a/video/out/gl_hwdec_vda.c
+++ b/video/out/gl_hwdec_vda.c
@@ -85,7 +85,7 @@ static struct mp_image *download_image(struct mp_hwdec_ctx *ctx,
                                        struct mp_image *hw_image,
                                        struct mp_image_pool *swpool)
 {
-    if (hw_image->imgfmt != IMGFMT_VDA)
+    if (hw_image->imgfmt != IMGFMT_VDA || hw_image->imgfmt != IMGFMT_VIDEOTOOLBOX)
         return NULL;
 
     CVPixelBufferRef pbuf = (CVPixelBufferRef)hw_image->planes[3];
@@ -129,26 +129,19 @@ static bool check_hwdec(struct gl_hwdec *hw)
     return true;
 }
 
-static int create(struct gl_hwdec *hw)
+static int create_common(struct gl_hwdec *hw, struct vda_format *format)
 {
     struct priv *p = talloc_zero(hw, struct priv);
 
     hw->priv = p;
     hw->gl_texture_target = GL_TEXTURE_RECTANGLE;
 
-#if HAVE_VDA_DEFAULT_INIT2
-    struct vda_format *f = vda_get_gl_format_from_imgfmt(IMGFMT_NV12);
-#else
-    struct vda_format *f = vda_get_gl_format_from_imgfmt(IMGFMT_UYVY);
-#endif
-
-    hw->converted_imgfmt = f->imgfmt;
+    hw->converted_imgfmt = format->imgfmt;
 
     if (!check_hwdec(hw))
         return -1;
 
     hw->hwctx = &p->hwctx;
-    hw->hwctx->type = HWDEC_VDA;
     hw->hwctx->download_image = download_image;
 
     GL *gl = hw->gl;
@@ -157,6 +150,36 @@ static int create(struct gl_hwdec *hw)
     return 0;
 }
 
+#if HAVE_VDA_GL
+static int create_vda(struct gl_hwdec *hw)
+{
+#if HAVE_VDA_DEFAULT_INIT2
+    struct vda_format *f = vda_get_gl_format_from_imgfmt(IMGFMT_NV12);
+#else
+    struct vda_format *f = vda_get_gl_format_from_imgfmt(IMGFMT_UYVY);
+#endif
+    if (create_common(hw, f))
+        return -1;
+
+    hw->hwctx->type = HWDEC_VDA;
+
+    return 0;
+}
+#endif
+
+#if HAVE_VIDEOTOOLBOX_GL
+static int create_videotoolbox(struct gl_hwdec *hw)
+{
+    struct vda_format *f = vda_get_gl_format_from_imgfmt(IMGFMT_NV12);
+    if (create_common(hw, f))
+        return -1;
+
+    hw->hwctx->type = HWDEC_VIDEOTOOLBOX;
+
+    return 0;
+}
+#endif
+
 static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 {
     params->imgfmt = hw->driver->imgfmt;
@@ -219,11 +242,24 @@ static void destroy(struct gl_hwdec *hw)
     gl->DeleteTextures(MP_MAX_PLANES, p->gl_planes);
 }
 
+#if HAVE_VDA_GL
 const struct gl_hwdec_driver gl_hwdec_vda = {
     .api_name = "vda",
     .imgfmt = IMGFMT_VDA,
-    .create = create,
+    .create = create_vda,
     .reinit = reinit,
     .map_image = map_image,
     .destroy = destroy,
 };
+#endif
+
+#if HAVE_VIDEOTOOLBOX_GL
+const struct gl_hwdec_driver gl_hwdec_videotoolbox = {
+    .api_name = "videotoolbox",
+    .imgfmt = IMGFMT_VIDEOTOOLBOX,
+    .create = create_videotoolbox,
+    .reinit = reinit,
+    .map_image = map_image,
+    .destroy = destroy,
+};
+#endif
diff --git a/wscript b/wscript
index f6d5b6c194..687e2bfff3 100644
--- a/wscript
+++ b/wscript
@@ -728,6 +728,25 @@ hwaccel_features = [
         'deps': [ 'gl-cocoa', 'vda-hwaccel' ],
         'func': check_true
     }, {
+        'name': '--videotoolbox-hwaccel',
+        'desc': 'libavcodec videotoolbox hwaccel',
+        'func': compose_checks(
+            check_headers('VideoToolbox/VideoToolbox.h'),
+            check_statement('libavcodec/videotoolbox.h',
+                            'av_videotoolbox_alloc_context()',
+                            framework='IOSurface',
+                            use='libav')),
+    } , {
+        'name': '--videotoolbox-gl',
+        'desc': 'Videotoolbox with OpenGL',
+        'deps': [ 'gl-cocoa', 'videotoolbox-hwaccel' ],
+        'func': check_true
+    } , {
+        'name': 'videotoolbox-vda-gl',
+        'desc': 'Videotoolbox or VDA with OpenGL',
+        'deps': [ 'videotoolbox-gl', 'vda-gl' ],
+        'func': check_true
+    }, {
         'name': '--vdpau-hwaccel',
         'desc': 'libavcodec VDPAU hwaccel',
         'deps': [ 'vdpau' ],
diff --git a/wscript_build.py b/wscript_build.py
index 771ce5726c..5278f777eb 100644
--- a/wscript_build.py
+++ b/wscript_build.py
@@ -292,6 +292,7 @@ def build(ctx):
         ( "video/decode/vaapi.c",                "vaapi-hwaccel" ),
         ( "video/decode/vd_lavc.c" ),
         ( "video/decode/vda.c",                  "vda-hwaccel" ),
+        ( "video/decode/videotoolbox.c",         "videotoolbox-hwaccel" ),
         ( "video/decode/vdpau.c",                "vdpau-hwaccel" ),
         ( "video/decode/vdpau_old.c",            "vdpau-old-hwaccel" ),
         ( "video/filter/vf.c" ),
@@ -335,7 +336,7 @@ def build(ctx):
         ( "video/out/gl_hwdec.c",                "gl" ),
         ( "video/out/gl_hwdec_dxva2.c",          "gl-win32" ),
         ( "video/out/gl_hwdec_vaglx.c",          "vaapi-glx" ),
-        ( "video/out/gl_hwdec_vda.c",            "vda-gl" ),
+        ( "video/out/gl_hwdec_vda.c",            "videotoolbox-vda-gl" ),
         ( "video/out/gl_hwdec_vdpau.c",          "vdpau-gl-x11" ),
         ( "video/out/gl_lcms.c",                 "gl" ),
         ( "video/out/gl_osd.c",                  "gl" ),