videotoolbox: make decoder format customizable

Because apparently there's no ideal universally working format.

The weird OpenGL texture format for kCVPixelFormatType_32BGRA is from:

http://stackoverflow.com/questions/22077544/draw-an-iosurface-to-an-opengl-context

(Which apparently got it from the linked Apple example code.)

3 files changed, 33 insertions, 8 deletions

diff --git a/video/out/opengl/hwdec.c b/video/out/opengl/hwdec.c
index 2a8c1eb14b..3c3a9785a6 100644
--- a/video/out/opengl/hwdec.c
+++ b/video/out/opengl/hwdec.c
@@ -55,6 +55,7 @@ static const struct gl_hwdec_driver *const mpgl_hwdec_drivers[] = {
 };
 
 static struct gl_hwdec *load_hwdec_driver(struct mp_log *log, GL *gl,
+                                          struct mpv_global *global,
                                           const struct gl_hwdec_driver *drv,
                                           bool is_auto)
 {
@@ -62,6 +63,7 @@ static struct gl_hwdec *load_hwdec_driver(struct mp_log *log, GL *gl,
     *hwdec = (struct gl_hwdec) {
         .driver = drv,
         .log = mp_log_new(hwdec, log, drv->api_name),
+        .global = global,
         .gl = gl,
         .gl_texture_target = GL_TEXTURE_2D,
         .probing = is_auto,
@@ -76,13 +78,13 @@ static struct gl_hwdec *load_hwdec_driver(struct mp_log *log, GL *gl,
 }
 
 struct gl_hwdec *gl_hwdec_load_api(struct mp_log *log, GL *gl,
-                                   const char *api_name)
+                                   struct mpv_global *g, const char *api_name)
 {
     bool is_auto = api_name && strcmp(api_name, "auto") == 0;
     for (int n = 0; mpgl_hwdec_drivers[n]; n++) {
         const struct gl_hwdec_driver *drv = mpgl_hwdec_drivers[n];
         if (is_auto || (api_name && strcmp(drv->api_name, api_name) == 0)) {
-            struct gl_hwdec *r = load_hwdec_driver(log, gl, drv, is_auto);
+            struct gl_hwdec *r = load_hwdec_driver(log, gl, g, drv, is_auto);
             if (r)
                 return r;
         }
@@ -91,9 +93,10 @@ struct gl_hwdec *gl_hwdec_load_api(struct mp_log *log, GL *gl,
 }
 
 // Like gl_hwdec_load_api(), but use HWDEC_... identifiers.
-struct gl_hwdec *gl_hwdec_load_api_id(struct mp_log *log, GL *gl, int id)
+struct gl_hwdec *gl_hwdec_load_api_id(struct mp_log *log, GL *gl,
+                                      struct mpv_global *g, int id)
 {
-    return gl_hwdec_load_api(log, gl, m_opt_choice_str(mp_hwdec_names, id));
+    return gl_hwdec_load_api(log, gl, g, m_opt_choice_str(mp_hwdec_names, id));
 }
 
 void gl_hwdec_uninit(struct gl_hwdec *hwdec)
diff --git a/video/out/opengl/hwdec.h b/video/out/opengl/hwdec.h
index 6f07536f57..ac1771c28d 100644
--- a/video/out/opengl/hwdec.h
+++ b/video/out/opengl/hwdec.h
@@ -9,6 +9,7 @@ struct mp_hwdec_info;
 struct gl_hwdec {
     const struct gl_hwdec_driver *driver;
     struct mp_log *log;
+    struct mpv_global *global;
     GL *gl;
     struct mp_hwdec_ctx *hwctx;
     // For free use by hwdec driver
@@ -45,8 +46,9 @@ struct gl_hwdec_driver {
 };
 
 struct gl_hwdec *gl_hwdec_load_api(struct mp_log *log, GL *gl,
-                                   const char *api_name);
-struct gl_hwdec *gl_hwdec_load_api_id(struct mp_log *log, GL *gl, int id);
+                                   struct mpv_global *g, const char *api_name);
+struct gl_hwdec *gl_hwdec_load_api_id(struct mp_log *log, GL *gl,
+                                      struct mpv_global *g, int id);
 
 void gl_hwdec_uninit(struct gl_hwdec *hwdec);
 
diff --git a/video/out/opengl/hwdec_osx.c b/video/out/opengl/hwdec_osx.c
index 84e8abca41..d1212e220f 100644
--- a/video/out/opengl/hwdec_osx.c
+++ b/video/out/opengl/hwdec_osx.c
@@ -26,6 +26,8 @@
 
 #include "video/mp_image_pool.h"
 #include "hwdec.h"
+#include "common/global.h"
+#include "options/options.h"
 
 struct vt_gl_plane_format {
     GLenum gl_format;
@@ -55,7 +57,23 @@ static struct vt_format vt_formats[] = {
             { GL_RED, GL_UNSIGNED_BYTE, GL_RED },
             { GL_RG,  GL_UNSIGNED_BYTE, GL_RG } ,
         }
-    }
+    },
+    {
+        .cvpixfmt = kCVPixelFormatType_422YpCbCr8,
+        .imgfmt = IMGFMT_UYVY,
+        .planes = 1,
+        .gl = {
+            { GL_RGB_422_APPLE, GL_UNSIGNED_SHORT_8_8_APPLE, GL_RGB }
+        }
+    },
+    {
+        .cvpixfmt = kCVPixelFormatType_32BGRA,
+        .imgfmt = IMGFMT_RGB0,
+        .planes = 1,
+        .gl = {
+            { GL_BGRA, GL_UNSIGNED_INT_8_8_8_8_REV, GL_RGBA }
+        }
+    },
 };
 
 static struct vt_format *vt_get_gl_format(uint32_t cvpixfmt)
@@ -134,7 +152,8 @@ static int create(struct gl_hwdec *hw)
         return -1;
 
     struct priv *p = talloc_zero(hw, struct priv);
-    struct vt_format *f = vt_get_gl_format_from_imgfmt(IMGFMT_NV12);
+    struct vt_format *f =
+        vt_get_gl_format_from_imgfmt(hw->global->opts->videotoolbox_format);
     if (!f)
         return -1;
 
@@ -143,6 +162,7 @@ static int create(struct gl_hwdec *hw)
     hw->hwctx = &p->hwctx;
     hw->hwctx->download_image = download_image;
     hw->hwctx->type = HWDEC_VIDEOTOOLBOX;
+    hw->hwctx->priv = (void *)(uintptr_t)f->cvpixfmt;
 
     hw->gl_texture_target = GL_TEXTURE_RECTANGLE;
     hw->gl->GenTextures(MP_MAX_PLANES, p->gl_planes);