4 files changed, 36 insertions, 6 deletions

diff --git a/video/decode/videotoolbox.c b/video/decode/videotoolbox.c
index 2867204ecd..2d2f5f735c 100644
--- a/video/decode/videotoolbox.c
+++ b/video/decode/videotoolbox.c
@@ -31,7 +31,7 @@ static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
                  const char *codec)
 {
     hwdec_request_api(info, "videotoolbox");
-    if (!info || !info->hwctx || info->hwctx->type != HWDEC_VIDEOTOOLBOX)
+    if (!info || !info->hwctx || !info->hwctx->get_vt_fmt)
         return HWDEC_ERR_NO_CTX;
     switch (mp_codec_to_av_codec_id(codec)) {
     case AV_CODEC_ID_H264:
@@ -88,9 +88,11 @@ static int init_decoder(struct lavc_ctx *ctx, int w, int h)
     av_videotoolbox_default_free(ctx->avctx);
 
     AVVideotoolboxContext *vtctx = av_videotoolbox_alloc_context();
-    vtctx->cv_pix_fmt_type = (uintptr_t)ctx->hwdec_info->hwctx->priv;
-    int err = av_videotoolbox_default_init2(ctx->avctx, vtctx);
 
+    struct mp_hwdec_ctx *hwctx = ctx->hwdec_info->hwctx;
+    vtctx->cv_pix_fmt_type = hwctx->get_vt_fmt(hwctx);
+
+    int err = av_videotoolbox_default_init2(ctx->avctx, vtctx);
     if (err < 0) {
         print_videotoolbox_error(ctx->log, MSGL_ERR, "failed to init videotoolbox decoder", err);
         return -1;
@@ -105,6 +107,15 @@ static void uninit(struct lavc_ctx *ctx)
         av_videotoolbox_default_free(ctx->avctx);
 }
 
+static struct mp_image *process_image(struct lavc_ctx *ctx, struct mp_image *img)
+{
+    if (img->imgfmt == IMGFMT_VIDEOTOOLBOX) {
+        CVPixelBufferRef pbuf = (CVPixelBufferRef)img->planes[3];
+        img->params.hw_subfmt = CVPixelBufferGetPixelFormatType(pbuf);
+    }
+    return img;
+}
+
 const struct vd_lavc_hwdec mp_vd_lavc_videotoolbox = {
     .type = HWDEC_VIDEOTOOLBOX,
     .image_format = IMGFMT_VIDEOTOOLBOX,
@@ -112,4 +123,5 @@ const struct vd_lavc_hwdec mp_vd_lavc_videotoolbox = {
     .init = init,
     .uninit = uninit,
     .init_decoder = init_decoder,
+    .process_image = process_image,
 };
diff --git a/video/hwdec.h b/video/hwdec.h
index 9585e32e25..898b035361 100644
--- a/video/hwdec.h
+++ b/video/hwdec.h
@@ -32,6 +32,7 @@ struct mp_hwdec_ctx {
     struct mp_vdpau_ctx *vdpau_ctx;
     struct mp_vaapi_ctx *vaapi_ctx;
     struct mp_d3d_ctx *d3d_ctx;
+    uint32_t (*get_vt_fmt)(struct mp_hwdec_ctx *ctx);
 
     // Optional.
     // Allocates a software image from the pool, downloads the hw image from
diff --git a/video/out/opengl/hwdec.h b/video/out/opengl/hwdec.h
index c04962dd76..5126d7f0fa 100644
--- a/video/out/opengl/hwdec.h
+++ b/video/out/opengl/hwdec.h
@@ -39,6 +39,7 @@ struct gl_hwdec_driver {
     // Prepare for rendering video. (E.g. create textures.)
     // Called on initialization, and every time the video size changes.
     // *params must be set to the format the hw textures return.
+    // This also can update hw->converted_imgfmt.
     int (*reinit)(struct gl_hwdec *hw, struct mp_image_params *params);
     // Return textures that contain a copy or reference of the given hw_image.
     int (*map_image)(struct gl_hwdec *hw, struct mp_image *hw_image,
diff --git a/video/out/opengl/hwdec_osx.c b/video/out/opengl/hwdec_osx.c
index 78d01b3894..3046768604 100644
--- a/video/out/opengl/hwdec_osx.c
+++ b/video/out/opengl/hwdec_osx.c
@@ -146,14 +146,21 @@ static bool check_hwdec(struct gl_hwdec *hw)
     return true;
 }
 
+static uint32_t get_vt_fmt(struct mp_hwdec_ctx *ctx)
+{
+    struct gl_hwdec *hw = ctx->priv;
+    struct vt_format *f =
+        vt_get_gl_format_from_imgfmt(hw->global->opts->videotoolbox_format);
+    return f ? f->cvpixfmt : (uint32_t)-1;
+}
+
 static int create(struct gl_hwdec *hw)
 {
     if (!check_hwdec(hw))
         return -1;
 
     struct priv *p = talloc_zero(hw, struct priv);
-    struct vt_format *f =
-        vt_get_gl_format_from_imgfmt(hw->global->opts->videotoolbox_format);
+    struct vt_format *f = vt_get_gl_format_from_imgfmt(IMGFMT_NV12);
     if (!f)
         return -1;
 
@@ -162,17 +169,26 @@ static int create(struct gl_hwdec *hw)
     hw->hwctx = &p->hwctx;
     hw->hwctx->download_image = download_image;
     hw->hwctx->type = HWDEC_VIDEOTOOLBOX;
-    hw->hwctx->priv = (void *)(uintptr_t)f->cvpixfmt;
+    hw->hwctx->get_vt_fmt = get_vt_fmt;
 
     hw->gl_texture_target = GL_TEXTURE_RECTANGLE;
     hw->gl->GenTextures(MP_MAX_PLANES, p->gl_planes);
 
+    hw->hwctx->priv = hw;
     return 0;
 }
 
 static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 {
     assert(params->imgfmt == hw->driver->imgfmt);
+
+    struct vt_format *f = vt_get_gl_format(params->hw_subfmt);
+    if (!f) {
+        MP_ERR(hw, "Unsupported CVPixelBuffer format.\n");
+        return -1;
+    }
+
+    hw->converted_imgfmt = f->imgfmt;
     return 0;
 }