videotoolbox: change how videotoolbox format is managed

The underlying intention of this code is to make changing
--videotoolbox-format at runtime work. For this reason, the format can't
just be statically setup, but must be read from the option at runtime.

This means the format is not fixed anymore, and we have to make sure the
renderer is property reinitialized if the format changes. There is
currently no way to trigger reinit on this level, which is why the
mp_image_params.hw_subfmt field was introduced.

One sketchy thing remains: normally, the renderer is supposed to be
involved with VO format negotiation, which would ensure that the VO
can take the format at all. Since the hw_subfmt is not part of this
format negotiation, it's implied the get_vt_fmt() callback only
returns formats supported by the renderer. This is not necessarily
clear because vo_opengl checks this with converted_imgfmt separately.
None of this matters in practice though, because we know all formats
are always supported.

(This still requires somehow triggering decoder reinit to make the
change effective.)

1 files changed, 15 insertions, 3 deletions

diff --git a/video/decode/videotoolbox.c b/video/decode/videotoolbox.c
index 2867204ecd..2d2f5f735c 100644
--- a/video/decode/videotoolbox.c
+++ b/video/decode/videotoolbox.c
@@ -31,7 +31,7 @@ static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
                  const char *codec)
 {
     hwdec_request_api(info, "videotoolbox");
-    if (!info || !info->hwctx || info->hwctx->type != HWDEC_VIDEOTOOLBOX)
+    if (!info || !info->hwctx || !info->hwctx->get_vt_fmt)
         return HWDEC_ERR_NO_CTX;
     switch (mp_codec_to_av_codec_id(codec)) {
     case AV_CODEC_ID_H264:
@@ -88,9 +88,11 @@ static int init_decoder(struct lavc_ctx *ctx, int w, int h)
     av_videotoolbox_default_free(ctx->avctx);
 
     AVVideotoolboxContext *vtctx = av_videotoolbox_alloc_context();
-    vtctx->cv_pix_fmt_type = (uintptr_t)ctx->hwdec_info->hwctx->priv;
-    int err = av_videotoolbox_default_init2(ctx->avctx, vtctx);
 
+    struct mp_hwdec_ctx *hwctx = ctx->hwdec_info->hwctx;
+    vtctx->cv_pix_fmt_type = hwctx->get_vt_fmt(hwctx);
+
+    int err = av_videotoolbox_default_init2(ctx->avctx, vtctx);
     if (err < 0) {
         print_videotoolbox_error(ctx->log, MSGL_ERR, "failed to init videotoolbox decoder", err);
         return -1;
@@ -105,6 +107,15 @@ static void uninit(struct lavc_ctx *ctx)
         av_videotoolbox_default_free(ctx->avctx);
 }
 
+static struct mp_image *process_image(struct lavc_ctx *ctx, struct mp_image *img)
+{
+    if (img->imgfmt == IMGFMT_VIDEOTOOLBOX) {
+        CVPixelBufferRef pbuf = (CVPixelBufferRef)img->planes[3];
+        img->params.hw_subfmt = CVPixelBufferGetPixelFormatType(pbuf);
+    }
+    return img;
+}
+
 const struct vd_lavc_hwdec mp_vd_lavc_videotoolbox = {
     .type = HWDEC_VIDEOTOOLBOX,
     .image_format = IMGFMT_VIDEOTOOLBOX,
@@ -112,4 +123,5 @@ const struct vd_lavc_hwdec mp_vd_lavc_videotoolbox = {
     .init = init,
     .uninit = uninit,
     .init_decoder = init_decoder,
+    .process_image = process_image,
 };