videotoolbox: remove weird format-negotiation between VO and decoder

Originally, there was probably some sort of intention to restrict it to
formats supported by the interop, or something. But in the end it was
overcomplicated nonsense.

In the future, we could use mp_hwdec_ctx.supported_formats or other
mechanisms to handle this in a better way.

mp_hwdec_ctx.ctx is not set to a dummy pointer - hwdec_devices_load() is
only used to detect whether to vo_opengl interop is present, and the
common hwdec code expects that the .ctx field is not NULL.

This also changes videotoolbox-copy to use --videotoolbox-format,
instead of the FFmpeg-set default.

1 files changed, 10 insertions, 16 deletions

diff --git a/video/decode/hw_videotoolbox.c b/video/decode/hw_videotoolbox.c
index 8edffb5453..8d8c165a84 100644
--- a/video/decode/hw_videotoolbox.c
+++ b/video/decode/hw_videotoolbox.c
@@ -22,6 +22,7 @@
 
 #include "common/av_common.h"
 #include "common/msg.h"
+#include "options/options.h"
 #include "video/mp_image.h"
 #include "video/decode/lavc.h"
 #include "video/mp_image_pool.h"
@@ -96,10 +97,17 @@ static void print_videotoolbox_error(struct mp_log *log, int lev, char *message,
     mp_msg(log, lev, "%s: %d\n", message, error_code);
 }
 
-static int init_decoder_common(struct lavc_ctx *ctx, int w, int h, AVVideotoolboxContext *vtctx)
+static int init_decoder(struct lavc_ctx *ctx, int w, int h)
 {
     av_videotoolbox_default_free(ctx->avctx);
 
+    AVVideotoolboxContext *vtctx = av_videotoolbox_alloc_context();
+    if (!vtctx)
+        return -1;
+
+    vtctx->cv_pix_fmt_type =
+        mp_imgfmt_to_cvpixelformat(ctx->opts->videotoolbox_format);
+
     int err = av_videotoolbox_default_init2(ctx->avctx, vtctx);
     if (err < 0) {
         print_videotoolbox_error(ctx->log, MSGL_ERR, "failed to init videotoolbox decoder", err);
@@ -109,20 +117,6 @@ static int init_decoder_common(struct lavc_ctx *ctx, int w, int h, AVVideotoolbo
     return 0;
 }
 
-static int init_decoder(struct lavc_ctx *ctx, int w, int h)
-{
-    AVVideotoolboxContext *vtctx = av_videotoolbox_alloc_context();
-    struct mp_vt_ctx *vt = hwdec_devices_load(ctx->hwdec_devs, HWDEC_VIDEOTOOLBOX);
-    vtctx->cv_pix_fmt_type = vt->get_vt_fmt(vt);
-
-    return init_decoder_common(ctx, w, h, vtctx);
-}
-
-static int init_decoder_copy(struct lavc_ctx *ctx, int w, int h)
-{
-    return init_decoder_common(ctx, w, h, NULL);
-}
-
 static void uninit(struct lavc_ctx *ctx)
 {
     if (ctx->avctx)
@@ -179,7 +173,7 @@ const struct vd_lavc_hwdec mp_vd_lavc_videotoolbox_copy = {
     .probe = probe_copy,
     .init = init,
     .uninit = uninit,
-    .init_decoder = init_decoder_copy,
+    .init_decoder = init_decoder,
     .process_image = copy_image,
     .delay_queue = HWDEC_DELAY_QUEUE_COUNT,
 };