vd_ffmpeg: autoselect output colorspaces without codecs.conf

Selecting the colorspace to output from a decoder is done in the
function mpcodecs_config_vo(). Add a new version of this function,
mpcodecs_config_vo2(), that allows the decoder to specify a list of
candidate colorspaces instead of always using a hardcoded list
specified in the codecs.conf entry. If the codecs.conf entry has any
"out" lines then those still take priority and the decoder-provided
list (if any) is ignored. Make vd_ffmpeg provide a list of the
colorspaces it's willing to output. Remove "out" lines from most
entries for libavcodec video decoders in codecs.conf, so that the
automatic values are now used instead.

1 files changed, 12 insertions, 1 deletions

diff --git a/libmpcodecs/vd_ffmpeg.c b/libmpcodecs/vd_ffmpeg.c
index 5a17bc243e..5460d20fd1 100644
--- a/libmpcodecs/vd_ffmpeg.c
+++ b/libmpcodecs/vd_ffmpeg.c
@@ -501,7 +501,18 @@ static int init_vo(sh_video_t *sh, enum PixelFormat pix_fmt){
         sh->disp_h = height;
         ctx->pix_fmt = pix_fmt;
         ctx->best_csp = pixfmt2imgfmt(pix_fmt);
-        if (!mpcodecs_config_vo(sh, sh->disp_w, sh->disp_h, ctx->best_csp))
+        const unsigned int *supported_fmts;
+        if (ctx->best_csp == IMGFMT_YV12)
+            supported_fmts = (const unsigned int[])
+                {IMGFMT_YV12, IMGFMT_I420, IMGFMT_IYUV, 0xffffffff};
+        else if (ctx->best_csp == IMGFMT_422P)
+            supported_fmts = (const unsigned int[])
+                {IMGFMT_422P, IMGFMT_YV12, IMGFMT_I420, IMGFMT_IYUV,
+                 0xffffffff};
+        else
+            supported_fmts = (const unsigned int[]){ctx->best_csp, 0xffffffff};
+        if (!mpcodecs_config_vo2(sh, sh->disp_w, sh->disp_h, supported_fmts,
+                                 ctx->best_csp))
             return -1;
         ctx->vo_initialized = 1;
     }