vd_ffmpeg: autoselect output colorspaces without codecs.conf

Selecting the colorspace to output from a decoder is done in the
function mpcodecs_config_vo(). Add a new version of this function,
mpcodecs_config_vo2(), that allows the decoder to specify a list of
candidate colorspaces instead of always using a hardcoded list
specified in the codecs.conf entry. If the codecs.conf entry has any
"out" lines then those still take priority and the decoder-provided
list (if any) is ignored. Make vd_ffmpeg provide a list of the
colorspaces it's willing to output. Remove "out" lines from most
entries for libavcodec video decoders in codecs.conf, so that the
automatic values are now used instead.

3 files changed, 36 insertions, 13 deletions

diff --git a/libmpcodecs/vd.c b/libmpcodecs/vd.c
index c54ae5260b..3ba72790a5 100644
--- a/libmpcodecs/vd.c
+++ b/libmpcodecs/vd.c
@@ -35,6 +35,7 @@
 
 #include "vd.h"
 #include "vf.h"
+#include "libvo/video_out.h"
 
 extern const vd_functions_t mpcodecs_vd_null;
 extern const vd_functions_t mpcodecs_vd_ffmpeg;
@@ -108,13 +109,12 @@ const vd_functions_t * const mpcodecs_vd_drivers[] = {
     NULL
 };
 
-#include "libvo/video_out.h"
-
-int mpcodecs_config_vo(sh_video_t *sh, int w, int h,
-                       unsigned int preferred_outfmt)
+int mpcodecs_config_vo2(sh_video_t *sh, int w, int h,
+                        const unsigned int *outfmts,
+                        unsigned int preferred_outfmt)
 {
     struct MPOpts *opts = sh->opts;
-    int i, j;
+    int j;
     unsigned int out_fmt = 0;
     int screen_size_x = 0;
     int screen_size_y = 0;
@@ -138,6 +138,10 @@ int mpcodecs_config_vo(sh_video_t *sh, int w, int h,
         // user wants postprocess but no pp filter yet:
         sh->vfilter = vf = vf_open_filter(opts, vf, "pp", NULL);
     }
+
+    if (!outfmts || sh->codec->outfmt[0] != 0xffffffff)
+        outfmts = sh->codec->outfmt;
+
     // check if libvo and codec has common outfmt (no conversion):
   csp_again:
 
@@ -150,11 +154,11 @@ int mpcodecs_config_vo(sh_video_t *sh, int w, int h,
     }
 
     j = -1;
-    for (i = 0; i < CODECS_MAX_OUTFMT; i++) {
+    for (int i = 0; i < CODECS_MAX_OUTFMT; i++) {
         int flags;
-        out_fmt = sh->codec->outfmt[i];
+        out_fmt = outfmts[i];
         if (out_fmt == (unsigned int) 0xFFFFFFFF)
-            continue;
+            break;
         flags = vf->query_format(vf, out_fmt);
         mp_msg(MSGT_CPLAYER, MSGL_DBG2,
                "vo_debug: query(%s) returned 0x%X (i=%d) \n",
@@ -234,7 +238,8 @@ int mpcodecs_config_vo(sh_video_t *sh, int w, int h,
         sh->vf_initialized = -1;
         return 0;               // failed
     }
-    out_fmt = sh->codec->outfmt[j];
+    out_fmt = outfmts[j];
+    sh->outfmt = out_fmt;
     mp_msg(MSGT_CPLAYER, MSGL_V, "VDec: using %s as output csp (no %d)\n",
            vo_format_name(out_fmt), j);
     sh->outfmtidx = j;
@@ -354,8 +359,7 @@ mp_image_t *mpcodecs_get_image(sh_video_t *sh, int mp_imgtype, int mp_imgflag,
                                int w, int h)
 {
     mp_image_t *mpi =
-        vf_get_image(sh->vfilter, sh->codec->outfmt[sh->outfmtidx], mp_imgtype,
-                     mp_imgflag, w, h);
+        vf_get_image(sh->vfilter, sh->outfmt, mp_imgtype, mp_imgflag, w, h);
     if (mpi)
         mpi->x = mpi->y = 0;
     return mpi;
diff --git a/libmpcodecs/vd.h b/libmpcodecs/vd.h
index 4615a9dc6e..76f3d00553 100644
--- a/libmpcodecs/vd.h
+++ b/libmpcodecs/vd.h
@@ -49,7 +49,15 @@ extern const vd_functions_t * const mpcodecs_vd_drivers[];
 #define VDCTRL_QUERY_UNSEEN_FRAMES 9 /* current decoder lag */
 
 // callbacks:
-int mpcodecs_config_vo(sh_video_t *sh, int w, int h, unsigned int preferred_outfmt);
+int mpcodecs_config_vo2(sh_video_t *sh, int w, int h,
+                        const unsigned int *outfmts,
+                        unsigned int preferred_outfmt);
+static inline int mpcodecs_config_vo(sh_video_t *sh, int w, int h,
+                                     unsigned int preferred_outfmt)
+{
+    return mpcodecs_config_vo2(sh, w, h, NULL, preferred_outfmt);
+}
+
 mp_image_t* mpcodecs_get_image(sh_video_t *sh, int mp_imgtype, int mp_imgflag, int w, int h);
 void mpcodecs_draw_slice(sh_video_t *sh, unsigned char** src, int* stride, int w,int h, int x, int y);
 
diff --git a/libmpcodecs/vd_ffmpeg.c b/libmpcodecs/vd_ffmpeg.c
index 5a17bc243e..5460d20fd1 100644
--- a/libmpcodecs/vd_ffmpeg.c
+++ b/libmpcodecs/vd_ffmpeg.c
@@ -501,7 +501,18 @@ static int init_vo(sh_video_t *sh, enum PixelFormat pix_fmt){
         sh->disp_h = height;
         ctx->pix_fmt = pix_fmt;
         ctx->best_csp = pixfmt2imgfmt(pix_fmt);
-        if (!mpcodecs_config_vo(sh, sh->disp_w, sh->disp_h, ctx->best_csp))
+        const unsigned int *supported_fmts;
+        if (ctx->best_csp == IMGFMT_YV12)
+            supported_fmts = (const unsigned int[])
+                {IMGFMT_YV12, IMGFMT_I420, IMGFMT_IYUV, 0xffffffff};
+        else if (ctx->best_csp == IMGFMT_422P)
+            supported_fmts = (const unsigned int[])
+                {IMGFMT_422P, IMGFMT_YV12, IMGFMT_I420, IMGFMT_IYUV,
+                 0xffffffff};
+        else
+            supported_fmts = (const unsigned int[]){ctx->best_csp, 0xffffffff};
+        if (!mpcodecs_config_vo2(sh, sh->disp_w, sh->disp_h, supported_fmts,
+                                 ctx->best_csp))
             return -1;
         ctx->vo_initialized = 1;
     }