ad_lavc, vd_lavc: move mpv->lavc decoder parameter setup to common code

This can be useful in other contexts.

Note that we end up setting AVCodecContext.width/height instead of
coded_width/coded_height now. AVCodecParameters can't set coded_width,
but this is probably more correct anyway.

4 files changed, 79 insertions, 38 deletions

diff --git a/audio/decode/ad_lavc.c b/audio/decode/ad_lavc.c
index 820559a06b..7f3abfd612 100644
--- a/audio/decode/ad_lavc.c
+++ b/audio/decode/ad_lavc.c
@@ -122,19 +122,11 @@ static int init(struct dec_audio *da, const char *decoder)
 
     mp_set_avopts(da->log, lavc_context, opts->avopts);
 
-    lavc_context->codec_tag = c->codec_tag;
-    lavc_context->sample_rate = c->samplerate;
-    lavc_context->bit_rate = c->bitrate;
-    lavc_context->block_align = c->block_align;
-    lavc_context->bits_per_coded_sample = c->bits_per_coded_sample;
-    lavc_context->channels = c->channels.num;
-    if (!mp_chmap_is_unknown(&c->channels))
-        lavc_context->channel_layout = mp_chmap_to_lavc(&c->channels);
-
-    // demux_mkv
-    mp_lavc_set_extradata(lavc_context, c->extradata, c->extradata_size);
-
-    mp_set_lav_codec_headers(lavc_context, c);
+    if (mp_set_avctx_codec_headers(lavc_context, c) < 0) {
+        MP_ERR(da, "Could not set decoder parameters.\n");
+        uninit(da);
+        return 0;
+    }
 
     mp_set_avcodec_threads(da->log, lavc_context, opts->threads);
 
diff --git a/common/av_common.c b/common/av_common.c
index 84a416abf3..e8a0e76ed2 100644
--- a/common/av_common.c
+++ b/common/av_common.c
@@ -33,6 +33,7 @@
 #include "common/msg.h"
 #include "demux/packet.h"
 #include "demux/stheader.h"
+#include "video/fmt-conversion.h"
 #include "av_common.h"
 #include "codecs.h"
 
@@ -50,12 +51,73 @@ int mp_lavc_set_extradata(AVCodecContext *avctx, void *ptr, int size)
     return 0;
 }
 
-// This only copies ffmpeg-native codec parameters. Parameters produced by
-// other demuxers must be handled manually.
-void mp_set_lav_codec_headers(AVCodecContext *avctx, struct mp_codec_params *c)
+enum AVMediaType mp_to_av_stream_type(int type)
 {
-    if (c->lav_codecpar)
-        avcodec_parameters_to_context(avctx, c->lav_codecpar);
+    switch (type) {
+    case STREAM_VIDEO: return AVMEDIA_TYPE_VIDEO;
+    case STREAM_AUDIO: return AVMEDIA_TYPE_AUDIO;
+    case STREAM_SUB:   return AVMEDIA_TYPE_SUBTITLE;
+    default:           return AVMEDIA_TYPE_UNKNOWN;
+    }
+}
+
+AVCodecParameters *mp_codec_params_to_av(struct mp_codec_params *c)
+{
+    AVCodecParameters *avp = avcodec_parameters_alloc();
+    if (!avp)
+        return NULL;
+
+    // If we have lavf demuxer params, they overwrite by definition any others.
+    if (c->lav_codecpar) {
+        if (avcodec_parameters_copy(avp, c->lav_codecpar) < 0)
+            goto error;
+        return avp;
+    }
+
+    avp->codec_type = mp_to_av_stream_type(c->type);
+    avp->codec_id = mp_codec_to_av_codec_id(c->codec);
+    avp->codec_tag = c->codec_tag;
+    if (c->extradata_size) {
+        avp->extradata =
+            av_mallocz(c->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
+        if (!avp->extradata)
+            goto error;
+        avp->extradata_size = c->extradata_size;
+        memcpy(avp->extradata, c->extradata, avp->extradata_size);
+    }
+    avp->bits_per_coded_sample = c->bits_per_coded_sample;
+
+    // Video only
+    avp->width = c->disp_w;
+    avp->height = c->disp_h;
+    if (c->codec && strcmp(c->codec, "mp-rawvideo") == 0) {
+        avp->format = imgfmt2pixfmt(c->codec_tag);
+        avp->codec_tag = 0;
+    }
+
+    // Audio only
+    avp->sample_rate = c->samplerate;
+    avp->bit_rate = c->bitrate;
+    avp->block_align = c->block_align;
+    avp->channels = c->channels.num;
+    if (!mp_chmap_is_unknown(&c->channels))
+        avp->channel_layout = mp_chmap_to_lavc(&c->channels);
+
+    return avp;
+error:
+    avcodec_parameters_free(&avp);
+    return NULL;
+}
+
+// Set avctx codec headers for decoding. Returns <0 on failure.
+int mp_set_avctx_codec_headers(AVCodecContext *avctx, struct mp_codec_params *c)
+{
+    AVCodecParameters *avp = mp_codec_params_to_av(c);
+    if (!avp)
+        return -1;
+    int r = avcodec_parameters_to_context(avctx, avp) < 0 ? -1 : 0;
+    avcodec_parameters_free(&avp);
+    return r;
 }
 
 // Pick a "good" timebase, which will be used to convert double timestamps
diff --git a/common/av_common.h b/common/av_common.h
index 553c01a931..1d30fab71f 100644
--- a/common/av_common.h
+++ b/common/av_common.h
@@ -31,7 +31,9 @@ struct AVDictionary;
 struct mp_log;
 
 int mp_lavc_set_extradata(AVCodecContext *avctx, void *ptr, int size);
-void mp_set_lav_codec_headers(AVCodecContext *avctx, struct mp_codec_params *c);
+enum AVMediaType mp_to_av_stream_type(int type);
+AVCodecParameters *mp_codec_params_to_av(struct mp_codec_params *c);
+int mp_set_avctx_codec_headers(AVCodecContext *avctx, struct mp_codec_params *c);
 AVRational mp_get_codec_timebase(struct mp_codec_params *c);
 void mp_set_av_packet(AVPacket *dst, struct demux_packet *mpkt, AVRational *tb);
 int64_t mp_pts_to_av(double mp_pts, AVRational *tb);
diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index c4cfbc30d3..7e3c36ecfc 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -468,15 +468,12 @@ static void init_avctx(struct dec_video *vd, const char *decoder,
 {
     vd_ffmpeg_ctx *ctx = vd->priv;
     struct vd_lavc_params *lavc_param = vd->opts->vd_lavc_params;
-    bool mp_rawvideo = false;
     struct mp_codec_params *c = vd->codec;
 
     assert(!ctx->avctx);
 
-    if (strcmp(decoder, "mp-rawvideo") == 0) {
-        mp_rawvideo = true;
+    if (strcmp(decoder, "mp-rawvideo") == 0)
         decoder = "rawvideo";
-    }
 
     AVCodec *lavc_codec = avcodec_find_decoder_by_name(decoder);
     if (!lavc_codec)
@@ -536,23 +533,11 @@ static void init_avctx(struct dec_video *vd, const char *decoder,
     // Do this after the above avopt handling in case it changes values
     ctx->skip_frame = avctx->skip_frame;
 
-    avctx->codec_tag = c->codec_tag;
-    avctx->coded_width  = c->disp_w;
-    avctx->coded_height = c->disp_h;
-    avctx->bits_per_coded_sample = c->bits_per_coded_sample;
-
-    mp_lavc_set_extradata(avctx, c->extradata, c->extradata_size);
-
-    if (mp_rawvideo) {
-        avctx->pix_fmt = imgfmt2pixfmt(c->codec_tag);
-        avctx->codec_tag = 0;
-        if (avctx->pix_fmt == AV_PIX_FMT_NONE && c->codec_tag)
-            MP_ERR(vd, "Image format %s not supported by lavc.\n",
-                   mp_imgfmt_to_name(c->codec_tag));
+    if (mp_set_avctx_codec_headers(avctx, c) < 0) {
+        MP_ERR(vd, "Could not set codec parameters.\n");
+        goto error;
     }
 
-    mp_set_lav_codec_headers(avctx, c);
-
     /* open it */
     if (avcodec_open2(avctx, lavc_codec, NULL) < 0)
         goto error;