vd_ffmpeg, demux_mng: allow general raw formats, fix MNG demuxer

Change vd_ffmpeg such that if sh_video->format is a mplayer pixel
format, and there's no other codec information, try to play it as raw
video. (The case of no codec information happens if the "generic" ffmpeg
decoder is instantiated, which is tried last. This means clashes with
actual existing formats are less likely.)

demux_mng did not initialize all fields of the bih, which made vd_ffmpeg
do invalid memory accesses when trying to copy the extradata. Also, use
IMGFMT_RGB32 instead of creating the FourCC directly. (They should be
the same, but what if mplayer changes the IMGFMT_* values.)

This also fixes demux_rawvideo.

2 files changed, 20 insertions, 9 deletions

diff --git a/libmpcodecs/vd_ffmpeg.c b/libmpcodecs/vd_ffmpeg.c
index d4d816c52b..b30648b6f6 100644
--- a/libmpcodecs/vd_ffmpeg.c
+++ b/libmpcodecs/vd_ffmpeg.c
@@ -125,7 +125,8 @@ static int init(sh_video_t *sh)
     struct lavc_param *lavc_param = &sh->opts->lavc_param;
     AVCodecContext *avctx;
     vd_ffmpeg_ctx *ctx;
-    AVCodec *lavc_codec;
+    AVCodec *lavc_codec = NULL;
+    enum PixelFormat rawfmt = PIX_FMT_NONE;
     int do_vis_debug = lavc_param->vismv ||
             (lavc_param->debug & (FF_DEBUG_VIS_MB_TYPE | FF_DEBUG_VIS_QP));
 
@@ -140,17 +141,22 @@ static int init(sh_video_t *sh)
             uninit(sh);
             return 0;
         }
-    } else if (!sh->libav_codec_id) {
-        mp_tmsg(MSGT_DECVIDEO, MSGL_INFO, "No Libav codec ID known. "
-                "Generic lavc decoder is not applicable.\n");
-        return 0;
-    } else {
+    } else if (sh->libav_codec_id) {
         lavc_codec = avcodec_find_decoder(sh->libav_codec_id);
         if (!lavc_codec) {
             mp_tmsg(MSGT_DECVIDEO, MSGL_INFO, "Libavcodec has no decoder "
                    "for this codec\n");
+            uninit(sh);
             return 0;
         }
+    } else if (!IMGFMT_IS_HWACCEL(sh->format)) {
+        rawfmt = imgfmt2pixfmt(sh->format);
+        if (rawfmt != PIX_FMT_NONE)
+            lavc_codec = avcodec_find_decoder_by_name("rawvideo");
+    }
+    if (!lavc_codec) {
+        uninit(sh);
+        return 0;
     }
 
     sh->codecname = lavc_codec->long_name;
@@ -230,7 +236,11 @@ static int init(sh_video_t *sh)
     if (lavc_param->gray)
         avctx->flags |= CODEC_FLAG_GRAY;
     avctx->flags2 |= lavc_param->fast;
-    avctx->codec_tag = sh->format;
+    if (rawfmt == PIX_FMT_NONE) {
+        avctx->codec_tag = sh->format;
+    } else {
+        avctx->pix_fmt = rawfmt;
+    }
     avctx->stream_codec_tag = sh->video.fccHandler;
     avctx->idct_algo = lavc_param->idct_algo;
     avctx->error_concealment = lavc_param->error_concealment;
diff --git a/libmpdemux/demux_mng.c b/libmpdemux/demux_mng.c
index d45add658f..ad08ba8c4b 100644
--- a/libmpdemux/demux_mng.c
+++ b/libmpdemux/demux_mng.c
@@ -31,6 +31,7 @@
 #include "stream/stream.h"
 #include "demuxer.h"
 #include "stheader.h"
+#include "libmpcodecs/img_format.h"
 
 #define MNG_SUPPORT_READ
 #define MNG_SUPPORT_DISPLAY
@@ -436,14 +437,14 @@ static demuxer_t * demux_mng_open(demuxer_t * demuxer)
     sh_video->ds = demuxer->video;
 
     // set format of pixels in video packets
-    sh_video->format = mmioFOURCC(32, 'B', 'G', 'R');
+    sh_video->format = IMGFMT_RGB32;
 
     // set framerate to some value (MNG does not have a fixed framerate)
     sh_video->fps       = 5.0f;
     sh_video->frametime = 1.0f / sh_video->fps;
 
     // set video frame parameters
-    sh_video->bih                = malloc(sizeof(*sh_video->bih));
+    sh_video->bih                = calloc(1, sizeof(*sh_video->bih));
     sh_video->bih->biCompression = sh_video->format;
     sh_video->bih->biWidth       = mng_priv->width;
     sh_video->bih->biHeight      = mng_priv->height;