Fix extradata passing to lavc RV20 decoder

Pass video codec extradata unchanged from demux_real, sync vd_realvid to 
the new format
Sync mkv demuxer to the changes above (cmsg24 extradata was totally 
broken before)
Detect cmsg24 size from extradata (was fixed)
Based on a patch by elupus >> elupus >a< ecce se <<


git-svn-id: svn://svn.mplayerhq.hu/mplayer/trunk@20936 b3059339-0415-0410-9bf9-f77b7e298cf2

2 files changed, 33 insertions, 21 deletions

diff --git a/libmpcodecs/vd_ffmpeg.c b/libmpcodecs/vd_ffmpeg.c
index 19192f8669..64154aaff7 100644
--- a/libmpcodecs/vd_ffmpeg.c
+++ b/libmpcodecs/vd_ffmpeg.c
@@ -354,21 +354,20 @@ static int init(sh_video_t *sh){
     case mmioFOURCC('R', 'V', '2', '0'):
     case mmioFOURCC('R', 'V', '3', '0'):
     case mmioFOURCC('R', 'V', '4', '0'):
-        avctx->extradata_size= 8;
-        avctx->extradata = av_mallocz(avctx->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
-        if(sh->bih->biSize!=sizeof(*sh->bih)+8){
+        if(sh->bih->biSize<sizeof(*sh->bih)+8){
             /* only 1 packet per frame & sub_id from fourcc */
+            avctx->extradata_size= 8;
+            avctx->extradata = av_mallocz(avctx->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
 	    ((uint32_t*)avctx->extradata)[0] = 0;
-	    avctx->sub_id=
 	    ((uint32_t*)avctx->extradata)[1] =
         	(sh->format == mmioFOURCC('R', 'V', '1', '3')) ? 0x10003001 : 0x10000000;
         } else {
 	    /* has extra slice header (demux_rm or rm->avi streamcopy) */
-	    unsigned int* extrahdr=(unsigned int*)(sh->bih+1);
-	    ((uint32_t*)avctx->extradata)[0] = be2me_32(extrahdr[0]);
-	    avctx->sub_id= extrahdr[1];
-	    ((uint32_t*)avctx->extradata)[1] = be2me_32(extrahdr[1]);
+	    avctx->extradata_size = sh->bih->biSize-sizeof(BITMAPINFOHEADER);
+	    avctx->extradata = av_mallocz(avctx->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
+	    memcpy(avctx->extradata, sh->bih+1, avctx->extradata_size);
 	}
+	avctx->sub_id= be2me_32(avctx->extradata+4);
 
 //        printf("%X %X %d %d\n", extrahdr[0], extrahdr[1]);
         break;
@@ -748,7 +747,7 @@ static mp_image_t* decode(sh_video_t *sh,void* data,int len,int flags){
        || sh->format == mmioFOURCC('R', 'V', '2', '0')
        || sh->format == mmioFOURCC('R', 'V', '3', '0')
        || sh->format == mmioFOURCC('R', 'V', '4', '0'))
-    if(sh->bih->biSize==sizeof(*sh->bih)+8){
+    if(sh->bih->biSize>=sizeof(*sh->bih)+8){
         int i;
         dp_hdr_t *hdr= (dp_hdr_t*)data;
 
diff --git a/libmpcodecs/vd_realvid.c b/libmpcodecs/vd_realvid.c
index 37e68bc4f8..2ef9733c8f 100644
--- a/libmpcodecs/vd_realvid.c
+++ b/libmpcodecs/vd_realvid.c
@@ -210,12 +210,17 @@ static int init(sh_video_t *sh){
 	char *path;
 	int result;
 	// we export codec id and sub-id from demuxer in bitmapinfohdr:
-	unsigned int* extrahdr=(unsigned int*)(sh->bih+1);
-	struct rv_init_t init_data={
-		11, sh->disp_w, sh->disp_h,0,0,extrahdr[0],
-		1,extrahdr[1]}; // rv30
+	unsigned char* extrahdr=(unsigned char*)(sh->bih+1);
+	unsigned int extrahdr_size = sh->bih->biSize - sizeof(BITMAPINFOHEADER);
+	struct rv_init_t init_data;
 
-	mp_msg(MSGT_DECVIDEO,MSGL_V,"realvideo codec id: 0x%08X  sub-id: 0x%08X\n",extrahdr[1],extrahdr[0]);
+	if(extrahdr_size < 8) {
+	    mp_msg(MSGT_DECVIDEO,MSGL_ERR,"realvideo: extradata too small (%u)\n", sh->bih->biSize - sizeof(BITMAPINFOHEADER));
+	    return 0;
+	}
+	init_data = (struct rv_init_t){11, sh->disp_w, sh->disp_h, 0, 0, be2me_32(((unsigned int*)extrahdr)[0]), 1, be2me_32(((unsigned int*)extrahdr)[1])}; // rv30
+
+	mp_msg(MSGT_DECVIDEO,MSGL_V,"realvideo codec id: 0x%08X  sub-id: 0x%08X\n",be2me_32(((unsigned int*)extrahdr)[1]),be2me_32(((unsigned int*)extrahdr)[0]));
 
 	path = malloc(strlen(REALCODEC_PATH)+strlen(sh->codec->dll)+2);
 	if (!path) return 0;
@@ -251,13 +256,21 @@ static int init(sh_video_t *sh){
 	    return 0;
 	}
 	// setup rv30 codec (codec sub-type and image dimensions):
-	if((sh->format<=0x30335652) && (extrahdr[1]>=0x20200002)){
-	    // We could read nonsense data while filling this, but input is big enough so no sig11
-	    uint32_t cmsg24[10]={sh->disp_w,sh->disp_h,((unsigned char *)extrahdr)[8]*4,((unsigned char *)extrahdr)[9]*4,
-	                        ((unsigned char *)extrahdr)[10]*4,((unsigned char *)extrahdr)[11]*4,
-	                        ((unsigned char *)extrahdr)[12]*4,((unsigned char *)extrahdr)[13]*4,
-	                        ((unsigned char *)extrahdr)[14]*4,((unsigned char *)extrahdr)[15]*4};
-	    cmsg_data_t cmsg_data={0x24,1+((extrahdr[0]>>16)&7), &cmsg24[0]};
+	if((sh->format<=0x30335652) && (be2me_32(((unsigned int*)extrahdr)[1])>=0x20200002)){
+	    int i, cmsg_cnt;
+	    uint32_t cmsg24[16]={sh->disp_w,sh->disp_h};
+	    cmsg_data_t cmsg_data={0x24,1+(extrahdr[1]&7), &cmsg24[0]};
+
+	    mp_msg(MSGT_DECVIDEO,MSGL_V,"realvideo: using cmsg24 with %u elements.\n",extrahdr[1]&7);
+	    cmsg_cnt = (extrahdr[1]&7)*2;
+	    if (extrahdr_size-8 < cmsg_cnt) {
+	        mp_msg(MSGT_DECVIDEO,MSGL_WARN,"realvideo: not enough extradata (%u) to make %u cmsg24 elements.\n",extrahdr_size-8,extrahdr[1]&7);
+	        cmsg_cnt = extrahdr_size-8;
+	    }
+	    for (i = 0; i < cmsg_cnt; i++)
+	        cmsg24[2+i] = extrahdr[8+i]*4;
+	    if (extrahdr_size-8 > cmsg_cnt)
+	        mp_msg(MSGT_DECVIDEO,MSGL_WARN,"realvideo: %u bytes of unknown extradata remaining.\n",extrahdr_size-8-cmsg_cnt);
 
 #ifdef USE_WIN32DLL
 	    if (dll_type == 1)