Fix extradata passing to lavc RV20 decoder

Pass video codec extradata unchanged from demux_real, sync vd_realvid to 
the new format
Sync mkv demuxer to the changes above (cmsg24 extradata was totally 
broken before)
Detect cmsg24 size from extradata (was fixed)
Based on a patch by elupus >> elupus >a< ecce se <<


git-svn-id: svn://svn.mplayerhq.hu/mplayer/trunk@20936 b3059339-0415-0410-9bf9-f77b7e298cf2

4 files changed, 50 insertions, 82 deletions

diff --git a/libmpcodecs/vd_ffmpeg.c b/libmpcodecs/vd_ffmpeg.c
index 19192f8669..64154aaff7 100644
--- a/libmpcodecs/vd_ffmpeg.c
+++ b/libmpcodecs/vd_ffmpeg.c
@@ -354,21 +354,20 @@ static int init(sh_video_t *sh){
     case mmioFOURCC('R', 'V', '2', '0'):
     case mmioFOURCC('R', 'V', '3', '0'):
     case mmioFOURCC('R', 'V', '4', '0'):
-        avctx->extradata_size= 8;
-        avctx->extradata = av_mallocz(avctx->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
-        if(sh->bih->biSize!=sizeof(*sh->bih)+8){
+        if(sh->bih->biSize<sizeof(*sh->bih)+8){
             /* only 1 packet per frame & sub_id from fourcc */
+            avctx->extradata_size= 8;
+            avctx->extradata = av_mallocz(avctx->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
 	    ((uint32_t*)avctx->extradata)[0] = 0;
-	    avctx->sub_id=
 	    ((uint32_t*)avctx->extradata)[1] =
         	(sh->format == mmioFOURCC('R', 'V', '1', '3')) ? 0x10003001 : 0x10000000;
         } else {
 	    /* has extra slice header (demux_rm or rm->avi streamcopy) */
-	    unsigned int* extrahdr=(unsigned int*)(sh->bih+1);
-	    ((uint32_t*)avctx->extradata)[0] = be2me_32(extrahdr[0]);
-	    avctx->sub_id= extrahdr[1];
-	    ((uint32_t*)avctx->extradata)[1] = be2me_32(extrahdr[1]);
+	    avctx->extradata_size = sh->bih->biSize-sizeof(BITMAPINFOHEADER);
+	    avctx->extradata = av_mallocz(avctx->extradata_size + FF_INPUT_BUFFER_PADDING_SIZE);
+	    memcpy(avctx->extradata, sh->bih+1, avctx->extradata_size);
 	}
+	avctx->sub_id= be2me_32(avctx->extradata+4);
 
 //        printf("%X %X %d %d\n", extrahdr[0], extrahdr[1]);
         break;
@@ -748,7 +747,7 @@ static mp_image_t* decode(sh_video_t *sh,void* data,int len,int flags){
        || sh->format == mmioFOURCC('R', 'V', '2', '0')
        || sh->format == mmioFOURCC('R', 'V', '3', '0')
        || sh->format == mmioFOURCC('R', 'V', '4', '0'))
-    if(sh->bih->biSize==sizeof(*sh->bih)+8){
+    if(sh->bih->biSize>=sizeof(*sh->bih)+8){
         int i;
         dp_hdr_t *hdr= (dp_hdr_t*)data;
 
diff --git a/libmpcodecs/vd_realvid.c b/libmpcodecs/vd_realvid.c
index 37e68bc4f8..2ef9733c8f 100644
--- a/libmpcodecs/vd_realvid.c
+++ b/libmpcodecs/vd_realvid.c
@@ -210,12 +210,17 @@ static int init(sh_video_t *sh){
 	char *path;
 	int result;
 	// we export codec id and sub-id from demuxer in bitmapinfohdr:
-	unsigned int* extrahdr=(unsigned int*)(sh->bih+1);
-	struct rv_init_t init_data={
-		11, sh->disp_w, sh->disp_h,0,0,extrahdr[0],
-		1,extrahdr[1]}; // rv30
+	unsigned char* extrahdr=(unsigned char*)(sh->bih+1);
+	unsigned int extrahdr_size = sh->bih->biSize - sizeof(BITMAPINFOHEADER);
+	struct rv_init_t init_data;
 
-	mp_msg(MSGT_DECVIDEO,MSGL_V,"realvideo codec id: 0x%08X  sub-id: 0x%08X\n",extrahdr[1],extrahdr[0]);
+	if(extrahdr_size < 8) {
+	    mp_msg(MSGT_DECVIDEO,MSGL_ERR,"realvideo: extradata too small (%u)\n", sh->bih->biSize - sizeof(BITMAPINFOHEADER));
+	    return 0;
+	}
+	init_data = (struct rv_init_t){11, sh->disp_w, sh->disp_h, 0, 0, be2me_32(((unsigned int*)extrahdr)[0]), 1, be2me_32(((unsigned int*)extrahdr)[1])}; // rv30
+
+	mp_msg(MSGT_DECVIDEO,MSGL_V,"realvideo codec id: 0x%08X  sub-id: 0x%08X\n",be2me_32(((unsigned int*)extrahdr)[1]),be2me_32(((unsigned int*)extrahdr)[0]));
 
 	path = malloc(strlen(REALCODEC_PATH)+strlen(sh->codec->dll)+2);
 	if (!path) return 0;
@@ -251,13 +256,21 @@ static int init(sh_video_t *sh){
 	    return 0;
 	}
 	// setup rv30 codec (codec sub-type and image dimensions):
-	if((sh->format<=0x30335652) && (extrahdr[1]>=0x20200002)){
-	    // We could read nonsense data while filling this, but input is big enough so no sig11
-	    uint32_t cmsg24[10]={sh->disp_w,sh->disp_h,((unsigned char *)extrahdr)[8]*4,((unsigned char *)extrahdr)[9]*4,
-	                        ((unsigned char *)extrahdr)[10]*4,((unsigned char *)extrahdr)[11]*4,
-	                        ((unsigned char *)extrahdr)[12]*4,((unsigned char *)extrahdr)[13]*4,
-	                        ((unsigned char *)extrahdr)[14]*4,((unsigned char *)extrahdr)[15]*4};
-	    cmsg_data_t cmsg_data={0x24,1+((extrahdr[0]>>16)&7), &cmsg24[0]};
+	if((sh->format<=0x30335652) && (be2me_32(((unsigned int*)extrahdr)[1])>=0x20200002)){
+	    int i, cmsg_cnt;
+	    uint32_t cmsg24[16]={sh->disp_w,sh->disp_h};
+	    cmsg_data_t cmsg_data={0x24,1+(extrahdr[1]&7), &cmsg24[0]};
+
+	    mp_msg(MSGT_DECVIDEO,MSGL_V,"realvideo: using cmsg24 with %u elements.\n",extrahdr[1]&7);
+	    cmsg_cnt = (extrahdr[1]&7)*2;
+	    if (extrahdr_size-8 < cmsg_cnt) {
+	        mp_msg(MSGT_DECVIDEO,MSGL_WARN,"realvideo: not enough extradata (%u) to make %u cmsg24 elements.\n",extrahdr_size-8,extrahdr[1]&7);
+	        cmsg_cnt = extrahdr_size-8;
+	    }
+	    for (i = 0; i < cmsg_cnt; i++)
+	        cmsg24[2+i] = extrahdr[8+i]*4;
+	    if (extrahdr_size-8 > cmsg_cnt)
+	        mp_msg(MSGT_DECVIDEO,MSGL_WARN,"realvideo: %u bytes of unknown extradata remaining.\n",extrahdr_size-8-cmsg_cnt);
 
 #ifdef USE_WIN32DLL
 	    if (dll_type == 1)
diff --git a/libmpdemux/demux_mkv.c b/libmpdemux/demux_mkv.c
index a72387253b..acbba13ce3 100644
--- a/libmpdemux/demux_mkv.c
+++ b/libmpdemux/demux_mkv.c
@@ -1869,12 +1869,14 @@ demux_mkv_open_video (demuxer_t *demuxer, mkv_track_t *track, int vid)
           unsigned char *dst, *src;
           real_video_props_t *rvp;
           uint32_t type2;
+          unsigned int cnt;
 
           rvp = (real_video_props_t *) track->private_data;
           src = (unsigned char *) (rvp + 1);
 
-          bih = realloc(bih, sizeof (BITMAPINFOHEADER)+12);
-          bih->biSize = 48;
+          cnt = track->private_size - sizeof (real_video_props_t);
+          bih = realloc(bih, sizeof (BITMAPINFOHEADER)+8+cnt);
+          bih->biSize = 48+cnt;
           bih->biPlanes = 1;
           type2 = be2me_32 (rvp->type2);
           if (type2 == 0x10003000 || type2 == 0x10003001)
@@ -1882,17 +1884,9 @@ demux_mkv_open_video (demuxer_t *demuxer, mkv_track_t *track, int vid)
           else
             bih->biCompression=mmioFOURCC('R','V',track->codec_id[9],'0');
           dst = (unsigned char *) (bih + 1);
-          ((unsigned int *) dst)[0] = be2me_32 (rvp->type1);
-          ((unsigned int *) dst)[1] = type2;
-
-          if (bih->biCompression <= 0x30335652 && type2 >= 0x20200002)
-            {
-              /* read secondary WxH for the cmsg24[] (see vd_realvid.c) */
-              ((unsigned short *)(bih+1))[4] = 4 * (unsigned short) src[0];
-              ((unsigned short *)(bih+1))[5] = 4 * (unsigned short) src[1];
-            }
-          else
-            memset(&dst[8], 0, 4);
+          ((unsigned int *) dst)[0] = rvp->type1;
+          ((unsigned int *) dst)[1] = rvp->type2;
+          stream_read(demuxer->stream, dst+8, cnt);
           track->realmedia = 1;
 
 #ifdef USE_QTX_CODECS
diff --git a/libmpdemux/demux_real.c b/libmpdemux/demux_real.c
index 2c7ddd8a44..64356ef73f 100644
--- a/libmpdemux/demux_real.c
+++ b/libmpdemux/demux_real.c
@@ -1514,9 +1514,9 @@ static demuxer_t* demux_open_real(demuxer_t* demuxer)
 		    mp_msg(MSGT_DEMUX,MSGL_V,"video fourcc: %.4s (%x)\n", (char *)&sh->format, sh->format);
 
 		    /* emulate BITMAPINFOHEADER */
-		    sh->bih = malloc(sizeof(BITMAPINFOHEADER)+16);
-		    memset(sh->bih, 0, sizeof(BITMAPINFOHEADER)+16);
-	    	    sh->bih->biSize = 48;
+		    sh->bih = malloc(sizeof(BITMAPINFOHEADER));
+		    memset(sh->bih, 0, sizeof(BITMAPINFOHEADER));
+	    	    sh->bih->biSize = sizeof(BITMAPINFOHEADER);
 		    sh->disp_w = sh->bih->biWidth = stream_read_word(demuxer->stream);
 		    sh->disp_h = sh->bih->biHeight = stream_read_word(demuxer->stream);
 		    sh->bih->biPlanes = 1;
@@ -1548,57 +1548,19 @@ static demuxer_t* demux_open_real(demuxer_t* demuxer)
 		    }
 		    stream_skip(demuxer->stream, 2);
 		    
-		    // read codec sub-format (to make difference between low and high rate codec)
-		    ((unsigned int*)(sh->bih+1))[0]=stream_read_dword(demuxer->stream);
-
-		    /* h263 hack */
-		    tmp = stream_read_dword(demuxer->stream);
-		    ((unsigned int*)(sh->bih+1))[1]=tmp;
-		    mp_msg(MSGT_DEMUX,MSGL_V,"H.263 ID: %x\n", tmp);
-		    switch (tmp)
 		    {
-			case 0x10000000:
-			    /* sub id: 0 */
-			    /* codec id: rv10 */
-			    break;
-			case 0x10003000:
-			case 0x10003001:
-			    /* sub id: 3 */
-			    /* codec id: rv10 */
-			    sh->bih->biCompression = sh->format = mmioFOURCC('R', 'V', '1', '3');
-			    break;
-			case 0x20001000:
-			case 0x20100001:
-			case 0x20200002:
-			    /* codec id: rv20 */
-			    break;
-			case 0x30202002:
-			    /* codec id: rv30 */
-			    break;
-			case 0x40000000:
-			    /* codec id: rv40 */
-			    break;
-			default:
-			    /* codec id: none */
-			    mp_msg(MSGT_DEMUX,MSGL_V,"unknown id: %x\n", tmp);
-		    }
-
-		    if((sh->format<=0x30335652) && (tmp>=0x20200002)){
-			    // read data for the cmsg24[] (see vd_realvid.c)
+			    // read and store codec extradata
 			    unsigned int cnt = codec_data_size - (stream_tell(demuxer->stream) - codec_pos);
-			    if (cnt < 2) {
-			        mp_msg(MSGT_DEMUX, MSGL_ERR,"realvid: cmsg24 data too short (size %u)\n", cnt);
+			    if (cnt > 0x7fffffff - sizeof(BITMAPINFOHEADER)) {
+			        mp_msg(MSGT_DEMUX, MSGL_ERR,"Extradata too big (%u)\n", cnt);
 			    } else  {
-			        int ii;
-			        if (cnt > 8) {
-			            mp_msg(MSGT_DEMUX, MSGL_WARN,"realvid: cmsg24 data too big, please report (size %u)\n", cnt);
-			            cnt = 8;
-			        }
-			        for (ii = 0; ii < cnt; ii++)
-			            ((unsigned char*)(sh->bih+1))[8+ii]=(unsigned short)stream_read_char(demuxer->stream);
+				sh->bih = realloc(sh->bih, sizeof(BITMAPINFOHEADER) + cnt);
 			        sh->bih->biSize += cnt;
+				stream_read(demuxer->stream, ((unsigned char*)(sh->bih+1)), cnt);
 			    }
 		    } 
+		    if(sh->format == 0x30315652 && ((unsigned char*)(sh->bih+1))[6] == 0x30)
+			    sh->bih->biCompression = sh->format = mmioFOURCC('R', 'V', '1', '3');
 		    
 		    /* Select video stream with highest bitrate if multirate file*/
 		    if (priv->is_multirate && ((demuxer->video->id == -1) ||