From de3f6718124372f64443a0b86527c1abf86d6f8c Mon Sep 17 00:00:00 2001 From: reimar Date: Sat, 5 Nov 2011 18:20:44 +0000 Subject: vd_ffmpeg, vf: fix crashes with some game formats Fixes for palette allocation handling. git-svn-id: svn://svn.mplayerhq.hu/mplayer/trunk@34304 b3059339-0415-0410-9bf9-f77b7e298cf2 This caused a crash with http://samples.ffmpeg.org/cdxl/fruit.cdxl if direct rendering was used. (Which is rarely these days.) With small changes: avoid av_freep() use, as this function is not sane. --- libmpcodecs/vd_ffmpeg.c | 3 --- libmpcodecs/vf.c | 10 ++++++++-- 2 files changed, 8 insertions(+), 5 deletions(-) (limited to 'libmpcodecs') diff --git a/libmpcodecs/vd_ffmpeg.c b/libmpcodecs/vd_ffmpeg.c index d36e4122a0..343a26d8d3 100644 --- a/libmpcodecs/vd_ffmpeg.c +++ b/libmpcodecs/vd_ffmpeg.c @@ -583,9 +583,6 @@ static void release_buffer(struct AVCodecContext *avctx, AVFrame *pic) } if (mpi) { - // Palette support: free palette buffer allocated in get_buffer - if (mpi->bpp == 8) - av_freep(&mpi->planes[1]); // release mpi (in case MPI_IMGTYPE_NUMBERED is used, e.g. for VDPAU) mpi->usage_count--; } diff --git a/libmpcodecs/vf.c b/libmpcodecs/vf.c index 5ec7d19570..8cf6002246 100644 --- a/libmpcodecs/vf.c +++ b/libmpcodecs/vf.c @@ -252,6 +252,8 @@ mp_image_t *vf_get_image(vf_instance_t *vf, unsigned int outfmt, break; } if (mpi) { + int missing_palette = !(mpi->flags & MP_IMGFLAG_RGB_PALETTE) && + (mp_imgflag & MP_IMGFLAG_RGB_PALETTE); mpi->type = mp_imgtype; mpi->w = vf->w; mpi->h = vf->h; @@ -263,11 +265,15 @@ mp_image_t *vf_get_image(vf_instance_t *vf, unsigned int outfmt, MP_IMGFLAG_DRAW_CALLBACK | MP_IMGFLAG_RGB_PALETTE); if (!vf->draw_slice) mpi->flags &= ~MP_IMGFLAG_DRAW_CALLBACK; - if (mpi->width != w2 || mpi->height != h) { + if (mpi->width != w2 || mpi->height != h || missing_palette) { if (mpi->flags & MP_IMGFLAG_ALLOCATED) { - if (mpi->width < w2 || mpi->height < h) { + if (mpi->width < w2 || mpi->height < h || missing_palette) { // need to re-allocate buffer memory: av_free(mpi->planes[0]); + if (mpi->flags & MP_IMGFLAG_RGB_PALETTE) + av_free(mpi->planes[1]); + for (int n = 0; n < MP_MAX_PLANES; n++) + mpi->planes[n] = NULL; mpi->flags &= ~MP_IMGFLAG_ALLOCATED; mp_msg(MSGT_VFILTER, MSGL_V, "vf.c: have to REALLOCATE buffer memory :(\n"); -- cgit v1.2.3