From 72aac9ae8a0053e7c30199044cc2c9493a39b793 Mon Sep 17 00:00:00 2001 From: wm4 Date: Tue, 17 Jun 2014 22:43:43 +0200 Subject: video: introduce failure path for image allocations Until now, failure to allocate image data resulted in a crash (i.e. abort() was called). This was intentional, because it's pretty silly to degrade playback, and in almost all situations, the OOM will probably kill you anyway. (And then there's the standard Linux overcommit behavior, which also will kill you at some point.) But I changed my opinion, so here we go. This change does not affect _all_ memory allocations, just image data. Now in most failure cases, the output will just be skipped. For video filters, this coincidentally means that failure is treated as EOF (because the playback core assumes EOF if nothing comes out of the video filter chain). In other situations, output might be in some way degraded, like skipping frames, not scaling OSD, and such. Functions whose return values changed semantics: mp_image_alloc mp_image_new_copy mp_image_new_ref mp_image_make_writeable mp_image_setrefp mp_image_to_av_frame_and_unref mp_image_from_av_frame mp_image_new_external_ref mp_image_new_custom_ref mp_image_pool_make_writeable mp_image_pool_get mp_image_pool_new_copy mp_vdpau_mixed_frame_create vf_alloc_out_image vf_make_out_image_writeable glGetWindowScreenshot --- sub/draw_bmp.c | 19 ++++++++++++++++--- sub/img_convert.c | 34 ++++++++++++++++++++++++++-------- sub/osd.c | 7 ++----- video/decode/vd_lavc.c | 2 +- video/filter/vf.c | 8 +++++--- video/filter/vf.h | 2 +- video/filter/vf_divtc.c | 5 ++++- video/filter/vf_dlopen.c | 8 +++++++- video/filter/vf_eq.c | 6 ++++-- video/filter/vf_expand.c | 2 ++ video/filter/vf_ilpack.c | 2 ++ video/filter/vf_lavfi.c | 4 ++++ video/filter/vf_mirror.c | 2 ++ video/filter/vf_noise.c | 2 ++ video/filter/vf_phase.c | 2 ++ video/filter/vf_pp.c | 2 ++ video/filter/vf_pullup.c | 2 ++ video/filter/vf_scale.c | 2 ++ video/filter/vf_softpulldown.c | 15 +++++++++++++-- video/filter/vf_stereo3d.c | 2 ++ video/filter/vf_sub.c | 2 ++ video/filter/vf_vapoursynth.c | 3 ++- video/filter/vf_vdpaupp.c | 4 ++++ video/image_writer.c | 4 ++++ video/mp_image.c | 38 +++++++++++++++++++++++++++++--------- video/mp_image.h | 2 +- video/mp_image_pool.c | 22 +++++++++++++++++----- video/mp_image_pool.h | 4 +++- video/out/gl_common.c | 2 ++ video/out/gl_video.c | 17 +++++++++-------- video/out/vo.c | 2 +- video/out/vo.h | 2 +- video/out/vo_corevideo.c | 3 ++- video/out/vo_direct3d.c | 5 ++++- video/out/vo_image.c | 4 ++++ video/out/vo_lavc.c | 2 ++ video/out/vo_opengl_old.c | 2 ++ video/out/vo_sdl.c | 2 ++ video/out/vo_vaapi.c | 10 ++++++---- video/out/vo_vdpau.c | 3 +++ video/out/vo_x11.c | 3 +++ video/out/vo_xv.c | 3 +++ video/vaapi.c | 3 ++- video/vdpau.c | 10 ++++++---- video/vdpau_mixer.c | 6 ++++-- 45 files changed, 219 insertions(+), 67 deletions(-) diff --git a/sub/draw_bmp.c b/sub/draw_bmp.c index 0b24689313..02b7f9ec20 100644 --- a/sub/draw_bmp.c +++ b/sub/draw_bmp.c @@ -223,12 +223,18 @@ static void scale_sb_rgba(struct sub_bitmap *sb, struct mp_image *dst_format, sbisrc.planes[0] = sb->bitmap; sbisrc.stride[0] = sb->stride; struct mp_image *sbisrc2 = mp_image_alloc(IMGFMT_BGR32, sb->dw, sb->dh); - mp_image_swscale(sbisrc2, &sbisrc, SWS_BILINEAR); - struct mp_image *sba = mp_image_alloc(IMGFMT_Y8, sb->dw, sb->dh); + struct mp_image *sbi = mp_image_alloc(dst_format->imgfmt, sb->dw, sb->dh); + if (!sbisrc2 || !sba || !sbi) { + talloc_free(sbisrc2); + talloc_free(sba); + talloc_free(sbi); + return; + } + + mp_image_swscale(sbisrc2, &sbisrc, SWS_BILINEAR); unpremultiply_and_split_BGR32(sbisrc2, sba); - struct mp_image *sbi = mp_image_alloc(dst_format->imgfmt, sb->dw, sb->dh); sbi->params.colorspace = dst_format->params.colorspace; sbi->params.colorlevels = dst_format->params.colorlevels; mp_image_swscale(sbi, sbisrc2, SWS_BILINEAR); @@ -262,6 +268,9 @@ static void draw_rgba(struct mp_draw_sub_cache *cache, struct mp_rect bb, if (!(sbi && sba)) scale_sb_rgba(sb, temp, &sbi, &sba); + // on OOM, skip drawing + if (!(sbi && sba)) + continue; int bytes = (bits + 7) / 8; uint8_t *alpha_p = sba->planes[0] + src_y * sba->stride[0] + src_x; @@ -452,6 +461,8 @@ static struct mp_image *chroma_up(struct mp_draw_sub_cache *cache, int imgfmt, talloc_free(cache->upsample_img); cache->upsample_img = mp_image_alloc(imgfmt, src->w, src->h); talloc_steal(cache, cache->upsample_img); + if (!cache->upsample_img) + return NULL; } cache->upsample_temp = *cache->upsample_img; @@ -547,6 +558,8 @@ void mp_draw_sub_bitmaps(struct mp_draw_sub_cache **cache, struct mp_image *dst, struct mp_image dst_region = *dst; mp_image_crop_rc(&dst_region, bb); struct mp_image *temp = chroma_up(cache_, format, &dst_region); + if (!temp) + continue; // on OOM, skip region if (sbs->format == SUBBITMAP_RGBA) { draw_rgba(cache_, bb, temp, bits, sbs); diff --git a/sub/img_convert.c b/sub/img_convert.c index 8c42c5ed4b..a29db4a6f2 100644 --- a/sub/img_convert.c +++ b/sub/img_convert.c @@ -77,6 +77,12 @@ bool osd_conv_idx_to_rgba(struct osd_conv_cache *c, struct sub_bitmaps *imgs) *d = *s; struct mp_image *image = mp_image_alloc(IMGFMT_BGRA, s->w, s->h); talloc_steal(c->parts, image); + if (!image) { + // on OOM, skip the region by making it 0 sized + d->w = d->h = d->dw = d->dh = 0; + continue; + } + d->stride = image->stride[0]; d->bitmap = image->planes[0]; @@ -108,6 +114,8 @@ bool osd_conv_blur_rgba(struct osd_conv_cache *c, struct sub_bitmaps *imgs, int pad = 5; struct mp_image *temp = mp_image_alloc(IMGFMT_BGRA, s->w + pad * 2, s->h + pad * 2); + if (!temp) + continue; // on OOM, skip region memset_pic(temp->planes[0], 0, temp->w * 4, temp->h, temp->stride[0]); uint8_t *p0 = temp->planes[0] + pad * 4 + pad * temp->stride[0]; memcpy_pic(p0, s->bitmap, s->w * 4, s->h, temp->stride[0], s->stride); @@ -121,10 +129,15 @@ bool osd_conv_blur_rgba(struct osd_conv_cache *c, struct sub_bitmaps *imgs, d->h = d->dh = s->dh + pad * 2 * sy; struct mp_image *image = mp_image_alloc(IMGFMT_BGRA, d->w, d->h); talloc_steal(c->parts, image); - d->stride = image->stride[0]; - d->bitmap = image->planes[0]; - - mp_image_sw_blur_scale(image, temp, gblur); + if (image) { + d->stride = image->stride[0]; + d->bitmap = image->planes[0]; + + mp_image_sw_blur_scale(image, temp, gblur); + } else { + // on OOM, skip region + *d = *s; + } talloc_free(temp); } @@ -168,10 +181,15 @@ bool osd_scale_rgba(struct osd_conv_cache *c, struct sub_bitmaps *imgs) d->h = d->dh = s->dh; struct mp_image *image = mp_image_alloc(IMGFMT_BGRA, d->w, d->h); talloc_steal(c->parts, image); - d->stride = image->stride[0]; - d->bitmap = image->planes[0]; - - mp_image_swscale(image, &src_image, mp_sws_fast_flags); + if (image) { + d->stride = image->stride[0]; + d->bitmap = image->planes[0]; + + mp_image_swscale(image, &src_image, mp_sws_fast_flags); + } else { + // on OOM, skip the region; just don't scale it + *d = *s; + } } return true; } diff --git a/sub/osd.c b/sub/osd.c index 28a98892ce..e88df98afe 100644 --- a/sub/osd.c +++ b/sub/osd.c @@ -353,11 +353,8 @@ static void draw_on_image(void *ctx, struct sub_bitmaps *imgs) { struct draw_on_image_closure *closure = ctx; struct osd_state *osd = closure->osd; - if (closure->pool) { - mp_image_pool_make_writeable(closure->pool, closure->dest); - } else { - mp_image_make_writeable(closure->dest); - } + if (!mp_image_pool_make_writeable(closure->pool, closure->dest)) + return; // on OOM, skip mp_draw_sub_bitmaps(&osd->draw_cache, closure->dest, imgs); talloc_steal(osd, osd->draw_cache); closure->changed = true; diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c index 5af70aebe5..721461fdb7 100644 --- a/video/decode/vd_lavc.c +++ b/video/decode/vd_lavc.c @@ -628,7 +628,7 @@ static int decode(struct dec_video *vd, struct demux_packet *packet, struct mp_image *mpi = mp_image_from_av_frame(ctx->pic); av_frame_unref(ctx->pic); if (!mpi) - return 0; + return 0; // mpi==NULL, or OOM assert(mpi->planes[0] || mpi->planes[3]); mp_image_set_params(mpi, ¶ms); diff --git a/video/filter/vf.c b/video/filter/vf.c index 5326483444..045b68f774 100644 --- a/video/filter/vf.c +++ b/video/filter/vf.c @@ -200,17 +200,19 @@ struct mp_image *vf_alloc_out_image(struct vf_instance *vf) struct mp_image_params *p = &vf->fmt_out; assert(p->imgfmt); struct mp_image *img = mp_image_pool_get(vf->out_pool, p->imgfmt, p->w, p->h); - vf_fix_img_params(img, p); + if (img) + vf_fix_img_params(img, p); return img; } -void vf_make_out_image_writeable(struct vf_instance *vf, struct mp_image *img) +// Returns false on failure; then the image can't be written to. +bool vf_make_out_image_writeable(struct vf_instance *vf, struct mp_image *img) { struct mp_image_params *p = &vf->fmt_out; assert(p->imgfmt); assert(p->imgfmt == img->imgfmt); assert(p->w == img->w && p->h == img->h); - mp_image_pool_make_writeable(vf->out_pool, img); + return mp_image_pool_make_writeable(vf->out_pool, img); } //============================================================================ diff --git a/video/filter/vf.h b/video/filter/vf.h index bfe573e618..71dcb699c2 100644 --- a/video/filter/vf.h +++ b/video/filter/vf.h @@ -155,7 +155,7 @@ void vf_print_filter_chain(struct vf_chain *c, int msglevel, // Filter internal API struct mp_image *vf_alloc_out_image(struct vf_instance *vf); -void vf_make_out_image_writeable(struct vf_instance *vf, struct mp_image *img); +bool vf_make_out_image_writeable(struct vf_instance *vf, struct mp_image *img); void vf_add_output_frame(struct vf_instance *vf, struct mp_image *img); // default wrappers: diff --git a/video/filter/vf_divtc.c b/video/filter/vf_divtc.c index e93950ff6f..093aa2883b 100644 --- a/video/filter/vf_divtc.c +++ b/video/filter/vf_divtc.c @@ -221,6 +221,8 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi) mp_image_unrefp(&p->buffer); p->buffer = mp_image_alloc(mpi->imgfmt, mpi->w, mpi->h); talloc_steal(vf, p->buffer); + if (!p->buffer) + return NULL; // skip on OOM } struct mp_image *dmpi = p->buffer; @@ -317,7 +319,8 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi) if(p->deghost>0) { imgop(copyop, dmpi, mpi, 0); - vf_make_out_image_writeable(vf, mpi); + if (!vf_make_out_image_writeable(vf, mpi)) + return NULL; // oom: eof imgop(deghost_plane, mpi, dmpi, p->deghost); mpi->pts = vf_detc_adjust_pts(&p->ptsbuf, pts, 0, 0); diff --git a/video/filter/vf_dlopen.c b/video/filter/vf_dlopen.c index 9f9cd20318..78bde8eb64 100644 --- a/video/filter/vf_dlopen.c +++ b/video/filter/vf_dlopen.c @@ -160,8 +160,10 @@ static int config(struct vf_instance *vf, vf->priv->outpic[i] = mp_image_alloc(vf->priv->out_width, vf->priv->out_height, vf->priv->outfmt); - set_imgprop(&vf->priv->filter.outpic[i], vf->priv->outpic[i]); + if (!vf->priv->outpic[i]) + return 0; // OOM talloc_steal(vf, vf->priv->outpic[i]); + set_imgprop(&vf->priv->filter.outpic[i], vf->priv->outpic[i]); } return vf_next_config(vf, vf->priv->out_width, @@ -235,6 +237,10 @@ static int filter(struct vf_instance *vf, struct mp_image *mpi) for (int n = 0; n < vf->priv->out_cnt; n++) { out[n] = vf_alloc_out_image(vf); + if (!out[n]) { + talloc_free(mpi); + return -1; + } mp_image_copy_attributes(out[n], mpi); set_imgprop(&vf->priv->filter.outpic[n], out[n]); } diff --git a/video/filter/vf_eq.c b/video/filter/vf_eq.c index 80eefab9b9..3381f248f7 100644 --- a/video/filter/vf_eq.c +++ b/video/filter/vf_eq.c @@ -216,8 +216,10 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *src) } struct mp_image *new = vf_alloc_out_image(vf); - mp_image_copy(new, &dst); - mp_image_copy_attributes(new, &dst); + if (new) { + mp_image_copy(new, &dst); + mp_image_copy_attributes(new, &dst); + } talloc_free(src); return new; diff --git a/video/filter/vf_expand.c b/video/filter/vf_expand.c index ba636ee279..4b8e89c88c 100644 --- a/video/filter/vf_expand.c +++ b/video/filter/vf_expand.c @@ -115,6 +115,8 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi) return mpi; struct mp_image *dmpi = vf_alloc_out_image(vf); + if (!dmpi) + return NULL; mp_image_copy_attributes(dmpi, mpi); struct mp_image cropped = *dmpi; diff --git a/video/filter/vf_ilpack.c b/video/filter/vf_ilpack.c index e27b846735..157f02eb7c 100644 --- a/video/filter/vf_ilpack.c +++ b/video/filter/vf_ilpack.c @@ -116,6 +116,8 @@ static void ilpack(unsigned char *dst, unsigned char *src[3], static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi) { mp_image_t *dmpi = vf_alloc_out_image(vf); + if (!dmpi) + return NULL; mp_image_copy_attributes(dmpi, mpi); diff --git a/video/filter/vf_lavfi.c b/video/filter/vf_lavfi.c index 066afa28dd..c6f0783460 100644 --- a/video/filter/vf_lavfi.c +++ b/video/filter/vf_lavfi.c @@ -273,6 +273,8 @@ static AVFrame *mp_to_av(struct vf_instance *vf, struct mp_image *img) uint64_t pts = img->pts == MP_NOPTS_VALUE ? AV_NOPTS_VALUE : img->pts * av_q2d(av_inv_q(p->timebase_in)); AVFrame *frame = mp_image_to_av_frame_and_unref(img); + if (!frame) + return NULL; // OOM is (coincidentally) handled as EOF frame->pts = pts; frame->sample_aspect_ratio = p->par_in; return frame; @@ -282,6 +284,8 @@ static struct mp_image *av_to_mp(struct vf_instance *vf, AVFrame *av_frame) { struct vf_priv_s *p = vf->priv; struct mp_image *img = mp_image_from_av_frame(av_frame); + if (!img) + return NULL; // OOM img->pts = av_frame->pts == AV_NOPTS_VALUE ? MP_NOPTS_VALUE : av_frame->pts * av_q2d(p->timebase_out); av_frame_free(&av_frame); diff --git a/video/filter/vf_mirror.c b/video/filter/vf_mirror.c index 2cf9e4a78d..90316b2be9 100644 --- a/video/filter/vf_mirror.c +++ b/video/filter/vf_mirror.c @@ -58,6 +58,8 @@ static inline void mirror(uint8_t *dst, uint8_t *src, int bp, int w) static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi) { mp_image_t *dmpi = vf_alloc_out_image(vf); + if (!dmpi) + return NULL; mp_image_copy_attributes(dmpi, mpi); for (int p = 0; p < mpi->num_planes; p++) { diff --git a/video/filter/vf_noise.c b/video/filter/vf_noise.c index f612469b8d..025de86bfc 100644 --- a/video/filter/vf_noise.c +++ b/video/filter/vf_noise.c @@ -216,6 +216,8 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi) struct mp_image *dmpi = mpi; if (!mp_image_is_writeable(mpi)) { dmpi = vf_alloc_out_image(vf); + if (!dmpi) + return NULL; mp_image_copy_attributes(dmpi, mpi); } diff --git a/video/filter/vf_phase.c b/video/filter/vf_phase.c index d0b6e33312..5d2179d92a 100644 --- a/video/filter/vf_phase.c +++ b/video/filter/vf_phase.c @@ -203,6 +203,8 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi) enum mode mode; struct mp_image *dmpi = vf_alloc_out_image(vf); + if (!dmpi) + return NULL; mp_image_copy_attributes(dmpi, mpi); int pw[MP_MAX_PLANES] = {0}; diff --git a/video/filter/vf_pp.c b/video/filter/vf_pp.c index f60b8d8257..4acce1c272 100644 --- a/video/filter/vf_pp.c +++ b/video/filter/vf_pp.c @@ -89,6 +89,8 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi) struct mp_image *dmpi = mpi; if (!mp_image_is_writeable(mpi) || non_local) { dmpi = vf_alloc_out_image(vf); + if (!dmpi) + return NULL; mp_image_copy_attributes(dmpi, mpi); } diff --git a/video/filter/vf_pullup.c b/video/filter/vf_pullup.c index 7e04722f1c..2d131842e7 100644 --- a/video/filter/vf_pullup.c +++ b/video/filter/vf_pullup.c @@ -204,6 +204,8 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi) // safe thing and always copy. Code outside the filter might // hold a buffer reference even if the filter chain is destroyed. dmpi = vf_alloc_out_image(vf); + if (!dmpi) + return NULL; mp_image_copy_attributes(dmpi, mpi); struct mp_image data = *dmpi; diff --git a/video/filter/vf_scale.c b/video/filter/vf_scale.c index 0090eb775f..11c4044483 100644 --- a/video/filter/vf_scale.c +++ b/video/filter/vf_scale.c @@ -334,6 +334,8 @@ static int reconfig(struct vf_instance *vf, struct mp_image_params *in, static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi) { struct mp_image *dmpi = vf_alloc_out_image(vf); + if (!dmpi) + return NULL; mp_image_copy_attributes(dmpi, mpi); mp_sws_scale(vf->priv->sws, dmpi, mpi); diff --git a/video/filter/vf_softpulldown.c b/video/filter/vf_softpulldown.c index afa6bf31f0..cc2742ee72 100644 --- a/video/filter/vf_softpulldown.c +++ b/video/filter/vf_softpulldown.c @@ -63,6 +63,8 @@ static int filter(struct vf_instance *vf, struct mp_image *mpi) mp_image_unrefp(&p->buffer); p->buffer = mp_image_alloc(mpi->imgfmt, mpi->w, mpi->h); talloc_steal(vf, p->buffer); + if (!p->buffer) + goto failed; // OOM } mp_image_copy_attributes(p->buffer, mpi); @@ -83,6 +85,8 @@ static int filter(struct vf_instance *vf, struct mp_image *mpi) if (state == 0) { struct mp_image *new = mp_image_new_ref(mpi); + if (!new) + goto failed; new->pts = vf_softpulldown_adjust_pts(&vf->priv->ptsbuf, mpi->pts, 0, 0, vf->priv->last_frame_duration); vf_add_output_frame(vf, new); vf->priv->out++; @@ -93,13 +97,17 @@ static int filter(struct vf_instance *vf, struct mp_image *mpi) } else { copy_pic_field(dmpi, mpi, 1); struct mp_image *new = mp_image_new_ref(mpi); + if (!new) + goto failed; new->pts = vf_softpulldown_adjust_pts(&vf->priv->ptsbuf, mpi->pts, 0, 0, vf->priv->last_frame_duration); vf_add_output_frame(vf, new); vf->priv->out++; if (flags & MP_IMGFIELD_REPEAT_FIRST) { struct mp_image *new2 = mp_image_new_ref(mpi); - new2->pts = vf_softpulldown_adjust_pts(&vf->priv->ptsbuf, mpi->pts, 0, 0, 3); - vf_add_output_frame(vf, new2); + if (new2) { + new2->pts = vf_softpulldown_adjust_pts(&vf->priv->ptsbuf, mpi->pts, 0, 0, 3); + vf_add_output_frame(vf, new2); + } vf->priv->out++; vf->priv->state=0; } else { @@ -116,6 +124,9 @@ static int filter(struct vf_instance *vf, struct mp_image *mpi) talloc_free(mpi); return 0; +failed: + talloc_free(mpi); + return -1; } static int control(vf_instance_t *vf, int request, void *data) diff --git a/video/filter/vf_stereo3d.c b/video/filter/vf_stereo3d.c index 4126cd2f4e..15d2095f5d 100644 --- a/video/filter/vf_stereo3d.c +++ b/video/filter/vf_stereo3d.c @@ -294,6 +294,8 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi) vf->priv->in.off_right; struct mp_image *dmpi = vf_alloc_out_image(vf); + if (!dmpi) + return NULL; mp_image_copy_attributes(dmpi, mpi); out_off_left = vf->priv->out.row_left * dmpi->stride[0] + diff --git a/video/filter/vf_sub.c b/video/filter/vf_sub.c index 1f31d7b67f..e2d633dce7 100644 --- a/video/filter/vf_sub.c +++ b/video/filter/vf_sub.c @@ -96,6 +96,8 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi) if (vf->priv->opt_top_margin || vf->priv->opt_bottom_margin) { struct mp_image *dmpi = vf_alloc_out_image(vf); + if (!dmpi) + return NULL; mp_image_copy_attributes(dmpi, mpi); prepare_image(vf, dmpi, mpi); talloc_free(mpi); diff --git a/video/filter/vf_vapoursynth.c b/video/filter/vf_vapoursynth.c index 9d871c2173..8af3d02bb0 100644 --- a/video/filter/vf_vapoursynth.c +++ b/video/filter/vf_vapoursynth.c @@ -205,7 +205,8 @@ static void VS_CC vs_frame_done(void *userData, const VSFrameRef *f, int n, MP_ERR(vf, "No PTS after filter at frame %d!\n", n); res = mp_image_new_copy(&img); p->vsapi->freeFrame(f); - } else { + } + if (!res) { p->failed = true; MP_ERR(vf, "Filter error at frame %d: %s\n", n, errorMsg); } diff --git a/video/filter/vf_vdpaupp.c b/video/filter/vf_vdpaupp.c index e0349c0c21..05cabf7998 100644 --- a/video/filter/vf_vdpaupp.c +++ b/video/filter/vf_vdpaupp.c @@ -70,6 +70,8 @@ static VdpVideoSurface ref_field(struct vf_priv_s *p, if (!FIELD_VALID(p, pos)) return VDP_INVALID_HANDLE; struct mp_image *mpi = mp_image_new_ref(p->buffered[pos / 2]); + if (!mpi) + return VDP_INVALID_HANDLE; talloc_steal(frame, mpi); return (uintptr_t)mpi->planes[3]; } @@ -84,6 +86,8 @@ static bool output_field(struct vf_instance *vf, int pos) return false; struct mp_image *mpi = mp_vdpau_mixed_frame_create(p->buffered[pos / 2]); + if (!mpi) + return false; // skip output on OOM struct mp_vdpau_mixer_frame *frame = mp_vdpau_mixed_frame_get(mpi); frame->field = VDP_VIDEO_MIXER_PICTURE_STRUCTURE_FRAME; diff --git a/video/image_writer.c b/video/image_writer.c index f769fd9f27..c6da4ff7e5 100644 --- a/video/image_writer.c +++ b/video/image_writer.c @@ -289,6 +289,10 @@ int write_image(struct mp_image *image, const struct image_writer_opts *opts, // it's unclear what colorspace/levels the target wants if (image->imgfmt != destfmt || is_anamorphic) { struct mp_image *dst = mp_image_alloc(destfmt, d_w, d_h); + if (!dst) { + mp_err(log, "Out of memory.\n"); + return 0; + } mp_image_copy_attributes(dst, image); mp_image_swscale(dst, image, mp_sws_hq_flags); diff --git a/video/mp_image.c b/video/mp_image.c index 13b59b2c4e..dc123b7bf2 100644 --- a/video/mp_image.c +++ b/video/mp_image.c @@ -117,7 +117,7 @@ static bool m_refcount_is_unique(struct m_refcount *ref) return true; } -static void mp_image_alloc_planes(struct mp_image *mpi) +static bool mp_image_alloc_planes(struct mp_image *mpi) { assert(!mpi->planes[0]); @@ -141,12 +141,13 @@ static void mp_image_alloc_planes(struct mp_image *mpi) uint8_t *data = av_malloc(FFMAX(sum, 1)); if (!data) - abort(); //out of memory + return false; for (int n = 0; n < MP_MAX_PLANES; n++) { mpi->planes[n] = plane_size[n] ? data : NULL; data += plane_size[n]; } + return true; } void mp_image_setfmt(struct mp_image *mpi, int out_fmt) @@ -203,11 +204,14 @@ struct mp_image *mp_image_alloc(int imgfmt, int w, int h) { struct mp_image *mpi = talloc_zero(NULL, struct mp_image); talloc_set_destructor(mpi, mp_image_destructor); + mpi->refcount = m_refcount_new(); + mp_image_set_size(mpi, w, h); mp_image_setfmt(mpi, imgfmt); - mp_image_alloc_planes(mpi); - - mpi->refcount = m_refcount_new(); + if (!mp_image_alloc_planes(mpi)) { + talloc_free(mpi); + return NULL; + } mpi->refcount->free = av_free; mpi->refcount->arg = mpi->planes[0]; return mpi; @@ -216,6 +220,8 @@ struct mp_image *mp_image_alloc(int imgfmt, int w, int h) struct mp_image *mp_image_new_copy(struct mp_image *img) { struct mp_image *new = mp_image_alloc(img->imgfmt, img->w, img->h); + if (!new) + return NULL; mp_image_copy(new, img); mp_image_copy_attributes(new, img); @@ -265,6 +271,7 @@ struct mp_image *mp_image_new_ref(struct mp_image *img) // Return a reference counted reference to img. If the reference count reaches // 0, call free(free_arg). The data passed by img must not be free'd before // that. The new reference will be writeable. +// On allocation failure, unref the frame and return NULL. struct mp_image *mp_image_new_custom_ref(struct mp_image *img, void *free_arg, void (*free)(void *arg)) { @@ -275,6 +282,7 @@ struct mp_image *mp_image_new_custom_ref(struct mp_image *img, void *free_arg, // connect to an external refcounting API. It is assumed that the new object // has an initial reference to that external API. If free is given, that is // called after the last unref. All function pointers are optional. +// On allocation failure, unref the frame and return NULL. struct mp_image *mp_image_new_external_ref(struct mp_image *img, void *arg, void (*ref)(void *arg), void (*unref)(void *arg), @@ -304,15 +312,22 @@ bool mp_image_is_writeable(struct mp_image *img) // Make the image data referenced by img writeable. This allocates new data // if the data wasn't already writeable, and img->planes[] and img->stride[] // will be set to the copy. -void mp_image_make_writeable(struct mp_image *img) +// Returns success; if false is returned, the image could not be made writeable. +bool mp_image_make_writeable(struct mp_image *img) { if (mp_image_is_writeable(img)) - return; + return true; - mp_image_steal_data(img, mp_image_new_copy(img)); + struct mp_image *new = mp_image_new_copy(img); + if (!new) + return false; + mp_image_steal_data(img, new); assert(mp_image_is_writeable(img)); + return true; } +// Helper function: unrefs *p_img, and sets *p_img to a new ref of new_value. +// Only unrefs *p_img and sets it to NULL if out of memory. void mp_image_setrefp(struct mp_image **p_img, struct mp_image *new_value) { if (*p_img != new_value) { @@ -586,7 +601,7 @@ struct mp_image *mp_image_from_av_frame(struct AVFrame *av_frame) { AVFrame *new_ref = av_frame_clone(av_frame); if (!new_ref) - abort(); // OOM + return NULL; struct mp_image t = {0}; mp_image_copy_fields_from_av_frame(&t, new_ref); return mp_image_new_external_ref(&t, new_ref, NULL, NULL, frame_is_unique, @@ -604,10 +619,13 @@ static void free_img(void *opaque, uint8_t *data) // mp_image_from_av_frame(). It's done this way to allow marking the // resulting AVFrame as writeable if img is the only reference (in // other words, it's an optimization). +// On failure, img is only unreffed. struct AVFrame *mp_image_to_av_frame_and_unref(struct mp_image *img) { struct mp_image *new_ref = mp_image_new_ref(img); // ensure it's refcounted talloc_free(img); + if (!new_ref) + return NULL; AVFrame *frame = av_frame_alloc(); mp_image_copy_fields_to_av_frame(frame, new_ref); // Caveat: if img has shared references, and all other references disappear @@ -619,6 +637,8 @@ struct AVFrame *mp_image_to_av_frame_and_unref(struct mp_image *img) // Make it so that the actual image data is freed only if _all_ buffers // are unreferenced. struct mp_image *dummy_ref = mp_image_new_ref(new_ref); + if (!dummy_ref) + abort(); // out of memory (for the ref, not real image data) void *ptr = new_ref->planes[n]; size_t size = new_ref->stride[n] * new_ref->h; frame->buf[n] = av_buffer_create(ptr, size, free_img, dummy_ref, flags); diff --git a/video/mp_image.h b/video/mp_image.h index 9b2ef38c80..70e024bf7f 100644 --- a/video/mp_image.h +++ b/video/mp_image.h @@ -114,7 +114,7 @@ void mp_image_copy_attributes(struct mp_image *dmpi, struct mp_image *mpi); struct mp_image *mp_image_new_copy(struct mp_image *img); struct mp_image *mp_image_new_ref(struct mp_image *img); bool mp_image_is_writeable(struct mp_image *img); -void mp_image_make_writeable(struct mp_image *img); +bool mp_image_make_writeable(struct mp_image *img); void mp_image_setrefp(struct mp_image **p_img, struct mp_image *new_value); void mp_image_unrefp(struct mp_image **p_img); diff --git a/video/mp_image_pool.c b/video/mp_image_pool.c index c2d1fca080..62d55cc372 100644 --- a/video/mp_image_pool.c +++ b/video/mp_image_pool.c @@ -147,6 +147,7 @@ struct mp_image *mp_image_pool_get_no_alloc(struct mp_image_pool *pool, int fmt, // mp_image_alloc() is that there is a transparent mechanism to recycle image // data allocations through this pool. // The image can be free'd with talloc_free(). +// Returns NULL on OOM. struct mp_image *mp_image_pool_get(struct mp_image_pool *pool, int fmt, int w, int h) { @@ -171,24 +172,35 @@ struct mp_image *mp_image_pool_get(struct mp_image_pool *pool, int fmt, } // Like mp_image_new_copy(), but allocate the image out of the pool. +// Returns NULL on OOM. struct mp_image *mp_image_pool_new_copy(struct mp_image_pool *pool, struct mp_image *img) { struct mp_image *new = mp_image_pool_get(pool, img->imgfmt, img->w, img->h); - mp_image_copy(new, img); - mp_image_copy_attributes(new, img); + if (new) { + mp_image_copy(new, img); + mp_image_copy_attributes(new, img); + } return new; } // Like mp_image_make_writeable(), but if a copy has to be made, allocate it // out of the pool. -void mp_image_pool_make_writeable(struct mp_image_pool *pool, +// If pool==NULL, mp_image_make_writeable() is called (for convenience). +// Returns false on failure (see mp_image_make_writeable()). +bool mp_image_pool_make_writeable(struct mp_image_pool *pool, struct mp_image *img) { if (mp_image_is_writeable(img)) - return; - mp_image_steal_data(img, mp_image_pool_new_copy(pool, img)); + return true; + if (!pool) + return mp_image_make_writeable(img); + struct mp_image *new = mp_image_pool_new_copy(pool, img); + if (!new) + return false; + mp_image_steal_data(img, new); assert(mp_image_is_writeable(img)); + return true; } void mp_image_pool_set_allocator(struct mp_image_pool *pool, diff --git a/video/mp_image_pool.h b/video/mp_image_pool.h index d9ab2ee9e1..c2307da887 100644 --- a/video/mp_image_pool.h +++ b/video/mp_image_pool.h @@ -1,6 +1,8 @@ #ifndef MPV_MP_IMAGE_POOL_H #define MPV_MP_IMAGE_POOL_H +#include + struct mp_image_pool; struct mp_image_pool *mp_image_pool_new(int max_count); @@ -19,7 +21,7 @@ void mp_image_pool_set_allocator(struct mp_image_pool *pool, struct mp_image *mp_image_pool_new_copy(struct mp_image_pool *pool, struct mp_image *img); -void mp_image_pool_make_writeable(struct mp_image_pool *pool, +bool mp_image_pool_make_writeable(struct mp_image_pool *pool, struct mp_image *img); #endif diff --git a/video/out/gl_common.c b/video/out/gl_common.c index 243f442b4d..131a35daaf 100644 --- a/video/out/gl_common.c +++ b/video/out/gl_common.c @@ -833,6 +833,8 @@ mp_image_t *glGetWindowScreenshot(GL *gl) GLint vp[4]; //x, y, w, h gl->GetIntegerv(GL_VIEWPORT, vp); mp_image_t *image = mp_image_alloc(IMGFMT_RGB24, vp[2], vp[3]); + if (!image) + return NULL; gl->BindBuffer(GL_PIXEL_UNPACK_BUFFER, 0); gl->PixelStorei(GL_PACK_ALIGNMENT, 1); gl->PixelStorei(GL_PACK_ROW_LENGTH, 0); diff --git a/video/out/gl_video.c b/video/out/gl_video.c index 2b403b265d..b7a597654e 100644 --- a/video/out/gl_video.c +++ b/video/out/gl_video.c @@ -1727,7 +1727,7 @@ void gl_video_upload_image(struct gl_video *p, struct mp_image *mpi) if (p->hwdec_active) { mp_image_setrefp(&vimg->hwimage, mpi); - p->have_image = true; + p->have_image = !!vimg->hwimage; return; } @@ -1791,14 +1791,15 @@ struct mp_image *gl_video_download_image(struct gl_video *p) mp_image_t *image = mp_image_alloc(p->image_format, p->texture_w, p->texture_h); - - for (int n = 0; n < p->plane_count; n++) { - struct texplane *plane = &vimg->planes[n]; - gl->ActiveTexture(GL_TEXTURE0 + n); - glDownloadTex(gl, p->gl_target, plane->gl_format, plane->gl_type, - image->planes[n], image->stride[n]); + if (image) { + for (int n = 0; n < p->plane_count; n++) { + struct texplane *plane = &vimg->planes[n]; + gl->ActiveTexture(GL_TEXTURE0 + n); + glDownloadTex(gl, p->gl_target, plane->gl_format, plane->gl_type, + image->planes[n], image->stride[n]); + } + mp_image_set_attributes(image, &p->image_params); } - mp_image_set_attributes(image, &p->image_params); unset_image_textures(p); diff --git a/video/out/vo.c b/video/out/vo.c index 98601f0324..1af050f050 100644 --- a/video/out/vo.c +++ b/video/out/vo.c @@ -288,7 +288,7 @@ void vo_queue_image(struct vo *vo, struct mp_image *mpi) return; assert(mp_image_params_equals(vo->params, &mpi->params)); mpi = mp_image_new_ref(mpi); - if (vo->driver->filter_image) + if (vo->driver->filter_image && mpi) mpi = vo->driver->filter_image(vo, mpi); if (!mpi) { MP_ERR(vo, "Could not upload image.\n"); diff --git a/video/out/vo.h b/video/out/vo.h index 38b393a0a8..3d10171ed2 100644 --- a/video/out/vo.h +++ b/video/out/vo.h @@ -114,7 +114,7 @@ struct voctrl_screenshot_args { // The caller has to free the image with talloc_free(). // It is not specified whether the image data is a copy or references the // image data directly. - // Is never NULL. (Failure has to be indicated by returning VO_FALSE.) + // Can be NULL on failure. struct mp_image *out_image; // Whether the VO rendered OSD/subtitles into out_image bool has_osd; diff --git a/video/out/vo_corevideo.c b/video/out/vo_corevideo.c index 25555e3ee1..423fbcc6cb 100644 --- a/video/out/vo_corevideo.c +++ b/video/out/vo_corevideo.c @@ -287,7 +287,8 @@ static mp_image_t *get_screenshot(struct vo *vo, CVPixelBufferRef pbuf) img.stride[0] = stride; struct mp_image *image = mp_image_new_copy(&img); - mp_image_set_attributes(image, vo->params); + if (image) + mp_image_set_attributes(image, vo->params); CVPixelBufferUnlockBaseAddress(pbuf, 0); return image; diff --git a/video/out/vo_direct3d.c b/video/out/vo_direct3d.c index b34c1d49ef..c65620f1d2 100644 --- a/video/out/vo_direct3d.c +++ b/video/out/vo_direct3d.c @@ -1435,7 +1435,8 @@ static mp_image_t *get_screenshot(d3d_priv *priv) return NULL; struct mp_image *image = mp_image_new_copy(&buffer); - mp_image_set_attributes(image, priv->vo->params); + if (image) + mp_image_set_attributes(image, priv->vo->params); d3d_unlock_video_objects(priv); return image; @@ -1491,6 +1492,8 @@ static mp_image_t *get_window_screenshot(d3d_priv *priv) goto error_exit; image = mp_image_alloc(IMGFMT_BGR32, width, height); + if (!image) + goto error_exit; IDirect3DSurface9_LockRect(surface, &locked_rect, NULL, 0); diff --git a/video/out/vo_image.c b/video/out/vo_image.c index b732c417a9..db4337b97e 100644 --- a/video/out/vo_image.c +++ b/video/out/vo_image.c @@ -77,6 +77,8 @@ static void draw_image(struct vo *vo, mp_image_t *mpi) struct priv *p = vo->priv; mp_image_setrefp(&p->current, mpi); + if (!p->current) + return; struct mp_osd_res dim = osd_res_from_image_params(vo->params); osd_draw_on_image(vo->osd, dim, mpi->pts, OSD_DRAW_SUB_ONLY, p->current); @@ -85,6 +87,8 @@ static void draw_image(struct vo *vo, mp_image_t *mpi) static void flip_page(struct vo *vo) { struct priv *p = vo->priv; + if (!p->current) + return; (p->frame)++; diff --git a/video/out/vo_lavc.c b/video/out/vo_lavc.c index 7bc9c1a3b7..a387cd6457 100644 --- a/video/out/vo_lavc.c +++ b/video/out/vo_lavc.c @@ -468,6 +468,8 @@ static void draw_image_unlocked(struct vo *vo, mp_image_t *mpi) MP_INFO(vo, "Frame at pts %d got displayed %d times\n", (int) vc->lastframeipts, vc->lastdisplaycount); mp_image_setrefp(&vc->lastimg, mpi); + if (!vc->lastimg) + MP_ERR(vo, "Out of memory\n"); vc->lastimg_wants_osd = true; vc->lastframeipts = vc->lastipts = frameipts; diff --git a/video/out/vo_opengl_old.c b/video/out/vo_opengl_old.c index 3f9e4b0f38..16da97cc35 100644 --- a/video/out/vo_opengl_old.c +++ b/video/out/vo_opengl_old.c @@ -2015,6 +2015,8 @@ static mp_image_t *get_screenshot(struct vo *vo) mp_image_t *image = mp_image_alloc(p->image_format, p->texture_width, p->texture_height); + if (!image) + return NULL; glDownloadTex(gl, p->target, p->gl_format, p->gl_type, image->planes[0], image->stride[0]); diff --git a/video/out/vo_sdl.c b/video/out/vo_sdl.c index 57302b0a41..21b99dcf81 100644 --- a/video/out/vo_sdl.c +++ b/video/out/vo_sdl.c @@ -917,6 +917,8 @@ static struct mp_image *get_window_screenshot(struct vo *vo) struct priv *vc = vo->priv; struct mp_image *image = mp_image_alloc(vc->osd_format.mpv, vo->dwidth, vo->dheight); + if (!image) + return NULL; if (SDL_RenderReadPixels(vc->renderer, NULL, vc->osd_format.sdl, image->planes[0], image->stride[0])) { MP_ERR(vo, "SDL_RenderReadPixels failed\n"); diff --git a/video/out/vo_vaapi.c b/video/out/vo_vaapi.c index df96121137..c1c34370fd 100644 --- a/video/out/vo_vaapi.c +++ b/video/out/vo_vaapi.c @@ -196,10 +196,12 @@ static bool render_to_screen(struct priv *p, struct mp_image *mpi) p->black_surface = mp_image_pool_get(p->pool, IMGFMT_VAAPI, w, h); if (p->black_surface) { struct mp_image *img = mp_image_alloc(fmt, w, h); - mp_image_clear(img, 0, 0, w, h); - if (va_surface_upload(p->black_surface, img) < 0) - mp_image_unrefp(&p->black_surface); - talloc_free(img); + if (img) { + mp_image_clear(img, 0, 0, w, h); + if (va_surface_upload(p->black_surface, img) < 0) + mp_image_unrefp(&p->black_surface); + talloc_free(img); + } } } surface = va_surface_id(p->black_surface); diff --git a/video/out/vo_vdpau.c b/video/out/vo_vdpau.c index 406ed5f9ed..9e5807e320 100644 --- a/video/out/vo_vdpau.c +++ b/video/out/vo_vdpau.c @@ -883,6 +883,9 @@ static struct mp_image *read_output_surface(struct vo *vo, return NULL; struct mp_image *image = mp_image_alloc(IMGFMT_BGR32, width, height); + if (!image) + return NULL; + image->params.colorspace = MP_CSP_RGB; // hardcoded with conv. matrix image->params.colorlevels = vo->params->outputlevels; diff --git a/video/out/vo_x11.c b/video/out/vo_x11.c index 543e762fce..ab188bee6f 100644 --- a/video/out/vo_x11.c +++ b/video/out/vo_x11.c @@ -490,6 +490,9 @@ static int redraw_frame(struct vo *vo) { struct priv *p = vo->priv; + if (!p->original_image) + return false; + draw_image(vo, p->original_image); return true; } diff --git a/video/out/vo_xv.c b/video/out/vo_xv.c index e353dc27ff..870c3a1082 100644 --- a/video/out/vo_xv.c +++ b/video/out/vo_xv.c @@ -679,6 +679,9 @@ static int redraw_frame(struct vo *vo) { struct xvctx *ctx = vo->priv; + if (!ctx->original_image) + return false; + draw_image(vo, ctx->original_image); return true; } diff --git a/video/vaapi.c b/video/vaapi.c index 56bffaa05f..212ed4ff73 100644 --- a/video/vaapi.c +++ b/video/vaapi.c @@ -384,7 +384,8 @@ static struct mp_image *try_download(struct mp_image *src, if (va_image_map(p->ctx, image, &tmp)) { dst = pool ? mp_image_pool_get(pool, tmp.imgfmt, tmp.w, tmp.h) : mp_image_alloc(tmp.imgfmt, tmp.w, tmp.h); - mp_image_copy(dst, &tmp); + if (dst) + mp_image_copy(dst, &tmp); va_image_unmap(p->ctx, image); } return dst; diff --git a/video/vdpau.c b/video/vdpau.c index 0cf91ffe86..90aba50e63 100644 --- a/video/vdpau.c +++ b/video/vdpau.c @@ -184,10 +184,12 @@ static struct mp_image *create_ref(struct mp_vdpau_ctx *ctx, int index) struct mp_image *res = mp_image_new_custom_ref(&(struct mp_image){0}, ref, release_decoder_surface); - mp_image_setfmt(res, e->rgb ? IMGFMT_VDPAU_OUTPUT : IMGFMT_VDPAU); - mp_image_set_size(res, e->w, e->h); - res->planes[0] = (void *)"dummy"; // must be non-NULL, otherwise arbitrary - res->planes[3] = (void *)(intptr_t)(e->rgb ? e->osurface : e->surface); + if (res) { + mp_image_setfmt(res, e->rgb ? IMGFMT_VDPAU_OUTPUT : IMGFMT_VDPAU); + mp_image_set_size(res, e->w, e->h); + res->planes[0] = (void *)"dummy"; // must be non-NULL, otherwise arbitrary + res->planes[3] = (void *)(intptr_t)(e->rgb ? e->osurface : e->surface); + } return res; } diff --git a/video/vdpau_mixer.c b/video/vdpau_mixer.c index 1a53374622..2d60f32912 100644 --- a/video/vdpau_mixer.c +++ b/video/vdpau_mixer.c @@ -45,8 +45,10 @@ struct mp_image *mp_vdpau_mixed_frame_create(struct mp_image *base) frame->field = VDP_VIDEO_MIXER_PICTURE_STRUCTURE_FRAME; struct mp_image *mpi = mp_image_new_custom_ref(base, frame, free_mixed_frame); - mpi->planes[2] = (void *)frame; - mpi->planes[3] = (void *)(uintptr_t)VDP_INVALID_HANDLE; + if (mpi) { + mpi->planes[2] = (void *)frame; + mpi->planes[3] = (void *)(uintptr_t)VDP_INVALID_HANDLE; + } return mpi; } -- cgit v1.2.3