video: introduce failure path for image allocations

Until now, failure to allocate image data resulted in a crash (i.e.
abort() was called). This was intentional, because it's pretty silly to
degrade playback, and in almost all situations, the OOM will probably
kill you anyway. (And then there's the standard Linux overcommit
behavior, which also will kill you at some point.)

But I changed my opinion, so here we go. This change does not affect
_all_ memory allocations, just image data. Now in most failure cases,
the output will just be skipped. For video filters, this coincidentally
means that failure is treated as EOF (because the playback core assumes
EOF if nothing comes out of the video filter chain). In other
situations, output might be in some way degraded, like skipping frames,
not scaling OSD, and such.

Functions whose return values changed semantics:

  mp_image_alloc
  mp_image_new_copy
  mp_image_new_ref
  mp_image_make_writeable
  mp_image_setrefp
  mp_image_to_av_frame_and_unref
  mp_image_from_av_frame
  mp_image_new_external_ref
  mp_image_new_custom_ref
  mp_image_pool_make_writeable
  mp_image_pool_get
  mp_image_pool_new_copy
  mp_vdpau_mixed_frame_create
  vf_alloc_out_image
  vf_make_out_image_writeable
  glGetWindowScreenshot

14 files changed, 44 insertions, 16 deletions

diff --git a/video/out/gl_common.c b/video/out/gl_common.c
index 243f442b4d..131a35daaf 100644
--- a/video/out/gl_common.c
+++ b/video/out/gl_common.c
@@ -833,6 +833,8 @@ mp_image_t *glGetWindowScreenshot(GL *gl)
     GLint vp[4]; //x, y, w, h
     gl->GetIntegerv(GL_VIEWPORT, vp);
     mp_image_t *image = mp_image_alloc(IMGFMT_RGB24, vp[2], vp[3]);
+    if (!image)
+        return NULL;
     gl->BindBuffer(GL_PIXEL_UNPACK_BUFFER, 0);
     gl->PixelStorei(GL_PACK_ALIGNMENT, 1);
     gl->PixelStorei(GL_PACK_ROW_LENGTH, 0);
diff --git a/video/out/gl_video.c b/video/out/gl_video.c
index 2b403b265d..b7a597654e 100644
--- a/video/out/gl_video.c
+++ b/video/out/gl_video.c
@@ -1727,7 +1727,7 @@ void gl_video_upload_image(struct gl_video *p, struct mp_image *mpi)
 
     if (p->hwdec_active) {
         mp_image_setrefp(&vimg->hwimage, mpi);
-        p->have_image = true;
+        p->have_image = !!vimg->hwimage;
         return;
     }
 
@@ -1791,14 +1791,15 @@ struct mp_image *gl_video_download_image(struct gl_video *p)
 
     mp_image_t *image = mp_image_alloc(p->image_format, p->texture_w,
                                                         p->texture_h);
-
-    for (int n = 0; n < p->plane_count; n++) {
-        struct texplane *plane = &vimg->planes[n];
-        gl->ActiveTexture(GL_TEXTURE0 + n);
-        glDownloadTex(gl, p->gl_target, plane->gl_format, plane->gl_type,
-                      image->planes[n], image->stride[n]);
+    if (image) {
+        for (int n = 0; n < p->plane_count; n++) {
+            struct texplane *plane = &vimg->planes[n];
+            gl->ActiveTexture(GL_TEXTURE0 + n);
+            glDownloadTex(gl, p->gl_target, plane->gl_format, plane->gl_type,
+                          image->planes[n], image->stride[n]);
+        }
+        mp_image_set_attributes(image, &p->image_params);
     }
-    mp_image_set_attributes(image, &p->image_params);
 
     unset_image_textures(p);
 
diff --git a/video/out/vo.c b/video/out/vo.c
index 98601f0324..1af050f050 100644
--- a/video/out/vo.c
+++ b/video/out/vo.c
@@ -288,7 +288,7 @@ void vo_queue_image(struct vo *vo, struct mp_image *mpi)
         return;
     assert(mp_image_params_equals(vo->params, &mpi->params));
     mpi = mp_image_new_ref(mpi);
-    if (vo->driver->filter_image)
+    if (vo->driver->filter_image && mpi)
         mpi = vo->driver->filter_image(vo, mpi);
     if (!mpi) {
         MP_ERR(vo, "Could not upload image.\n");
diff --git a/video/out/vo.h b/video/out/vo.h
index 38b393a0a8..3d10171ed2 100644
--- a/video/out/vo.h
+++ b/video/out/vo.h
@@ -114,7 +114,7 @@ struct voctrl_screenshot_args {
     // The caller has to free the image with talloc_free().
     // It is not specified whether the image data is a copy or references the
     // image data directly.
-    // Is never NULL. (Failure has to be indicated by returning VO_FALSE.)
+    // Can be NULL on failure.
     struct mp_image *out_image;
     // Whether the VO rendered OSD/subtitles into out_image
     bool has_osd;
diff --git a/video/out/vo_corevideo.c b/video/out/vo_corevideo.c
index 25555e3ee1..423fbcc6cb 100644
--- a/video/out/vo_corevideo.c
+++ b/video/out/vo_corevideo.c
@@ -287,7 +287,8 @@ static mp_image_t *get_screenshot(struct vo *vo, CVPixelBufferRef pbuf)
     img.stride[0] = stride;
 
     struct mp_image *image = mp_image_new_copy(&img);
-    mp_image_set_attributes(image, vo->params);
+    if (image)
+        mp_image_set_attributes(image, vo->params);
     CVPixelBufferUnlockBaseAddress(pbuf, 0);
 
     return image;
diff --git a/video/out/vo_direct3d.c b/video/out/vo_direct3d.c
index b34c1d49ef..c65620f1d2 100644
--- a/video/out/vo_direct3d.c
+++ b/video/out/vo_direct3d.c
@@ -1435,7 +1435,8 @@ static mp_image_t *get_screenshot(d3d_priv *priv)
         return NULL;
 
     struct mp_image *image = mp_image_new_copy(&buffer);
-    mp_image_set_attributes(image, priv->vo->params);
+    if (image)
+        mp_image_set_attributes(image, priv->vo->params);
 
     d3d_unlock_video_objects(priv);
     return image;
@@ -1491,6 +1492,8 @@ static mp_image_t *get_window_screenshot(d3d_priv *priv)
         goto error_exit;
 
     image = mp_image_alloc(IMGFMT_BGR32, width, height);
+    if (!image)
+        goto error_exit;
 
     IDirect3DSurface9_LockRect(surface, &locked_rect, NULL, 0);
 
diff --git a/video/out/vo_image.c b/video/out/vo_image.c
index b732c417a9..db4337b97e 100644
--- a/video/out/vo_image.c
+++ b/video/out/vo_image.c
@@ -77,6 +77,8 @@ static void draw_image(struct vo *vo, mp_image_t *mpi)
     struct priv *p = vo->priv;
 
     mp_image_setrefp(&p->current, mpi);
+    if (!p->current)
+        return;
 
     struct mp_osd_res dim = osd_res_from_image_params(vo->params);
     osd_draw_on_image(vo->osd, dim, mpi->pts, OSD_DRAW_SUB_ONLY, p->current);
@@ -85,6 +87,8 @@ static void draw_image(struct vo *vo, mp_image_t *mpi)
 static void flip_page(struct vo *vo)
 {
     struct priv *p = vo->priv;
+    if (!p->current)
+        return;
 
     (p->frame)++;
 
diff --git a/video/out/vo_lavc.c b/video/out/vo_lavc.c
index 7bc9c1a3b7..a387cd6457 100644
--- a/video/out/vo_lavc.c
+++ b/video/out/vo_lavc.c
@@ -468,6 +468,8 @@ static void draw_image_unlocked(struct vo *vo, mp_image_t *mpi)
                 MP_INFO(vo, "Frame at pts %d got displayed %d times\n",
                         (int) vc->lastframeipts, vc->lastdisplaycount);
             mp_image_setrefp(&vc->lastimg, mpi);
+            if (!vc->lastimg)
+                MP_ERR(vo, "Out of memory\n");
             vc->lastimg_wants_osd = true;
 
             vc->lastframeipts = vc->lastipts = frameipts;
diff --git a/video/out/vo_opengl_old.c b/video/out/vo_opengl_old.c
index 3f9e4b0f38..16da97cc35 100644
--- a/video/out/vo_opengl_old.c
+++ b/video/out/vo_opengl_old.c
@@ -2015,6 +2015,8 @@ static mp_image_t *get_screenshot(struct vo *vo)
 
     mp_image_t *image = mp_image_alloc(p->image_format, p->texture_width,
                                                         p->texture_height);
+    if (!image)
+        return NULL;
 
     glDownloadTex(gl, p->target, p->gl_format, p->gl_type, image->planes[0],
                   image->stride[0]);
diff --git a/video/out/vo_sdl.c b/video/out/vo_sdl.c
index 57302b0a41..21b99dcf81 100644
--- a/video/out/vo_sdl.c
+++ b/video/out/vo_sdl.c
@@ -917,6 +917,8 @@ static struct mp_image *get_window_screenshot(struct vo *vo)
     struct priv *vc = vo->priv;
     struct mp_image *image = mp_image_alloc(vc->osd_format.mpv, vo->dwidth,
                                                                 vo->dheight);
+    if (!image)
+        return NULL;
     if (SDL_RenderReadPixels(vc->renderer, NULL, vc->osd_format.sdl,
                              image->planes[0], image->stride[0])) {
         MP_ERR(vo, "SDL_RenderReadPixels failed\n");
diff --git a/video/out/vo_vaapi.c b/video/out/vo_vaapi.c
index df96121137..c1c34370fd 100644
--- a/video/out/vo_vaapi.c
+++ b/video/out/vo_vaapi.c
@@ -196,10 +196,12 @@ static bool render_to_screen(struct priv *p, struct mp_image *mpi)
             p->black_surface = mp_image_pool_get(p->pool, IMGFMT_VAAPI, w, h);
             if (p->black_surface) {
                 struct mp_image *img = mp_image_alloc(fmt, w, h);
-                mp_image_clear(img, 0, 0, w, h);
-                if (va_surface_upload(p->black_surface, img) < 0)
-                    mp_image_unrefp(&p->black_surface);
-                talloc_free(img);
+                if (img) {
+                    mp_image_clear(img, 0, 0, w, h);
+                    if (va_surface_upload(p->black_surface, img) < 0)
+                        mp_image_unrefp(&p->black_surface);
+                    talloc_free(img);
+                }
             }
         }
         surface = va_surface_id(p->black_surface);
diff --git a/video/out/vo_vdpau.c b/video/out/vo_vdpau.c
index 406ed5f9ed..9e5807e320 100644
--- a/video/out/vo_vdpau.c
+++ b/video/out/vo_vdpau.c
@@ -883,6 +883,9 @@ static struct mp_image *read_output_surface(struct vo *vo,
         return NULL;
 
     struct mp_image *image = mp_image_alloc(IMGFMT_BGR32, width, height);
+    if (!image)
+        return NULL;
+
     image->params.colorspace = MP_CSP_RGB;
     // hardcoded with conv. matrix
     image->params.colorlevels = vo->params->outputlevels;
diff --git a/video/out/vo_x11.c b/video/out/vo_x11.c
index 543e762fce..ab188bee6f 100644
--- a/video/out/vo_x11.c
+++ b/video/out/vo_x11.c
@@ -490,6 +490,9 @@ static int redraw_frame(struct vo *vo)
 {
     struct priv *p = vo->priv;
 
+    if (!p->original_image)
+        return false;
+
     draw_image(vo, p->original_image);
     return true;
 }
diff --git a/video/out/vo_xv.c b/video/out/vo_xv.c
index e353dc27ff..870c3a1082 100644
--- a/video/out/vo_xv.c
+++ b/video/out/vo_xv.c
@@ -679,6 +679,9 @@ static int redraw_frame(struct vo *vo)
 {
     struct xvctx *ctx = vo->priv;
 
+    if (!ctx->original_image)
+        return false;
+
     draw_image(vo, ctx->original_image);
     return true;
 }