From f353ccf1f3efa6c4ccddee1713b62f1c49627619 Mon Sep 17 00:00:00 2001 From: wm4 Date: Tue, 4 Feb 2020 21:24:30 +0100 Subject: ass_blur: check for memory allocation size overflows Check for overflows that could happen with alignment and the multiplication. The INT_MAX / 4 is somewhat approximate and assumes that degenerate alignment values won't happen. This still assumes that a possibly overflowing end_w/end_h calculation doesn't make the compiler's optimizer destroy the overflow checks. --- libass/ass_blur.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libass/ass_blur.c b/libass/ass_blur.c index aa0489b..0a622ea 100644 --- a/libass/ass_blur.c +++ b/libass/ass_blur.c @@ -849,8 +849,14 @@ bool ass_gaussian_blur(const BitmapEngine *engine, Bitmap *bm, double r2) int end_w = ((w + offset) & ~((1 << blur.level) - 1)) - 4; int end_h = ((h + offset) & ~((1 << blur.level) - 1)) - 4; + if (end_w >= INT_MAX / 4) + return false; + const int stripe_width = 1 << (engine->align_order - 1); - int size = end_h * ((end_w + stripe_width - 1) & ~(stripe_width - 1)); + int aligned_end_w = (end_w + stripe_width - 1) & ~(stripe_width - 1); + if (end_h >= INT_MAX / 8 / aligned_end_w) + return false; + int size = end_h * aligned_end_w; int16_t *tmp = ass_aligned_alloc(2 * stripe_width, 4 * size, false); if (!tmp) return false; -- cgit v1.2.3