summaryrefslogtreecommitdiffstats
path: root/libass
diff options
context:
space:
mode:
authorwm4 <wm4@nowhere>2020-02-04 21:24:30 +0100
committerwm4 <wm4@nowhere>2020-03-06 13:07:13 +0100
commitf353ccf1f3efa6c4ccddee1713b62f1c49627619 (patch)
tree870197ca99dd09dcff1721f2b55fc555986adf55 /libass
parent6a670b485536e0398b3fc414fc4317a7b65b5622 (diff)
downloadlibass-f353ccf1f3efa6c4ccddee1713b62f1c49627619.tar.bz2
libass-f353ccf1f3efa6c4ccddee1713b62f1c49627619.tar.xz
ass_blur: check for memory allocation size overflows
Check for overflows that could happen with alignment and the multiplication. The INT_MAX / 4 is somewhat approximate and assumes that degenerate alignment values won't happen. This still assumes that a possibly overflowing end_w/end_h calculation doesn't make the compiler's optimizer destroy the overflow checks.
Diffstat (limited to 'libass')
-rw-r--r--libass/ass_blur.c8
1 files changed, 7 insertions, 1 deletions
diff --git a/libass/ass_blur.c b/libass/ass_blur.c
index aa0489b..0a622ea 100644
--- a/libass/ass_blur.c
+++ b/libass/ass_blur.c
@@ -849,8 +849,14 @@ bool ass_gaussian_blur(const BitmapEngine *engine, Bitmap *bm, double r2)
int end_w = ((w + offset) & ~((1 << blur.level) - 1)) - 4;
int end_h = ((h + offset) & ~((1 << blur.level) - 1)) - 4;
+ if (end_w >= INT_MAX / 4)
+ return false;
+
const int stripe_width = 1 << (engine->align_order - 1);
- int size = end_h * ((end_w + stripe_width - 1) & ~(stripe_width - 1));
+ int aligned_end_w = (end_w + stripe_width - 1) & ~(stripe_width - 1);
+ if (end_h >= INT_MAX / 8 / aligned_end_w)
+ return false;
+ int size = end_h * aligned_end_w;
int16_t *tmp = ass_aligned_alloc(2 * stripe_width, 4 * size, false);
if (!tmp)
return false;