From 676f9dc5b52ef406c5527bdadbcb947f11392929 Mon Sep 17 00:00:00 2001 From: "Dr.Smile" Date: Sun, 27 Sep 2020 03:07:47 +0300 Subject: outline: fix overflows in outline processing This commit enforces strict invariant on ASS_Outline to contain point coordinates into predetermined range. Fixes https://github.com/libass/libass/issues/431. --- libass/ass_outline.c | 62 ++++++++++++++++++++++++++++++++++++++++++------- libass/ass_outline.h | 3 ++- libass/ass_rasterizer.c | 10 ++++---- libass/ass_render.c | 8 +++++-- 4 files changed, 65 insertions(+), 18 deletions(-) (limited to 'libass') diff --git a/libass/ass_outline.c b/libass/ass_outline.c index 5fbad2d..f70cea1 100644 --- a/libass/ass_outline.c +++ b/libass/ass_outline.c @@ -48,6 +48,11 @@ static void outline_clear(ASS_Outline *outline) outline->n_segments = outline->max_segments = 0; } +static bool valid_point(const FT_Vector *pt) +{ + return abs(pt->x) <= OUTLINE_MAX && abs(pt->y) <= OUTLINE_MAX; +} + bool outline_convert(ASS_Outline *outline, const FT_Outline *source) { if (!source || !source->n_points) { @@ -77,12 +82,16 @@ bool outline_convert(ASS_Outline *outline, const FT_Outline *source) continue; } + if (!valid_point(source->points + j)) + goto fail; switch (FT_CURVE_TAG(source->tags[j])) { case FT_CURVE_TAG_ON: st = S_ON; break; case FT_CURVE_TAG_CONIC: + if (!valid_point(source->points + last)) + goto fail; pt.x = source->points[last].x; pt.y = -source->points[last].y; switch (FT_CURVE_TAG(source->tags[last])) { @@ -110,6 +119,8 @@ bool outline_convert(ASS_Outline *outline, const FT_Outline *source) outline->points[outline->n_points++] = pt; for (j++; j <= last; j++) { + if (!valid_point(source->points + j)) + goto fail; switch (FT_CURVE_TAG(source->tags[j])) { case FT_CURVE_TAG_ON: switch (st) { @@ -209,6 +220,23 @@ bool outline_scale_pow2(ASS_Outline *outline, const ASS_Outline *source, return true; } + int32_t lim_x = OUTLINE_MAX; + if (scale_ord_x > 0) + lim_x = scale_ord_x < 32 ? lim_x >> scale_ord_x : 0; + else + scale_ord_x = FFMAX(scale_ord_x, -32); + + int32_t lim_y = OUTLINE_MAX; + if (scale_ord_y > 0) + lim_y = scale_ord_y < 32 ? lim_y >> scale_ord_y : 0; + else + scale_ord_y = FFMAX(scale_ord_y, -32); + + if (!lim_x || !lim_y) { + outline_clear(outline); + return false; + } + if (!outline_alloc(outline, source->n_points, source->n_segments)) return false; @@ -216,6 +244,10 @@ bool outline_scale_pow2(ASS_Outline *outline, const ASS_Outline *source, int sy = scale_ord_y + 32; const ASS_Vector *pt = source->points; for (size_t i = 0; i < source->n_points; i++) { + if (abs(pt[i].x) > lim_x || abs(pt[i].y) > lim_y) { + outline_free(outline); + return false; + } // that's equivalent to pt[i].x << scale_ord_x, // but works even for negative coordinate and/or shift amount outline->points[i].x = pt[i].x * ((int64_t) 1 << sx) >> 32; @@ -244,6 +276,10 @@ bool outline_transform_2d(ASS_Outline *outline, const ASS_Outline *source, for (int k = 0; k < 2; k++) v[k] = m[k][0] * pt[i].x + m[k][1] * pt[i].y + m[k][2]; + if (!(fabs(v[0]) < OUTLINE_MAX && fabs(v[1]) < OUTLINE_MAX)) { + outline_free(outline); + return false; + } outline->points[i].x = lrint(v[0]); outline->points[i].y = lrint(v[1]); } @@ -271,8 +307,15 @@ bool outline_transform_3d(ASS_Outline *outline, const ASS_Outline *source, v[k] = m[k][0] * pt[i].x + m[k][1] * pt[i].y + m[k][2]; double w = 1 / FFMAX(v[2], 0.1); - outline->points[i].x = lrint(v[0] * w); - outline->points[i].y = lrint(v[1] * w); + v[0] *= w; + v[1] *= w; + + if (!(fabs(v[0]) < OUTLINE_MAX && fabs(v[1]) < OUTLINE_MAX)) { + outline_free(outline); + return false; + } + outline->points[i].x = lrint(v[0]); + outline->points[i].y = lrint(v[1]); } memcpy(outline->segments, source->segments, source->n_segments); outline->n_points = source->n_points; @@ -299,6 +342,9 @@ void outline_free(ASS_Outline *outline) */ bool outline_add_point(ASS_Outline *outline, ASS_Vector pt, char segment) { + if(abs(pt.x) > OUTLINE_MAX || abs(pt.y) > OUTLINE_MAX) + return false; + if (outline->n_points >= outline->max_points) { size_t new_size = 2 * outline->max_points; if (!ASS_REALLOC_ARRAY(outline->points, new_size)) @@ -1351,7 +1397,7 @@ bool outline_stroke(ASS_Outline *result, ASS_Outline *result1, const int dir = 3; int rad = FFMAX(xbord, ybord); - assert(rad >= eps); + assert(rad >= eps && rad <= OUTLINE_MAX); StrokerState str; str.result[0] = result; @@ -1374,12 +1420,10 @@ bool outline_stroke(ASS_Outline *result, ASS_Outline *result1, str.err_c = 390 * rel_err * rel_err; str.err_a = e; - for (size_t i = 0; i < path->n_points; i++) { - if (path->points[i].x < OUTLINE_MIN || path->points[i].x > OUTLINE_MAX) - return false; - if (path->points[i].y < OUTLINE_MIN || path->points[i].y > OUTLINE_MAX) - return false; - } +#ifndef NDEBUG + for (size_t i = 0; i < path->n_points; i++) + assert(abs(path->points[i].x) <= OUTLINE_MAX && abs(path->points[i].y) <= OUTLINE_MAX); +#endif ASS_Vector *start = path->points, *cur = start; for (size_t i = 0; i < path->n_segments; i++) { diff --git a/libass/ass_outline.h b/libass/ass_outline.h index 414103a..841f9de 100644 --- a/libass/ass_outline.h +++ b/libass/ass_outline.h @@ -83,8 +83,9 @@ typedef struct { char *segments; } ASS_Outline; -#define OUTLINE_MIN (-((int32_t) 1 << 28)) +// ouline point coordinates should always be in [-OUTLINE_MAX, +OUTLINE_MAX] range #define OUTLINE_MAX (((int32_t) 1 << 28) - 1) +// cubic spline splitting requires 8 * OUTLINE_MAX + 4 <= INT32_MAX bool outline_alloc(ASS_Outline *outline, size_t n_points, size_t n_segments); bool outline_convert(ASS_Outline *outline, const FT_Outline *source); diff --git a/libass/ass_rasterizer.c b/libass/ass_rasterizer.c index df11753..5310258 100644 --- a/libass/ass_rasterizer.c +++ b/libass/ass_rasterizer.c @@ -269,12 +269,10 @@ bool rasterizer_set_outline(RasterizerData *rst, } rst->size[0] = rst->n_first; - for (size_t i = 0; i < path->n_points; i++) { - if (path->points[i].x < OUTLINE_MIN || path->points[i].x > OUTLINE_MAX) - return false; - if (path->points[i].y < OUTLINE_MIN || path->points[i].y > OUTLINE_MAX) - return false; - } +#ifndef NDEBUG + for (size_t i = 0; i < path->n_points; i++) + assert(abs(path->points[i].x) <= OUTLINE_MAX && abs(path->points[i].y) <= OUTLINE_MAX); +#endif ASS_Vector *start = path->points, *cur = start; for (size_t i = 0; i < path->n_segments; i++) { diff --git a/libass/ass_render.c b/libass/ass_render.c index cff7736..6246274 100644 --- a/libass/ass_render.c +++ b/libass/ass_render.c @@ -1438,8 +1438,12 @@ get_bitmap_glyph(ASS_Renderer *render_priv, GlyphInfo *info, w *= STROKER_PRECISION / POSITION_PRECISION; frexp(w * (FFMAX(mxx, myx) + mzx * rz), &k->scale_ord_x); frexp(w * (FFMAX(mxy, myy) + mzy * rz), &k->scale_ord_y); - k->border.x = lrint(ldexp(bord_x, k->scale_ord_x) / STROKER_PRECISION); - k->border.y = lrint(ldexp(bord_y, k->scale_ord_y) / STROKER_PRECISION); + bord_x = ldexp(bord_x, k->scale_ord_x); + bord_y = ldexp(bord_y, k->scale_ord_y); + if (!(bord_x < OUTLINE_MAX && bord_y < OUTLINE_MAX)) + return; + k->border.x = lrint(bord_x / STROKER_PRECISION); + k->border.y = lrint(bord_y / STROKER_PRECISION); if (!k->border.x && !k->border.y) { ass_cache_inc_ref(info->bm); info->bm_o = info->bm; -- cgit v1.2.3