Check additional mallocsscan-build

4 files changed, 85 insertions, 40 deletions

diff --git a/libass/ass_font.c b/libass/ass_font.c
index 69675c6..f1c2069 100644
--- a/libass/ass_font.c
+++ b/libass/ass_font.c
@@ -392,16 +392,23 @@ static int ass_strike_outline_glyph(FT_Face face, ASS_Font *font,
     return 0;
 }
 
-void outline_copy(FT_Library lib, FT_Outline *source, FT_Outline **dest)
+FT_Outline *outline_copy(FT_Library lib, FT_Outline *source)
 {
-    if (source == NULL) {
-        *dest = NULL;
-        return;
+    FT_Outline *dest;
+    if (!source)
+        return NULL;
+
+    dest = calloc(1, sizeof(*dest));
+    if (!dest)
+        return NULL;
+
+    if (FT_Outline_New(lib, source->n_points, source->n_contours, dest)) {
+        free(dest);
+        return NULL;
     }
-    *dest = calloc(1, sizeof(**dest));
 
-    FT_Outline_New(lib, source->n_points, source->n_contours, *dest);
-    FT_Outline_Copy(source, *dest);
+    FT_Outline_Copy(source, dest);
+    return dest;
 }
 
 void outline_free(FT_Library lib, FT_Outline *outline)
@@ -679,7 +686,7 @@ static int get_contour_direction(FT_Vector *points, int start, int end)
  * \param border_x border size, x direction, d6 format
  * \param border_x border size, y direction, d6 format
  */
-void fix_freetype_stroker(FT_Outline *outline, int border_x, int border_y)
+bool fix_freetype_stroker(FT_Outline *outline, int border_x, int border_y)
 {
     int nc = outline->n_contours;
     int begin, stop;
@@ -691,6 +698,12 @@ void fix_freetype_stroker(FT_Outline *outline, int border_x, int border_y)
     int i, j;
     int inside_direction;
 
+    if (!valid_cont || !boxes) {
+        free(valid_cont);
+        free(boxes);
+        return false;
+    }
+
     inside_direction = FT_Outline_Get_Orientation(outline) ==
         FT_ORIENTATION_TRUETYPE;
 
@@ -769,5 +782,7 @@ void fix_freetype_stroker(FT_Outline *outline, int border_x, int border_y)
 
     free(boxes);
     free(valid_cont);
+
+    return true;
 }
 
diff --git a/libass/ass_font.h b/libass/ass_font.h
index f80b887..2054d16 100644
--- a/libass/ass_font.h
+++ b/libass/ass_font.h
@@ -20,12 +20,14 @@
 #define LIBASS_FONT_H
 
 #include <stdint.h>
+#include <stdbool.h>
 #include <ft2build.h>
 #include FT_GLYPH_H
 #include FT_OUTLINE_H
 
 #include "ass.h"
 #include "ass_types.h"
+#include "ass_utils.h"
 
 #define VERTICAL_LOWER_BOUND 0x02f1
 
@@ -74,8 +76,9 @@ FT_Glyph ass_font_get_glyph(void *fontconfig_priv, ASS_Font *font,
                             ASS_Hinting hinting, int deco);
 FT_Vector ass_font_get_kerning(ASS_Font *font, uint32_t c1, uint32_t c2);
 void ass_font_free(ASS_Font *font);
-void fix_freetype_stroker(FT_Outline *outline, int border_x, int border_y);
-void outline_copy(FT_Library lib, FT_Outline *source, FT_Outline **dest);
+ASS_WARN_UNUSED
+bool fix_freetype_stroker(FT_Outline *outline, int border_x, int border_y);
+FT_Outline *outline_copy(FT_Library lib, FT_Outline *source);
 void outline_free(FT_Library lib, FT_Outline *outline);
 
 #endif                          /* LIBASS_FONT_H */
diff --git a/libass/ass_render.c b/libass/ass_render.c
index b5684a1..617c47e 100644
--- a/libass/ass_render.c
+++ b/libass/ass_render.c
@@ -952,7 +952,8 @@ static void free_render_context(ASS_Renderer *render_priv)
  * Replace the outline of a glyph by a contour which makes up a simple
  * opaque rectangle.
  */
-static void draw_opaque_box(ASS_Renderer *render_priv, GlyphInfo *info,
+ASS_WARN_UNUSED
+static bool draw_opaque_box(ASS_Renderer *render_priv, GlyphInfo *info,
                             int asc, int desc, FT_Outline *ol,
                             FT_Vector advance, int sx, int sy)
 {
@@ -981,7 +982,8 @@ static void draw_opaque_box(ASS_Renderer *render_priv, GlyphInfo *info,
         { .x = -sx,         .y = -desc - sy },
     };
 
-    FT_Outline_New(render_priv->ftlibrary, 4, 1, ol);
+    if (FT_Outline_New(render_priv->ftlibrary, 4, 1, ol))
+        return false;
 
     ol->n_points = ol->n_contours = 0;
     for (i = 0; i < 4; i++) {
@@ -989,19 +991,22 @@ static void draw_opaque_box(ASS_Renderer *render_priv, GlyphInfo *info,
         ol->tags[ol->n_points++] = 1;
     }
     ol->contours[ol->n_contours++] = ol->n_points - 1;
+    return true;
 }
 
 /*
  * Stroke an outline glyph in x/y direction.  Applies various fixups to get
  * around limitations of the FreeType stroker.
  */
-static void stroke_outline(ASS_Renderer *render_priv, FT_Outline *outline,
+ASS_WARN_UNUSED
+static bool stroke_outline(ASS_Renderer *render_priv, FT_Outline *outline,
                            int sx, int sy)
 {
     if (sx <= 0 && sy <= 0)
-        return;
+        return false;
 
-    fix_freetype_stroker(outline, sx, sy);
+    if (!fix_freetype_stroker(outline, sx, sy))
+        return false;
 
     // Borders are equal; use the regular stroker
     if (sx == sy && render_priv->state.stroker) {
@@ -1013,12 +1018,14 @@ static void stroke_outline(ASS_Renderer *render_priv, FT_Outline *outline,
         if (error) {
             ass_msg(render_priv->library, MSGL_WARN,
                     "FT_Stroker_ParseOutline failed, error: %d", error);
+            return false;
         }
         error = FT_Stroker_GetBorderCounts(render_priv->state.stroker, border,
                 &n_points, &n_contours);
         if (error) {
             ass_msg(render_priv->library, MSGL_WARN,
                     "FT_Stroker_GetBorderCounts failed, error: %d", error);
+            return false;
         }
         FT_Outline_Done(render_priv->ftlibrary, outline);
         error = FT_Outline_New(render_priv->ftlibrary, n_points, n_contours, outline);
@@ -1026,7 +1033,7 @@ static void stroke_outline(ASS_Renderer *render_priv, FT_Outline *outline,
         if (error) {
             ass_msg(render_priv->library, MSGL_WARN,
                     "FT_Outline_New failed, error: %d", error);
-            return;
+            return false;
         }
         FT_Stroker_ExportBorder(render_priv->state.stroker, border, outline);
 
@@ -1042,8 +1049,9 @@ static void stroke_outline(ASS_Renderer *render_priv, FT_Outline *outline,
         int i;
         FT_Outline nol;
 
-        FT_Outline_New(render_priv->ftlibrary, outline->n_points,
-                       outline->n_contours, &nol);
+        if (FT_Outline_New(render_priv->ftlibrary, outline->n_points,
+                           outline->n_contours, &nol))
+            return false;
         FT_Outline_Copy(outline, &nol);
 
         FT_Outline_Embolden(outline, sx * 2);
@@ -1057,6 +1065,7 @@ static void stroke_outline(ASS_Renderer *render_priv, FT_Outline *outline,
         FT_Outline_Done(render_priv->ftlibrary, &nol);
 #endif
     }
+    return true;
 }
 
 /**
@@ -1161,15 +1170,15 @@ get_outline_glyph(ASS_Renderer *priv, GlyphInfo *info)
 
     if (!val) {
         OutlineHashValue v;
+        ASS_Drawing *drawing = info->drawing;
         memset(&v, 0, sizeof(v));
 
-        if (info->drawing) {
-            ASS_Drawing *drawing = info->drawing;
+        if (drawing) {
             ass_drawing_hash(drawing);
             if(!ass_drawing_parse(drawing, 0))
                 return;
-            outline_copy(priv->ftlibrary, &drawing->outline,
-                    &v.outline);
+            if (!(v.outline = outline_copy(priv->ftlibrary, &drawing->outline)))
+                return;
             v.advance.x = drawing->advance.x;
             v.advance.y = drawing->advance.y;
             v.asc = drawing->asc;
@@ -1185,8 +1194,10 @@ get_outline_glyph(ASS_Renderer *priv, GlyphInfo *info)
                         info->symbol, info->face_index, info->glyph_index,
                         priv->settings.hinting, info->flags);
             if (glyph != NULL) {
-                outline_copy(priv->ftlibrary,
-                        &((FT_OutlineGlyph)glyph)->outline, &v.outline);
+                if (!(v.outline =
+                          outline_copy(priv->ftlibrary,
+                                       &((FT_OutlineGlyph)glyph)->outline)))
+                    return;
                 if (priv->settings.shaper == ASS_SHAPING_SIMPLE) {
                     v.advance.x = d16_to_d6(glyph->advance.x);
                     v.advance.y = d16_to_d6(glyph->advance.y);
@@ -1199,36 +1210,44 @@ get_outline_glyph(ASS_Renderer *priv, GlyphInfo *info)
             }
         }
 
-        if (!v.outline) {
-            if (info->drawing)
-                free(key.u.drawing.text);
-            return;
-        }
-
         FT_Outline_Get_CBox(v.outline, &v.bbox_scaled);
 
         if (info->border_style == 3) {
             FT_Vector advance;
 
-            v.border = calloc(1, sizeof(FT_Outline));
+            if (!(v.border = calloc(1, sizeof(FT_Outline))))
+                goto cleanup_border;
 
-            if (priv->settings.shaper == ASS_SHAPING_SIMPLE || info->drawing)
+            if (priv->settings.shaper == ASS_SHAPING_SIMPLE || drawing)
                 advance = v.advance;
             else
                 advance = info->advance;
 
-            draw_opaque_box(priv, info, v.asc, v.desc, v.border, advance,
+            if (draw_opaque_box(priv, info, v.asc, v.desc, v.border, advance,
                     double_to_d6(info->border_x * priv->border_scale),
-                    double_to_d6(info->border_y * priv->border_scale));
-
+                    double_to_d6(info->border_y * priv->border_scale)))
+            {
+                free(v.border);
+                goto cleanup_border;
+            }
         } else if ((info->border_x > 0 || info->border_y > 0)
                 && double_to_d6(info->scale_x) && double_to_d6(info->scale_y)) {
 
             change_border(priv, info->border_x, info->border_y);
-            outline_copy(priv->ftlibrary, v.outline, &v.border);
-            stroke_outline(priv, v.border,
+            if (!(v.border = outline_copy(priv->ftlibrary, v.outline))) {
+            cleanup_border:
+                outline_free(priv->ftlibrary, v.outline);
+                if (drawing)
+                    free(key.u.drawing.text);
+                return;
+            }
+
+            if (stroke_outline(priv, v.border,
                     double_to_d6(info->border_x * priv->border_scale),
-                    double_to_d6(info->border_y * priv->border_scale));
+                    double_to_d6(info->border_y * priv->border_scale))) {
+                free(v.border);
+                goto cleanup_border;
+            }
         }
 
         v.lib = priv->ftlibrary;
@@ -1351,8 +1370,8 @@ get_bitmap_glyph(ASS_Renderer *render_priv, GlyphInfo *info)
 
         hash_val.bm = hash_val.bm_o = hash_val.bm_s = 0;
 
-        outline_copy(render_priv->ftlibrary, info->outline, &outline);
-        outline_copy(render_priv->ftlibrary, info->border, &border);
+        outline = outline_copy(render_priv->ftlibrary, info->outline);
+        border = outline_copy(render_priv->ftlibrary, info->border);
 
         // calculating rotation shift vector (from rotation origin to the glyph basepoint)
         shift.x = key->shift_x;
diff --git a/libass/ass_utils.h b/libass/ass_utils.h
index 579ac40..e8711cf 100644
--- a/libass/ass_utils.h
+++ b/libass/ass_utils.h
@@ -194,4 +194,12 @@ static inline unsigned fnv_32a_str(char *str, unsigned hval)
     return hval;
 }
 
+#if defined(__GNUC__) && (__GNUC__ >= 4)
+#define ASS_WARN_UNUSED __attribute__ ((warn_unused_result))
+#elif defined(_MSC_VER) && (_MSC_VER >= 1700)
+#define ASS_WARN_UNUSED _Check_return_
+#else
+#define ASS_WARN_UNUSED
+#endif
+
 #endif                          /* LIBASS_UTILS_H */