renderer: consolidate processing of glyph decoration flags

Processing of DECO_ROTATE has moved after ASS_Outline conversion too.
All relevant outline processing functions have moved into ass_outline.c.
outline_convert() now expects preallocated outline to reduce reallocations.

5 files changed, 147 insertions, 134 deletions

diff --git a/libass/ass_font.c b/libass/ass_font.c
index 68e5fe4..36c7eaa 100644
--- a/libass/ass_font.c
+++ b/libass/ass_font.c
@@ -373,87 +373,6 @@ void ass_font_get_asc_desc(ASS_Font *font, int face_index,
     *desc = FT_MulFix(-face->descender, y_scale);
 }
 
-static void add_line(ASS_Outline *ol, int bear, int advance, int dir, int pos, int size) {
-    ASS_Vector points[4] = {
-        {.x = bear,      .y = pos - size},
-        {.x = advance,   .y = pos - size},
-        {.x = advance,   .y = pos + size},
-        {.x = bear,      .y = pos + size},
-    };
-
-    if (dir == FT_ORIENTATION_TRUETYPE) {
-        for (int i = 0; i < 4; i++)
-            ol->points[ol->n_points++] = points[i];
-    } else {
-        for (int i = 3; i >= 0; i--)
-            ol->points[ol->n_points++] = points[i];
-    }
-
-    for (int i = 0; i < 4; i++)
-        ol->segments[ol->n_segments++] = OUTLINE_LINE_SEGMENT;
-    ol->segments[ol->n_segments - 1] |= OUTLINE_CONTOUR_END;
-}
-
-/*
- * Strike a glyph with a horizontal line; it's possible to underline it
- * and/or strike through it.  For the line's position and size, truetype
- * tables are consulted.  Obviously this relies on the data in the tables
- * being accurate.
- *
- */
-int ass_strike_outline_glyph(ASS_Font *font, int face_index,
-                             FT_Glyph glyph, ASS_Outline *ol,
-                             int under, int through)
-{
-    FT_Face face = font->faces[face_index];
-    TT_OS2 *os2 = FT_Get_Sfnt_Table(face, ft_sfnt_os2);
-    TT_Postscript *ps = FT_Get_Sfnt_Table(face, ft_sfnt_post);
-    int advance, y_scale, i, dir;
-
-    if (!under && !through)
-        return 0;
-
-    // Grow outline
-    i = (under ? 4 : 0) + (through ? 4 : 0);
-    if (ol->n_points > SIZE_MAX - i)
-        return 0;
-    if (ol->n_segments > SIZE_MAX - i)
-        return 0;
-    if (!ASS_REALLOC_ARRAY(ol->points, ol->n_points + i))
-        return 0;
-    if (!ASS_REALLOC_ARRAY(ol->segments, ol->n_segments + i))
-        return 0;
-
-    advance = d16_to_d6(glyph->advance.x);
-    y_scale = face->size->metrics.y_scale;
-
-    // Reverse drawing direction for non-truetype fonts
-    dir = FT_Outline_Get_Orientation(&((FT_OutlineGlyph) glyph)->outline);
-
-    // Add points to the outline
-    if (under && ps) {
-        int pos = FT_MulFix(ps->underlinePosition, y_scale);
-        int size = FT_MulFix(ps->underlineThickness, y_scale / 2);
-
-        if (pos > 0 || size <= 0)
-            return 1;
-
-        add_line(ol, 0, advance, dir, -pos, size);
-    }
-
-    if (through && os2) {
-        int pos = FT_MulFix(os2->yStrikeoutPosition, y_scale);
-        int size = FT_MulFix(os2->yStrikeoutSize, y_scale / 2);
-
-        if (pos < 0 || size <= 0)
-            return 1;
-
-        add_line(ol, 0, advance, dir, -pos, size);
-    }
-
-    return 0;
-}
-
 /**
  * Slightly embold a glyph without touching its metrics
  */
@@ -543,7 +462,7 @@ int ass_font_get_index(ASS_FontSelector *fontsel, ASS_Font *font,
  * \param ch character code
  **/
 FT_Glyph ass_font_get_glyph(ASS_Font *font, int face_index, int index,
-                            ASS_Hinting hinting, int deco)
+                            ASS_Hinting hinting)
 {
     int error;
     FT_Glyph glyph;
@@ -587,22 +506,6 @@ FT_Glyph ass_font_get_glyph(ASS_Font *font, int face_index, int index,
         return 0;
     }
 
-    // Rotate glyph, if needed
-    if (deco & DECO_ROTATE) {
-        FT_Matrix m = { 0, double_to_d16(-1.0), double_to_d16(1.0), 0 };
-        TT_OS2 *os2 = FT_Get_Sfnt_Table(face, ft_sfnt_os2);
-        int desc = 0;
-
-        if (os2)
-            desc = FT_MulFix(os2->sTypoDescender, face->size->metrics.y_scale);
-
-        FT_Outline_Translate(&((FT_OutlineGlyph) glyph)->outline, 0, -desc);
-        FT_Outline_Transform(&((FT_OutlineGlyph) glyph)->outline, &m);
-        FT_Outline_Translate(&((FT_OutlineGlyph) glyph)->outline,
-                             face->glyph->metrics.vertAdvance, desc);
-        glyph->advance.x = face->glyph->linearVertAdvance;
-    }
-
     return glyph;
 }
 
@@ -620,3 +523,86 @@ void ass_font_clear(ASS_Font *font)
     }
     free((char *) font->desc.family.str);
 }
+
+/**
+ * \brief Convert glyph into ASS_Outline according to decoration flags
+ **/
+bool ass_get_glyph_outline(ASS_Outline *outline, int32_t *advance,
+                           FT_Face face, FT_Glyph glyph, unsigned flags)
+{
+    int32_t y_scale = face->size->metrics.y_scale;
+    int32_t adv = flags & DECO_ROTATE ? face->glyph->linearVertAdvance : glyph->advance.x;
+    *advance = adv = d16_to_d6(adv);
+
+    int n_lines = 0;
+    int32_t line_y[2][2];
+    if (adv > 0 && (flags & DECO_UNDERLINE)) {
+        TT_Postscript *ps = FT_Get_Sfnt_Table(face, ft_sfnt_post);
+        if (ps && ps->underlinePosition <= 0 && ps->underlineThickness > 0) {
+            int64_t pos  = ((int64_t) ps->underlinePosition  * y_scale + 0x8000) >> 16;
+            int64_t size = ((int64_t) ps->underlineThickness * y_scale + 0x8000) >> 16;
+            pos = -pos - (size >> 1);
+            if (pos >= -OUTLINE_MAX && pos + size <= OUTLINE_MAX) {
+                line_y[n_lines][0] = pos;
+                line_y[n_lines][1] = pos + size;
+                n_lines++;
+            }
+        }
+    }
+    if (adv > 0 && (flags & DECO_STRIKETHROUGH)) {
+        TT_OS2 *os2 = FT_Get_Sfnt_Table(face, ft_sfnt_os2);
+        if (os2 && os2->yStrikeoutPosition >= 0 && os2->yStrikeoutSize > 0) {
+            int64_t pos  = ((int64_t) os2->yStrikeoutPosition * y_scale + 0x8000) >> 16;
+            int64_t size = ((int64_t) os2->yStrikeoutSize     * y_scale + 0x8000) >> 16;
+            pos = -pos - (size >> 1);
+            if (pos >= -OUTLINE_MAX && pos + size <= OUTLINE_MAX) {
+                line_y[n_lines][0] = pos;
+                line_y[n_lines][1] = pos + size;
+                n_lines++;
+            }
+        }
+    }
+
+    assert(glyph->format == FT_GLYPH_FORMAT_OUTLINE);
+    FT_Outline *source = &((FT_OutlineGlyph) glyph)->outline;
+    if (!source->n_points && !n_lines) {
+        outline_clear(outline);
+        return true;
+    }
+
+    size_t max_points = 2 * source->n_points + 4 * n_lines;
+    size_t max_segments = source->n_points + 4 * n_lines;
+    if (!outline_alloc(outline, max_points, max_segments))
+        return false;
+
+    if (!outline_convert(outline, source))
+        goto fail;
+
+    if (flags & DECO_ROTATE) {
+        TT_OS2 *os2 = FT_Get_Sfnt_Table(face, ft_sfnt_os2);
+        int64_t desc = 0;
+        if (os2) {
+            desc = ((int64_t) os2->sTypoDescender * y_scale + 0x8000) >> 16;
+            if (llabs(desc) > 2 * OUTLINE_MAX)
+                goto fail;
+        }
+        int64_t dv = face->glyph->metrics.vertAdvance + desc;
+        if (llabs(dv) > 2 * OUTLINE_MAX)
+            goto fail;
+        ASS_Vector offs = { dv, -desc };
+        if (!outline_rotate_90(outline, offs))
+            goto fail;
+    }
+
+    if (!n_lines)
+        return true;
+    FT_Orientation dir = FT_Outline_Get_Orientation(source);
+    int iy = (dir == FT_ORIENTATION_TRUETYPE ? 0 : 1);
+    for (int i = 0; i < n_lines; i++)
+        outline_add_rect(outline, 0, line_y[i][iy], adv, line_y[i][iy ^ 1]);
+    return true;
+
+fail:
+    outline_free(outline);
+    return false;
+}
diff --git a/libass/ass_font.h b/libass/ass_font.h
index 57f832d..783d6f9 100644
--- a/libass/ass_font.h
+++ b/libass/ass_font.h
@@ -22,7 +22,6 @@
 #include <stdint.h>
 #include <ft2build.h>
 #include FT_GLYPH_H
-#include FT_OUTLINE_H
 
 typedef struct ass_font ASS_Font;
 
@@ -61,12 +60,11 @@ int ass_font_get_index(ASS_FontSelector *fontsel, ASS_Font *font,
                        uint32_t symbol, int *face_index, int *glyph_index);
 uint32_t ass_font_index_magic(FT_Face face, uint32_t symbol);
 FT_Glyph ass_font_get_glyph(ASS_Font *font, int face_index, int index,
-                            ASS_Hinting hinting, int deco);
+                            ASS_Hinting hinting);
 void ass_font_clear(ASS_Font *font);
 
-int ass_strike_outline_glyph(ASS_Font *font, int face_index,
-                             FT_Glyph glyph, ASS_Outline *ol,
-                             int under, int through);
+bool ass_get_glyph_outline(ASS_Outline *outline, int32_t *advance,
+                           FT_Face face, FT_Glyph glyph, unsigned flags);
 
 FT_Face ass_face_open(ASS_Library *lib, FT_Library ftlib, const char *path,
                       const char *postscript_name, int index);
diff --git a/libass/ass_outline.c b/libass/ass_outline.c
index ae5c06a..b967089 100644
--- a/libass/ass_outline.c
+++ b/libass/ass_outline.c
@@ -39,7 +39,7 @@ bool outline_alloc(ASS_Outline *outline, size_t n_points, size_t n_segments)
     return true;
 }
 
-static void outline_clear(ASS_Outline *outline)
+void outline_clear(ASS_Outline *outline)
 {
     outline->points = NULL;
     outline->segments = NULL;
@@ -55,14 +55,6 @@ static bool valid_point(const FT_Vector *pt)
 
 bool outline_convert(ASS_Outline *outline, const FT_Outline *source)
 {
-    if (!source || !source->n_points) {
-        outline_clear(outline);
-        return true;
-    }
-
-    if (!outline_alloc(outline, 2 * source->n_points, source->n_points))
-        return false;
-
     enum Status {
         S_ON, S_Q, S_C1, S_C2
     };
@@ -74,7 +66,7 @@ bool outline_convert(ASS_Outline *outline, const FT_Outline *source)
 
         int last = source->contours[i];
         if (j > last || last >= source->n_points)
-            goto fail;
+            return false;
 
         // skip degenerate 2-point contours from broken fonts
         if (last - j < 2) {
@@ -83,7 +75,7 @@ bool outline_convert(ASS_Outline *outline, const FT_Outline *source)
         }
 
         if (!valid_point(source->points + j))
-            goto fail;
+            return false;
         switch (FT_CURVE_TAG(source->tags[j])) {
         case FT_CURVE_TAG_ON:
             st = S_ON;
@@ -91,7 +83,7 @@ bool outline_convert(ASS_Outline *outline, const FT_Outline *source)
 
         case FT_CURVE_TAG_CONIC:
             if (!valid_point(source->points + last))
-                goto fail;
+                return false;
             pt.x =  source->points[last].x;
             pt.y = -source->points[last].y;
             switch (FT_CURVE_TAG(source->tags[last])) {
@@ -105,14 +97,14 @@ bool outline_convert(ASS_Outline *outline, const FT_Outline *source)
                 break;
 
             default:
-                goto fail;
+                return false;
             }
             outline->points[outline->n_points++] = pt;
             st = S_Q;
             break;
 
         default:
-            goto fail;
+            return false;
         }
         pt.x =  source->points[j].x;
         pt.y = -source->points[j].y;
@@ -120,7 +112,7 @@ bool outline_convert(ASS_Outline *outline, const FT_Outline *source)
 
         for (j++; j <= last; j++) {
             if (!valid_point(source->points + j))
-                goto fail;
+                return false;
             switch (FT_CURVE_TAG(source->tags[j])) {
             case FT_CURVE_TAG_ON:
                 switch (st) {
@@ -137,7 +129,7 @@ bool outline_convert(ASS_Outline *outline, const FT_Outline *source)
                     break;
 
                 default:
-                    goto fail;
+                    return false;
                 }
                 st = S_ON;
                 break;
@@ -156,7 +148,7 @@ bool outline_convert(ASS_Outline *outline, const FT_Outline *source)
                     break;
 
                 default:
-                    goto fail;
+                    return false;
                 }
                 break;
 
@@ -171,12 +163,12 @@ bool outline_convert(ASS_Outline *outline, const FT_Outline *source)
                     break;
 
                 default:
-                    goto fail;
+                    return false;
                 }
                 break;
 
             default:
-                goto fail;
+                return false;
             }
             pt.x =  source->points[j].x;
             pt.y = -source->points[j].y;
@@ -201,15 +193,50 @@ bool outline_convert(ASS_Outline *outline, const FT_Outline *source)
             break;
 
         default:
-            goto fail;
+            return false;
         }
         outline->segments[outline->n_segments - 1] |= OUTLINE_CONTOUR_END;
     }
     return true;
+}
 
-fail:
-    outline_free(outline);
-    return false;
+bool outline_rotate_90(ASS_Outline *outline, ASS_Vector offs)
+{
+    assert(abs(offs.x) <= INT32_MAX - OUTLINE_MAX);
+    assert(abs(offs.y) <= INT32_MAX - OUTLINE_MAX);
+    for (size_t i = 0; i < outline->n_points; i++) {
+        ASS_Vector pt = { offs.x + outline->points[i].y,
+                          offs.y - outline->points[i].x };
+        if (abs(pt.x) > OUTLINE_MAX || abs(pt.y) > OUTLINE_MAX)
+            return false;
+        outline->points[i] = pt;
+    }
+    return true;
+}
+
+void outline_add_rect(ASS_Outline *outline,
+                      int32_t x0, int32_t y0, int32_t x1, int32_t y1)
+{
+    assert(outline->n_points + 4 <= outline->max_points);
+    assert(outline->n_segments + 4 <= outline->max_segments);
+    assert(abs(x0) <= OUTLINE_MAX && abs(y0) <= OUTLINE_MAX);
+    assert(abs(x1) <= OUTLINE_MAX && abs(y1) <= OUTLINE_MAX);
+    assert(!outline->n_segments ||
+        (outline->segments[outline->n_segments - 1] & OUTLINE_CONTOUR_END));
+
+    size_t pos = outline->n_points;
+    outline->points[pos + 0].x = outline->points[pos + 3].x = x0;
+    outline->points[pos + 1].x = outline->points[pos + 2].x = x1;
+    outline->points[pos + 0].y = outline->points[pos + 1].y = y0;
+    outline->points[pos + 2].y = outline->points[pos + 3].y = y1;
+    outline->n_points = pos + 4;
+
+    pos = outline->n_segments;
+    outline->segments[pos + 0] = OUTLINE_LINE_SEGMENT;
+    outline->segments[pos + 1] = OUTLINE_LINE_SEGMENT;
+    outline->segments[pos + 2] = OUTLINE_LINE_SEGMENT;
+    outline->segments[pos + 3] = OUTLINE_LINE_SEGMENT | OUTLINE_CONTOUR_END;
+    outline->n_segments = pos + 4;
 }
 
 bool outline_scale_pow2(ASS_Outline *outline, const ASS_Outline *source,
diff --git a/libass/ass_outline.h b/libass/ass_outline.h
index 6f80c41..89df528 100644
--- a/libass/ass_outline.h
+++ b/libass/ass_outline.h
@@ -88,7 +88,13 @@ typedef struct {
 // cubic spline splitting requires 8 * OUTLINE_MAX + 4 <= INT32_MAX
 
 bool outline_alloc(ASS_Outline *outline, size_t n_points, size_t n_segments);
+void outline_clear(ASS_Outline *outline);
+
 bool outline_convert(ASS_Outline *outline, const FT_Outline *source);
+bool outline_rotate_90(ASS_Outline *outline, ASS_Vector offs);
+void outline_add_rect(ASS_Outline *outline,
+                      int32_t x0, int32_t y0, int32_t x1, int32_t y1);
+
 bool outline_scale_pow2(ASS_Outline *outline, const ASS_Outline *source,
                         int scale_ord_x, int scale_ord_y);
 bool outline_transform_2d(ASS_Outline *outline, const ASS_Outline *source,
diff --git a/libass/ass_render.c b/libass/ass_render.c
index f986b97..5d78106 100644
--- a/libass/ass_render.c
+++ b/libass/ass_render.c
@@ -1166,16 +1166,12 @@ size_t ass_outline_construct(void *key, void *value, void *priv)
             ass_face_set_size(k->font->faces[k->face_index], k->size);
             FT_Glyph glyph =
                 ass_font_get_glyph(k->font, k->face_index, k->glyph_index,
-                                   render_priv->settings.hinting, k->flags);
+                                   render_priv->settings.hinting);
             if (glyph != NULL) {
-                FT_Outline *src = &((FT_OutlineGlyph) glyph)->outline;
-                if (!outline_convert(&v->outline[0], src))
+                if (!ass_get_glyph_outline(&v->outline[0], &v->advance,
+                                           k->font->faces[k->face_index],
+                                           glyph, k->flags))
                     return 1;
-                v->advance = d16_to_d6(glyph->advance.x);
-                ass_strike_outline_glyph(k->font, k->face_index,
-                                         glyph, &v->outline[0],
-                                         k->flags & DECO_UNDERLINE,
-                                         k->flags & DECO_STRIKETHROUGH);
                 FT_Done_Glyph(glyph);
                 ass_font_get_asc_desc(k->font, k->face_index,
                                       &v->asc, &v->desc);