Consolidate and quantize all transformations

This commit defers all outline transformations until rasterization stage.
Combined transformation is then quantized and used as bitmap key.
That should improve performance of slow animations.

Also caching of initial and stroked outlines and bitmaps is now separate
in preparation to proper error estimation for stroker stage.

Note that Z-clipping for perspective transformations is now done
differently compared to VSFilter. That clipping is mostly safety feature
to protect from overflows and divisions by zero and is almost never
triggered in real-world subtitles.

1 files changed, 3 insertions, 30 deletions

diff --git a/libass/ass_bitmap.c b/libass/ass_bitmap.c
index cc40ed6..3e3a5a0 100644
--- a/libass/ass_bitmap.c
+++ b/libass/ass_bitmap.c
@@ -200,6 +200,9 @@ Bitmap *outline_to_bitmap(ASS_Renderer *render_priv,
         return NULL;
     }
 
+    if (rst->bbox.x_min > rst->bbox.x_max || rst->bbox.y_min > rst->bbox.y_max)
+        return NULL;
+
     if (bord < 0 || bord > INT_MAX / 2)
         return NULL;
     if (rst->bbox.x_max > INT_MAX - 63 || rst->bbox.y_max > INT_MAX - 63)
@@ -434,36 +437,6 @@ int be_padding(int be)
     return FFMAX(128 - be, 0);
 }
 
-bool outline_to_bitmap2(ASS_Renderer *render_priv, ASS_Outline *outline,
-                        ASS_Outline *border1, ASS_Outline *border2,
-                        Bitmap **bm_g, Bitmap **bm_o)
-{
-    assert(bm_g && bm_o);
-    *bm_g = *bm_o = NULL;
-
-    if (outline && !outline->n_points)
-        outline = NULL;
-    if (border1 && !border1->n_points)
-        border1 = NULL;
-    if (border2 && !border2->n_points)
-        border2 = NULL;
-
-    if (outline) {
-        *bm_g = outline_to_bitmap(render_priv, outline, NULL, 1);
-        if (!*bm_g)
-            return false;
-    }
-
-    if (border1 || border2) {
-        *bm_o = outline_to_bitmap(render_priv, border1, border2, 1);
-        if (!*bm_o) {
-            return false;
-        }
-    }
-
-    return true;
-}
-
 /**
  * \brief Add two bitmaps together at a given position
  * Uses additive blending, clipped to [0,255]. Pure C implementation.