libass: Fix cache lookup problem causing memory bloat

The cache code did hash lookups by storing key values in struct fields
and then hashing and comparing the struct as a single memory block. In
at least one case such a struct contained uninitialized padding bytes
which prevented the complete memory area of the struct from matching
even though the fields did. As a result the code failed to find
existing objects in the cache and stored new versions of them, causing
gigabytes of memory use in some circumstances. Initializing the struct
memory to zero before writing the fields avoided such memory use in
tests but is not guaranteed to work if I interpret the C standard
correctly (the compiler is allowed to write garbage over padding bytes
when changing struct member values).

Change the code to use struct-specific hashing and comparison
functions that work field by field to guarantee correct behavior.
Create these by replacing the struct definition with a template that
lists the fields and can be used the generate each of struct
definition, hash function and compare function with some preprocessor
magic (otherwise every field would need to be listed separately in all
three).

1 files changed, 87 insertions, 0 deletions

diff --git a/libass/ass_cache_template.c b/libass/ass_cache_template.c
new file mode 100644
index 0000000000..5f77190fa1
--- /dev/null
+++ b/libass/ass_cache_template.c
@@ -0,0 +1,87 @@
+#ifdef CREATE_STRUCT_DEFINITIONS
+#undef CREATE_STRUCT_DEFINITIONS
+#define START(funcname, structname) \
+    typedef struct structname {
+#define GENERIC(type, member) \
+        type member;
+#define FTVECTOR(member) \
+        FT_Vector member;
+#define END(typedefnamename) \
+    } typedefnamename;
+
+#elif defined(CREATE_COMPARISON_FUNCTIONS)
+#undef CREATE_COMPARISON_FUNCTIONS
+#define START(funcname, structname) \
+    static int funcname##_compare(void *key1, void *key2, size_t key_size) \
+    { \
+        struct structname *a = key1; \
+        struct structname *b = key2; \
+        return // conditions follow
+#define GENERIC(type, member) \
+            a->member == b->member &&
+#define FTVECTOR(member) \
+            a->member.x == b->member.x && a->member.y == b->member.y &&
+#define END(typedefname) \
+            1; \
+    }
+
+#elif defined(CREATE_HASH_FUNCTIONS)
+#undef CREATE_HASH_FUNCTIONS
+#define START(funcname, structname) \
+    static unsigned funcname##_hash(void *buf, size_t len) \
+    { \
+        struct structname *p = buf; \
+        unsigned hval = FNV1_32A_INIT;
+#define GENERIC(type, member) \
+        hval = fnv_32a_buf(&p->member, sizeof(p->member), hval);
+#define FTVECTOR(member) GENERIC(, member.x); GENERIC(, member.y);
+#define END(typedefname) \
+        return hval; \
+    }
+
+#else
+#error missing defines
+#endif
+
+
+
+// describes a bitmap; bitmaps with equivalents structs are considered identical
+START(bitmap, bipmap_hash_key_s)
+    GENERIC(char, bitmap) // bool : true = bitmap, false = outline
+    GENERIC(ass_font_t *, font)
+    GENERIC(double, size) // font size
+    GENERIC(uint32_t, ch) // character code
+    GENERIC(unsigned, outline) // border width, 16.16 fixed point value
+    GENERIC(int, bold)
+    GENERIC(int, italic)
+    GENERIC(char, be) // blur edges
+    GENERIC(unsigned, scale_x) // 16.16
+    GENERIC(unsigned, scale_y) // 16.16
+    GENERIC(int, frx) // signed 16.16
+    GENERIC(int, fry) // signed 16.16
+    GENERIC(int, frz) // signed 16.16
+    // shift vector that was added to glyph before applying rotation
+    // = 0, if frx = fry = frx = 0
+    // = (glyph base point) - (rotation origin), otherwise
+    GENERIC(int, shift_x)
+    GENERIC(int, shift_y)
+    FTVECTOR(advance) // subpixel shift vector
+END(bitmap_hash_key_t)
+
+// describes an outline glyph
+START(glyph, glyph_hash_key_s)
+    GENERIC(ass_font_t *, font)
+    GENERIC(double, size) // font size
+    GENERIC(uint32_t, ch) // character code
+    GENERIC(int, bold)
+    GENERIC(int, italic)
+    GENERIC(unsigned, scale_x) // 16.16
+    GENERIC(unsigned, scale_y) // 16.16
+    FTVECTOR(advance) // subpixel shift vector
+    GENERIC(unsigned, outline) // border width, 16.16
+END(glyph_hash_key_t)
+
+#undef START
+#undef GENERIC
+#undef FTVECTOR
+#undef END