render: match VSFilter's behavior when painting fill in shadow/border

Some releases rely on this.
See corresponding VSFilter code:
https://github.com/Cyberbeing/xy-VSFilter/blob/cf8f5b27de77fe649341bfab0fdfd498e1ad2fa6/src/subtitles/RTS.cpp#L1270
https://github.com/Cyberbeing/xy-VSFilter/blob/cf8f5b27de77fe649341bfab0fdfd498e1ad2fa6/src/subtitles/RTS.cpp#L1291

3 files changed, 33 insertions, 13 deletions

diff --git a/libass/ass_cache.h b/libass/ass_cache.h
index 314d7b8..fc2777c 100644
--- a/libass/ass_cache.h
+++ b/libass/ass_cache.h
@@ -69,9 +69,11 @@ typedef struct outline_hash_key {
 } OutlineHashKey;
 
 enum {
-    FILTER_BORDER_STYLE_3 = 1,
-    FILTER_NONZERO_BORDER = 2,
-    FILTER_NONZERO_SHADOW = 4,
+    FILTER_BORDER_STYLE_3 = 0x01,
+    FILTER_NONZERO_BORDER = 0x02,
+    FILTER_NONZERO_SHADOW = 0x04,
+    FILTER_FILL_IN_SHADOW = 0x08,
+    FILTER_FILL_IN_BORDER = 0x10,
 };
 
 typedef struct {
diff --git a/libass/ass_render.c b/libass/ass_render.c
index 28b6461..1efbcd0 100644
--- a/libass/ass_render.c
+++ b/libass/ass_render.c
@@ -1954,6 +1954,7 @@ static bool parse_events(ASS_Renderer *render_priv, ASS_Event *event)
         for (int i = 0; i < 4; i++) {
             uint32_t clr = render_priv->state.c[i];
             // VSFilter compatibility: apply fade only when it's positive
+            info->a_pre_fade[i] = _a(clr);
             if (render_priv->state.fade > 0)
                 change_alpha(&clr,
                              mult_alpha(_a(clr), render_priv->state.fade), 1.);
@@ -2285,10 +2286,21 @@ static void render_and_combine_glyphs(ASS_Renderer *render_priv,
                 flags |= FILTER_NONZERO_BORDER;
             if (info->shadow_x || info->shadow_y)
                 flags |= FILTER_NONZERO_SHADOW;
-            // VSFilter compatibility: invisible fill and no border?
-            // In this case no shadow is supposed to be rendered.
-            if (flags == FILTER_NONZERO_SHADOW && (info->c[0] & 0xFF) == 0xFF)
-                flags = 0;
+            if (flags & FILTER_NONZERO_SHADOW &&
+                (info->effect_type == EF_KARAOKE_KF ||
+                 info->effect_type == EF_KARAOKE_KO ||
+                 (info->a_pre_fade[0]) != 0xFF ||
+                 info->border_style == 3))
+                flags |= FILTER_FILL_IN_SHADOW;
+            if (!(flags & FILTER_NONZERO_BORDER) &&
+                !(flags & FILTER_FILL_IN_SHADOW))
+                flags &= ~FILTER_NONZERO_SHADOW;
+            if ((flags & FILTER_NONZERO_BORDER &&
+                 info->a_pre_fade[0] == 0 &&
+                 info->a_pre_fade[1] == 0 &&
+                 _a(info->c[2]) == 0) ||
+                info->border_style == 3)
+                flags |= FILTER_FILL_IN_BORDER;
 
             if (linebreak || is_new_bm_run(info, last_info)) {
                 linebreak = 0;
@@ -2491,19 +2503,24 @@ size_t ass_composite_construct(void *key, void *value, void *priv)
 
     int flags = k->filter.flags;
     double r2 = restore_blur(k->filter.blur);
-    bool no_blur = (flags & ~FILTER_NONZERO_SHADOW) == FILTER_NONZERO_BORDER;
-    if (!no_blur)
+    if (!(flags & FILTER_NONZERO_BORDER) || (flags & FILTER_BORDER_STYLE_3))
         ass_synth_blur(render_priv->engine, &v->bm, k->filter.be, r2);
     ass_synth_blur(render_priv->engine, &v->bm_o, k->filter.be, r2);
 
+    if (!(flags & FILTER_FILL_IN_BORDER) && !(flags & FILTER_FILL_IN_SHADOW))
+        fix_outline(&v->bm, &v->bm_o);
+
     if (flags & FILTER_NONZERO_SHADOW) {
-        if (flags & FILTER_NONZERO_BORDER)
+        if (flags & FILTER_NONZERO_BORDER) {
             copy_bitmap(render_priv->engine, &v->bm_s, &v->bm_o);
-        else if (flags & FILTER_BORDER_STYLE_3) {
+            if ((flags & FILTER_FILL_IN_BORDER) && !(flags & FILTER_FILL_IN_SHADOW))
+                fix_outline(&v->bm, &v->bm_s);
+        } else if (flags & FILTER_BORDER_STYLE_3) {
             v->bm_s = v->bm_o;
             memset(&v->bm_o, 0, sizeof(v->bm_o));
-        } else
+        } else {
             copy_bitmap(render_priv->engine, &v->bm_s, &v->bm);
+        }
 
         // Works right even for negative offsets
         // '>>' rounds toward negative infinity, '&' returns correct remainder
@@ -2512,7 +2529,7 @@ size_t ass_composite_construct(void *key, void *value, void *priv)
         shift_bitmap(&v->bm_s, k->filter.shadow.x & SUBPIXEL_MASK, k->filter.shadow.y & SUBPIXEL_MASK);
     }
 
-    if (no_blur)
+    if ((flags & FILTER_FILL_IN_SHADOW) && !(flags & FILTER_FILL_IN_BORDER))
         fix_outline(&v->bm, &v->bm_o);
 
     return sizeof(CompositeHashKey) + sizeof(CompositeHashValue) +
diff --git a/libass/ass_render.h b/libass/ass_render.h
index 170258f..7bdf1e3 100644
--- a/libass/ass_render.h
+++ b/libass/ass_render.h
@@ -142,6 +142,7 @@ typedef struct glyph_info {
     ASS_Vector offset;
     char linebreak;             // the first (leading) glyph of some line ?
     uint32_t c[4];              // colors
+    uint8_t a_pre_fade[4];      // alpha values before applying fades
     ASS_Vector advance;         // 26.6
     ASS_Vector cluster_advance;
     char effect;                // the first (leading) glyph of some effect ?