Fix scaling in x direction

VSFilter scales several features based on width/PlayResX
but libass until now didn't. This affects:
 - explicit vector drawing coordinates
 - font spacing
 - borders
 - shadows

This list was obtained by checking against Cyberbeing/xy-VSFilter
code and searching forevery occurence of m_scalex.
This m_scalex factor is also used for the Banner effect,
to make the delay PlayRes-relative, but in libass the value
is PlayRes-relative by default without needing to be scaled.

Additionally, libass neglected to compensate anamorphic stretching
for some features which is also being fixed now.

Both omissions combined actually happened to cancel each other out
for some PlayRes-relative feature in specific but not unusual cases.
If subs for an anamorphic video use isotropically scaled storage
resolution as PlayRes, libass already rendered e.g. vector drawings
correctly. However, other features like e.g. \xshad used to only render
correctly if isotropically scaled display res was used. Thus, more
complex typesetting actually was never rendered correctly on anamorphic
video before this change.

Fixes https://github.com/libass/libass/issues/171.

2 files changed, 53 insertions, 34 deletions

diff --git a/libass/ass_render.c b/libass/ass_render.c
index 682c588..07efa88 100644
--- a/libass/ass_render.c
+++ b/libass/ass_render.c
@@ -672,9 +672,9 @@ static void blend_vector_clip(ASS_Renderer *render_priv, ASS_Image *head)
 
     double m[3][3] = {{0}};
     int32_t scale_base = lshiftwrapi(1, render_priv->state.clip_drawing_scale - 1);
-    double w = scale_base > 0 ? (render_priv->font_scale / scale_base) : 0;
-    m[0][0] = render_priv->font_scale_x * w;
-    m[1][1] = w;
+    double w = scale_base > 0 ? (1.0 / scale_base) : 0;
+    m[0][0] = render_priv->screen_scale_x * w;
+    m[1][1] = render_priv->screen_scale_y * w;
     m[2][2] = 1;
 
     m[0][2] = int_to_d6(render_priv->settings.left_margin);
@@ -999,21 +999,31 @@ static void init_font_scale(ASS_Renderer *render_priv)
 {
     ASS_Settings *settings_priv = &render_priv->settings;
 
+    double font_scr_w = render_priv->orig_width;
     double font_scr_h = render_priv->orig_height;
-    if (!render_priv->state.explicit && render_priv->settings.use_margins)
+    if (!render_priv->state.explicit && render_priv->settings.use_margins) {
+        font_scr_w = render_priv->fit_width;
         font_scr_h = render_priv->fit_height;
+    }
 
-    render_priv->font_scale = font_scr_h / render_priv->track->PlayResY;
-    render_priv->blur_scale = font_scr_h / ass_layout_res(render_priv).y;
-    if (render_priv->track->ScaledBorderAndShadow)
-        render_priv->border_scale =
-            font_scr_h / render_priv->track->PlayResY;
-    else
-        render_priv->border_scale = render_priv->blur_scale;
+    render_priv->screen_scale_x = font_scr_w / render_priv->track->PlayResX;
+    render_priv->screen_scale_y = font_scr_h / render_priv->track->PlayResY;
+
+    ASS_Vector layout_res = ass_layout_res(render_priv);
+    render_priv->blur_scale = font_scr_h / layout_res.y;
+    if (render_priv->track->ScaledBorderAndShadow) {
+        render_priv->border_scale_x = render_priv->screen_scale_x;
+        render_priv->border_scale_y = render_priv->screen_scale_y;
+    } else {
+        render_priv->border_scale_x = font_scr_w / layout_res.x;
+        render_priv->border_scale_y = render_priv->blur_scale;
+    }
 
     if (render_priv->state.apply_font_scale) {
-        render_priv->font_scale *= settings_priv->font_size_coeff;
-        render_priv->border_scale *= settings_priv->font_size_coeff;
+        render_priv->screen_scale_x *= settings_priv->font_size_coeff;
+        render_priv->screen_scale_y *= settings_priv->font_size_coeff;
+        render_priv->border_scale_x *= settings_priv->font_size_coeff;
+        render_priv->border_scale_y *= settings_priv->font_size_coeff;
         render_priv->blur_scale *= settings_priv->font_size_coeff;
     }
 }
@@ -1137,9 +1147,9 @@ get_outline_glyph(ASS_Renderer *priv, GlyphInfo *info)
         }
 
         int32_t scale_base = lshiftwrapi(1, info->drawing_scale - 1);
-        double w = scale_base > 0 ? (priv->font_scale / scale_base) : 0;
-        scale.x = info->scale_x * w;
-        scale.y = info->scale_y * w;
+        double w = scale_base > 0 ? (1.0 / scale_base) : 0;
+        scale.x = info->scale_x * w * priv->screen_scale_x / priv->font_scale_x;
+        scale.y = info->scale_y * w * priv->screen_scale_y;
         desc = 64 * info->drawing_pbo;
         asc = val->asc - desc;
 
@@ -1375,8 +1385,11 @@ get_bitmap_glyph(ASS_Renderer *render_priv, GlyphInfo *info,
 
         ol_key.type = OUTLINE_BOX;
 
-        double w = 64 * render_priv->border_scale;
-        ASS_DVector bord = { info->border_x * w, info->border_y * w };
+        ASS_DVector bord = {
+            64 * info->border_x * render_priv->border_scale_x /
+                render_priv->font_scale_x,
+            64 * info->border_y * render_priv->border_scale_y,
+        };
         double width = info->hspacing_scaled + info->advance.x;
         double height = info->asc + info->desc;
 
@@ -1413,9 +1426,11 @@ get_bitmap_glyph(ASS_Renderer *render_priv, GlyphInfo *info,
         BorderHashKey *k = &ol_key.u.border;
         k->outline = info->outline;
 
-        double w = 64 * render_priv->border_scale;
-        double bord_x = w * info->border_x / tr->scale.x;
-        double bord_y = w * info->border_y / tr->scale.y;
+        double bord_x =
+            64 * render_priv->border_scale_x * info->border_x / tr->scale.x /
+                render_priv->font_scale_x;
+        double bord_y =
+            64 * render_priv->border_scale_y * info->border_y / tr->scale.y;
 
         const ASS_Rect *bbox = &info->outline->cbox;
         // Estimate bounding box half size after stroking
@@ -1429,7 +1444,7 @@ get_bitmap_glyph(ASS_Renderer *render_priv, GlyphInfo *info,
         double mzx = fabs(m[2][0]), mzy = fabs(m[2][1]);
 
         double z0 = m[2][2] - mzx * dx - mzy * dy;
-        w = 1 / FFMAX(z0, m[2][2] / MAX_PERSP_SCALE);
+        double w = 1 / FFMAX(z0, m[2][2] / MAX_PERSP_SCALE);
 
         // Notation from quantize_transform().
         // Note that goal here is to estimate acceptable error for stroking, i. e. D(x) and D(y).
@@ -1527,12 +1542,12 @@ static void measure_text_on_eol(ASS_Renderer *render_priv, double scale, int cur
     render_priv->text_info.height += scale * max_asc + scale * max_desc;
     // For *VSFilter compatibility do biased rounding on max_border*
     // https://github.com/Cyberbeing/xy-VSFilter/blob/xy_sub_filter_rc4@%7B2020-05-17%7D/src/subtitles/RTS.cpp#L1465
-    render_priv->text_info.border_bottom = (int) (render_priv->border_scale * max_border_y + 0.5);
+    render_priv->text_info.border_bottom = (int) (render_priv->border_scale_y * max_border_y + 0.5);
     if (cur_line == 0)
         render_priv->text_info.border_top = render_priv->text_info.border_bottom;
     // VSFilter takes max \bordx into account for collision, even if far from edge
     render_priv->text_info.border_x = FFMAX(render_priv->text_info.border_x,
-            (int) (render_priv->border_scale * max_border_x + 0.5));
+            (int) (render_priv->border_scale_x * max_border_x + 0.5));
 }
 
 
@@ -2098,8 +2113,9 @@ static bool parse_events(ASS_Renderer *render_priv, ASS_Event *event)
         info->effect_type = render_priv->state.effect_type;
         info->effect_timing = render_priv->state.effect_timing;
         info->effect_skip_timing = render_priv->state.effect_skip_timing;
+        // VSFilter compatibility: font glyphs use PlayResY scaling in both dimensions
         info->font_size =
-            fabs(render_priv->state.font_size * render_priv->font_scale);
+            fabs(render_priv->state.font_size * render_priv->screen_scale_y);
         info->be = render_priv->state.be;
         info->blur = render_priv->state.blur;
         info->shadow_x = render_priv->state.shadow_x;
@@ -2127,7 +2143,8 @@ static bool parse_events(ASS_Renderer *render_priv, ASS_Event *event)
 
         if (!drawing_text.str) {
             info->hspacing_scaled = double_to_d6(info->hspacing *
-                    render_priv->font_scale * info->scale_x);
+                    render_priv->screen_scale_x / render_priv->font_scale_x *
+                    info->scale_x);
             fix_glyph_scaling(render_priv, info);
         }
 
@@ -2349,10 +2366,10 @@ static void calculate_rotation_params(ASS_Renderer *render_priv, ASS_DRect *bbox
         GlyphInfo *info = text_info->glyphs + i;
         while (info) {
             info->shift.x = info->pos.x + double_to_d6(device_x - center.x +
-                    info->shadow_x * render_priv->border_scale /
+                    info->shadow_x * render_priv->border_scale_x /
                     render_priv->font_scale_x);
             info->shift.y = info->pos.y + double_to_d6(device_y - center.y +
-                    info->shadow_y * render_priv->border_scale);
+                    info->shadow_y * render_priv->border_scale_y);
             info = info->next;
         }
     }
@@ -2476,8 +2493,8 @@ static void render_and_combine_glyphs(ASS_Renderer *render_priv,
                 double blur_scale = render_priv->blur_scale * (2 / sqrt(log(256)));
                 filter->blur = quantize_blur(info->blur * blur_scale, &shadow_mask);
                 if (flags & FILTER_NONZERO_SHADOW) {
-                    int32_t x = double_to_d6(info->shadow_x * render_priv->border_scale);
-                    int32_t y = double_to_d6(info->shadow_y * render_priv->border_scale);
+                    int32_t x = double_to_d6(info->shadow_x * render_priv->border_scale_x);
+                    int32_t y = double_to_d6(info->shadow_y * render_priv->border_scale_y);
                     filter->shadow.x = (x + (shadow_mask >> 1)) & ~shadow_mask;
                     filter->shadow.y = (y + (shadow_mask >> 1)) & ~shadow_mask;
                 } else
@@ -2695,9 +2712,9 @@ size_t ass_composite_construct(void *key, void *value, void *priv)
 static void add_background(ASS_Renderer *render_priv, EventImages *event_images)
 {
     int size_x = render_priv->state.shadow_x > 0 ?
-        lround(render_priv->state.shadow_x * render_priv->border_scale) : 0;
+        lround(render_priv->state.shadow_x * render_priv->border_scale_x) : 0;
     int size_y = render_priv->state.shadow_y > 0 ?
-        lround(render_priv->state.shadow_y * render_priv->border_scale) : 0;
+        lround(render_priv->state.shadow_y * render_priv->border_scale_y) : 0;
     int left    = event_images->left - size_x;
     int top     = event_images->top  - size_y;
     int right   = event_images->left + event_images->width  + size_x;
diff --git a/libass/ass_render.h b/libass/ass_render.h
index af1516c..9034b35 100644
--- a/libass/ass_render.h
+++ b/libass/ass_render.h
@@ -315,9 +315,11 @@ struct ass_renderer {
     double fit_width;           // frame width without zoom & pan (fit to screen & letterboxed)
     ASS_Track *track;
     long long time;             // frame's timestamp, ms
-    double font_scale;
+    double screen_scale_x;
+    double screen_scale_y;
     double font_scale_x;        // x scale applied to all glyphs to preserve text aspect ratio
-    double border_scale;
+    double border_scale_x;
+    double border_scale_y;
     double blur_scale;
 
     RenderContext state;