renderer: fix subtitles to full screen frame iff use_margins

Based on a commit by wm4.

Nowadays, margins are used by players such as mpv to implement
video zoom & pan, although this was not expected when margins
were first implemented in libass. This results in unpleasant
rendering when panning too far, and it is argued that subtitles
should not change size or move when panning and zooming at all.

libass also makes an attempt to keep subtitles on screen even when
the use of margins is disabled. This is unintuitive and prone to break.

Fix this by strictly separating events which render as if they were
part of the video, and events which should use margins. The latter
will now use the entire screen as canvas, rather than using the video
frame. This actually simplifies the various y2scr functions.

To preserve scaling (mainly for styled subtitles where line breaks are
carefully chosen based on font/video size ratio) and to avoid badly
stretching out things like ASS Margins due to aspect ratio differences
between video and screen, estimate the unpanned & unzoomed video size
from the video aspect ratio and the screen size, and base all scaling
on that. This means that if the user plays a video in letterboxed mode
without extra margins, they get the same scaling as if they were playing
the same video with the same video rectangle size without any margins
at all (with some elements merely spaced out to make use of the black
bars); and when they zoom & pan afterwards, the subtitles don't move
or change size.

This changes behavior even with ass_set_use_margins(_, 0). Before this,
normal dialogue was forced into the visible video area (if negative
margins were set); now it renders it as if it were part of the video.

This also changes the behavior of left and right margins even with
ass_set_use_margins(_, 1). Before this, normal dialogue was forced
into the visible video width (with both positive and negative margins);
now it renders across the entire width of the screen/window.
For 4:3 video letterboxed on 16:9 screen, this means text will cross
the edges of the video, which may look worse than before.

3 files changed, 62 insertions, 54 deletions

diff --git a/libass/ass_render.c b/libass/ass_render.c
index 1a92717..37a67dd 100644
--- a/libass/ass_render.c
+++ b/libass/ass_render.c
@@ -203,26 +203,40 @@ static double x2scr_pos(ASS_Renderer *render_priv, double x)
     return x * render_priv->orig_width / render_priv->font_scale_x / render_priv->track->PlayResX +
         render_priv->settings.left_margin;
 }
-static double x2scr(ASS_Renderer *render_priv, double x)
+static double x2scr_left(ASS_Renderer *render_priv, double x)
 {
-    if (render_priv->state.explicit)
+    if (render_priv->state.explicit || !render_priv->settings.use_margins)
         return x2scr_pos(render_priv, x);
-    return x * render_priv->orig_width_nocrop / render_priv->font_scale_x /
+    return x * render_priv->fit_width / render_priv->font_scale_x /
+        render_priv->track->PlayResX;
+}
+static double x2scr_right(ASS_Renderer *render_priv, double x)
+{
+    if (render_priv->state.explicit || !render_priv->settings.use_margins)
+        return x2scr_pos(render_priv, x);
+    return x * render_priv->fit_width / render_priv->font_scale_x /
         render_priv->track->PlayResX +
-        FFMAX(render_priv->settings.left_margin, 0);
+        (render_priv->width - render_priv->fit_width);
 }
 static double x2scr_pos_scaled(ASS_Renderer *render_priv, double x)
 {
     return x * render_priv->orig_width / render_priv->track->PlayResX +
         render_priv->settings.left_margin;
 }
-static double x2scr_scaled(ASS_Renderer *render_priv, double x)
+static double x2scr_left_scaled(ASS_Renderer *render_priv, double x)
+{
+    if (render_priv->state.explicit || !render_priv->settings.use_margins)
+        return x2scr_pos_scaled(render_priv, x);
+    return x * render_priv->fit_width /
+        render_priv->track->PlayResX;
+}
+static double x2scr_right_scaled(ASS_Renderer *render_priv, double x)
 {
-    if (render_priv->state.explicit)
+    if (render_priv->state.explicit || !render_priv->settings.use_margins)
         return x2scr_pos_scaled(render_priv, x);
-    return x * render_priv->orig_width_nocrop /
+    return x * render_priv->fit_width /
         render_priv->track->PlayResX +
-        FFMAX(render_priv->settings.left_margin, 0);
+        (render_priv->width - render_priv->fit_width);
 }
 /**
  * \brief Mapping between script and screen coordinates
@@ -234,40 +248,29 @@ static double y2scr_pos(ASS_Renderer *render_priv, double y)
 }
 static double y2scr(ASS_Renderer *render_priv, double y)
 {
-    if (render_priv->state.explicit)
+    if (render_priv->state.explicit || !render_priv->settings.use_margins)
         return y2scr_pos(render_priv, y);
-    return y * render_priv->orig_height_nocrop /
+    return y * render_priv->fit_height /
         render_priv->track->PlayResY +
-        FFMAX(render_priv->settings.top_margin, 0);
+        (render_priv->height - render_priv->fit_height) * 0.5;
 }
 
 // the same for toptitles
 static double y2scr_top(ASS_Renderer *render_priv, double y)
 {
-    if (render_priv->state.explicit)
+    if (render_priv->state.explicit || !render_priv->settings.use_margins)
         return y2scr_pos(render_priv, y);
-    if (render_priv->settings.use_margins)
-        return y * render_priv->orig_height_nocrop /
-            render_priv->track->PlayResY;
-    else
-        return y * render_priv->orig_height_nocrop /
-            render_priv->track->PlayResY +
-            FFMAX(render_priv->settings.top_margin, 0);
+    return y * render_priv->fit_height /
+        render_priv->track->PlayResY;
 }
 // the same for subtitles
 static double y2scr_sub(ASS_Renderer *render_priv, double y)
 {
-    if (render_priv->state.explicit)
+    if (render_priv->state.explicit || !render_priv->settings.use_margins)
         return y2scr_pos(render_priv, y);
-    if (render_priv->settings.use_margins)
-        return y * render_priv->orig_height_nocrop /
-            render_priv->track->PlayResY +
-            FFMAX(render_priv->settings.top_margin, 0)
-            + FFMAX(render_priv->settings.bottom_margin, 0);
-    else
-        return y * render_priv->orig_height_nocrop /
-            render_priv->track->PlayResY +
-            FFMAX(render_priv->settings.top_margin, 0);
+    return y * render_priv->fit_height /
+        render_priv->track->PlayResY +
+        (render_priv->height - render_priv->fit_height);
 }
 
 /*
@@ -971,17 +974,18 @@ static void init_font_scale(ASS_Renderer *render_priv)
 {
     ASS_Settings *settings_priv = &render_priv->settings;
 
-    render_priv->font_scale = ((double) render_priv->orig_height) /
-                              render_priv->track->PlayResY;
+    double font_scr_h = render_priv->orig_height;
+    if (!render_priv->state.explicit && render_priv->settings.use_margins)
+        font_scr_h = render_priv->fit_height;
+
+    render_priv->font_scale = font_scr_h / render_priv->track->PlayResY;
     if (settings_priv->storage_height)
-        render_priv->blur_scale = ((double) render_priv->orig_height) /
-            settings_priv->storage_height;
+        render_priv->blur_scale = font_scr_h / settings_priv->storage_height;
     else
         render_priv->blur_scale = 1.;
     if (render_priv->track->ScaledBorderAndShadow)
         render_priv->border_scale =
-            ((double) render_priv->orig_height) /
-            render_priv->track->PlayResY;
+            font_scr_h / render_priv->track->PlayResY;
     else
         render_priv->border_scale = render_priv->blur_scale;
     if (!settings_priv->storage_height)
@@ -2160,8 +2164,8 @@ static void calculate_rotation_params(ASS_Renderer *render_priv, ASS_DRect *bbox
 {
     ASS_DVector center;
     if (render_priv->state.have_origin) {
-        center.x = x2scr(render_priv, render_priv->state.org_x);
-        center.y = y2scr(render_priv, render_priv->state.org_y);
+        center.x = x2scr_pos(render_priv, render_priv->state.org_x);
+        center.y = y2scr_pos(render_priv, render_priv->state.org_y);
     } else {
         double bx = 0., by = 0.;
         get_base_point(bbox, render_priv->state.alignment, &bx, &by);
@@ -2583,8 +2587,8 @@ ass_render_event(ASS_Renderer *render_priv, ASS_Event *event,
 
     // calculate max length of a line
     double max_text_width =
-        x2scr(render_priv, render_priv->track->PlayResX - MarginR) -
-        x2scr(render_priv, MarginL);
+        x2scr_right(render_priv, render_priv->track->PlayResX - MarginR) -
+        x2scr_left(render_priv, MarginL);
 
     // wrap lines
     if (render_priv->state.evt_type != EVENT_HSCROLL) {
@@ -2612,16 +2616,16 @@ ass_render_event(ASS_Renderer *render_priv, ASS_Event *event,
     double device_x = 0;
     if (render_priv->state.evt_type == EVENT_NORMAL ||
         render_priv->state.evt_type == EVENT_VSCROLL) {
-        device_x = x2scr(render_priv, MarginL);
+        device_x = x2scr_left(render_priv, MarginL);
     } else if (render_priv->state.evt_type == EVENT_HSCROLL) {
         if (render_priv->state.scroll_direction == SCROLL_RL)
             device_x =
-                x2scr(render_priv,
+                x2scr_pos(render_priv,
                       render_priv->track->PlayResX -
                       render_priv->state.scroll_shift);
         else if (render_priv->state.scroll_direction == SCROLL_LR)
             device_x =
-                x2scr(render_priv, render_priv->state.scroll_shift) -
+                x2scr_pos(render_priv, render_priv->state.scroll_shift) -
                 (bbox.x_max - bbox.x_min);
     }
 
@@ -2689,9 +2693,9 @@ ass_render_event(ASS_Renderer *render_priv, ASS_Event *event,
         render_priv->state.evt_type == EVENT_HSCROLL ||
         render_priv->state.evt_type == EVENT_VSCROLL) {
         render_priv->state.clip_x0 =
-            x2scr_scaled(render_priv, render_priv->state.clip_x0);
+            x2scr_left_scaled(render_priv, render_priv->state.clip_x0);
         render_priv->state.clip_x1 =
-            x2scr_scaled(render_priv, render_priv->state.clip_x1);
+            x2scr_right_scaled(render_priv, render_priv->state.clip_x1);
         if (valign == VALIGN_TOP) {
             render_priv->state.clip_y0 =
                 y2scr_top(render_priv, render_priv->state.clip_y0);
diff --git a/libass/ass_render.h b/libass/ass_render.h
index e64fd6d..417cfdf 100644
--- a/libass/ass_render.h
+++ b/libass/ass_render.h
@@ -64,12 +64,12 @@ typedef struct {
     double font_size_coeff;     // font size multiplier
     double line_spacing;        // additional line spacing (in frame pixels)
     double line_position;       // vertical position for subtitles, 0-100 (0 = no change)
-    int top_margin;             // height of top margin. Everything except toptitles is shifted down by top_margin.
+    int top_margin;             // height of top margin. Video frame is shifted down by top_margin.
     int bottom_margin;          // height of bottom margin. (frame_height - top_margin - bottom_margin) is original video height.
     int left_margin;
     int right_margin;
     int use_margins;            // 0 - place all subtitles inside original frame
-    // 1 - use margins for placing toptitles and subtitles
+    // 1 - place subtitles (incl. toptitles) in full display frame incl. margins
     double par;                 // user defined pixel aspect ratio (0 = unset)
     ASS_Hinting hinting;
     ASS_ShapingLevel shaper;
@@ -302,8 +302,8 @@ struct ass_renderer {
     int width, height;          // screen dimensions
     int orig_height;            // frame height ( = screen height - margins )
     int orig_width;             // frame width ( = screen width - margins )
-    int orig_height_nocrop;     // frame height ( = screen height - margins + cropheight)
-    int orig_width_nocrop;      // frame width ( = screen width - margins + cropwidth)
+    double fit_height;          // frame height without zoom & pan (fit to screen & letterboxed)
+    double fit_width;           // frame width without zoom & pan (fit to screen & letterboxed)
     ASS_Track *track;
     long long time;             // frame's timestamp, ms
     double font_scale;
diff --git a/libass/ass_render_api.c b/libass/ass_render_api.c
index 4930c44..29465a9 100644
--- a/libass/ass_render_api.c
+++ b/libass/ass_render_api.c
@@ -37,12 +37,16 @@ static void ass_reconfigure(ASS_Renderer *priv)
         settings->right_margin;
     priv->orig_height = settings->frame_height - settings->top_margin -
         settings->bottom_margin;
-    priv->orig_width_nocrop =
-        settings->frame_width - FFMAX(settings->left_margin, 0) -
-        FFMAX(settings->right_margin, 0);
-    priv->orig_height_nocrop =
-        settings->frame_height - FFMAX(settings->top_margin, 0) -
-        FFMAX(settings->bottom_margin, 0);
+    priv->fit_width =
+        (long long) priv->orig_width * priv->height >=
+        (long long) priv->orig_height * priv->width ?
+            priv->width :
+            (double) priv->orig_width * priv->height / priv->orig_height;
+    priv->fit_height =
+        (long long) priv->orig_width * priv->height <=
+        (long long) priv->orig_height * priv->width ?
+            priv->height :
+            (double) priv->orig_height * priv->width / priv->orig_width;
 }
 
 void ass_set_frame_size(ASS_Renderer *priv, int w, int h)