subs: use correct font aspect ratio for libass + converted subs

Rendering of ASS subtitles tries to be bug compatible with VSFilter
and stretches fonts when the video is anamorphic (some scripts try to
compensate for this VSFilter behavior, so trying to render them
"correctly" would give the wrong result). However this behavior is not
appropriate for subtitles we converted to ASS format ourselves for
libass rendering, as they certainly don't have VSFilter bug
workarounds. Change the code to use different behavior for "native"
ASS tracks and converted ones. It's questionable whether the
VSFilter-compatible behavior is appropriate for external .ass files
either, as there could be anamorphic and non-anamorphic versions of
the same video and the bug-compatible behavior can only be correct for
one alternative at most. However it's probably better to keep it as a
default at least, so that extracting a muxed subtitle track and using
that does not give behavior different from the original muxed one.

The aspect ratio setting is per ASS_Renderer, and changing it resets
libass caches. For that reason this commit adds separate renderer
instances to use for the "correct" and "VSFilter bug compatible"
cases.

1 files changed, 28 insertions, 12 deletions

diff --git a/libmpcodecs/vf_ass.c b/libmpcodecs/vf_ass.c
index f1a43b52bd..d695fd206d 100644
--- a/libmpcodecs/vf_ass.c
+++ b/libmpcodecs/vf_ass.c
@@ -61,7 +61,8 @@ static const struct vf_priv_s {
     int auto_insert;
 
     struct osd_state *osd;
-    ASS_Renderer *ass_priv;
+    ASS_Renderer *renderer_realaspect;
+    ASS_Renderer *renderer_vsfilter;
 
     unsigned char *planes[3];
     struct line_limits {
@@ -92,9 +93,14 @@ static int config(struct vf_instance *vf,
     vf->priv->planes[2]   = malloc(vf->priv->outw * vf->priv->outh);
     vf->priv->line_limits = malloc((vf->priv->outh + 1) / 2 * sizeof(*vf->priv->line_limits));
 
-    if (vf->priv->ass_priv) {
-        ass_configure(vf->priv->ass_priv, vf->priv->outw, vf->priv->outh, 0);
-        ass_set_aspect_ratio(vf->priv->ass_priv, 1, 1);
+    if (vf->priv->renderer_realaspect) {
+        ass_configure(vf->priv->renderer_realaspect,
+                      vf->priv->outw, vf->priv->outh, 0);
+        ass_configure(vf->priv->renderer_vsfilter,
+                      vf->priv->outw, vf->priv->outh, 0);
+        ass_set_aspect_ratio(vf->priv->renderer_realaspect,
+                             (double)width / height * d_height / d_width, 1);
+        ass_set_aspect_ratio(vf->priv->renderer_vsfilter, 1, 1);
     }
 
     return vf_next_config(vf, vf->priv->outw, vf->priv->outh, d_width,
@@ -351,9 +357,11 @@ static int render_frame(struct vf_instance *vf, mp_image_t *mpi,
 static int put_image(struct vf_instance *vf, mp_image_t *mpi, double pts)
 {
     ASS_Image *images = 0;
-    if (sub_visibility && vf->priv->ass_priv && vf->priv->osd->ass_track
+    ASS_Renderer *renderer = vf->priv->osd->vsfilter_aspect ?
+        vf->priv->renderer_vsfilter : vf->priv->renderer_realaspect;
+    if (sub_visibility && renderer && vf->priv->osd->ass_track
 	&& (pts != MP_NOPTS_VALUE))
-        images = ass_mp_render_frame(vf->priv->ass_priv,
+        images = ass_mp_render_frame(renderer,
                                      vf->priv->osd->ass_track,
 				     (pts + sub_delay) * 1000 + .5, NULL);
 
@@ -382,13 +390,19 @@ static int control(vf_instance_t *vf, int request, void *data)
         vf->priv->osd = data;
         break;
     case VFCTRL_INIT_EOSD:
-        vf->priv->ass_priv = ass_renderer_init((ASS_Library *)data);
-        if (!vf->priv->ass_priv)
+        vf->priv->renderer_realaspect = ass_renderer_init((ASS_Library *)data);
+        if (!vf->priv->renderer_realaspect)
             return CONTROL_FALSE;
-        ass_configure_fonts(vf->priv->ass_priv);
+        vf->priv->renderer_vsfilter = ass_renderer_init((ASS_Library *)data);
+        if (!vf->priv->renderer_vsfilter) {
+            ass_renderer_done(vf->priv->renderer_realaspect);
+            return CONTROL_FALSE;
+        }
+        ass_configure_fonts(vf->priv->renderer_realaspect);
+        ass_configure_fonts(vf->priv->renderer_vsfilter);
         return CONTROL_TRUE;
     case VFCTRL_DRAW_EOSD:
-        if (vf->priv->ass_priv)
+        if (vf->priv->renderer_realaspect)
             return CONTROL_TRUE;
         break;
     }
@@ -397,8 +411,10 @@ static int control(vf_instance_t *vf, int request, void *data)
 
 static void uninit(struct vf_instance *vf)
 {
-    if (vf->priv->ass_priv)
-        ass_renderer_done(vf->priv->ass_priv);
+    if (vf->priv->renderer_realaspect) {
+        ass_renderer_done(vf->priv->renderer_realaspect);
+        ass_renderer_done(vf->priv->renderer_vsfilter);
+    }
     free(vf->priv->planes[1]);
     free(vf->priv->planes[2]);
     free(vf->priv->line_limits);