subs: use correct font aspect ratio for libass + converted subs

Rendering of ASS subtitles tries to be bug compatible with VSFilter
and stretches fonts when the video is anamorphic (some scripts try to
compensate for this VSFilter behavior, so trying to render them
"correctly" would give the wrong result). However this behavior is not
appropriate for subtitles we converted to ASS format ourselves for
libass rendering, as they certainly don't have VSFilter bug
workarounds. Change the code to use different behavior for "native"
ASS tracks and converted ones. It's questionable whether the
VSFilter-compatible behavior is appropriate for external .ass files
either, as there could be anamorphic and non-anamorphic versions of
the same video and the bug-compatible behavior can only be correct for
one alternative at most. However it's probably better to keep it as a
default at least, so that extracting a muxed subtitle track and using
that does not give behavior different from the original muxed one.

The aspect ratio setting is per ASS_Renderer, and changing it resets
libass caches. For that reason this commit adds separate renderer
instances to use for the "correct" and "VSFilter bug compatible"
cases.

2 files changed, 67 insertions, 25 deletions

diff --git a/libmpcodecs/vf_ass.c b/libmpcodecs/vf_ass.c
index f1a43b52bd..d695fd206d 100644
--- a/libmpcodecs/vf_ass.c
+++ b/libmpcodecs/vf_ass.c
@@ -61,7 +61,8 @@ static const struct vf_priv_s {
     int auto_insert;
 
     struct osd_state *osd;
-    ASS_Renderer *ass_priv;
+    ASS_Renderer *renderer_realaspect;
+    ASS_Renderer *renderer_vsfilter;
 
     unsigned char *planes[3];
     struct line_limits {
@@ -92,9 +93,14 @@ static int config(struct vf_instance *vf,
     vf->priv->planes[2]   = malloc(vf->priv->outw * vf->priv->outh);
     vf->priv->line_limits = malloc((vf->priv->outh + 1) / 2 * sizeof(*vf->priv->line_limits));
 
-    if (vf->priv->ass_priv) {
-        ass_configure(vf->priv->ass_priv, vf->priv->outw, vf->priv->outh, 0);
-        ass_set_aspect_ratio(vf->priv->ass_priv, 1, 1);
+    if (vf->priv->renderer_realaspect) {
+        ass_configure(vf->priv->renderer_realaspect,
+                      vf->priv->outw, vf->priv->outh, 0);
+        ass_configure(vf->priv->renderer_vsfilter,
+                      vf->priv->outw, vf->priv->outh, 0);
+        ass_set_aspect_ratio(vf->priv->renderer_realaspect,
+                             (double)width / height * d_height / d_width, 1);
+        ass_set_aspect_ratio(vf->priv->renderer_vsfilter, 1, 1);
     }
 
     return vf_next_config(vf, vf->priv->outw, vf->priv->outh, d_width,
@@ -351,9 +357,11 @@ static int render_frame(struct vf_instance *vf, mp_image_t *mpi,
 static int put_image(struct vf_instance *vf, mp_image_t *mpi, double pts)
 {
     ASS_Image *images = 0;
-    if (sub_visibility && vf->priv->ass_priv && vf->priv->osd->ass_track
+    ASS_Renderer *renderer = vf->priv->osd->vsfilter_aspect ?
+        vf->priv->renderer_vsfilter : vf->priv->renderer_realaspect;
+    if (sub_visibility && renderer && vf->priv->osd->ass_track
 	&& (pts != MP_NOPTS_VALUE))
-        images = ass_mp_render_frame(vf->priv->ass_priv,
+        images = ass_mp_render_frame(renderer,
                                      vf->priv->osd->ass_track,
 				     (pts + sub_delay) * 1000 + .5, NULL);
 
@@ -382,13 +390,19 @@ static int control(vf_instance_t *vf, int request, void *data)
         vf->priv->osd = data;
         break;
     case VFCTRL_INIT_EOSD:
-        vf->priv->ass_priv = ass_renderer_init((ASS_Library *)data);
-        if (!vf->priv->ass_priv)
+        vf->priv->renderer_realaspect = ass_renderer_init((ASS_Library *)data);
+        if (!vf->priv->renderer_realaspect)
             return CONTROL_FALSE;
-        ass_configure_fonts(vf->priv->ass_priv);
+        vf->priv->renderer_vsfilter = ass_renderer_init((ASS_Library *)data);
+        if (!vf->priv->renderer_vsfilter) {
+            ass_renderer_done(vf->priv->renderer_realaspect);
+            return CONTROL_FALSE;
+        }
+        ass_configure_fonts(vf->priv->renderer_realaspect);
+        ass_configure_fonts(vf->priv->renderer_vsfilter);
         return CONTROL_TRUE;
     case VFCTRL_DRAW_EOSD:
-        if (vf->priv->ass_priv)
+        if (vf->priv->renderer_realaspect)
             return CONTROL_TRUE;
         break;
     }
@@ -397,8 +411,10 @@ static int control(vf_instance_t *vf, int request, void *data)
 
 static void uninit(struct vf_instance *vf)
 {
-    if (vf->priv->ass_priv)
-        ass_renderer_done(vf->priv->ass_priv);
+    if (vf->priv->renderer_realaspect) {
+        ass_renderer_done(vf->priv->renderer_realaspect);
+        ass_renderer_done(vf->priv->renderer_vsfilter);
+    }
     free(vf->priv->planes[1]);
     free(vf->priv->planes[2]);
     free(vf->priv->line_limits);
diff --git a/libmpcodecs/vf_vo.c b/libmpcodecs/vf_vo.c
index 7e6697d527..b10181cf14 100644
--- a/libmpcodecs/vf_vo.c
+++ b/libmpcodecs/vf_vo.c
@@ -39,7 +39,8 @@ extern float sub_delay;
 struct vf_priv_s {
     struct vo *vo;
 #ifdef CONFIG_ASS
-    ASS_Renderer *ass_priv;
+    ASS_Renderer *renderer_realaspect;
+    ASS_Renderer *renderer_vsfilter;
     bool prev_visibility;
     double scale_ratio;
 #endif
@@ -83,9 +84,12 @@ static int config(struct vf_instance *vf,
 #ifdef CONFIG_ASS
     vf->priv->scale_ratio = (double) d_width / d_height * height / width;
 
-    if (vf->priv->ass_priv)
-	ass_configure(vf->priv->ass_priv, width, height,
+    if (vf->priv->renderer_realaspect) {
+	ass_configure(vf->priv->renderer_realaspect, width, height,
                       vf->default_caps & VFCAP_EOSD_UNSCALED);
+	ass_configure(vf->priv->renderer_vsfilter, width, height,
+                      vf->default_caps & VFCAP_EOSD_UNSCALED);
+    }
 #endif
 
     return 1;
@@ -131,9 +135,16 @@ static int control(struct vf_instance *vf, int request, void* data)
 #ifdef CONFIG_ASS
     case VFCTRL_INIT_EOSD:
     {
-        vf->priv->ass_priv = ass_renderer_init((ASS_Library*)data);
-        if (!vf->priv->ass_priv) return CONTROL_FALSE;
-        ass_configure_fonts(vf->priv->ass_priv);
+        vf->priv->renderer_realaspect = ass_renderer_init(data);
+        if (!vf->priv->renderer_realaspect)
+            return CONTROL_FALSE;
+        vf->priv->renderer_vsfilter = ass_renderer_init(data);
+        if (!vf->priv->renderer_vsfilter) {
+            ass_renderer_done(vf->priv->renderer_realaspect);
+            return CONTROL_FALSE;
+        }
+        ass_configure_fonts(vf->priv->renderer_realaspect);
+        ass_configure_fonts(vf->priv->renderer_vsfilter);
         vf->priv->prev_visibility = false;
         return CONTROL_TRUE;
     }
@@ -142,16 +153,29 @@ static int control(struct vf_instance *vf, int request, void* data)
         struct osd_state *osd = data;
         mp_eosd_images_t images = {NULL, 2};
         double pts = video_out->next_pts;
-        if (!video_out->config_ok || !vf->priv->ass_priv) return CONTROL_FALSE;
+        ASS_Renderer *renderer;
+        double scale;
+        if (osd->vsfilter_aspect) {
+            renderer = vf->priv->renderer_vsfilter;
+            scale = vf->priv->scale_ratio;
+        } else {
+            renderer = vf->priv->renderer_realaspect;
+            scale = 1;
+        }
+        if (!video_out->config_ok || !renderer)
+            return CONTROL_FALSE;
+        if (osd->ass_track_changed)
+            vf->priv->prev_visibility = false;
+        osd->ass_track_changed = false;
         if (sub_visibility && osd->ass_track && (pts != MP_NOPTS_VALUE)) {
             struct mp_eosd_res res = {0};
             if (vo_control(video_out, VOCTRL_GET_EOSD_RES, &res) == VO_TRUE) {
-                ass_set_frame_size(vf->priv->ass_priv, res.w, res.h);
-                ass_set_margins(vf->priv->ass_priv, res.mt, res.mb, res.ml, res.mr);
-                ass_set_aspect_ratio(vf->priv->ass_priv, vf->priv->scale_ratio, 1);
+                ass_set_frame_size(renderer, res.w, res.h);
+                ass_set_margins(renderer, res.mt, res.mb, res.ml, res.mr);
+                ass_set_aspect_ratio(renderer, scale, 1);
             }
 
-            images.imgs = ass_mp_render_frame(vf->priv->ass_priv,
+            images.imgs = ass_mp_render_frame(renderer,
                                               osd->ass_track,
                                               (pts+sub_delay) * 1000 + .5,
                                               &images.changed);
@@ -223,8 +247,10 @@ static void uninit(struct vf_instance *vf)
          * to get rid of numbered-mpi references that will now be invalid. */
         vo_seek_reset(video_out);
 #ifdef CONFIG_ASS
-        if (vf->priv->ass_priv)
-            ass_renderer_done(vf->priv->ass_priv);
+        if (vf->priv->renderer_realaspect) {
+            ass_renderer_done(vf->priv->renderer_realaspect);
+            ass_renderer_done(vf->priv->renderer_vsfilter);
+        }
 #endif
         free(vf->priv);
     }