subs: use correct font aspect ratio for libass + converted subs

Rendering of ASS subtitles tries to be bug compatible with VSFilter
and stretches fonts when the video is anamorphic (some scripts try to
compensate for this VSFilter behavior, so trying to render them
"correctly" would give the wrong result). However this behavior is not
appropriate for subtitles we converted to ASS format ourselves for
libass rendering, as they certainly don't have VSFilter bug
workarounds. Change the code to use different behavior for "native"
ASS tracks and converted ones. It's questionable whether the
VSFilter-compatible behavior is appropriate for external .ass files
either, as there could be anamorphic and non-anamorphic versions of
the same video and the bug-compatible behavior can only be correct for
one alternative at most. However it's probably better to keep it as a
default at least, so that extracting a muxed subtitle track and using
that does not give behavior different from the original muxed one.

The aspect ratio setting is per ASS_Renderer, and changing it resets
libass caches. For that reason this commit adds separate renderer
instances to use for the "correct" and "VSFilter bug compatible"
cases.

7 files changed, 85 insertions, 28 deletions

diff --git a/command.c b/command.c
index 40ea205c00..0cb5c35f15 100644
--- a/command.c
+++ b/command.c
@@ -1664,9 +1664,14 @@ static int mp_property_sub(m_option_t *prop, int action, void *arg,
     } else if (source == SUB_SOURCE_SUBS) {
         mpctx->set_of_sub_pos = source_pos;
 #ifdef CONFIG_ASS
-        if (opts->ass_enabled && mpctx->set_of_ass_tracks[mpctx->set_of_sub_pos])
-            mpctx->osd->ass_track = mpctx->set_of_ass_tracks[mpctx->set_of_sub_pos];
-        else
+        if (opts->ass_enabled
+            && mpctx->set_of_ass_tracks[mpctx->set_of_sub_pos]) {
+            mpctx->osd->ass_track =
+                mpctx->set_of_ass_tracks[mpctx->set_of_sub_pos];
+            mpctx->osd->ass_track_changed = true;
+            mpctx->osd->vsfilter_aspect =
+                mpctx->track_was_native_ass[mpctx->set_of_sub_pos];
+        } else
 #endif
         {
             mpctx->subdata = mpctx->set_of_subtitles[mpctx->set_of_sub_pos];
diff --git a/libmpcodecs/vf_ass.c b/libmpcodecs/vf_ass.c
index f1a43b52bd..d695fd206d 100644
--- a/libmpcodecs/vf_ass.c
+++ b/libmpcodecs/vf_ass.c
@@ -61,7 +61,8 @@ static const struct vf_priv_s {
     int auto_insert;
 
     struct osd_state *osd;
-    ASS_Renderer *ass_priv;
+    ASS_Renderer *renderer_realaspect;
+    ASS_Renderer *renderer_vsfilter;
 
     unsigned char *planes[3];
     struct line_limits {
@@ -92,9 +93,14 @@ static int config(struct vf_instance *vf,
     vf->priv->planes[2]   = malloc(vf->priv->outw * vf->priv->outh);
     vf->priv->line_limits = malloc((vf->priv->outh + 1) / 2 * sizeof(*vf->priv->line_limits));
 
-    if (vf->priv->ass_priv) {
-        ass_configure(vf->priv->ass_priv, vf->priv->outw, vf->priv->outh, 0);
-        ass_set_aspect_ratio(vf->priv->ass_priv, 1, 1);
+    if (vf->priv->renderer_realaspect) {
+        ass_configure(vf->priv->renderer_realaspect,
+                      vf->priv->outw, vf->priv->outh, 0);
+        ass_configure(vf->priv->renderer_vsfilter,
+                      vf->priv->outw, vf->priv->outh, 0);
+        ass_set_aspect_ratio(vf->priv->renderer_realaspect,
+                             (double)width / height * d_height / d_width, 1);
+        ass_set_aspect_ratio(vf->priv->renderer_vsfilter, 1, 1);
     }
 
     return vf_next_config(vf, vf->priv->outw, vf->priv->outh, d_width,
@@ -351,9 +357,11 @@ static int render_frame(struct vf_instance *vf, mp_image_t *mpi,
 static int put_image(struct vf_instance *vf, mp_image_t *mpi, double pts)
 {
     ASS_Image *images = 0;
-    if (sub_visibility && vf->priv->ass_priv && vf->priv->osd->ass_track
+    ASS_Renderer *renderer = vf->priv->osd->vsfilter_aspect ?
+        vf->priv->renderer_vsfilter : vf->priv->renderer_realaspect;
+    if (sub_visibility && renderer && vf->priv->osd->ass_track
 	&& (pts != MP_NOPTS_VALUE))
-        images = ass_mp_render_frame(vf->priv->ass_priv,
+        images = ass_mp_render_frame(renderer,
                                      vf->priv->osd->ass_track,
 				     (pts + sub_delay) * 1000 + .5, NULL);
 
@@ -382,13 +390,19 @@ static int control(vf_instance_t *vf, int request, void *data)
         vf->priv->osd = data;
         break;
     case VFCTRL_INIT_EOSD:
-        vf->priv->ass_priv = ass_renderer_init((ASS_Library *)data);
-        if (!vf->priv->ass_priv)
+        vf->priv->renderer_realaspect = ass_renderer_init((ASS_Library *)data);
+        if (!vf->priv->renderer_realaspect)
             return CONTROL_FALSE;
-        ass_configure_fonts(vf->priv->ass_priv);
+        vf->priv->renderer_vsfilter = ass_renderer_init((ASS_Library *)data);
+        if (!vf->priv->renderer_vsfilter) {
+            ass_renderer_done(vf->priv->renderer_realaspect);
+            return CONTROL_FALSE;
+        }
+        ass_configure_fonts(vf->priv->renderer_realaspect);
+        ass_configure_fonts(vf->priv->renderer_vsfilter);
         return CONTROL_TRUE;
     case VFCTRL_DRAW_EOSD:
-        if (vf->priv->ass_priv)
+        if (vf->priv->renderer_realaspect)
             return CONTROL_TRUE;
         break;
     }
@@ -397,8 +411,10 @@ static int control(vf_instance_t *vf, int request, void *data)
 
 static void uninit(struct vf_instance *vf)
 {
-    if (vf->priv->ass_priv)
-        ass_renderer_done(vf->priv->ass_priv);
+    if (vf->priv->renderer_realaspect) {
+        ass_renderer_done(vf->priv->renderer_realaspect);
+        ass_renderer_done(vf->priv->renderer_vsfilter);
+    }
     free(vf->priv->planes[1]);
     free(vf->priv->planes[2]);
     free(vf->priv->line_limits);
diff --git a/libmpcodecs/vf_vo.c b/libmpcodecs/vf_vo.c
index 7e6697d527..b10181cf14 100644
--- a/libmpcodecs/vf_vo.c
+++ b/libmpcodecs/vf_vo.c
@@ -39,7 +39,8 @@ extern float sub_delay;
 struct vf_priv_s {
     struct vo *vo;
 #ifdef CONFIG_ASS
-    ASS_Renderer *ass_priv;
+    ASS_Renderer *renderer_realaspect;
+    ASS_Renderer *renderer_vsfilter;
     bool prev_visibility;
     double scale_ratio;
 #endif
@@ -83,9 +84,12 @@ static int config(struct vf_instance *vf,
 #ifdef CONFIG_ASS
     vf->priv->scale_ratio = (double) d_width / d_height * height / width;
 
-    if (vf->priv->ass_priv)
-	ass_configure(vf->priv->ass_priv, width, height,
+    if (vf->priv->renderer_realaspect) {
+	ass_configure(vf->priv->renderer_realaspect, width, height,
                       vf->default_caps & VFCAP_EOSD_UNSCALED);
+	ass_configure(vf->priv->renderer_vsfilter, width, height,
+                      vf->default_caps & VFCAP_EOSD_UNSCALED);
+    }
 #endif
 
     return 1;
@@ -131,9 +135,16 @@ static int control(struct vf_instance *vf, int request, void* data)
 #ifdef CONFIG_ASS
     case VFCTRL_INIT_EOSD:
     {
-        vf->priv->ass_priv = ass_renderer_init((ASS_Library*)data);
-        if (!vf->priv->ass_priv) return CONTROL_FALSE;
-        ass_configure_fonts(vf->priv->ass_priv);
+        vf->priv->renderer_realaspect = ass_renderer_init(data);
+        if (!vf->priv->renderer_realaspect)
+            return CONTROL_FALSE;
+        vf->priv->renderer_vsfilter = ass_renderer_init(data);
+        if (!vf->priv->renderer_vsfilter) {
+            ass_renderer_done(vf->priv->renderer_realaspect);
+            return CONTROL_FALSE;
+        }
+        ass_configure_fonts(vf->priv->renderer_realaspect);
+        ass_configure_fonts(vf->priv->renderer_vsfilter);
         vf->priv->prev_visibility = false;
         return CONTROL_TRUE;
     }
@@ -142,16 +153,29 @@ static int control(struct vf_instance *vf, int request, void* data)
         struct osd_state *osd = data;
         mp_eosd_images_t images = {NULL, 2};
         double pts = video_out->next_pts;
-        if (!video_out->config_ok || !vf->priv->ass_priv) return CONTROL_FALSE;
+        ASS_Renderer *renderer;
+        double scale;
+        if (osd->vsfilter_aspect) {
+            renderer = vf->priv->renderer_vsfilter;
+            scale = vf->priv->scale_ratio;
+        } else {
+            renderer = vf->priv->renderer_realaspect;
+            scale = 1;
+        }
+        if (!video_out->config_ok || !renderer)
+            return CONTROL_FALSE;
+        if (osd->ass_track_changed)
+            vf->priv->prev_visibility = false;
+        osd->ass_track_changed = false;
         if (sub_visibility && osd->ass_track && (pts != MP_NOPTS_VALUE)) {
             struct mp_eosd_res res = {0};
             if (vo_control(video_out, VOCTRL_GET_EOSD_RES, &res) == VO_TRUE) {
-                ass_set_frame_size(vf->priv->ass_priv, res.w, res.h);
-                ass_set_margins(vf->priv->ass_priv, res.mt, res.mb, res.ml, res.mr);
-                ass_set_aspect_ratio(vf->priv->ass_priv, vf->priv->scale_ratio, 1);
+                ass_set_frame_size(renderer, res.w, res.h);
+                ass_set_margins(renderer, res.mt, res.mb, res.ml, res.mr);
+                ass_set_aspect_ratio(renderer, scale, 1);
             }
 
-            images.imgs = ass_mp_render_frame(vf->priv->ass_priv,
+            images.imgs = ass_mp_render_frame(renderer,
                                               osd->ass_track,
                                               (pts+sub_delay) * 1000 + .5,
                                               &images.changed);
@@ -223,8 +247,10 @@ static void uninit(struct vf_instance *vf)
          * to get rid of numbered-mpi references that will now be invalid. */
         vo_seek_reset(video_out);
 #ifdef CONFIG_ASS
-        if (vf->priv->ass_priv)
-            ass_renderer_done(vf->priv->ass_priv);
+        if (vf->priv->renderer_realaspect) {
+            ass_renderer_done(vf->priv->renderer_realaspect);
+            ass_renderer_done(vf->priv->renderer_vsfilter);
+        }
 #endif
         free(vf->priv);
     }
diff --git a/libvo/sub.h b/libvo/sub.h
index 12d5474e5f..8309613ea6 100644
--- a/libvo/sub.h
+++ b/libvo/sub.h
@@ -19,6 +19,8 @@
 #ifndef MPLAYER_SUB_H
 #define MPLAYER_SUB_H
 
+#include <stdbool.h>
+
 typedef struct mp_osd_bbox_s {
     int x1,y1,x2,y2;
 } mp_osd_bbox_t;
@@ -70,6 +72,8 @@ struct osd_state {
     unsigned char osd_text[128];
     struct font_desc *sub_font;
     struct ass_track *ass_track;
+    bool ass_track_changed;
+    bool vsfilter_aspect;
 };
 
 #include "subreader.h"
diff --git a/mp_core.h b/mp_core.h
index 3e8b17132a..5ad01a0964 100644
--- a/mp_core.h
+++ b/mp_core.h
@@ -198,6 +198,7 @@ typedef struct MPContext {
     // parsed by libass or NULL if format unsupported
     struct ass_track *set_of_ass_tracks[MAX_SUBTITLE_FILES];
     sub_data* set_of_subtitles[MAX_SUBTITLE_FILES];
+    bool track_was_native_ass[MAX_SUBTITLE_FILES];
 
     int file_format;
 
diff --git a/mplayer.c b/mplayer.c
index 00d568b359..90e454961a 100644
--- a/mplayer.c
+++ b/mplayer.c
@@ -1092,6 +1092,7 @@ void add_subtitles(struct MPContext *mpctx, char *filename, float fps, int noerr
     struct MPOpts *opts = &mpctx->opts;
     sub_data *subd = NULL;
     struct ass_track *asst = NULL;
+    bool is_native_ass = false;
 
     if (filename == NULL || mpctx->set_of_sub_size >= MAX_SUBTITLE_FILES)
 	return;
@@ -1103,6 +1104,7 @@ void add_subtitles(struct MPContext *mpctx, char *filename, float fps, int noerr
 #else
         asst = ass_read_stream(ass_library, filename, 0);
 #endif
+        is_native_ass = asst;
         if (!asst) {
             subd = sub_read_file(filename, fps, &mpctx->opts);
             if (subd) {
@@ -1124,6 +1126,7 @@ void add_subtitles(struct MPContext *mpctx, char *filename, float fps, int noerr
 
     mpctx->set_of_ass_tracks[mpctx->set_of_sub_size] = asst;
     mpctx->set_of_subtitles[mpctx->set_of_sub_size] = subd;
+    mpctx->track_was_native_ass[mpctx->set_of_sub_size] = is_native_ass;
     mp_msg(MSGT_IDENTIFY, MSGL_INFO, "ID_FILE_SUB_ID=%d\n", mpctx->set_of_sub_size);
     mp_msg(MSGT_IDENTIFY, MSGL_INFO, "ID_FILE_SUB_FILENAME=%s\n",
 	   filename_recode(filename));
diff --git a/sub/sd_ass.c b/sub/sd_ass.c
index 5eae2ebc28..a5879873a9 100644
--- a/sub/sd_ass.c
+++ b/sub/sd_ass.c
@@ -62,6 +62,8 @@ static void init(struct sh_sub *sh, struct osd_state *osd)
 
     assert(osd->ass_track == NULL);
     osd->ass_track = ctx->ass_track;
+    osd->vsfilter_aspect = sh->type == 'a';
+    osd->ass_track_changed = true;
 }
 
 static void decode(struct sh_sub *sh, struct osd_state *osd, void *data,