subs: libass: use a single persistent renderer for subtitles

To draw libass subtitles, the code used ASS_Renderer objects created
in vf_vo (VO rendering) or vf_ass. They were destroyed and recreated
together with the video filter  chain. Change the code to use a single
persistent renderer instance stored in the main osd_state struct.
Because libass seems to misbehave if fonts are changed while a
renderer exists (even if ass_set_fonts() is called on the renderer
afterwards), the renderer is recreated after adding embedded fonts.

The known benefits are simpler code and avoiding delays when switching
between timeline parts from different files (libass fontconfig
initialization, needed when creating a new renderer, can take a long
time in some cases; switching between files rebuilds the video filter
chain, and this required recreating the renderers). On the other hand,
I'm not sure whether this could cause inefficient bitmap caching in
libass; explicitly resetting the renderer in some cases could be
beneficial. The new code does not keep the distinction of separate
renderers for vsfilter munged aspect vs normal; this means that
changing subtitle tracks can lose cache for the previous track.

The new code always sets some libass parameters on each rendering
call, which were previously only set if they had potentially changed.
This should be harmless as libass itself has checks to see if the
values differ from previous ones.

Conflicts:
	command.c
	libmpcodecs/vf_ass.c
	libmpcodecs/vf_vo.c
	mplayer.c
	sub/ass_mp.c

10 files changed, 72 insertions, 119 deletions

diff --git a/command.c b/command.c
index f0b17719db..f733bbc0f0 100644
--- a/command.c
+++ b/command.c
@@ -1519,7 +1519,7 @@ static int mp_property_ass_use_margins(m_option_t *prop, int action,
             return M_PROPERTY_ERROR;
     case M_PROPERTY_STEP_UP:
     case M_PROPERTY_STEP_DOWN:
-        mpctx->osd->ass_force_reload = true;
+        vo_osd_changed(OSDTYPE_SUBTITLE);
     default:
         return m_property_flag(prop, action, arg, &opts->ass_use_margins);
     }
@@ -1537,8 +1537,7 @@ static int mp_property_ass_vsfilter_aspect_compat(m_option_t *prop, int action,
             return M_PROPERTY_ERROR;
     case M_PROPERTY_STEP_UP:
     case M_PROPERTY_STEP_DOWN:
-        //has to re-render subs with new aspect ratio
-        mpctx->osd->ass_force_reload = 1;
+        vo_osd_changed(OSDTYPE_SUBTITLE);
     default:
         return m_property_flag(prop, action, arg,
                                &mpctx->opts.ass_vsfilter_aspect_compat);
@@ -1581,10 +1580,8 @@ static int mp_property_sub_scale(m_option_t *prop, int action, void *arg,
             return M_PROPERTY_ERROR;
         M_PROPERTY_CLAMP(prop, *(float *) arg);
 #ifdef CONFIG_ASS
-        if (opts->ass_enabled) {
+        if (opts->ass_enabled)
             opts->ass_font_scale = *(float *) arg;
-            mpctx->osd->ass_force_reload = true;
-        }
 #endif
         text_font_scale_factor = *(float *) arg;
         vo_osd_resized();
@@ -1596,7 +1593,6 @@ static int mp_property_sub_scale(m_option_t *prop, int action, void *arg,
             opts->ass_font_scale += (arg ? *(float *) arg : 0.1) *
                               (action == M_PROPERTY_STEP_UP ? 1.0 : -1.0);
             M_PROPERTY_CLAMP(prop, opts->ass_font_scale);
-            mpctx->osd->ass_force_reload = true;
         }
 #endif
         text_font_scale_factor += (arg ? *(float *) arg : 0.1) *
diff --git a/libmpcodecs/vf_ass.c b/libmpcodecs/vf_ass.c
index f7e3e62ec0..a346f658cf 100644
--- a/libmpcodecs/vf_ass.c
+++ b/libmpcodecs/vf_ass.c
@@ -62,9 +62,7 @@ static const struct vf_priv_s {
     int auto_insert;
 
     struct osd_state *osd;
-    ASS_Renderer *renderer;
-
-    double realaspect;
+    double aspect_correction;
 
     unsigned char *planes[3];
     struct line_limits {
@@ -93,11 +91,7 @@ static int config(struct vf_instance *vf,
     vf->priv->planes[2]   = malloc(vf->priv->outw * vf->priv->outh);
     vf->priv->line_limits = malloc((vf->priv->outh + 1) / 2 * sizeof(*vf->priv->line_limits));
 
-    if (vf->priv->renderer) {
-        mp_ass_configure(vf->priv->renderer, opts,
-                         vf->priv->outw, vf->priv->outh, 0);
-        vf->priv->realaspect = (double)width / height * d_height / d_width;
-    }
+    vf->priv->aspect_correction = (double)width / height * d_height / d_width;
 
     return vf_next_config(vf, vf->priv->outw, vf->priv->outh, d_width,
 			  d_height, flags, outfmt);
@@ -360,22 +354,21 @@ static int put_image(struct vf_instance *vf, mp_image_t *mpi, double pts)
 {
     struct osd_state *osd = vf->priv->osd;
     ASS_Image *images = 0;
-    double scale = osd->vsfilter_aspect
-            && vf->opts->ass_vsfilter_aspect_compat
-            ? 1 : vf->priv->realaspect;
-    if (sub_visibility && vf->priv->renderer && osd->ass_track
-            && (pts != MP_NOPTS_VALUE)) {
-        ass_set_aspect_ratio(vf->priv->renderer, scale, 1);
-        if (osd->ass_force_reload)
-            mp_ass_reload_options(vf->priv->renderer, vf->opts);
-        osd->ass_force_reload = false;
-        images = ass_render_frame(vf->priv->renderer, osd->ass_track,
+    ASS_Renderer *renderer = osd->ass_renderer;
+    bool vs = osd->vsfilter_aspect && vf->opts->ass_vsfilter_aspect_compat;
+    if (sub_visibility && osd->ass_track && (pts != MP_NOPTS_VALUE)) {
+        struct mp_eosd_res dim = { .w = vf->priv->outw, .h = vf->priv->outh,
+                                   .mt = vf->opts->ass_top_margin,
+                                   .mb = vf->opts->ass_bottom_margin };
+        mp_ass_configure(renderer, vf->opts, &dim, 0);
+        ass_set_aspect_ratio(renderer,
+                             vs ? 1. : vf->priv->aspect_correction, 1);
+        images = ass_render_frame(renderer, osd->ass_track,
                         (pts - osd->sub_offset + sub_delay) * 1000 + .5, NULL);
     }
 
     prepare_image(vf, mpi);
-    if (images)
-        render_frame(vf, mpi, images);
+    render_frame(vf, mpi, images);
 
     return vf_next_put_image(vf, vf->dmpi, pts);
 }
@@ -398,23 +391,15 @@ static int control(vf_instance_t *vf, int request, void *data)
         vf->priv->osd = data;
         break;
     case VFCTRL_INIT_EOSD:
-        vf->priv->renderer = ass_renderer_init((ASS_Library *)data);
-        if (!vf->priv->renderer)
-            return CONTROL_FALSE;
-        mp_ass_configure_fonts(vf->priv->renderer);
         return CONTROL_TRUE;
     case VFCTRL_DRAW_EOSD:
-        if (vf->priv->renderer)
-            return CONTROL_TRUE;
-        break;
+        return CONTROL_TRUE;
     }
     return vf_next_control(vf, request, data);
 }
 
 static void uninit(struct vf_instance *vf)
 {
-    if (vf->priv->renderer)
-        ass_renderer_done(vf->priv->renderer);
     free(vf->priv->planes[1]);
     free(vf->priv->planes[2]);
     free(vf->priv->line_limits);
diff --git a/libmpcodecs/vf_vo.c b/libmpcodecs/vf_vo.c
index 0c747beb83..0db75b670e 100644
--- a/libmpcodecs/vf_vo.c
+++ b/libmpcodecs/vf_vo.c
@@ -35,11 +35,8 @@
 
 struct vf_priv_s {
     struct vo *vo;
-#ifdef CONFIG_ASS
-    ASS_Renderer *renderer;
     bool prev_visibility;
     double scale_ratio;
-#endif
 };
 #define video_out (vf->priv->vo)
 
@@ -79,17 +76,10 @@ static int config(struct vf_instance *vf,
     if (vo_config(video_out, width, height, d_width, d_height, flags, outfmt))
         return 0;
 
-#ifdef CONFIG_ASS
     vf->priv->scale_ratio = (double) d_width / d_height * height / width;
 
-    if (vf->priv->renderer) {
-        mp_ass_configure(vf->priv->renderer, vf->opts, width, height,
-                         vf->default_caps & VFCAP_EOSD_UNSCALED);
-    }
-
     // force EOSD change detection reset
     vf->priv->prev_visibility = false;
-#endif
 
     return 1;
 }
@@ -133,46 +123,33 @@ static int control(struct vf_instance *vf, int request, void *data)
         return vo_control(video_out, VOCTRL_GET_EQUALIZER, &param) == VO_TRUE;
     }
 #ifdef CONFIG_ASS
-    case VFCTRL_INIT_EOSD:
-    {
-        vf->priv->renderer = ass_renderer_init(data);
-        if (!vf->priv->renderer)
-            return CONTROL_FALSE;
-        mp_ass_configure_fonts(vf->priv->renderer);
+    case VFCTRL_INIT_EOSD: {
         vf->priv->prev_visibility = false;
         return CONTROL_TRUE;
     }
+
     case VFCTRL_DRAW_EOSD: {
+        struct mp_eosd_res dim = { 0 };
+        if (!video_out->config_ok ||
+                vo_control(video_out, VOCTRL_GET_EOSD_RES, &dim) != true) {
+            vf->priv->prev_visibility = false;
+            return CONTROL_FALSE;
+        }
         struct osd_state *osd = data;
-        mp_eosd_images_t images = {NULL, 2};
-        ASS_Renderer *renderer = vf->priv->renderer;
-        double scale;
-        if (osd->vsfilter_aspect && vf->opts->ass_vsfilter_aspect_compat) {
+        mp_eosd_images_t images = { NULL, 2 };
+        ASS_Renderer *renderer = osd->ass_renderer;
+        double scale = 1;
+        if (osd->vsfilter_aspect && vf->opts->ass_vsfilter_aspect_compat)
             scale = vf->priv->scale_ratio;
-        } else {
-            scale = 1;
-        }
-        if (!video_out->config_ok || !renderer)
-            return CONTROL_FALSE;
-        if (osd->ass_track_changed)
-            vf->priv->prev_visibility = false;
-        osd->ass_track_changed = false;
         if (sub_visibility && osd->ass_track && (osd->pts != MP_NOPTS_VALUE)) {
-            struct mp_eosd_res res = { 0 };
-            if (vo_control(video_out, VOCTRL_GET_EOSD_RES, &res) == VO_TRUE) {
-                ass_set_frame_size(renderer, res.w, res.h);
-                ass_set_margins(renderer, res.mt, res.mb, res.ml, res.mr);
-                ass_set_aspect_ratio(renderer, scale, 1);
-            }
-
-            if (osd->ass_force_reload)
-                mp_ass_reload_options(vf->priv->renderer, vf->opts);
+            mp_ass_configure(renderer, vf->opts, &dim,
+                             vf->default_caps & VFCAP_EOSD_UNSCALED);
+            ass_set_aspect_ratio(renderer, scale, 1);
             images.imgs = ass_render_frame(renderer, osd->ass_track,
                                            (osd->pts + sub_delay) * 1000 + .5,
                                            &images.changed);
-            if (!vf->priv->prev_visibility || osd->ass_force_reload)
+            if (!vf->priv->prev_visibility)
                 images.changed = 2;
-            osd->ass_force_reload = false;
             vf->priv->prev_visibility = true;
         } else
             vf->priv->prev_visibility = false;
@@ -242,10 +219,6 @@ static void uninit(struct vf_instance *vf)
         /* Allow VO (which may live on to work with another instance of vf_vo)
          * to get rid of numbered-mpi references that will now be invalid. */
         vo_seek_reset(video_out);
-#ifdef CONFIG_ASS
-        if (vf->priv->renderer)
-            ass_renderer_done(vf->priv->renderer);
-#endif
         free(vf->priv);
     }
 }
diff --git a/libvo/video_out.h b/libvo/video_out.h
index 3042efd8d2..3e46722dfd 100644
--- a/libvo/video_out.h
+++ b/libvo/video_out.h
@@ -92,12 +92,6 @@ struct voctrl_get_equalizer_args {
     int *valueptr;
 };
 
-// VOCTRL_GET_EOSD_RES
-typedef struct mp_eosd_res {
-    int w, h; // screen dimensions, including black borders
-    int mt, mb, ml, mr; // borders (top, bottom, left, right)
-} mp_eosd_res_t;
-
 // VOCTRL_SCREENSHOT
 struct voctrl_screenshot_args {
     // 0: Save image of the currently displayed video frame, in original
diff --git a/mplayer.c b/mplayer.c
index 497517077c..0734defbd6 100644
--- a/mplayer.c
+++ b/mplayer.c
@@ -1971,13 +1971,10 @@ static void reinit_subs(struct MPContext *mpctx)
 #ifdef CONFIG_ASS
         if (opts->ass_enabled && track->ass_track) {
             mpctx->osd->ass_track = track->ass_track;
-            mpctx->osd->ass_track_changed = true;
             mpctx->osd->vsfilter_aspect = track->native_ass_track;
-        } else
-#endif
-        {
-            vo_osd_changed(OSDTYPE_SUBTITLE);
         }
+#endif
+        vo_osd_changed(OSDTYPE_SUBTITLE);
     } else if (track->stream) {
         if (mpctx->sh_sub->type == 'v')
             init_vo_spudec(mpctx);
@@ -3679,7 +3676,7 @@ static void print_timeline(struct MPContext *mpctx)
 static void add_subtitle_fonts_from_sources(struct MPContext *mpctx)
 {
 #ifdef CONFIG_ASS
-    if (mpctx->opts.ass_enabled && mpctx->ass_library) {
+    if (mpctx->opts.ass_enabled) {
         for (int j = 0; j < mpctx->num_sources; j++) {
             struct demuxer *d = mpctx->sources[j];
             for (int i = 0; i < d->num_attachments; i++) {
@@ -3690,6 +3687,13 @@ static void add_subtitle_fonts_from_sources(struct MPContext *mpctx)
             }
         }
     }
+
+    // libass seems to misbehave if fonts are changed while a renderer
+    // exists, so we (re)create the renderer after fonts are set.
+    assert(!mpctx->osd->ass_renderer);
+    mpctx->osd->ass_renderer = ass_renderer_init(mpctx->osd->ass_library);
+    if (mpctx->osd->ass_renderer)
+        mp_ass_configure_fonts(mpctx->osd->ass_renderer);
 #endif
 }
 
@@ -4045,8 +4049,10 @@ terminate_playback:  // don't jump here after ao/vo/getch initialization!
     vo_sub = NULL;
 #ifdef CONFIG_ASS
     mpctx->osd->ass_track = NULL;
-    if (mpctx->ass_library)
-        ass_clear_fonts(mpctx->ass_library);
+    if (mpctx->osd->ass_renderer)
+        ass_renderer_done(mpctx->osd->ass_renderer);
+    mpctx->osd->ass_renderer = NULL;
+    ass_clear_fonts(mpctx->ass_library);
 #endif
 }
 
diff --git a/sub/ass_mp.c b/sub/ass_mp.c
index 8266f45d2e..202664578b 100644
--- a/sub/ass_mp.c
+++ b/sub/ass_mp.c
@@ -225,13 +225,14 @@ ASS_Track *mp_ass_read_stream(ASS_Library *library, const char *fname,
     return track;
 }
 
-void mp_ass_configure(ASS_Renderer *priv, struct MPOpts *opts, int w, int h,
-                      bool unscaled)
+void mp_ass_configure(ASS_Renderer *priv, struct MPOpts *opts,
+                      struct mp_eosd_res *dim, bool unscaled)
 {
     int hinting;
-    ass_set_frame_size(priv, w, h);
-    ass_set_margins(priv, opts->ass_top_margin, opts->ass_bottom_margin, 0, 0);
-    mp_ass_reload_options(priv, opts);
+    ass_set_frame_size(priv, dim->w, dim->h);
+    ass_set_margins(priv, dim->mt, dim->mb, dim->ml, dim->mr);
+    ass_set_use_margins(priv, opts->ass_use_margins);
+    ass_set_font_scale(priv, opts->ass_font_scale);
     if (!unscaled && (opts->ass_hinting & 4))
         hinting = 0;
     else
@@ -294,14 +295,3 @@ ASS_Library *mp_ass_init(struct MPOpts *opts)
     free(path);
     return priv;
 }
-
-void mp_ass_reload_options(ASS_Renderer *priv, struct MPOpts *opts)
-{
-    /* This could be needed for vf_ass case if the margins were actually
-     * runtime configurable, but would be wrong with EOSD:
-     * ass_set_margins(priv, opts->ass_top_margin, opts->ass_bottom_margin,
-     *                 0, 0);
-     */
-    ass_set_use_margins(priv, opts->ass_use_margins);
-    ass_set_font_scale(priv, opts->ass_font_scale);
-}
diff --git a/sub/ass_mp.h b/sub/ass_mp.h
index a222d61bbb..095cf4311d 100644
--- a/sub/ass_mp.h
+++ b/sub/ass_mp.h
@@ -31,6 +31,9 @@
 #include <ass/ass.h>
 #include <ass/ass_types.h>
 
+struct MPOpts;
+struct mp_eosd_res;
+
 ASS_Track *mp_ass_default_track(ASS_Library *library, struct MPOpts *opts);
 ASS_Track *mp_ass_read_subdata(ASS_Library *library, struct MPOpts *opts,
                                sub_data *subdata, double fps);
@@ -38,13 +41,11 @@ ASS_Track *mp_ass_read_stream(ASS_Library *library, const char *fname,
                               char *charset);
 
 struct MPOpts;
-void mp_ass_configure(ASS_Renderer *priv, struct MPOpts *opts, int w, int h,
-                      bool unscaled);
+void mp_ass_configure(ASS_Renderer *priv, struct MPOpts *opts,
+                      struct mp_eosd_res *dim, bool unscaled);
 void mp_ass_configure_fonts(ASS_Renderer *priv);
 ASS_Library *mp_ass_init(struct MPOpts *opts);
 
-void mp_ass_reload_options(ASS_Renderer *priv, struct MPOpts *opts);
-
 #else /* CONFIG_ASS */
 
 /* Needed for EOSD code using this type to compile */
diff --git a/sub/dec_sub.h b/sub/dec_sub.h
index d6fbef25f0..e58ad65550 100644
--- a/sub/dec_sub.h
+++ b/sub/dec_sub.h
@@ -1,6 +1,14 @@
+#ifndef MPLAYER_DEC_SUB_H
+#define MPLAYER_DEC_SUB_H
+
 struct sh_sub;
 struct osd_state;
 
+typedef struct mp_eosd_res {
+    int w, h; // screen dimensions, including black borders
+    int mt, mb, ml, mr; // borders (top, bottom, left, right)
+} mp_eosd_res_t;
+
 static inline bool is_text_sub(int type)
 {
     return type == 't' || type == 'm' || type == 'a';
@@ -12,3 +20,5 @@ void sub_init(struct sh_sub *sh, struct osd_state *osd);
 void sub_reset(struct sh_sub *sh, struct osd_state *osd);
 void sub_switchoff(struct sh_sub *sh, struct osd_state *osd);
 void sub_uninit(struct sh_sub *sh);
+
+#endif
diff --git a/sub/sd_ass.c b/sub/sd_ass.c
index b8924fbb3f..f54c18e805 100644
--- a/sub/sd_ass.c
+++ b/sub/sd_ass.c
@@ -64,7 +64,6 @@ static int init(struct sh_sub *sh, struct osd_state *osd)
     assert(osd->ass_track == NULL);
     osd->ass_track = ctx->ass_track;
     osd->vsfilter_aspect = sh->type == 'a';
-    osd->ass_track_changed = true;
     return 0;
 }
 
diff --git a/sub/sub.h b/sub/sub.h
index 1fe5ec50e7..3ad8dc9217 100644
--- a/sub/sub.h
+++ b/sub/sub.h
@@ -21,6 +21,9 @@
 
 #include <stdbool.h>
 
+#include "subreader.h"
+#include "dec_sub.h"
+
 typedef struct mp_osd_bbox_s {
     int x1,y1,x2,y2;
 } mp_osd_bbox_t;
@@ -58,14 +61,12 @@ typedef struct mp_osd_obj_s {
 
 struct osd_state {
     struct ass_library *ass_library;
-    // flag to signal reinitialization due to ass-related option changes
-    bool ass_force_reload;
+    struct ass_renderer *ass_renderer;
     int w, h;
     char *osd_text;
     struct ass_track *ass_track;
     double pts;
     double sub_offset;
-    bool ass_track_changed;
     bool vsfilter_aspect;
 
     struct ass_renderer *osd_render;
@@ -74,8 +75,6 @@ struct osd_state {
     struct MPOpts *opts;
 };
 
-#include "subreader.h"
-
 extern subtitle* vo_sub;
 
 extern int vo_osd_progbar_type;