sub: protect ASS_Renderer state

Each subtitle track gets its own decoder instance (sd_ass). But they use
a shared ASS_Renderer. This is done mainly because of fontconfig.
Initializing fontconfig is very slow when using it with memory fonts, so
there's a practical need to cache this memory font state, which is done
by not creating separate ASS_Renderers. This is very dirty and very
evil, but we probably can't get rid of it any time soon.

The shared ASS_Renderer was not properly synchronized. While the program
logic guarantees that only one sd_ass instance is visible at a time,
there are other interactions that require synchronization. In
particular, I suspect concurrent execution of mp_ass_configure_fonts()
and sd_ass.get_bitmaps cause issues in a newer libass development
branch.

So here's a shitty hack that hopefully fixes things, hopefully only
until libass becomes less dependent on fontconfig.

4 files changed, 16 insertions, 3 deletions

diff --git a/sub/dec_sub.c b/sub/dec_sub.c
index f03526ca8b..c16f7b5887 100644
--- a/sub/dec_sub.c
+++ b/sub/dec_sub.c
@@ -156,11 +156,13 @@ void sub_set_extradata(struct dec_sub *sub, void *data, int data_len)
 }
 
 void sub_set_ass_renderer(struct dec_sub *sub, struct ass_library *ass_library,
-                          struct ass_renderer *ass_renderer)
+                          struct ass_renderer *ass_renderer,
+                          pthread_mutex_t *ass_lock)
 {
     pthread_mutex_lock(&sub->lock);
     sub->init_sd.ass_library = ass_library;
     sub->init_sd.ass_renderer = ass_renderer;
+    sub->init_sd.ass_lock = ass_lock;
     pthread_mutex_unlock(&sub->lock);
 }
 
diff --git a/sub/dec_sub.h b/sub/dec_sub.h
index 495ac46402..ddec46e243 100644
--- a/sub/dec_sub.h
+++ b/sub/dec_sub.h
@@ -3,6 +3,7 @@
 
 #include <stdbool.h>
 #include <stdint.h>
+#include <pthread.h>
 
 #include "osd.h"
 
@@ -31,7 +32,8 @@ void sub_set_video_res(struct dec_sub *sub, int w, int h);
 void sub_set_video_fps(struct dec_sub *sub, double fps);
 void sub_set_extradata(struct dec_sub *sub, void *data, int data_len);
 void sub_set_ass_renderer(struct dec_sub *sub, struct ass_library *ass_library,
-                          struct ass_renderer *ass_renderer);
+                          struct ass_renderer *ass_renderer,
+                          pthread_mutex_t *ass_lock);
 void sub_init_from_sh(struct dec_sub *sub, struct sh_stream *sh);
 
 bool sub_is_initialized(struct dec_sub *sub);
diff --git a/sub/sd.h b/sub/sd.h
index 7b2399056a..a77028a43c 100644
--- a/sub/sd.h
+++ b/sub/sd.h
@@ -28,6 +28,7 @@ struct sd {
     // Shared renderer for ASS - done to avoid reloading embedded fonts.
     struct ass_library *ass_library;
     struct ass_renderer *ass_renderer;
+    pthread_mutex_t *ass_lock;
 
     // If false, try to remove multiple subtitles.
     // (Only for decoders which have accept_packets_in_advance set.)
diff --git a/sub/sd_ass.c b/sub/sd_ass.c
index 6f3e24a6f9..f8b1180560 100644
--- a/sub/sd_ass.c
+++ b/sub/sd_ass.c
@@ -59,7 +59,7 @@ static bool supports_format(const char *format)
 static int init(struct sd *sd)
 {
     struct MPOpts *opts = sd->opts;
-    if (!sd->ass_library || !sd->ass_renderer || !sd->codec)
+    if (!sd->ass_library || !sd->ass_renderer || !sd->ass_lock || !sd->codec)
         return -1;
 
     struct sd_ass_priv *ctx = talloc_zero(NULL, struct sd_ass_priv);
@@ -67,6 +67,8 @@ static int init(struct sd *sd)
 
     ctx->is_converted = sd->converted_from != NULL;
 
+    pthread_mutex_lock(sd->ass_lock);
+
     ctx->ass_track = ass_new_track(sd->ass_library);
     if (!ctx->is_converted)
         ctx->ass_track->track_type = TRACK_TYPE_ASS;
@@ -78,6 +80,8 @@ static int init(struct sd *sd)
 
     mp_ass_add_default_styles(ctx->ass_track, opts);
 
+    pthread_mutex_unlock(sd->ass_lock);
+
     return 0;
 }
 
@@ -197,6 +201,8 @@ static void get_bitmaps(struct sd *sd, struct mp_osd_res dim, double pts,
     if (pts == MP_NOPTS_VALUE || !sd->ass_renderer)
         return;
 
+    pthread_mutex_lock(sd->ass_lock);
+
     ASS_Renderer *renderer = sd->ass_renderer;
     double scale = dim.display_par;
     if (!ctx->is_converted && (!opts->ass_style_override ||
@@ -224,6 +230,8 @@ static void get_bitmaps(struct sd *sd, struct mp_osd_res dim, double pts,
 
     if (!ctx->is_converted)
         mangle_colors(sd, res);
+
+    pthread_mutex_unlock(sd->ass_lock);
 }
 
 struct buf {