sub: always go through sub.c for OSD rendering

Before this commit, vf_vo.c and vf_ass.c were manually calling the
subtitle decoder to retrieve images to render. In particular, this
circumvented the sub-bitmap conversion & caching layer in sub.c.

Change this so that subtitle decoding isn't special anymore, and draws
all subtitles with the normal OSD drawing API.

This is also a step towards removing the need for vf_ass auto-insertion.
In fact, if auto-insertion would be disabled now, VOs with "old" OSD
rendering could still render ASS subtitles in monochrome, because
there is still ASS -> old-OSD bitmap conversion in the sub.c mechanism.

The code is written with the assumption that the subtitle rendering
filter (vf_ass) can render all subtitle formats. Since vf_ass knows the
ASS format only, rendering image subs (i.e. RGBA subs) with it simply
fails. This means that with vo_xv (vf_ass auto-inserted), image subs
wouldn't be rendered. Use a dumb hack to disable rendering subs with a
filter, if we detect that the subs are not in ASS format. (Trying to
render the subs first would probably result in purging the conversion
cache on every frame.)

3 files changed, 102 insertions, 10 deletions

diff --git a/sub/dec_sub.h b/sub/dec_sub.h
index 9c75506c4c..7ccf513438 100644
--- a/sub/dec_sub.h
+++ b/sub/dec_sub.h
@@ -60,6 +60,15 @@ typedef struct sub_bitmaps {
     int bitmap_id, bitmap_pos_id;
 } mp_eosd_images_t;
 
+struct sub_render_params {
+    double pts;
+    struct mp_eosd_res dim;
+    double normal_scale;
+    double vsfilter_scale;
+
+    bool support_rgba;
+};
+
 static inline bool is_text_sub(int type)
 {
     return type == 't' || type == 'm' || type == 'a';
diff --git a/sub/sub.c b/sub/sub.c
index fb1c76c91d..64dba7df20 100644
--- a/sub/sub.c
+++ b/sub/sub.c
@@ -114,6 +114,7 @@ struct osd_state *osd_create(struct MPOpts *opts, struct ass_library *asslib)
         .osd_text = talloc_strdup(osd, ""),
         .progbar_type = -1,
     };
+
     for (int n = 0; n < MAX_OSD_PARTS; n++) {
         struct osd_object *obj = talloc_struct(osd, struct osd_object, {
             .type = n,
@@ -122,6 +123,11 @@ struct osd_state *osd_create(struct MPOpts *opts, struct ass_library *asslib)
             obj->cache[i] = talloc_steal(obj, osd_conv_cache_new());
         osd->objs[n] = obj;
     }
+
+    // OSDTYPE_SPU is an odd case, because vf_ass.c can't render it.
+    osd->objs[OSDTYPE_SUB]->is_sub = true;      // dec_sub.c
+    osd->objs[OSDTYPE_SUBTITLE]->is_sub = true; // osd_libass.c
+
     osd_init_backend(osd);
     global_osd = osd;
     return osd;
@@ -157,6 +163,7 @@ static bool spu_visible(struct osd_state *osd, struct osd_object *obj)
 // Return false on format mismatch, or if nothing to be renderer.
 static bool render_object(struct osd_state *osd, struct osd_object *obj,
                           struct sub_bitmaps *out_imgs,
+                          struct sub_render_params *sub_params,
                           const bool formats[SUBBITMAP_COUNT])
 {
     memset(out_imgs, 0x55, sizeof(*out_imgs));
@@ -170,6 +177,19 @@ static bool render_object(struct osd_state *osd, struct osd_object *obj,
             out_imgs->bitmap_id++;
             out_imgs->bitmap_pos_id++;
         }
+    } else if (obj->type == OSDTYPE_SUB) {
+        double pts = sub_params->pts;
+        if (pts != MP_NOPTS_VALUE)
+            pts += sub_delay - osd->sub_offset;
+
+        // passing the parameters is a big temporary hack
+        osd->sub_pts = pts;
+        osd->dim = sub_params->dim;
+        osd->normal_scale = sub_params->normal_scale;
+        osd->vsfilter_scale = sub_params->vsfilter_scale;
+        osd->support_rgba = formats[SUBBITMAP_RGBA];
+
+        sub_get_bitmaps(osd, out_imgs);
     } else {
         osd_object_get_bitmaps(osd, obj, out_imgs);
     }
@@ -216,13 +236,46 @@ static bool render_object(struct osd_state *osd, struct osd_object *obj,
     if (cached)
         obj->cached = *out_imgs;
 
-    return formats[out_imgs->format];
+    if (!formats[out_imgs->format]) {
+        mp_msg(MSGT_OSD, MSGL_ERR, "Can't render OSD part %d (format %d).\n",
+               obj->type, out_imgs->format);
+        return false;
+    }
+    return true;
+}
+
+// This is a hack to render the first subtitle OSD object, which is not empty.
+// It's a hack because it's really only useful for subtitles: normal OSD can
+// have multiple objects, and rendering one object may invalidate data of a
+// previously rendered, different object (that's how osd_libass.c works).
+// Also, it assumes this is called from a filter: it disables VO rendering of
+// subtitles, because we don't want to render both.
+bool osd_draw_sub(struct osd_state *osd, struct sub_bitmaps *out_imgs,
+                  struct sub_render_params *sub_params,
+                  const bool formats[SUBBITMAP_COUNT])
+{
+    *out_imgs = (struct sub_bitmaps) {0};
+    for (int n = 0; n < MAX_OSD_PARTS; n++) {
+        struct osd_object *obj = osd->objs[n];
+        if (obj->is_sub) {
+            // hack to allow rendering non-ASS subs with vf_ass inserted
+            // (vf_ass is auto-inserted if VOs don't support EOSD)
+#ifdef CONFIG_ASS
+            osd->render_subs_in_filter = !!sub_get_ass_track(osd);
+            if (!osd->render_subs_in_filter)
+                return false;
+#endif
+            if (render_object(osd, obj, out_imgs, sub_params, formats))
+                return true;
+        }
+    }
+    return false;
 }
 
 void draw_osd_with_eosd(struct vo *vo, struct osd_state *osd)
 {
-    mp_eosd_res_t res = {0};
-    if (vo_control(vo, VOCTRL_GET_EOSD_RES, &res) != VO_TRUE)
+    mp_eosd_res_t dim = {0};
+    if (vo_control(vo, VOCTRL_GET_EOSD_RES, &dim) != VO_TRUE)
         return;
 
     bool formats[SUBBITMAP_COUNT];
@@ -231,12 +284,24 @@ void draw_osd_with_eosd(struct vo *vo, struct osd_state *osd)
         formats[n] = vo_control(vo, VOCTRL_QUERY_EOSD_FORMAT, &data) == VO_TRUE;
     }
 
-    osd_update_ext(osd, res);
+    osd_update_ext(osd, dim);
+
+    struct aspect_data asp = vo->aspdat;
+
+    struct sub_render_params subparams = {
+        .pts = osd->vo_sub_pts,
+        .dim = dim,
+        .normal_scale = 1,
+        .vsfilter_scale = (double) asp.prew / asp.preh * asp.orgh / asp.orgw,
+        .support_rgba = formats[SUBBITMAP_RGBA],
+    };
 
     for (int n = 0; n < MAX_OSD_PARTS; n++) {
         struct osd_object *obj = osd->objs[n];
+        if (obj->is_sub && osd->render_subs_in_filter)
+            continue;
         struct sub_bitmaps imgs;
-        if (render_object(osd, obj, &imgs, formats))
+        if (render_object(osd, obj, &imgs, &subparams, formats))
             vo_control(vo, VOCTRL_DRAW_EOSD, &imgs);
     }
 }
@@ -249,14 +314,22 @@ void osd_draw_text_ext(struct osd_state *osd, int w, int h,
                                           int stride),
                    void *ctx)
 {
-    struct mp_eosd_res res =
+    struct mp_eosd_res dim =
         {.w = w, .h = h, .ml = ml, .mt = mt, .mr = mr, .mb = mb};
-    osd_update_ext(osd, res);
+    osd_update_ext(osd, dim);
+    struct sub_render_params subparams = {
+        .pts = osd->vo_sub_pts,
+        .dim = dim,
+        .normal_scale = 1,
+        .vsfilter_scale = 1, // unknown
+    };
     for (int n = 0; n < MAX_OSD_PARTS; n++) {
         struct osd_object *obj = osd->objs[n];
+        if (obj->is_sub && osd->render_subs_in_filter)
+            continue;
         struct sub_bitmaps imgs;
         bool formats[SUBBITMAP_COUNT] = {[SUBBITMAP_OLD_PLANAR] = true};
-        if (render_object(osd, obj, &imgs, formats)) {
+        if (render_object(osd, obj, &imgs, &subparams, formats)) {
             assert(imgs.num_parts == 1);
             struct sub_bitmap *part = &imgs.parts[0];
             struct old_osd_planar *bmp = part->bitmap;
diff --git a/sub/sub.h b/sub/sub.h
index badc21d7a6..4f055558e9 100644
--- a/sub/sub.h
+++ b/sub/sub.h
@@ -27,7 +27,7 @@
 struct vo;
 
 enum mp_osdtype {
-    OSDTYPE_ASS,
+    OSDTYPE_SUB,
     OSDTYPE_OSD,
     OSDTYPE_SUBTITLE,
     OSDTYPE_PROGBAR,
@@ -40,6 +40,8 @@ enum mp_osdtype {
 
 struct osd_object {
     int type; // OSDTYPE_*
+    bool is_sub;
+
     bool force_redraw;
 
     // caches for OSD conversion (internal to render_object())
@@ -62,13 +64,17 @@ struct osd_state {
     struct ass_library *ass_library;
     struct ass_renderer *ass_renderer;
     struct sh_sub *sh_sub;
-    double sub_pts;
     double sub_offset;
+    double vo_sub_pts;
+
+    double sub_pts;
     struct mp_eosd_res dim;
     double normal_scale;
     double vsfilter_scale;
     bool support_rgba;
 
+    bool render_subs_in_filter;
+
     int w, h;
 
     char *osd_text;             // OSDTYPE_OSD
@@ -171,6 +177,10 @@ void vo_osd_reset_changed(void);
 bool vo_osd_has_changed(struct osd_state *osd);
 void osd_free(struct osd_state *osd);
 
+bool osd_draw_sub(struct osd_state *osd, struct sub_bitmaps *out_imgs,
+                  struct sub_render_params *sub_params,
+                  const bool formats[SUBBITMAP_COUNT]);
+
 bool sub_bitmaps_bb(struct sub_bitmaps *imgs, int *x1, int *y1,
                     int *x2, int *y2);