sub: always go through sub.c for OSD rendering

Before this commit, vf_vo.c and vf_ass.c were manually calling the
subtitle decoder to retrieve images to render. In particular, this
circumvented the sub-bitmap conversion & caching layer in sub.c.

Change this so that subtitle decoding isn't special anymore, and draws
all subtitles with the normal OSD drawing API.

This is also a step towards removing the need for vf_ass auto-insertion.
In fact, if auto-insertion would be disabled now, VOs with "old" OSD
rendering could still render ASS subtitles in monochrome, because
there is still ASS -> old-OSD bitmap conversion in the sub.c mechanism.

The code is written with the assumption that the subtitle rendering
filter (vf_ass) can render all subtitle formats. Since vf_ass knows the
ASS format only, rendering image subs (i.e. RGBA subs) with it simply
fails. This means that with vo_xv (vf_ass auto-inserted), image subs
wouldn't be rendered. Use a dumb hack to disable rendering subs with a
filter, if we detect that the subs are not in ASS format. (Trying to
render the subs first would probably result in purging the conversion
cache on every frame.)

7 files changed, 124 insertions, 56 deletions

diff --git a/libmpcodecs/vf.h b/libmpcodecs/vf.h
index ec8144b22f..8b18470e62 100644
--- a/libmpcodecs/vf.h
+++ b/libmpcodecs/vf.h
@@ -104,8 +104,7 @@ struct vf_ctrl_screenshot {
 #define VFCTRL_SKIP_NEXT_FRAME 12  // For encoding - drop the next frame that passes thru
 #define VFCTRL_FLUSH_FRAMES    13  // For encoding - flush delayed frames
 #define VFCTRL_SCREENSHOT      14  // Take screenshot, arg is vf_ctrl_screenshot
-#define VFCTRL_INIT_EOSD       15  // Select EOSD renderer
-#define VFCTRL_DRAW_EOSD       16  // Render EOSD */
+#define VFCTRL_INIT_OSD        15  // Filter OSD renderer present?
 #define VFCTRL_SET_DEINTERLACE 18  // Set deinterlacing status
 #define VFCTRL_GET_DEINTERLACE 19  // Get deinterlacing status
 /* Hack to make the OSD state object available to vf_expand and vf_ass which
diff --git a/libmpcodecs/vf_ass.c b/libmpcodecs/vf_ass.c
index f0b2e6ab77..937912313b 100644
--- a/libmpcodecs/vf_ass.c
+++ b/libmpcodecs/vf_ass.c
@@ -355,23 +355,23 @@ static int put_image(struct vf_instance *vf, mp_image_t *mpi, double pts)
     struct vf_priv_s *priv = vf->priv;
     struct MPOpts *opts = vf->opts;
     struct osd_state *osd = priv->osd;
-    ASS_Image *images = 0;
+    struct sub_bitmaps imgs = (struct sub_bitmaps) {0};
     if (pts != MP_NOPTS_VALUE) {
-        osd->dim = (struct mp_eosd_res){ .w = vf->priv->outw,
-                                         .h = vf->priv->outh,
-                                         .mt = opts->ass_top_margin,
-                                         .mb = opts->ass_bottom_margin };
-        osd->normal_scale = vf->priv->aspect_correction;
-        osd->vsfilter_scale = 1;
-        osd->sub_pts = pts - osd->sub_offset;
-        osd->support_rgba = false;
-        struct sub_bitmaps b;
-        sub_get_bitmaps(osd, &b);
-        images = b.imgs;
+        struct sub_render_params subparams = {
+            .pts = pts,
+            .dim = { .w = vf->priv->outw,
+                     .h = vf->priv->outh,
+                     .mt = opts->ass_top_margin,
+                     .mb = opts->ass_bottom_margin },
+            .normal_scale = vf->priv->aspect_correction,
+            .vsfilter_scale = 1,
+        };
+        bool formats[SUBBITMAP_COUNT] = {[SUBBITMAP_LIBASS] = true};
+        osd_draw_sub(osd, &imgs, &subparams, formats);
     }
 
     prepare_image(vf, mpi);
-    render_frame(vf, mpi, images);
+    render_frame(vf, mpi, imgs.imgs);
 
     return vf_next_put_image(vf, vf->dmpi, pts);
 }
@@ -393,9 +393,7 @@ static int control(vf_instance_t *vf, int request, void *data)
     case VFCTRL_SET_OSD_OBJ:
         vf->priv->osd = data;
         break;
-    case VFCTRL_INIT_EOSD:
-        return CONTROL_TRUE;
-    case VFCTRL_DRAW_EOSD:
+    case VFCTRL_INIT_OSD:
         return CONTROL_TRUE;
     }
     return vf_next_control(vf, request, data);
diff --git a/libmpcodecs/vf_vo.c b/libmpcodecs/vf_vo.c
index 49668d4640..998b4b6dfa 100644
--- a/libmpcodecs/vf_vo.c
+++ b/libmpcodecs/vf_vo.c
@@ -34,7 +34,6 @@
 
 struct vf_priv_s {
     struct vo *vo;
-    double scale_ratio;
 };
 #define video_out (vf->priv->vo)
 
@@ -74,8 +73,6 @@ static int config(struct vf_instance *vf,
     if (vo_config(video_out, width, height, d_width, d_height, flags, outfmt))
         return 0;
 
-    vf->priv->scale_ratio = (double) d_width / d_height * height / width;
-
     return 1;
 }
 
@@ -117,19 +114,6 @@ static int control(struct vf_instance *vf, int request, void *data)
         };
         return vo_control(video_out, VOCTRL_GET_EQUALIZER, &param) == VO_TRUE;
     }
-    case VFCTRL_DRAW_EOSD: {
-        struct osd_state *osd = data;
-        osd->support_rgba = vf->default_caps & VFCAP_EOSD_RGBA;
-        osd->dim = (struct mp_eosd_res){0};
-        if (!video_out->config_ok ||
-                vo_control(video_out, VOCTRL_GET_EOSD_RES, &osd->dim) != true)
-            return CONTROL_FALSE;
-        osd->normal_scale = 1;
-        osd->vsfilter_scale = vf->priv->scale_ratio;
-        struct sub_bitmaps images;
-        sub_get_bitmaps(osd, &images);
-        return vo_control(video_out, VOCTRL_DRAW_EOSD, &images) == VO_TRUE;
-    }
     }
     return CONTROL_UNKNOWN;
 }
diff --git a/mplayer.c b/mplayer.c
index a885fac8ca..ebf911cb54 100644
--- a/mplayer.c
+++ b/mplayer.c
@@ -2397,9 +2397,9 @@ int reinit_video_chain(struct MPContext *mpctx)
 
     sh_video->vfilter = append_filters(sh_video->vfilter, opts->vf_settings);
 
-    if (opts->ass_enabled)
-        sh_video->vfilter->control(sh_video->vfilter, VFCTRL_INIT_EOSD,
-                                   mpctx->ass_library);
+    struct vf_instance *vf = sh_video->vfilter;
+    mpctx->osd->render_subs_in_filter
+        = vf->control(vf, VFCTRL_INIT_OSD, NULL) == VO_TRUE;
 
     init_best_video_codec(sh_video, video_codec_list, video_fm_list);
 
@@ -2644,12 +2644,10 @@ static int redraw_osd(struct MPContext *mpctx)
     struct vf_instance *vf = sh_video->vfilter;
     if (vo_redraw_frame(mpctx->video_out) < 0)
         return -1;
-    mpctx->osd->sub_pts = mpctx->video_pts;
-    if (mpctx->osd->sub_pts != MP_NOPTS_VALUE)
-        mpctx->osd->sub_pts += sub_delay - mpctx->osd->sub_offset;
+    mpctx->osd->vo_sub_pts = mpctx->video_pts;
 
-    if (!(sh_video->output_flags & VFCAP_EOSD_FILTER))
-        vf->control(vf, VFCTRL_DRAW_EOSD, mpctx->osd);
+    // NOTE: probably should use VO ctrl directly, and/or check if a filter does
+    //       OSD rendering (a filter can't redraw anything here)
     vf->control(vf, VFCTRL_DRAW_OSD, mpctx->osd);
     vo_osd_reset_changed();
     vo_flip_page(mpctx->video_out, 0, -1);
@@ -3257,10 +3255,7 @@ static void run_playloop(struct MPContext *mpctx)
         update_subtitles(mpctx, sh_video->pts);
         update_osd_msg(mpctx);
         struct vf_instance *vf = sh_video->vfilter;
-        mpctx->osd->sub_pts = mpctx->video_pts;
-        if (mpctx->osd->sub_pts != MP_NOPTS_VALUE)
-            mpctx->osd->sub_pts += sub_delay - mpctx->osd->sub_offset;
-        vf->control(vf, VFCTRL_DRAW_EOSD, mpctx->osd);
+        mpctx->osd->vo_sub_pts = mpctx->video_pts;
         vf->control(vf, VFCTRL_DRAW_OSD, mpctx->osd);
         vo_osd_reset_changed();
 
diff --git a/sub/dec_sub.h b/sub/dec_sub.h
index 9c75506c4c..7ccf513438 100644
--- a/sub/dec_sub.h
+++ b/sub/dec_sub.h
@@ -60,6 +60,15 @@ typedef struct sub_bitmaps {
     int bitmap_id, bitmap_pos_id;
 } mp_eosd_images_t;
 
+struct sub_render_params {
+    double pts;
+    struct mp_eosd_res dim;
+    double normal_scale;
+    double vsfilter_scale;
+
+    bool support_rgba;
+};
+
 static inline bool is_text_sub(int type)
 {
     return type == 't' || type == 'm' || type == 'a';
diff --git a/sub/sub.c b/sub/sub.c
index fb1c76c91d..64dba7df20 100644
--- a/sub/sub.c
+++ b/sub/sub.c
@@ -114,6 +114,7 @@ struct osd_state *osd_create(struct MPOpts *opts, struct ass_library *asslib)
         .osd_text = talloc_strdup(osd, ""),
         .progbar_type = -1,
     };
+
     for (int n = 0; n < MAX_OSD_PARTS; n++) {
         struct osd_object *obj = talloc_struct(osd, struct osd_object, {
             .type = n,
@@ -122,6 +123,11 @@ struct osd_state *osd_create(struct MPOpts *opts, struct ass_library *asslib)
             obj->cache[i] = talloc_steal(obj, osd_conv_cache_new());
         osd->objs[n] = obj;
     }
+
+    // OSDTYPE_SPU is an odd case, because vf_ass.c can't render it.
+    osd->objs[OSDTYPE_SUB]->is_sub = true;      // dec_sub.c
+    osd->objs[OSDTYPE_SUBTITLE]->is_sub = true; // osd_libass.c
+
     osd_init_backend(osd);
     global_osd = osd;
     return osd;
@@ -157,6 +163,7 @@ static bool spu_visible(struct osd_state *osd, struct osd_object *obj)
 // Return false on format mismatch, or if nothing to be renderer.
 static bool render_object(struct osd_state *osd, struct osd_object *obj,
                           struct sub_bitmaps *out_imgs,
+                          struct sub_render_params *sub_params,
                           const bool formats[SUBBITMAP_COUNT])
 {
     memset(out_imgs, 0x55, sizeof(*out_imgs));
@@ -170,6 +177,19 @@ static bool render_object(struct osd_state *osd, struct osd_object *obj,
             out_imgs->bitmap_id++;
             out_imgs->bitmap_pos_id++;
         }
+    } else if (obj->type == OSDTYPE_SUB) {
+        double pts = sub_params->pts;
+        if (pts != MP_NOPTS_VALUE)
+            pts += sub_delay - osd->sub_offset;
+
+        // passing the parameters is a big temporary hack
+        osd->sub_pts = pts;
+        osd->dim = sub_params->dim;
+        osd->normal_scale = sub_params->normal_scale;
+        osd->vsfilter_scale = sub_params->vsfilter_scale;
+        osd->support_rgba = formats[SUBBITMAP_RGBA];
+
+        sub_get_bitmaps(osd, out_imgs);
     } else {
         osd_object_get_bitmaps(osd, obj, out_imgs);
     }
@@ -216,13 +236,46 @@ static bool render_object(struct osd_state *osd, struct osd_object *obj,
     if (cached)
         obj->cached = *out_imgs;
 
-    return formats[out_imgs->format];
+    if (!formats[out_imgs->format]) {
+        mp_msg(MSGT_OSD, MSGL_ERR, "Can't render OSD part %d (format %d).\n",
+               obj->type, out_imgs->format);
+        return false;
+    }
+    return true;
+}
+
+// This is a hack to render the first subtitle OSD object, which is not empty.
+// It's a hack because it's really only useful for subtitles: normal OSD can
+// have multiple objects, and rendering one object may invalidate data of a
+// previously rendered, different object (that's how osd_libass.c works).
+// Also, it assumes this is called from a filter: it disables VO rendering of
+// subtitles, because we don't want to render both.
+bool osd_draw_sub(struct osd_state *osd, struct sub_bitmaps *out_imgs,
+                  struct sub_render_params *sub_params,
+                  const bool formats[SUBBITMAP_COUNT])
+{
+    *out_imgs = (struct sub_bitmaps) {0};
+    for (int n = 0; n < MAX_OSD_PARTS; n++) {
+        struct osd_object *obj = osd->objs[n];
+        if (obj->is_sub) {
+            // hack to allow rendering non-ASS subs with vf_ass inserted
+            // (vf_ass is auto-inserted if VOs don't support EOSD)
+#ifdef CONFIG_ASS
+            osd->render_subs_in_filter = !!sub_get_ass_track(osd);
+            if (!osd->render_subs_in_filter)
+                return false;
+#endif
+            if (render_object(osd, obj, out_imgs, sub_params, formats))
+                return true;
+        }
+    }
+    return false;
 }
 
 void draw_osd_with_eosd(struct vo *vo, struct osd_state *osd)
 {
-    mp_eosd_res_t res = {0};
-    if (vo_control(vo, VOCTRL_GET_EOSD_RES, &res) != VO_TRUE)
+    mp_eosd_res_t dim = {0};
+    if (vo_control(vo, VOCTRL_GET_EOSD_RES, &dim) != VO_TRUE)
         return;
 
     bool formats[SUBBITMAP_COUNT];
@@ -231,12 +284,24 @@ void draw_osd_with_eosd(struct vo *vo, struct osd_state *osd)
         formats[n] = vo_control(vo, VOCTRL_QUERY_EOSD_FORMAT, &data) == VO_TRUE;
     }
 
-    osd_update_ext(osd, res);
+    osd_update_ext(osd, dim);
+
+    struct aspect_data asp = vo->aspdat;
+
+    struct sub_render_params subparams = {
+        .pts = osd->vo_sub_pts,
+        .dim = dim,
+        .normal_scale = 1,
+        .vsfilter_scale = (double) asp.prew / asp.preh * asp.orgh / asp.orgw,
+        .support_rgba = formats[SUBBITMAP_RGBA],
+    };
 
     for (int n = 0; n < MAX_OSD_PARTS; n++) {
         struct osd_object *obj = osd->objs[n];
+        if (obj->is_sub && osd->render_subs_in_filter)
+            continue;
         struct sub_bitmaps imgs;
-        if (render_object(osd, obj, &imgs, formats))
+        if (render_object(osd, obj, &imgs, &subparams, formats))
             vo_control(vo, VOCTRL_DRAW_EOSD, &imgs);
     }
 }
@@ -249,14 +314,22 @@ void osd_draw_text_ext(struct osd_state *osd, int w, int h,
                                           int stride),
                    void *ctx)
 {
-    struct mp_eosd_res res =
+    struct mp_eosd_res dim =
         {.w = w, .h = h, .ml = ml, .mt = mt, .mr = mr, .mb = mb};
-    osd_update_ext(osd, res);
+    osd_update_ext(osd, dim);
+    struct sub_render_params subparams = {
+        .pts = osd->vo_sub_pts,
+        .dim = dim,
+        .normal_scale = 1,
+        .vsfilter_scale = 1, // unknown
+    };
     for (int n = 0; n < MAX_OSD_PARTS; n++) {
         struct osd_object *obj = osd->objs[n];
+        if (obj->is_sub && osd->render_subs_in_filter)
+            continue;
         struct sub_bitmaps imgs;
         bool formats[SUBBITMAP_COUNT] = {[SUBBITMAP_OLD_PLANAR] = true};
-        if (render_object(osd, obj, &imgs, formats)) {
+        if (render_object(osd, obj, &imgs, &subparams, formats)) {
             assert(imgs.num_parts == 1);
             struct sub_bitmap *part = &imgs.parts[0];
             struct old_osd_planar *bmp = part->bitmap;
diff --git a/sub/sub.h b/sub/sub.h
index badc21d7a6..4f055558e9 100644
--- a/sub/sub.h
+++ b/sub/sub.h
@@ -27,7 +27,7 @@
 struct vo;
 
 enum mp_osdtype {
-    OSDTYPE_ASS,
+    OSDTYPE_SUB,
     OSDTYPE_OSD,
     OSDTYPE_SUBTITLE,
     OSDTYPE_PROGBAR,
@@ -40,6 +40,8 @@ enum mp_osdtype {
 
 struct osd_object {
     int type; // OSDTYPE_*
+    bool is_sub;
+
     bool force_redraw;
 
     // caches for OSD conversion (internal to render_object())
@@ -62,13 +64,17 @@ struct osd_state {
     struct ass_library *ass_library;
     struct ass_renderer *ass_renderer;
     struct sh_sub *sh_sub;
-    double sub_pts;
     double sub_offset;
+    double vo_sub_pts;
+
+    double sub_pts;
     struct mp_eosd_res dim;
     double normal_scale;
     double vsfilter_scale;
     bool support_rgba;
 
+    bool render_subs_in_filter;
+
     int w, h;
 
     char *osd_text;             // OSDTYPE_OSD
@@ -171,6 +177,10 @@ void vo_osd_reset_changed(void);
 bool vo_osd_has_changed(struct osd_state *osd);
 void osd_free(struct osd_state *osd);
 
+bool osd_draw_sub(struct osd_state *osd, struct sub_bitmaps *out_imgs,
+                  struct sub_render_params *sub_params,
+                  const bool formats[SUBBITMAP_COUNT]);
+
 bool sub_bitmaps_bb(struct sub_bitmaps *imgs, int *x1, int *y1,
                     int *x2, int *y2);