sub: fix text subtitle aspect ratio with vo_xv and vo_lavc, refactor

This fixes that vo_xv didn't display text subtitles correctly when
using anamorphic video. It didn't pass the aspect information to the
subtitle renderer. Also, try to render OSD correctly with respect to
aspect ratio settings: on vo_xv, the OSD is rendered into the video,
and needs to be "stretched" too when playing anamorphic video. When
the -monitorpixelaspect option is used, even with VOs such as vo_opengl
the OSD has to be rendered with that aspect ratio.

As preparation for future commits, replace the weird vsfilter_scale
value with a somewhat more sensible video_par member.

Also, struct mp_eosd_res is a better place for the aspect ratio
parameters, as OSD needs this too.

Use osd_draw_on_image() directly in vo_lavc, which fixes aspect ratio
issues as well.

13 files changed, 79 insertions, 70 deletions

diff --git a/libmpcodecs/vf_ass.c b/libmpcodecs/vf_ass.c
index 77248b49c0..167ebd6e15 100644
--- a/libmpcodecs/vf_ass.c
+++ b/libmpcodecs/vf_ass.c
@@ -63,7 +63,7 @@ static const struct vf_priv_s {
     int auto_insert;
 
     struct osd_state *osd;
-    double aspect_correction;
+    struct mp_eosd_res dim;
 
     unsigned char *planes[3];
     struct line_limits {
@@ -92,7 +92,17 @@ static int config(struct vf_instance *vf,
     vf->priv->planes[2]   = malloc(vf->priv->outw * vf->priv->outh);
     vf->priv->line_limits = malloc((vf->priv->outh + 1) / 2 * sizeof(*vf->priv->line_limits));
 
-    vf->priv->aspect_correction = (double)width / height * d_height / d_width;
+    double dar = (double)d_width / d_height;
+    double sar = (double)width / height;
+
+    vf->priv->dim = (struct mp_eosd_res) {
+        .w = vf->priv->outw,
+        .h = vf->priv->outh,
+        .mt = opts->ass_top_margin,
+        .mb = opts->ass_bottom_margin,
+        .display_par = sar / dar,
+        .video_par = dar / sar,
+    };
 
     return vf_next_config(vf, vf->priv->outw, vf->priv->outh, d_width,
 			  d_height, flags, outfmt);
@@ -354,18 +364,12 @@ static int render_frame(struct vf_instance *vf, mp_image_t *mpi,
 static int put_image(struct vf_instance *vf, mp_image_t *mpi, double pts)
 {
     struct vf_priv_s *priv = vf->priv;
-    struct MPOpts *opts = vf->opts;
     struct osd_state *osd = priv->osd;
     struct sub_bitmaps imgs = (struct sub_bitmaps) {0};
     if (pts != MP_NOPTS_VALUE) {
         struct sub_render_params subparams = {
             .pts = pts,
-            .dim = { .w = vf->priv->outw,
-                     .h = vf->priv->outh,
-                     .mt = opts->ass_top_margin,
-                     .mb = opts->ass_bottom_margin },
-            .normal_scale = vf->priv->aspect_correction,
-            .vsfilter_scale = 1,
+            .dim = priv->dim,
         };
         bool formats[SUBBITMAP_COUNT] = {[SUBBITMAP_LIBASS] = true};
         osd_draw_sub(osd, &imgs, &subparams, formats);
diff --git a/libvo/aspect.c b/libvo/aspect.c
index 8a26a5ac00..f3cd00a5e5 100644
--- a/libvo/aspect.c
+++ b/libvo/aspect.c
@@ -25,18 +25,13 @@
 
 #include "video_out.h"
 
-void aspect_save_orig(struct vo *vo, int orgw, int orgh)
+void aspect_save_videores(struct vo *vo, int w, int h, int d_w, int d_h)
 {
-    mp_msg(MSGT_VO, MSGL_DBG2, "aspect_save_orig %dx%d\n", orgw, orgh);
-    vo->aspdat.orgw = orgw;
-    vo->aspdat.orgh = orgh;
-}
-
-void aspect_save_prescale(struct vo *vo, int prew, int preh)
-{
-    mp_msg(MSGT_VO, MSGL_DBG2, "aspect_save_prescale %dx%d\n", prew, preh);
-    vo->aspdat.prew = prew;
-    vo->aspdat.preh = preh;
+    vo->aspdat.orgw = w;
+    vo->aspdat.orgh = h;
+    vo->aspdat.prew = d_w;
+    vo->aspdat.preh = d_h;
+    vo->aspdat.par = (double)d_w / d_h * h / w;
 }
 
 void aspect_save_screenres(struct vo *vo, int scrw, int scrh)
@@ -52,9 +47,9 @@ void aspect_save_screenres(struct vo *vo, int scrw, int scrh)
     vo->aspdat.scrw = scrw;
     vo->aspdat.scrh = scrh;
     if (opts->force_monitor_aspect)
-        vo->monitor_aspect = opts->force_monitor_aspect;
+        vo->monitor_par = opts->force_monitor_aspect * scrh / scrw;
     else
-        vo->monitor_aspect = opts->monitor_pixel_aspect * scrw / scrh;
+        vo->monitor_par = 1.0 / opts->monitor_pixel_aspect;
 }
 
 /* aspect is called with the source resolution and the
@@ -64,17 +59,17 @@ void aspect_save_screenres(struct vo *vo, int scrw, int scrh)
 void aspect_fit(struct vo *vo, int *srcw, int *srch, int fitw, int fith)
 {
     struct aspect_data *aspdat = &vo->aspdat;
-    float pixelaspect = vo->monitor_aspect * aspdat->scrh / aspdat->scrw;
+    float pixelaspect = vo->monitor_par;
 
-    mp_msg(MSGT_VO, MSGL_DBG2, "aspect(0) fitin: %dx%d screenaspect: %.2f\n",
-           fitw, fith, vo->monitor_aspect);
+    mp_msg(MSGT_VO, MSGL_DBG2, "aspect(0) fitin: %dx%d monitor_par: %.2f\n",
+           fitw, fith, vo->monitor_par);
     *srcw = fitw;
-    *srch = (float)fitw / aspdat->prew * aspdat->preh * pixelaspect;
+    *srch = (float)fitw / aspdat->prew * aspdat->preh / pixelaspect;
     *srch += *srch % 2; // round
     mp_msg(MSGT_VO, MSGL_DBG2, "aspect(1) wh: %dx%d (org: %dx%d)\n",
            *srcw, *srch, aspdat->prew, aspdat->preh);
     if (*srch > fith || *srch < aspdat->orgh) {
-        int tmpw = (float)fith / aspdat->preh * aspdat->prew / pixelaspect;
+        int tmpw = (float)fith / aspdat->preh * aspdat->prew * pixelaspect;
         tmpw += tmpw % 2; // round
         if (tmpw <= fitw) {
             *srch = fith;
diff --git a/libvo/aspect.h b/libvo/aspect.h
index aa117833f3..c5247421d2 100644
--- a/libvo/aspect.h
+++ b/libvo/aspect.h
@@ -24,10 +24,7 @@ struct vo;
 void panscan_init(struct vo *vo);
 void panscan_calc_windowed(struct vo *vo);
 
-void aspect_save_orig(struct vo *vo, int orgw, int orgh);
-
-void aspect_save_prescale(struct vo *vo, int prew, int preh);
-
+void aspect_save_videores(struct vo *vo, int w, int h, int d_w, int d_h);
 void aspect_save_screenres(struct vo *vo, int scrw, int scrh);
 
 #define A_WINZOOM 2 ///< zoom to fill window size
diff --git a/libvo/video_out.c b/libvo/video_out.c
index 18a8482c18..2503fa93e3 100644
--- a/libvo/video_out.c
+++ b/libvo/video_out.c
@@ -370,8 +370,7 @@ int vo_config(struct vo *vo, uint32_t width, uint32_t height,
 {
     struct MPOpts *opts = vo->opts;
     panscan_init(vo);
-    aspect_save_orig(vo, width, height);
-    aspect_save_prescale(vo, d_width, d_height);
+    aspect_save_videores(vo, width, height, d_width, d_height);
 
     if (vo_control(vo, VOCTRL_UPDATE_SCREENINFO, NULL) == VO_TRUE) {
         aspect(vo, &d_width, &d_height, A_NOZOOM);
diff --git a/libvo/video_out.h b/libvo/video_out.h
index 5bd84ab4af..2689c244a8 100644
--- a/libvo/video_out.h
+++ b/libvo/video_out.h
@@ -265,12 +265,13 @@ struct vo {
     int panscan_x;
     int panscan_y;
     float panscan_amount;
-    float monitor_aspect;
+    float monitor_par;
     struct aspect_data {
         int orgw; // real width
         int orgh; // real height
         int prew; // prescaled width
         int preh; // prescaled height
+        float par; // pixel aspect ratio out of orgw/orgh and prew/preh
         int scrw; // horizontal resolution
         int scrh; // vertical resolution
         float asp;
diff --git a/libvo/vo_lavc.c b/libvo/vo_lavc.c
index 5ba548926e..72a1351d85 100644
--- a/libvo/vo_lavc.c
+++ b/libvo/vo_lavc.c
@@ -34,7 +34,7 @@
 #include "encode_lavc.h"
 
 #include "sub/sub.h"
-#include "sub/draw_bmp.h"
+#include "sub/dec_sub.h"
 
 struct priv {
     uint8_t *buffer;
@@ -485,6 +485,29 @@ static void check_events(struct vo *vo)
 {
 }
 
+static void draw_osd(struct vo *vo, struct osd_state *osd)
+{
+    struct priv *vc = vo->priv;
+
+    if (vc->lastimg && vc->lastimg_wants_osd) {
+        struct aspect_data asp = vo->aspdat;
+        double sar = (double)asp.orgw / asp.orgh;
+        double dar = (double)asp.prew / asp.preh;
+
+        struct sub_render_params subparams = {
+            .pts = osd->vo_sub_pts,
+            .dim = {
+                .w = asp.orgw,
+                .h = asp.orgh,
+                .display_par = sar / dar,
+                .video_par = dar / sar,
+            },
+        };
+
+        osd_draw_on_image(osd, vc->lastimg, &vc->colorspace, &subparams);
+    }
+}
+
 static int control(struct vo *vo, uint32_t request, void *data)
 {
     struct priv *vc = vo->priv;
@@ -506,25 +529,6 @@ static int control(struct vo *vo, uint32_t request, void *data)
     case VOCTRL_GET_YUV_COLORSPACE:
         *(struct mp_csp_details *)data = vc->colorspace;
         return 1;
-    case VOCTRL_DRAW_EOSD:
-        if (vc->lastimg && vc->lastimg_wants_osd) {
-            struct sub_bitmaps *imgs = data;
-            mp_draw_sub_bitmaps(vc->lastimg, imgs, &vc->colorspace);
-        }
-        return VO_TRUE;
-    case VOCTRL_GET_EOSD_RES: {
-        struct mp_eosd_res *r = data;
-        r->w = vc->stream->codec->width;
-        r->h = vc->stream->codec->height;
-        r->ml = r->mr = 0;
-        r->mt = r->mb = 0;
-        return VO_TRUE;
-    }
-    case VOCTRL_QUERY_EOSD_FORMAT: {
-        int format = *(int *)data;
-        return (format == SUBBITMAP_LIBASS || format == SUBBITMAP_RGBA)
-               ? VO_TRUE : VO_NOTIMPL;
-    }
     }
     return VO_NOTIMPL;
 }
@@ -543,7 +547,7 @@ const struct vo_driver video_out_lavc = {
     .control = control,
     .uninit = uninit,
     .check_events = check_events,
-    .draw_osd = draw_osd_with_eosd,
+    .draw_osd = draw_osd,
     .flip_page_timed = flip_page_timed,
 };
 
diff --git a/libvo/vo_x11.c b/libvo/vo_x11.c
index 8725279bb8..6ea95527f3 100644
--- a/libvo/vo_x11.c
+++ b/libvo/vo_x11.c
@@ -43,7 +43,6 @@
 
 #include "sub/sub.h"
 #include "sub/dec_sub.h"
-#include "sub/draw_bmp.h"
 
 #include "libmpcodecs/sws_utils.h"
 #define MODE_RGB  0x1
@@ -446,9 +445,12 @@ static void draw_osd(struct vo *vo, struct osd_state *osd)
 
     struct sub_render_params subparams = {
         .pts = osd->vo_sub_pts,
-        .dim = {.w = img.w, .h = img.h},
-        .normal_scale = 1,
-        .vsfilter_scale = 1,
+        .dim = {
+            .w = img.w,
+            .h = img.h,
+            .display_par = vo->monitor_par,
+            .video_par = vo->aspdat.par,
+        },
     };
 
     osd_draw_on_image(osd, &img, &csp, &subparams);
diff --git a/libvo/vo_xv.c b/libvo/vo_xv.c
index b62eaf18d7..8ea22c4da4 100644
--- a/libvo/vo_xv.c
+++ b/libvo/vo_xv.c
@@ -344,11 +344,18 @@ static void draw_osd(struct vo *vo, struct osd_state *osd)
     struct mp_csp_details csp = {0};
     vo_control(vo, VOCTRL_GET_YUV_COLORSPACE, &csp);
 
+    struct vo_rect *src = &ctx->src_rect;
+    struct vo_rect *dst = &ctx->dst_rect;
+    double xvpar = (double)dst->width / dst->height * src->height / src->width;
+
     struct sub_render_params subparams = {
         .pts = osd->vo_sub_pts,
-        .dim = {.w = ctx->image_width, .h = ctx->image_height},
-        .normal_scale = 1,
-        .vsfilter_scale = 1,
+        .dim = {
+            .w = ctx->image_width,
+            .h = ctx->image_height,
+            .display_par = vo->monitor_par / xvpar,
+            .video_par = vo->aspdat.par,
+        },
     };
 
     if (osd_draw_on_image(osd, &img, &csp, &subparams))
diff --git a/sub/dec_sub.h b/sub/dec_sub.h
index 0c5c1ab7ee..389851503e 100644
--- a/sub/dec_sub.h
+++ b/sub/dec_sub.h
@@ -13,8 +13,6 @@ struct MPOpts *opts;
 struct sub_render_params {
     double pts;
     struct mp_eosd_res dim;
-    double normal_scale;
-    double vsfilter_scale;
 };
 
 static inline bool is_text_sub(int type)
diff --git a/sub/osd_libass.c b/sub/osd_libass.c
index d77c0d1af2..a182c6bdf2 100644
--- a/sub/osd_libass.c
+++ b/sub/osd_libass.c
@@ -283,6 +283,7 @@ void osd_object_get_bitmaps(struct osd_state *osd, struct osd_object *obj,
         return;
 
     ass_set_frame_size(osd->osd_render, osd->res.w, osd->res.h);
+    ass_set_aspect_ratio(osd->osd_render, osd->res.display_par, 1.0);
     mp_ass_render_frame(osd->osd_render, obj->osd_track, 0,
                         &obj->parts_cache, out_imgs);
     talloc_steal(obj, obj->parts_cache);
diff --git a/sub/sd_ass.c b/sub/sd_ass.c
index a5e23d74da..7926820ece 100644
--- a/sub/sd_ass.c
+++ b/sub/sd_ass.c
@@ -137,11 +137,11 @@ static void get_bitmaps(struct sh_sub *sh, struct osd_state *osd,
     if (params->pts == MP_NOPTS_VALUE)
         return;
 
-    double scale = params->normal_scale;
+    double scale = params->dim.display_par;
     bool use_vs_aspect = opts->ass_style_override
                          ? opts->ass_vsfilter_aspect_compat : 1;
     if (ctx->vsfilter_aspect && use_vs_aspect)
-        scale = params->vsfilter_scale;
+        scale = scale * params->dim.video_par;
     ASS_Renderer *renderer = osd->ass_renderer;
     mp_ass_configure(renderer, opts, &params->dim);
     ass_set_aspect_ratio(renderer, scale, 1);
diff --git a/sub/sub.c b/sub/sub.c
index 862f5d1f6e..8599002927 100644
--- a/sub/sub.c
+++ b/sub/sub.c
@@ -270,15 +270,14 @@ void draw_osd_with_eosd(struct vo *vo, struct osd_state *osd)
         formats[n] = vo_control(vo, VOCTRL_QUERY_EOSD_FORMAT, &data) == VO_TRUE;
     }
 
-    osd_update_ext(osd, dim);
+    dim.display_par = vo->monitor_par;
+    dim.video_par = vo->aspdat.par;
 
-    struct aspect_data asp = vo->aspdat;
+    osd_update_ext(osd, dim);
 
     struct sub_render_params subparams = {
         .pts = osd->vo_sub_pts,
         .dim = dim,
-        .normal_scale = 1,
-        .vsfilter_scale = (double) asp.prew / asp.preh * asp.orgh / asp.orgw,
     };
 
     for (int n = 0; n < MAX_OSD_PARTS; n++) {
diff --git a/sub/sub.h b/sub/sub.h
index 4f3ca905b6..3d40186b60 100644
--- a/sub/sub.h
+++ b/sub/sub.h
@@ -82,6 +82,8 @@ struct sub_bitmaps {
 struct mp_eosd_res {
     int w, h; // screen dimensions, including black borders
     int mt, mb, ml, mr; // borders (top, bottom, left, right)
+    double display_par;
+    double video_par; // PAR of the original video (for some sub decoders)
 };
 
 enum mp_osdtype {