sub: pack libass bitmaps directly in sd_ass.c and osd_libass.c

Change all producer of libass images to packing the bitmaps into a
single larger bitmap directly when they're output. This is supposed to
help working towards refcounted sub bitmaps.

This will reduce performance for VOs like vo_xv, but not for vo_opengl.
vo_opengl simply will pick up the pre-packed sub bitmaps, and skip
packing them again. vo_xv will copy and pack the sub bitmaps
unnecessarily - but if we want sub bitmap refcounting, they'd have to be
copied anyway.

The packing code cannot be removed yet from vo_opengl, because there are
certain corner cases that still produce unpackad other sub bitmaps.
Actual refcounting will also require more work.

6 files changed, 148 insertions, 54 deletions

diff --git a/sub/ass_mp.c b/sub/ass_mp.c
index 2f9b39740f..4e3c526c64 100644
--- a/sub/ass_mp.c
+++ b/sub/ass_mp.c
@@ -35,6 +35,8 @@
 #include "osd.h"
 #include "stream/stream.h"
 #include "options/options.h"
+#include "video/out/bitmap_packer.h"
+#include "video/mp_image.h"
 
 // res_y should be track->PlayResY
 // It determines scaling of font sizes and more.
@@ -97,37 +99,6 @@ void mp_ass_configure_fonts(ASS_Renderer *priv, struct osd_style_opts *opts,
     talloc_free(tmp);
 }
 
-void mp_ass_render_frame(ASS_Renderer *renderer, ASS_Track *track, double time,
-                         struct sub_bitmaps *res)
-{
-    int changed;
-    ASS_Image *imgs = ass_render_frame(renderer, track, time, &changed);
-    if (changed)
-        res->change_id++;
-    assert(res->format == 0 || res->format == SUBBITMAP_LIBASS);
-    res->format = SUBBITMAP_LIBASS;
-
-    int num_parts_alloc = MP_TALLOC_AVAIL(res->parts);
-    for (struct ass_image *img = imgs; img; img = img->next) {
-        if (img->w == 0 || img->h == 0)
-            continue;
-        if (res->num_parts >= num_parts_alloc) {
-            num_parts_alloc = MPMAX(num_parts_alloc * 2, 32);
-            res->parts = talloc_realloc(NULL, res->parts, struct sub_bitmap,
-                                        num_parts_alloc);
-        }
-        struct sub_bitmap *p = &res->parts[res->num_parts];
-        p->bitmap = img->bitmap;
-        p->stride = img->stride;
-        p->libass.color = img->color;
-        p->dw = p->w = img->w;
-        p->dh = p->h = img->h;
-        p->x = img->dst_x;
-        p->y = img->dst_y;
-        res->num_parts++;
-    }
-}
-
 static const int map_ass_level[] = {
     MSGL_ERR,           // 0 "FATAL errors"
     MSGL_WARN,
@@ -177,3 +148,104 @@ void mp_ass_flush_old_events(ASS_Track *track, long long ts)
         track->events[i] = track->events[i+n];
     }
 }
+
+struct mp_ass_packer {
+    struct sub_bitmap *cached_parts; // only for the array memory
+    struct mp_image *cached_img;
+    struct sub_bitmaps cached_subs;
+    bool cached_subs_valid;
+    struct bitmap_packer *packer;
+};
+
+// Free with talloc_free().
+struct mp_ass_packer *mp_ass_packer_alloc(void *ta_parent)
+{
+    struct mp_ass_packer *p = talloc_zero(ta_parent, struct mp_ass_packer);
+    p->packer = talloc_zero(p, struct bitmap_packer);
+    return p;
+}
+
+// Pack the contents of image_lists[0] to image_lists[num_image_lists-1] into
+// a single image, and make *out point to it. *out is completely overwritten.
+// If libass reported any change, image_lists_changed must be set (it then
+// repacks all images). preferred_osd_format can be set to a desired
+// sub_bitmap_format. Currently, only SUBBITMAP_LIBASS is supported.
+void mp_ass_packer_pack(struct mp_ass_packer *p, ASS_Image **image_lists,
+                        int num_image_lists, bool image_lists_changed,
+                        int preferred_osd_format, struct sub_bitmaps *out)
+{
+    if (p->cached_subs_valid && !image_lists_changed) {
+        *out = p->cached_subs;
+        return;
+    }
+
+    *out = (struct sub_bitmaps){.change_id = 1};
+    p->cached_subs_valid = false;
+
+    struct sub_bitmaps res = {
+        .change_id = image_lists_changed,
+        .format = SUBBITMAP_LIBASS,
+        .parts = p->cached_parts,
+    };
+
+    for (int n = 0; n < num_image_lists; n++) {
+        for (struct ass_image *img = image_lists[n]; img; img = img->next) {
+            if (img->w == 0 || img->h == 0)
+                continue;
+            MP_TARRAY_GROW(p, p->cached_parts, res.num_parts);
+            res.parts = p->cached_parts;
+            struct sub_bitmap *b = &res.parts[res.num_parts];
+            b->bitmap = img->bitmap;
+            b->stride = img->stride;
+            b->libass.color = img->color;
+            b->dw = b->w = img->w;
+            b->dh = b->h = img->h;
+            b->x = img->dst_x;
+            b->y = img->dst_y;
+            res.num_parts++;
+        }
+    }
+
+    packer_set_size(p->packer, res.num_parts);
+
+    for (int n = 0; n < res.num_parts; n++)
+        p->packer->in[n] = (struct pos){res.parts[n].w, res.parts[n].h};
+
+    if (p->packer->count == 0 || packer_pack(p->packer) < 0)
+        return;
+
+    struct pos bb[2];
+    packer_get_bb(p->packer, bb);
+
+    res.packed_w = bb[1].x;
+    res.packed_h = bb[1].y;
+
+    if (!p->cached_img || p->cached_img->w < res.packed_w ||
+                          p->cached_img->h < res.packed_h)
+    {
+        talloc_free(p->cached_img);
+        p->cached_img = mp_image_alloc(IMGFMT_Y8, p->packer->w, p->packer->h);
+        if (!p->cached_img)
+            return;
+        talloc_steal(p, p->cached_img);
+    }
+
+    res.packed = p->cached_img;
+
+    for (int n = 0; n < res.num_parts; n++) {
+        struct sub_bitmap *b = &res.parts[n];
+        struct pos pos = p->packer->result[n];
+
+        int stride = res.packed->stride[0];
+        void *pdata = (uint8_t *)res.packed->planes[0] + pos.y * stride + pos.x;
+        memcpy_pic(pdata, b->bitmap, b->w, b->h, stride, b->stride);
+
+        b->src_x = pos.x;
+        b->src_y = pos.y;
+    }
+
+    *out = res;
+    p->cached_subs = res;
+    p->cached_subs.change_id = 0;
+    p->cached_subs_valid = true;
+}
diff --git a/sub/ass_mp.h b/sub/ass_mp.h
index 20f0ebe9bd..50397bdc6b 100644
--- a/sub/ass_mp.h
+++ b/sub/ass_mp.h
@@ -49,9 +49,11 @@ void mp_ass_configure_fonts(ASS_Renderer *priv, struct osd_style_opts *opts,
                             struct mpv_global *global, struct mp_log *log);
 ASS_Library *mp_ass_init(struct mpv_global *global, struct mp_log *log);
 
-struct sub_bitmap;
 struct sub_bitmaps;
-void mp_ass_render_frame(ASS_Renderer *renderer, ASS_Track *track, double time,
-                         struct sub_bitmaps *res);
+struct mp_ass_packer;
+struct mp_ass_packer *mp_ass_packer_alloc(void *ta_parent);
+void mp_ass_packer_pack(struct mp_ass_packer *p, ASS_Image **image_lists,
+                        int num_image_lists, bool changed,
+                        int preferred_osd_format, struct sub_bitmaps *out);
 
 #endif                          /* MPLAYER_ASS_MP_H */
diff --git a/sub/osd.h b/sub/osd.h
index 7cfc695873..ade546f4f4 100644
--- a/sub/osd.h
+++ b/sub/osd.h
@@ -69,6 +69,7 @@ struct sub_bitmaps {
     // parts[].bitmap pointer points into the image data here (and stride will
     // correspond to packed->stride[0]).
     //  SUBBITMAP_RGBA: IMGFMT_BGRA (exact match)
+    //  SUBBITMAP_LIBASS: IMGFMT_Y8 (not the same, but compatible layout)
     // Other formats have this set to NULL.
     struct mp_image *packed;
 
diff --git a/sub/osd_libass.c b/sub/osd_libass.c
index 6a2efa6603..f940b6117f 100644
--- a/sub/osd_libass.c
+++ b/sub/osd_libass.c
@@ -90,7 +90,6 @@ void osd_destroy_backend(struct osd_state *osd)
     for (int n = 0; n < MAX_OSD_PARTS; n++) {
         struct osd_object *obj = osd->objs[n];
         destroy_ass_renderer(&obj->ass);
-        talloc_free(obj->parts_cache.parts);
         for (int i = 0; i < obj->num_externals; i++)
             destroy_external(&obj->externals[i]);
         obj->num_externals = 0;
@@ -501,7 +500,7 @@ void osd_set_external(struct osd_state *osd, void *id, int res_x, int res_y,
         entry->res_x = res_x;
         entry->res_y = res_y;
         update_external(osd, obj, entry);
-        obj->parts_cache.change_id = 1;
+        obj->changed = true;
         osd_changed_unlocked(osd, obj->type);
     }
 
@@ -510,14 +509,19 @@ done:
 }
 
 static void append_ass(struct ass_state *ass, struct mp_osd_res *res,
-                       struct sub_bitmaps *imgs)
+                       ASS_Image **img_list, bool *changed)
 {
-    if (!ass->render || !ass->track)
+    if (!ass->render || !ass->track) {
+        *img_list = NULL;
         return;
+    }
 
     ass_set_frame_size(ass->render, res->w, res->h);
     ass_set_aspect_ratio(ass->render, res->display_par, 1.0);
-    mp_ass_render_frame(ass->render, ass->track, 0, imgs);
+
+    int ass_changed;
+    *img_list = ass_render_frame(ass->render, ass->track, 0, &ass_changed);
+    *changed |= ass_changed;
 }
 
 void osd_object_get_bitmaps(struct osd_state *osd, struct osd_object *obj,
@@ -526,12 +530,19 @@ void osd_object_get_bitmaps(struct osd_state *osd, struct osd_object *obj,
     if (obj->force_redraw && obj->type == OSDTYPE_OSD)
         update_osd(osd, obj);
 
-    append_ass(&obj->ass, &obj->vo_res, &obj->parts_cache);
-    for (int n = 0; n < obj->num_externals; n++)
-        append_ass(&obj->externals[n].ass, &obj->vo_res, &obj->parts_cache);
+    if (!obj->ass_packer)
+        obj->ass_packer = mp_ass_packer_alloc(obj);
+
+    MP_TARRAY_GROW(obj, obj->ass_imgs, obj->num_externals + 1);
+
+    append_ass(&obj->ass, &obj->vo_res, &obj->ass_imgs[0], &obj->changed);
+    for (int n = 0; n < obj->num_externals; n++) {
+        append_ass(&obj->externals[n].ass, &obj->vo_res, &obj->ass_imgs[n + 1],
+                   &obj->changed);
+    }
 
-    *out_imgs = obj->parts_cache;
+    mp_ass_packer_pack(obj->ass_packer, obj->ass_imgs, obj->num_externals + 1,
+                       obj->changed, SUBBITMAP_LIBASS, out_imgs);
 
-    obj->parts_cache.change_id = 0;
-    obj->parts_cache.num_parts = 0;
+    obj->changed = false;
 }
diff --git a/sub/osd_state.h b/sub/osd_state.h
index 0fff668d2d..c2e7911d1b 100644
--- a/sub/osd_state.h
+++ b/sub/osd_state.h
@@ -57,8 +57,10 @@ struct osd_object {
     struct mp_osd_res vo_res;
 
     // Internally used by osd_libass.c
-    struct sub_bitmaps parts_cache;
+    bool changed;
     struct ass_state ass;
+    struct mp_ass_packer *ass_packer;
+    struct ass_image **ass_imgs;
 };
 
 struct osd_external {
diff --git a/sub/sd_ass.c b/sub/sd_ass.c
index 6f35053370..879d528fb7 100644
--- a/sub/sd_ass.c
+++ b/sub/sd_ass.c
@@ -44,7 +44,8 @@ struct sd_ass_priv {
     bool is_converted;
     struct lavc_conv *converter;
     bool on_top;
-    struct sub_bitmaps part_cache;
+    struct mp_ass_packer *packer;
+    struct sub_bitmap *bs;
     char last_text[500];
     struct mp_image_params video_params;
     struct mp_image_params last_params;
@@ -212,6 +213,8 @@ static int init(struct sd *sd)
 
     enable_output(sd, true);
 
+    ctx->packer = mp_ass_packer_alloc(ctx);
+
     return 0;
 }
 
@@ -459,15 +462,18 @@ static void get_bitmaps(struct sd *sd, struct mp_osd_res dim, double pts,
     if (no_ass)
         fill_plaintext(sd, pts);
 
-    ctx->part_cache.change_id = 0;
-    ctx->part_cache.num_parts = 0;
-    mp_ass_render_frame(renderer, track, ts, &ctx->part_cache);
-    talloc_steal(ctx, ctx->part_cache.parts);
+    int changed;
+    ASS_Image *imgs = ass_render_frame(renderer, track, ts, &changed);
+    mp_ass_packer_pack(ctx->packer, &imgs, 1, changed, SUBBITMAP_LIBASS, res);
 
-    if (!converted)
-        mangle_colors(sd, &ctx->part_cache);
+    if (!converted && res->num_parts > 0) {
+        // mangle_colors() modifies the color field, so copy the thing.
+        MP_TARRAY_GROW(ctx, ctx->bs, res->num_parts);
+        memcpy(ctx->bs, res->parts, sizeof(ctx->bs[0]) * res->num_parts);
+        res->parts = ctx->bs;
 
-    *res = ctx->part_cache;
+        mangle_colors(sd, res);
+    }
 }
 
 struct buf {