sub: create sub_bitmap array even when using libass

One sub_bitmaps struct could contain either a libass ASS_Image list, or
a mplayer native list of sub-bitmaps. This caused code duplication in
vo_vdpau.c and bitmap_packer.c.

Avoid this by creating such a sub_bitmap array even with libass. This
basically copies the list and recreates it in mplayer's native format.
It gets rid of the code duplication, and will make implementing extended
subtitle and OSD rendering in other VOs easier.

Also do some cosmetic changes and other preparations for the following
commits.

6 files changed, 63 insertions, 90 deletions

diff --git a/libvo/bitmap_packer.c b/libvo/bitmap_packer.c
index eedc2e2242..5e5bafea0c 100644
--- a/libvo/bitmap_packer.c
+++ b/libvo/bitmap_packer.c
@@ -30,6 +30,24 @@
 #include "sub/ass_mp.h"
 #include "sub/dec_sub.h"
 
+void packer_reset(struct bitmap_packer *packer)
+{
+    struct bitmap_packer old = *packer;
+    *packer = (struct bitmap_packer) {
+        .w_max = old.w_max,
+        .h_max = old.h_max,
+    };
+    talloc_free_children(packer);
+}
+
+void packer_get_bb(struct bitmap_packer *packer, struct pos out_bb[2])
+{
+    out_bb[0] = (struct pos) {0};
+    out_bb[1] = (struct pos) {
+        FFMIN(packer->used_width + packer->padding, packer->w),
+        FFMIN(packer->used_height + packer->padding, packer->h),
+    };
+}
 
 #define HEIGHT_SORT_BITS 4
 static int size_index(int s)
@@ -171,36 +189,15 @@ void packer_set_size(struct bitmap_packer *packer, int size)
                                            packer->asize + 16);
 }
 
-static int packer_pack_from_assimg(struct bitmap_packer *packer,
-                                   struct ass_image *imglist)
-{
-    int count = 0;
-    struct ass_image *img = imglist;
-    while (img) {
-        if (count >= packer->asize)
-            packer_set_size(packer, FFMAX(packer->asize * 2, 32));
-        packer->in[count].x = img->w;
-        packer->in[count].y = img->h;
-        img = img->next;
-        count++;
-    }
-    packer->count = count;
-    return packer_pack(packer);
-}
-
 int packer_pack_from_subbitmaps(struct bitmap_packer *packer,
-                                struct sub_bitmaps *b, int padding_pixels)
+                                struct sub_bitmaps *b)
 {
-    packer->padding = 0;
     packer->count = 0;
-    if (b->type == SUBBITMAP_EMPTY)
+    if (b->format == SUBBITMAP_EMPTY)
         return 0;
-    if (b->type == SUBBITMAP_LIBASS)
-        return packer_pack_from_assimg(packer, b->imgs);
-    packer->padding = padding_pixels;
-    packer_set_size(packer, b->part_count);
+    packer_set_size(packer, b->num_parts);
     int a = packer->padding;
-    for (int i = 0; i < b->part_count; i++)
+    for (int i = 0; i < b->num_parts; i++)
         packer->in[i] = (struct pos){b->parts[i].w + a, b->parts[i].h + a};
     return packer_pack(packer);
 }
diff --git a/libvo/bitmap_packer.h b/libvo/bitmap_packer.h
index c7c377cbd0..99c7b514b4 100644
--- a/libvo/bitmap_packer.h
+++ b/libvo/bitmap_packer.h
@@ -26,6 +26,13 @@ struct bitmap_packer {
 struct ass_image;
 struct sub_bitmaps;
 
+// Clear all internal state. Leave the following fields: w_max, h_max
+void packer_reset(struct bitmap_packer *packer);
+
+// Get the bounding box used for bitmap data (including padding).
+// The bounding box doesn't exceed (0,0)-(packer->w,packer->h).
+void packer_get_bb(struct bitmap_packer *packer, struct pos out_bb[2]);
+
 /* Reallocate packer->in for at least to desired number of items.
  * Also sets packer->count to the same value.
  */
@@ -46,6 +53,6 @@ int packer_pack(struct bitmap_packer *packer);
  * given image list.
  */
 int packer_pack_from_subbitmaps(struct bitmap_packer *packer,
-                                struct sub_bitmaps *b, int padding_pixels);
+                                struct sub_bitmaps *b);
 
 #endif
diff --git a/libvo/vo_corevideo.m b/libvo/vo_corevideo.m
index 1d10330bcb..fd667acb61 100644
--- a/libvo/vo_corevideo.m
+++ b/libvo/vo_corevideo.m
@@ -45,11 +45,10 @@ struct quad {
 };
 
 #define CV_VERTICES_PER_QUAD 6
-#define CV_MAX_OSD_PARTS 20
 
 struct osd_p {
-    GLuint tex[CV_MAX_OSD_PARTS];
-    NSRect tex_rect[CV_MAX_OSD_PARTS];
+    GLuint tex[MAX_OSD_PARTS];
+    NSRect tex_rect[MAX_OSD_PARTS];
     int tex_cnt;
 };
 
@@ -185,7 +184,7 @@ static void create_osd_texture(void *ctx, int x0, int y0, int w, int h,
         return;
     }
 
-    if (osd->tex_cnt >= CV_MAX_OSD_PARTS) {
+    if (osd->tex_cnt >= MAX_OSD_PARTS) {
         mp_msg(MSGT_VO, MSGL_ERR, "Too many OSD parts, contact the"
                                   " developers!\n");
         return;
diff --git a/libvo/vo_gl.c b/libvo/vo_gl.c
index 5453943fe9..236471d537 100644
--- a/libvo/vo_gl.c
+++ b/libvo/vo_gl.c
@@ -44,9 +44,6 @@
 #include "fastmemcpy.h"
 #include "sub/ass_mp.h"
 
-//! How many parts the OSD may consist of at most
-#define MAX_OSD_PARTS 20
-
 //for gl_priv.use_yuv
 #define MASK_ALL_YUV (~(1 << YUV_CONVERSION_NONE))
 #define MASK_NOT_COMBINERS (~((1 << YUV_CONVERSION_NONE) | (1 << YUV_CONVERSION_COMBINERS)))
diff --git a/libvo/vo_gl3.c b/libvo/vo_gl3.c
index dea0c6637a..ebce32677b 100644
--- a/libvo/vo_gl3.c
+++ b/libvo/vo_gl3.c
@@ -59,9 +59,6 @@ static const char vo_gl3_shaders[] =
 #include "libvo/vo_gl3_shaders.h"
 ;
 
-// How many parts the OSD may consist of at most.
-#define MAX_OSD_PARTS 20
-
 // Pixel width of 1D lookup textures.
 #define LOOKUP_TEXTURE_SIZE 256
 
diff --git a/libvo/vo_vdpau.c b/libvo/vo_vdpau.c
index 594829b394..d5856e2dfa 100644
--- a/libvo/vo_vdpau.c
+++ b/libvo/vo_vdpau.c
@@ -92,8 +92,6 @@ struct vdp_functions {
 #undef VDP_FUNCTION
 };
 
-#define MAX_OLD_OSD_BITMAPS 6
-
 struct vdpctx {
     struct vdp_functions *vdp;
 
@@ -165,7 +163,7 @@ struct vdpctx {
         int x0, y0, w, h;
         unsigned char *src, *srca;
         int stride;
-    } old_osd_elements[MAX_OLD_OSD_BITMAPS];
+    } old_osd_elements[MAX_OSD_PARTS];
     int old_osd_count;
     unsigned char                     *osd_data_temp;
     int                                osd_data_size;
@@ -184,7 +182,7 @@ struct vdpctx {
         VdpRect source;
         VdpRect dest;
         VdpColor color;
-    } *eosd_targets, osd_targets[MAX_OLD_OSD_BITMAPS][2];
+    } *eosd_targets, osd_targets[MAX_OSD_PARTS][2];
     int eosd_targets_size;
     int eosd_render_count;
     int bitmap_id;
@@ -1008,7 +1006,7 @@ static void generate_eosd(struct vo *vo, mp_eosd_images_t *imgs)
 
     vc->eosd_render_count = 0;
 
-    if (imgs->type == SUBBITMAP_EMPTY)
+    if (imgs->format == SUBBITMAP_EMPTY)
         return;
 
     if (imgs->bitmap_id == vc->bitmap_id)
@@ -1017,7 +1015,7 @@ static void generate_eosd(struct vo *vo, mp_eosd_images_t *imgs)
     need_upload = true;
     VdpRGBAFormat format;
     int format_size;
-    switch (imgs->type) {
+    switch (imgs->format) {
     case SUBBITMAP_LIBASS:
         format = VDP_RGBA_FORMAT_A8;
         format_size = 1;
@@ -1036,7 +1034,8 @@ static void generate_eosd(struct vo *vo, mp_eosd_images_t *imgs)
     sfc->format = format;
     if (!sfc->packer)
         sfc->packer = make_packer(vo, format);
-    int r = packer_pack_from_subbitmaps(sfc->packer, imgs, imgs->scaled);
+    sfc->packer->padding = imgs->scaled; // assume 2x2 filter on scaling
+    int r = packer_pack_from_subbitmaps(sfc->packer, imgs);
     if (r < 0) {
         mp_msg(MSGT_VO, MSGL_ERR, "[vdpau] EOSD bitmaps do not fit on "
                "a surface with the maximum supported size\n");
@@ -1072,57 +1071,34 @@ eosd_skip_upload:
         vc->eosd_targets_size = sfc->packer->count;
         vc->eosd_targets = talloc_size(vc, vc->eosd_targets_size
                                            * sizeof(*vc->eosd_targets));
-        }
+    }
 
-    if (imgs->type == SUBBITMAP_LIBASS) {
-        int i = 0;
-        for (ASS_Image *p = imgs->imgs; p; p = p->next, i++) {
-            if (p->w == 0 || p->h == 0)
-                continue;
-            struct eosd_target *target = vc->eosd_targets +
-                                         vc->eosd_render_count;
-            int x = sfc->packer->result[i].x;
-            int y = sfc->packer->result[i].y;
-            target->source = (VdpRect){x, y, x + p->w, y + p->h};
-            if (need_upload) {
-                vdp_st = vdp->
-                    bitmap_surface_put_bits_native(sfc->surface,
-                                                   (const void *) &p->bitmap,
-                                                   &p->stride, &target->source);
-                CHECK_ST_WARNING("EOSD: putbits failed");
-            }
-            // Render dest, color, etc.
-            target->color.alpha = 1.0 - ((p->color >> 0) & 0xff) / 255.0;
-            target->color.blue  = ((p->color >>  8) & 0xff) / 255.0;
-            target->color.green = ((p->color >> 16) & 0xff) / 255.0;
-            target->color.red   = ((p->color >> 24) & 0xff) / 255.0;
-            target->dest.x0 = p->dst_x;
-            target->dest.y0 = p->dst_y;
-            target->dest.x1 = p->w + p->dst_x;
-            target->dest.y1 = p->h + p->dst_y;
-            vc->eosd_render_count++;
+    for (int i = 0 ;i < sfc->packer->count; i++) {
+        struct sub_bitmap *b = &imgs->parts[i];
+        struct eosd_target *target = vc->eosd_targets + vc->eosd_render_count;
+        int x = sfc->packer->result[i].x;
+        int y = sfc->packer->result[i].y;
+        target->source = (VdpRect){x, y, x + b->w, y + b->h};
+        target->dest = (VdpRect){b->x, b->y, b->x + b->dw, b->y + b->dh};
+        target->color = (VdpColor){1, 1, 1, 1};
+        if (imgs->format == SUBBITMAP_LIBASS) {
+            uint32_t color = b->libass.color;
+            target->color.alpha = 1.0 - ((color >> 0) & 0xff) / 255.0;
+            target->color.blue  = ((color >>  8) & 0xff) / 255.0;
+            target->color.green = ((color >> 16) & 0xff) / 255.0;
+            target->color.red   = ((color >> 24) & 0xff) / 255.0;
         }
-    } else {
-        for (int i = 0 ;i < sfc->packer->count; i++) {
-            struct sub_bitmap *b = &imgs->parts[i];
-            struct eosd_target *target = vc->eosd_targets +
-                                         vc->eosd_render_count;
-            int x = sfc->packer->result[i].x;
-            int y = sfc->packer->result[i].y;
-            target->source = (VdpRect){x, y, x + b->w, y + b->h};
-            if (need_upload) {
-                vdp_st = vdp->
-                    bitmap_surface_put_bits_native(sfc->surface,
-                                                   &(const void *){b->bitmap},
-                                                   &(uint32_t){b->w * 4},
-                                                   &target->source);
+        if (need_upload) {
+            vdp_st = vdp->
+                bitmap_surface_put_bits_native(sfc->surface,
+                                               &(const void *){b->bitmap},
+                                               &(uint32_t){b->stride},
+                                               &target->source);
                 CHECK_ST_WARNING("EOSD: putbits failed");
-            }
-            target->color = (VdpColor){1, 1, 1, 1};
-            target->dest = (VdpRect){b->x, b->y, b->x + b->dw, b->y + b->dh};
-            vc->eosd_render_count++;
         }
+        vc->eosd_render_count++;
     }
+
     vc->bitmap_id = imgs->bitmap_id;
     vc->bitmap_pos_id = imgs->bitmap_pos_id;
 }
@@ -1133,7 +1109,7 @@ static void record_osd(void *ctx, int x0, int y0, int w, int h,
     struct vo *vo = ctx;
     struct vdpctx *vc = vo->priv;
 
-    assert(vc->old_osd_count < MAX_OLD_OSD_BITMAPS);
+    assert(vc->old_osd_count < MAX_OSD_PARTS);
     if (!w || !h)
         return;
     vc->old_osd_elements[vc->old_osd_count++] = (struct old_osd){