subs, vo_vdpau: support RGBA color for PGS subtitles

Support passing bitmap subtitles to VOs in full RGBA color, and
implement this for libavcodec-decoded subtitle formats on decoding
side and vo_vdpau on display side. Currently this is enabled for PGS
(blu-ray) and DVB subtitles.

VDPAU seems to have sampling issues similar to known GL ones when
drawing a sub-rectangle from a larger texture with scaling, where
adjacent pixels outside the specified source rectangle affect the
result. As the bitmap subtitles may be scaled, add padding support to
the bitmap packer code.

In principle, this could be used for colored DVD subtitles too.
However, the libavcodec DVD decoder lacks parts of the resolution and
palette handling that are present in spudec.c.

Conflicts:
	libvo/vo_gl.c
	sub/dec_sub.h
	sub/sd_lavc.c

3 files changed, 127 insertions, 40 deletions

diff --git a/libvo/bitmap_packer.c b/libvo/bitmap_packer.c
index 2ae8a1eee0..eedc2e2242 100644
--- a/libvo/bitmap_packer.c
+++ b/libvo/bitmap_packer.c
@@ -28,6 +28,7 @@
 #include "mp_msg.h"
 #include "mpcommon.h"
 #include "sub/ass_mp.h"
+#include "sub/dec_sub.h"
 
 
 #define HEIGHT_SORT_BITS 4
@@ -55,7 +56,7 @@ static int size_index(int s)
  * free rectangle with corners (13, 20)-(w, 50) is filled recursively.
  */
 static int pack_rectangles(struct pos *in, struct pos *out, int num_rects,
-                           int w, int h, int *scratch)
+                           int w, int h, int *scratch, int *used_width)
 {
     int bins[16 << HEIGHT_SORT_BITS];
     int sizes[16 << HEIGHT_SORT_BITS] = { 0 };
@@ -100,11 +101,12 @@ static int pack_rectangles(struct pos *in, struct pos *out, int num_rects,
                 s.x = right;
                 maxy = FFMAX(maxy, bottom);
             }
+            *used_width = FFMAX(*used_width, s.x);
             if (maxy > 0)
                 s.bottom = maxy;
         }
     }
-    return num_rects ? -1 : 0;
+    return num_rects ? -1 : y;
 }
 
 int packer_pack(struct bitmap_packer *packer)
@@ -115,7 +117,7 @@ int packer_pack(struct bitmap_packer *packer)
     struct pos *in = packer->in;
     int xmax = 0, ymax = 0;
     for (int i = 0; i < packer->count; i++) {
-        if (in[i].x == 0 || in[i].y == 0)
+        if (in[i].x <= packer->padding || in[i].y <= packer->padding)
             in[i] = (struct pos){0, 0};
         if (in[i].x < 0 || in [i].x > 65535 || in[i].y < 0 || in[i].y > 65535) {
             mp_msg(MSGT_VO, MSGL_FATAL, "Invalid OSD / subtitle bitmap size\n");
@@ -124,20 +126,33 @@ int packer_pack(struct bitmap_packer *packer)
         xmax = FFMAX(xmax, in[i].x);
         ymax = FFMAX(ymax, in[i].y);
     }
+    xmax = FFMAX(0, xmax - packer->padding);
+    ymax = FFMAX(0, ymax - packer->padding);
     if (xmax > packer->w)
         packer->w = 1 << av_log2(xmax - 1) + 1;
     if (ymax > packer->h)
         packer->h = 1 << av_log2(ymax - 1) + 1;
     while (1) {
-        if (pack_rectangles(in, packer->result, packer->count, packer->w,
-                            packer->h, packer->scratch) >= 0)
+        int used_width = 0;
+        int y = pack_rectangles(in, packer->result, packer->count,
+                                packer->w + packer->padding,
+                                packer->h + packer->padding,
+                                packer->scratch, &used_width);
+        if (y >= 0) {
+            // No padding at edges
+            packer->used_width = FFMIN(used_width, packer->w);
+            packer->used_height = FFMIN(y, packer->h);
             return packer->w != w_orig || packer->h != h_orig;
+        }
         if (packer->w <= packer->h && packer->w != packer->w_max)
             packer->w = FFMIN(packer->w * 2, packer->w_max);
         else if (packer->h != packer->h_max)
             packer->h = FFMIN(packer->h * 2, packer->h_max);
-        else
+        else {
+            packer->w = w_orig;
+            packer->h = h_orig;
             return -1;
+        }
     }
 }
 
@@ -156,8 +171,8 @@ void packer_set_size(struct bitmap_packer *packer, int size)
                                            packer->asize + 16);
 }
 
-int packer_pack_from_assimg(struct bitmap_packer *packer,
-                            struct ass_image *imglist)
+static int packer_pack_from_assimg(struct bitmap_packer *packer,
+                                   struct ass_image *imglist)
 {
     int count = 0;
     struct ass_image *img = imglist;
@@ -172,3 +187,20 @@ int packer_pack_from_assimg(struct bitmap_packer *packer,
     packer->count = count;
     return packer_pack(packer);
 }
+
+int packer_pack_from_subbitmaps(struct bitmap_packer *packer,
+                                struct sub_bitmaps *b, int padding_pixels)
+{
+    packer->padding = 0;
+    packer->count = 0;
+    if (b->type == SUBBITMAP_EMPTY)
+        return 0;
+    if (b->type == SUBBITMAP_LIBASS)
+        return packer_pack_from_assimg(packer, b->imgs);
+    packer->padding = padding_pixels;
+    packer_set_size(packer, b->part_count);
+    int a = packer->padding;
+    for (int i = 0; i < b->part_count; i++)
+        packer->in[i] = (struct pos){b->parts[i].w + a, b->parts[i].h + a};
+    return packer_pack(packer);
+}
diff --git a/libvo/bitmap_packer.h b/libvo/bitmap_packer.h
index 45b6fbc4ae..c7c377cbd0 100644
--- a/libvo/bitmap_packer.h
+++ b/libvo/bitmap_packer.h
@@ -11,9 +11,12 @@ struct bitmap_packer {
     int h;
     int w_max;
     int h_max;
+    int padding;
     int count;
     struct pos *in;
     struct pos *result;
+    int used_width;
+    int used_height;
 
     // internal
     int *scratch;
@@ -21,6 +24,7 @@ struct bitmap_packer {
 };
 
 struct ass_image;
+struct sub_bitmaps;
 
 /* Reallocate packer->in for at least to desired number of items.
  * Also sets packer->count to the same value.
@@ -41,7 +45,7 @@ int packer_pack(struct bitmap_packer *packer);
  * packer->in will be reallocated if needed and filled from the
  * given image list.
  */
-int packer_pack_from_assimg(struct bitmap_packer *packer,
-                            struct ass_image *imglist);
+int packer_pack_from_subbitmaps(struct bitmap_packer *packer,
+                                struct sub_bitmaps *b, int padding_pixels);
 
 #endif
diff --git a/libvo/vo_vdpau.c b/libvo/vo_vdpau.c
index 922ec24201..594829b394 100644
--- a/libvo/vo_vdpau.c
+++ b/libvo/vo_vdpau.c
@@ -172,6 +172,7 @@ struct vdpctx {
 
     // EOSD
     struct eosd_bitmap_surface {
+        VdpRGBAFormat format;
         VdpBitmapSurface surface;
         uint32_t max_width;
         uint32_t max_height;
@@ -1007,16 +1008,35 @@ static void generate_eosd(struct vo *vo, mp_eosd_images_t *imgs)
 
     vc->eosd_render_count = 0;
 
-    if (!imgs->imgs)
-        return; // There's nothing to render!
+    if (imgs->type == SUBBITMAP_EMPTY)
+        return;
 
     if (imgs->bitmap_id == vc->bitmap_id)
         goto eosd_skip_upload;
 
     need_upload = true;
+    VdpRGBAFormat format;
+    int format_size;
+    switch (imgs->type) {
+    case SUBBITMAP_LIBASS:
+        format = VDP_RGBA_FORMAT_A8;
+        format_size = 1;
+        break;
+    case SUBBITMAP_RGBA:
+        format = VDP_RGBA_FORMAT_B8G8R8A8;
+        format_size = 4;
+        break;
+    default:
+        abort();
+    };
+    if (sfc->format != format) {
+        talloc_free(sfc->packer);
+        sfc->packer = NULL;
+    };
+    sfc->format = format;
     if (!sfc->packer)
-        sfc->packer = make_packer(vo, VDP_RGBA_FORMAT_A8);
-    int r = packer_pack_from_assimg(sfc->packer, imgs->imgs);
+        sfc->packer = make_packer(vo, format);
+    int r = packer_pack_from_subbitmaps(sfc->packer, imgs, imgs->scaled);
     if (r < 0) {
         mp_msg(MSGT_VO, MSGL_ERR, "[vdpau] EOSD bitmaps do not fit on "
                "a surface with the maximum supported size\n");
@@ -1028,13 +1048,21 @@ static void generate_eosd(struct vo *vo, mp_eosd_images_t *imgs)
         }
         mp_msg(MSGT_VO, MSGL_V, "[vdpau] Allocating a %dx%d surface for "
                "EOSD bitmaps.\n", sfc->packer->w, sfc->packer->h);
-        vdp_st = vdp->bitmap_surface_create(vc->vdp_device, VDP_RGBA_FORMAT_A8,
+        vdp_st = vdp->bitmap_surface_create(vc->vdp_device, format,
                                             sfc->packer->w, sfc->packer->h,
                                             true, &sfc->surface);
         if (vdp_st != VDP_STATUS_OK)
             sfc->surface = VDP_INVALID_HANDLE;
         CHECK_ST_WARNING("EOSD: error when creating surface");
     }
+    if (imgs->scaled) {
+        char zeros[sfc->packer->used_width * format_size];
+        memset(zeros, 0, sizeof(zeros));
+        vdp_st = vdp->bitmap_surface_put_bits_native(sfc->surface,
+                &(const void *){zeros}, &(uint32_t){0},
+                &(VdpRect){0, 0, sfc->packer->used_width,
+                                 sfc->packer->used_height});
+    }
 
 eosd_skip_upload:
     if (sfc->surface == VDP_INVALID_HANDLE)
@@ -1046,31 +1074,54 @@ eosd_skip_upload:
                                            * sizeof(*vc->eosd_targets));
         }
 
-    int i = 0;
-    for (ASS_Image *p = imgs->imgs; p; p = p->next, i++) {
-        if (p->w == 0 || p->h == 0)
-            continue;
-        struct eosd_target *target = &vc->eosd_targets[vc->eosd_render_count];
-        int x = sfc->packer->result[i].x;
-        int y = sfc->packer->result[i].y;
-        target->source = (VdpRect){x, y, x + p->w, y + p->h};
-        if (need_upload) {
-            vdp_st = vdp->
-                bitmap_surface_put_bits_native(sfc->surface,
-                                               (const void *) &p->bitmap,
-                                               &p->stride, &target->source);
-            CHECK_ST_WARNING("EOSD: putbits failed");
+    if (imgs->type == SUBBITMAP_LIBASS) {
+        int i = 0;
+        for (ASS_Image *p = imgs->imgs; p; p = p->next, i++) {
+            if (p->w == 0 || p->h == 0)
+                continue;
+            struct eosd_target *target = vc->eosd_targets +
+                                         vc->eosd_render_count;
+            int x = sfc->packer->result[i].x;
+            int y = sfc->packer->result[i].y;
+            target->source = (VdpRect){x, y, x + p->w, y + p->h};
+            if (need_upload) {
+                vdp_st = vdp->
+                    bitmap_surface_put_bits_native(sfc->surface,
+                                                   (const void *) &p->bitmap,
+                                                   &p->stride, &target->source);
+                CHECK_ST_WARNING("EOSD: putbits failed");
+            }
+            // Render dest, color, etc.
+            target->color.alpha = 1.0 - ((p->color >> 0) & 0xff) / 255.0;
+            target->color.blue  = ((p->color >>  8) & 0xff) / 255.0;
+            target->color.green = ((p->color >> 16) & 0xff) / 255.0;
+            target->color.red   = ((p->color >> 24) & 0xff) / 255.0;
+            target->dest.x0 = p->dst_x;
+            target->dest.y0 = p->dst_y;
+            target->dest.x1 = p->w + p->dst_x;
+            target->dest.y1 = p->h + p->dst_y;
+            vc->eosd_render_count++;
+        }
+    } else {
+        for (int i = 0 ;i < sfc->packer->count; i++) {
+            struct sub_bitmap *b = &imgs->parts[i];
+            struct eosd_target *target = vc->eosd_targets +
+                                         vc->eosd_render_count;
+            int x = sfc->packer->result[i].x;
+            int y = sfc->packer->result[i].y;
+            target->source = (VdpRect){x, y, x + b->w, y + b->h};
+            if (need_upload) {
+                vdp_st = vdp->
+                    bitmap_surface_put_bits_native(sfc->surface,
+                                                   &(const void *){b->bitmap},
+                                                   &(uint32_t){b->w * 4},
+                                                   &target->source);
+                CHECK_ST_WARNING("EOSD: putbits failed");
+            }
+            target->color = (VdpColor){1, 1, 1, 1};
+            target->dest = (VdpRect){b->x, b->y, b->x + b->dw, b->y + b->dh};
+            vc->eosd_render_count++;
         }
-        // Render dest, color, etc.
-        target->color.alpha = 1.0 - ((p->color >> 0) & 0xff) / 255.0;
-        target->color.blue  = ((p->color >>  8) & 0xff) / 255.0;
-        target->color.green = ((p->color >> 16) & 0xff) / 255.0;
-        target->color.red   = ((p->color >> 24) & 0xff) / 255.0;
-        target->dest.x0 = p->dst_x;
-        target->dest.y0 = p->dst_y;
-        target->dest.x1 = p->w + p->dst_x;
-        target->dest.y1 = p->h + p->dst_y;
-        vc->eosd_render_count++;
     }
     vc->bitmap_id = imgs->bitmap_id;
     vc->bitmap_pos_id = imgs->bitmap_pos_id;
@@ -1529,7 +1580,7 @@ static int query_format(uint32_t format)
 {
     int default_flags = VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW
         | VFCAP_HWSCALE_UP | VFCAP_HWSCALE_DOWN | VFCAP_OSD | VFCAP_EOSD
-        | VFCAP_EOSD_UNSCALED | VFCAP_FLIP;
+        | VFCAP_EOSD_UNSCALED | VFCAP_EOSD_RGBA | VFCAP_FLIP;
     switch (format) {
     case IMGFMT_YV12:
     case IMGFMT_I420: