subs, vo_vdpau: support RGBA color for PGS subtitles

Support passing bitmap subtitles to VOs in full RGBA color, and
implement this for libavcodec-decoded subtitle formats on decoding
side and vo_vdpau on display side. Currently this is enabled for PGS
(blu-ray) and DVB subtitles.

VDPAU seems to have sampling issues similar to known GL ones when
drawing a sub-rectangle from a larger texture with scaling, where
adjacent pixels outside the specified source rectangle affect the
result. As the bitmap subtitles may be scaled, add padding support to
the bitmap packer code.

In principle, this could be used for colored DVD subtitles too.
However, the libavcodec DVD decoder lacks parts of the resolution and
palette handling that are present in spudec.c.

Conflicts:
	libvo/vo_gl.c
	sub/dec_sub.h
	sub/sd_lavc.c

5 files changed, 143 insertions, 12 deletions

diff --git a/sub/dec_sub.c b/sub/dec_sub.c
index 3278c10d85..4a048b27a6 100644
--- a/sub/dec_sub.c
+++ b/sub/dec_sub.c
@@ -62,7 +62,8 @@ void sub_get_bitmaps(struct osd_state *osd, struct sub_bitmaps *res)
 {
     struct MPOpts *opts = osd->opts;
 
-    *res = (struct sub_bitmaps){ .bitmap_id = osd->bitmap_id,
+    *res = (struct sub_bitmaps){ .type = SUBBITMAP_EMPTY,
+                                 .bitmap_id = osd->bitmap_id,
                                  .bitmap_pos_id = osd->bitmap_pos_id };
     if (!opts->sub_visibility || !osd->sh_sub || !osd->sh_sub->active) {
         /* Change ID in case we just switched from visible subtitles
diff --git a/sub/dec_sub.h b/sub/dec_sub.h
index efbfdc9ce7..c71a2348aa 100644
--- a/sub/dec_sub.h
+++ b/sub/dec_sub.h
@@ -5,13 +5,32 @@ struct sh_sub;
 struct osd_state;
 struct ass_track;
 
+enum sub_bitmap_type {
+    SUBBITMAP_EMPTY,
+    SUBBITMAP_LIBASS,
+    SUBBITMAP_RGBA,
+};
+
 typedef struct mp_eosd_res {
     int w, h; // screen dimensions, including black borders
     int mt, mb, ml, mr; // borders (top, bottom, left, right)
 } mp_eosd_res_t;
 
 typedef struct sub_bitmaps {
+    enum sub_bitmap_type type;
+
     struct ass_image *imgs;
+
+    struct sub_bitmap {
+        int w, h;
+        int x, y;
+        // Note: not clipped, going outside the screen area is allowed
+        int dw, dh;
+        void *bitmap;
+    } *parts;
+    int part_count;
+
+    bool scaled;
     int bitmap_id, bitmap_pos_id;
 } mp_eosd_images_t;
 
diff --git a/sub/sd_ass.c b/sub/sd_ass.c
index 37c220a4b5..9295cab07d 100644
--- a/sub/sd_ass.c
+++ b/sub/sd_ass.c
@@ -147,6 +147,7 @@ static void get_bitmaps(struct sh_sub *sh, struct osd_state *osd,
         res->bitmap_id = ++res->bitmap_pos_id;
     else if (changed)
         res->bitmap_pos_id++;
+    res->type = SUBBITMAP_LIBASS;
 }
 
 static void reset(struct sh_sub *sh, struct osd_state *osd)
diff --git a/sub/sd_lavc.c b/sub/sd_lavc.c
index a4eafff429..1da33ffca1 100644
--- a/sub/sd_lavc.c
+++ b/sub/sd_lavc.c
@@ -19,6 +19,7 @@
 
 #include <libavcodec/avcodec.h>
 
+#include "talloc.h"
 #include "mp_msg.h"
 #include "libmpdemux/stheader.h"
 #include "sd.h"
@@ -26,8 +27,17 @@
 // Current code still pushes subs directly to global spudec
 #include "sub.h"
 
-static void avsub_to_spudec(AVSubtitleRect **rects, int num_rects,
-                            double pts, double endpts)
+struct sd_lavc_priv {
+    AVCodecContext *avctx;
+    int count;
+    struct sub_bitmap *inbitmaps;
+    struct sub_bitmap *outbitmaps;
+    bool bitmaps_changed;
+    double endpts;
+};
+
+static void old_avsub_to_spudec(AVSubtitleRect **rects, int num_rects,
+                                double pts, double endpts)
 {
     int i, xmin = INT_MAX, ymin = INT_MAX, xmax = INT_MIN, ymax = INT_MIN;
     struct spu_packet_t *packet;
@@ -69,6 +79,9 @@ static void avsub_to_spudec(AVSubtitleRect **rects, int num_rects,
 
 static int init(struct sh_sub *sh, struct osd_state *osd)
 {
+    if (sh->initialized)
+        return 0;
+    struct sd_lavc_priv *priv = talloc_zero(NULL, struct sd_lavc_priv);
     enum CodecID cid = CODEC_ID_NONE;
     switch (sh->type) {
     case 'b':
@@ -77,6 +90,8 @@ static int init(struct sh_sub *sh, struct osd_state *osd)
         cid = CODEC_ID_HDMV_PGS_SUBTITLE; break;
     case 'x':
         cid = CODEC_ID_XSUB; break;
+    case 'v':
+        cid = CODEC_ID_DVD_SUBTITLE; break;
     }
     AVCodecContext *ctx = NULL;
     AVCodec *sub_codec = avcodec_find_decoder(cid);
@@ -85,25 +100,41 @@ static int init(struct sh_sub *sh, struct osd_state *osd)
     ctx = avcodec_alloc_context3(sub_codec);
     if (!ctx)
         goto error;
+    ctx->extradata_size = sh->extradata_len;
+    ctx->extradata = sh->extradata;
     if (avcodec_open2(ctx, sub_codec, NULL) < 0)
         goto error;
-    sh->context = ctx;
+    priv->avctx = ctx;
+    sh->context = priv;
     return 0;
 
  error:
     mp_msg(MSGT_SUBREADER, MSGL_ERR,
            "Could not open libavcodec subtitle decoder\n");
     av_free(ctx);
+    talloc_free(priv);
     return -1;
 }
 
+static void clear(struct sd_lavc_priv *priv)
+{
+    priv->count = 0;
+    talloc_free(priv->inbitmaps);
+    talloc_free(priv->outbitmaps);
+    priv->inbitmaps = priv->outbitmaps = NULL;
+    priv->bitmaps_changed = true;
+    priv->endpts = MP_NOPTS_VALUE;
+}
+
 static void decode(struct sh_sub *sh, struct osd_state *osd, void *data,
                    int data_len, double pts, double duration)
 {
-    AVCodecContext *ctx = sh->context;
+    struct sd_lavc_priv *priv = sh->context;
+    AVCodecContext *ctx = priv->avctx;
     AVSubtitle sub;
     AVPacket pkt;
 
+    clear(priv);
     av_init_packet(&pkt);
     pkt.data = data;
     pkt.size = data_len;
@@ -127,10 +158,37 @@ static void decode(struct sh_sub *sh, struct osd_state *osd, void *data,
     if (sub.num_rects > 0) {
         switch (sub.rects[0]->type) {
         case SUBTITLE_BITMAP:
-            if (!vo_spudec)
-                vo_spudec = spudec_new_scaled(NULL, ctx->width, ctx->height, NULL, 0);
-            avsub_to_spudec(sub.rects, sub.num_rects, pts, endpts);
-            vo_osd_changed(OSDTYPE_SPU);
+            // Assume resolution heuristics only work for PGS and DVB
+            if (!osd->support_rgba || sh->type != 'p' && sh->type != 'b') {
+                if (!vo_spudec)
+                    vo_spudec = spudec_new_scaled(NULL, ctx->width, ctx->height,
+                                                  NULL, 0);
+                old_avsub_to_spudec(sub.rects, sub.num_rects, pts, endpts);
+                vo_osd_changed(OSDTYPE_SPU);
+                break;
+            }
+            priv->inbitmaps = talloc_array(priv, struct sub_bitmap,
+                                           sub.num_rects);
+            for (int i = 0; i < sub.num_rects; i++) {
+                struct AVSubtitleRect *r = sub.rects[i];
+                struct sub_bitmap *b = &priv->inbitmaps[i];
+                uint32_t *outbmp = talloc_size(priv->inbitmaps,
+                                               r->w * r->h * 4);
+                b->bitmap = outbmp;
+                b->w = r->w;
+                b->h = r->h;
+                b->x = r->x;
+                b->y = r->y;
+                uint8_t *inbmp = r->pict.data[0];
+                uint32_t *palette = (uint32_t *) r->pict.data[1];
+                for (int y = 0; y < r->h; y++) {
+                    for (int x = 0; x < r->w; x++)
+                        *outbmp++ = palette[*inbmp++];
+                    inbmp += r->pict.linesize[0] - r->w;
+                };
+            }
+            priv->count = sub.num_rects;
+            priv->endpts = endpts;
             break;
         default:
             mp_msg(MSGT_SUBREADER, MSGL_ERR, "sd_lavc: unsupported subtitle "
@@ -141,21 +199,72 @@ static void decode(struct sh_sub *sh, struct osd_state *osd, void *data,
     avsubtitle_free(&sub);
 }
 
+static void get_bitmaps(struct sh_sub *sh, struct osd_state *osd,
+                        struct sub_bitmaps *res)
+{
+    struct sd_lavc_priv *priv = sh->context;
+
+    if (priv->endpts != MP_NOPTS_VALUE && (osd->sub_pts >= priv->endpts ||
+                                           osd->sub_pts < priv->endpts - 300))
+        clear(priv);
+    if (!osd->support_rgba)
+        return;
+    if (priv->bitmaps_changed && priv->count > 0)
+        priv->outbitmaps = talloc_memdup(priv, priv->inbitmaps,
+                                         talloc_get_size(priv->inbitmaps));
+    bool pos_changed = false;
+    // Hope that PGS subs set these and 720/576 works for dvb subs
+    int inw = priv->avctx->width;
+    if (!inw)
+        inw = 720;
+    int inh = priv->avctx->height;
+    if (!inh)
+        inh = 576;
+    struct mp_eosd_res *d = &osd->dim;
+    double xscale = (double) (d->w - d->ml - d->mr) / inw;
+    double yscale = (double) (d->h - d->mt - d->mb) / inh;
+    for (int i = 0; i < priv->count; i++) {
+        struct sub_bitmap *bi = &priv->inbitmaps[i];
+        struct sub_bitmap *bo = &priv->outbitmaps[i];
+#define SET(var, val) pos_changed |= var != (int)(val); var = (val)
+        SET(bo->x, bi->x * xscale + d->ml);
+        SET(bo->y, bi->y * yscale + d->mt);
+        SET(bo->dw, bi->w * xscale);
+        SET(bo->dh, bi->h * yscale);
+    }
+    res->parts = priv->outbitmaps;
+    res->part_count = priv->count;
+    if (priv->bitmaps_changed)
+        res->bitmap_id = ++res->bitmap_pos_id;
+    else if (pos_changed)
+        res->bitmap_pos_id++;
+    priv->bitmaps_changed = false;
+    res->type = SUBBITMAP_RGBA;
+    res->scaled = xscale != 1 || yscale != 1;
+}
+
 static void reset(struct sh_sub *sh, struct osd_state *osd)
 {
+    struct sd_lavc_priv *priv = sh->context;
+
+    clear(priv);
     // lavc might not do this right for all codecs; may need close+reopen
-    avcodec_flush_buffers(sh->context);
+    avcodec_flush_buffers(priv->avctx);
 }
 
 static void uninit(struct sh_sub *sh)
 {
-    avcodec_close(sh->context);
-    av_free(sh->context);
+    struct sd_lavc_priv *priv = sh->context;
+
+    avcodec_close(priv->avctx);
+    av_free(priv->avctx);
+    talloc_free(priv);
 }
 
 const struct sd_functions sd_lavc = {
     .init = init,
     .decode = decode,
+    .get_bitmaps = get_bitmaps,
     .reset = reset,
     .switch_off = reset,
     .uninit = uninit,
diff --git a/sub/sub.h b/sub/sub.h
index 9fd6c5329e..fc0047bba6 100644
--- a/sub/sub.h
+++ b/sub/sub.h
@@ -71,6 +71,7 @@ struct osd_state {
     double normal_scale;
     double vsfilter_scale;
     bool unscaled;
+    bool support_rgba;
 
     struct ass_renderer *osd_render;
     struct ass_library *osd_ass_library;