subs, vo_vdpau: support RGBA color for PGS subtitles

Support passing bitmap subtitles to VOs in full RGBA color, and
implement this for libavcodec-decoded subtitle formats on decoding
side and vo_vdpau on display side. Currently this is enabled for PGS
(blu-ray) and DVB subtitles.

VDPAU seems to have sampling issues similar to known GL ones when
drawing a sub-rectangle from a larger texture with scaling, where
adjacent pixels outside the specified source rectangle affect the
result. As the bitmap subtitles may be scaled, add padding support to
the bitmap packer code.

In principle, this could be used for colored DVD subtitles too.
However, the libavcodec DVD decoder lacks parts of the resolution and
palette handling that are present in spudec.c.

Conflicts:
	libvo/vo_gl.c
	sub/dec_sub.h
	sub/sd_lavc.c

1 files changed, 120 insertions, 11 deletions

diff --git a/sub/sd_lavc.c b/sub/sd_lavc.c
index a4eafff429..1da33ffca1 100644
--- a/sub/sd_lavc.c
+++ b/sub/sd_lavc.c
@@ -19,6 +19,7 @@
 
 #include <libavcodec/avcodec.h>
 
+#include "talloc.h"
 #include "mp_msg.h"
 #include "libmpdemux/stheader.h"
 #include "sd.h"
@@ -26,8 +27,17 @@
 // Current code still pushes subs directly to global spudec
 #include "sub.h"
 
-static void avsub_to_spudec(AVSubtitleRect **rects, int num_rects,
-                            double pts, double endpts)
+struct sd_lavc_priv {
+    AVCodecContext *avctx;
+    int count;
+    struct sub_bitmap *inbitmaps;
+    struct sub_bitmap *outbitmaps;
+    bool bitmaps_changed;
+    double endpts;
+};
+
+static void old_avsub_to_spudec(AVSubtitleRect **rects, int num_rects,
+                                double pts, double endpts)
 {
     int i, xmin = INT_MAX, ymin = INT_MAX, xmax = INT_MIN, ymax = INT_MIN;
     struct spu_packet_t *packet;
@@ -69,6 +79,9 @@ static void avsub_to_spudec(AVSubtitleRect **rects, int num_rects,
 
 static int init(struct sh_sub *sh, struct osd_state *osd)
 {
+    if (sh->initialized)
+        return 0;
+    struct sd_lavc_priv *priv = talloc_zero(NULL, struct sd_lavc_priv);
     enum CodecID cid = CODEC_ID_NONE;
     switch (sh->type) {
     case 'b':
@@ -77,6 +90,8 @@ static int init(struct sh_sub *sh, struct osd_state *osd)
         cid = CODEC_ID_HDMV_PGS_SUBTITLE; break;
     case 'x':
         cid = CODEC_ID_XSUB; break;
+    case 'v':
+        cid = CODEC_ID_DVD_SUBTITLE; break;
     }
     AVCodecContext *ctx = NULL;
     AVCodec *sub_codec = avcodec_find_decoder(cid);
@@ -85,25 +100,41 @@ static int init(struct sh_sub *sh, struct osd_state *osd)
     ctx = avcodec_alloc_context3(sub_codec);
     if (!ctx)
         goto error;
+    ctx->extradata_size = sh->extradata_len;
+    ctx->extradata = sh->extradata;
     if (avcodec_open2(ctx, sub_codec, NULL) < 0)
         goto error;
-    sh->context = ctx;
+    priv->avctx = ctx;
+    sh->context = priv;
     return 0;
 
  error:
     mp_msg(MSGT_SUBREADER, MSGL_ERR,
            "Could not open libavcodec subtitle decoder\n");
     av_free(ctx);
+    talloc_free(priv);
     return -1;
 }
 
+static void clear(struct sd_lavc_priv *priv)
+{
+    priv->count = 0;
+    talloc_free(priv->inbitmaps);
+    talloc_free(priv->outbitmaps);
+    priv->inbitmaps = priv->outbitmaps = NULL;
+    priv->bitmaps_changed = true;
+    priv->endpts = MP_NOPTS_VALUE;
+}
+
 static void decode(struct sh_sub *sh, struct osd_state *osd, void *data,
                    int data_len, double pts, double duration)
 {
-    AVCodecContext *ctx = sh->context;
+    struct sd_lavc_priv *priv = sh->context;
+    AVCodecContext *ctx = priv->avctx;
     AVSubtitle sub;
     AVPacket pkt;
 
+    clear(priv);
     av_init_packet(&pkt);
     pkt.data = data;
     pkt.size = data_len;
@@ -127,10 +158,37 @@ static void decode(struct sh_sub *sh, struct osd_state *osd, void *data,
     if (sub.num_rects > 0) {
         switch (sub.rects[0]->type) {
         case SUBTITLE_BITMAP:
-            if (!vo_spudec)
-                vo_spudec = spudec_new_scaled(NULL, ctx->width, ctx->height, NULL, 0);
-            avsub_to_spudec(sub.rects, sub.num_rects, pts, endpts);
-            vo_osd_changed(OSDTYPE_SPU);
+            // Assume resolution heuristics only work for PGS and DVB
+            if (!osd->support_rgba || sh->type != 'p' && sh->type != 'b') {
+                if (!vo_spudec)
+                    vo_spudec = spudec_new_scaled(NULL, ctx->width, ctx->height,
+                                                  NULL, 0);
+                old_avsub_to_spudec(sub.rects, sub.num_rects, pts, endpts);
+                vo_osd_changed(OSDTYPE_SPU);
+                break;
+            }
+            priv->inbitmaps = talloc_array(priv, struct sub_bitmap,
+                                           sub.num_rects);
+            for (int i = 0; i < sub.num_rects; i++) {
+                struct AVSubtitleRect *r = sub.rects[i];
+                struct sub_bitmap *b = &priv->inbitmaps[i];
+                uint32_t *outbmp = talloc_size(priv->inbitmaps,
+                                               r->w * r->h * 4);
+                b->bitmap = outbmp;
+                b->w = r->w;
+                b->h = r->h;
+                b->x = r->x;
+                b->y = r->y;
+                uint8_t *inbmp = r->pict.data[0];
+                uint32_t *palette = (uint32_t *) r->pict.data[1];
+                for (int y = 0; y < r->h; y++) {
+                    for (int x = 0; x < r->w; x++)
+                        *outbmp++ = palette[*inbmp++];
+                    inbmp += r->pict.linesize[0] - r->w;
+                };
+            }
+            priv->count = sub.num_rects;
+            priv->endpts = endpts;
             break;
         default:
             mp_msg(MSGT_SUBREADER, MSGL_ERR, "sd_lavc: unsupported subtitle "
@@ -141,21 +199,72 @@ static void decode(struct sh_sub *sh, struct osd_state *osd, void *data,
     avsubtitle_free(&sub);
 }
 
+static void get_bitmaps(struct sh_sub *sh, struct osd_state *osd,
+                        struct sub_bitmaps *res)
+{
+    struct sd_lavc_priv *priv = sh->context;
+
+    if (priv->endpts != MP_NOPTS_VALUE && (osd->sub_pts >= priv->endpts ||
+                                           osd->sub_pts < priv->endpts - 300))
+        clear(priv);
+    if (!osd->support_rgba)
+        return;
+    if (priv->bitmaps_changed && priv->count > 0)
+        priv->outbitmaps = talloc_memdup(priv, priv->inbitmaps,
+                                         talloc_get_size(priv->inbitmaps));
+    bool pos_changed = false;
+    // Hope that PGS subs set these and 720/576 works for dvb subs
+    int inw = priv->avctx->width;
+    if (!inw)
+        inw = 720;
+    int inh = priv->avctx->height;
+    if (!inh)
+        inh = 576;
+    struct mp_eosd_res *d = &osd->dim;
+    double xscale = (double) (d->w - d->ml - d->mr) / inw;
+    double yscale = (double) (d->h - d->mt - d->mb) / inh;
+    for (int i = 0; i < priv->count; i++) {
+        struct sub_bitmap *bi = &priv->inbitmaps[i];
+        struct sub_bitmap *bo = &priv->outbitmaps[i];
+#define SET(var, val) pos_changed |= var != (int)(val); var = (val)
+        SET(bo->x, bi->x * xscale + d->ml);
+        SET(bo->y, bi->y * yscale + d->mt);
+        SET(bo->dw, bi->w * xscale);
+        SET(bo->dh, bi->h * yscale);
+    }
+    res->parts = priv->outbitmaps;
+    res->part_count = priv->count;
+    if (priv->bitmaps_changed)
+        res->bitmap_id = ++res->bitmap_pos_id;
+    else if (pos_changed)
+        res->bitmap_pos_id++;
+    priv->bitmaps_changed = false;
+    res->type = SUBBITMAP_RGBA;
+    res->scaled = xscale != 1 || yscale != 1;
+}
+
 static void reset(struct sh_sub *sh, struct osd_state *osd)
 {
+    struct sd_lavc_priv *priv = sh->context;
+
+    clear(priv);
     // lavc might not do this right for all codecs; may need close+reopen
-    avcodec_flush_buffers(sh->context);
+    avcodec_flush_buffers(priv->avctx);
 }
 
 static void uninit(struct sh_sub *sh)
 {
-    avcodec_close(sh->context);
-    av_free(sh->context);
+    struct sd_lavc_priv *priv = sh->context;
+
+    avcodec_close(priv->avctx);
+    av_free(priv->avctx);
+    talloc_free(priv);
 }
 
 const struct sd_functions sd_lavc = {
     .init = init,
     .decode = decode,
+    .get_bitmaps = get_bitmaps,
     .reset = reset,
     .switch_off = reset,
     .uninit = uninit,