sub: handle vobsub-in-mp4

The mplayer decoder (spudec.c) actually handled this. There was explicit
code for binary palettes (16 32 bit values), and the subtitle resolution
was handled by video resolution coincidentally matching the subtitle
resolution.

Whoever puts vobsub into mp4 should be punished.

Fixes the sample gundam_sample.mp4, closes github issue #547.

5 files changed, 44 insertions, 0 deletions

diff --git a/demux/demux_lavf.c b/demux/demux_lavf.c
index e3ac84b50d..d85e4b8095 100644
--- a/demux/demux_lavf.c
+++ b/demux/demux_lavf.c
@@ -458,6 +458,8 @@ static void handle_stream(demuxer_t *demuxer, int i)
             memcpy(sh_sub->extradata, codec->extradata, codec->extradata_size);
             sh_sub->extradata_len = codec->extradata_size;
         }
+        sh_sub->w = codec->width;
+        sh_sub->h = codec->height;
 
         // Hack for MicroDVD: if time_base matches the ffmpeg microdvd reader's
         // default FPS (23.976), assume the MicroDVD file did not declare a
diff --git a/demux/stheader.h b/demux/stheader.h
index 25f60ba032..177bf50b39 100644
--- a/demux/stheader.h
+++ b/demux/stheader.h
@@ -87,6 +87,7 @@ typedef struct sh_video {
 typedef struct sh_sub {
     unsigned char *extradata;   // extra header data passed from demuxer
     int extradata_len;
+    int w, h;                   // mp4 vobsubs
     int frame_based;            // timestamps are frame-based
     bool is_utf8;               // if false, subtitle packet charset is unknown
     struct ass_track *track;    // loaded by libass
diff --git a/sub/dec_sub.c b/sub/dec_sub.c
index 46c5b3fafb..fdb8d8f648 100644
--- a/sub/dec_sub.c
+++ b/sub/dec_sub.c
@@ -210,6 +210,8 @@ void sub_init_from_sh(struct dec_sub *sub, struct sh_stream *sh)
     struct sd init_sd = sub->init_sd;
     init_sd.codec = sh->codec;
     init_sd.ass_track = sh->sub->track;
+    init_sd.sub_stream_w = sh->sub->w;
+    init_sd.sub_stream_h = sh->sub->h;
 
     while (sub->num_sd < MAX_NUM_SD) {
         struct sd *sd = talloc(NULL, struct sd);
diff --git a/sub/sd.h b/sub/sd.h
index 088dc166d8..78b9825ead 100644
--- a/sub/sd.h
+++ b/sub/sd.h
@@ -25,6 +25,9 @@ struct sd {
     // the resolution of the VO, nor does it have to be the OSD resolution.
     int sub_video_w, sub_video_h;
 
+    // Resolution hints stored in mp4 files.
+    int sub_stream_w, sub_stream_h;
+
     // Make sd_ass use an existing track
     struct ass_track *ass_track;
 
diff --git a/sub/sd_lavc.c b/sub/sd_lavc.c
index 9f57d9ec38..0252a37207 100644
--- a/sub/sd_lavc.c
+++ b/sub/sd_lavc.c
@@ -21,12 +21,14 @@
 
 #include <libavcodec/avcodec.h>
 #include <libavutil/common.h>
+#include <libavutil/intreadwrite.h>
 
 #include "talloc.h"
 #include "common/msg.h"
 #include "common/av_common.h"
 #include "options/options.h"
 #include "video/mp_image.h"
+#include "video/csputils.h"
 #include "sd.h"
 #include "dec_sub.h"
 
@@ -92,6 +94,35 @@ static void get_resolution(struct sd *sd, int wh[2])
     guess_resolution(priv->avctx->codec_id, &wh[0], &wh[1]);
 }
 
+static void set_mp4_vobsub_idx(AVCodecContext *avctx, char *src, int w, int h)
+{
+    char pal_s[128];
+    int pal_s_pos = 0;
+    for (int i = 0; i < 16; i++) {
+        unsigned int e = AV_RB32(src + i * 4);
+
+        // lavc doesn't accept YUV palette - "does god hate me?"
+        struct mp_csp_params csp = MP_CSP_PARAMS_DEFAULTS;
+        csp.int_bits_in = 8;
+        csp.int_bits_out = 8;
+        float cmatrix[3][4];
+        mp_get_yuv2rgb_coeffs(&csp, cmatrix);
+        int c[3] = {(e >> 16) & 0xff, (e >> 8) & 0xff, e & 0xff};
+        mp_map_int_color(cmatrix, 8, c);
+        e = (c[2] << 16) | (c[1] << 8) | c[0];
+
+        snprintf(pal_s + pal_s_pos, sizeof(pal_s) - pal_s_pos, "%06x%s", e,
+                 i != 15 ? ", " : "");
+        pal_s_pos = strlen(pal_s);
+        if (pal_s_pos >= sizeof(pal_s))
+            break;
+    }
+
+    char buf[256] = "";
+    snprintf(buf, sizeof(buf), "size: %dx%d\npalette: %s\n", w, h, pal_s);
+    mp_lavc_set_extradata(avctx, buf, strlen(buf));
+}
+
 static int init(struct sd *sd)
 {
     struct sd_lavc_priv *priv = talloc_zero(NULL, struct sd_lavc_priv);
@@ -104,6 +135,11 @@ static int init(struct sd *sd)
     if (!ctx)
         goto error;
     mp_lavc_set_extradata(ctx, sd->extradata, sd->extradata_len);
+    if (sd->extradata_len == 64 && sd->sub_stream_w && sd->sub_stream_h &&
+        cid == AV_CODEC_ID_DVD_SUBTITLE)
+    {
+        set_mp4_vobsub_idx(ctx, sd->extradata, sd->sub_stream_w, sd->sub_stream_h);
+    }
     if (avcodec_open2(ctx, sub_codec, NULL) < 0)
         goto error;
     priv->avctx = ctx;