demux_libass: do charset conversion by -subcp

Old code used to use libass' recoding feature, which is a copy of the
old MPlayer code. We dropped that a few commits ago. Unfortunately,
this made it impossible to load some subtitle files, like UTF-16 files.

Make .ass loading respect -subcp again. We do this by recoding the
probe buffer to UTF-8, and then trying to load it normally. (Yep.)

Since UTF-16 in particular will effectively half the probe buffer size,
double the probe size.

1 files changed, 16 insertions, 3 deletions

diff --git a/demux/demux_libass.c b/demux/demux_libass.c
index 1b7dc24452..cbc85b3abe 100644
--- a/demux/demux_libass.c
+++ b/demux/demux_libass.c
@@ -23,10 +23,11 @@
 
 #include "core/options.h"
 #include "core/mp_msg.h"
+#include "core/charset_conv.h"
 #include "stream/stream.h"
 #include "demux.h"
 
-#define PROBE_SIZE (4 * 1024)
+#define PROBE_SIZE (8 * 1024)
 
 struct priv {
     ASS_Track *track;
@@ -34,6 +35,7 @@ struct priv {
 
 static int d_check_file(struct demuxer *demuxer)
 {
+    const char *user_cp = demuxer->opts->sub_cp;
     struct stream *s = demuxer->stream;
     // Older versions of libass will behave strange if renderer and track
     // library handles mismatch, so make sure everything uses a global handle.
@@ -53,7 +55,13 @@ static int d_check_file(struct demuxer *demuxer)
     memcpy(tmp, buf.start, buf.len);
     buf.start = tmp;
     buf.start[buf.len] = '\0';
-    ASS_Track *track = ass_read_memory(lib, buf.start, buf.len, NULL);
+    bstr cbuf =
+        mp_charset_guess_and_conv_to_utf8(buf, user_cp,  MP_ICONV_ALLOW_CUTOFF);
+    if (cbuf.start == NULL)
+        cbuf = buf;
+    ASS_Track *track = ass_read_memory(lib, cbuf.start, cbuf.len, NULL);
+    if (cbuf.start != buf.start)
+        talloc_free(cbuf.start);
     talloc_free(buf.start);
     if (!track)
         return 0;
@@ -67,7 +75,12 @@ static int d_check_file(struct demuxer *demuxer)
                 "larger than 100 MB: %s\n", demuxer->filename);
         return 0;
     }
-    track = ass_read_memory(lib, buf.start, buf.len, NULL);
+    cbuf = mp_charset_guess_and_conv_to_utf8(buf, user_cp,  MP_ICONV_VERBOSE);
+    if (cbuf.start == NULL)
+        cbuf = buf;
+    track = ass_read_memory(lib, cbuf.start, cbuf.len, NULL);
+    if (cbuf.start != buf.start)
+        talloc_free(cbuf.start);
     talloc_free(buf.start);
     if (!track)
         return 0;