sd_lavc: use subtitle framework for former av_sub.c code

Change libavcodec subtitle decoding code (used for some bitmap
subtitle types) to use the same decoding framework as sd_ass. The
functionality that was previously in av_sub.c and was directly called
from mplayer.c is now in sd_lavc.c.

Conflicts:
	mplayer.c
	sub/av_sub.h
	sub/sd_lavc.c

Merged from mplayer2. The remaining use of is_av_sub() is replaced by
a check whether a subtitle decoder is active, which should give the
same results.

5 files changed, 72 insertions, 94 deletions

diff --git a/sub/av_sub.h b/sub/av_sub.h
deleted file mode 100644
index ee6205c87d..0000000000
--- a/sub/av_sub.h
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * This file is part of MPlayer.
- *
- * MPlayer is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- *
- * MPlayer is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License along
- * with MPlayer; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
- */
-
-#ifndef MPLAYER_AV_SUB_H
-#define MPLAYER_AV_SUB_H
-
-#include <stdint.h>
-
-struct sh_sub;
-
-void reset_avsub(struct sh_sub *sh);
-int decode_avsub(struct sh_sub *sh, uint8_t *data, int size,
-                 double pts, double endpts);
-bool is_av_sub(int type);
-
-#endif /* MPLAYER_AV_SUB_H */
diff --git a/sub/dec_sub.c b/sub/dec_sub.c
index 6a6d1d77e5..7528e90e58 100644
--- a/sub/dec_sub.c
+++ b/sub/dec_sub.c
@@ -27,6 +27,7 @@
 #include "options.h"
 
 extern const struct sd_functions sd_ass;
+extern const struct sd_functions sd_lavc;
 
 void sub_init(struct sh_sub *sh, struct osd_state *osd)
 {
@@ -36,8 +37,11 @@ void sub_init(struct sh_sub *sh, struct osd_state *osd)
     if (opts->ass_enabled && is_text_sub(sh->type))
         sh->sd_driver = &sd_ass;
 #endif
+    if (strchr("bpx", sh->type))
+        sh->sd_driver = &sd_lavc;
     if (sh->sd_driver) {
-        sh->sd_driver->init(sh, osd);
+        if (sh->sd_driver->init(sh, osd) < 0)
+            return;
         sh->initialized = true;
         sh->active = true;
     }
diff --git a/sub/sd.h b/sub/sd.h
index d5aea5c1a6..c36110277b 100644
--- a/sub/sd.h
+++ b/sub/sd.h
@@ -5,7 +5,7 @@ struct osd_state;
 struct sh_sub;
 
 struct sd_functions {
-    void (*init)(struct sh_sub *sh, struct osd_state *osd);
+    int  (*init)(struct sh_sub *sh, struct osd_state *osd);
     void (*decode)(struct sh_sub *sh, struct osd_state *osd,
                    void *data, int data_len, double pts, double duration);
     void (*reset)(struct sh_sub *sh, struct osd_state *osd);
diff --git a/sub/sd_ass.c b/sub/sd_ass.c
index 69587646e4..b8924fbb3f 100644
--- a/sub/sd_ass.c
+++ b/sub/sd_ass.c
@@ -43,7 +43,7 @@ static void free_last_event(ASS_Track *track)
     track->n_events--;
 }
 
-static void init(struct sh_sub *sh, struct osd_state *osd)
+static int init(struct sh_sub *sh, struct osd_state *osd)
 {
     struct sd_ass_priv *ctx;
 
@@ -65,6 +65,7 @@ static void init(struct sh_sub *sh, struct osd_state *osd)
     osd->ass_track = ctx->ass_track;
     osd->vsfilter_aspect = sh->type == 'a';
     osd->ass_track_changed = true;
+    return 0;
 }
 
 static void decode(struct sh_sub *sh, struct osd_state *osd, void *data,
diff --git a/sub/av_sub.c b/sub/sd_lavc.c
index dbf349133f..a4eafff429 100644
--- a/sub/av_sub.c
+++ b/sub/sd_lavc.c
@@ -1,41 +1,30 @@
 /*
- * This file is part of MPlayer.
+ * This file is part of mplayer2.
  *
- * MPlayer is free software; you can redistribute it and/or modify
+ * mplayer2 is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
  * the Free Software Foundation; either version 2 of the License, or
  * (at your option) any later version.
  *
- * MPlayer is distributed in the hope that it will be useful,
+ * mplayer2 is distributed in the hope that it will be useful,
  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  * GNU General Public License for more details.
  *
  * You should have received a copy of the GNU General Public License along
- * with MPlayer; if not, write to the Free Software Foundation, Inc.,
- * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+ * with mplayer2.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <stdlib.h>
+
 #include <libavcodec/avcodec.h>
 
 #include "mp_msg.h"
 #include "libmpdemux/stheader.h"
-#include "sub.h"
+#include "sd.h"
 #include "spudec.h"
-#include "av_sub.h"
-
-bool is_av_sub(int type)
-{
-    return type == 'b' || type == 'p' || type == 'x';
-}
-
-void reset_avsub(struct sh_sub *sh)
-{
-    if (sh->context) {
-        avcodec_close(sh->context);
-        av_freep(&sh->context);
-    }
-}
+// Current code still pushes subs directly to global spudec
+#include "sub.h"
 
 static void avsub_to_spudec(AVSubtitleRect **rects, int num_rects,
                             double pts, double endpts)
@@ -78,20 +67,9 @@ static void avsub_to_spudec(AVSubtitleRect **rects, int num_rects,
     spudec_packet_send(vo_spudec, packet, pts, endpts);
 }
 
-/**
- * Decode a subtitle packet via libavcodec.
- * \return < 0 on error, > 0 if further processing is needed
- */
-int decode_avsub(struct sh_sub *sh, uint8_t *data, int size,
-                 double pts, double duration)
+static int init(struct sh_sub *sh, struct osd_state *osd)
 {
-    AVCodecContext *ctx = sh->context;
     enum CodecID cid = CODEC_ID_NONE;
-    int res;
-    int got_sub;
-    AVSubtitle sub;
-    AVPacket pkt;
-
     switch (sh->type) {
     case 'b':
         cid = CODEC_ID_DVB_SUBTITLE; break;
@@ -100,33 +78,42 @@ int decode_avsub(struct sh_sub *sh, uint8_t *data, int size,
     case 'x':
         cid = CODEC_ID_XSUB; break;
     }
+    AVCodecContext *ctx = NULL;
+    AVCodec *sub_codec = avcodec_find_decoder(cid);
+    if (!sub_codec)
+        goto error;
+    ctx = avcodec_alloc_context3(sub_codec);
+    if (!ctx)
+        goto error;
+    if (avcodec_open2(ctx, sub_codec, NULL) < 0)
+        goto error;
+    sh->context = ctx;
+    return 0;
+
+ error:
+    mp_msg(MSGT_SUBREADER, MSGL_ERR,
+           "Could not open libavcodec subtitle decoder\n");
+    av_free(ctx);
+    return -1;
+}
+
+static void decode(struct sh_sub *sh, struct osd_state *osd, void *data,
+                   int data_len, double pts, double duration)
+{
+    AVCodecContext *ctx = sh->context;
+    AVSubtitle sub;
+    AVPacket pkt;
 
     av_init_packet(&pkt);
     pkt.data = data;
-    pkt.size = size;
+    pkt.size = data_len;
     pkt.pts = pts * 1000;
     if (duration >= 0)
         pkt.convergence_duration = duration * 1000;
-    if (!ctx) {
-        AVCodec *sub_codec;
-        sub_codec = avcodec_find_decoder(cid);
-        if (!sub_codec)
-            goto error;
-        ctx = avcodec_alloc_context3(sub_codec);
-        if (!ctx)
-            goto error;
-        if (avcodec_open2(ctx, sub_codec, NULL) < 0) {
-         error:
-            mp_msg(MSGT_SUBREADER, MSGL_FATAL,
-                   "Could not open subtitle decoder\n");
-            av_freep(&ctx);
-            return -1;
-        }
-        sh->context = ctx;
-    }
-    res = avcodec_decode_subtitle2(ctx, &sub, &got_sub, &pkt);
-    if (res < 0)
-        return res;
+    int got_sub;
+    int res = avcodec_decode_subtitle2(ctx, &sub, &got_sub, &pkt);
+    if (res < 0 || !got_sub)
+        return;
     if (pts != MP_NOPTS_VALUE) {
         if (sub.end_display_time > sub.start_display_time)
             duration = (sub.end_display_time - sub.start_display_time) / 1000.0;
@@ -135,9 +122,9 @@ int decode_avsub(struct sh_sub *sh, uint8_t *data, int size,
     double endpts = MP_NOPTS_VALUE;
     if (pts != MP_NOPTS_VALUE && duration >= 0)
         endpts = pts + duration;
-    if (got_sub && vo_spudec && sub.num_rects == 0)
+    if (vo_spudec && sub.num_rects == 0)
         spudec_set_paletted(vo_spudec, NULL, 0, NULL, 0, 0, 0, 0, pts, endpts);
-    if (got_sub && sub.num_rects > 0) {
+    if (sub.num_rects > 0) {
         switch (sub.rects[0]->type) {
         case SUBTITLE_BITMAP:
             if (!vo_spudec)
@@ -146,13 +133,30 @@ int decode_avsub(struct sh_sub *sh, uint8_t *data, int size,
             vo_osd_changed(OSDTYPE_SPU);
             break;
         default:
-            mp_msg(MSGT_SUBREADER, MSGL_ERR, "sd_avsub: unsupported subtitle "
+            mp_msg(MSGT_SUBREADER, MSGL_ERR, "sd_lavc: unsupported subtitle "
                    "type from libavcodec\n");
-            res = -1;
             break;
         }
     }
-    if (got_sub)
-        avsubtitle_free(&sub);
-    return res;
+    avsubtitle_free(&sub);
+}
+
+static void reset(struct sh_sub *sh, struct osd_state *osd)
+{
+    // lavc might not do this right for all codecs; may need close+reopen
+    avcodec_flush_buffers(sh->context);
 }
+
+static void uninit(struct sh_sub *sh)
+{
+    avcodec_close(sh->context);
+    av_free(sh->context);
+}
+
+const struct sd_functions sd_lavc = {
+    .init = init,
+    .decode = decode,
+    .reset = reset,
+    .switch_off = reset,
+    .uninit = uninit,
+};