From 0af5335383887cda7650d4b33bc42759c1a5891f Mon Sep 17 00:00:00 2001 From: wm4 Date: Mon, 15 Feb 2016 21:04:07 +0100 Subject: Rewrite ordered chapters and timeline stuff This uses a different method to piece segments together. The old approach basically changes to a new file (with a new start offset) any time a segment ends. This meant waiting for audio/video end on segment end, and then changing to the new segment all at once. It had a very weird impact on the playback core, and some things (like truly gapless segment transitions, or frame backstepping) just didn't work. The new approach adds the demux_timeline pseudo-demuxer, which presents an uniform packet stream from the many segments. This is pretty similar to how ordered chapters are implemented everywhere else. It also reminds of the FFmpeg concat pseudo-demuxer. The "pure" version of this approach doesn't work though. Segments can actually have different codec configurations (different extradata), and subtitles are most likely broken too. (Subtitles have multiple corner cases which break the pure stream-concatenation approach completely.) To counter this, we do two things: - Reinit the decoder with each segment. We go as far as allowing concatenating files with completely different codecs for the sake of EDL (which also uses the timeline infrastructure). A "lighter" approach would try to make use of decoder mechanism to update e.g. the extradata, but that seems fragile. - Clip decoded data to segment boundaries. This is equivalent to normal playback core mechanisms like hr-seek, but now the playback core doesn't need to care about these things. These two mechanisms are equivalent to what happened in the old implementation, except they don't happen in the playback core anymore. In other words, the playback core is completely relieved from timeline implementation details. (Which honestly is exactly what I'm trying to do here. I don't think ordered chapter behavior deserves improvement, even if it's bad - but I want to get it out from the playback core.) There is code duplication between audio and video decoder common code. This is awful and could be shareable - but this will happen later. Note that the audio path has some code to clip audio frames for the purpose of codec preroll/gapless handling, but it's not shared as sharing it would cause more pain than it would help. --- demux/demux.c | 17 ++- demux/demux.h | 1 + demux/demux_timeline.c | 392 +++++++++++++++++++++++++++++++++++++++++++++++++ demux/packet.c | 5 + demux/packet.h | 5 + 5 files changed, 418 insertions(+), 2 deletions(-) create mode 100644 demux/demux_timeline.c (limited to 'demux') diff --git a/demux/demux.c b/demux/demux.c index 41b0f8f910..61becd7f04 100644 --- a/demux/demux.c +++ b/demux/demux.c @@ -36,6 +36,7 @@ #include "stream/stream.h" #include "demux.h" +#include "timeline.h" #include "stheader.h" #include "cue.h" @@ -52,6 +53,7 @@ extern const demuxer_desc_t demuxer_desc_playlist; extern const demuxer_desc_t demuxer_desc_disc; extern const demuxer_desc_t demuxer_desc_rar; extern const demuxer_desc_t demuxer_desc_libarchive; +extern const demuxer_desc_t demuxer_desc_timeline; /* Please do not add any new demuxers here. If you want to implement a new * demuxer, add it to libavformat, except for wrappers around external @@ -782,7 +784,7 @@ bool demux_has_packet(struct sh_stream *sh) return has_packet; } -// Read and return any packet we find. +// Read and return any packet we find. NULL means EOF. struct demux_packet *demux_read_any_packet(struct demuxer *demuxer) { struct demux_internal *in = demuxer->in; @@ -1082,7 +1084,8 @@ static struct demuxer *open_given_type(struct mpv_global *global, mp_dbg(log, "Trying demuxer: %s (force-level: %s)\n", desc->name, d_level(check)); - if (stream->seekable) // not for DVD/BD/DVB in particular + // not for DVD/BD/DVB in particular + if (stream->seekable && (!params || !params->timeline)) stream_seek(stream, 0); // Peek this much data to avoid that stream_read() run by some demuxers @@ -1110,6 +1113,16 @@ static struct demuxer *open_given_type(struct mpv_global *global, demux_changed(in->d_thread, DEMUX_EVENT_ALL); demux_update(demuxer); stream_control(demuxer->stream, STREAM_CTRL_SET_READAHEAD, &(int){false}); + struct timeline *tl = timeline_load(global, log, demuxer); + if (tl) { + struct demuxer_params params2 = {0}; + params2.timeline = tl; + struct demuxer *sub = open_given_type(global, log, + &demuxer_desc_timeline, stream, + ¶ms2, DEMUX_CHECK_FORCE); + if (sub) + return sub; + } return demuxer; } diff --git a/demux/demux.h b/demux/demux.h index 72ed15888b..05e645f728 100644 --- a/demux/demux.h +++ b/demux/demux.h @@ -160,6 +160,7 @@ struct demuxer_params { struct matroska_segment_uid *matroska_wanted_uids; int matroska_wanted_segment; bool *matroska_was_valid; + struct timeline *timeline; // -- demux_open_url() only int stream_flags; bool allow_capture; diff --git a/demux/demux_timeline.c b/demux/demux_timeline.c new file mode 100644 index 0000000000..9dbbff2158 --- /dev/null +++ b/demux/demux_timeline.c @@ -0,0 +1,392 @@ +/* + * This file is part of mpv. + * + * mpv is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * mpv is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with mpv. If not, see . + */ + +#include +#include + +#include "common/common.h" +#include "common/msg.h" + +#include "demux.h" +#include "timeline.h" +#include "stheader.h" + +struct segment { + int index; + double start, end; + double d_start; + struct demuxer *d; + // stream_map[sh_stream.index] = index into priv.streams, where sh_stream + // is a stream from the source d. It's used to map the streams of the + // source onto the set of streams of the virtual timeline. + // Uses -1 for streams that do not appear in the virtual timeline. + int *stream_map; + int num_stream_map; +}; + +// Information for each stream on the virtual timeline. (Mirrors streams +// exposed by demux_timeline.) +struct virtual_stream { + struct sh_stream *sh; // stream exported by demux_timeline + bool selected; // ==demux_stream_is_selected(sh) + bool new_segment; // whether a new segment needs to be signaled + int eos_packets; // deal with b-frame delay +}; + +struct priv { + struct timeline *tl; + + double duration; + + struct segment **segments; + int num_segments; + struct segment *current; + + // As the demuxer user sees it. + struct virtual_stream *streams; + int num_streams; + + // Total number of packets received past end of segment. Used + // to be clever about determining when to switch segments. + int eos_packets; + + double seek_pts; +}; + +static void switch_segment(struct demuxer *demuxer, struct segment *new, + double start_pts, int flags) +{ + struct priv *p = demuxer->priv; + + if (p->current == new) + return; + + if (!(flags & (SEEK_FORWARD | SEEK_BACKWARD))) + flags |= SEEK_BACKWARD; + + MP_VERBOSE(demuxer, "switch to segment %d\n", new->index); + + p->current = new; + demux_set_ts_offset(new->d, new->start - new->d_start); + demux_seek(new->d, start_pts, flags | SEEK_ABSOLUTE); + + for (int n = 0; n < p->num_streams; n++) { + struct virtual_stream *vs = &p->streams[n]; + vs->new_segment = true; + vs->eos_packets = 0; + } + + p->eos_packets = 0; +} + +static void d_seek(struct demuxer *demuxer, double rel_seek_secs, int flags) +{ + struct priv *p = demuxer->priv; + + double pts = p->seek_pts; + if (flags & SEEK_ABSOLUTE) + pts = 0.0f; + + if (flags & SEEK_FACTOR) { + pts += p->duration * rel_seek_secs; + } else { + pts += rel_seek_secs; + } + + flags &= SEEK_FORWARD | SEEK_BACKWARD | SEEK_HR; + + struct segment *new = p->segments[p->num_segments - 1]; + for (int n = 0; n < p->num_segments; n++) { + if (pts < p->segments[n]->end) { + new = p->segments[n]; + break; + } + } + + p->current = NULL; // force seek + switch_segment(demuxer, new, pts, flags); + + p->seek_pts = pts; +} + +static int d_fill_buffer(struct demuxer *demuxer) +{ + struct priv *p = demuxer->priv; + + if (!p->current) + switch_segment(demuxer, p->segments[0], 0, 0); + + struct segment *seg = p->current; + + struct demux_packet *pkt = demux_read_any_packet(seg->d); + if (!pkt || pkt->pts >= seg->end) + p->eos_packets += 1; + + // Test for EOF. Do this here to properly run into EOF even if other + // streams are disabled etc. If it somehow doesn't manage to reach the end + // after demuxing a high (bit arbitrary) number of packets, assume one of + // the streams went EOF early. + bool eos_reached = p->eos_packets > 0; + if (eos_reached && p->eos_packets < 100) { + for (int n = 0; n < p->num_streams; n++) { + struct virtual_stream *vs = &p->streams[n]; + if (vs->selected) { + int max_packets = 0; + if (vs->sh->type == STREAM_AUDIO) + max_packets = 1; + if (vs->sh->type == STREAM_VIDEO) + max_packets = 16; + eos_reached &= vs->eos_packets >= max_packets; + } + } + } + + if (eos_reached || !pkt) { + talloc_free(pkt); + + struct segment *next = NULL; + for (int n = 0; n < p->num_segments - 1; n++) { + if (p->segments[n] == seg) { + next = p->segments[n + 1]; + break; + } + } + if (!next) + return 0; + switch_segment(demuxer, next, next->start, 0); + return 1; // reader will retry + } + + if (pkt->stream < 0 || pkt->stream > seg->num_stream_map) + goto drop; + + if (!pkt->codec) + pkt->codec = demux_get_stream(seg->d, pkt->stream)->codec; + + if (pkt->start == MP_NOPTS_VALUE || pkt->start < seg->start) + pkt->start = seg->start; + if (pkt->end == MP_NOPTS_VALUE || pkt->end > seg->end) + pkt->end = seg->end; + + pkt->stream = seg->stream_map[pkt->stream]; + if (pkt->stream < 0) + goto drop; + + struct virtual_stream *vs = &p->streams[pkt->stream]; + + if (pkt->pts != MP_NOPTS_VALUE && pkt->pts >= seg->end) { + // Trust the keyframe flag. Might not always be a good idea, but will + // be sufficient at least with mkv. The problem is that this flag is + // not well-defined in libavformat and is container-dependent. + if (pkt->keyframe || vs->eos_packets == INT_MAX) { + vs->eos_packets = INT_MAX; + goto drop; + } else { + vs->eos_packets += 1; + } + } + + pkt->new_segment = vs->new_segment; + vs->new_segment = false; + + demux_add_packet(vs->sh, pkt); + return 1; + +drop: + talloc_free(pkt); + return 1; +} + +static void print_timeline(struct demuxer *demuxer) +{ + struct priv *p = demuxer->priv; + + MP_VERBOSE(demuxer, "Timeline segments:\n"); + for (int n = 0; n < p->num_segments; n++) { + struct segment *seg = p->segments[n]; + int src_num = -1; + for (int i = 0; i < p->tl->num_sources; i++) { + if (p->tl->sources[i] == seg->d) { + src_num = i; + break; + } + } + MP_VERBOSE(demuxer, " %2d: %12f [%12f] (", n, seg->start, seg->d_start); + for (int i = 0; i < seg->num_stream_map; i++) + MP_VERBOSE(demuxer, "%s%d", i ? " " : "", seg->stream_map[i]); + MP_VERBOSE(demuxer, ") %d:'%s'\n", src_num, seg->d->filename); + } + MP_VERBOSE(demuxer, "Total duration: %f\n", p->duration); +} + +static bool target_stream_used(struct segment *seg, int target_index) +{ + for (int n = 0; n < seg->num_stream_map; n++) { + if (seg->stream_map[n] == target_index) + return true; + } + return false; +} + +// Create mapping from segment streams to virtual timeline streams. +static void associate_streams(struct demuxer *demuxer, struct segment *seg) +{ + struct priv *p = demuxer->priv; + + int counts[STREAM_TYPE_COUNT] = {0}; + + int num_streams = demux_get_num_stream(seg->d); + for (int n = 0; n < num_streams; n++) { + struct sh_stream *sh = demux_get_stream(seg->d, n); + // Try associating by demuxer ID (supposedly useful for ordered chapters). + struct sh_stream *other = + demuxer_stream_by_demuxer_id(demuxer, sh->type, sh->demuxer_id); + if (!other || !target_stream_used(seg, other->index)) { + // Try to associate the first unused stream with matching media type. + for (int i = 0; i < p->num_streams; i++) { + struct sh_stream *cur = p->streams[i].sh; + if (cur->type == sh->type && !target_stream_used(seg, cur->index)) + { + other = cur; + break; + } + } + } + + MP_TARRAY_APPEND(seg, seg->stream_map, seg->num_stream_map, + other ? other->index : -1); + + counts[sh->type] += 1; + } +} + +static int d_open(struct demuxer *demuxer, enum demux_check check) +{ + struct priv *p = demuxer->priv = talloc_zero(demuxer, struct priv); + p->tl = demuxer->params ? demuxer->params->timeline : NULL; + if (!p->tl || p->tl->num_parts < 1) + return -1; + + p->duration = p->tl->parts[p->tl->num_parts].start; + + demuxer->chapters = p->tl->chapters; + demuxer->num_chapters = p->tl->num_chapters; + + struct demuxer *meta = p->tl->track_layout; + demuxer->metadata = meta->metadata; + demuxer->attachments = meta->attachments; + demuxer->num_attachments = meta->num_attachments; + + int num_streams = demux_get_num_stream(meta); + for (int n = 0; n < num_streams; n++) { + struct sh_stream *sh = demux_get_stream(meta, n); + struct sh_stream *new = demux_alloc_sh_stream(sh->type); + new->demuxer_id = sh->demuxer_id; + new->codec = sh->codec; + new->title = sh->title; + new->lang = sh->lang; + new->default_track = sh->default_track; + new->forced_track = sh->forced_track; + new->hls_bitrate = sh->hls_bitrate; + new->missing_timestamps = sh->missing_timestamps; + demux_add_sh_stream(demuxer, new); + struct virtual_stream vs = { + .sh = new, + }; + MP_TARRAY_APPEND(p, p->streams, p->num_streams, vs); + } + + for (int n = 0; n < p->tl->num_parts; n++) { + struct timeline_part *part = &p->tl->parts[n]; + struct timeline_part *next = &p->tl->parts[n + 1]; + + struct segment *seg = talloc_ptrtype(p, seg); + *seg = (struct segment){ + .d = part->source, + .d_start = part->source_start, + .start = part->start, + .end = next->start, + }; + + associate_streams(demuxer, seg); + + seg->index = n; + MP_TARRAY_APPEND(p, p->segments, p->num_segments, seg); + } + + print_timeline(demuxer); + + demuxer->seekable = true; + demuxer->partially_seekable = true; + + return 0; +} + +static void d_close(struct demuxer *demuxer) +{ + struct priv *p = demuxer->priv; + struct demuxer *master = p->tl->demuxer; + timeline_destroy(p->tl); + free_demuxer(master); +} + +static void reselect_streams(struct demuxer *demuxer) +{ + struct priv *p = demuxer->priv; + + for (int n = 0; n < p->num_streams; n++) { + struct virtual_stream *vs = &p->streams[n]; + vs->selected = demux_stream_is_selected(vs->sh); + } + + for (int n = 0; n < p->num_segments; n++) { + struct segment *seg = p->segments[n]; + for (int i = 0; i < seg->num_stream_map; i++) { + struct sh_stream *sh = demux_get_stream(seg->d, i); + bool selected = false; + if (seg->stream_map[i] >= 0) + selected = p->streams[seg->stream_map[i]].selected; + demuxer_select_track(seg->d, sh, selected); + } + } +} + +static int d_control(struct demuxer *demuxer, int cmd, void *arg) +{ + struct priv *p = demuxer->priv; + + switch (cmd) { + case DEMUXER_CTRL_GET_TIME_LENGTH: { + *(double *)arg = p->duration; + return DEMUXER_CTRL_OK; + } + case DEMUXER_CTRL_SWITCHED_TRACKS: + reselect_streams(demuxer); + return DEMUXER_CTRL_OK; + } + return DEMUXER_CTRL_NOTIMPL; +} + +const demuxer_desc_t demuxer_desc_timeline = { + .name = "timeline", + .desc = "timeline segment merging wrapper", + .fill_buffer = d_fill_buffer, + .open = d_open, + .close = d_close, + .seek = d_seek, + .control = d_control, +}; diff --git a/demux/packet.c b/demux/packet.c index 22b111b0ce..32fabc4f78 100644 --- a/demux/packet.c +++ b/demux/packet.c @@ -49,6 +49,8 @@ struct demux_packet *new_demux_packet_from_avpacket(struct AVPacket *avpkt) .dts = MP_NOPTS_VALUE, .duration = -1, .pos = -1, + .start = MP_NOPTS_VALUE, + .end = MP_NOPTS_VALUE, .stream = -1, .avpacket = talloc_zero(dp, AVPacket), }; @@ -106,6 +108,9 @@ void demux_packet_copy_attribs(struct demux_packet *dst, struct demux_packet *sr dst->dts = src->dts; dst->duration = src->duration; dst->pos = src->pos; + dst->start = src->start; + dst->end = src->end; + dst->new_segment = src->new_segment; dst->keyframe = src->keyframe; dst->stream = src->stream; } diff --git a/demux/packet.h b/demux/packet.h index 82dad6f90f..723dbc7e54 100644 --- a/demux/packet.h +++ b/demux/packet.h @@ -35,6 +35,11 @@ typedef struct demux_packet { int64_t pos; // position in source file byte stream int stream; // source stream index + // segmentation (ordered chapters, EDL) + struct mp_codec_params *codec; + double start, end; + bool new_segment; + // private struct demux_packet *next; struct AVPacket *avpacket; // keep the buffer allocation and sidedata -- cgit v1.2.3