From f37f4de8496556afaa024e39e2efb433eb1680d4 Mon Sep 17 00:00:00 2001 From: wm4 Date: Wed, 6 Nov 2019 21:36:02 +0100 Subject: stream: turn into a ring buffer, make size configurable In some corner cases (see #6802), it can be beneficial to use a larger stream buffer size. Use this as argument to rewrite everything for no reason. Turn stream.c itself into a ring buffer, with configurable size. The latter would have been easily achievable with minimal changes, and the ring buffer is the hard part. There is no reason to have a ring buffer at all, except possibly if ffmpeg don't fix their awful mp4 demuxer, and some subtle issues with demux_mkv.c wanting to seek back by small offsets (the latter was handled with small stream_peek() calls, which are unneeded now). In addition, this turns small forward seeks into reads (where data is simply skipped). Before this commit, only stream_skip() did this (which also mean that stream_skip() simply calls stream_seek() now). Replace all stream_peek() calls with something else (usually stream_read_peek()). The function was a problem, because it returned a pointer to the internal buffer, which is now a ring buffer with wrapping. The new function just copies the data into a buffer, and in some cases requires callers to dynamically allocate memory. (The most common case, demux_lavf.c, required a separate buffer allocation anyway due to FFmpeg "idiosyncrasies".) This is the bulk of the demuxer_* changes. I'm not happy with this. There still isn't a good reason why there should be a ring buffer, that is complex, and most of the time just wastes half of the available memory. Maybe another rewrite soon. It also contains bugs; you're an alpha tester now. --- stream/stream.h | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) (limited to 'stream/stream.h') diff --git a/stream/stream.h b/stream/stream.h index 66f151962a..7437e1c86b 100644 --- a/stream/stream.h +++ b/stream/stream.h @@ -28,7 +28,11 @@ #include "misc/bstr.h" +// Minimum guaranteed buffer and seek-back size. For any reads <= of this size, +// it's guaranteed that you can seek back by <= of this size again. #define STREAM_BUFFER_SIZE 2048 +// (Half of this is typically reserved for seeking back.) +#define STREAM_FIXED_BUFFER_SIZE (STREAM_BUFFER_SIZE * 2) // stream->mode #define STREAM_READ 0 @@ -119,7 +123,6 @@ typedef struct stream { void (*close)(struct stream *s); int read_chunk; // maximum amount of data to read at once to limit latency - unsigned int buf_pos, buf_len; int64_t pos; int eof; int mode; //STREAM_READ or STREAM_WRITE @@ -145,20 +148,46 @@ typedef struct stream { // added to this. The user can reset this as needed. uint64_t total_unbuffered_read_bytes; + // Buffer size requested by user; s->buffer may have a different size + int requested_buffer_size; + + // This is a ring buffer. It is reset only on seeks (or when buffers are + // dropped). Otherwise old contents always stay valid. + // The valid buffer is from buf_start to buf_end; buf_end can be larger + // then the buffer size (requires wrap around). buf_cur is a value in the + // range [buf_start, buf_end]. + // When reading more data from the stream, buf_start is advanced as old + // data is overwritten with new data. + // Example: + // 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 + // +===========================+---------------------------+ + // + 05 06 07 08 | 01 02 03 04 + 05 06 07 08 | 01 02 03 04 + + // +===========================+---------------------------+ + // ^ buf_start (4) | | + // | ^ buf_end (12 % 8 => 4) + // ^ buf_cur (9 % 8 => 1) + // Here, the entire 8 byte buffer is filled, i.e. buf_end - buf_start = 8. + // buffer_mask == 7, so (x & buffer_mask) == (x % buffer_size) + unsigned int buf_start; // index of oldest byte in buffer (is <= buffer_mask) + unsigned int buf_cur; // current read pos (can be > buffer_mask) + unsigned int buf_end; // end position (can be > buffer_mask) + + unsigned int buffer_mask; // buffer_size-1, where buffer_size == 2**n uint8_t *buffer; - int buffer_alloc; - uint8_t buffer_inline[STREAM_BUFFER_SIZE]; + uint8_t buffer_inline[STREAM_FIXED_BUFFER_SIZE]; } stream_t; -int stream_fill_buffer(stream_t *s); +// Non-inline version with stream_read_char(). +int stream_read_char_fallback(stream_t *s); int stream_write_buffer(stream_t *s, unsigned char *buf, int len); inline static int stream_read_char(stream_t *s) { - return (s->buf_pos < s->buf_len) ? s->buffer[s->buf_pos++] : - (stream_fill_buffer(s) ? s->buffer[s->buf_pos++] : -256); + return s->buf_cur < s->buf_end + ? s->buffer[(s->buf_cur++) & s->buffer_mask] + : stream_read_char_fallback(s); } int stream_skip_bom(struct stream *s); @@ -170,15 +199,14 @@ inline static int stream_eof(stream_t *s) inline static int64_t stream_tell(stream_t *s) { - return s->pos + s->buf_pos - s->buf_len; + return s->pos + s->buf_cur - s->buf_end; } bool stream_skip(stream_t *s, int64_t len); bool stream_seek(stream_t *s, int64_t pos); int stream_read(stream_t *s, char *mem, int total); int stream_read_partial(stream_t *s, char *buf, int buf_size); -struct bstr stream_peek(stream_t *s, int len); -struct bstr stream_peek_buffer(stream_t *s); +int stream_read_peek(stream_t *s, void* buf, int buf_size); void stream_drop_buffers(stream_t *s); int64_t stream_get_size(stream_t *s); -- cgit v1.2.3