demux: add sanity checks to packet allocation functions

Change new_demux_packet() and resize_demux_packet() length parameter
type from int to size_t and add a check to abort() if the size is over
1 GB. This should make integer overflow problems leading to memory
corruption in demuxers less likely; and aborting should be no worse
than insane memory consumption. Also make the functions abort() if the
actual allocation fails instead of trying to continue with a
zero-sized buffer.

1 files changed, 24 insertions, 9 deletions

diff --git a/libmpdemux/demuxer.c b/libmpdemux/demuxer.c
index 85f8e875ee..056822e1ab 100644
--- a/libmpdemux/demuxer.c
+++ b/libmpdemux/demuxer.c
@@ -177,8 +177,13 @@ const demuxer_desc_t *const demuxer_list[] = {
     NULL
 };
 
-struct demux_packet *new_demux_packet(int len)
+struct demux_packet *new_demux_packet(size_t len)
 {
+    if (len > 1000000000) {
+        mp_msg(MSGT_DEMUXER, MSGL_FATAL, "Attempt to allocate demux packet "
+               "over 1 GB!\n");
+        abort();
+    }
     struct demux_packet *dp = malloc(sizeof(struct demux_packet));
     dp->len = len;
     dp->next = NULL;
@@ -190,26 +195,36 @@ struct demux_packet *new_demux_packet(int len)
     dp->refcount = 1;
     dp->master = NULL;
     dp->buffer = NULL;
-    if (len > 0 && (dp->buffer = malloc(len + MP_INPUT_BUFFER_PADDING_SIZE)))
+    if (len > 0) {
+        dp->buffer = malloc(len + MP_INPUT_BUFFER_PADDING_SIZE);
+        if (!dp->buffer) {
+            mp_msg(MSGT_DEMUXER, MSGL_FATAL, "Memory allocation failure!\n");
+            abort();
+        }
         memset(dp->buffer + len, 0, 8);
-    else
-        dp->len = 0;
+    }
     return dp;
 }
 
-void resize_demux_packet(struct demux_packet *dp, int len)
+void resize_demux_packet(struct demux_packet *dp, size_t len)
 {
+    if (len > 1000000000) {
+        mp_msg(MSGT_DEMUXER, MSGL_FATAL, "Attempt to realloc demux packet "
+               "over 1 GB!\n");
+        abort();
+    }
     if (len > 0) {
         dp->buffer = realloc(dp->buffer, len + 8);
+        if (!dp->buffer) {
+            mp_msg(MSGT_DEMUXER, MSGL_FATAL, "Memory allocation failure!\n");
+            abort();
+        }
+        memset(dp->buffer + len, 0, 8);
     } else {
         free(dp->buffer);
         dp->buffer = NULL;
     }
     dp->len = len;
-    if (dp->buffer)
-        memset(dp->buffer + len, 0, 8);
-    else
-        dp->len = 0;
 }
 
 struct demux_packet *clone_demux_packet(struct demux_packet *pack)