demux: add sanity checks to packet allocation functions

Change new_demux_packet() and resize_demux_packet() length parameter
type from int to size_t and add a check to abort() if the size is over
1 GB. This should make integer overflow problems leading to memory
corruption in demuxers less likely; and aborting should be no worse
than insane memory consumption. Also make the functions abort() if the
actual allocation fails instead of trying to continue with a
zero-sized buffer.

2 files changed, 26 insertions, 11 deletions

diff --git a/libmpdemux/demuxer.c b/libmpdemux/demuxer.c
index 85f8e875ee..056822e1ab 100644
--- a/libmpdemux/demuxer.c
+++ b/libmpdemux/demuxer.c
@@ -177,8 +177,13 @@ const demuxer_desc_t *const demuxer_list[] = {
     NULL
 };
 
-struct demux_packet *new_demux_packet(int len)
+struct demux_packet *new_demux_packet(size_t len)
 {
+    if (len > 1000000000) {
+        mp_msg(MSGT_DEMUXER, MSGL_FATAL, "Attempt to allocate demux packet "
+               "over 1 GB!\n");
+        abort();
+    }
     struct demux_packet *dp = malloc(sizeof(struct demux_packet));
     dp->len = len;
     dp->next = NULL;
@@ -190,26 +195,36 @@ struct demux_packet *new_demux_packet(int len)
     dp->refcount = 1;
     dp->master = NULL;
     dp->buffer = NULL;
-    if (len > 0 && (dp->buffer = malloc(len + MP_INPUT_BUFFER_PADDING_SIZE)))
+    if (len > 0) {
+        dp->buffer = malloc(len + MP_INPUT_BUFFER_PADDING_SIZE);
+        if (!dp->buffer) {
+            mp_msg(MSGT_DEMUXER, MSGL_FATAL, "Memory allocation failure!\n");
+            abort();
+        }
         memset(dp->buffer + len, 0, 8);
-    else
-        dp->len = 0;
+    }
     return dp;
 }
 
-void resize_demux_packet(struct demux_packet *dp, int len)
+void resize_demux_packet(struct demux_packet *dp, size_t len)
 {
+    if (len > 1000000000) {
+        mp_msg(MSGT_DEMUXER, MSGL_FATAL, "Attempt to realloc demux packet "
+               "over 1 GB!\n");
+        abort();
+    }
     if (len > 0) {
         dp->buffer = realloc(dp->buffer, len + 8);
+        if (!dp->buffer) {
+            mp_msg(MSGT_DEMUXER, MSGL_FATAL, "Memory allocation failure!\n");
+            abort();
+        }
+        memset(dp->buffer + len, 0, 8);
     } else {
         free(dp->buffer);
         dp->buffer = NULL;
     }
     dp->len = len;
-    if (dp->buffer)
-        memset(dp->buffer + len, 0, 8);
-    else
-        dp->len = 0;
 }
 
 struct demux_packet *clone_demux_packet(struct demux_packet *pack)
diff --git a/libmpdemux/demuxer.h b/libmpdemux/demuxer.h
index 3a7a6157c2..742e42aa74 100644
--- a/libmpdemux/demuxer.h
+++ b/libmpdemux/demuxer.h
@@ -286,8 +286,8 @@ typedef struct {
   int aid, vid, sid; //audio, video and subtitle id
 } demux_program_t;
 
-struct demux_packet *new_demux_packet(int len);
-void resize_demux_packet(struct demux_packet *dp, int len);
+struct demux_packet *new_demux_packet(size_t len);
+void resize_demux_packet(struct demux_packet *dp, size_t len);
 struct demux_packet *clone_demux_packet(struct demux_packet *pack);
 void free_demux_packet(struct demux_packet *dp);