malloc/nano-malloc: correctly check for out-of-bounds allocation reqs

The overflow check in mEMALIGn erroneously checks for INT_MAX,
albeit the input parameter is size_t.  Fix this to check for
__SIZE_MAX__ instead.  Also, it misses to check the req against
adding the alignment before calling mALLOc.

While at it, add out-of-bounds checks to pvALLOc, nano_memalign,
nano_valloc, and Cygwin's (unused) dlpvalloc.

Signed-off-by: Corinna Vinschen <corinna@vinschen.de>

newlib/libc/stdlib/mallocr.c

@@ -3055,7 +3055,7 @@ Void_t* mEMALIGn(RARG alignment, bytes) RDECL size_t alignment; size_t bytes;
   nb = request2size(bytes);
 
   /* Check for overflow. */
-  if (nb > INT_MAX || nb < bytes)
+  if (nb > __SIZE_MAX__ - (alignment + MINSIZE) || nb < bytes)
   {
     RERRNO = ENOMEM;
     return 0;
@@ -3172,6 +3172,11 @@ Void_t* pvALLOc(RARG bytes) RDECL size_t bytes;
 #endif
 {
   size_t pagesize = malloc_getpagesize;
+  if (bytes > __SIZE_MAX__ - pagesize)
+  {
+    RERRNO = ENOMEM;
+    return 0;
+  }
   return mEMALIGn (RCALL pagesize, (bytes + pagesize - 1) & ~(pagesize - 1));
 }
 

newlib/libc/stdlib/nano-mallocr.c

@@ -580,8 +580,22 @@ void * nano_memalign(RARG size_t align, size_t s)
     if ((align & (align-1)) != 0) return NULL;
 
     align = MAX(align, MALLOC_ALIGN);
+
+    /* Make sure ma_size does not overflow */
+    if (s > __SIZE_MAX__ - CHUNK_ALIGN)
+    {
+	RERRNO = ENOMEM;
+	return NULL;
+    }
     ma_size = ALIGN_SIZE(MAX(s, MALLOC_MINSIZE), CHUNK_ALIGN);
-    size_with_padding = ma_size + align - MALLOC_ALIGN;
+
+    /* Make sure size_with_padding does not overflow */
+    if (ma_size > __SIZE_MAX__ - (align - MALLOC_ALIGN))
+    {
+	RERRNO = ENOMEM;
+	return NULL;
+    }
+    size_with_padding = ma_size + (align - MALLOC_ALIGN);
 
     allocated = nano_malloc(RCALL size_with_padding);
     if (allocated == NULL) return NULL;
@@ -644,6 +658,12 @@ void * nano_valloc(RARG size_t s)
 #ifdef DEFINE_PVALLOC
 void * nano_pvalloc(RARG size_t s)
 {
+    /* Make sure size given to nano_valloc does not overflow */
+    if (s > __SIZE_MAX__ - MALLOC_PAGE_ALIGN)
+    {
+	RERRNO = ENOMEM;
+	return NULL;
+    }
     return nano_valloc(RCALL ALIGN_SIZE(s, MALLOC_PAGE_ALIGN));
 }
 #endif /* DEFINE_PVALLOC */

winsup/cygwin/malloc.cc

@@ -5298,6 +5298,10 @@ void* dlpvalloc(size_t bytes) {
   size_t pagesz;
   ensure_initialization();
   pagesz = mparams.page_size;
+  if (bytes > MAX_REQUEST) {
+    MALLOC_FAILURE_ACTION;
+    return NULL;
+  }
   return dlmemalign(pagesz, (bytes + pagesz - SIZE_T_ONE) & ~(pagesz - SIZE_T_ONE));
 }