ruby.h: check integer overflow

* include/ruby/ruby.h (ALLOCV_N): check integer overflow, as well
  as ruby_xmalloc2.  pointed out by Paul <pawlkt AT gmail.com>.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@51533 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 19 insertions, 2 deletions

diff --git a/include/ruby/ruby.h b/include/ruby/ruby.h
index 01b3c2efc6..cfc1005919 100644
--- a/include/ruby/ruby.h
+++ b/include/ruby/ruby.h
@@ -1431,14 +1431,31 @@ rb_num2char_inline(VALUE x)
 
 void *rb_alloc_tmp_buffer(volatile VALUE *store, long len) RUBY_ATTR_ALLOC_SIZE((2));
 void rb_free_tmp_buffer(volatile VALUE *store);
+NORETURN(void ruby_malloc_size_overflow(size_t, size_t));
+static inline size_t
+ruby_xmalloc2_size(const size_t count, const size_t elsize)
+{
+    if (count > SIZE_MAX / elsize) {
+	ruby_malloc_size_overflow(count, elsize);
+    }
+    return count * elsize;
+}
 /* allocates _n_ bytes temporary buffer and stores VALUE including it
  * in _v_.  _n_ may be evaluated twice. */
 #ifdef C_ALLOCA
 # define ALLOCV(v, n) rb_alloc_tmp_buffer(&(v), (n))
+# define ALLOCV_N(type, v, n) \
+    ((type*)ALLOCV((v), ruby_xmalloc2_size((n), sizeof(type))))
 #else
-# define ALLOCV(v, n) ((n) < 1024 ? (RB_GC_GUARD(v) = 0, alloca(n)) : rb_alloc_tmp_buffer(&(v), (n)))
+# define ALLOCV_LIMIT 1024
+# define ALLOCV(v, n) ((n) < ALLOCV_LIMIT ? \
+		       (RB_GC_GUARD(v) = 0, alloca(n)) : \
+		       rb_alloc_tmp_buffer(&(v), (n)))
+# define ALLOCV_N(type, v, n) \
+    ((type*)(ruby_xmalloc2_size((n), sizeof(type)) < ALLOCV_LIMIT ? \
+	     (RB_GC_GUARD(v) = 0, alloca((n) * sizeof(type))) : \
+	     rb_alloc_tmp_buffer(&(v), (n) * sizeof(type))))
 #endif
-#define ALLOCV_N(type, v, n) ((type*)ALLOCV((v), sizeof(type)*(n)))
 #define ALLOCV_END(v) rb_free_tmp_buffer(&(v))
 
 #define MEMZERO(p,type,n) memset((p), 0, sizeof(type)*(n))