From 7e0ae1698d4db0baec858a46de8d1ae875360cf5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=8D=9C=E9=83=A8=E6=98=8C=E5=B9=B3?= Date: Mon, 7 Oct 2019 16:56:08 +0900 Subject: avoid overflow in integer multiplication This changeset basically replaces `ruby_xmalloc(x * y)` into `ruby_xmalloc2(x, y)`. Some convenient functions are also provided for instance `rb_xmalloc_mul_add(x, y, z)` which allocates x * y + z byes. --- transient_heap.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'transient_heap.c') diff --git a/transient_heap.c b/transient_heap.c index dcf65fc7ab..6cdd34284f 100644 --- a/transient_heap.c +++ b/transient_heap.c @@ -440,6 +440,9 @@ Init_TransientHeap(void) theap->promoted_objects_index = 0; /* should not use ALLOC_N to be free from GC */ theap->promoted_objects = malloc(sizeof(VALUE) * theap->promoted_objects_size); + STATIC_ASSERT( + integer_overflow, + sizeof(VALUE) <= SIZE_MAX / TRANSIENT_HEAP_PROMOTED_DEFAULT_SIZE); if (theap->promoted_objects == NULL) rb_bug("Init_TransientHeap: malloc failed."); } @@ -618,7 +621,13 @@ transient_heap_promote_add(struct transient_heap* theap, VALUE obj) if (theap->promoted_objects_size <= theap->promoted_objects_index) { theap->promoted_objects_size *= 2; if (TRANSIENT_HEAP_DEBUG >= 1) fprintf(stderr, "rb_transient_heap_promote: expand table to %d\n", theap->promoted_objects_size); - theap->promoted_objects = realloc(theap->promoted_objects, theap->promoted_objects_size * sizeof(VALUE)); + if (UNLIKELY((size_t)theap->promoted_objects_size > SIZE_MAX / sizeof(VALUE))) { + /* realloc failure due to integer overflow */ + theap->promoted_objects = NULL; + } + else { + theap->promoted_objects = realloc(theap->promoted_objects, theap->promoted_objects_size * sizeof(VALUE)); + } if (theap->promoted_objects == NULL) rb_bug("rb_transient_heap_promote: realloc failed"); } theap->promoted_objects[theap->promoted_objects_index++] = obj; -- cgit v1.2.3