array.c: fix array size overflow

* array.c (ary_ensure_room_for_push): check if array size will
  exceed maxmum size to get rid of buffer overflow.
  [ruby-dev:49043] [Bug #11235]
* array.c (ary_ensure_room_for_unshift, rb_ary_splice): ditto.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@50827 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 12 insertions, 1 deletions

diff --git a/array.c b/array.c
index 32f9f9ebe4..ea233a7ec2 100644
--- a/array.c
+++ b/array.c
@@ -353,9 +353,13 @@ rb_ary_modify(VALUE ary)
 static VALUE
 ary_ensure_room_for_push(VALUE ary, long add_len)
 {
-    long new_len = RARRAY_LEN(ary) + add_len;
+    long old_len = RARRAY_LEN(ary);
+    long new_len = old_len + add_len;
     long capa;
 
+    if (old_len > ARY_MAX_SIZE - add_len) {
+	rb_raise(rb_eIndexError, "index %ld too big", new_len);
+    }
     if (ARY_SHARED_P(ary)) {
 	if (new_len > RARRAY_EMBED_LEN_MAX) {
 	    VALUE shared = ARY_SHARED(ary);
@@ -1095,6 +1099,10 @@ ary_ensure_room_for_unshift(VALUE ary, int argc)
     long capa;
     const VALUE *head, *sharedp;
 
+    if (len > ARY_MAX_SIZE - argc) {
+	rb_raise(rb_eIndexError, "index %ld too big", new_len);
+    }
+
     if (ARY_SHARED_P(ary)) {
 	VALUE shared = ARY_SHARED(ary);
 	capa = RARRAY_LEN(shared);
@@ -1592,6 +1600,9 @@ rb_ary_splice(VALUE ary, long beg, long len, VALUE rpl)
     else {
 	long alen;
 
+	if (olen - len > ARY_MAX_SIZE - rlen) {
+	    rb_raise(rb_eIndexError, "index %ld too big", olen + rlen - len);
+	}
 	rb_ary_modify(ary);
 	alen = olen + rlen - len;
 	if (alen >= ARY_CAPA(ary)) {