array.c: fix array size overflow

* array.c (ary_ensure_room_for_push): check if array size will
  exceed maxmum size to get rid of buffer overflow.
  [ruby-dev:49043] [Bug #11235]
* array.c (ary_ensure_room_for_unshift, rb_ary_splice): ditto.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@50827 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 28 insertions, 0 deletions

diff --git a/test/ruby/test_array.rb b/test/ruby/test_array.rb
index bf14d3dfc9..dd0357d132 100644
--- a/test/ruby/test_array.rb
+++ b/test/ruby/test_array.rb
@@ -2553,6 +2553,34 @@ class TestArray < Test::Unit::TestCase
     end
   end
 
+  sizeof_long = [0].pack("l!").size
+  sizeof_voidp = [""].pack("p").size
+  if sizeof_long < sizeof_voidp
+    ARY_MAX = (1<<(8*sizeof_long-1)) / sizeof_voidp - 1
+    Bug11235 = '[ruby-dev:49043] [Bug #11235]'
+
+    def test_push_over_ary_max
+      assert_separately(['-', ARY_MAX.to_s, Bug11235], <<-"end;")
+        a = Array.new(ARGV[0].to_i)
+        assert_raise(IndexError, ARGV[1]) {0x1000.times {a.push(1)}}
+      end;
+    end
+
+    def test_unshift_over_ary_max
+      assert_separately(['-', ARY_MAX.to_s, Bug11235], <<-"end;")
+        a = Array.new(ARGV[0].to_i)
+        assert_raise(IndexError, ARGV[1]) {0x1000.times {a.unshift(1)}}
+      end;
+    end
+
+    def test_splice_over_ary_max
+      assert_separately(['-', ARY_MAX.to_s, Bug11235], <<-"end;")
+        a = Array.new(ARGV[0].to_i)
+        assert_raise(IndexError, ARGV[1]) {a[0, 0] = Array.new(0x1000)}
+      end;
+    end
+  end
+
   private
   def need_continuation
     unless respond_to?(:callcc, true)