stringio.c: fix signed integer overflow

* ext/stringio/stringio.c (strio_seek): Avoid signed integer overflow.
  It's not harmful in practice here, but is still undefined behavior.

* ext/stringio/stringio.c (strio_extend): Check that the new length does
  not exceed LONG_MAX. This fixes the invalid write on the overflow.

* test/stringio/test_stringio.rb (test_write_integer_overflow): Add a
  test case for the above fix in strio_extend().

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@56253 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

3 files changed, 30 insertions, 6 deletions

diff --git a/ChangeLog b/ChangeLog
index a39f2dc505..98b8b5e878 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+Mon Sep 26 16:23:49 2016  Kazuki Yamaguchi  <k@rhe.jp>
+
+	* ext/stringio/stringio.c (strio_seek): Avoid signed integer overflow.
+	  It's not harmful in practice here, but is still undefined behavior.
+
+	* ext/stringio/stringio.c (strio_extend): Check that the new length does
+	  not exceed LONG_MAX. This fixes the invalid write on the overflow.
+
+	* test/stringio/test_stringio.rb (test_write_integer_overflow): Add a
+	  test case for the above fix in strio_extend().
+
 Mon Sep 26 15:43:34 2016  Kazuki Yamaguchi  <k@rhe.jp>
 
 	* eval_intern.h (TH_PUSH_TAG): Initialize struct rb_vm_tag::tag with
diff --git a/ext/stringio/stringio.c b/ext/stringio/stringio.c
index a5a2327366..9273a8effb 100644
--- a/ext/stringio/stringio.c
+++ b/ext/stringio/stringio.c
@@ -610,29 +610,30 @@ strio_seek(int argc, VALUE *argv, VALUE self)
 {
     VALUE whence;
     struct StringIO *ptr = StringIO(self);
-    long offset;
+    long amount, offset;
 
     rb_scan_args(argc, argv, "11", NULL, &whence);
-    offset = NUM2LONG(argv[0]);
+    amount = NUM2LONG(argv[0]);
     if (CLOSED(self)) {
 	rb_raise(rb_eIOError, "closed stream");
     }
     switch (NIL_P(whence) ? 0 : NUM2LONG(whence)) {
       case 0:
+	offset = 0;
 	break;
       case 1:
-	offset += ptr->pos;
+	offset = ptr->pos;
 	break;
       case 2:
-	offset += RSTRING_LEN(ptr->string);
+	offset = RSTRING_LEN(ptr->string);
 	break;
       default:
 	error_inval("invalid whence");
     }
-    if (offset < 0) {
+    if (amount > LONG_MAX - offset || amount + offset < 0) {
 	error_inval(0);
     }
-    ptr->pos = offset;
+    ptr->pos = amount + offset;
     return INT2FIX(0);
 }
 
@@ -734,6 +735,9 @@ strio_extend(struct StringIO *ptr, long pos, long len)
 {
     long olen;
 
+    if (len > LONG_MAX - pos)
+	rb_raise(rb_eArgError, "string size too big");
+
     check_modifiable(ptr);
     olen = RSTRING_LEN(ptr->string);
     if (pos + len > olen) {
diff --git a/test/stringio/test_stringio.rb b/test/stringio/test_stringio.rb
index a3f6dfc3e8..a900362157 100644
--- a/test/stringio/test_stringio.rb
+++ b/test/stringio/test_stringio.rb
@@ -160,6 +160,15 @@ class TestStringIO < Test::Unit::TestCase
     assert_equal(Encoding::UTF_8, s.encoding, "honor the original encoding over ASCII-8BIT")
   end
 
+  def test_write_integer_overflow
+    long_max = (1 << (RbConfig::SIZEOF["long"] * 8 - 1)) - 1
+    f = StringIO.new
+    f.pos = long_max
+    assert_raise(ArgumentError) {
+      f.write("pos + len overflows")
+    }
+  end
+
   def test_set_encoding
     bug10285 = '[ruby-core:65240] [Bug #10285]'
     f = StringIO.new()