stringio.c: fix signed integer overflow

* ext/stringio/stringio.c (strio_seek): Avoid signed integer overflow.
  It's not harmful in practice here, but is still undefined behavior.

* ext/stringio/stringio.c (strio_extend): Check that the new length does
  not exceed LONG_MAX. This fixes the invalid write on the overflow.

* test/stringio/test_stringio.rb (test_write_integer_overflow): Add a
  test case for the above fix in strio_extend().

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@56253 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 10 insertions, 6 deletions

diff --git a/ext/stringio/stringio.c b/ext/stringio/stringio.c
index a5a2327366..9273a8effb 100644
--- a/ext/stringio/stringio.c
+++ b/ext/stringio/stringio.c
@@ -610,29 +610,30 @@ strio_seek(int argc, VALUE *argv, VALUE self)
 {
     VALUE whence;
     struct StringIO *ptr = StringIO(self);
-    long offset;
+    long amount, offset;
 
     rb_scan_args(argc, argv, "11", NULL, &whence);
-    offset = NUM2LONG(argv[0]);
+    amount = NUM2LONG(argv[0]);
     if (CLOSED(self)) {
 	rb_raise(rb_eIOError, "closed stream");
     }
     switch (NIL_P(whence) ? 0 : NUM2LONG(whence)) {
       case 0:
+	offset = 0;
 	break;
       case 1:
-	offset += ptr->pos;
+	offset = ptr->pos;
 	break;
       case 2:
-	offset += RSTRING_LEN(ptr->string);
+	offset = RSTRING_LEN(ptr->string);
 	break;
       default:
 	error_inval("invalid whence");
     }
-    if (offset < 0) {
+    if (amount > LONG_MAX - offset || amount + offset < 0) {
 	error_inval(0);
     }
-    ptr->pos = offset;
+    ptr->pos = amount + offset;
     return INT2FIX(0);
 }
 
@@ -734,6 +735,9 @@ strio_extend(struct StringIO *ptr, long pos, long len)
 {
     long olen;
 
+    if (len > LONG_MAX - pos)
+	rb_raise(rb_eArgError, "string size too big");
+
     check_modifiable(ptr);
     olen = RSTRING_LEN(ptr->string);
     if (pos + len > olen) {