string.c: fix integer overflow in enc_strlen() and rb_enc_strlen_cr()

* string.c (enc_strlen, rb_enc_strlen_cr): Avoid signed integer
  overflow. The result type of a pointer subtraction may have the same
  size as long. This fixes String#size returning an negative value on
  i686-linux environment:

    str = "\x00" * ((1<<31)-2))
    str.slice!(-3, 3)
    str.force_encoding("UTF-32BE")
    str << 1234
    p str.size

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@56247 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 4 insertions, 2 deletions

diff --git a/string.c b/string.c
index 6988e08ec1..c537bbf4ab 100644
--- a/string.c
+++ b/string.c
@@ -1527,7 +1527,8 @@ enc_strlen(const char *p, const char *e, rb_encoding *enc, int cr)
     const char *q;
 
     if (rb_enc_mbmaxlen(enc) == rb_enc_mbminlen(enc)) {
-        return (e - p + rb_enc_mbminlen(enc) - 1) / rb_enc_mbminlen(enc);
+	long diff = (long)(e - p);
+	return diff / rb_enc_mbminlen(enc) + !!(diff % rb_enc_mbminlen(enc));
     }
 #ifdef NONASCII_MASK
     else if (cr == ENC_CODERANGE_VALID && enc == rb_utf8_encoding()) {
@@ -1609,7 +1610,8 @@ rb_enc_strlen_cr(const char *p, const char *e, rb_encoding *enc, int *cr)
 
     *cr = 0;
     if (rb_enc_mbmaxlen(enc) == rb_enc_mbminlen(enc)) {
-	return (e - p + rb_enc_mbminlen(enc) - 1) / rb_enc_mbminlen(enc);
+	long diff = (long)(e - p);
+	return diff / rb_enc_mbminlen(enc) + !!(diff % rb_enc_mbminlen(enc));
     }
     else if (rb_enc_asciicompat(enc)) {
 	c = 0;