From 537fea9921c8b18ec975d9520881167796d1bbd9 Mon Sep 17 00:00:00 2001 From: rhe Date: Mon, 26 Sep 2016 02:09:50 +0000 Subject: string.c: fix integer overflow in enc_strlen() and rb_enc_strlen_cr() * string.c (enc_strlen, rb_enc_strlen_cr): Avoid signed integer overflow. The result type of a pointer subtraction may have the same size as long. This fixes String#size returning an negative value on i686-linux environment: str = "\x00" * ((1<<31)-2)) str.slice!(-3, 3) str.force_encoding("UTF-32BE") str << 1234 p str.size git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@56247 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- string.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'string.c') diff --git a/string.c b/string.c index 6988e08ec1..c537bbf4ab 100644 --- a/string.c +++ b/string.c @@ -1527,7 +1527,8 @@ enc_strlen(const char *p, const char *e, rb_encoding *enc, int cr) const char *q; if (rb_enc_mbmaxlen(enc) == rb_enc_mbminlen(enc)) { - return (e - p + rb_enc_mbminlen(enc) - 1) / rb_enc_mbminlen(enc); + long diff = (long)(e - p); + return diff / rb_enc_mbminlen(enc) + !!(diff % rb_enc_mbminlen(enc)); } #ifdef NONASCII_MASK else if (cr == ENC_CODERANGE_VALID && enc == rb_utf8_encoding()) { @@ -1609,7 +1610,8 @@ rb_enc_strlen_cr(const char *p, const char *e, rb_encoding *enc, int *cr) *cr = 0; if (rb_enc_mbmaxlen(enc) == rb_enc_mbminlen(enc)) { - return (e - p + rb_enc_mbminlen(enc) - 1) / rb_enc_mbminlen(enc); + long diff = (long)(e - p); + return diff / rb_enc_mbminlen(enc) + !!(diff % rb_enc_mbminlen(enc)); } else if (rb_enc_asciicompat(enc)) { c = 0; -- cgit v1.2.3