summaryrefslogtreecommitdiff
path: root/bignum.c
diff options
context:
space:
mode:
authorshyouhei <shyouhei@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2018-11-12 01:08:35 +0000
committershyouhei <shyouhei@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2018-11-12 01:08:35 +0000
commit199c5cc18506f0f92e2b76f76e08608f7a09e5ca (patch)
tree8422fde996a19ba10833c33ecac7ef57a3664439 /bignum.c
parent623ecdace6c9201aec8ae5dca88770cee6c85a37 (diff)
~(unsigned char) is not unsigned char
The unary ~ operator excercises integer promotion of the operand _before_ actually applying bitwise complement (cf: ISO/IEC 9899:1990 section 6.3.3.3). Which means `~buf[i]` is in fact `(int)~(int)buf[i]`. The problem is, when buf[i] is 0xFF: buf[i] 0xFF (int)buf[i] 0x0000_00FF ~(int)buf[i] 0xFFFF_FF00 This is -256, out of unsigned char range. The proposed fix is to change the char signed. By doing so, buf[i] 0xFF (signed char)buf[i] 0xFF (int)(signed char)buf[i] 0xFFFF_FFFF ~(int)(signed char)buf[i] 0x0000_0000 This is 0, does not overflow. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@65675 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'bignum.c')
-rw-r--r--bignum.c8
1 files changed, 6 insertions, 2 deletions
diff --git a/bignum.c b/bignum.c
index 8f9f2ce737..868c5dba88 100644
--- a/bignum.c
+++ b/bignum.c
@@ -616,8 +616,12 @@ static int
bytes_2comp(unsigned char *buf, size_t len)
{
size_t i;
- for (i = 0; i < len; i++)
- buf[i] = ~buf[i];
+ for (i = 0; i < len; i++) {
+ signed char c = buf[i];
+ signed int d = ~c;
+ unsigned int e = d & 0xFF;
+ buf[i] = e;
+ }
for (i = 0; i < len; i++) {
buf[i]++;
if (buf[i] != 0)