diff options
author | Yusuke Endoh <mame@ruby-lang.org> | 2019-10-21 21:22:53 +0900 |
---|---|---|
committer | Yusuke Endoh <mame@ruby-lang.org> | 2019-10-21 21:24:21 +0900 |
commit | f364564e66d1db1de8e80d669287386595c8bc46 (patch) | |
tree | b02c5be041ae8a1761c021fe9a655ec89d792bed | |
parent | c8f97d16202a953a66a326b450a82d926f0fea9c (diff) |
bignum.c (estimate_initial_sqrt): prevent integer overflow
`Integer.sqrt(0xffff_ffff_ffff_ffff ** 2)` caused assertion failure
because of integer overflow. [ruby-core:95453] [Bug #16269]
-rw-r--r-- | bignum.c | 10 | ||||
-rw-r--r-- | test/ruby/test_integer.rb | 3 |
2 files changed, 12 insertions, 1 deletions
@@ -6888,7 +6888,15 @@ estimate_initial_sqrt(VALUE *xp, const size_t xn, const BDIGIT *nds, size_t len) rshift /= 2; rshift += (2-(len&1))*BITSPERDIG/2; if (rshift >= 0) { - d <<= rshift; + if (nlz((BDIGIT)d) + rshift >= BITSPERDIG) { + /* (d << rshift) does cause overflow. + * example: Integer.sqrt(0xffff_ffff_ffff_ffff ** 2) + */ + d = ~(BDIGIT_DBL)0; + } + else { + d <<= rshift; + } } BDIGITS_ZERO(xds, xn-2); bdigitdbl2bary(&xds[xn-2], 2, d); diff --git a/test/ruby/test_integer.rb b/test/ruby/test_integer.rb index 9a4f560ed5..a111698771 100644 --- a/test/ruby/test_integer.rb +++ b/test/ruby/test_integer.rb @@ -640,6 +640,9 @@ class TestInteger < Test::Unit::TestCase failures << n unless root*root <= n && (root+1)*(root+1) > n end assert_empty(failures, bug13440) + + x = 0xffff_ffff_ffff_ffff + assert_equal(x, Integer.sqrt(x ** 2), "[ruby-core:95453]") end def test_fdiv |