summaryrefslogtreecommitdiff
path: root/bignum.c
diff options
context:
space:
mode:
authorusa <usa@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2021-02-28 13:47:29 +0000
committerusa <usa@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2021-02-28 13:47:29 +0000
commitb64876ae559b05949c6857fb3d76822c2f7c4df2 (patch)
treec98c09d45db6f43eeee3254f8b80b0b7c613e412 /bignum.c
parent8b49c3e4bc767bec8a66ac81cbda033330fb2703 (diff)
merge revision(s) f364564e66d1db1de8e80d669287386595c8bc46: [Backport #16269]
bignum.c (estimate_initial_sqrt): prevent integer overflow `Integer.sqrt(0xffff_ffff_ffff_ffff ** 2)` caused assertion failure because of integer overflow. [ruby-core:95453] [Bug #16269] --- bignum.c | 10 +++++++++- test/ruby/test_integer.rb | 3 +++ 2 files changed, 12 insertions(+), 1 deletion(-) git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/branches/ruby_2_6@67894 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'bignum.c')
-rw-r--r--bignum.c10
1 files changed, 9 insertions, 1 deletions
diff --git a/bignum.c b/bignum.c
index ee3b49fd04..302774b6e2 100644
--- a/bignum.c
+++ b/bignum.c
@@ -6875,7 +6875,15 @@ estimate_initial_sqrt(VALUE *xp, const size_t xn, const BDIGIT *nds, size_t len)
rshift /= 2;
rshift += (2-(len&1))*BITSPERDIG/2;
if (rshift >= 0) {
- d <<= rshift;
+ if (nlz((BDIGIT)d) + rshift >= BITSPERDIG) {
+ /* (d << rshift) does cause overflow.
+ * example: Integer.sqrt(0xffff_ffff_ffff_ffff ** 2)
+ */
+ d = ~(BDIGIT_DBL)0;
+ }
+ else {
+ d <<= rshift;
+ }
}
BDIGITS_ZERO(xds, xn-2);
bdigitdbl2bary(&xds[xn-2], 2, d);