merge revision(s) f364564e66d1db1de8e80d669287386595c8bc46: [Backport #16269]

	bignum.c (estimate_initial_sqrt): prevent integer overflow

	`Integer.sqrt(0xffff_ffff_ffff_ffff ** 2)` caused assertion failure
	because of integer overflow.  [ruby-core:95453] [Bug #16269]
	---
	 bignum.c                  | 10 +++++++++-
	 test/ruby/test_integer.rb |  3 +++
	 2 files changed, 12 insertions(+), 1 deletion(-)

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/branches/ruby_2_6@67894 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 9 insertions, 1 deletions

diff --git a/bignum.c b/bignum.c
index ee3b49fd04..302774b6e2 100644
--- a/bignum.c
+++ b/bignum.c
@@ -6875,7 +6875,15 @@ estimate_initial_sqrt(VALUE *xp, const size_t xn, const BDIGIT *nds, size_t len)
     rshift /= 2;
     rshift += (2-(len&1))*BITSPERDIG/2;
     if (rshift >= 0) {
-	d <<= rshift;
+        if (nlz((BDIGIT)d) + rshift >= BITSPERDIG) {
+            /* (d << rshift) does cause overflow.
+             * example: Integer.sqrt(0xffff_ffff_ffff_ffff ** 2)
+             */
+            d = ~(BDIGIT_DBL)0;
+        }
+        else {
+            d <<= rshift;
+        }
     }
     BDIGITS_ZERO(xds, xn-2);
     bdigitdbl2bary(&xds[xn-2], 2, d);