From dc06b7b7156e0b1e32f32acf0b89925fd1c4b79f Mon Sep 17 00:00:00 2001 From: matz Date: Fri, 7 Jan 2011 00:17:06 +0000 Subject: * bignum.c (bigmul1_karatsuba): avoid overflow that make assertion fail in certain case. this patch is contributed from Ray Chason in personal communication. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@30483 b2dd03c8-39d4-4d8f-98ff-823fe69b080e --- bignum.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'bignum.c') diff --git a/bignum.c b/bignum.c index 2f804db5df..ede0de6523 100644 --- a/bignum.c +++ b/bignum.c @@ -2157,14 +2157,14 @@ bigmul1_karatsuba(VALUE x, VALUE y) t3 = bigmul0(xh, yh); i = xn + yn - n; - /* add t3 to middle bytes of the result (z1) */ - bigadd_core(zds + n, i, BDIGITS(t3), big_real_len(t3), zds + n, i); + /* subtract t1 from t3 */ + bigsub_core(BDIGITS(t3), big_real_len(t3), BDIGITS(t1), t1n, BDIGITS(t3), big_real_len(t3)); - /* subtract t1 from middle bytes of the result (z1) */ - bigsub_core(zds + n, i, BDIGITS(t1), t1n, zds + n, i); + /* subtract t2 from t3; t3 is now the middle term of the product */ + if (t2 != Qundef) bigsub_core(BDIGITS(t3), big_real_len(t3), BDIGITS(t2), t2n, BDIGITS(t3), big_real_len(t3)); - /* subtract t2 from middle bytes of the result (z1) */ - if (t2 != Qundef) bigsub_core(zds + n, i, BDIGITS(t2), t2n, zds + n, i); + /* add t3 to middle bytes of the result (z1) */ + bigadd_core(zds + n, i, BDIGITS(t3), big_real_len(t3), zds + n, i); return z; } -- cgit v1.2.3