summaryrefslogtreecommitdiff
path: root/ext/bigdecimal
diff options
context:
space:
mode:
authormrkn <mrkn@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2013-12-12 17:02:27 +0000
committermrkn <mrkn@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2013-12-12 17:02:27 +0000
commitdf33325192b6f09828ccdbe537397cd38679e979 (patch)
tree43a9b423005620436a76cce5cfee861066199974 /ext/bigdecimal
parent7d3538db3fc8fb5890ed0f3ae19904cf05bb9d24 (diff)
* ext/bigdecimal/bigdecimal.c (VpSetPTR): fix for limitation of the resulting
precision. * test/bigdecimal/test_bigdecimal.rb (test_limit): add tests for the above change. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@44153 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'ext/bigdecimal')
-rw-r--r--ext/bigdecimal/bigdecimal.c16
1 files changed, 14 insertions, 2 deletions
diff --git a/ext/bigdecimal/bigdecimal.c b/ext/bigdecimal/bigdecimal.c
index 266983c..e0b7c01 100644
--- a/ext/bigdecimal/bigdecimal.c
+++ b/ext/bigdecimal/bigdecimal.c
@@ -4189,6 +4189,7 @@ VpAddAbs(Real *a, Real *b, Real *c)
a_pos = ap;
b_pos = bp;
c_pos = cp;
+
if (word_shift == (size_t)-1L) return 0; /* Overflow */
if (b_pos == (size_t)-1L) goto Assign_a;
@@ -4380,12 +4381,19 @@ static size_t
VpSetPTR(Real *a, Real *b, Real *c, size_t *a_pos, size_t *b_pos, size_t *c_pos, BDIGIT *av, BDIGIT *bv)
{
size_t left_word, right_word, word_shift;
+
+ size_t const round_limit = (VpGetPrecLimit() + BASE_FIG - 1) / BASE_FIG;
+
+ assert(a->exponent >= b->expoennt);
+
c->frac[0] = 0;
*av = *bv = 0;
+
word_shift = (a->exponent - b->exponent);
left_word = b->Prec + word_shift;
right_word = Max(a->Prec, left_word);
left_word = c->MaxPrec - 1; /* -1 ... prepare for round up */
+
/*
* check if 'round' is needed.
*/
@@ -4408,7 +4416,9 @@ VpSetPTR(Real *a, Real *b, Real *c, size_t *a_pos, size_t *b_pos, size_t *c_pos,
* a_pos = |
*/
*a_pos = left_word;
- *av = a->frac[*a_pos]; /* av is 'A' shown in above. */
+ if (*a_pos <= round_limit) {
+ *av = a->frac[*a_pos]; /* av is 'A' shown in above. */
+ }
}
else {
/*
@@ -4427,7 +4437,9 @@ VpSetPTR(Real *a, Real *b, Real *c, size_t *a_pos, size_t *b_pos, size_t *c_pos,
*/
if (c->MaxPrec >= word_shift + 1) {
*b_pos = c->MaxPrec - word_shift - 1;
- *bv = b->frac[*b_pos];
+ if (*b_pos + word_shift <= round_limit) {
+ *bv = b->frac[*b_pos];
+ }
}
else {
*b_pos = -1L;