* include/ruby/intern.h: add the prototype declaration of

  rb_num_coerce_bit.

* numeric.c (rb_num_coerce_bit): the new coerce function for bitwise
  binary operation.

* bignum.c (rb_big_and): use coerce to convert the argument, which isn't
  a Fixnum nor a Bignum, to the corresponding Integer object so that
  bitwise operations can support Integer-mimic objects.
  [Bug #1792] [ruby-core:39491]

* bignum.c (rb_big_or): ditto.

* bignum.c (rb_big_xor): ditto.

* numeric.c (bit_coerce): ditto.

* numeric.c (fix_and): ditto.

* numeric.c (fix_or): ditto.

* numeric.c (fix_xor): ditto.

* test/ruby/test_integer.rb: add tests for the above changes.

* test/ruby/test_bignum.rb: ditto.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@38560 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 46 insertions, 20 deletions

diff --git a/numeric.c b/numeric.c
index c2562f4698..1447f2ff68 100644
--- a/numeric.c
+++ b/numeric.c
@@ -3165,15 +3165,29 @@ fix_rev(VALUE num)
     return ~num | FIXNUM_FLAG;
 }
 
-static VALUE
-bit_coerce(VALUE x)
-{
-    while (!FIXNUM_P(x) && !RB_TYPE_P(x, T_BIGNUM)) {
-	rb_raise(rb_eTypeError,
-		 "can't convert %s into Integer for bitwise arithmetic",
-		 rb_obj_classname(x));
+static int
+bit_coerce(VALUE *x, VALUE *y, int err)
+{
+    if (!FIXNUM_P(*y) && !RB_TYPE_P(*y, T_BIGNUM)) {
+	do_coerce(x, y, err);
+	if (!FIXNUM_P(*x) && !RB_TYPE_P(*x, T_BIGNUM)
+	    && !FIXNUM_P(*y) && !RB_TYPE_P(*y, T_BIGNUM)) {
+	    if (!err) return FALSE;
+	    rb_raise(rb_eTypeError,
+		     "%s can't be coerced into %s for bitwise arithmetic",
+		     rb_special_const_p(*y) ?
+			RSTRING_PTR(rb_inspect(*y)) : rb_obj_classname(*y),
+		     rb_obj_classname(*x));
+	}
     }
-    return x;
+    return TRUE;
+}
+
+VALUE
+rb_num_coerce_bit(VALUE x, VALUE y, ID func)
+{
+    bit_coerce(&x, &y, TRUE);
+    return rb_funcall(x, func, 1, y);
 }
 
 /*
@@ -3186,13 +3200,17 @@ bit_coerce(VALUE x)
 static VALUE
 fix_and(VALUE x, VALUE y)
 {
-    long val;
+    if (FIXNUM_P(y)) {
+	long val = FIX2LONG(x) & FIX2LONG(y);
+	return LONG2NUM(val);
+    }
 
-    if (!FIXNUM_P(y = bit_coerce(y))) {
+    if (RB_TYPE_P(y, T_BIGNUM)) {
 	return rb_big_and(y, x);
     }
-    val = FIX2LONG(x) & FIX2LONG(y);
-    return LONG2NUM(val);
+
+    bit_coerce(&x, &y, TRUE);
+    return rb_funcall(x, rb_intern("&"), 1, y);
 }
 
 /*
@@ -3205,13 +3223,17 @@ fix_and(VALUE x, VALUE y)
 static VALUE
 fix_or(VALUE x, VALUE y)
 {
-    long val;
+    if (FIXNUM_P(y)) {
+	long val = FIX2LONG(x) | FIX2LONG(y);
+	return LONG2NUM(val);
+    }
 
-    if (!FIXNUM_P(y = bit_coerce(y))) {
+    if (RB_TYPE_P(y, T_BIGNUM)) {
 	return rb_big_or(y, x);
     }
-    val = FIX2LONG(x) | FIX2LONG(y);
-    return LONG2NUM(val);
+
+    bit_coerce(&x, &y, TRUE);
+    return rb_funcall(x, rb_intern("|"), 1, y);
 }
 
 /*
@@ -3224,13 +3246,17 @@ fix_or(VALUE x, VALUE y)
 static VALUE
 fix_xor(VALUE x, VALUE y)
 {
-    long val;
+    if (FIXNUM_P(y)) {
+	long val = FIX2LONG(x) ^ FIX2LONG(y);
+	return LONG2NUM(val);
+    }
 
-    if (!FIXNUM_P(y = bit_coerce(y))) {
+    if (RB_TYPE_P(y, T_BIGNUM)) {
 	return rb_big_xor(y, x);
     }
-    val = FIX2LONG(x) ^ FIX2LONG(y);
-    return LONG2NUM(val);
+
+    bit_coerce(&x, &y, TRUE);
+    return rb_funcall(x, rb_intern("^"), 1, y);
 }
 
 static VALUE fix_lshift(long, unsigned long);