Fix Range#{max,minmax} for range with integer beginning and non-integer end

Previously, for inclusive ranges, the max would show up as the
end of the range, even though the end was not an integer and would
not be the maximum value.  For exclusive ranges, max/minmax would
previously raise a TypeError, even though it is possible to get the
correct maximum.

This change to max/minmax also uncovered a similar error in cover?,
which calls max in certain cases, so adjust the code there so that
cover? still works as expected.

Fixes [Bug #17017]

2 files changed, 29 insertions, 5 deletions

diff --git a/range.c b/range.c
index 59cae0ae34..93cf126f51 100644
--- a/range.c
+++ b/range.c
@@ -1234,6 +1234,13 @@ range_max(int argc, VALUE *argv, VALUE range)
         if (c > 0)
             return Qnil;
         if (EXCL(range)) {
+            if (RB_INTEGER_TYPE_P(b) && !RB_INTEGER_TYPE_P(e)) {
+                VALUE end = e;
+                e = rb_funcall(e, rb_intern("floor"), 0);
+                if (!RTEST(rb_funcall(e, rb_intern("=="), 1, end))) {
+                    return e;
+                }
+            }
             if (!RB_INTEGER_TYPE_P(e)) {
                 rb_raise(rb_eTypeError, "cannot exclude non Integer end value");
             }
@@ -1246,6 +1253,9 @@ range_max(int argc, VALUE *argv, VALUE range)
             }
             return rb_funcall(e, '-', 1, INT2FIX(1));
         }
+        if (RB_INTEGER_TYPE_P(b) && !RB_INTEGER_TYPE_P(e)) {
+            e = rb_funcall(e, rb_intern("floor"), 0);
+        }
         return e;
     }
 }
@@ -1593,9 +1603,14 @@ r_cover_range_p(VALUE range, VALUE beg, VALUE end, VALUE val)
     else if (cmp_end >= 0) {
         return TRUE;
     }
-
-    val_max = rb_rescue2(r_call_max, val, 0, Qnil, rb_eTypeError, (VALUE)0);
-    if (val_max == Qnil) return FALSE;
+    if (RB_INTEGER_TYPE_P(val_beg) && RB_INTEGER_TYPE_P(beg) &&
+            RB_INTEGER_TYPE_P(val_end) != RB_INTEGER_TYPE_P(end)) {
+        val_max = val_end;
+    }
+    else {
+        val_max = rb_rescue2(r_call_max, val, 0, Qnil, rb_eTypeError, (VALUE)0);
+        if (val_max == Qnil) return FALSE;
+    }
 
     return r_less(end, val_max) >= 0;
 }
diff --git a/test/ruby/test_range.rb b/test/ruby/test_range.rb
index 3953b3ecc2..9052fe6174 100644
--- a/test/ruby/test_range.rb
+++ b/test/ruby/test_range.rb
@@ -107,11 +107,13 @@ class TestRange < Test::Unit::TestCase
     assert_equal(1, (1...2).max)
     assert_raise(RangeError) { (1..).max }
     assert_raise(RangeError) { (1...).max }
+    assert_equal(2, (1..2.1).max)
+    assert_equal(2, (1...2.1).max)
 
     assert_equal(2.0, (1.0..2.0).max)
     assert_equal(nil, (2.0..1.0).max)
     assert_raise(TypeError) { (1.0...2.0).max }
-    assert_raise(TypeError) { (1...1.5).max }
+    assert_equal(1, (1...1.5).max)
     assert_raise(TypeError) { (1.5...2).max }
 
     assert_equal(-0x80000002, ((-0x80000002)...(-0x80000001)).max)
@@ -133,11 +135,13 @@ class TestRange < Test::Unit::TestCase
     assert_equal([1, 1], (1...2).minmax)
     assert_raise(RangeError) { (1..).minmax }
     assert_raise(RangeError) { (1...).minmax }
+    assert_equal([1, 2], (1..2.1).minmax)
+    assert_equal([1, 2], (1...2.1).minmax)
 
     assert_equal([1.0, 2.0], (1.0..2.0).minmax)
     assert_equal([nil, nil], (2.0..1.0).minmax)
     assert_raise(TypeError) { (1.0...2.0).minmax }
-    assert_raise(TypeError) { (1...1.5).minmax }
+    assert_equal([1, 1], (1..1.5).minmax)
     assert_raise(TypeError) { (1.5...2).minmax }
 
     assert_equal([-0x80000002, -0x80000002], ((-0x80000002)...(-0x80000001)).minmax)
@@ -652,7 +656,12 @@ class TestRange < Test::Unit::TestCase
     assert_not_operator(1..10, :cover?, 3...2)
     assert_not_operator(1..10, :cover?, 3...3)
     assert_not_operator('aa'..'zz', :cover?, 'aa'...'zzz')
+
     assert_not_operator(1..10, :cover?, 1...10.1)
+    assert_not_operator(1...10.1, :cover?, 1..10.1)
+    assert_operator(1..10.1, :cover?, 1...10.1)
+    assert_operator(1..10.1, :cover?, 1...10)
+    assert_operator(1..10.1, :cover?, 1..10)
   end
 
   def test_beg_len