Fix Range#max for beginless Integer ranges [Bug #17034]

* Fix Range#max for beginless Integer ranges
* Update test/ruby/test_range.rb
* Fix formatting

https://github.com/ruby/ruby/pull/3328

Co-authored-by: Nobuyoshi Nakada <nobu@ruby-lang.org>

2 files changed, 8 insertions, 3 deletions

diff --git a/range.c b/range.c
index 93cf126f51..124d9faa51 100644
--- a/range.c
+++ b/range.c
@@ -1220,16 +1220,17 @@ range_max(int argc, VALUE *argv, VALUE range)
 	rb_raise(rb_eRangeError, "cannot get the maximum of endless range");
     }
 
+    VALUE b = RANGE_BEG(range);
+
     if (rb_block_given_p() || (EXCL(range) && !nm) || argc) {
-        if (NIL_P(RANGE_BEG(range))) {
+        if (NIL_P(b)) {
             rb_raise(rb_eRangeError, "cannot get the maximum of beginless range with custom comparison method");
         }
         return rb_call_super(argc, argv);
     }
     else {
         struct cmp_opt_data cmp_opt = { 0, 0 };
-        VALUE b = RANGE_BEG(range);
-        int c = OPTIMIZED_CMP(b, e, cmp_opt);
+        int c = NIL_P(b) ? -1 : OPTIMIZED_CMP(b, e, cmp_opt);
 
         if (c > 0)
             return Qnil;
diff --git a/test/ruby/test_range.rb b/test/ruby/test_range.rb
index 9052fe6174..0b3f6c68f6 100644
--- a/test/ruby/test_range.rb
+++ b/test/ruby/test_range.rb
@@ -127,6 +127,10 @@ class TestRange < Test::Unit::TestCase
     assert_raise(RangeError) { (1...).max(3) }
 
     assert_raise(RangeError) { (..0).min {|a, b| a <=> b } }
+
+    assert_equal(2, (..2).max)
+    assert_raise(TypeError) { (...2).max }
+    assert_raise(TypeError) { (...2.0).max }
   end
 
   def test_minmax