* numeric.c (check_uint): fix off-by-one bug of NUM2UINT.

* bignum.c (rb_big2ulong): fix off-by-one bug of NUM2ULONG.

* test/-ext-/num2int/test_num2int.rb: add a testcase for NUM2INT()
  NUM2UINT(), NUM2LONG(), NUM2ULONG(), NUM2LL and NUM2ULL().
* ext/-test-/num2int/depend: ditto.
* ext/-test-/num2int/extconf.rb: ditto.
* ext/-test-/num2int/num2int.c: ditto.



git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@33739 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

7 files changed, 249 insertions, 4 deletions

diff --git a/ChangeLog b/ChangeLog
index b5df0cd2d1..4df8469576 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,14 @@
+Sun Nov 13 09:57:29 2011  KOSAKI Motohiro  <kosaki.motohiro@gmail.com>
+
+	* numeric.c (check_uint): fix off-by-one bug of NUM2UINT.
+	* bignum.c (rb_big2ulong): fix off-by-one bug of NUM2ULONG.
+
+	* test/-ext-/num2int/test_num2int.rb: add a testcase for NUM2INT()
+	  NUM2UINT(), NUM2LONG(), NUM2ULONG(), NUM2LL and NUM2ULL().
+	* ext/-test-/num2int/depend: ditto.
+	* ext/-test-/num2int/extconf.rb: ditto.
+	* ext/-test-/num2int/num2int.c: ditto.
+
 Sun Nov 13 23:47:29 2011  Nobuyoshi Nakada  <nobu@ruby-lang.org>
 
 	* ext/dbm/extconf.rb: use convertible_int.
diff --git a/bignum.c b/bignum.c
index db7215ac15..7bcc4c7935 100644
--- a/bignum.c
+++ b/bignum.c
@@ -1210,10 +1210,11 @@ rb_big2ulong(VALUE x)
     VALUE num = big2ulong(x, "unsigned long", TRUE);
 
     if (!RBIGNUM_SIGN(x)) {
-	if ((long)num < 0) {
+	VALUE v = (VALUE)(-(SIGNED_VALUE)num);
+
+	if (v <= LONG_MAX)
 	    rb_raise(rb_eRangeError, "bignum out of range of unsigned long");
-	}
-	return (VALUE)(-(SIGNED_VALUE)num);
+	return v;
     }
     return num;
 }
diff --git a/ext/-test-/num2int/depend b/ext/-test-/num2int/depend
new file mode 100644
index 0000000000..7a85cdb6c0
--- /dev/null
+++ b/ext/-test-/num2int/depend
@@ -0,0 +1 @@
+num2int.o: $(top_srcdir)/numeric.c $(hdrdir)/ruby/ruby.h
diff --git a/ext/-test-/num2int/extconf.rb b/ext/-test-/num2int/extconf.rb
new file mode 100644
index 0000000000..2bc820e480
--- /dev/null
+++ b/ext/-test-/num2int/extconf.rb
@@ -0,0 +1 @@
+create_makefile("-test-/num2int/num2int")
diff --git a/ext/-test-/num2int/num2int.c b/ext/-test-/num2int/num2int.c
new file mode 100644
index 0000000000..a7242a982c
--- /dev/null
+++ b/ext/-test-/num2int/num2int.c
@@ -0,0 +1,86 @@
+#include <ruby.h>
+
+extern VALUE rb_stdout;
+
+static VALUE
+print_num2int(VALUE obj, VALUE num)
+{
+    char buf[128];
+    VALUE str;
+
+    sprintf(buf, "%d", NUM2INT(num));
+    str = rb_str_new_cstr(buf);
+    rb_io_write(rb_stdout, str);
+}
+
+static VALUE
+print_num2uint(VALUE obj, VALUE num)
+{
+    char buf[128];
+    VALUE str;
+
+    sprintf(buf, "%u", NUM2UINT(num));
+    str = rb_str_new_cstr(buf);
+    rb_io_write(rb_stdout, str);
+}
+
+static VALUE
+print_num2long(VALUE obj, VALUE num)
+{
+    char buf[128];
+    VALUE str;
+
+    sprintf(buf, "%ld", NUM2LONG(num));
+    str = rb_str_new_cstr(buf);
+    rb_io_write(rb_stdout, str);
+}
+
+static VALUE
+print_num2ulong(VALUE obj, VALUE num)
+{
+    char buf[128];
+    VALUE str;
+
+    sprintf(buf, "%lu", NUM2ULONG(num));
+    str = rb_str_new_cstr(buf);
+    rb_io_write(rb_stdout, str);
+}
+
+static VALUE
+print_num2ll(VALUE obj, VALUE num)
+{
+    char buf[128];
+    VALUE str;
+
+    sprintf(buf, "%lld", NUM2LL(num));
+    str = rb_str_new_cstr(buf);
+    rb_io_write(rb_stdout, str);
+}
+
+static VALUE
+print_num2ull(VALUE obj, VALUE num)
+{
+    char buf[128];
+    VALUE str;
+
+    sprintf(buf, "%llu", NUM2ULL(num));
+    str = rb_str_new_cstr(buf);
+    rb_io_write(rb_stdout, str);
+}
+
+
+void
+Init_num2int(void)
+{
+    VALUE cNum2int = rb_path2class("TestNum2int::Num2int");
+
+    rb_define_singleton_method(cNum2int, "print_num2int", print_num2int, 1);
+    rb_define_singleton_method(cNum2int, "print_num2uint", print_num2uint, 1);
+
+    rb_define_singleton_method(cNum2int, "print_num2long", print_num2long, 1);
+    rb_define_singleton_method(cNum2int, "print_num2ulong", print_num2ulong, 1);
+
+    rb_define_singleton_method(cNum2int, "print_num2ll", print_num2ll, 1);
+    rb_define_singleton_method(cNum2int, "print_num2ull", print_num2ull, 1);
+}
+
diff --git a/numeric.c b/numeric.c
index e59226c48a..af50fbafde 100644
--- a/numeric.c
+++ b/numeric.c
@@ -1888,7 +1888,7 @@ check_uint(VALUE num, VALUE sign)
 
     if (RTEST(sign)) {
 	/* minus */
-	if ((num & mask) != mask || (num & ~mask) <= INT_MAX + 1UL)
+	if ((num & mask) != mask || (num & ~mask) <= INT_MAX)
 #define VALUE_MSBMASK   ((VALUE)1 << ((sizeof(VALUE) * CHAR_BIT) - 1))
 	    rb_raise(rb_eRangeError, "integer %"PRIdVALUE " too small to convert to `unsigned int'", num|VALUE_MSBMASK);
     }
diff --git a/test/-ext-/num2int/test_num2int.rb b/test/-ext-/num2int/test_num2int.rb
new file mode 100644
index 0000000000..7d021d1955
--- /dev/null
+++ b/test/-ext-/num2int/test_num2int.rb
@@ -0,0 +1,145 @@
+require 'test/unit'
+
+class TestNum2int < Test::Unit::TestCase
+  module Num2int
+  end
+  require '-test-/num2int/num2int'
+
+  INT_MIN = -2147483648
+  INT_MAX = 2147483647
+  UINT_MAX = 4294967295
+
+  case [0].pack('L!').size
+  when 4
+    LONG_MAX = 2147483647
+    LONG_MIN = -2147483648
+    ULONG_MAX = 4294967295
+  when 8
+    LONG_MAX = 9223372036854775807
+    LONG_MIN = -9223372036854775808
+    ULONG_MAX = 18446744073709551615
+  end
+
+  LLONG_MAX = 9223372036854775807
+  LLONG_MIN = -9223372036854775808
+  ULLONG_MAX = 18446744073709551615
+
+  def test_num2int
+    assert_output(INT_MIN.to_s) do
+      Num2int.print_num2int(INT_MIN)
+    end
+    assert_output(INT_MAX.to_s) do
+      Num2int.print_num2int(INT_MAX)
+    end
+    assert_raise(RangeError) do
+      Num2int.print_num2int(INT_MIN-1)
+    end
+    assert_raise(RangeError) do
+      Num2int.print_num2int(INT_MAX+1)
+    end
+  end
+
+  def test_num2uint
+    assert_output("0") do
+      Num2int.print_num2uint(0)
+    end
+    assert_output(UINT_MAX.to_s) do
+      Num2int.print_num2uint(UINT_MAX)
+    end
+    assert_output(UINT_MAX.to_s) do
+      Num2int.print_num2uint(-1)
+    end
+    assert_output((INT_MAX+1).to_s) do
+      Num2int.print_num2uint(INT_MIN)
+    end
+    assert_raise(RangeError) do
+      Num2int.print_num2uint(INT_MIN-1)
+    end
+    assert_raise(RangeError) do
+      Num2int.print_num2uint(UINT_MAX+1)
+    end
+  end
+
+  def test_num2long
+    assert_output(LONG_MIN.to_s) do
+      Num2int.print_num2long(LONG_MIN)
+    end
+    assert_output(LONG_MAX.to_s) do
+      Num2int.print_num2long(LONG_MAX)
+    end
+    assert_raise(RangeError) do
+      Num2int.print_num2long(LONG_MIN-1)
+    end
+    assert_raise(RangeError) do
+      Num2int.print_num2long(LONG_MAX+1)
+    end
+  end
+
+  def test_num2ulong
+    assert_output("0") do
+      Num2int.print_num2ulong(0)
+    end
+    assert_output(ULONG_MAX.to_s) do
+      Num2int.print_num2ulong(ULONG_MAX)
+    end
+    assert_output(ULONG_MAX.to_s) do
+      Num2int.print_num2ulong(-1)
+    end
+    assert_output((LONG_MAX+1).to_s) do
+      Num2int.print_num2ulong(LONG_MIN)
+    end
+    assert_raise(RangeError) do
+      Num2int.print_num2ulong(LONG_MIN-1)
+    end
+    assert_raise(RangeError) do
+      Num2int.print_num2ulong(ULONG_MAX+1)
+    end
+  end
+
+  def test_num2ll
+    assert_output(LONG_MIN.to_s) do
+      Num2int.print_num2ll(LONG_MIN)
+    end
+    assert_output(LLONG_MAX.to_s) do
+      Num2int.print_num2ll(LLONG_MAX)
+    end
+    assert_raise(RangeError) do
+      Num2int.print_num2ll(LLONG_MIN-1)
+    end
+    assert_raise(RangeError) do
+      Num2int.print_num2ll(LLONG_MAX+1)
+    end
+  end
+
+  def test_num2ull
+    assert_output("0") do
+      Num2int.print_num2ull(0)
+    end
+    assert_output(ULLONG_MAX.to_s) do
+      Num2int.print_num2ull(ULLONG_MAX)
+    end
+    assert_output(ULLONG_MAX.to_s) do
+      Num2int.print_num2ull(-1)
+    end
+    assert_output((LLONG_MAX+2).to_s) do
+      Num2int.print_num2ull(LLONG_MIN+1)
+    end
+
+    # maybe bug
+    assert_output((LLONG_MAX).to_s) do
+      Num2int.print_num2ull(LLONG_MIN-1)
+    end
+    # maybe bug
+    assert_output(1.to_s) do
+      Num2int.print_num2ull(LLONG_MIN*2+1)
+    end
+    assert_raise(RangeError) do
+      Num2int.print_num2ull(LLONG_MIN*2)
+    end
+    assert_raise(RangeError) do
+      Num2int.print_num2ull(ULLONG_MAX+1)
+    end
+  end
+end
+
+