Check the suffix argument always if given

It also must be ASCII-compatible and must not contain null byte, as
well as the path argument.

2 files changed, 9 insertions, 12 deletions

diff --git a/file.c b/file.c
index 6b4e137860..70c3892986 100644
--- a/file.c
+++ b/file.c
@@ -5020,23 +5020,21 @@ ruby_enc_find_basename(const char *name, long *baselen, long *alllen, rb_encodin
 static VALUE
 rb_file_s_basename(int argc, VALUE *argv, VALUE _)
 {
-    VALUE fname, fext;
-    const char *name, *p;
+    VALUE fname, fext = Qnil;
+    const char *name, *p, *fp = 0;
     long f = 0, n;
     rb_encoding *enc;
 
-    fext = Qnil;
-    if (rb_check_arity(argc, 1, 2) == 2) {
+    argc = rb_check_arity(argc, 1, 2);
+    fname = argv[0];
+    CheckPath(fname, name);
+    if (argc == 2) {
         fext = argv[1];
-        StringValue(fext);
+        fp = StringValueCStr(fext);
         check_path_encoding(fext);
-        enc = rb_str_enc_get(fext);
     }
-    fname = argv[0];
-    CheckPath(fname, name);
     if (NIL_P(fext) || !(enc = rb_enc_compatible(fname, fext))) {
         enc = rb_str_enc_get(fname);
-        fext = Qnil;
     }
 
     n = RSTRING_LEN(fname);
@@ -5047,12 +5045,10 @@ rb_file_s_basename(int argc, VALUE *argv, VALUE _)
     bool mb_enc = !rb_str_encindex_fastpath(rb_enc_to_index(enc));
     p = enc_find_basename(name, &f, &n, mb_enc, enc);
     if (n >= 0) {
-        if (NIL_P(fext)) {
+        if (!fp) {
             f = n;
         }
         else {
-            const char *fp;
-            fp = StringValueCStr(fext);
             if (!(f = rmext(p, f, n, fp, RSTRING_LEN(fext), enc))) {
                 f = n;
             }
diff --git a/test/ruby/test_file_exhaustive.rb b/test/ruby/test_file_exhaustive.rb
index 394dc47603..be9e6bd44e 100644
--- a/test/ruby/test_file_exhaustive.rb
+++ b/test/ruby/test_file_exhaustive.rb
@@ -1235,6 +1235,7 @@ class TestFileExhaustive < Test::Unit::TestCase
     assert_equal("foo", File.basename("foo", ".ext"))
     assert_equal("foo", File.basename("foo.ext", ".ext"))
     assert_equal("foo", File.basename("foo.ext", ".*"))
+    assert_raise(ArgumentError) {File.basename("", "\0")}
   end
 
   if NTFS