Actually ignore FNM_CASEFOLD flag in Dir.glob

This was already documented as being ignored, but it wasn't being
ignored, causing an issue in a particular case where a UTF-8
pattern was provided and a filename was tested that wasn't valid
UTF-8.

Fixes [Bug #14456]

2 files changed, 12 insertions, 1 deletions

diff --git a/dir.c b/dir.c
index d6a963f9d6..8ac10d2360 100644
--- a/dir.c
+++ b/dir.c
@@ -2955,7 +2955,7 @@ static VALUE
 dir_s_glob(rb_execution_context_t *ec, VALUE obj, VALUE str, VALUE rflags, VALUE base, VALUE sort)
 {
     VALUE ary = rb_check_array_type(str);
-    const int flags = NUM2INT(rflags) | dir_glob_option_sort(sort);
+    const int flags = (NUM2INT(rflags) | dir_glob_option_sort(sort)) & ~FNM_CASEFOLD;
     base = dir_glob_option_base(base);
     if (NIL_P(ary)) {
 	ary = rb_push_glob(str, base, flags);
diff --git a/test/ruby/test_dir.rb b/test/ruby/test_dir.rb
index bc91be4abf..bb268f2ab3 100644
--- a/test/ruby/test_dir.rb
+++ b/test/ruby/test_dir.rb
@@ -342,6 +342,17 @@ class TestDir < Test::Unit::TestCase
     assert_equal(%w[dir/], Dir.chdir(@root) {Dir.open("a") {|d| Dir.glob("**/*/", base: d, sort: false).sort}})
   end
 
+  def test_glob_ignore_casefold_invalid_encoding
+    bug14456 = "[ruby-core:85448]"
+    filename = "\u00AAa123".encode('ISO-8859-1')
+    File.write(File.join(@root, filename), "")
+    matches = Dir.chdir(@root) {|d| Dir.glob("*a123".encode('UTF-8'), File::FNM_CASEFOLD)}
+    assert_equal(1, matches.size, bug14456)
+    matches.each{|f| f.force_encoding('ISO-8859-1')}
+    # Handle MacOS/Windows, which saves under a different filename
+    assert_include([filename, "\u00C2\u00AAa123".encode('ISO-8859-1')], matches.first, bug14456)
+  end
+
   def assert_entries(entries, children_only = false)
     entries.sort!
     expected = ("a".."z").to_a