Fix infinite loop at illegal sequence [Bug #17729]

As mblen returns -1 on failure, skip the first byte and try the
succeeding bytes in that case.

Close https://github.com/ruby/ruby/pull/4281

1 files changed, 5 insertions, 0 deletions

diff --git a/test/ruby/test_rubyoptions.rb b/test/ruby/test_rubyoptions.rb
index 0d42604b87..e26a24e8ad 100644
--- a/test/ruby/test_rubyoptions.rb
+++ b/test/ruby/test_rubyoptions.rb
@@ -1089,6 +1089,11 @@ class TestRubyOptions < Test::Unit::TestCase
     end
   end
 
+  def test_rubylib_invalid_encoding
+    env = {"RUBYLIB"=>"\xFF", "LOCALE"=>"en_US.UTF-8", "LC_ALL"=>"en_US.UTF-8"}
+    assert_ruby_status([env, "-e;"])
+  end
+
   def test_null_script
     skip "#{IO::NULL} is not a character device" unless File.chardev?(IO::NULL)
     assert_in_out_err([IO::NULL], success: true)