[ruby/yarp] Fix lex compat with BOM

* BOM should not impact looking for the encoding string
* We should re-encode tokens when the encoding changes
* BOM should change the column of comments only

https://github.com/ruby/yarp/commit/119fc2d7b2

1 files changed, 57 insertions, 0 deletions

diff --git a/test/bom_test.rb b/test/bom_test.rb
new file mode 100644
index 0000000000..7dc7eabe92
--- /dev/null
+++ b/test/bom_test.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+# Don't bother checking this on these engines, this is such a specific Ripper
+# test.
+return if RUBY_ENGINE == "jruby" || RUBY_ENGINE == "truffleruby"
+
+require "yarp_test_helper"
+
+class BOMTest < Test::Unit::TestCase
+  def test_ident
+    assert_bom("foo")
+  end
+
+  def test_back_reference
+    assert_bom("$+")
+  end
+
+  def test_instance_variable
+    assert_bom("@foo")
+  end
+
+  def test_class_variable
+    assert_bom("@@foo")
+  end
+
+  def test_global_variable
+    assert_bom("$foo")
+  end
+
+  def test_numbered_reference
+    assert_bom("$1")
+  end
+
+  def test_percents
+    assert_bom("%i[]")
+    assert_bom("%r[]")
+    assert_bom("%s[]")
+    assert_bom("%q{}")
+    assert_bom("%w[]")
+    assert_bom("%x[]")
+    assert_bom("%I[]")
+    assert_bom("%W[]")
+    assert_bom("%Q{}")
+  end
+
+  def test_string
+    assert_bom("\"\"")
+    assert_bom("''")
+  end
+
+  private
+
+  def assert_bom(source)
+    bommed = "\xEF\xBB\xBF#{source}"
+    assert_equal YARP.lex_ripper(bommed), YARP.lex_compat(bommed).value
+  end
+end