Reapply "[ruby/prism] Add Ripper :on_sp events for Prism.lex_compat and Prism::Translation::Ripper"

This reverts commit 58f1127b51cf4fbb1f334f8701a041f40701dca2.

3 files changed, 10 insertions, 4 deletions

diff --git a/test/prism/fixtures/bom_leading_space.txt b/test/prism/fixtures/bom_leading_space.txt
new file mode 100644
index 0000000000..48d3ee50ea
--- /dev/null
+++ b/test/prism/fixtures/bom_leading_space.txt
@@ -0,0 +1 @@
+ p (42)
diff --git a/test/prism/fixtures/bom_spaces.txt b/test/prism/fixtures/bom_spaces.txt
new file mode 100644
index 0000000000..c18ad4c21a
--- /dev/null
+++ b/test/prism/fixtures/bom_spaces.txt
@@ -0,0 +1 @@
+p ( 42 )
diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb
index 2a0504c19f..280abd94ea 100644
--- a/test/prism/ruby/ripper_test.rb
+++ b/test/prism/ruby/ripper_test.rb
@@ -39,6 +39,8 @@ module Prism
 
     # Skip these tests that we haven't implemented yet.
     omitted_sexp_raw = [
+      "bom_leading_space.txt",
+      "bom_spaces.txt",
       "dos_endings.txt",
       "heredocs_with_fake_newlines.txt",
       "heredocs_with_ignored_newlines.txt",
@@ -92,7 +94,7 @@ module Prism
       assert_equal(expected, lexer.parse[0].to_a)
       assert_equal(lexer.parse[0].to_a, lexer.scan[0].to_a)
 
-      assert_equal(%i[on_int on_op], Translation::Ripper::Lexer.new("1 +").lex.map(&:event))
+      assert_equal(%i[on_int on_sp on_op], Translation::Ripper::Lexer.new("1 +").lex.map(&:event))
       assert_raise(SyntaxError) { Translation::Ripper::Lexer.new("1 +").lex(raise_errors: true) }
     end
 
@@ -121,15 +123,17 @@ module Prism
     def assert_ripper_lex(source)
       prism = Translation::Ripper.lex(source)
       ripper = Ripper.lex(source)
-      ripper.reject! { |elem| elem[1] == :on_sp } # Prism doesn't emit on_sp
-      ripper.sort_by! { |elem| elem[0] } # Prism emits tokens by their order in the code, not in parse order
+
+      # Prism emits tokens by their order in the code, not in parse order
+      ripper.sort_by! { |elem| elem[0] }
 
       [prism.size, ripper.size].max.times do |i|
         expected = ripper[i]
         actual = prism[i]
+
         # Since tokens related to heredocs are not emitted in the same order,
         # the state also doesn't line up.
-        if expected[1] == :on_heredoc_end && actual[1] == :on_heredoc_end
+        if expected && actual && expected[1] == :on_heredoc_end && actual[1] == :on_heredoc_end
           expected[3] = actual[3] = nil
         end