[ruby/prism] Skip missing heredoc end in ripper translator

Prism inserts these to make bookkeeping easier. Ripper does not do so.

https://github.com/ruby/prism/commit/0a3b560218

2 files changed, 16 insertions, 3 deletions

diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb
index e8d2ce1b19..3d5cbfcddc 100644
--- a/lib/prism/lex_compat.rb
+++ b/lib/prism/lex_compat.rb
@@ -768,21 +768,24 @@ module Prism
         source.byte_offset(line, column)
       end
 
-      # Add :on_sp tokens
-      tokens = insert_on_sp(tokens, source, result.data_loc, bom, eof_token)
+      tokens = post_process_tokens(tokens, source, result.data_loc, bom, eof_token)
 
       Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, source)
     end
 
     private
 
-    def insert_on_sp(tokens, source, data_loc, bom, eof_token)
+    def post_process_tokens(tokens, source, data_loc, bom, eof_token)
       new_tokens = []
 
       prev_token_state = Translation::Ripper::Lexer::State[Translation::Ripper::EXPR_BEG]
       prev_token_end = bom ? 3 : 0
 
       tokens.each do |token|
+        # Skip missing heredoc ends.
+        next if token[1] == :on_heredoc_end && token[2] == ""
+
+        # Add :on_sp tokens.
         line, column = token[0]
         start_offset = source.byte_offset(line, column)
 
diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb
index 15f535f3d6..39cb9395ab 100644
--- a/test/prism/ruby/ripper_test.rb
+++ b/test/prism/ruby/ripper_test.rb
@@ -81,6 +81,16 @@ module Prism
       define_method("#{fixture.test_name}_lex") { assert_ripper_lex(fixture.read) }
     end
 
+    def test_lex_ignored_missing_heredoc_end
+      ["", "-", "~"].each do |type|
+        source = "<<#{type}FOO\n"
+        assert_ripper_lex(source)
+
+        source = "<<#{type}'FOO'\n"
+        assert_ripper_lex(source)
+      end
+    end
+
     module Events
       attr_reader :events