[ruby/irb] Fix ripper_lex_without_warning duplicated heredoc token

https://github.com/ruby/irb/commit/45b539af39

2 files changed, 17 insertions, 8 deletions

diff --git a/lib/irb/ruby-lex.rb b/lib/irb/ruby-lex.rb
index 29862f5507..766e797fca 100644
--- a/lib/irb/ruby-lex.rb
+++ b/lib/irb/ruby-lex.rb
@@ -152,17 +152,13 @@ class RubyLex
       lexer = Ripper::Lexer.new(inner_code, '-', line_no)
       if lexer.respond_to?(:scan) # Ruby 2.7+
         tokens = []
-        pos_to_index = {}
         lexer.scan.each do |t|
           next if t.pos.first == 0
-          if pos_to_index.has_key?(t.pos)
-            index = pos_to_index[t.pos]
-            found_tk = tokens[index]
-            if ERROR_TOKENS.include?(found_tk.event) && !ERROR_TOKENS.include?(t.event)
-              tokens[index] = t
-            end
+          prev_tk = tokens.last
+          position_overlapped = prev_tk && t.pos[0] == prev_tk.pos[0] && t.pos[1] < prev_tk.pos[1] + prev_tk.tok.bytesize
+          if position_overlapped
+            tokens[-1] = t if ERROR_TOKENS.include?(prev_tk.event) && !ERROR_TOKENS.include?(t.event)
           else
-            pos_to_index[t.pos] = tokens.size
             tokens << t
           end
         end
diff --git a/test/irb/test_ruby_lex.rb b/test/irb/test_ruby_lex.rb
index 47435d675e..2c94a36a5d 100644
--- a/test/irb/test_ruby_lex.rb
+++ b/test/irb/test_ruby_lex.rb
@@ -619,5 +619,18 @@ module TestIRB
         pos_to_index[t.pos] = i
       }
     end
+
+    def test_unterminated_code
+      if Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.7.0')
+        pend 'This test needs Ripper::Lexer#scan to take broken tokens'
+      end
+
+      ['do', '<<A'].each do |code|
+        tokens = RubyLex.ripper_lex_without_warning(code)
+        assert_equal(code, tokens.map(&:tok).join, "Cannot reconstruct code from tokens")
+        error_tokens = tokens.map(&:event).grep(/error/)
+        assert_empty(error_tokens, 'Error tokens must be ignored if there is corresponding non-error token')
+      end
+    end
   end
 end