From b0e2b7a5ff0df14b3c8062c31ebb526a73a03763 Mon Sep 17 00:00:00 2001
From: Nobuyoshi Nakada <nobu@ruby-lang.org>
Date: Thu, 30 May 2019 21:49:08 +0900
Subject: Include stack elements left after errors

---
 ext/ripper/lib/ripper/lexer.rb |  2 +-
 test/ripper/test_lexer.rb      | 22 ++++++++++++++++++++++
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index d1d257582d..696471a75d 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -103,7 +103,7 @@ class Ripper
 
     # parse the code and returns elements including errors.
     def scan
-      (parse() + errors).sort_by {|e| [*e.pos, (e.message ? -1 : 0)]}
+      (parse() + errors + @stack.flatten).uniq.sort_by {|e| [*e.pos, (e.message ? -1 : 0)]}
     end
 
     def parse
diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb
index 30b85eff89..4b70915851 100644
--- a/test/ripper/test_lexer.rb
+++ b/test/ripper/test_lexer.rb
@@ -113,4 +113,26 @@ class TestRipper::Lexer < Test::Unit::TestCase
     assert_equal [[1,0],:on_cvar,"@@1",state(:EXPR_END)], Ripper.lex("@@1").last
     assert_equal [[1,1],:on_cvar,"@@1",state(:EXPR_ENDFN)], Ripper.lex(":@@1").last
   end
+
+  def test_token_aftr_error_heredoc
+    code = "<<A.upcase\n"
+    result = Ripper::Lexer.new(code).scan
+    message = proc {result.pretty_inspect}
+    expected = [
+      [[1, 0], :on_heredoc_beg, "<<A", state(:EXPR_BEG)],
+      [[1, 3], :on_period, ".", state(:EXPR_DOT)],
+      [[1, 4], :on_ident, "upcase", state(:EXPR_ARG)],
+      [[1, 10], :on_nl, "\n", state(:EXPR_BEG)],
+      [[1, 11], :compile_error, "", state(:EXPR_BEG), "can't find string \"A\" anywhere before EOF"],
+    ]
+    pos = 0
+    expected.each_with_index do |ex, i|
+      s = result[i]
+      assert_equal ex, s.to_a, message
+      assert_equal pos, s.pos[1], message
+      pos += s.tok.bytesize
+    end
+    assert_equal pos, code.bytesize
+    assert_equal expected.size, result.size
+  end
 end
-- 
cgit v1.2.3