From cee5beab1d7bf6f99530957494ab9775696d42ce Mon Sep 17 00:00:00 2001
From: Nobuyoshi Nakada <nobu@ruby-lang.org>
Date: Fri, 6 Jan 2023 20:26:11 +0900
Subject: [Bug #19312] Return end-of-input at `__END__`

---
 test/ruby/test_ast.rb | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

(limited to 'test/ruby')

diff --git a/test/ruby/test_ast.rb b/test/ruby/test_ast.rb
index e43dac28b3..875cf7138e 100644
--- a/test/ruby/test_ast.rb
+++ b/test/ruby/test_ast.rb
@@ -624,6 +624,27 @@ dummy
     assert_equal("def test_keep_script_lines_for_of\n", node_method.source.lines.first)
   end
 
+  def test_keep_tokens_for_parse
+    node = RubyVM::AbstractSyntaxTree.parse(<<~END, keep_tokens: true)
+    1.times do
+    end
+    __END__
+    dummy
+    END
+
+    expected = [
+      [:tINTEGER, "1"],
+      [:".", "."],
+      [:tIDENTIFIER, "times"],
+      [:tSP, " "],
+      [:keyword_do, "do"],
+      [:tIGNORED_NL, "\n"],
+      [:keyword_end, "end"],
+      [:nl, "\n"],
+    ]
+    assert_equal(expected, node.all_tokens.map { [_2, _3]})
+  end
+
   def test_encoding_with_keep_script_lines
     # Stop a warning "possibly useless use of a literal in void context"
     verbose_bak, $VERBOSE = $VERBOSE, nil
-- 
cgit v1.2.3