[Bug #21202] Fix wrong token concat while tokenizing nested unterminated heredoc (#13000)

1 files changed, 41 insertions, 0 deletions

diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb
index a371e8c42d..7a2c22ff2d 100644
--- a/test/ripper/test_lexer.rb
+++ b/test/ripper/test_lexer.rb
@@ -344,6 +344,47 @@ world"
     ]
 
     assert_lexer(expected, code)
+
+    code = <<~'HEREDOC'
+      <<H1
+      #{<<H2}a
+      H2
+      b
+    HEREDOC
+
+    expected = [
+      [[1, 0], :on_heredoc_beg, "<<H1", state(:EXPR_BEG)],
+      [[1, 4], :on_nl, "\n", state(:EXPR_BEG)],
+      [[2, 0], :on_embexpr_beg, "\#{", state(:EXPR_BEG)],
+      [[2, 2], :on_heredoc_beg, "<<H2", state(:EXPR_BEG)],
+      [[2, 6], :on_embexpr_end, "}", state(:EXPR_END)],
+      [[2, 7], :on_tstring_content, "a\n", state(:EXPR_BEG)],
+      [[3, 0], :on_heredoc_end, "H2\n", state(:EXPR_BEG)],
+      [[4, 0], :on_tstring_content, "b\n", state(:EXPR_BEG)]
+    ]
+
+    assert_lexer(expected, code)
+
+    code = <<~'HEREDOC'
+      <<H1
+      #{<<H2}a
+      H2
+      b
+      c
+    HEREDOC
+
+    expected = [
+      [[1, 0], :on_heredoc_beg, "<<H1", state(:EXPR_BEG)],
+      [[1, 4], :on_nl, "\n", state(:EXPR_BEG)],
+      [[2, 0], :on_embexpr_beg, "\#{", state(:EXPR_BEG)],
+      [[2, 2], :on_heredoc_beg, "<<H2", state(:EXPR_BEG)],
+      [[2, 6], :on_embexpr_end, "}", state(:EXPR_END)],
+      [[2, 7], :on_tstring_content, "a\n", state(:EXPR_BEG)],
+      [[3, 0], :on_heredoc_end, "H2\n", state(:EXPR_BEG)],
+      [[4, 0], :on_tstring_content, "b\nc\n", state(:EXPR_BEG)]
+    ]
+
+    assert_lexer(expected, code)
   end
 
   def test_invalid_escape_ctrl_mbchar