[Bug #18963] Separate string contents by here document terminator

2 files changed, 23 insertions, 0 deletions

diff --git a/parse.y b/parse.y
index 405a83a8cf..e6fee29595 100644
--- a/parse.y
+++ b/parse.y
@@ -7196,6 +7196,10 @@ tokadd_string(struct parser_params *p,
 {
     int c;
     bool erred = false;
+#ifdef RIPPER
+    const int heredoc_end = (p->heredoc_end ? p->heredoc_end + 1 : 0);
+    int top_of_line = FALSE;
+#endif
 
 #define mixed_error(enc1, enc2) \
     (void)(erred || (parser_mixed_error(p, enc1, enc2), erred = true))
@@ -7206,6 +7210,12 @@ tokadd_string(struct parser_params *p,
 	if (p->heredoc_indent > 0) {
 	    parser_update_heredoc_indent(p, c);
 	}
+#ifdef RIPPER
+        if (top_of_line && heredoc_end == p->ruby_sourceline) {
+            pushback(p, c);
+            break;
+        }
+#endif
 
 	if (paren && c == paren) {
 	    ++*nest;
@@ -7332,6 +7342,9 @@ tokadd_string(struct parser_params *p,
 	    }
         }
 	tokadd(p, c);
+#ifdef RIPPER
+	top_of_line = (c == '\n');
+#endif
     }
   terminate:
     if (*enc) *encp = *enc;
diff --git a/test/ripper/test_lexer.rb b/test/ripper/test_lexer.rb
index 4f3f4657ef..27e0007023 100644
--- a/test/ripper/test_lexer.rb
+++ b/test/ripper/test_lexer.rb
@@ -242,4 +242,14 @@ class TestRipper::Lexer < Test::Unit::TestCase
     EOF
     assert_equal([[5, 0], :on_heredoc_end, "EOS\n", state(:EXPR_BEG)], Ripper.lex(s).last, bug)
   end
+
+  def test_tokenize_with_here_document
+    bug = '[Bug #18963]'
+    code = %[
+<<A + "hello
+A
+world"
+]
+    assert_equal(code, Ripper.tokenize(code).join(""), bug)
+  end
 end