[Bug #19563] Yield words separators per lines

So that newlines across a here-doc terminator will be separated
tokens.

Cf. https://github.com/ruby/irb/pull/558

2 files changed, 16 insertions, 11 deletions

diff --git a/parse.y b/parse.y
index f371185ee0..a332c31838 100644
--- a/parse.y
+++ b/parse.y
@@ -5025,7 +5025,11 @@ regexp		: tREGEXP_BEG regexp_contents tREGEXP_END
                     }
                 ;
 
-words		: tWORDS_BEG ' ' word_list tSTRING_END
+words_sep	: ' ' {}
+                | words_sep ' '
+                ;
+
+words		: tWORDS_BEG words_sep word_list tSTRING_END
                     {
                     /*%%%*/
                         $$ = make_list($3, &@$);
@@ -5041,7 +5045,7 @@ word_list	: /* none */
                     /*% %*/
                     /*% ripper: words_new! %*/
                     }
-                | word_list word ' '
+                | word_list word words_sep
                     {
                     /*%%%*/
                         $$ = list_append(p, $1, evstr2dstr(p, $2));
@@ -5061,7 +5065,7 @@ word		: string_content
                     }
                 ;
 
-symbols 	: tSYMBOLS_BEG ' ' symbol_list tSTRING_END
+symbols 	: tSYMBOLS_BEG words_sep symbol_list tSTRING_END
                     {
                     /*%%%*/
                         $$ = make_list($3, &@$);
@@ -5077,7 +5081,7 @@ symbol_list	: /* none */
                     /*% %*/
                     /*% ripper: symbols_new! %*/
                     }
-                | symbol_list word ' '
+                | symbol_list word words_sep
                     {
                     /*%%%*/
                         $$ = symbol_append(p, $1, evstr2dstr(p, $2));
@@ -5086,7 +5090,7 @@ symbol_list	: /* none */
                     }
                 ;
 
-qwords		: tQWORDS_BEG ' ' qword_list tSTRING_END
+qwords		: tQWORDS_BEG words_sep qword_list tSTRING_END
                     {
                     /*%%%*/
                         $$ = make_list($3, &@$);
@@ -5095,7 +5099,7 @@ qwords		: tQWORDS_BEG ' ' qword_list tSTRING_END
                     }
                 ;
 
-qsymbols	: tQSYMBOLS_BEG ' ' qsym_list tSTRING_END
+qsymbols	: tQSYMBOLS_BEG words_sep qsym_list tSTRING_END
                     {
                     /*%%%*/
                         $$ = make_list($3, &@$);
@@ -5111,7 +5115,7 @@ qword_list	: /* none */
                     /*% %*/
                     /*% ripper: qwords_new! %*/
                     }
-                | qword_list tSTRING_CONTENT ' '
+                | qword_list tSTRING_CONTENT words_sep
                     {
                     /*%%%*/
                         $$ = list_append(p, $1, $2);
@@ -5127,7 +5131,7 @@ qsym_list	: /* none */
                     /*% %*/
                     /*% ripper: qsymbols_new! %*/
                     }
-                | qsym_list tSTRING_CONTENT ' '
+                | qsym_list tSTRING_CONTENT words_sep
                     {
                     /*%%%*/
                         $$ = symbol_append(p, $1, $2);
@@ -7909,7 +7913,8 @@ parse_string(struct parser_params *p, rb_strterm_literal_t *quote)
     }
     c = nextc(p);
     if ((func & STR_FUNC_QWORDS) && ISSPACE(c)) {
-        do {c = nextc(p);} while (ISSPACE(c));
+        ruby_debug_breakpoint();
+        while (c != '\n' && ISSPACE(c = nextc(p)));
         space = 1;
     }
     if (func & STR_FUNC_LIST) {
@@ -7926,7 +7931,7 @@ parse_string(struct parser_params *p, rb_strterm_literal_t *quote)
         return parser_string_term(p, func);
     }
     if (space) {
-        pushback(p, c);
+        if (!ISSPACE(c)) pushback(p, c);
         add_delayed_token(p, p->lex.ptok, p->lex.pcur, __LINE__);
         return ' ';
     }
diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb
index da3dbfb66c..5d6ac615ca 100644
--- a/test/ripper/test_scanner_events.rb
+++ b/test/ripper/test_scanner_events.rb
@@ -712,7 +712,7 @@ class TestRipper::ScannerEvents < Test::Unit::TestCase
                  scan('words_sep', '%w( w w w )')
     assert_equal [' ', "\n", ' ', ' '],
                  scan('words_sep', "%w( w\nw w )")
-    assert_equal ["\n\n", "\n ", ' ', ' '],
+    assert_equal ["\n", "\n", "\n", ' ', ' ', ' '],
                  scan('words_sep', "%w(\n\nw\n w w )")
   end