[ruby/prism] Fix `on_words_sep` for ripper translator with newlines

Ripper emits a token each per line.

https://github.com/ruby/prism/commit/4b5c9f5437

2 files changed, 9 insertions, 7 deletions

diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb
index 0d78023eba..775be2759a 100644
--- a/lib/prism/lex_compat.rb
+++ b/lib/prism/lex_compat.rb
@@ -657,6 +657,15 @@ module Prism
             IgnoreStateToken.new([[lineno, column], event, value, lex_state])
           when :on_embexpr_end
             IgnoreStateToken.new([[lineno, column], event, value, lex_state])
+          when :on_words_sep
+            # Ripper emits one token each per line.
+            lines = value.lines
+            lines[0...-1].each do |whitespace|
+              tokens << Token.new([[lineno, column], event, whitespace, lex_state])
+              lineno += 1
+              column = 0
+            end
+            Token.new([[lineno, column], event, lines.last, lex_state])
           when :on_regexp_end
             # On regex end, Ripper scans and then sets end state, so the ripper
             # lexed output is begin, when it should be end. prism sets lex state
diff --git a/lib/prism/lex_ripper.rb b/lib/prism/lex_ripper.rb
index 2054cf55ac..f069e50ba9 100644
--- a/lib/prism/lex_ripper.rb
+++ b/lib/prism/lex_ripper.rb
@@ -26,13 +26,6 @@ module Prism
             results << token
             previous = token
           end
-        when :on_words_sep
-          if previous[1] == :on_words_sep
-            previous[2] << token[2]
-          else
-            results << token
-            previous = token
-          end
         else
           results << token
           previous = token