diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/prism/lex_compat.rb | 9 | ||||
| -rw-r--r-- | lib/prism/lex_ripper.rb | 7 |
2 files changed, 9 insertions, 7 deletions
diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index 0d78023eba..775be2759a 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -657,6 +657,15 @@ module Prism IgnoreStateToken.new([[lineno, column], event, value, lex_state]) when :on_embexpr_end IgnoreStateToken.new([[lineno, column], event, value, lex_state]) + when :on_words_sep + # Ripper emits one token each per line. + lines = value.lines + lines[0...-1].each do |whitespace| + tokens << Token.new([[lineno, column], event, whitespace, lex_state]) + lineno += 1 + column = 0 + end + Token.new([[lineno, column], event, lines.last, lex_state]) when :on_regexp_end # On regex end, Ripper scans and then sets end state, so the ripper # lexed output is begin, when it should be end. prism sets lex state diff --git a/lib/prism/lex_ripper.rb b/lib/prism/lex_ripper.rb index 2054cf55ac..f069e50ba9 100644 --- a/lib/prism/lex_ripper.rb +++ b/lib/prism/lex_ripper.rb @@ -26,13 +26,6 @@ module Prism results << token previous = token end - when :on_words_sep - if previous[1] == :on_words_sep - previous[2] << token[2] - else - results << token - previous = token - end else results << token previous = token |