diff options
| author | Earlopain <14981592+Earlopain@users.noreply.github.com> | 2026-01-26 12:16:34 +0100 |
|---|---|---|
| committer | git <svn-admin@ruby-lang.org> | 2026-01-26 11:34:49 +0000 |
| commit | 5f254209184f4bcddc3155a85da7dc970f5708bf (patch) | |
| tree | a5de32cbcff938612aca50906e076aa50e023e42 | |
| parent | e410d938fa1b8474fd4763b9b8383defa7f1db3f (diff) | |
[ruby/prism] Fix `on_words_sep` for ripper translator with newlines
Ripper emits a token each per line.
https://github.com/ruby/prism/commit/4b5c9f5437
| -rw-r--r-- | lib/prism/lex_compat.rb | 9 | ||||
| -rw-r--r-- | lib/prism/lex_ripper.rb | 7 | ||||
| -rw-r--r-- | test/prism/ruby/ripper_test.rb | 8 |
3 files changed, 9 insertions, 15 deletions
diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index 0d78023eba..775be2759a 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -657,6 +657,15 @@ module Prism IgnoreStateToken.new([[lineno, column], event, value, lex_state]) when :on_embexpr_end IgnoreStateToken.new([[lineno, column], event, value, lex_state]) + when :on_words_sep + # Ripper emits one token each per line. + lines = value.lines + lines[0...-1].each do |whitespace| + tokens << Token.new([[lineno, column], event, whitespace, lex_state]) + lineno += 1 + column = 0 + end + Token.new([[lineno, column], event, lines.last, lex_state]) when :on_regexp_end # On regex end, Ripper scans and then sets end state, so the ripper # lexed output is begin, when it should be end. prism sets lex state diff --git a/lib/prism/lex_ripper.rb b/lib/prism/lex_ripper.rb index 2054cf55ac..f069e50ba9 100644 --- a/lib/prism/lex_ripper.rb +++ b/lib/prism/lex_ripper.rb @@ -26,13 +26,6 @@ module Prism results << token previous = token end - when :on_words_sep - if previous[1] == :on_words_sep - previous[2] << token[2] - else - results << token - previous = token - end else results << token previous = token diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb index 6e9dcee4c9..c8d259135f 100644 --- a/test/prism/ruby/ripper_test.rb +++ b/test/prism/ruby/ripper_test.rb @@ -62,18 +62,10 @@ module Prism ] omitted_lex = [ - "comments.txt", - "heredoc_percent_q_newline_delimiter.txt", "heredoc_with_escaped_newline_at_start.txt", "heredocs_with_fake_newlines.txt", "indented_file_end.txt", - "seattlerb/TestRubyParserShared.txt", - "seattlerb/class_comments.txt", - "seattlerb/module_comments.txt", - "seattlerb/parse_line_block_inline_comment_leading_newlines.txt", - "seattlerb/parse_line_block_inline_multiline_comment.txt", "spanning_heredoc_newlines.txt", - "strings.txt", "whitequark/dedenting_heredoc.txt", "whitequark/procarg0.txt", ] |