From 5f254209184f4bcddc3155a85da7dc970f5708bf Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Mon, 26 Jan 2026 12:16:34 +0100 Subject: [ruby/prism] Fix `on_words_sep` for ripper translator with newlines Ripper emits a token each per line. https://github.com/ruby/prism/commit/4b5c9f5437 --- lib/prism/lex_compat.rb | 9 +++++++++ lib/prism/lex_ripper.rb | 7 ------- test/prism/ruby/ripper_test.rb | 8 -------- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index 0d78023eba..775be2759a 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -657,6 +657,15 @@ module Prism IgnoreStateToken.new([[lineno, column], event, value, lex_state]) when :on_embexpr_end IgnoreStateToken.new([[lineno, column], event, value, lex_state]) + when :on_words_sep + # Ripper emits one token each per line. + lines = value.lines + lines[0...-1].each do |whitespace| + tokens << Token.new([[lineno, column], event, whitespace, lex_state]) + lineno += 1 + column = 0 + end + Token.new([[lineno, column], event, lines.last, lex_state]) when :on_regexp_end # On regex end, Ripper scans and then sets end state, so the ripper # lexed output is begin, when it should be end. prism sets lex state diff --git a/lib/prism/lex_ripper.rb b/lib/prism/lex_ripper.rb index 2054cf55ac..f069e50ba9 100644 --- a/lib/prism/lex_ripper.rb +++ b/lib/prism/lex_ripper.rb @@ -26,13 +26,6 @@ module Prism results << token previous = token end - when :on_words_sep - if previous[1] == :on_words_sep - previous[2] << token[2] - else - results << token - previous = token - end else results << token previous = token diff --git a/test/prism/ruby/ripper_test.rb b/test/prism/ruby/ripper_test.rb index 6e9dcee4c9..c8d259135f 100644 --- a/test/prism/ruby/ripper_test.rb +++ b/test/prism/ruby/ripper_test.rb @@ -62,18 +62,10 @@ module Prism ] omitted_lex = [ - "comments.txt", - "heredoc_percent_q_newline_delimiter.txt", "heredoc_with_escaped_newline_at_start.txt", "heredocs_with_fake_newlines.txt", "indented_file_end.txt", - "seattlerb/TestRubyParserShared.txt", - "seattlerb/class_comments.txt", - "seattlerb/module_comments.txt", - "seattlerb/parse_line_block_inline_comment_leading_newlines.txt", - "seattlerb/parse_line_block_inline_multiline_comment.txt", "spanning_heredoc_newlines.txt", - "strings.txt", "whitequark/dedenting_heredoc.txt", "whitequark/procarg0.txt", ] -- cgit v1.2.3