From 5f254209184f4bcddc3155a85da7dc970f5708bf Mon Sep 17 00:00:00 2001 From: Earlopain <14981592+Earlopain@users.noreply.github.com> Date: Mon, 26 Jan 2026 12:16:34 +0100 Subject: [ruby/prism] Fix `on_words_sep` for ripper translator with newlines Ripper emits a token each per line. https://github.com/ruby/prism/commit/4b5c9f5437 --- lib/prism/lex_compat.rb | 9 +++++++++ lib/prism/lex_ripper.rb | 7 ------- 2 files changed, 9 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index 0d78023eba..775be2759a 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -657,6 +657,15 @@ module Prism IgnoreStateToken.new([[lineno, column], event, value, lex_state]) when :on_embexpr_end IgnoreStateToken.new([[lineno, column], event, value, lex_state]) + when :on_words_sep + # Ripper emits one token each per line. + lines = value.lines + lines[0...-1].each do |whitespace| + tokens << Token.new([[lineno, column], event, whitespace, lex_state]) + lineno += 1 + column = 0 + end + Token.new([[lineno, column], event, lines.last, lex_state]) when :on_regexp_end # On regex end, Ripper scans and then sets end state, so the ripper # lexed output is begin, when it should be end. prism sets lex state diff --git a/lib/prism/lex_ripper.rb b/lib/prism/lex_ripper.rb index 2054cf55ac..f069e50ba9 100644 --- a/lib/prism/lex_ripper.rb +++ b/lib/prism/lex_ripper.rb @@ -26,13 +26,6 @@ module Prism results << token previous = token end - when :on_words_sep - if previous[1] == :on_words_sep - previous[2] << token[2] - else - results << token - previous = token - end else results << token previous = token -- cgit v1.2.3