From c9da8d67fdb9fab82f76d583239f5b9761f60350 Mon Sep 17 00:00:00 2001 From: Koichi ITO Date: Sat, 16 Mar 2024 00:30:01 +0900 Subject: [ruby/prism] Fix a token incompatibility for `Prism::Translation::Parser::Lexer` This PR fixes a token incompatibility between Parser gem and `Prism::Translation::Parser` for the heredocs_leading_whitespace.txt test. https://github.com/ruby/prism/commit/7d45fb1eed --- lib/prism/translation/parser/lexer.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb index 7febca449e..9e5d27ef29 100644 --- a/lib/prism/translation/parser/lexer.rb +++ b/lib/prism/translation/parser/lexer.rb @@ -217,6 +217,8 @@ module Prism index = 0 length = lexed.length + heredoc_identifier_stack = [] + while index < length token, state = lexed[index] index += 1 @@ -275,6 +277,9 @@ module Prism when :tSPACE value = nil when :tSTRING_BEG + if token.type == :HEREDOC_START + heredoc_identifier_stack.push(value.match(/<<[-~]?["']?(?.*?)["']?\z/)[:heredoc_identifier]) + end if ["\"", "'"].include?(value) && (next_token = lexed[index][0]) && next_token.type == :STRING_END next_location = token.location.join(next_token.location) type = :tSTRING @@ -322,7 +327,7 @@ module Prism when :tSTRING_END if token.type == :HEREDOC_END && value.end_with?("\n") newline_length = value.end_with?("\r\n") ? 2 : 1 - value = value.sub(/\r?\n\z/, '') + value = heredoc_identifier_stack.pop location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.end_offset - newline_length]) elsif token.type == :REGEXP_END value = value[0] -- cgit v1.2.3