diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/prism.rb | 8 | ||||
| -rw-r--r-- | lib/prism/lex_compat.rb | 101 | ||||
| -rw-r--r-- | lib/prism/lex_ripper.rb | 2 | ||||
| -rw-r--r-- | lib/prism/translation/ripper.rb | 81 | ||||
| -rw-r--r-- | lib/rubygems/ext/builder.rb | 5 | ||||
| -rw-r--r-- | lib/rubygems/specification_policy.rb | 8 | ||||
| -rw-r--r-- | lib/syntax_suggest/code_line.rb | 19 | ||||
| -rw-r--r-- | lib/syntax_suggest/version.rb | 2 |
8 files changed, 159 insertions, 67 deletions
diff --git a/lib/prism.rb b/lib/prism.rb index d809557fce..dab3420377 100644 --- a/lib/prism.rb +++ b/lib/prism.rb @@ -61,8 +61,7 @@ module Prism # Prism::lex_compat(source, **options) -> LexCompat::Result # # Returns a parse result whose value is an array of tokens that closely - # resembles the return value of Ripper::lex. The main difference is that the - # `:on_sp` token is not emitted. + # resembles the return value of Ripper::lex. # # For supported options, see Prism::parse. def self.lex_compat(source, **options) @@ -72,9 +71,8 @@ module Prism # :call-seq: # Prism::lex_ripper(source) -> Array # - # This lexes with the Ripper lex. It drops any space events but otherwise - # returns the same tokens. Raises SyntaxError if the syntax in source is - # invalid. + # This wraps the result of Ripper.lex. It produces almost exactly the + # same tokens. Raises SyntaxError if the syntax in source is invalid. def self.lex_ripper(source) LexRipper.new(source).result # steep:ignore end diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index f7b9a0effc..597e63c73e 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -226,7 +226,7 @@ module Prism end # Tokens where state should be ignored - # used for :on_comment, :on_heredoc_end, :on_embexpr_end + # used for :on_sp, :on_comment, :on_heredoc_end, :on_embexpr_end class IgnoreStateToken < Token def ==(other) # :nodoc: self[0...-1] == other[0...-1] @@ -611,10 +611,10 @@ module Prism BOM_FLUSHED = RUBY_VERSION >= "3.3.0" private_constant :BOM_FLUSHED - attr_reader :source, :options + attr_reader :options - def initialize(source, **options) - @source = source + def initialize(code, **options) + @code = code @options = options end @@ -624,12 +624,14 @@ module Prism state = :default heredoc_stack = [[]] #: Array[Array[Heredoc::PlainHeredoc | Heredoc::DashHeredoc | Heredoc::DedentingHeredoc]] - result = Prism.lex(source, **options) + result = Prism.lex(@code, **options) + source = result.source result_value = result.value previous_state = nil #: State? last_heredoc_end = nil #: Integer? + eof_token = nil - bom = source.byteslice(0..2) == "\xEF\xBB\xBF" + bom = source.slice(0, 3) == "\xEF\xBB\xBF" result_value.each_with_index do |(token, lex_state), index| lineno = token.location.start_line @@ -741,6 +743,7 @@ module Prism Token.new([[lineno, column], event, value, lex_state]) when :on_eof + eof_token = token previous_token = result_value[index - 1][0] # If we're at the end of the file and the previous token was a @@ -763,7 +766,7 @@ module Prism end_offset += 3 end - tokens << Token.new([[lineno, 0], :on_nl, source.byteslice(start_offset...end_offset), lex_state]) + tokens << Token.new([[lineno, 0], :on_nl, source.slice(start_offset, end_offset - start_offset), lex_state]) end end @@ -857,7 +860,89 @@ module Prism # We sort by location to compare against Ripper's output tokens.sort_by!(&:location) - Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, Source.for(source)) + # Add :on_sp tokens + tokens = add_on_sp_tokens(tokens, source, result.data_loc, bom, eof_token) + + Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, source) + end + + def add_on_sp_tokens(tokens, source, data_loc, bom, eof_token) + new_tokens = [] + + prev_token_state = Translation::Ripper::Lexer::State.cached(Translation::Ripper::EXPR_BEG) + prev_token_end = bom ? 3 : 0 + + tokens.each do |token| + line, column = token.location + start_offset = source.line_to_byte_offset(line) + column + # Ripper reports columns on line 1 without counting the BOM, so we adjust to get the real offset + start_offset += 3 if line == 1 && bom + + if start_offset > prev_token_end + sp_value = source.slice(prev_token_end, start_offset - prev_token_end) + sp_line = source.line(prev_token_end) + sp_column = source.column(prev_token_end) + # Ripper reports columns on line 1 without counting the BOM + sp_column -= 3 if sp_line == 1 && bom + continuation_index = sp_value.byteindex("\\") + + # ripper emits up to three :on_sp tokens when line continuations are used + if continuation_index + next_whitespace_index = continuation_index + 1 + next_whitespace_index += 1 if sp_value.byteslice(next_whitespace_index) == "\r" + next_whitespace_index += 1 + first_whitespace = sp_value[0...continuation_index] + continuation = sp_value[continuation_index...next_whitespace_index] + second_whitespace = sp_value[next_whitespace_index..] + + new_tokens << IgnoreStateToken.new([ + [sp_line, sp_column], + :on_sp, + first_whitespace, + prev_token_state + ]) unless first_whitespace.empty? + + new_tokens << IgnoreStateToken.new([ + [sp_line, sp_column + continuation_index], + :on_sp, + continuation, + prev_token_state + ]) + + new_tokens << IgnoreStateToken.new([ + [sp_line + 1, 0], + :on_sp, + second_whitespace, + prev_token_state + ]) unless second_whitespace.empty? + else + new_tokens << IgnoreStateToken.new([ + [sp_line, sp_column], + :on_sp, + sp_value, + prev_token_state + ]) + end + end + + new_tokens << token + prev_token_state = token.state + prev_token_end = start_offset + token.value.bytesize + end + + unless data_loc # no trailing :on_sp with __END__ as it is always preceded by :on_nl + end_offset = eof_token.location.end_offset + if prev_token_end < end_offset + new_tokens << IgnoreStateToken.new([ + [source.line(prev_token_end), source.column(prev_token_end)], + :on_sp, + source.slice(prev_token_end, end_offset - prev_token_end), + prev_token_state + ]) + end + end + + new_tokens end end diff --git a/lib/prism/lex_ripper.rb b/lib/prism/lex_ripper.rb index 4b5c3b77fd..2054cf55ac 100644 --- a/lib/prism/lex_ripper.rb +++ b/lib/prism/lex_ripper.rb @@ -19,8 +19,6 @@ module Prism lex(source).each do |token| case token[1] - when :on_sp - # skip when :on_tstring_content if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@")) previous[2] << token[2] diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb index c8f9fa7731..735217d2e0 100644 --- a/lib/prism/translation/ripper.rb +++ b/lib/prism/translation/ripper.rb @@ -832,7 +832,7 @@ module Prism # foo(bar) # ^^^ def visit_arguments_node(node) - arguments, _ = visit_call_node_arguments(node, nil, false) + arguments, _, _ = visit_call_node_arguments(node, nil, false) arguments end @@ -1042,16 +1042,16 @@ module Prism case node.name when :[] receiver = visit(node.receiver) - arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) bounds(node.location) call = on_aref(receiver, arguments) - if block.nil? - call - else + if has_ripper_block bounds(node.location) on_method_add_block(call, block) + else + call end when :[]= receiver = visit(node.receiver) @@ -1110,9 +1110,9 @@ module Prism if node.variable_call? on_vcall(message) else - arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location)) + arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location)) call = - if node.opening_loc.nil? && arguments&.any? + if node.opening_loc.nil? && get_arguments_and_block(node.arguments, node.block).first.any? bounds(node.location) on_command(message, arguments) elsif !node.opening_loc.nil? @@ -1123,11 +1123,11 @@ module Prism on_method_add_arg(on_fcall(message), on_args_new) end - if block.nil? - call - else + if has_ripper_block bounds(node.block.location) on_method_add_block(call, block) + else + call end end end @@ -1151,7 +1151,7 @@ module Prism bounds(node.location) on_assign(on_field(receiver, call_operator, message), value) else - arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location)) + arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location)) call = if node.opening_loc.nil? bounds(node.location) @@ -1169,27 +1169,35 @@ module Prism on_method_add_arg(on_call(receiver, call_operator, message), arguments) end - if block.nil? - call - else + if has_ripper_block bounds(node.block.location) on_method_add_block(call, block) + else + call end end end end - # Visit the arguments and block of a call node and return the arguments - # and block as they should be used. - private def visit_call_node_arguments(arguments_node, block_node, trailing_comma) + # Extract the arguments and block Ripper-style, which means if the block + # is like `&b` then it's moved to arguments. + private def get_arguments_and_block(arguments_node, block_node) arguments = arguments_node&.arguments || [] block = block_node if block.is_a?(BlockArgumentNode) - arguments << block + arguments += [block] block = nil end + [arguments, block] + end + + # Visit the arguments and block of a call node and return the arguments + # and block as they should be used. + private def visit_call_node_arguments(arguments_node, block_node, trailing_comma) + arguments, block = get_arguments_and_block(arguments_node, block_node) + [ if arguments.length == 1 && arguments.first.is_a?(ForwardingArgumentsNode) visit(arguments.first) @@ -1203,7 +1211,8 @@ module Prism on_args_add_block(args, false) end end, - visit(block) + visit(block), + block != nil, ] end @@ -1640,10 +1649,10 @@ module Prism end bounds(node.location) - if receiver.nil? - on_def(name, parameters, bodystmt) - else + if receiver on_defs(receiver, operator, name, parameters, bodystmt) + else + on_def(name, parameters, bodystmt) end end @@ -2041,7 +2050,7 @@ module Prism # ^^^^^^^^^^^^^^^ def visit_index_operator_write_node(node) receiver = visit(node.receiver) - arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) bounds(node.location) target = on_aref_field(receiver, arguments) @@ -2058,7 +2067,7 @@ module Prism # ^^^^^^^^^^^^^^^^ def visit_index_and_write_node(node) receiver = visit(node.receiver) - arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) bounds(node.location) target = on_aref_field(receiver, arguments) @@ -2075,7 +2084,7 @@ module Prism # ^^^^^^^^^^^^^^^^ def visit_index_or_write_node(node) receiver = visit(node.receiver) - arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) bounds(node.location) target = on_aref_field(receiver, arguments) @@ -2092,7 +2101,7 @@ module Prism # ^^^^^^^^ def visit_index_target_node(node) receiver = visit(node.receiver) - arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + arguments, _, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) bounds(node.location) on_aref_field(receiver, arguments) @@ -3122,7 +3131,7 @@ module Prism # super(foo) # ^^^^^^^^^^ def visit_super_node(node) - arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.rparen_loc || node.location)) + arguments, block, has_ripper_block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.rparen_loc || node.location)) if !node.lparen_loc.nil? bounds(node.lparen_loc) @@ -3132,11 +3141,11 @@ module Prism bounds(node.location) call = on_super(arguments) - if block.nil? - call - else + if has_ripper_block bounds(node.block.location) on_method_add_block(call, block) + else + call end end @@ -3446,12 +3455,12 @@ module Prism # :stopdoc: def _dispatch_0; end - def _dispatch_1(_); end - def _dispatch_2(_, _); end - def _dispatch_3(_, _, _); end - def _dispatch_4(_, _, _, _); end - def _dispatch_5(_, _, _, _, _); end - def _dispatch_7(_, _, _, _, _, _, _); end + def _dispatch_1(arg); arg end + def _dispatch_2(arg, _); arg end + def _dispatch_3(arg, _, _); arg end + def _dispatch_4(arg, _, _, _); arg end + def _dispatch_5(arg, _, _, _, _); arg end + def _dispatch_7(arg, _, _, _, _, _, _); arg end # :startdoc: # diff --git a/lib/rubygems/ext/builder.rb b/lib/rubygems/ext/builder.rb index 350daf1e16..62d36bcf48 100644 --- a/lib/rubygems/ext/builder.rb +++ b/lib/rubygems/ext/builder.rb @@ -163,8 +163,6 @@ class Gem::Ext::Builder @gem_dir = spec.full_gem_path @target_rbconfig = target_rbconfig @build_jobs = build_jobs - - @ran_rake = false end ## @@ -177,7 +175,6 @@ class Gem::Ext::Builder when /configure/ then Gem::Ext::ConfigureBuilder when /rakefile/i, /mkrf_conf/i then - @ran_rake = true Gem::Ext::RakeBuilder when /CMakeLists.txt/ then Gem::Ext::CmakeBuilder.new @@ -250,8 +247,6 @@ EOF FileUtils.rm_f @spec.gem_build_complete_path @spec.extensions.each do |extension| - break if @ran_rake - build_extension extension, dest_path end diff --git a/lib/rubygems/specification_policy.rb b/lib/rubygems/specification_policy.rb index e5008a24db..0fcb635394 100644 --- a/lib/rubygems/specification_policy.rb +++ b/lib/rubygems/specification_policy.rb @@ -436,6 +436,7 @@ or set it to nil if you don't want to specify a license. warning "deprecated autorequire specified" if @specification.autorequire @specification.executables.each do |executable| + validate_executable(executable) validate_shebang_line_in(executable) end @@ -449,6 +450,13 @@ or set it to nil if you don't want to specify a license. warning("no #{attribute} specified") if value.nil? || value.empty? end + def validate_executable(executable) + separators = [File::SEPARATOR, File::ALT_SEPARATOR, File::PATH_SEPARATOR].compact.map {|sep| Regexp.escape(sep) }.join + return unless executable.match?(/[\s#{separators}]/) + + error "executable \"#{executable}\" contains invalid characters" + end + def validate_shebang_line_in(executable) executable_path = File.join(@specification.bindir, executable) return if File.read(executable_path, 2) == "#!" diff --git a/lib/syntax_suggest/code_line.rb b/lib/syntax_suggest/code_line.rb index 58197e95d0..76ca892ac3 100644 --- a/lib/syntax_suggest/code_line.rb +++ b/lib/syntax_suggest/code_line.rb @@ -180,18 +180,17 @@ module SyntaxSuggest # EOM # expect(lines.first.trailing_slash?).to eq(true) # - if SyntaxSuggest.use_prism_parser? - def trailing_slash? - last = @lex.last - last&.type == :on_tstring_end - end - else - def trailing_slash? - last = @lex.last - return false unless last - return false unless last.type == :on_sp + def trailing_slash? + last = @lex.last + # Older versions of prism diverged slightly from Ripper in compatibility mode + case last&.type + when :on_sp last.token == TRAILING_SLASH + when :on_tstring_end + true + else + false end end diff --git a/lib/syntax_suggest/version.rb b/lib/syntax_suggest/version.rb index 1aa908f4e5..db50a1a89a 100644 --- a/lib/syntax_suggest/version.rb +++ b/lib/syntax_suggest/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxSuggest - VERSION = "2.0.2" + VERSION = "2.0.3" end |