diff options
Diffstat (limited to 'lib/prism/translation/ruby_parser.rb')
| -rw-r--r-- | lib/prism/translation/ruby_parser.rb | 316 |
1 files changed, 227 insertions, 89 deletions
diff --git a/lib/prism/translation/ruby_parser.rb b/lib/prism/translation/ruby_parser.rb index 5c59fe3181..42bc5ee658 100644 --- a/lib/prism/translation/ruby_parser.rb +++ b/lib/prism/translation/ruby_parser.rb @@ -1,6 +1,17 @@ # frozen_string_literal: true +# :markup: markdown -require "ruby_parser" +begin + require "sexp" +rescue LoadError + warn(%q{Error: Unable to load sexp. Add `gem "sexp_processor"` to your Gemfile.}) + exit(1) +end + +class RubyParser # :nodoc: + class SyntaxError < RuntimeError # :nodoc: + end +end module Prism module Translation @@ -8,9 +19,9 @@ module Prism # seattlerb/ruby_parser gem's syntax tree. class RubyParser # A prism visitor that builds Sexp objects. - class Compiler < ::Prism::Compiler + class Compiler < ::Prism::Compiler # :nodoc: # This is the name of the file that we are compiling. We set it on every - # Sexp object that is generated, and also use it to compile __FILE__ + # Sexp object that is generated, and also use it to compile `__FILE__` # nodes. attr_reader :file @@ -50,7 +61,19 @@ module Prism # a and b # ^^^^^^^ def visit_and_node(node) - s(node, :and, visit(node.left), visit(node.right)) + left = visit(node.left) + + if left[0] == :and + # ruby_parser has the and keyword as right-associative as opposed to + # prism which has it as left-associative. We reverse that + # associativity here. + nest = left + nest = nest[2] while nest[2][0] == :and + nest[2] = s(node, :and, nest[2], visit(node.right)) + left + else + s(node, :and, left, visit(node.right)) + end end # [] @@ -114,7 +137,7 @@ module Prism # $+ # ^^ def visit_back_reference_read_node(node) - s(node, :back_ref, node.name.name.delete_prefix("$").to_sym) + s(node, :back_ref, node.name.to_s.delete_prefix("$").to_sym) end # begin end @@ -130,7 +153,7 @@ module Prism end current = node.rescue_clause - until (current = current.consequent).nil? + until (current = current.subsequent).nil? result << visit(current) end end @@ -246,6 +269,11 @@ module Prism when RegularExpressionNode, InterpolatedRegularExpressionNode return s(node, :match2, visit(node.receiver), visit(node.arguments.arguments.first)) end + + case node.arguments.arguments.first + when RegularExpressionNode, InterpolatedRegularExpressionNode + return s(node, :match3, visit(node.arguments.arguments.first), visit(node.receiver)) + end end end @@ -271,9 +299,9 @@ module Prism # ^^^^^^^^^^^^^^^ def visit_call_operator_write_node(node) if op_asgn?(node) - s(node, op_asgn_type(node, :op_asgn), visit(node.receiver), visit_write_value(node.value), node.read_name, node.operator) + s(node, op_asgn_type(node, :op_asgn), visit(node.receiver), visit_write_value(node.value), node.read_name, node.binary_operator) else - s(node, op_asgn_type(node, :op_asgn2), visit(node.receiver), node.write_name, node.operator, visit_write_value(node.value)) + s(node, op_asgn_type(node, :op_asgn2), visit(node.receiver), node.write_name, node.binary_operator, visit_write_value(node.value)) end end @@ -325,13 +353,13 @@ module Prism # case foo; when bar; end # ^^^^^^^^^^^^^^^^^^^^^^^ def visit_case_node(node) - s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.consequent) + s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.else_clause) end # case foo; in bar; end # ^^^^^^^^^^^^^^^^^^^^^ def visit_case_match_node(node) - s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.consequent) + s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.else_clause) end # class Foo; end @@ -344,14 +372,18 @@ module Prism visit(node.constant_path) end - if node.body.nil? - s(node, :class, name, visit(node.superclass)) - elsif node.body.is_a?(StatementsNode) - compiler = copy_compiler(in_def: false) - s(node, :class, name, visit(node.superclass)).concat(node.body.body.map { |child| child.accept(compiler) }) - else - s(node, :class, name, visit(node.superclass), node.body.accept(copy_compiler(in_def: false))) - end + result = + if node.body.nil? + s(node, :class, name, visit(node.superclass)) + elsif node.body.is_a?(StatementsNode) + compiler = copy_compiler(in_def: false) + s(node, :class, name, visit(node.superclass)).concat(node.body.body.map { |child| child.accept(compiler) }) + else + s(node, :class, name, visit(node.superclass), node.body.accept(copy_compiler(in_def: false))) + end + + attach_comments(result, node) + result end # @@foo @@ -362,9 +394,6 @@ module Prism # @@foo = 1 # ^^^^^^^^^ - # - # @@foo, @@bar = 1 - # ^^^^^ ^^^^^ def visit_class_variable_write_node(node) s(node, class_variable_write_type, node.name, visit_write_value(node.value)) end @@ -372,7 +401,7 @@ module Prism # @@foo += bar # ^^^^^^^^^^^^ def visit_class_variable_operator_write_node(node) - s(node, class_variable_write_type, node.name, s(node, :call, s(node, :cvar, node.name), node.operator, visit_write_value(node.value))) + s(node, class_variable_write_type, node.name, s(node, :call, s(node, :cvar, node.name), node.binary_operator, visit_write_value(node.value))) end # @@foo &&= bar @@ -417,7 +446,7 @@ module Prism # Foo += bar # ^^^^^^^^^^^ def visit_constant_operator_write_node(node) - s(node, :cdecl, node.name, s(node, :call, s(node, :const, node.name), node.operator, visit_write_value(node.value))) + s(node, :cdecl, node.name, s(node, :call, s(node, :const, node.name), node.binary_operator, visit_write_value(node.value))) end # Foo &&= bar @@ -442,9 +471,9 @@ module Prism # ^^^^^^^^ def visit_constant_path_node(node) if node.parent.nil? - s(node, :colon3, node.child.name) + s(node, :colon3, node.name) else - s(node, :colon2, visit(node.parent), node.child.name) + s(node, :colon2, visit(node.parent), node.name) end end @@ -460,7 +489,7 @@ module Prism # Foo::Bar += baz # ^^^^^^^^^^^^^^^ def visit_constant_path_operator_write_node(node) - s(node, :op_asgn, visit(node.target), node.operator, visit_write_value(node.value)) + s(node, :op_asgn, visit(node.target), node.binary_operator, visit_write_value(node.value)) end # Foo::Bar &&= baz @@ -480,9 +509,9 @@ module Prism def visit_constant_path_target_node(node) inner = if node.parent.nil? - s(node, :colon3, node.child.name) + s(node, :colon3, node.name) else - s(node, :colon2, visit(node.parent), node.child.name) + s(node, :colon2, visit(node.parent), node.name) end s(node, :const, inner) @@ -502,7 +531,9 @@ module Prism s(node, :defs, visit(node.receiver), name) end + attach_comments(result, node) result.line(node.name_loc.start_line) + if node.parameters.nil? result << s(node, :args).line(node.name_loc.start_line) else @@ -617,9 +648,6 @@ module Prism # $foo = 1 # ^^^^^^^^ - # - # $foo, $bar = 1 - # ^^^^ ^^^^ def visit_global_variable_write_node(node) s(node, :gasgn, node.name, visit_write_value(node.value)) end @@ -627,7 +655,7 @@ module Prism # $foo += bar # ^^^^^^^^^^^ def visit_global_variable_operator_write_node(node) - s(node, :gasgn, node.name, s(node, :call, s(node, :gvar, node.name), node.operator, visit(node.value))) + s(node, :gasgn, node.name, s(node, :call, s(node, :gvar, node.name), node.binary_operator, visit(node.value))) end # $foo &&= bar @@ -678,7 +706,7 @@ module Prism # foo ? bar : baz # ^^^^^^^^^^^^^^^ def visit_if_node(node) - s(node, :if, visit(node.predicate), visit(node.statements), visit(node.consequent)) + s(node, :if, visit(node.predicate), visit(node.statements), visit(node.subsequent)) end # 1i @@ -719,7 +747,7 @@ module Prism arglist << visit(node.block) if !node.block.nil? end - s(node, :op_asgn1, visit(node.receiver), arglist, node.operator, visit_write_value(node.value)) + s(node, :op_asgn1, visit(node.receiver), arglist, node.binary_operator, visit_write_value(node.value)) end # foo[bar] &&= baz @@ -765,9 +793,6 @@ module Prism # @foo = 1 # ^^^^^^^^ - # - # @foo, @bar = 1 - # ^^^^ ^^^^ def visit_instance_variable_write_node(node) s(node, :iasgn, node.name, visit_write_value(node.value)) end @@ -775,7 +800,7 @@ module Prism # @foo += bar # ^^^^^^^^^^^ def visit_instance_variable_operator_write_node(node) - s(node, :iasgn, node.name, s(node, :call, s(node, :ivar, node.name), node.operator, visit_write_value(node.value))) + s(node, :iasgn, node.name, s(node, :call, s(node, :ivar, node.name), node.binary_operator, visit_write_value(node.value))) end # @foo &&= bar @@ -805,17 +830,29 @@ module Prism # if /foo #{bar}/ then end # ^^^^^^^^^^^^ def visit_interpolated_match_last_line_node(node) - s(node, :match, s(node, :dregx).concat(visit_interpolated_parts(node.parts))) + parts = visit_interpolated_parts(node.parts) + regexp = + if parts.length == 1 + s(node, :lit, Regexp.new(parts.first, node.options)) + else + s(node, :dregx).concat(parts).tap do |result| + options = node.options + result << options if options != 0 + end + end + + s(node, :match, regexp) end # /foo #{bar}/ # ^^^^^^^^^^^^ def visit_interpolated_regular_expression_node(node) - if node.parts.all? { |part| part.is_a?(StringNode) || (part.is_a?(EmbeddedStatementsNode) && part.statements&.body&.length == 1 && part.statements.body.first.is_a?(StringNode)) } - unescaped = node.parts.map { |part| part.is_a?(StringNode) ? part.unescaped : part.statements.body.first.unescaped }.join - s(node, :lit, Regexp.new(unescaped, node.options)) + parts = visit_interpolated_parts(node.parts) + + if parts.length == 1 + s(node, :lit, Regexp.new(parts.first, node.options)) else - s(node, :dregx).concat(visit_interpolated_parts(node.parts)).tap do |result| + s(node, :dregx).concat(parts).tap do |result| options = node.options result << options if options != 0 end @@ -825,47 +862,102 @@ module Prism # "foo #{bar}" # ^^^^^^^^^^^^ def visit_interpolated_string_node(node) - if (node.parts.all? { |part| part.is_a?(StringNode) || (part.is_a?(EmbeddedStatementsNode) && part.statements&.body&.length == 1 && part.statements.body.first.is_a?(StringNode)) }) || - (node.opening.nil? && node.parts.all? { |part| part.is_a?(StringNode) && !part.opening_loc.nil? }) - unescaped = node.parts.map { |part| part.is_a?(StringNode) ? part.unescaped : part.statements.body.first.unescaped }.join - s(node, :str, unescaped) - else - s(node, :dstr).concat(visit_interpolated_parts(node.parts)) - end + parts = visit_interpolated_parts(node.parts) + parts.length == 1 ? s(node, :str, parts.first) : s(node, :dstr).concat(parts) end # :"foo #{bar}" # ^^^^^^^^^^^^^ def visit_interpolated_symbol_node(node) - if node.parts.all? { |part| part.is_a?(StringNode) || (part.is_a?(EmbeddedStatementsNode) && part.statements&.body&.length == 1 && part.statements.body.first.is_a?(StringNode)) } - unescaped = node.parts.map { |part| part.is_a?(StringNode) ? part.unescaped : part.statements.body.first.unescaped }.join - s(node, :lit, unescaped.to_sym) - else - s(node, :dsym).concat(visit_interpolated_parts(node.parts)) - end + parts = visit_interpolated_parts(node.parts) + parts.length == 1 ? s(node, :lit, parts.first.to_sym) : s(node, :dsym).concat(parts) end # `foo #{bar}` # ^^^^^^^^^^^^ def visit_interpolated_x_string_node(node) - children = visit_interpolated_parts(node.parts) - s(node.heredoc? ? node.parts.first : node, :dxstr).concat(children) + source = node.heredoc? ? node.parts.first : node + parts = visit_interpolated_parts(node.parts) + parts.length == 1 ? s(source, :xstr, parts.first) : s(source, :dxstr).concat(parts) end # Visit the interpolated content of the string-like node. private def visit_interpolated_parts(parts) - parts.each_with_object([]).with_index do |(part, results), index| - if index == 0 - if part.is_a?(StringNode) - results << part.unescaped + visited = [] + + parts.each do |part| + result = visit(part) + + if result[0] == :evstr && result[1] + if result[1][0] == :str + visited << result[1] + elsif result[1][0] == :dstr + visited.concat(result[1][1..-1]) else - results << "" - results << visit(part) + visited << result + end + visited << :space + elsif result[0] == :dstr + if !visited.empty? && part.parts[0].is_a?(StringNode) + # If we are in the middle of an implicitly concatenated string, + # we should not have a bare string as the first part. In this + # case we need to visit just that first part and then we can + # push the rest of the parts onto the visited array. + result[1] = visit(part.parts[0]) end + visited.concat(result[1..-1]) else - results << visit(part) + visited << result end end + + state = :beginning #: :beginning | :string_content | :interpolated_content + results = [] + + visited.each_with_index do |result, index| + case state + when :beginning + if result.is_a?(String) + results << result + state = :string_content + elsif result.is_a?(Array) && result[0] == :str + results << result[1] + state = :string_content + else + results << "" + results << result + state = :interpolated_content + end + when :string_content + if result == :space + # continue + elsif result.is_a?(String) + results[0] = "#{results[0]}#{result}" + elsif result.is_a?(Array) && result[0] == :str + results[0] = "#{results[0]}#{result[1]}" + else + results << result + state = :interpolated_content + end + when :interpolated_content + if result == :space + # continue + elsif visited[index - 1] != :space && result.is_a?(Array) && result[0] == :str && results[-1][0] == :str && (results[-1].line_max == result.line) + results[-1][1] = "#{results[-1][1]}#{result[1]}" + results[-1].line_max = result.line_max + else + results << result + end + end + end + + results + end + + # -> { it } + # ^^ + def visit_it_local_variable_read_node(node) + s(node, :call, nil, :it) end # foo(bar: baz) @@ -887,8 +979,8 @@ module Prism def visit_lambda_node(node) parameters = case node.parameters - when nil, NumberedParametersNode - s(node, :args) + when nil, ItParametersNode, NumberedParametersNode + 0 else visit(node.parameters) end @@ -912,9 +1004,6 @@ module Prism # foo = 1 # ^^^^^^^ - # - # foo, bar = 1 - # ^^^ ^^^ def visit_local_variable_write_node(node) s(node, :lasgn, node.name, visit_write_value(node.value)) end @@ -922,7 +1011,7 @@ module Prism # foo += bar # ^^^^^^^^^^ def visit_local_variable_operator_write_node(node) - s(node, :lasgn, node.name, s(node, :call, s(node, :lvar, node.name), node.operator, visit_write_value(node.value))) + s(node, :lasgn, node.name, s(node, :call, s(node, :lvar, node.name), node.binary_operator, visit_write_value(node.value))) end # foo &&= bar @@ -970,8 +1059,8 @@ module Prism # A node that is missing from the syntax tree. This is only used in the # case of a syntax error. The parser gem doesn't have such a concept, so # we invent our own here. - def visit_missing_node(node) - raise "Cannot visit missing node directly" + def visit_error_recovery_node(node) + raise "Cannot visit error recovery node directly" end # module Foo; end @@ -984,14 +1073,18 @@ module Prism visit(node.constant_path) end - if node.body.nil? - s(node, :module, name) - elsif node.body.is_a?(StatementsNode) - compiler = copy_compiler(in_def: false) - s(node, :module, name).concat(node.body.body.map { |child| child.accept(compiler) }) - else - s(node, :module, name, node.body.accept(copy_compiler(in_def: false))) - end + result = + if node.body.nil? + s(node, :module, name) + elsif node.body.is_a?(StatementsNode) + compiler = copy_compiler(in_def: false) + s(node, :module, name).concat(node.body.body.map { |child| child.accept(compiler) }) + else + s(node, :module, name, node.body.accept(copy_compiler(in_def: false))) + end + + attach_comments(result, node) + result end # foo, bar = baz @@ -1047,6 +1140,12 @@ module Prism s(node, :nil) end + # def foo(&nil); end + # ^^^^ + def visit_no_block_parameter_node(node) + :"&nil" + end + # def foo(**nil); end # ^^^^^ def visit_no_keywords_parameter_node(node) @@ -1080,14 +1179,26 @@ module Prism # a or b # ^^^^^^ def visit_or_node(node) - s(node, :or, visit(node.left), visit(node.right)) + left = visit(node.left) + + if left[0] == :or + # ruby_parser has the or keyword as right-associative as opposed to + # prism which has it as left-associative. We reverse that + # associativity here. + nest = left + nest = nest[2] while nest[2][0] == :or + nest[2] = s(node, :or, nest[2], visit(node.right)) + left + else + s(node, :or, left, visit(node.right)) + end end # def foo(bar, *baz); end # ^^^^^^^^^ def visit_parameters_node(node) children = - node.compact_child_nodes.map do |element| + node.each_child_node.map do |element| if element.is_a?(MultiTargetNode) visit_destructured_parameter(element) else @@ -1297,7 +1408,7 @@ module Prism # __FILE__ # ^^^^^^^^ def visit_source_file_node(node) - s(node, :str, file) + s(node, :str, node.filepath) end # __LINE__ @@ -1336,7 +1447,14 @@ module Prism # "foo" # ^^^^^ def visit_string_node(node) - s(node, :str, node.unescaped) + unescaped = node.unescaped + + if node.forced_binary_encoding? + unescaped = unescaped.dup + unescaped.force_encoding(Encoding::BINARY) + end + + s(node, :str, unescaped) end # super(foo) @@ -1378,7 +1496,7 @@ module Prism # bar unless foo # ^^^^^^^^^^^^^^ def visit_unless_node(node) - s(node, :if, visit(node.predicate), visit(node.consequent), visit(node.statements)) + s(node, :if, visit(node.predicate), visit(node.else_clause), visit(node.statements)) end # until foo; bar end @@ -1429,6 +1547,17 @@ module Prism private + # Attach prism comments to the given sexp. + def attach_comments(sexp, node) + return unless node.comments + return if node.comments.empty? + + extra = node.location.start_line - node.comments.last.location.start_line + comments = node.comments.map(&:slice) + comments.concat([nil] * [0, extra].max) + sexp.comments = comments.join("\n") + end + # Create a new compiler with the given options. def copy_compiler(in_def: self.in_def, in_pattern: self.in_pattern) Compiler.new(file, in_def: in_def, in_pattern: in_pattern) @@ -1451,7 +1580,7 @@ module Prism else parameters = case block.parameters - when nil, NumberedParametersNode + when nil, ItParametersNode, NumberedParametersNode 0 else visit(block.parameters) @@ -1498,13 +1627,21 @@ module Prism # Parse the given source and translate it into the seattlerb/ruby_parser # gem's Sexp format. def parse(source, filepath = "(string)") - translate(Prism.parse(source), filepath) + translate(Prism.parse(source, filepath: filepath, partial_script: true), filepath) end # Parse the given file and translate it into the seattlerb/ruby_parser # gem's Sexp format. def parse_file(filepath) - translate(Prism.parse_file(filepath), filepath) + translate(Prism.parse_file(filepath, partial_script: true), filepath) + end + + # Parse the give file and translate it into the + # seattlerb/ruby_parser gem's Sexp format. This method is + # provided for API compatibility to RubyParser and takes an + # optional +timeout+ argument. + def process(ruby, file = "(string)", timeout = nil) + Timeout.timeout(timeout) { parse(ruby, file) } end class << self @@ -1531,6 +1668,7 @@ module Prism raise ::RubyParser::SyntaxError, "#{filepath}:#{error.location.start_line} :: #{error.message}" end + result.attach_comments! result.value.accept(Compiler.new(filepath)) end end |