diff options
Diffstat (limited to 'spec/syntax_suggest')
30 files changed, 14962 insertions, 0 deletions
diff --git a/spec/syntax_suggest/fixtures/derailed_require_tree.rb.txt b/spec/syntax_suggest/fixtures/derailed_require_tree.rb.txt new file mode 100644 index 0000000000..668ac4010b --- /dev/null +++ b/spec/syntax_suggest/fixtures/derailed_require_tree.rb.txt @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +# Tree structure used to store and sort require memory costs +# RequireTree.new('get_process_mem') +module DerailedBenchmarks + class RequireTree + REQUIRED_BY = {} + + attr_reader :name + attr_writer :cost + attr_accessor :parent + + def initialize(name) + @name = name + @children = {} + @cost = 0 + + def self.reset! + REQUIRED_BY.clear + if defined?(Kernel::REQUIRE_STACK) + Kernel::REQUIRE_STACK.clear + + Kernel::REQUIRE_STACK.push(TOP_REQUIRE) + end + end + + def <<(tree) + @children[tree.name.to_s] = tree + tree.parent = self + (REQUIRED_BY[tree.name.to_s] ||= []) << self.name + end + + def [](name) + @children[name.to_s] + end + + # Returns array of child nodes + def children + @children.values + end + + def cost + @cost || 0 + end + + # Returns sorted array of child nodes from Largest to Smallest + def sorted_children + children.sort { |c1, c2| c2.cost <=> c1.cost } + end + + def to_string + str = String.new("#{name}: #{cost.round(4)} MiB") + if parent && REQUIRED_BY[self.name.to_s] + names = REQUIRED_BY[self.name.to_s].uniq - [parent.name.to_s] + if names.any? + str << " (Also required by: #{ names.first(2).join(", ") }" + str << ", and #{names.count - 2} others" if names.count > 3 + str << ")" + end + end + str + end + + # Recursively prints all child nodes + def print_sorted_children(level = 0, out = STDOUT) + return if cost < ENV['CUT_OFF'].to_f + out.puts " " * level + self.to_string + level += 1 + sorted_children.each do |child| + child.print_sorted_children(level, out) + end + end + end +end diff --git a/spec/syntax_suggest/fixtures/rexe.rb.txt b/spec/syntax_suggest/fixtures/rexe.rb.txt new file mode 100755 index 0000000000..92e44d4d1e --- /dev/null +++ b/spec/syntax_suggest/fixtures/rexe.rb.txt @@ -0,0 +1,569 @@ +#!/usr/bin/env ruby +# +# rexe - Ruby Command Line Executor Filter +# +# Inspired by https://github.com/thisredone/rb + +# frozen_string_literal: true + + +require 'bundler' +require 'date' +require 'optparse' +require 'ostruct' +require 'shellwords' + +class Rexe + + VERSION = '1.5.1' + + PROJECT_URL = 'https://github.com/keithrbennett/rexe' + + + module Helpers + + # Try executing code. If error raised, print message (but not stack trace) & exit -1. + def try + begin + yield + rescue Exception => e + unless e.class == SystemExit + $stderr.puts("rexe: #{e}") + $stderr.puts("Use the -h option to get help.") + exit(-1) + end + end + end + end + + + class Options < Struct.new( + :input_filespec, + :input_format, + :input_mode, + :loads, + :output_format, + :output_format_tty, + :output_format_block, + :requires, + :log_format, + :noop) + + + def initialize + super + clear + end + + + def clear + self.input_filespec = nil + self.input_format = :none + self.input_mode = :none + self.output_format = :none + self.output_format_tty = :none + self.output_format_block = :none + self.loads = [] + self.requires = [] + self.log_format = :none + self.noop = false + end + end + + + + + + class Lookups + def input_modes + @input_modes ||= { + 'l' => :line, + 'e' => :enumerator, + 'b' => :one_big_string, + 'n' => :none + } + end + + + def input_formats + @input_formats ||= { + 'j' => :json, + 'm' => :marshal, + 'n' => :none, + 'y' => :yaml, + } + end + + + def input_parsers + @input_parsers ||= { + json: ->(string) { JSON.parse(string) }, + marshal: ->(string) { Marshal.load(string) }, + none: ->(string) { string }, + yaml: ->(string) { YAML.load(string) }, + } + end + + + def output_formats + @output_formats ||= { + 'a' => :amazing_print, + 'i' => :inspect, + 'j' => :json, + 'J' => :pretty_json, + 'm' => :marshal, + 'n' => :none, + 'p' => :puts, # default + 'P' => :pretty_print, + 's' => :to_s, + 'y' => :yaml, + } + end + + + def formatters + @formatters ||= { + amazing_print: ->(obj) { obj.ai + "\n" }, + inspect: ->(obj) { obj.inspect + "\n" }, + json: ->(obj) { obj.to_json }, + marshal: ->(obj) { Marshal.dump(obj) }, + none: ->(_obj) { nil }, + pretty_json: ->(obj) { JSON.pretty_generate(obj) }, + pretty_print: ->(obj) { obj.pretty_inspect }, + puts: ->(obj) { require 'stringio'; sio = StringIO.new; sio.puts(obj); sio.string }, + to_s: ->(obj) { obj.to_s + "\n" }, + yaml: ->(obj) { obj.to_yaml }, + } + end + + + def format_requires + @format_requires ||= { + json: 'json', + pretty_json: 'json', + amazing_print: 'amazing_print', + pretty_print: 'pp', + yaml: 'yaml' + } + end + end + + + + class CommandLineParser + + include Helpers + + attr_reader :lookups, :options + + def initialize + @lookups = Lookups.new + @options = Options.new + end + + + # Inserts contents of REXE_OPTIONS environment variable at the beginning of ARGV. + private def prepend_environment_options + env_opt_string = ENV['REXE_OPTIONS'] + if env_opt_string + args_to_prepend = Shellwords.shellsplit(env_opt_string) + ARGV.unshift(args_to_prepend).flatten! + end + end + + + private def add_format_requires_to_requires_list + formats = [options.input_format, options.output_format, options.log_format] + requires = formats.map { |format| lookups.format_requires[format] }.uniq.compact + requires.each { |r| options.requires << r } + end + + + private def help_text + unless @help_text + @help_text ||= <<~HEREDOC + + rexe -- Ruby Command Line Executor/Filter -- v#{VERSION} -- #{PROJECT_URL} + + Executes Ruby code on the command line, + optionally automating management of standard input and standard output, + and optionally parsing input and formatting output with YAML, JSON, etc. + + rexe [options] [Ruby source code] + + Options: + + -c --clear_options Clear all previous command line options specified up to now + -f --input_file Use this file instead of stdin for preprocessed input; + if filespec has a YAML and JSON file extension, + sets input format accordingly and sets input mode to -mb + -g --log_format FORMAT Log format, logs to stderr, defaults to -gn (none) + (see -o for format options) + -h, --help Print help and exit + -i, --input_format FORMAT Input format, defaults to -in (None) + -ij JSON + -im Marshal + -in None (default) + -iy YAML + -l, --load RUBY_FILE(S) Ruby file(s) to load, comma separated; + ! to clear all, or precede a name with '-' to remove + -m, --input_mode MODE Input preprocessing mode (determines what `self` will be) + defaults to -mn (none) + -ml line; each line is ingested as a separate string + -me enumerator (each_line on STDIN or File) + -mb big string; all lines combined into one string + -mn none (default); no input preprocessing; + self is an Object.new + -n, --[no-]noop Do not execute the code (useful with -g); + For true: yes, true, y, +; for false: no, false, n + -o, --output_format FORMAT Output format, defaults to -on (no output): + -oa Amazing Print + -oi Inspect + -oj JSON + -oJ Pretty JSON + -om Marshal + -on No Output (default) + -op Puts + -oP Pretty Print + -os to_s + -oy YAML + If 2 letters are provided, 1st is for tty devices, 2nd for block + --project-url Outputs project URL on Github, then exits + -r, --require REQUIRE(S) Gems and built-in libraries to require, comma separated; + ! to clear all, or precede a name with '-' to remove + -v, --version Prints version and exits + + --------------------------------------------------------------------------------------- + + In many cases you will need to enclose your source code in single or double quotes. + + If source code is not specified, it will default to 'self', + which is most likely useful only in a filter mode (-ml, -me, -mb). + + If there is a .rexerc file in your home directory, it will be run as Ruby code + before processing the input. + + If there is a REXE_OPTIONS environment variable, its content will be prepended + to the command line so that you can specify options implicitly + (e.g. `export REXE_OPTIONS="-r amazing_print,yaml"`) + + HEREDOC + + @help_text.freeze + end + + @help_text + end + + + # File file input mode; detects the input mode (JSON, YAML, or None) from the extension. + private def autodetect_file_format(filespec) + extension = File.extname(filespec).downcase + if extension == '.json' + :json + elsif extension == '.yml' || extension == '.yaml' + :yaml + else + :none + end + end + + + private def open_resource(resource_identifier) + command = case (`uname`.chomp) + when 'Darwin' + 'open' + when 'Linux' + 'xdg-open' + else + 'start' + end + + `#{command} #{resource_identifier}` + end + + + # Using 'optparse', parses the command line. + # Settings go into this instance's properties (see Struct declaration). + def parse + + prepend_environment_options + + OptionParser.new do |parser| + + parser.on('-c', '--clear_options', "Clear all previous command line options") do |v| + options.clear + end + + parser.on('-f', '--input_file FILESPEC', + 'Use this file instead of stdin; autodetects YAML and JSON file extensions') do |v| + unless File.exist?(v) + raise "File #{v} does not exist." + end + options.input_filespec = v + options.input_format = autodetect_file_format(v) + if [:json, :yaml].include?(options.input_format) + options.input_mode = :one_big_string + end + end + + parser.on('-g', '--log_format FORMAT', 'Log format, logs to stderr, defaults to none (see -o for format options)') do |v| + options.log_format = lookups.output_formats[v] + if options.log_format.nil? + raise("Output mode was '#{v}' but must be one of #{lookups.output_formats.keys}.") + end + end + + parser.on("-h", "--help", "Show help") do |_help_requested| + puts help_text + exit + end + + parser.on('-i', '--input_format FORMAT', + 'Mode with which to parse input values (n = none (default), j = JSON, m = Marshal, y = YAML') do |v| + + options.input_format = lookups.input_formats[v] + if options.input_format.nil? + raise("Input mode was '#{v}' but must be one of #{lookups.input_formats.keys}.") + end + end + + parser.on('-l', '--load RUBY_FILE(S)', 'Ruby file(s) to load, comma separated, or ! to clear') do |v| + if v == '!' + options.loads.clear + else + loadfiles = v.split(',').map(&:strip).map { |s| File.expand_path(s) } + removes, adds = loadfiles.partition { |filespec| filespec[0] == '-' } + + existent, nonexistent = adds.partition { |filespec| File.exists?(filespec) } + if nonexistent.any? + raise("\nDid not find the following files to load: #{nonexistent}\n\n") + else + existent.each { |filespec| options.loads << filespec } + end + + removes.each { |filespec| options.loads -= [filespec[1..-1]] } + end + end + + parser.on('-m', '--input_mode MODE', + 'Mode with which to handle input (-ml, -me, -mb, -mn (default)') do |v| + + options.input_mode = lookups.input_modes[v] + if options.input_mode.nil? + raise("Input mode was '#{v}' but must be one of #{lookups.input_modes.keys}.") + end + end + + # See https://stackoverflow.com/questions/54576873/ruby-optionparser-short-code-for-boolean-option + # for an excellent explanation of this optparse incantation. + # According to the answer, valid options are: + # -n no, -n yes, -n false, -n true, -n n, -n y, -n +, but not -n -. + parser.on('-n', '--[no-]noop [FLAG]', TrueClass, "Do not execute the code (useful with -g)") do |v| + options.noop = (v.nil? ? true : v) + end + + parser.on('-o', '--output_format FORMAT', + 'Mode with which to format values for output (`-o` + [aijJmnpsy])') do |v| + options.output_format_tty = lookups.output_formats[v[0]] + options.output_format_block = lookups.output_formats[v[-1]] + options.output_format = ($stdout.tty? ? options.output_format_tty : options.output_format_block) + if [options.output_format_tty, options.output_format_block].include?(nil) + raise("Bad output mode '#{v}'; each must be one of #{lookups.output_formats.keys}.") + end + end + + parser.on('-r', '--require REQUIRE(S)', + 'Gems and built-in libraries (e.g. shellwords, yaml) to require, comma separated, or ! to clear') do |v| + if v == '!' + options.requires.clear + else + v.split(',').map(&:strip).each do |r| + if r[0] == '-' + options.requires -= [r[1..-1]] + else + options.requires << r + end + end + end + end + + parser.on('-v', '--version', 'Print version') do + puts VERSION + exit(0) + end + + # Undocumented feature: open Github project with default web browser on a Mac + parser.on('', '--open-project') do + open_resource(PROJECT_URL) + exit(0) + end + + parser.on('', '--project-url') do + puts PROJECT_URL + exit(0) + end + + end.parse! + + # We want to do this after all options have been processed because we don't want any clearing of the + # options (by '-c', etc.) to result in exclusion of these needed requires. + add_format_requires_to_requires_list + + options.requires = options.requires.sort.uniq + options.loads.uniq! + + options + + end + end + + + class Main + + include Helpers + + attr_reader :callable, :input_parser, :lookups, + :options, :output_formatter, + :log_formatter, :start_time, :user_source_code + + + def initialize + @lookups = Lookups.new + @start_time = DateTime.now + end + + + private def load_global_config_if_exists + filespec = File.join(Dir.home, '.rexerc') + load(filespec) if File.exists?(filespec) + end + + + private def init_parser_and_formatters + @input_parser = lookups.input_parsers[options.input_format] + @output_formatter = lookups.formatters[options.output_format] + @log_formatter = lookups.formatters[options.log_format] + end + + + # Executes the user specified code in the manner appropriate to the input mode. + # Performs any optionally specified parsing on input and formatting on output. + private def execute(eval_context_object, code) + if options.input_format != :none && options.input_mode != :none + eval_context_object = input_parser.(eval_context_object) + end + + value = eval_context_object.instance_eval(&code) + + unless options.output_format == :none + print output_formatter.(value) + end + rescue Errno::EPIPE + exit(-13) + end + + + # The global $RC (Rexe Context) OpenStruct is available in your user code. + # In order to make it possible to access this object in your loaded files, we are not creating + # it here; instead we add properties to it. This way, you can initialize an OpenStruct yourself + # in your loaded code and it will still work. If you do that, beware, any properties you add will be + # included in the log output. If the to_s of your added objects is large, that might be a pain. + private def init_rexe_context + $RC ||= OpenStruct.new + $RC.count = 0 + $RC.rexe_version = VERSION + $RC.start_time = start_time.iso8601 + $RC.source_code = user_source_code + $RC.options = options.to_h + + def $RC.i; count end # `i` aliases `count` so you can more concisely get the count in your user code + end + + + private def create_callable + eval("Proc.new { #{user_source_code} }") + end + + + private def lookup_action(mode) + input = options.input_filespec ? File.open(options.input_filespec) : STDIN + { + line: -> { input.each { |l| execute(l.chomp, callable); $RC.count += 1 } }, + enumerator: -> { execute(input.each_line, callable); $RC.count += 1 }, + one_big_string: -> { big_string = input.read; execute(big_string, callable); $RC.count += 1 }, + none: -> { execute(Object.new, callable) } + }.fetch(mode) + end + + + private def output_log_entry + if options.log_format != :none + $RC.duration_secs = Time.now - start_time.to_time + STDERR.puts(log_formatter.($RC.to_h)) + end + end + + + # Bypasses Bundler's restriction on loading gems + # (see https://stackoverflow.com/questions/55144094/bundler-doesnt-permit-using-gems-in-project-home-directory) + private def require!(the_require) + begin + require the_require + rescue LoadError => error + gem_path = `gem which #{the_require}` + if gem_path.chomp.strip.empty? + raise error # re-raise the error, can't fix it + else + load_dir = File.dirname(gem_path) + $LOAD_PATH += load_dir + require the_require + end + end + end + + + # This class' entry point. + def call + + try do + + @options = CommandLineParser.new.parse + + options.requires.each { |r| require!(r) } + load_global_config_if_exists + options.loads.each { |file| load(file) } + + @user_source_code = ARGV.join(' ') + @user_source_code = 'self' if @user_source_code == '' + + @callable = create_callable + + init_rexe_context + init_parser_and_formatters + + # This is where the user's source code will be executed; the action will in turn call `execute`. + lookup_action(options.input_mode).call unless options.noop + + output_log_entry + end + end + end +end + + +def bundler_run(&block) + # This used to be an unconditional call to with_clean_env but that method is now deprecated: + # [DEPRECATED] `Bundler.with_clean_env` has been deprecated in favor of `Bundler.with_unbundled_env`. + # If you instead want the environment before bundler was originally loaded, + # use `Bundler.with_original_env` + + if Bundler.respond_to?(:with_unbundled_env) + Bundler.with_unbundled_env { block.call } + else + Bundler.with_clean_env { block.call } + end +end + + +bundler_run { Rexe::Main.new.call } diff --git a/spec/syntax_suggest/fixtures/routes.rb.txt b/spec/syntax_suggest/fixtures/routes.rb.txt new file mode 100644 index 0000000000..86733821c0 --- /dev/null +++ b/spec/syntax_suggest/fixtures/routes.rb.txt @@ -0,0 +1,121 @@ +Rails.application.routes.draw do + constraints -> { Rails.application.config.non_production } do + namespace :foo do + resource :bar + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + constraints -> { Rails.application.config.non_production } do + namespace :bar do + resource :baz + end + end + + namespace :admin do + resource :session + + match "/foobar(*path)", via: :all, to: redirect { |_params, req| + uri = URI(req.path.gsub("foobar", "foobaz")) + uri.query = req.query_string.presence + uri.to_s + } +end diff --git a/spec/syntax_suggest/fixtures/ruby_buildpack.rb.txt b/spec/syntax_suggest/fixtures/ruby_buildpack.rb.txt new file mode 100644 index 0000000000..9acdbf3a61 --- /dev/null +++ b/spec/syntax_suggest/fixtures/ruby_buildpack.rb.txt @@ -0,0 +1,1344 @@ +require "tmpdir" +require "digest/md5" +require "benchmark" +require "rubygems" +require "language_pack" +require "language_pack/base" +require "language_pack/ruby_version" +require "language_pack/helpers/nodebin" +require "language_pack/helpers/node_installer" +require "language_pack/helpers/yarn_installer" +require "language_pack/helpers/layer" +require "language_pack/helpers/binstub_check" +require "language_pack/version" + +# base Ruby Language Pack. This is for any base ruby app. +class LanguagePack::Ruby < LanguagePack::Base + NAME = "ruby" + LIBYAML_VERSION = "0.1.7" + LIBYAML_PATH = "libyaml-#{LIBYAML_VERSION}" + RBX_BASE_URL = "http://binaries.rubini.us/heroku" + NODE_BP_PATH = "vendor/node/bin" + + Layer = LanguagePack::Helpers::Layer + + # detects if this is a valid Ruby app + # @return [Boolean] true if it's a Ruby app + def self.use? + instrument "ruby.use" do + File.exist?("Gemfile") + end + end + + def self.bundler + @@bundler ||= LanguagePack::Helpers::BundlerWrapper.new.install + end + + def bundler + self.class.bundler + end + + def initialize(*args) + super(*args) + @fetchers[:mri] = LanguagePack::Fetcher.new(VENDOR_URL, @stack) + @fetchers[:rbx] = LanguagePack::Fetcher.new(RBX_BASE_URL, @stack) + @node_installer = LanguagePack::Helpers::NodeInstaller.new + @yarn_installer = LanguagePack::Helpers::YarnInstaller.new + end + + def name + "Ruby" + end + + def default_addons + instrument "ruby.default_addons" do + add_dev_database_addon + end + end + + def default_config_vars + instrument "ruby.default_config_vars" do + vars = { + "LANG" => env("LANG") || "en_US.UTF-8", + } + + ruby_version.jruby? ? vars.merge({ + "JRUBY_OPTS" => default_jruby_opts + }) : vars + end + end + + def default_process_types + instrument "ruby.default_process_types" do + { + "rake" => "bundle exec rake", + "console" => "bundle exec irb" + } + end + end + + def best_practice_warnings + if bundler.has_gem?("asset_sync") + warn(<<-WARNING) +You are using the `asset_sync` gem. +This is not recommended. +See https://devcenter.heroku.com/articles/please-do-not-use-asset-sync for more information. +WARNING + end + end + + def compile + instrument 'ruby.compile' do + # check for new app at the beginning of the compile + new_app? + Dir.chdir(build_path) + remove_vendor_bundle + warn_bundler_upgrade + warn_bad_binstubs + install_ruby(slug_vendor_ruby, build_ruby_path) + setup_language_pack_environment( + ruby_layer_path: File.expand_path("."), + gem_layer_path: File.expand_path("."), + bundle_path: "vendor/bundle", + bundle_default_without: "development:test" + ) + allow_git do + install_bundler_in_app(slug_vendor_base) + load_bundler_cache + build_bundler + post_bundler + create_database_yml + install_binaries + run_assets_precompile_rake_task + end + config_detect + best_practice_warnings + warn_outdated_ruby + setup_profiled(ruby_layer_path: "$HOME", gem_layer_path: "$HOME") # $HOME is set to /app at run time + setup_export + cleanup + super + end + rescue => e + warn_outdated_ruby + raise e + end + + + def build + new_app? + remove_vendor_bundle + warn_bad_binstubs + ruby_layer = Layer.new(@layer_dir, "ruby", launch: true) + install_ruby("#{ruby_layer.path}/#{slug_vendor_ruby}") + ruby_layer.metadata[:version] = ruby_version.version + ruby_layer.metadata[:patchlevel] = ruby_version.patchlevel if ruby_version.patchlevel + ruby_layer.metadata[:engine] = ruby_version.engine.to_s + ruby_layer.metadata[:engine_version] = ruby_version.engine_version + ruby_layer.write + + gem_layer = Layer.new(@layer_dir, "gems", launch: true, cache: true, build: true) + setup_language_pack_environment( + ruby_layer_path: ruby_layer.path, + gem_layer_path: gem_layer.path, + bundle_path: "#{gem_layer.path}/vendor/bundle", + bundle_default_without: "development:test" + ) + allow_git do + # TODO install bundler in separate layer + topic "Loading Bundler Cache" + gem_layer.validate! do |metadata| + valid_bundler_cache?(gem_layer.path, gem_layer.metadata) + end + install_bundler_in_app("#{gem_layer.path}/#{slug_vendor_base}") + build_bundler + # TODO post_bundler might need to be done in a new layer + bundler.clean + gem_layer.metadata[:gems] = Digest::SHA2.hexdigest(File.read("Gemfile.lock")) + gem_layer.metadata[:stack] = @stack + gem_layer.metadata[:ruby_version] = run_stdout(%q(ruby -v)).strip + gem_layer.metadata[:rubygems_version] = run_stdout(%q(gem -v)).strip + gem_layer.metadata[:buildpack_version] = BUILDPACK_VERSION + gem_layer.write + + create_database_yml + # TODO replace this with multibuildpack stuff? put binaries in their own layer? + install_binaries + run_assets_precompile_rake_task + end + setup_profiled(ruby_layer_path: ruby_layer.path, gem_layer_path: gem_layer.path) + setup_export(gem_layer) + config_detect + best_practice_warnings + cleanup + + super + end + + def cleanup + end + + def config_detect + end + +private + + # A bad shebang line looks like this: + # + # ``` + # #!/usr/bin/env ruby2.5 + # ``` + # + # Since `ruby2.5` is not a valid binary name + # + def warn_bad_binstubs + check = LanguagePack::Helpers::BinstubCheck.new(app_root_dir: Dir.pwd, warn_object: self) + check.call + end + + def default_malloc_arena_max? + return true if @metadata.exists?("default_malloc_arena_max") + return @metadata.touch("default_malloc_arena_max") if new_app? + + return false + end + + def warn_bundler_upgrade + old_bundler_version = @metadata.read("bundler_version").strip if @metadata.exists?("bundler_version") + + if old_bundler_version && old_bundler_version != bundler.version + warn(<<-WARNING, inline: true) +Your app was upgraded to bundler #{ bundler.version }. +Previously you had a successful deploy with bundler #{ old_bundler_version }. + +If you see problems related to the bundler version please refer to: +https://devcenter.heroku.com/articles/bundler-version#known-upgrade-issues + +WARNING + end + end + + # For example "vendor/bundle/ruby/2.6.0" + def self.slug_vendor_base + @slug_vendor_base ||= begin + command = %q(ruby -e "require 'rbconfig';puts \"vendor/bundle/#{RUBY_ENGINE}/#{RbConfig::CONFIG['ruby_version']}\"") + out = run_no_pipe(command, user_env: true).strip + error "Problem detecting bundler vendor directory: #{out}" unless $?.success? + out + end + end + + # the relative path to the bundler directory of gems + # @return [String] resulting path + def slug_vendor_base + instrument 'ruby.slug_vendor_base' do + @slug_vendor_base ||= self.class.slug_vendor_base + end + end + + # the relative path to the vendored ruby directory + # @return [String] resulting path + def slug_vendor_ruby + "vendor/#{ruby_version.version_without_patchlevel}" + end + + # the absolute path of the build ruby to use during the buildpack + # @return [String] resulting path + def build_ruby_path + "/tmp/#{ruby_version.version_without_patchlevel}" + end + + # fetch the ruby version from bundler + # @return [String, nil] returns the ruby version if detected or nil if none is detected + def ruby_version + instrument 'ruby.ruby_version' do + return @ruby_version if @ruby_version + new_app = !File.exist?("vendor/heroku") + last_version_file = "buildpack_ruby_version" + last_version = nil + last_version = @metadata.read(last_version_file).strip if @metadata.exists?(last_version_file) + + @ruby_version = LanguagePack::RubyVersion.new(bundler.ruby_version, + is_new: new_app, + last_version: last_version) + return @ruby_version + end + end + + def set_default_web_concurrency + <<-EOF +case $(ulimit -u) in +256) + export HEROKU_RAM_LIMIT_MB=${HEROKU_RAM_LIMIT_MB:-512} + export WEB_CONCURRENCY=${WEB_CONCURRENCY:-2} + ;; +512) + export HEROKU_RAM_LIMIT_MB=${HEROKU_RAM_LIMIT_MB:-1024} + export WEB_CONCURRENCY=${WEB_CONCURRENCY:-4} + ;; +16384) + export HEROKU_RAM_LIMIT_MB=${HEROKU_RAM_LIMIT_MB:-2560} + export WEB_CONCURRENCY=${WEB_CONCURRENCY:-8} + ;; +32768) + export HEROKU_RAM_LIMIT_MB=${HEROKU_RAM_LIMIT_MB:-6144} + export WEB_CONCURRENCY=${WEB_CONCURRENCY:-16} + ;; +*) + ;; +esac +EOF + end + + # default JRUBY_OPTS + # return [String] string of JRUBY_OPTS + def default_jruby_opts + "-Xcompile.invokedynamic=false" + end + + # sets up the environment variables for the build process + def setup_language_pack_environment(ruby_layer_path:, gem_layer_path:, bundle_path:, bundle_default_without:) + instrument 'ruby.setup_language_pack_environment' do + if ruby_version.jruby? + ENV["PATH"] += ":bin" + ENV["JRUBY_OPTS"] = env('JRUBY_BUILD_OPTS') || env('JRUBY_OPTS') + end + setup_ruby_install_env(ruby_layer_path) + + # By default Node can address 1.5GB of memory, a limitation it inherits from + # the underlying v8 engine. This can occasionally cause issues during frontend + # builds where memory use can exceed this threshold. + # + # This passes an argument to all Node processes during the build, so that they + # can take advantage of all available memory on the build dynos. + ENV["NODE_OPTIONS"] ||= "--max_old_space_size=2560" + + # TODO when buildpack-env-args rolls out, we can get rid of + # ||= and the manual setting below + default_config_vars.each do |key, value| + ENV[key] ||= value + end + + paths = [] + gem_path = "#{gem_layer_path}/#{slug_vendor_base}" + ENV["GEM_PATH"] = gem_path + ENV["GEM_HOME"] = gem_path + + ENV["DISABLE_SPRING"] = "1" + + # Rails has a binstub for yarn that doesn't work for all applications + # we need to ensure that yarn comes before local bin dir for that case + paths << yarn_preinstall_bin_path if yarn_preinstalled? + + # Need to remove `./bin` folder since it links to the wrong --prefix ruby binstubs breaking require in Ruby 1.9.2 and 1.8.7. + # Because for 1.9.2 and 1.8.7 there is a "build" ruby and a non-"build" Ruby + paths << "#{File.expand_path(".")}/bin" unless ruby_version.ruby_192_or_lower? + + paths << "#{gem_layer_path}/#{bundler_binstubs_path}" # Binstubs from bundler, eg. vendor/bundle/bin + paths << "#{gem_layer_path}/#{slug_vendor_base}/bin" # Binstubs from rubygems, eg. vendor/bundle/ruby/2.6.0/bin + paths << ENV["PATH"] + + ENV["PATH"] = paths.join(":") + + ENV["BUNDLE_WITHOUT"] = env("BUNDLE_WITHOUT") || bundle_default_without + if ENV["BUNDLE_WITHOUT"].include?(' ') + ENV["BUNDLE_WITHOUT"] = ENV["BUNDLE_WITHOUT"].tr(' ', ':') + + warn("Your BUNDLE_WITHOUT contains a space, we are converting it to a colon `:` BUNDLE_WITHOUT=#{ENV["BUNDLE_WITHOUT"]}", inline: true) + end + ENV["BUNDLE_PATH"] = bundle_path + ENV["BUNDLE_BIN"] = bundler_binstubs_path + ENV["BUNDLE_DEPLOYMENT"] = "1" + ENV["BUNDLE_GLOBAL_PATH_APPENDS_RUBY_SCOPE"] = "1" if bundler.needs_ruby_global_append_path? + end + end + + # Sets up the environment variables for subsequent processes run by + # muiltibuildpack. We can't use profile.d because $HOME isn't set up + def setup_export(layer = nil) + instrument 'ruby.setup_export' do + if layer + paths = ENV["PATH"] + else + paths = ENV["PATH"].split(":").map do |path| + /^\/.*/ !~ path ? "#{build_path}/#{path}" : path + end.join(":") + end + + # TODO ensure path exported is correct + set_export_path "PATH", paths, layer + + if layer + gem_path = "#{layer.path}/#{slug_vendor_base}" + else + gem_path = "#{build_path}/#{slug_vendor_base}" + end + set_export_path "GEM_PATH", gem_path, layer + set_export_default "LANG", "en_US.UTF-8", layer + + # TODO handle jruby + if ruby_version.jruby? + set_export_default "JRUBY_OPTS", default_jruby_opts + end + + set_export_default "BUNDLE_PATH", ENV["BUNDLE_PATH"], layer + set_export_default "BUNDLE_WITHOUT", ENV["BUNDLE_WITHOUT"], layer + set_export_default "BUNDLE_BIN", ENV["BUNDLE_BIN"], layer + set_export_default "BUNDLE_GLOBAL_PATH_APPENDS_RUBY_SCOPE", ENV["BUNDLE_GLOBAL_PATH_APPENDS_RUBY_SCOPE"], layer if bundler.needs_ruby_global_append_path? + set_export_default "BUNDLE_DEPLOYMENT", ENV["BUNDLE_DEPLOYMENT"], layer if ENV["BUNDLE_DEPLOYMENT"] # Unset on windows since we delete the Gemfile.lock + end + end + + # sets up the profile.d script for this buildpack + def setup_profiled(ruby_layer_path: , gem_layer_path: ) + instrument 'setup_profiled' do + profiled_path = [] + + # Rails has a binstub for yarn that doesn't work for all applications + # we need to ensure that yarn comes before local bin dir for that case + if yarn_preinstalled? + profiled_path << yarn_preinstall_bin_path.gsub(File.expand_path("."), "$HOME") + elsif has_yarn_binary? + profiled_path << "#{ruby_layer_path}/vendor/#{@yarn_installer.binary_path}" + end + profiled_path << "$HOME/bin" # /app in production + profiled_path << "#{gem_layer_path}/#{bundler_binstubs_path}" # Binstubs from bundler, eg. vendor/bundle/bin + profiled_path << "#{gem_layer_path}/#{slug_vendor_base}/bin" # Binstubs from rubygems, eg. vendor/bundle/ruby/2.6.0/bin + profiled_path << "$PATH" + + set_env_default "LANG", "en_US.UTF-8" + set_env_override "GEM_PATH", "#{gem_layer_path}/#{slug_vendor_base}:$GEM_PATH" + set_env_override "PATH", profiled_path.join(":") + set_env_override "DISABLE_SPRING", "1" + + set_env_default "MALLOC_ARENA_MAX", "2" if default_malloc_arena_max? + + web_concurrency = env("SENSIBLE_DEFAULTS") ? set_default_web_concurrency : "" + add_to_profiled(web_concurrency, filename: "WEB_CONCURRENCY.sh", mode: "w") # always write that file, even if its empty (meaning no defaults apply), for interop with other buildpacks - and we overwrite the file rather than appending (which is the default) + + # TODO handle JRUBY + if ruby_version.jruby? + set_env_default "JRUBY_OPTS", default_jruby_opts + end + + set_env_default "BUNDLE_PATH", ENV["BUNDLE_PATH"] + set_env_default "BUNDLE_WITHOUT", ENV["BUNDLE_WITHOUT"] + set_env_default "BUNDLE_BIN", ENV["BUNDLE_BIN"] + set_env_default "BUNDLE_GLOBAL_PATH_APPENDS_RUBY_SCOPE", ENV["BUNDLE_GLOBAL_PATH_APPENDS_RUBY_SCOPE"] if bundler.needs_ruby_global_append_path? + set_env_default "BUNDLE_DEPLOYMENT", ENV["BUNDLE_DEPLOYMENT"] if ENV["BUNDLE_DEPLOYMENT"] # Unset on windows since we delete the Gemfile.lock + end + end + + def warn_outdated_ruby + return unless defined?(@outdated_version_check) + + @warn_outdated ||= begin + @outdated_version_check.join + + warn_outdated_minor + warn_outdated_eol + warn_stack_upgrade + true + end + end + + def warn_stack_upgrade + return unless defined?(@ruby_download_check) + return unless @ruby_download_check.next_stack(current_stack: stack) + return if @ruby_download_check.exists_on_next_stack?(current_stack: stack) + + warn(<<~WARNING) + Your Ruby version is not present on the next stack + + You are currently using #{ruby_version.version_for_download} on #{stack} stack. + This version does not exist on #{@ruby_download_check.next_stack(current_stack: stack)}. In order to upgrade your stack you will + need to upgrade to a supported Ruby version. + + For a list of supported Ruby versions see: + https://devcenter.heroku.com/articles/ruby-support#supported-runtimes + + For a list of the oldest Ruby versions present on a given stack see: + https://devcenter.heroku.com/articles/ruby-support#oldest-available-runtimes + WARNING + end + + def warn_outdated_eol + return unless @outdated_version_check.maybe_eol? + + if @outdated_version_check.eol? + warn(<<~WARNING) + EOL Ruby Version + + You are using a Ruby version that has reached its End of Life (EOL) + + We strongly suggest you upgrade to Ruby #{@outdated_version_check.suggest_ruby_eol_version} or later + + Your current Ruby version no longer receives security updates from + Ruby Core and may have serious vulnerabilities. While you will continue + to be able to deploy on Heroku with this Ruby version you must upgrade + to a non-EOL version to be eligible to receive support. + + Upgrade your Ruby version as soon as possible. + + For a list of supported Ruby versions see: + https://devcenter.heroku.com/articles/ruby-support#supported-runtimes + WARNING + else + # Maybe EOL + warn(<<~WARNING) + Potential EOL Ruby Version + + You are using a Ruby version that has either reached its End of Life (EOL) + or will reach its End of Life on December 25th of this year. + + We suggest you upgrade to Ruby #{@outdated_version_check.suggest_ruby_eol_version} or later + + Once a Ruby version becomes EOL, it will no longer receive + security updates from Ruby core and may have serious vulnerabilities. + + Please upgrade your Ruby version. + + For a list of supported Ruby versions see: + https://devcenter.heroku.com/articles/ruby-support#supported-runtimes + WARNING + end + end + + def warn_outdated_minor + return if @outdated_version_check.latest_minor_version? + + warn(<<~WARNING) + There is a more recent Ruby version available for you to use: + + #{@outdated_version_check.suggested_ruby_minor_version} + + The latest version will include security and bug fixes. We always recommend + running the latest version of your minor release. + + Please upgrade your Ruby version. + + For all available Ruby versions see: + https://devcenter.heroku.com/articles/ruby-support#supported-runtimes + WARNING + end + + # install the vendored ruby + # @return [Boolean] true if it installs the vendored ruby and false otherwise + def install_ruby(install_path, build_ruby_path = nil) + instrument 'ruby.install_ruby' do + # Could do a compare operation to avoid re-downloading ruby + return false unless ruby_version + installer = LanguagePack::Installers::RubyInstaller.installer(ruby_version).new(@stack) + + @ruby_download_check = LanguagePack::Helpers::DownloadPresence.new(ruby_version.file_name) + @ruby_download_check.call + + if ruby_version.build? + installer.fetch_unpack(ruby_version, build_ruby_path, true) + end + + installer.install(ruby_version, install_path) + + @outdated_version_check = LanguagePack::Helpers::OutdatedRubyVersion.new( + current_ruby_version: ruby_version, + fetcher: installer.fetcher + ) + @outdated_version_check.call + + @metadata.write("buildpack_ruby_version", ruby_version.version_for_download) + + topic "Using Ruby version: #{ruby_version.version_for_download}" + if !ruby_version.set + warn(<<~WARNING) + You have not declared a Ruby version in your Gemfile. + + To declare a Ruby version add this line to your Gemfile: + + ``` + ruby "#{LanguagePack::RubyVersion::DEFAULT_VERSION_NUMBER}" + ``` + + For more information see: + https://devcenter.heroku.com/articles/ruby-versions + WARNING + end + + if ruby_version.warn_ruby_26_bundler? + warn(<<~WARNING, inline: true) + There is a known bundler bug with your version of Ruby + + Your version of Ruby contains a problem with the built-in integration of bundler. If + you encounter a bundler error you need to upgrade your Ruby version. We suggest you upgrade to: + + #{@outdated_version_check.suggested_ruby_minor_version} + + For more information see: + https://devcenter.heroku.com/articles/bundler-version#known-upgrade-issues + WARNING + end + end + + true + rescue LanguagePack::Fetcher::FetchError + if @ruby_download_check.does_not_exist? + message = <<~ERROR + The Ruby version you are trying to install does not exist: #{ruby_version.version_for_download} + ERROR + else + message = <<~ERROR + The Ruby version you are trying to install does not exist on this stack. + + You are trying to install #{ruby_version.version_for_download} on #{stack}. + + Ruby #{ruby_version.version_for_download} is present on the following stacks: + + - #{@ruby_download_check.valid_stack_list.join("\n - ")} + ERROR + + if env("CI") + message << <<~ERROR + + On Heroku CI you can set your stack in the `app.json`. For example: + + ``` + "stack": "heroku-20" + ``` + ERROR + end + end + + message << <<~ERROR + + Heroku recommends you use the latest supported Ruby version listed here: + https://devcenter.heroku.com/articles/ruby-support#supported-runtimes + + For more information on syntax for declaring a Ruby version see: + https://devcenter.heroku.com/articles/ruby-versions + ERROR + + error message + end + + # TODO make this compatible with CNB + def new_app? + @new_app ||= !File.exist?("vendor/heroku") + end + + # find the ruby install path for its binstubs during build + # @return [String] resulting path or empty string if ruby is not vendored + def ruby_install_binstub_path(ruby_layer_path = ".") + @ruby_install_binstub_path ||= + if ruby_version.build? + "#{build_ruby_path}/bin" + elsif ruby_version + "#{ruby_layer_path}/#{slug_vendor_ruby}/bin" + else + "" + end + end + + # setup the environment so we can use the vendored ruby + def setup_ruby_install_env(ruby_layer_path = ".") + instrument 'ruby.setup_ruby_install_env' do + ENV["PATH"] = "#{File.expand_path(ruby_install_binstub_path(ruby_layer_path))}:#{ENV["PATH"]}" + end + end + + # installs vendored gems into the slug + def install_bundler_in_app(bundler_dir) + instrument 'ruby.install_language_pack_gems' do + FileUtils.mkdir_p(bundler_dir) + Dir.chdir(bundler_dir) do |dir| + `cp -R #{bundler.bundler_path}/. .` + end + + # write bundler shim, so we can control the version bundler used + # Ruby 2.6.0 started vendoring bundler + write_bundler_shim("vendor/bundle/bin") if ruby_version.vendored_bundler? + end + end + + # default set of binaries to install + # @return [Array] resulting list + def binaries + add_node_js_binary + add_yarn_binary + end + + # vendors binaries into the slug + def install_binaries + instrument 'ruby.install_binaries' do + binaries.each {|binary| install_binary(binary) } + Dir["bin/*"].each {|path| run("chmod +x #{path}") } + end + end + + # vendors individual binary into the slug + # @param [String] name of the binary package from S3. + # Example: https://s3.amazonaws.com/language-pack-ruby/node-0.4.7.tgz, where name is "node-0.4.7" + def install_binary(name) + topic "Installing #{name}" + bin_dir = "bin" + FileUtils.mkdir_p bin_dir + Dir.chdir(bin_dir) do |dir| + if name.match(/^node\-/) + @node_installer.install + # need to set PATH here b/c `node-gyp` can change the CWD, but still depends on executing node. + # the current PATH is relative, but it needs to be absolute for this. + # doing this here also prevents it from being exported during runtime + node_bin_path = File.absolute_path(".") + # this needs to be set after so other binaries in bin/ don't take precedence" + ENV["PATH"] = "#{ENV["PATH"]}:#{node_bin_path}" + elsif name.match(/^yarn\-/) + FileUtils.mkdir_p("../vendor") + Dir.chdir("../vendor") do |vendor_dir| + @yarn_installer.install + yarn_path = File.absolute_path("#{vendor_dir}/#{@yarn_installer.binary_path}") + ENV["PATH"] = "#{yarn_path}:#{ENV["PATH"]}" + end + else + @fetchers[:buildpack].fetch_untar("#{name}.tgz") + end + end + end + + # removes a binary from the slug + # @param [String] relative path of the binary on the slug + def uninstall_binary(path) + FileUtils.rm File.join('bin', File.basename(path)), :force => true + end + + def load_default_cache? + new_app? && ruby_version.default? + end + + # loads a default bundler cache for new apps to speed up initial bundle installs + def load_default_cache + instrument "ruby.load_default_cache" do + if false # load_default_cache? + puts "New app detected loading default bundler cache" + patchlevel = run("ruby -e 'puts RUBY_PATCHLEVEL'").strip + cache_name = "#{LanguagePack::RubyVersion::DEFAULT_VERSION}-p#{patchlevel}-default-cache" + @fetchers[:buildpack].fetch_untar("#{cache_name}.tgz") + end + end + end + + # remove `vendor/bundle` that comes from the git repo + # in case there are native ext. + # users should be using `bundle pack` instead. + # https://github.com/heroku/heroku-buildpack-ruby/issues/21 + def remove_vendor_bundle + if File.exists?("vendor/bundle") + warn(<<-WARNING) +Removing `vendor/bundle`. +Checking in `vendor/bundle` is not supported. Please remove this directory +and add it to your .gitignore. To vendor your gems with Bundler, use +`bundle pack` instead. +WARNING + FileUtils.rm_rf("vendor/bundle") + end + end + + def bundler_binstubs_path + "vendor/bundle/bin" + end + + def bundler_path + @bundler_path ||= "#{slug_vendor_base}/gems/#{bundler.dir_name}" + end + + def write_bundler_shim(path) + FileUtils.mkdir_p(path) + shim_path = "#{path}/bundle" + File.open(shim_path, "w") do |file| + file.print <<-BUNDLE +#!/usr/bin/env ruby +require 'rubygems' + +version = "#{bundler.version}" + +if ARGV.first + str = ARGV.first + str = str.dup.force_encoding("BINARY") if str.respond_to? :force_encoding + if str =~ /\A_(.*)_\z/ and Gem::Version.correct?($1) then + version = $1 + ARGV.shift + end +end + +if Gem.respond_to?(:activate_bin_path) +load Gem.activate_bin_path('bundler', 'bundle', version) +else +gem "bundler", version +load Gem.bin_path("bundler", "bundle", version) +end +BUNDLE + end + FileUtils.chmod(0755, shim_path) + end + + # runs bundler to install the dependencies + def build_bundler + instrument 'ruby.build_bundler' do + log("bundle") do + if File.exist?("#{Dir.pwd}/.bundle/config") + warn(<<~WARNING, inline: true) + You have the `.bundle/config` file checked into your repository + It contains local state like the location of the installed bundle + as well as configured git local gems, and other settings that should + not be shared between multiple checkouts of a single repo. Please + remove the `.bundle/` folder from your repo and add it to your `.gitignore` file. + + https://devcenter.heroku.com/articles/bundler-configuration + WARNING + end + + if bundler.windows_gemfile_lock? + log("bundle", "has_windows_gemfile_lock") + + File.unlink("Gemfile.lock") + ENV.delete("BUNDLE_DEPLOYMENT") + + warn(<<~WARNING, inline: true) + Removing `Gemfile.lock` because it was generated on Windows. + Bundler will do a full resolve so native gems are handled properly. + This may result in unexpected gem versions being used in your app. + In rare occasions Bundler may not be able to resolve your dependencies at all. + + https://devcenter.heroku.com/articles/bundler-windows-gemfile + WARNING + end + + bundle_command = String.new("") + bundle_command << "BUNDLE_WITHOUT='#{ENV["BUNDLE_WITHOUT"]}' " + bundle_command << "BUNDLE_PATH=#{ENV["BUNDLE_PATH"]} " + bundle_command << "BUNDLE_BIN=#{ENV["BUNDLE_BIN"]} " + bundle_command << "BUNDLE_DEPLOYMENT=#{ENV["BUNDLE_DEPLOYMENT"]} " if ENV["BUNDLE_DEPLOYMENT"] # Unset on windows since we delete the Gemfile.lock + bundle_command << "BUNDLE_GLOBAL_PATH_APPENDS_RUBY_SCOPE=#{ENV["BUNDLE_GLOBAL_PATH_APPENDS_RUBY_SCOPE"]} " if bundler.needs_ruby_global_append_path? + bundle_command << "bundle install -j4" + + topic("Installing dependencies using bundler #{bundler.version}") + + bundler_output = String.new("") + bundle_time = nil + env_vars = {} + Dir.mktmpdir("libyaml-") do |tmpdir| + libyaml_dir = "#{tmpdir}/#{LIBYAML_PATH}" + + # need to setup compile environment for the psych gem + yaml_include = File.expand_path("#{libyaml_dir}/include").shellescape + yaml_lib = File.expand_path("#{libyaml_dir}/lib").shellescape + pwd = Dir.pwd + bundler_path = "#{pwd}/#{slug_vendor_base}/gems/#{bundler.dir_name}/lib" + + # we need to set BUNDLE_CONFIG and BUNDLE_GEMFILE for + # codon since it uses bundler. + env_vars["BUNDLE_GEMFILE"] = "#{pwd}/Gemfile" + env_vars["BUNDLE_CONFIG"] = "#{pwd}/.bundle/config" + env_vars["CPATH"] = noshellescape("#{yaml_include}:$CPATH") + env_vars["CPPATH"] = noshellescape("#{yaml_include}:$CPPATH") + env_vars["LIBRARY_PATH"] = noshellescape("#{yaml_lib}:$LIBRARY_PATH") + env_vars["RUBYOPT"] = syck_hack + env_vars["NOKOGIRI_USE_SYSTEM_LIBRARIES"] = "true" + env_vars["BUNDLE_DISABLE_VERSION_CHECK"] = "true" + env_vars["BUNDLER_LIB_PATH"] = "#{bundler_path}" if ruby_version.ruby_version == "1.8.7" + env_vars["BUNDLE_DISABLE_VERSION_CHECK"] = "true" + + puts "Running: #{bundle_command}" + instrument "ruby.bundle_install" do + bundle_time = Benchmark.realtime do + bundler_output << pipe("#{bundle_command} --no-clean", out: "2>&1", env: env_vars, user_env: true) + end + end + end + + if $?.success? + puts "Bundle completed (#{"%.2f" % bundle_time}s)" + log "bundle", :status => "success" + puts "Cleaning up the bundler cache." + instrument "ruby.bundle_clean" do + # Only show bundle clean output when not using default cache + if load_default_cache? + run("bundle clean > /dev/null", user_env: true, env: env_vars) + else + pipe("bundle clean", out: "2> /dev/null", user_env: true, env: env_vars) + end + end + @bundler_cache.store + + # Keep gem cache out of the slug + FileUtils.rm_rf("#{slug_vendor_base}/cache") + else + mcount "fail.bundle.install" + log "bundle", :status => "failure" + error_message = "Failed to install gems via Bundler." + puts "Bundler Output: #{bundler_output}" + if bundler_output.match(/An error occurred while installing sqlite3/) + mcount "fail.sqlite3" + error_message += <<~ERROR + + Detected sqlite3 gem which is not supported on Heroku: + https://devcenter.heroku.com/articles/sqlite3 + ERROR + end + + if bundler_output.match(/but your Gemfile specified/) + mcount "fail.ruby_version_mismatch" + error_message += <<~ERROR + + Detected a mismatch between your Ruby version installed and + Ruby version specified in Gemfile or Gemfile.lock. You can + correct this by running: + + $ bundle update --ruby + $ git add Gemfile.lock + $ git commit -m "update ruby version" + + If this does not solve the issue please see this documentation: + + https://devcenter.heroku.com/articles/ruby-versions#your-ruby-version-is-x-but-your-gemfile-specified-y + ERROR + end + + error error_message + end + end + end + end + + def post_bundler + instrument "ruby.post_bundler" do + Dir[File.join(slug_vendor_base, "**", ".git")].each do |dir| + FileUtils.rm_rf(dir) + end + bundler.clean + end + end + + # RUBYOPT line that requires syck_hack file + # @return [String] require string if needed or else an empty string + def syck_hack + instrument "ruby.syck_hack" do + syck_hack_file = File.expand_path(File.join(File.dirname(__FILE__), "../../vendor/syck_hack")) + rv = run_stdout('ruby -e "puts RUBY_VERSION"').strip + # < 1.9.3 includes syck, so we need to use the syck hack + if Gem::Version.new(rv) < Gem::Version.new("1.9.3") + "-r#{syck_hack_file}" + else + "" + end + end + end + + # writes ERB based database.yml for Rails. The database.yml uses the DATABASE_URL from the environment during runtime. + def create_database_yml + instrument 'ruby.create_database_yml' do + return false unless File.directory?("config") + return false if bundler.has_gem?('activerecord') && bundler.gem_version('activerecord') >= Gem::Version.new('4.1.0.beta1') + + log("create_database_yml") do + topic("Writing config/database.yml to read from DATABASE_URL") + File.open("config/database.yml", "w") do |file| + file.puts <<-DATABASE_YML +<% + +require 'cgi' +require 'uri' + +begin + uri = URI.parse(ENV["DATABASE_URL"]) +rescue URI::InvalidURIError + raise "Invalid DATABASE_URL" +end + +raise "No RACK_ENV or RAILS_ENV found" unless ENV["RAILS_ENV"] || ENV["RACK_ENV"] + +def attribute(name, value, force_string = false) + if value + value_string = + if force_string + '"' + value + '"' + else + value + end + "\#{name}: \#{value_string}" + else + "" + end +end + +adapter = uri.scheme +adapter = "postgresql" if adapter == "postgres" + +database = (uri.path || "").split("/")[1] + +username = uri.user +password = uri.password + +host = uri.host +port = uri.port + +params = CGI.parse(uri.query || "") + +%> + +<%= ENV["RAILS_ENV"] || ENV["RACK_ENV"] %>: + <%= attribute "adapter", adapter %> + <%= attribute "database", database %> + <%= attribute "username", username %> + <%= attribute "password", password, true %> + <%= attribute "host", host %> + <%= attribute "port", port %> + +<% params.each do |key, value| %> + <%= key %>: <%= value.first %> +<% end %> + DATABASE_YML + end + end + end + end + + def rake + @rake ||= begin + rake_gem_available = bundler.has_gem?("rake") || ruby_version.rake_is_vendored? + raise_on_fail = bundler.gem_version('railties') && bundler.gem_version('railties') > Gem::Version.new('3.x') + + topic "Detecting rake tasks" + rake = LanguagePack::Helpers::RakeRunner.new(rake_gem_available) + rake.load_rake_tasks!({ env: rake_env }, raise_on_fail) + rake + end + end + + def rake_env + if database_url + { "DATABASE_URL" => database_url } + else + {} + end.merge(user_env_hash) + end + + def database_url + env("DATABASE_URL") if env("DATABASE_URL") + end + + # executes the block with GIT_DIR environment variable removed since it can mess with the current working directory git thinks it's in + # @param [block] block to be executed in the GIT_DIR free context + def allow_git(&blk) + git_dir = ENV.delete("GIT_DIR") # can mess with bundler + blk.call + ENV["GIT_DIR"] = git_dir + end + + # decides if we need to enable the dev database addon + # @return [Array] the database addon if the pg gem is detected or an empty Array if it isn't. + def add_dev_database_addon + pg_adapters.any? {|a| bundler.has_gem?(a) } ? ['heroku-postgresql'] : [] + end + + def pg_adapters + [ + "pg", + "activerecord-jdbcpostgresql-adapter", + "jdbc-postgres", + "jdbc-postgresql", + "jruby-pg", + "rjack-jdbc-postgres", + "tgbyte-activerecord-jdbcpostgresql-adapter" + ] + end + + # decides if we need to install the node.js binary + # @note execjs will blow up if no JS RUNTIME is detected and is loaded. + # @return [Array] the node.js binary path if we need it or an empty Array + def add_node_js_binary + return [] if node_js_preinstalled? + + if Pathname(build_path).join("package.json").exist? || + bundler.has_gem?('execjs') || + bundler.has_gem?('webpacker') + [@node_installer.binary_path] + else + [] + end + end + + def add_yarn_binary + return [] if yarn_preinstalled? +| + if Pathname(build_path).join("yarn.lock").exist? || bundler.has_gem?('webpacker') + [@yarn_installer.name] + else + [] + end + end + + def has_yarn_binary? + add_yarn_binary.any? + end + + # checks if node.js is installed via the official heroku-buildpack-nodejs using multibuildpack + # @return String if it's detected and false if it isn't + def node_preinstall_bin_path + return @node_preinstall_bin_path if defined?(@node_preinstall_bin_path) + + legacy_path = "#{Dir.pwd}/#{NODE_BP_PATH}" + path = run("which node").strip + if path && $?.success? + @node_preinstall_bin_path = path + elsif run("#{legacy_path}/node -v") && $?.success? + @node_preinstall_bin_path = legacy_path + else + @node_preinstall_bin_path = false + end + end + alias :node_js_preinstalled? :node_preinstall_bin_path + + def node_not_preinstalled? + !node_js_preinstalled? + end + + # Example: tmp/build_8523f77fb96a956101d00988dfeed9d4/.heroku/yarn/bin/ (without the `yarn` at the end) + def yarn_preinstall_bin_path + (yarn_preinstall_binary_path || "").chomp("/yarn") + end + + # Example `tmp/build_8523f77fb96a956101d00988dfeed9d4/.heroku/yarn/bin/yarn` + def yarn_preinstall_binary_path + return @yarn_preinstall_binary_path if defined?(@yarn_preinstall_binary_path) + + path = run("which yarn").strip + if path && $?.success? + @yarn_preinstall_binary_path = path + else + @yarn_preinstall_binary_path = false + end + end + + def yarn_preinstalled? + yarn_preinstall_binary_path + end + + def yarn_not_preinstalled? + !yarn_preinstalled? + end + + def run_assets_precompile_rake_task + instrument 'ruby.run_assets_precompile_rake_task' do + + precompile = rake.task("assets:precompile") + return true unless precompile.is_defined? + + topic "Precompiling assets" + precompile.invoke(env: rake_env) + if precompile.success? + puts "Asset precompilation completed (#{"%.2f" % precompile.time}s)" + else + precompile_fail(precompile.output) + end + end + end + + def precompile_fail(output) + mcount "fail.assets_precompile" + log "assets_precompile", :status => "failure" + msg = "Precompiling assets failed.\n" + if output.match(/(127\.0\.0\.1)|(org\.postgresql\.util)/) + msg << "Attempted to access a nonexistent database:\n" + msg << "https://devcenter.heroku.com/articles/pre-provision-database\n" + end + + sprockets_version = bundler.gem_version('sprockets') + if output.match(/Sprockets::FileNotFound/) && (sprockets_version < Gem::Version.new('4.0.0.beta7') && sprockets_version > Gem::Version.new('4.0.0.beta4')) + mcount "fail.assets_precompile.file_not_found_beta" + msg << "If you have this file in your project\n" + msg << "try upgrading to Sprockets 4.0.0.beta7 or later:\n" + msg << "https://github.com/rails/sprockets/pull/547\n" + end + + error msg + end + + def bundler_cache + "vendor/bundle" + end + + def valid_bundler_cache?(path, metadata) + full_ruby_version = run_stdout(%q(ruby -v)).strip + rubygems_version = run_stdout(%q(gem -v)).strip + old_rubygems_version = nil + + old_rubygems_version = metadata[:ruby_version] + old_stack = metadata[:stack] + old_stack ||= DEFAULT_LEGACY_STACK + + stack_change = old_stack != @stack + if !new_app? && stack_change + return [false, "Purging Cache. Changing stack from #{old_stack} to #{@stack}"] + end + + # fix bug from v37 deploy + if File.exists?("#{path}/vendor/ruby_version") + puts "Broken cache detected. Purging build cache." + cache.clear("vendor") + FileUtils.rm_rf("#{path}/vendor/ruby_version") + return [false, "Broken cache detected. Purging build cache."] + # fix bug introduced in v38 + elsif !metadata.include?(:buildpack_version) && metadata.include?(:ruby_version) + puts "Broken cache detected. Purging build cache." + return [false, "Broken cache detected. Purging build cache."] + elsif (@bundler_cache.exists? || @bundler_cache.old?) && full_ruby_version != metadata[:ruby_version] + return [false, <<-MESSAGE] +Ruby version change detected. Clearing bundler cache. +Old: #{metadata[:ruby_version]} +New: #{full_ruby_version} +MESSAGE + end + + # fix git gemspec bug from Bundler 1.3.0+ upgrade + if File.exists?(bundler_cache) && !metadata.include?(:bundler_version) && !run("find #{path}/vendor/bundle/*/*/bundler/gems/*/ -name *.gemspec").include?("No such file or directory") + return [false, "Old bundler cache detected. Clearing bundler cache."] + end + + # fix for https://github.com/heroku/heroku-buildpack-ruby/issues/86 + if (!metadata.include?(:rubygems_version) || + (old_rubygems_version == "2.0.0" && old_rubygems_version != rubygems_version)) && + metadata.include?(:ruby_version) && metadata[:ruby_version].strip.include?("ruby 2.0.0p0") + return [false, "Updating to rubygems #{rubygems_version}. Clearing bundler cache."] + end + + # fix for https://github.com/sparklemotion/nokogiri/issues/923 + if metadata.include?(:buildpack_version) && (bv = metadata[:buildpack_version].sub('v', '').to_i) && bv != 0 && bv <= 76 + return [false, <<-MESSAGE] +Fixing nokogiri install. Clearing bundler cache. +See https://github.com/sparklemotion/nokogiri/issues/923. +MESSAGE + end + + # recompile nokogiri to use new libyaml + if metadata.include?(:buildpack_version) && (bv = metadata[:buildpack_version].sub('v', '').to_i) && bv != 0 && bv <= 99 && bundler.has_gem?("psych") + return [false, <<-MESSAGE] +Need to recompile psych for CVE-2013-6393. Clearing bundler cache. +See http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=737076. +MESSAGE + end + + # recompile gems for libyaml 0.1.7 update + if metadata.include?(:buildpack_version) && (bv = metadata[:buildpack_version].sub('v', '').to_i) && bv != 0 && bv <= 147 && + (metadata.include?(:ruby_version) && metadata[:ruby_version].match(/ruby 2\.1\.(9|10)/) || + bundler.has_gem?("psych") + ) + return [false, <<-MESSAGE] +Need to recompile gems for CVE-2014-2014-9130. Clearing bundler cache. +See https://devcenter.heroku.com/changelog-items/1016. +MESSAGE + end + + true + end + + def load_bundler_cache + instrument "ruby.load_bundler_cache" do + cache.load "vendor" + + full_ruby_version = run_stdout(%q(ruby -v)).strip + rubygems_version = run_stdout(%q(gem -v)).strip + heroku_metadata = "vendor/heroku" + old_rubygems_version = nil + ruby_version_cache = "ruby_version" + buildpack_version_cache = "buildpack_version" + bundler_version_cache = "bundler_version" + rubygems_version_cache = "rubygems_version" + stack_cache = "stack" + + # bundle clean does not remove binstubs + FileUtils.rm_rf("vendor/bundler/bin") + + old_rubygems_version = @metadata.read(ruby_version_cache).strip if @metadata.exists?(ruby_version_cache) + old_stack = @metadata.read(stack_cache).strip if @metadata.exists?(stack_cache) + old_stack ||= DEFAULT_LEGACY_STACK + + stack_change = old_stack != @stack + convert_stack = @bundler_cache.old? + @bundler_cache.convert_stack(stack_change) if convert_stack + if !new_app? && stack_change + puts "Purging Cache. Changing stack from #{old_stack} to #{@stack}" + purge_bundler_cache(old_stack) + elsif !new_app? && !convert_stack + @bundler_cache.load + end + + # fix bug from v37 deploy + if File.exists?("vendor/ruby_version") + puts "Broken cache detected. Purging build cache." + cache.clear("vendor") + FileUtils.rm_rf("vendor/ruby_version") + purge_bundler_cache + # fix bug introduced in v38 + elsif !@metadata.include?(buildpack_version_cache) && @metadata.exists?(ruby_version_cache) + puts "Broken cache detected. Purging build cache." + purge_bundler_cache + elsif (@bundler_cache.exists? || @bundler_cache.old?) && @metadata.exists?(ruby_version_cache) && full_ruby_version != @metadata.read(ruby_version_cache).strip + puts "Ruby version change detected. Clearing bundler cache." + puts "Old: #{@metadata.read(ruby_version_cache).strip}" + puts "New: #{full_ruby_version}" + purge_bundler_cache + end + + # fix git gemspec bug from Bundler 1.3.0+ upgrade + if File.exists?(bundler_cache) && !@metadata.exists?(bundler_version_cache) && !run("find vendor/bundle/*/*/bundler/gems/*/ -name *.gemspec").include?("No such file or directory") + puts "Old bundler cache detected. Clearing bundler cache." + purge_bundler_cache + end + + # fix for https://github.com/heroku/heroku-buildpack-ruby/issues/86 + if (!@metadata.exists?(rubygems_version_cache) || + (old_rubygems_version == "2.0.0" && old_rubygems_version != rubygems_version)) && + @metadata.exists?(ruby_version_cache) && @metadata.read(ruby_version_cache).strip.include?("ruby 2.0.0p0") + puts "Updating to rubygems #{rubygems_version}. Clearing bundler cache." + purge_bundler_cache + end + + # fix for https://github.com/sparklemotion/nokogiri/issues/923 + if @metadata.exists?(buildpack_version_cache) && (bv = @metadata.read(buildpack_version_cache).sub('v', '').to_i) && bv != 0 && bv <= 76 + puts "Fixing nokogiri install. Clearing bundler cache." + puts "See https://github.com/sparklemotion/nokogiri/issues/923." + purge_bundler_cache + end + + # recompile nokogiri to use new libyaml + if @metadata.exists?(buildpack_version_cache) && (bv = @metadata.read(buildpack_version_cache).sub('v', '').to_i) && bv != 0 && bv <= 99 && bundler.has_gem?("psych") + puts "Need to recompile psych for CVE-2013-6393. Clearing bundler cache." + puts "See http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=737076." + purge_bundler_cache + end + + # recompile gems for libyaml 0.1.7 update + if @metadata.exists?(buildpack_version_cache) && (bv = @metadata.read(buildpack_version_cache).sub('v', '').to_i) && bv != 0 && bv <= 147 && + (@metadata.exists?(ruby_version_cache) && @metadata.read(ruby_version_cache).strip.match(/ruby 2\.1\.(9|10)/) || + bundler.has_gem?("psych") + ) + puts "Need to recompile gems for CVE-2014-2014-9130. Clearing bundler cache." + puts "See https://devcenter.heroku.com/changelog-items/1016." + purge_bundler_cache + end + + FileUtils.mkdir_p(heroku_metadata) + @metadata.write(ruby_version_cache, full_ruby_version, false) + @metadata.write(buildpack_version_cache, BUILDPACK_VERSION, false) + @metadata.write(bundler_version_cache, bundler.version, false) + @metadata.write(rubygems_version_cache, rubygems_version, false) + @metadata.write(stack_cache, @stack, false) + @metadata.save + end + end + + def purge_bundler_cache(stack = nil) + instrument "ruby.purge_bundler_cache" do + @bundler_cache.clear(stack) + # need to reinstall language pack gems + install_bundler_in_app(slug_vendor_base) + end + end +end diff --git a/spec/syntax_suggest/fixtures/syntax_tree.rb.txt b/spec/syntax_suggest/fixtures/syntax_tree.rb.txt new file mode 100644 index 0000000000..1c110783f9 --- /dev/null +++ b/spec/syntax_suggest/fixtures/syntax_tree.rb.txt @@ -0,0 +1,9234 @@ +# frozen_string_literal: true + +require 'ripper' +require_relative 'syntax_tree/version' + +class SyntaxTree < Ripper + # Represents a line in the source. If this class is being used, it means that + # every character in the string is 1 byte in length, so we can just return the + # start of the line + the index. + class SingleByteString + def initialize(start) + @start = start + end + + def [](byteindex) + @start + byteindex + end + end + + # Represents a line in the source. If this class is being used, it means that + # there are characters in the string that are multi-byte, so we will build up + # an array of indices, such that array[byteindex] will be equal to the index + # of the character within the string. + class MultiByteString + def initialize(start, line) + @indices = [] + + line + .each_char + .with_index(start) do |char, index| + char.bytesize.times { @indices << index } + end + end + + def [](byteindex) + @indices[byteindex] + end + end + + # Represents the location of a node in the tree from the source code. + class Location + attr_reader :start_line, :start_char, :end_line, :end_char + + def initialize(start_line:, start_char:, end_line:, end_char:) + @start_line = start_line + @start_char = start_char + @end_line = end_line + @end_char = end_char + end + + def ==(other) + other.is_a?(Location) && start_line == other.start_line && + start_char == other.start_char && end_line == other.end_line && + end_char == other.end_char + end + + def to(other) + Location.new( + start_line: start_line, + start_char: start_char, + end_line: other.end_line, + end_char: other.end_char + ) + end + + def to_json(*opts) + [start_line, start_char, end_line, end_char].to_json(*opts) + end + + def self.token(line:, char:, size:) + new( + start_line: line, + start_char: char, + end_line: line, + end_char: char + size + ) + end + + def self.fixed(line:, char:) + new(start_line: line, start_char: char, end_line: line, end_char: char) + end + end + + # A special parser error so that we can get nice syntax displays on the error + # message when prettier prints out the results. + class ParseError < StandardError + attr_reader :lineno, :column + + def initialize(error, lineno, column) + super(error) + @lineno = lineno + @column = column + end + end + + attr_reader :source, :lines, :tokens + + # This is an attr_accessor so Stmts objects can grab comments out of this + # array and attach them to themselves. + attr_accessor :comments + + def initialize(source, *) + super + + # We keep the source around so that we can refer back to it when we're + # generating the AST. Sometimes it's easier to just reference the source + # string when you want to check if it contains a certain character, for + # example. + @source = source + + # Similarly, we keep the lines of the source string around to be able to + # check if certain lines contain certain characters. For example, we'll use + # this to generate the content that goes after the __END__ keyword. Or we'll + # use this to check if a comment has other content on its line. + @lines = source.split("\n") + + # This is the full set of comments that have been found by the parser. It's + # a running list. At the end of every block of statements, they will go in + # and attempt to grab any comments that are on their own line and turn them + # into regular statements. So at the end of parsing the only comments left + # in here will be comments on lines that also contain code. + @comments = [] + + # This is the current embdoc (comments that start with =begin and end with + # =end). Since they can't be nested, there's no need for a stack here, as + # there can only be one active. These end up getting dumped into the + # comments list before getting picked up by the statements that surround + # them. + @embdoc = nil + + # This is an optional node that can be present if the __END__ keyword is + # used in the file. In that case, this will represent the content after that + # keyword. + @__end__ = nil + + # Heredocs can actually be nested together if you're using interpolation, so + # this is a stack of heredoc nodes that are currently being created. When we + # get to the token that finishes off a heredoc node, we pop the top + # one off. If there are others surrounding it, then the body events will now + # be added to the correct nodes. + @heredocs = [] + + # This is a running list of tokens that have fired. It's useful + # mostly for maintaining location information. For example, if you're inside + # the handle of a def event, then in order to determine where the AST node + # started, you need to look backward in the tokens to find a def + # keyword. Most of the time, when a parser event consumes one of these + # events, it will be deleted from the list. So ideally, this list stays + # pretty short over the course of parsing a source string. + @tokens = [] + + # Here we're going to build up a list of SingleByteString or MultiByteString + # objects. They're each going to represent a string in the source. They are + # used by the `char_pos` method to determine where we are in the source + # string. + @line_counts = [] + last_index = 0 + + @source.lines.each do |line| + if line.size == line.bytesize + @line_counts << SingleByteString.new(last_index) + else + @line_counts << MultiByteString.new(last_index, line) + end + + last_index += line.size + end + end + + def self.parse(source) + parser = new(source) + response = parser.parse + response unless parser.error? + end + + private + + # ---------------------------------------------------------------------------- + # :section: Helper methods + # The following methods are used by the ripper event handlers to either + # determine their bounds or query other nodes. + # ---------------------------------------------------------------------------- + + # This represents the current place in the source string that we've gotten to + # so far. We have a memoized line_counts object that we can use to get the + # number of characters that we've had to go through to get to the beginning of + # this line, then we add the number of columns into this line that we've gone + # through. + def char_pos + @line_counts[lineno - 1][column] + end + + # As we build up a list of tokens, we'll periodically need to go backwards and + # find the ones that we've already hit in order to determine the location + # information for nodes that use them. For example, if you have a module node + # then you'll look backward for a kw token to determine your start location. + # + # This works with nesting since we're deleting tokens from the list once + # they've been used up. For example if you had nested module declarations then + # the innermost declaration would grab the last kw node that matches "module" + # (which would happen to be the innermost keyword). Then the outer one would + # only be able to grab the first one. In this way all of the tokens act as + # their own stack. + def find_token(type, value = :any, consume: true) + index = + tokens.rindex do |token| + token.is_a?(type) && (value == :any || (token.value == value)) + end + + if consume + # If we're expecting to be able to find a token and consume it, + # but can't actually find it, then we need to raise an error. This is + # _usually_ caused by a syntax error in the source that we're printing. It + # could also be caused by accidentally attempting to consume a token twice + # by two different parser event handlers. + unless index + message = "Cannot find expected #{value == :any ? type : value}" + raise ParseError.new(message, lineno, column) + end + + tokens.delete_at(index) + elsif index + tokens[index] + end + end + + # A helper function to find a :: operator. We do special handling instead of + # using find_token here because we don't pop off all of the :: + # operators so you could end up getting the wrong information if you have for + # instance ::X::Y::Z. + def find_colon2_before(const) + index = + tokens.rindex do |token| + token.is_a?(Op) && token.value == '::' && + token.location.start_char < const.location.start_char + end + + tokens[index] + end + + # Finds the next position in the source string that begins a statement. This + # is used to bind statements lists and make sure they don't include a + # preceding comment. For example, we want the following comment to be attached + # to the class node and not the statement node: + # + # class Foo # :nodoc: + # ... + # end + # + # By finding the next non-space character, we can make sure that the bounds of + # the statement list are correct. + def find_next_statement_start(position) + remaining = source[position..-1] + + if remaining.sub(/\A +/, '')[0] == '#' + return position + remaining.index("\n") + end + + position + end + + # ---------------------------------------------------------------------------- + # :section: Ripper event handlers + # The following methods all handle a dispatched ripper event. + # ---------------------------------------------------------------------------- + + # BEGINBlock represents the use of the +BEGIN+ keyword, which hooks into the + # lifecycle of the interpreter. Whatever is inside the block will get executed + # when the program starts. + # + # BEGIN { + # } + # + # Interestingly, the BEGIN keyword doesn't allow the do and end keywords for + # the block. Only braces are permitted. + class BEGINBlock + # [LBrace] the left brace that is seen after the keyword + attr_reader :lbrace + + # [Statements] the expressions to be executed + attr_reader :statements + + # [Location] the location of this node + attr_reader :location + + def initialize(lbrace:, statements:, location:) + @lbrace = lbrace + @statements = statements + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('BEGIN') + q.breakable + q.pp(statements) + end + end + + def to_json(*opts) + { + type: :BEGIN, + lbrace: lbrace, + stmts: statements, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_BEGIN: (Statements statements) -> BEGINBlock + def on_BEGIN(statements) + lbrace = find_token(LBrace) + rbrace = find_token(RBrace) + + statements.bind( + find_next_statement_start(lbrace.location.end_char), + rbrace.location.start_char + ) + + keyword = find_token(Kw, 'BEGIN') + + BEGINBlock.new( + lbrace: lbrace, + statements: statements, + location: keyword.location.to(rbrace.location) + ) + end + + # CHAR irepresents a single codepoint in the script encoding. + # + # ?a + # + # In the example above, the CHAR node represents the string literal "a". You + # can use control characters with this as well, as in ?\C-a. + class CHAR + # [String] the value of the character literal + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('CHAR') + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :CHAR, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_CHAR: (String value) -> CHAR + def on_CHAR(value) + node = + CHAR.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # ENDBlock represents the use of the +END+ keyword, which hooks into the + # lifecycle of the interpreter. Whatever is inside the block will get executed + # when the program ends. + # + # END { + # } + # + # Interestingly, the END keyword doesn't allow the do and end keywords for the + # block. Only braces are permitted. + class ENDBlock + # [LBrace] the left brace that is seen after the keyword + attr_reader :lbrace + + # [Statements] the expressions to be executed + attr_reader :statements + + # [Location] the location of this node + attr_reader :location + + def initialize(lbrace:, statements:, location:) + @lbrace = lbrace + @statements = statements + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('END') + q.breakable + q.pp(statements) + end + end + + def to_json(*opts) + { type: :END, lbrace: lbrace, stmts: statements, loc: location }.to_json( + *opts + ) + end + end + + # :call-seq: + # on_END: (Statements statements) -> ENDBlock + def on_END(statements) + lbrace = find_token(LBrace) + rbrace = find_token(RBrace) + + statements.bind( + find_next_statement_start(lbrace.location.end_char), + rbrace.location.start_char + ) + + keyword = find_token(Kw, 'END') + + ENDBlock.new( + lbrace: lbrace, + statements: statements, + location: keyword.location.to(rbrace.location) + ) + end + + # EndContent represents the use of __END__ syntax, which allows individual + # scripts to keep content after the main ruby code that can be read through + # the DATA constant. + # + # puts DATA.read + # + # __END__ + # some other content that is not executed by the program + # + class EndContent + # [String] the content after the script + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('__end__') + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :__end__, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on___end__: (String value) -> EndContent + def on___end__(value) + @__end__ = + EndContent.new( + value: lines[lineno..-1].join("\n"), + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + end + + # Alias represents the use of the +alias+ keyword with regular arguments (not + # global variables). The +alias+ keyword is used to make a method respond to + # another name as well as the current one. + # + # alias aliased_name name + # + # For the example above, in the current context you can now call aliased_name + # and it will execute the name method. When you're aliasing two methods, you + # can either provide bare words (like the example above) or you can provide + # symbols (note that this includes dynamic symbols like + # :"left-#{middle}-right"). + class Alias + # [DynaSymbol | SymbolLiteral] the new name of the method + attr_reader :left + + # [DynaSymbol | SymbolLiteral] the old name of the method + attr_reader :right + + # [Location] the location of this node + attr_reader :location + + def initialize(left:, right:, location:) + @left = left + @right = right + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('alias') + q.breakable + q.pp(left) + q.breakable + q.pp(right) + end + end + + def to_json(*opts) + { type: :alias, left: left, right: right, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_alias: ( + # (DynaSymbol | SymbolLiteral) left, + # (DynaSymbol | SymbolLiteral) right + # ) -> Alias + def on_alias(left, right) + keyword = find_token(Kw, 'alias') + + Alias.new( + left: left, + right: right, + location: keyword.location.to(right.location) + ) + end + + # ARef represents when you're pulling a value out of a collection at a + # specific index. Put another way, it's any time you're calling the method + # #[]. + # + # collection[index] + # + # The nodes usually contains two children, the collection and the index. In + # some cases, you don't necessarily have the second child node, because you + # can call procs with a pretty esoteric syntax. In the following example, you + # wouldn't have a second child node: + # + # collection[] + # + class ARef + # [untyped] the value being indexed + attr_reader :collection + + # [nil | Args | ArgsAddBlock] the value being passed within the brackets + attr_reader :index + + # [Location] the location of this node + attr_reader :location + + def initialize(collection:, index:, location:) + @collection = collection + @index = index + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('aref') + q.breakable + q.pp(collection) + q.breakable + q.pp(index) + end + end + + def to_json(*opts) + { + type: :aref, + collection: collection, + index: index, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_aref: (untyped collection, (nil | Args | ArgsAddBlock) index) -> ARef + def on_aref(collection, index) + find_token(LBracket) + rbracket = find_token(RBracket) + + ARef.new( + collection: collection, + index: index, + location: collection.location.to(rbracket.location) + ) + end + + # ARefField represents assigning values into collections at specific indices. + # Put another way, it's any time you're calling the method #[]=. The + # ARefField node itself is just the left side of the assignment, and they're + # always wrapped in assign nodes. + # + # collection[index] = value + # + class ARefField + # [untyped] the value being indexed + attr_reader :collection + + # [nil | ArgsAddBlock] the value being passed within the brackets + attr_reader :index + + # [Location] the location of this node + attr_reader :location + + def initialize(collection:, index:, location:) + @collection = collection + @index = index + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('aref_field') + q.breakable + q.pp(collection) + q.breakable + q.pp(index) + end + end + + def to_json(*opts) + { + type: :aref_field, + collection: collection, + index: index, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_aref_field: ( + # untyped collection, + # (nil | ArgsAddBlock) index + # ) -> ARefField + def on_aref_field(collection, index) + find_token(LBracket) + rbracket = find_token(RBracket) + + ARefField.new( + collection: collection, + index: index, + location: collection.location.to(rbracket.location) + ) + end + + # def on_arg_ambiguous(value) + # value + # end + + # ArgParen represents wrapping arguments to a method inside a set of + # parentheses. + # + # method(argument) + # + # In the example above, there would be an ArgParen node around the + # ArgsAddBlock node that represents the set of arguments being sent to the + # method method. The argument child node can be +nil+ if no arguments were + # passed, as in: + # + # method() + # + class ArgParen + # [nil | Args | ArgsAddBlock | ArgsForward] the arguments inside the + # parentheses + attr_reader :arguments + + # [Location] the location of this node + attr_reader :location + + def initialize(arguments:, location:) + @arguments = arguments + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('arg_paren') + q.breakable + q.pp(arguments) + end + end + + def to_json(*opts) + { type: :arg_paren, args: arguments, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_arg_paren: ( + # (nil | Args | ArgsAddBlock | ArgsForward) arguments + # ) -> ArgParen + def on_arg_paren(arguments) + lparen = find_token(LParen) + rparen = find_token(RParen) + + # If the arguments exceed the ending of the parentheses, then we know we + # have a heredoc in the arguments, and we need to use the bounds of the + # arguments to determine how large the arg_paren is. + ending = + if arguments && arguments.location.end_line > rparen.location.end_line + arguments + else + rparen + end + + ArgParen.new( + arguments: arguments, + location: lparen.location.to(ending.location) + ) + end + + # Args represents a list of arguments being passed to a method call or array + # literal. + # + # method(first, second, third) + # + class Args + # [Array[ untyped ]] the arguments that this node wraps + attr_reader :parts + + # [Location] the location of this node + attr_reader :location + + def initialize(parts:, location:) + @parts = parts + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('args') + q.breakable + q.group(2, '(', ')') { q.seplist(parts) { |part| q.pp(part) } } + end + end + + def to_json(*opts) + { type: :args, parts: parts, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_args_add: (Args arguments, untyped argument) -> Args + def on_args_add(arguments, argument) + if arguments.parts.empty? + # If this is the first argument being passed into the list of arguments, + # then we're going to use the bounds of the argument to override the + # parent node's location since this will be more accurate. + Args.new(parts: [argument], location: argument.location) + else + # Otherwise we're going to update the existing list with the argument + # being added as well as the new end bounds. + Args.new( + parts: arguments.parts << argument, + location: arguments.location.to(argument.location) + ) + end + end + + # ArgsAddBlock represents a list of arguments and potentially a block + # argument. ArgsAddBlock is commonly seen being passed to any method where you + # use parentheses (wrapped in an ArgParen node). It’s also used to pass + # arguments to the various control-flow keywords like +return+. + # + # method(argument, &block) + # + class ArgsAddBlock + # [Args] the arguments before the optional block + attr_reader :arguments + + # [nil | untyped] the optional block argument + attr_reader :block + + # [Location] the location of this node + attr_reader :location + + def initialize(arguments:, block:, location:) + @arguments = arguments + @block = block + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('args_add_block') + q.breakable + q.pp(arguments) + q.breakable + q.pp(block) + end + end + + def to_json(*opts) + { + type: :args_add_block, + args: arguments, + block: block, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_args_add_block: ( + # Args arguments, + # (false | untyped) block + # ) -> ArgsAddBlock + def on_args_add_block(arguments, block) + ending = block || arguments + + ArgsAddBlock.new( + arguments: arguments, + block: block || nil, + location: arguments.location.to(ending.location) + ) + end + + # Star represents using a splat operator on an expression. + # + # method(*arguments) + # + class ArgStar + # [untyped] the expression being splatted + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('arg_star') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :arg_star, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_args_add_star: (Args arguments, untyped star) -> Args + def on_args_add_star(arguments, argument) + beginning = find_token(Op, '*') + ending = argument || beginning + + location = + if arguments.parts.empty? + ending.location + else + arguments.location.to(ending.location) + end + + arg_star = + ArgStar.new( + value: argument, + location: beginning.location.to(ending.location) + ) + + Args.new(parts: arguments.parts << arg_star, location: location) + end + + # ArgsForward represents forwarding all kinds of arguments onto another method + # call. + # + # def request(method, path, **headers, &block); end + # + # def get(...) + # request(:GET, ...) + # end + # + # def post(...) + # request(:POST, ...) + # end + # + # In the example above, both the get and post methods are forwarding all of + # their arguments (positional, keyword, and block) on to the request method. + # The ArgsForward node appears in both the caller (the request method calls) + # and the callee (the get and post definitions). + class ArgsForward + # [String] the value of the operator + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('args_forward') + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :args_forward, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_args_forward: () -> ArgsForward + def on_args_forward + op = find_token(Op, '...') + + ArgsForward.new(value: op.value, location: op.location) + end + + # :call-seq: + # on_args_new: () -> Args + def on_args_new + Args.new(parts: [], location: Location.fixed(line: lineno, char: char_pos)) + end + + # ArrayLiteral represents any form of an array literal, and contains myriad + # child nodes because of the special array literal syntax like %w and %i. + # + # [] + # [one, two, three] + # [*one_two_three] + # %i[one two three] + # %w[one two three] + # %I[one two three] + # %W[one two three] + # + # Every line in the example above produces an ArrayLiteral node. In order, the + # child contents node of this ArrayLiteral node would be nil, Args, QSymbols, + # QWords, Symbols, and Words. + class ArrayLiteral + # [nil | Args | QSymbols | QWords | Symbols | Words] the + # contents of the array + attr_reader :contents + + # [Location] the location of this node + attr_reader :location + + def initialize(contents:, location:) + @contents = contents + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('array') + q.breakable + q.pp(contents) + end + end + + def to_json(*opts) + { type: :array, cnts: contents, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_array: ( + # (nil | Args | QSymbols | QWords | Symbols | Words) contents + # ) -> ArrayLiteral + def on_array(contents) + if !contents || contents.is_a?(Args) + lbracket = find_token(LBracket) + rbracket = find_token(RBracket) + + ArrayLiteral.new( + contents: contents, + location: lbracket.location.to(rbracket.location) + ) + else + tstring_end = find_token(TStringEnd) + contents = + contents.class.new( + elements: contents.elements, + location: contents.location.to(tstring_end.location) + ) + + ArrayLiteral.new(contents: contents, location: contents.location) + end + end + + # AryPtn represents matching against an array pattern using the Ruby 2.7+ + # pattern matching syntax. It’s one of the more complicated nodes, because + # the four parameters that it accepts can almost all be nil. + # + # case [1, 2, 3] + # in [Integer, Integer] + # "matched" + # in Container[Integer, Integer] + # "matched" + # in [Integer, *, Integer] + # "matched" + # end + # + # An AryPtn node is created with four parameters: an optional constant + # wrapper, an array of positional matches, an optional splat with identifier, + # and an optional array of positional matches that occur after the splat. + # All of the in clauses above would create an AryPtn node. + class AryPtn + # [nil | VarRef] the optional constant wrapper + attr_reader :constant + + # [Array[ untyped ]] the regular positional arguments that this array + # pattern is matching against + attr_reader :requireds + + # [nil | VarField] the optional starred identifier that grabs up a list of + # positional arguments + attr_reader :rest + + # [Array[ untyped ]] the list of positional arguments occurring after the + # optional star if there is one + attr_reader :posts + + # [Location] the location of this node + attr_reader :location + + def initialize(constant:, requireds:, rest:, posts:, location:) + @constant = constant + @requireds = requireds + @rest = rest + @posts = posts + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('aryptn') + + if constant + q.breakable + q.pp(constant) + end + + if requireds.any? + q.breakable + q.group(2, '(', ')') do + q.seplist(requireds) { |required| q.pp(required) } + end + end + + if rest + q.breakable + q.pp(rest) + end + + if posts.any? + q.breakable + q.group(2, '(', ')') { q.seplist(posts) { |post| q.pp(post) } } + end + end + end + + def to_json(*opts) + { + type: :aryptn, + constant: constant, + reqs: requireds, + rest: rest, + posts: posts, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_aryptn: ( + # (nil | VarRef) constant, + # (nil | Array[untyped]) requireds, + # (nil | VarField) rest, + # (nil | Array[untyped]) posts + # ) -> AryPtn + def on_aryptn(constant, requireds, rest, posts) + parts = [constant, *requireds, rest, *posts].compact + + AryPtn.new( + constant: constant, + requireds: requireds || [], + rest: rest, + posts: posts || [], + location: parts[0].location.to(parts[-1].location) + ) + end + + # Assign represents assigning something to a variable or constant. Generally, + # the left side of the assignment is going to be any node that ends with the + # name "Field". + # + # variable = value + # + class Assign + # [ARefField | ConstPathField | Field | TopConstField | VarField] the target + # to assign the result of the expression to + attr_reader :target + + # [untyped] the expression to be assigned + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(target:, value:, location:) + @target = target + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('assign') + q.breakable + q.pp(target) + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :assign, target: target, value: value, loc: location }.to_json( + *opts + ) + end + end + + # :call-seq: + # on_assign: ( + # (ARefField | ConstPathField | Field | TopConstField | VarField) target, + # untyped value + # ) -> Assign + def on_assign(target, value) + Assign.new( + target: target, + value: value, + location: target.location.to(value.location) + ) + end + + # Assoc represents a key-value pair within a hash. It is a child node of + # either an AssocListFromArgs or a BareAssocHash. + # + # { key1: value1, key2: value2 } + # + # In the above example, the would be two AssocNew nodes. + class Assoc + # [untyped] the key of this pair + attr_reader :key + + # [untyped] the value of this pair + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(key:, value:, location:) + @key = key + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('assoc') + q.breakable + q.pp(key) + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :assoc, key: key, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_assoc_new: (untyped key, untyped value) -> Assoc + def on_assoc_new(key, value) + Assoc.new( + key: key, + value: value, + location: key.location.to(value.location) + ) + end + + # AssocSplat represents double-splatting a value into a hash (either a hash + # literal or a bare hash in a method call). + # + # { **pairs } + # + class AssocSplat + # [untyped] the expression that is being splatted + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('assoc_splat') + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :assoc_splat, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_assoc_splat: (untyped value) -> AssocSplat + def on_assoc_splat(value) + operator = find_token(Op, '**') + + AssocSplat.new(value: value, location: operator.location.to(value.location)) + end + + # AssocListFromArgs represents the key-value pairs of a hash literal. Its + # parent node is always a hash. + # + # { key1: value1, key2: value2 } + # + class AssocListFromArgs + # [Array[ AssocNew | AssocSplat ]] + attr_reader :assocs + + # [Location] the location of this node + attr_reader :location + + def initialize(assocs:, location:) + @assocs = assocs + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('assoclist_from_args') + q.breakable + q.group(2, '(', ')') { q.seplist(assocs) { |assoc| q.pp(assoc) } } + end + end + + def to_json(*opts) + { type: :assoclist_from_args, assocs: assocs, loc: location }.to_json( + *opts + ) + end + end + + # :call-seq: + # on_assoclist_from_args: ( + # Array[AssocNew | AssocSplat] assocs + # ) -> AssocListFromArgs + def on_assoclist_from_args(assocs) + AssocListFromArgs.new( + assocs: assocs, + location: assocs[0].location.to(assocs[-1].location) + ) + end + + # Backref represents a global variable referencing a matched value. It comes + # in the form of a $ followed by a positive integer. + # + # $1 + # + class Backref + # [String] the name of the global backreference variable + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('backref') + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :backref, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_backref: (String value) -> Backref + def on_backref(value) + node = + Backref.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # Backtick represents the use of the ` operator. It's usually found being used + # for an XStringLiteral, but could also be found as the name of a method being + # defined. + class Backtick + # [String] the backtick in the string + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('backtick') + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :backtick, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_backtick: (String value) -> Backtick + def on_backtick(value) + node = + Backtick.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # BareAssocHash represents a hash of contents being passed as a method + # argument (and therefore has omitted braces). It's very similar to an + # AssocListFromArgs node. + # + # method(key1: value1, key2: value2) + # + class BareAssocHash + # [Array[ AssocNew | AssocSplat ]] + attr_reader :assocs + + # [Location] the location of this node + attr_reader :location + + def initialize(assocs:, location:) + @assocs = assocs + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('bare_assoc_hash') + q.breakable + q.group(2, '(', ')') { q.seplist(assocs) { |assoc| q.pp(assoc) } } + end + end + + def to_json(*opts) + { type: :bare_assoc_hash, assocs: assocs, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_bare_assoc_hash: (Array[AssocNew | AssocSplat] assocs) -> BareAssocHash + def on_bare_assoc_hash(assocs) + BareAssocHash.new( + assocs: assocs, + location: assocs[0].location.to(assocs[-1].location) + ) + end + + # Begin represents a begin..end chain. + # + # begin + # value + # end + # + class Begin + # [BodyStmt] the bodystmt that contains the contents of this begin block + attr_reader :bodystmt + + # [Location] the location of this node + attr_reader :location + + def initialize(bodystmt:, location:) + @bodystmt = bodystmt + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('begin') + q.breakable + q.pp(bodystmt) + end + end + + def to_json(*opts) + { type: :begin, bodystmt: bodystmt, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_begin: (BodyStmt bodystmt) -> Begin + def on_begin(bodystmt) + keyword = find_token(Kw, 'begin') + end_char = + if bodystmt.rescue_clause || bodystmt.ensure_clause || + bodystmt.else_clause + bodystmt.location.end_char + else + find_token(Kw, 'end').location.end_char + end + + bodystmt.bind(keyword.location.end_char, end_char) + + Begin.new( + bodystmt: bodystmt, + location: keyword.location.to(bodystmt.location) + ) + end + + # Binary represents any expression that involves two sub-expressions with an + # operator in between. This can be something that looks like a mathematical + # operation: + # + # 1 + 1 + # + # but can also be something like pushing a value onto an array: + # + # array << value + # + class Binary + # [untyped] the left-hand side of the expression + attr_reader :left + + # [String] the operator used between the two expressions + attr_reader :operator + + # [untyped] the right-hand side of the expression + attr_reader :right + + # [Location] the location of this node + attr_reader :location + + def initialize(left:, operator:, right:, location:) + @left = left + @operator = operator + @right = right + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('binary') + q.breakable + q.pp(left) + q.breakable + q.text(operator) + q.breakable + q.pp(right) + end + end + + def to_json(*opts) + { + type: :binary, + left: left, + op: operator, + right: right, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_binary: (untyped left, (Op | Symbol) operator, untyped right) -> Binary + def on_binary(left, operator, right) + # On most Ruby implementations, operator is a Symbol that represents that + # operation being performed. For instance in the example `1 < 2`, the + # `operator` object would be `:<`. However, on JRuby, it's an `@op` node, + # so here we're going to explicitly convert it into the same normalized + # form. + operator = tokens.delete(operator).value unless operator.is_a?(Symbol) + + Binary.new( + left: left, + operator: operator, + right: right, + location: left.location.to(right.location) + ) + end + + # BlockVar represents the parameters being declared for a block. Effectively + # this node is everything contained within the pipes. This includes all of the + # various parameter types, as well as block-local variable declarations. + # + # method do |positional, optional = value, keyword:, █ local| + # end + # + class BlockVar + # [Params] the parameters being declared with the block + attr_reader :params + + # [Array[ Ident ]] the list of block-local variable declarations + attr_reader :locals + + # [Location] the location of this node + attr_reader :location + + def initialize(params:, locals:, location:) + @params = params + @locals = locals + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('block_var') + q.breakable + q.pp(params) + + if locals.any? + q.breakable + q.group(2, '(', ')') { q.seplist(locals) { |local| q.pp(local) } } + end + end + end + + def to_json(*opts) + { + type: :block_var, + params: params, + locals: locals, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_block_var: (Params params, (nil | Array[Ident]) locals) -> BlockVar + def on_block_var(params, locals) + index = + tokens.rindex do |node| + node.is_a?(Op) && %w[| ||].include?(node.value) && + node.location.start_char < params.location.start_char + end + + beginning = tokens[index] + ending = tokens[-1] + + BlockVar.new( + params: params, + locals: locals || [], + location: beginning.location.to(ending.location) + ) + end + + # BlockArg represents declaring a block parameter on a method definition. + # + # def method(&block); end + # + class BlockArg + # [Ident] the name of the block argument + attr_reader :name + + # [Location] the location of this node + attr_reader :location + + def initialize(name:, location:) + @name = name + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('blockarg') + q.breakable + q.pp(name) + end + end + + def to_json(*opts) + { type: :blockarg, name: name, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_blockarg: (Ident name) -> BlockArg + def on_blockarg(name) + operator = find_token(Op, '&') + + BlockArg.new(name: name, location: operator.location.to(name.location)) + end + + # bodystmt can't actually determine its bounds appropriately because it + # doesn't necessarily know where it started. So the parent node needs to + # report back down into this one where it goes. + class BodyStmt + # [Statements] the list of statements inside the begin clause + attr_reader :statements + + # [nil | Rescue] the optional rescue chain attached to the begin clause + attr_reader :rescue_clause + + # [nil | Statements] the optional set of statements inside the else clause + attr_reader :else_clause + + # [nil | Ensure] the optional ensure clause + attr_reader :ensure_clause + + # [Location] the location of this node + attr_reader :location + + def initialize( + statements:, + rescue_clause:, + else_clause:, + ensure_clause:, + location: + ) + @statements = statements + @rescue_clause = rescue_clause + @else_clause = else_clause + @ensure_clause = ensure_clause + @location = location + end + + def bind(start_char, end_char) + @location = + Location.new( + start_line: location.start_line, + start_char: start_char, + end_line: location.end_line, + end_char: end_char + ) + + parts = [rescue_clause, else_clause, ensure_clause] + + # Here we're going to determine the bounds for the statements + consequent = parts.compact.first + statements.bind( + start_char, + consequent ? consequent.location.start_char : end_char + ) + + # Next we're going to determine the rescue clause if there is one + if rescue_clause + consequent = parts.drop(1).compact.first + rescue_clause.bind_end( + consequent ? consequent.location.start_char : end_char + ) + end + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('bodystmt') + q.breakable + q.pp(statements) + + if rescue_clause + q.breakable + q.pp(rescue_clause) + end + + if else_clause + q.breakable + q.pp(else_clause) + end + + if ensure_clause + q.breakable + q.pp(ensure_clause) + end + end + end + + def to_json(*opts) + { + type: :bodystmt, + stmts: statements, + rsc: rescue_clause, + els: else_clause, + ens: ensure_clause, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_bodystmt: ( + # Statements statements, + # (nil | Rescue) rescue_clause, + # (nil | Statements) else_clause, + # (nil | Ensure) ensure_clause + # ) -> BodyStmt + def on_bodystmt(statements, rescue_clause, else_clause, ensure_clause) + BodyStmt.new( + statements: statements, + rescue_clause: rescue_clause, + else_clause: else_clause, + ensure_clause: ensure_clause, + location: Location.fixed(line: lineno, char: char_pos) + ) + end + + # BraceBlock represents passing a block to a method call using the { } + # operators. + # + # method { |variable| variable + 1 } + # + class BraceBlock + # [LBrace] the left brace that opens this block + attr_reader :lbrace + + # [nil | BlockVar] the optional set of parameters to the block + attr_reader :block_var + + # [Statements] the list of expressions to evaluate within the block + attr_reader :statements + + # [Location] the location of this node + attr_reader :location + + def initialize(lbrace:, block_var:, statements:, location:) + @lbrace = lbrace + @block_var = block_var + @statements = statements + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('brace_block') + + if block_var + q.breakable + q.pp(block_var) + end + + q.breakable + q.pp(statements) + end + end + + def to_json(*opts) + { + type: :brace_block, + lbrace: lbrace, + block_var: block_var, + stmts: statements, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_brace_block: ( + # (nil | BlockVar) block_var, + # Statements statements + # ) -> BraceBlock + def on_brace_block(block_var, statements) + lbrace = find_token(LBrace) + rbrace = find_token(RBrace) + + statements.bind( + find_next_statement_start((block_var || lbrace).location.end_char), + rbrace.location.start_char + ) + + location = + Location.new( + start_line: lbrace.location.start_line, + start_char: lbrace.location.start_char, + end_line: [rbrace.location.end_line, statements.location.end_line].max, + end_char: rbrace.location.end_char + ) + + BraceBlock.new( + lbrace: lbrace, + block_var: block_var, + statements: statements, + location: location + ) + end + + # Break represents using the +break+ keyword. + # + # break + # + # It can also optionally accept arguments, as in: + # + # break 1 + # + class Break + # [Args | ArgsAddBlock] the arguments being sent to the keyword + attr_reader :arguments + + # [Location] the location of this node + attr_reader :location + + def initialize(arguments:, location:) + @arguments = arguments + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('break') + q.breakable + q.pp(arguments) + end + end + + def to_json(*opts) + { type: :break, args: arguments, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_break: ((Args | ArgsAddBlock) arguments) -> Break + def on_break(arguments) + keyword = find_token(Kw, 'break') + + location = keyword.location + location = location.to(arguments.location) unless arguments.is_a?(Args) + + Break.new(arguments: arguments, location: location) + end + + # Call represents a method call. This node doesn't contain the arguments being + # passed (if arguments are passed, this node will get nested under a + # MethodAddArg node). + # + # receiver.message + # + class Call + # [untyped] the receiver of the method call + attr_reader :receiver + + # [:"::" | Op | Period] the operator being used to send the message + attr_reader :operator + + # [:call | Backtick | Const | Ident | Op] the message being sent + attr_reader :message + + # [Location] the location of this node + attr_reader :location + + def initialize(receiver:, operator:, message:, location:) + @receiver = receiver + @operator = operator + @message = message + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('call') + q.breakable + q.pp(receiver) + q.breakable + q.pp(operator) + q.breakable + q.pp(message) + end + end + + def to_json(*opts) + { + type: :call, + receiver: receiver, + op: operator, + message: message, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_call: ( + # untyped receiver, + # (:"::" | Op | Period) operator, + # (:call | Backtick | Const | Ident | Op) message + # ) -> Call + def on_call(receiver, operator, message) + ending = message + ending = operator if message == :call + + Call.new( + receiver: receiver, + operator: operator, + message: message, + location: + Location.new( + start_line: receiver.location.start_line, + start_char: receiver.location.start_char, + end_line: [ending.location.end_line, receiver.location.end_line].max, + end_char: ending.location.end_char + ) + ) + end + + # Case represents the beginning of a case chain. + # + # case value + # when 1 + # "one" + # when 2 + # "two" + # else + # "number" + # end + # + class Case + # [nil | untyped] optional value being switched on + attr_reader :value + + # [In | When] the next clause in the chain + attr_reader :consequent + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, consequent:, location:) + @value = value + @consequent = consequent + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('case') + + if value + q.breakable + q.pp(value) + end + + q.breakable + q.pp(consequent) + end + end + + def to_json(*opts) + { type: :case, value: value, cons: consequent, loc: location }.to_json( + *opts + ) + end + end + + # RAssign represents a single-line pattern match. + # + # value in pattern + # value => pattern + # + class RAssign + # [untyped] the left-hand expression + attr_reader :value + + # [Kw | Op] the operator being used to match against the pattern, which is + # either => or in + attr_reader :operator + + # [untyped] the pattern on the right-hand side of the expression + attr_reader :pattern + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, operator:, pattern:, location:) + @value = value + @operator = operator + @pattern = pattern + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('rassign') + + q.breakable + q.pp(value) + + q.breakable + q.pp(operator) + + q.breakable + q.pp(pattern) + end + end + + def to_json(*opts) + { + type: :rassign, + value: value, + op: operator, + pattern: pattern, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_case: (untyped value, untyped consequent) -> Case | RAssign + def on_case(value, consequent) + if keyword = find_token(Kw, 'case', consume: false) + tokens.delete(keyword) + + Case.new( + value: value, + consequent: consequent, + location: keyword.location.to(consequent.location) + ) + else + operator = find_token(Kw, 'in', consume: false) || find_token(Op, '=>') + + RAssign.new( + value: value, + operator: operator, + pattern: consequent, + location: value.location.to(consequent.location) + ) + end + end + + # Class represents defining a class using the +class+ keyword. + # + # class Container + # end + # + # Classes can have path names as their class name in case it's being nested + # under a namespace, as in: + # + # class Namespace::Container + # end + # + # Classes can also be defined as a top-level path, in the case that it's + # already in a namespace but you want to define it at the top-level instead, + # as in: + # + # module OtherNamespace + # class ::Namespace::Container + # end + # end + # + # All of these declarations can also have an optional superclass reference, as + # in: + # + # class Child < Parent + # end + # + # That superclass can actually be any Ruby expression, it doesn't necessarily + # need to be a constant, as in: + # + # class Child < method + # end + # + class ClassDeclaration + # [ConstPathRef | ConstRef | TopConstRef] the name of the class being + # defined + attr_reader :constant + + # [nil | untyped] the optional superclass declaration + attr_reader :superclass + + # [BodyStmt] the expressions to execute within the context of the class + attr_reader :bodystmt + + # [Location] the location of this node + attr_reader :location + + def initialize(constant:, superclass:, bodystmt:, location:) + @constant = constant + @superclass = superclass + @bodystmt = bodystmt + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('class') + + q.breakable + q.pp(constant) + + if superclass + q.breakable + q.pp(superclass) + end + + q.breakable + q.pp(bodystmt) + end + end + + def to_json(*opts) + { + type: :class, + constant: constant, + superclass: superclass, + bodystmt: bodystmt, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_class: ( + # (ConstPathRef | ConstRef | TopConstRef) constant, + # untyped superclass, + # BodyStmt bodystmt + # ) -> ClassDeclaration + def on_class(constant, superclass, bodystmt) + beginning = find_token(Kw, 'class') + ending = find_token(Kw, 'end') + + bodystmt.bind( + find_next_statement_start((superclass || constant).location.end_char), + ending.location.start_char + ) + + ClassDeclaration.new( + constant: constant, + superclass: superclass, + bodystmt: bodystmt, + location: beginning.location.to(ending.location) + ) + end + + # Comma represents the use of the , operator. + class Comma + # [String] the comma in the string + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_comma: (String value) -> Comma + def on_comma(value) + node = + Comma.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # Command represents a method call with arguments and no parentheses. Note + # that Command nodes only happen when there is no explicit receiver for this + # method. + # + # method argument + # + class Command + # [Const | Ident] the message being sent to the implicit receiver + attr_reader :message + + # [Args | ArgsAddBlock] the arguments being sent with the message + attr_reader :arguments + + # [Location] the location of this node + attr_reader :location + + def initialize(message:, arguments:, location:) + @message = message + @arguments = arguments + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('command') + + q.breakable + q.pp(message) + + q.breakable + q.pp(arguments) + end + end + + def to_json(*opts) + { + type: :command, + message: message, + args: arguments, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_command: ( + # (Const | Ident) message, + # (Args | ArgsAddBlock) arguments + # ) -> Command + def on_command(message, arguments) + Command.new( + message: message, + arguments: arguments, + location: message.location.to(arguments.location) + ) + end + + # CommandCall represents a method call on an object with arguments and no + # parentheses. + # + # object.method argument + # + class CommandCall + # [untyped] the receiver of the message + attr_reader :receiver + + # [:"::" | Op | Period] the operator used to send the message + attr_reader :operator + + # [Const | Ident | Op] the message being send + attr_reader :message + + # [Args | ArgsAddBlock] the arguments going along with the message + attr_reader :arguments + + # [Location] the location of this node + attr_reader :location + + def initialize(receiver:, operator:, message:, arguments:, location:) + @receiver = receiver + @operator = operator + @message = message + @arguments = arguments + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('command_call') + + q.breakable + q.pp(receiver) + + q.breakable + q.pp(operator) + + q.breakable + q.pp(message) + + q.breakable + q.pp(arguments) + end + end + + def to_json(*opts) + { + type: :command_call, + receiver: receiver, + op: operator, + message: message, + args: arguments, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_command_call: ( + # untyped receiver, + # (:"::" | Op | Period) operator, + # (Const | Ident | Op) message, + # (Args | ArgsAddBlock) arguments + # ) -> CommandCall + def on_command_call(receiver, operator, message, arguments) + ending = arguments || message + + CommandCall.new( + receiver: receiver, + operator: operator, + message: message, + arguments: arguments, + location: receiver.location.to(ending.location) + ) + end + + # Comment represents a comment in the source. + # + # # comment + # + class Comment + # [String] the contents of the comment + attr_reader :value + + # [boolean] whether or not there is code on the same line as this comment. + # If there is, then inline will be true. + attr_reader :inline + alias inline? inline + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, inline:, location:) + @value = value + @inline = inline + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('comment') + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { + type: :comment, + value: value.force_encoding('UTF-8'), + inline: inline, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_comment: (String value) -> Comment + def on_comment(value) + line = lineno + comment = + Comment.new( + value: value[1..-1].chomp, + inline: value.strip != lines[line - 1], + location: + Location.token(line: line, char: char_pos, size: value.size - 1) + ) + + @comments << comment + comment + end + + # Const represents a literal value that _looks_ like a constant. This could + # actually be a reference to a constant: + # + # Constant + # + # It could also be something that looks like a constant in another context, as + # in a method call to a capitalized method: + # + # object.Constant + # + # or a symbol that starts with a capital letter: + # + # :Constant + # + class Const + # [String] the name of the constant + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('const') + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :const, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_const: (String value) -> Const + def on_const(value) + node = + Const.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # ConstPathField represents the child node of some kind of assignment. It + # represents when you're assigning to a constant that is being referenced as + # a child of another variable. + # + # object::Const = value + # + class ConstPathField + # [untyped] the source of the constant + attr_reader :parent + + # [Const] the constant itself + attr_reader :constant + + # [Location] the location of this node + attr_reader :location + + def initialize(parent:, constant:, location:) + @parent = parent + @constant = constant + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('const_path_field') + + q.breakable + q.pp(parent) + + q.breakable + q.pp(constant) + end + end + + def to_json(*opts) + { + type: :const_path_field, + parent: parent, + constant: constant, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_const_path_field: (untyped parent, Const constant) -> ConstPathField + def on_const_path_field(parent, constant) + ConstPathField.new( + parent: parent, + constant: constant, + location: parent.location.to(constant.location) + ) + end + + # ConstPathRef represents referencing a constant by a path. + # + # object::Const + # + class ConstPathRef + # [untyped] the source of the constant + attr_reader :parent + + # [Const] the constant itself + attr_reader :constant + + # [Location] the location of this node + attr_reader :location + + def initialize(parent:, constant:, location:) + @parent = parent + @constant = constant + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('const_path_ref') + + q.breakable + q.pp(parent) + + q.breakable + q.pp(constant) + end + end + + def to_json(*opts) + { + type: :const_path_ref, + parent: parent, + constant: constant, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_const_path_ref: (untyped parent, Const constant) -> ConstPathRef + def on_const_path_ref(parent, constant) + ConstPathRef.new( + parent: parent, + constant: constant, + location: parent.location.to(constant.location) + ) + end + + # ConstRef represents the name of the constant being used in a class or module + # declaration. + # + # class Container + # end + # + class ConstRef + # [Const] the constant itself + attr_reader :constant + + # [Location] the location of this node + attr_reader :location + + def initialize(constant:, location:) + @constant = constant + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('const_ref') + + q.breakable + q.pp(constant) + end + end + + def to_json(*opts) + { type: :const_ref, constant: constant, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_const_ref: (Const constant) -> ConstRef + def on_const_ref(constant) + ConstRef.new(constant: constant, location: constant.location) + end + + # CVar represents the use of a class variable. + # + # @@variable + # + class CVar + # [String] the name of the class variable + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('cvar') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :cvar, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_cvar: (String value) -> CVar + def on_cvar(value) + node = + CVar.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # Def represents defining a regular method on the current self object. + # + # def method(param) result end + # + class Def + # [Backtick | Const | Ident | Kw | Op] the name of the method + attr_reader :name + + # [Params | Paren] the parameter declaration for the method + attr_reader :params + + # [BodyStmt] the expressions to be executed by the method + attr_reader :bodystmt + + # [Location] the location of this node + attr_reader :location + + def initialize(name:, params:, bodystmt:, location:) + @name = name + @params = params + @bodystmt = bodystmt + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('def') + + q.breakable + q.pp(name) + + q.breakable + q.pp(params) + + q.breakable + q.pp(bodystmt) + end + end + + def to_json(*opts) + { + type: :def, + name: name, + params: params, + bodystmt: bodystmt, + loc: location + }.to_json(*opts) + end + end + + # DefEndless represents defining a single-line method since Ruby 3.0+. + # + # def method = result + # + class DefEndless + # [Backtick | Const | Ident | Kw | Op] the name of the method + attr_reader :name + + # [Paren] the parameter declaration for the method + attr_reader :paren + + # [untyped] the expression to be executed by the method + attr_reader :statement + + # [Location] the location of this node + attr_reader :location + + def initialize(name:, paren:, statement:, location:) + @name = name + @paren = paren + @statement = statement + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('def_endless') + + q.breakable + q.pp(name) + + q.breakable + q.pp(paren) + + q.breakable + q.pp(statement) + end + end + + def to_json(*opts) + { + type: :def_endless, + name: name, + paren: paren, + stmt: statement, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_def: ( + # (Backtick | Const | Ident | Kw | Op) name, + # (Params | Paren) params, + # untyped bodystmt + # ) -> Def | DefEndless + def on_def(name, params, bodystmt) + # Make sure to delete this token in case you're defining something like def + # class which would lead to this being a kw and causing all kinds of trouble + tokens.delete(name) + + # Find the beginning of the method definition, which works for single-line + # and normal method definitions. + beginning = find_token(Kw, 'def') + + # If we don't have a bodystmt node, then we have a single-line method + unless bodystmt.is_a?(BodyStmt) + node = + DefEndless.new( + name: name, + paren: params, + statement: bodystmt, + location: beginning.location.to(bodystmt.location) + ) + + return node + end + + # If there aren't any params then we need to correct the params node + # location information + if params.is_a?(Params) && params.empty? + end_char = name.location.end_char + location = + Location.new( + start_line: params.location.start_line, + start_char: end_char, + end_line: params.location.end_line, + end_char: end_char + ) + + params = Params.new(location: location) + end + + ending = find_token(Kw, 'end') + bodystmt.bind( + find_next_statement_start(params.location.end_char), + ending.location.start_char + ) + + Def.new( + name: name, + params: params, + bodystmt: bodystmt, + location: beginning.location.to(ending.location) + ) + end + + # Defined represents the use of the +defined?+ operator. It can be used with + # and without parentheses. + # + # defined?(variable) + # + class Defined + # [untyped] the value being sent to the keyword + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('defined') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :defined, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_defined: (untyped value) -> Defined + def on_defined(value) + beginning = find_token(Kw, 'defined?') + ending = value + + range = beginning.location.end_char...value.location.start_char + if source[range].include?('(') + find_token(LParen) + ending = find_token(RParen) + end + + Defined.new(value: value, location: beginning.location.to(ending.location)) + end + + # Defs represents defining a singleton method on an object. + # + # def object.method(param) result end + # + class Defs + # [untyped] the target where the method is being defined + attr_reader :target + + # [Op | Period] the operator being used to declare the method + attr_reader :operator + + # [Backtick | Const | Ident | Kw | Op] the name of the method + attr_reader :name + + # [Params | Paren] the parameter declaration for the method + attr_reader :params + + # [BodyStmt] the expressions to be executed by the method + attr_reader :bodystmt + + # [Location] the location of this node + attr_reader :location + + def initialize(target:, operator:, name:, params:, bodystmt:, location:) + @target = target + @operator = operator + @name = name + @params = params + @bodystmt = bodystmt + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('defs') + + q.breakable + q.pp(target) + + q.breakable + q.pp(operator) + + q.breakable + q.pp(name) + + q.breakable + q.pp(params) + + q.breakable + q.pp(bodystmt) + end + end + + def to_json(*opts) + { + type: :defs, + target: target, + op: operator, + name: name, + params: params, + bodystmt: bodystmt, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_defs: ( + # untyped target, + # (Op | Period) operator, + # (Backtick | Const | Ident | Kw | Op) name, + # (Params | Paren) params, + # BodyStmt bodystmt + # ) -> Defs + def on_defs(target, operator, name, params, bodystmt) + # Make sure to delete this token in case you're defining something + # like def class which would lead to this being a kw and causing all kinds + # of trouble + tokens.delete(name) + + # If there aren't any params then we need to correct the params node + # location information + if params.is_a?(Params) && params.empty? + end_char = name.location.end_char + location = + Location.new( + start_line: params.location.start_line, + start_char: end_char, + end_line: params.location.end_line, + end_char: end_char + ) + + params = Params.new(location: location) + end + + beginning = find_token(Kw, 'def') + ending = find_token(Kw, 'end') + + bodystmt.bind( + find_next_statement_start(params.location.end_char), + ending.location.start_char + ) + + Defs.new( + target: target, + operator: operator, + name: name, + params: params, + bodystmt: bodystmt, + location: beginning.location.to(ending.location) + ) + end + + # DoBlock represents passing a block to a method call using the +do+ and +end+ + # keywords. + # + # method do |value| + # end + # + class DoBlock + # [Kw] the do keyword that opens this block + attr_reader :keyword + + # [nil | BlockVar] the optional variable declaration within this block + attr_reader :block_var + + # [BodyStmt] the expressions to be executed within this block + attr_reader :bodystmt + + # [Location] the location of this node + attr_reader :location + + def initialize(keyword:, block_var:, bodystmt:, location:) + @keyword = keyword + @block_var = block_var + @bodystmt = bodystmt + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('do_block') + + if block_var + q.breakable + q.pp(block_var) + end + + q.breakable + q.pp(bodystmt) + end + end + + def to_json(*opts) + { + type: :do_block, + keyword: keyword, + block_var: block_var, + bodystmt: bodystmt, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_do_block: (BlockVar block_var, BodyStmt bodystmt) -> DoBlock + def on_do_block(block_var, bodystmt) + beginning = find_token(Kw, 'do') + ending = find_token(Kw, 'end') + + bodystmt.bind( + find_next_statement_start((block_var || beginning).location.end_char), + ending.location.start_char + ) + + DoBlock.new( + keyword: beginning, + block_var: block_var, + bodystmt: bodystmt, + location: beginning.location.to(ending.location) + ) + end + + # Dot2 represents using the .. operator between two expressions. Usually this + # is to create a range object. + # + # 1..2 + # + # Sometimes this operator is used to create a flip-flop. + # + # if value == 5 .. value == 10 + # end + # + # One of the sides of the expression may be nil, but not both. + class Dot2 + # [nil | untyped] the left side of the expression + attr_reader :left + + # [nil | untyped] the right side of the expression + attr_reader :right + + # [Location] the location of this node + attr_reader :location + + def initialize(left:, right:, location:) + @left = left + @right = right + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('dot2') + + if left + q.breakable + q.pp(left) + end + + if right + q.breakable + q.pp(right) + end + end + end + + def to_json(*opts) + { type: :dot2, left: left, right: right, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_dot2: ((nil | untyped) left, (nil | untyped) right) -> Dot2 + def on_dot2(left, right) + operator = find_token(Op, '..') + + beginning = left || operator + ending = right || operator + + Dot2.new( + left: left, + right: right, + location: beginning.location.to(ending.location) + ) + end + + # Dot3 represents using the ... operator between two expressions. Usually this + # is to create a range object. It's effectively the same event as the Dot2 + # node but with this operator you're asking Ruby to omit the final value. + # + # 1...2 + # + # Like Dot2 it can also be used to create a flip-flop. + # + # if value == 5 ... value == 10 + # end + # + # One of the sides of the expression may be nil, but not both. + class Dot3 + # [nil | untyped] the left side of the expression + attr_reader :left + + # [nil | untyped] the right side of the expression + attr_reader :right + + # [Location] the location of this node + attr_reader :location + + def initialize(left:, right:, location:) + @left = left + @right = right + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('dot3') + + if left + q.breakable + q.pp(left) + end + + if right + q.breakable + q.pp(right) + end + end + end + + def to_json(*opts) + { type: :dot3, left: left, right: right, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_dot3: ((nil | untyped) left, (nil | untyped) right) -> Dot3 + def on_dot3(left, right) + operator = find_token(Op, '...') + + beginning = left || operator + ending = right || operator + + Dot3.new( + left: left, + right: right, + location: beginning.location.to(ending.location) + ) + end + + # DynaSymbol represents a symbol literal that uses quotes to dynamically + # define its value. + # + # :"#{variable}" + # + # They can also be used as a special kind of dynamic hash key, as in: + # + # { "#{key}": value } + # + class DynaSymbol + # [Array[ StringDVar | StringEmbExpr | TStringContent ]] the parts of the + # dynamic symbol + attr_reader :parts + + # [String] the quote used to delimit the dynamic symbol + attr_reader :quote + + # [Location] the location of this node + attr_reader :location + + def initialize(parts:, quote:, location:) + @parts = parts + @quote = quote + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('dyna_symbol') + + q.breakable + q.group(2, '(', ')') { q.seplist(parts) { |part| q.pp(part) } } + end + end + + def to_json(*opts) + { type: :dyna_symbol, parts: parts, quote: quote, loc: location }.to_json( + *opts + ) + end + end + + # :call-seq: + # on_dyna_symbol: (StringContent string_content) -> DynaSymbol + def on_dyna_symbol(string_content) + if find_token(SymBeg, consume: false) + # A normal dynamic symbol + symbeg = find_token(SymBeg) + tstring_end = find_token(TStringEnd) + + DynaSymbol.new( + quote: symbeg.value, + parts: string_content.parts, + location: symbeg.location.to(tstring_end.location) + ) + else + # A dynamic symbol as a hash key + tstring_beg = find_token(TStringBeg) + label_end = find_token(LabelEnd) + + DynaSymbol.new( + parts: string_content.parts, + quote: label_end.value[0], + location: tstring_beg.location.to(label_end.location) + ) + end + end + + # Else represents the end of an +if+, +unless+, or +case+ chain. + # + # if variable + # else + # end + # + class Else + # [Statements] the expressions to be executed + attr_reader :statements + + # [Location] the location of this node + attr_reader :location + + def initialize(statements:, location:) + @statements = statements + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('else') + + q.breakable + q.pp(statements) + end + end + + def to_json(*opts) + { type: :else, stmts: statements, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_else: (Statements statements) -> Else + def on_else(statements) + beginning = find_token(Kw, 'else') + + # else can either end with an end keyword (in which case we'll want to + # consume that event) or it can end with an ensure keyword (in which case + # we'll leave that to the ensure to handle). + index = + tokens.rindex do |token| + token.is_a?(Kw) && %w[end ensure].include?(token.value) + end + + node = tokens[index] + ending = node.value == 'end' ? tokens.delete_at(index) : node + + statements.bind(beginning.location.end_char, ending.location.start_char) + + Else.new( + statements: statements, + location: beginning.location.to(ending.location) + ) + end + + # Elsif represents another clause in an +if+ or +unless+ chain. + # + # if variable + # elsif other_variable + # end + # + class Elsif + # [untyped] the expression to be checked + attr_reader :predicate + + # [Statements] the expressions to be executed + attr_reader :statements + + # [nil | Elsif | Else] the next clause in the chain + attr_reader :consequent + + # [Location] the location of this node + attr_reader :location + + def initialize(predicate:, statements:, consequent:, location:) + @predicate = predicate + @statements = statements + @consequent = consequent + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('elsif') + + q.breakable + q.pp(predicate) + + q.breakable + q.pp(statements) + + if consequent + q.breakable + q.pp(consequent) + end + end + end + + def to_json(*opts) + { + type: :elsif, + pred: predicate, + stmts: statements, + cons: consequent, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_elsif: ( + # untyped predicate, + # Statements statements, + # (nil | Elsif | Else) consequent + # ) -> Elsif + def on_elsif(predicate, statements, consequent) + beginning = find_token(Kw, 'elsif') + ending = consequent || find_token(Kw, 'end') + + statements.bind(predicate.location.end_char, ending.location.start_char) + + Elsif.new( + predicate: predicate, + statements: statements, + consequent: consequent, + location: beginning.location.to(ending.location) + ) + end + + # EmbDoc represents a multi-line comment. + # + # =begin + # first line + # second line + # =end + # + class EmbDoc + # [String] the contents of the comment + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def inline? + false + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('embdoc') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :embdoc, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_embdoc: (String value) -> EmbDoc + def on_embdoc(value) + @embdoc.value << value + @embdoc + end + + # :call-seq: + # on_embdoc_beg: (String value) -> EmbDoc + def on_embdoc_beg(value) + @embdoc = + EmbDoc.new( + value: value, + location: Location.fixed(line: lineno, char: char_pos) + ) + end + + # :call-seq: + # on_embdoc_end: (String value) -> EmbDoc + def on_embdoc_end(value) + location = @embdoc.location + embdoc = + EmbDoc.new( + value: @embdoc.value << value.chomp, + location: + Location.new( + start_line: location.start_line, + start_char: location.start_char, + end_line: lineno, + end_char: char_pos + value.length - 1 + ) + ) + + @comments << embdoc + @embdoc = nil + + embdoc + end + + # EmbExprBeg represents the beginning token for using interpolation inside of + # a parent node that accepts string content (like a string or regular + # expression). + # + # "Hello, #{person}!" + # + class EmbExprBeg + # [String] the #{ used in the string + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_embexpr_beg: (String value) -> EmbExprBeg + def on_embexpr_beg(value) + node = + EmbExprBeg.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # EmbExprEnd represents the ending token for using interpolation inside of a + # parent node that accepts string content (like a string or regular + # expression). + # + # "Hello, #{person}!" + # + class EmbExprEnd + # [String] the } used in the string + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_embexpr_end: (String value) -> EmbExprEnd + def on_embexpr_end(value) + node = + EmbExprEnd.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # EmbVar represents the use of shorthand interpolation for an instance, class, + # or global variable into a parent node that accepts string content (like a + # string or regular expression). + # + # "#@variable" + # + # In the example above, an EmbVar node represents the # because it forces + # @variable to be interpolated. + class EmbVar + # [String] the # used in the string + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_embvar: (String value) -> EmbVar + def on_embvar(value) + node = + EmbVar.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # Ensure represents the use of the +ensure+ keyword and its subsequent + # statements. + # + # begin + # ensure + # end + # + class Ensure + # [Kw] the ensure keyword that began this node + attr_reader :keyword + + # [Statements] the expressions to be executed + attr_reader :statements + + # [Location] the location of this node + attr_reader :location + + def initialize(keyword:, statements:, location:) + @keyword = keyword + @statements = statements + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('ensure') + + q.breakable + q.pp(statements) + end + end + + def to_json(*opts) + { + type: :ensure, + keyword: keyword, + stmts: statements, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_ensure: (Statements statements) -> Ensure + def on_ensure(statements) + keyword = find_token(Kw, 'ensure') + + # We don't want to consume the :@kw event, because that would break + # def..ensure..end chains. + ending = find_token(Kw, 'end', consume: false) + statements.bind( + find_next_statement_start(keyword.location.end_char), + ending.location.start_char + ) + + Ensure.new( + keyword: keyword, + statements: statements, + location: keyword.location.to(ending.location) + ) + end + + # ExcessedComma represents a trailing comma in a list of block parameters. It + # changes the block parameters such that they will destructure. + # + # [[1, 2, 3], [2, 3, 4]].each do |first, second,| + # end + # + # In the above example, an ExcessedComma node would appear in the third + # position of the Params node that is used to declare that block. The third + # position typically represents a rest-type parameter, but in this case is + # used to indicate that a trailing comma was used. + class ExcessedComma + # [String] the comma + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('excessed_comma') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :excessed_comma, value: value, loc: location }.to_json(*opts) + end + end + + # The handler for this event accepts no parameters (though in previous + # versions of Ruby it accepted a string literal with a value of ","). + # + # :call-seq: + # on_excessed_comma: () -> ExcessedComma + def on_excessed_comma(*) + comma = find_token(Comma) + + ExcessedComma.new(value: comma.value, location: comma.location) + end + + # FCall represents the piece of a method call that comes before any arguments + # (i.e., just the name of the method). It is used in places where the parser + # is sure that it is a method call and not potentially a local variable. + # + # method(argument) + # + # In the above example, it's referring to the +method+ segment. + class FCall + # [Const | Ident] the name of the method + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('fcall') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :fcall, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_fcall: ((Const | Ident) value) -> FCall + def on_fcall(value) + FCall.new(value: value, location: value.location) + end + + # Field is always the child of an assignment. It represents assigning to a + # “field” on an object. + # + # object.variable = value + # + class Field + # [untyped] the parent object that owns the field being assigned + attr_reader :parent + + # [:"::" | Op | Period] the operator being used for the assignment + attr_reader :operator + + # [Const | Ident] the name of the field being assigned + attr_reader :name + + # [Location] the location of this node + attr_reader :location + + def initialize(parent:, operator:, name:, location:) + @parent = parent + @operator = operator + @name = name + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('field') + + q.breakable + q.pp(parent) + + q.breakable + q.pp(operator) + + q.breakable + q.pp(name) + end + end + + def to_json(*opts) + { + type: :field, + parent: parent, + op: operator, + name: name, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_field: ( + # untyped parent, + # (:"::" | Op | Period) operator + # (Const | Ident) name + # ) -> Field + def on_field(parent, operator, name) + Field.new( + parent: parent, + operator: operator, + name: name, + location: parent.location.to(name.location) + ) + end + + # FloatLiteral represents a floating point number literal. + # + # 1.0 + # + class FloatLiteral + # [String] the value of the floating point number literal + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('float') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :float, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_float: (String value) -> FloatLiteral + def on_float(value) + node = + FloatLiteral.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # FndPtn represents matching against a pattern where you find a pattern in an + # array using the Ruby 3.0+ pattern matching syntax. + # + # case value + # in [*, 7, *] + # end + # + class FndPtn + # [nil | untyped] the optional constant wrapper + attr_reader :constant + + # [VarField] the splat on the left-hand side + attr_reader :left + + # [Array[ untyped ]] the list of positional expressions in the pattern that + # are being matched + attr_reader :values + + # [VarField] the splat on the right-hand side + attr_reader :right + + # [Location] the location of this node + attr_reader :location + + def initialize(constant:, left:, values:, right:, location:) + @constant = constant + @left = left + @values = values + @right = right + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('fndptn') + + if constant + q.breakable + q.pp(constant) + end + + q.breakable + q.pp(left) + + q.breakable + q.group(2, '(', ')') { q.seplist(values) { |value| q.pp(value) } } + + q.breakable + q.pp(right) + end + end + + def to_json(*opts) + { + type: :fndptn, + constant: constant, + left: left, + values: values, + right: right, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_fndptn: ( + # (nil | untyped) constant, + # VarField left, + # Array[untyped] values, + # VarField right + # ) -> FndPtn + def on_fndptn(constant, left, values, right) + beginning = constant || find_token(LBracket) + ending = find_token(RBracket) + + FndPtn.new( + constant: constant, + left: left, + values: values, + right: right, + location: beginning.location.to(ending.location) + ) + end + + # For represents using a +for+ loop. + # + # for value in list do + # end + # + class For + # [MLHS | MLHSAddStar | VarField] the variable declaration being used to + # pull values out of the object being enumerated + attr_reader :index + + # [untyped] the object being enumerated in the loop + attr_reader :collection + + # [Statements] the statements to be executed + attr_reader :statements + + # [Location] the location of this node + attr_reader :location + + def initialize(index:, collection:, statements:, location:) + @index = index + @collection = collection + @statements = statements + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('for') + + q.breakable + q.pp(index) + + q.breakable + q.pp(collection) + + q.breakable + q.pp(statements) + end + end + + def to_json(*opts) + { + type: :for, + index: index, + collection: collection, + stmts: statements, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_for: ( + # (MLHS | MLHSAddStar | VarField) value, + # untyped collection, + # Statements statements + # ) -> For + def on_for(index, collection, statements) + beginning = find_token(Kw, 'for') + ending = find_token(Kw, 'end') + + # Consume the do keyword if it exists so that it doesn't get confused for + # some other block + keyword = find_token(Kw, 'do', consume: false) + if keyword && keyword.location.start_char > collection.location.end_char && + keyword.location.end_char < ending.location.start_char + tokens.delete(keyword) + end + + statements.bind( + (keyword || collection).location.end_char, + ending.location.start_char + ) + + For.new( + index: index, + collection: collection, + statements: statements, + location: beginning.location.to(ending.location) + ) + end + + # GVar represents a global variable literal. + # + # $variable + # + class GVar + # [String] the name of the global variable + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('gvar') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :gvar, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_gvar: (String value) -> GVar + def on_gvar(value) + node = + GVar.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # HashLiteral represents a hash literal. + # + # { key => value } + # + class HashLiteral + # [nil | AssocListFromArgs] the contents of the hash + attr_reader :contents + + # [Location] the location of this node + attr_reader :location + + def initialize(contents:, location:) + @contents = contents + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('hash') + + q.breakable + q.pp(contents) + end + end + + def to_json(*opts) + { type: :hash, cnts: contents, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_hash: ((nil | AssocListFromArgs) contents) -> HashLiteral + def on_hash(contents) + lbrace = find_token(LBrace) + rbrace = find_token(RBrace) + + if contents + # Here we're going to expand out the location information for the contents + # node so that it can grab up any remaining comments inside the hash. + location = + Location.new( + start_line: contents.location.start_line, + start_char: lbrace.location.end_char, + end_line: contents.location.end_line, + end_char: rbrace.location.start_char + ) + + contents = contents.class.new(assocs: contents.assocs, location: location) + end + + HashLiteral.new( + contents: contents, + location: lbrace.location.to(rbrace.location) + ) + end + + # Heredoc represents a heredoc string literal. + # + # <<~DOC + # contents + # DOC + # + class Heredoc + # [HeredocBeg] the opening of the heredoc + attr_reader :beginning + + # [String] the ending of the heredoc + attr_reader :ending + + # [Array[ StringEmbExpr | StringDVar | TStringContent ]] the parts of the + # heredoc string literal + attr_reader :parts + + # [Location] the location of this node + attr_reader :location + + def initialize(beginning:, ending: nil, parts: [], location:) + @beginning = beginning + @ending = ending + @parts = parts + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('heredoc') + + q.breakable + q.group(2, '(', ')') { q.seplist(parts) { |part| q.pp(part) } } + end + end + + def to_json(*opts) + { + type: :heredoc, + beging: beginning, + ending: ending, + parts: parts, + loc: location + }.to_json(*opts) + end + end + + # HeredocBeg represents the beginning declaration of a heredoc. + # + # <<~DOC + # contents + # DOC + # + # In the example above the HeredocBeg node represents <<~DOC. + class HeredocBeg + # [String] the opening declaration of the heredoc + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('heredoc_beg') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :heredoc_beg, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_heredoc_beg: (String value) -> HeredocBeg + def on_heredoc_beg(value) + location = + Location.token(line: lineno, char: char_pos, size: value.size + 1) + + # Here we're going to artificially create an extra node type so that if + # there are comments after the declaration of a heredoc, they get printed. + beginning = HeredocBeg.new(value: value, location: location) + @heredocs << Heredoc.new(beginning: beginning, location: location) + + beginning + end + + # :call-seq: + # on_heredoc_dedent: (StringContent string, Integer width) -> Heredoc + def on_heredoc_dedent(string, width) + heredoc = @heredocs[-1] + + @heredocs[-1] = + Heredoc.new( + beginning: heredoc.beginning, + ending: heredoc.ending, + parts: string.parts, + location: heredoc.location + ) + end + + # :call-seq: + # on_heredoc_end: (String value) -> Heredoc + def on_heredoc_end(value) + heredoc = @heredocs[-1] + + @heredocs[-1] = + Heredoc.new( + beginning: heredoc.beginning, + ending: value.chomp, + parts: heredoc.parts, + location: + Location.new( + start_line: heredoc.location.start_line, + start_char: heredoc.location.start_char, + end_line: lineno, + end_char: char_pos + ) + ) + end + + # HshPtn represents matching against a hash pattern using the Ruby 2.7+ + # pattern matching syntax. + # + # case value + # in { key: } + # end + # + class HshPtn + # [nil | untyped] the optional constant wrapper + attr_reader :constant + + # [Array[ [Label, untyped] ]] the set of tuples representing the keywords + # that should be matched against in the pattern + attr_reader :keywords + + # [nil | VarField] an optional parameter to gather up all remaining keywords + attr_reader :keyword_rest + + # [Location] the location of this node + attr_reader :location + + def initialize(constant:, keywords:, keyword_rest:, location:) + @constant = constant + @keywords = keywords + @keyword_rest = keyword_rest + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('hshptn') + + if constant + q.breakable + q.pp(constant) + end + + if keywords.any? + q.breakable + q.group(2, '(', ')') do + q.seplist(keywords) { |keyword| q.pp(keyword) } + end + end + + if keyword_rest + q.breakable + q.pp(keyword_rest) + end + end + end + + def to_json(*opts) + { + type: :hshptn, + constant: constant, + keywords: keywords, + kwrest: keyword_rest, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_hshptn: ( + # (nil | untyped) constant, + # Array[[Label, untyped]] keywords, + # (nil | VarField) keyword_rest + # ) -> HshPtn + def on_hshptn(constant, keywords, keyword_rest) + parts = [constant, keywords, keyword_rest].flatten(2).compact + + HshPtn.new( + constant: constant, + keywords: keywords, + keyword_rest: keyword_rest, + location: parts[0].location.to(parts[-1].location) + ) + end + + # Ident represents an identifier anywhere in code. It can represent a very + # large number of things, depending on where it is in the syntax tree. + # + # value + # + class Ident + # [String] the value of the identifier + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('ident') + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { + type: :ident, + value: value.force_encoding('UTF-8'), + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_ident: (String value) -> Ident + def on_ident(value) + node = + Ident.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # If represents the first clause in an +if+ chain. + # + # if predicate + # end + # + class If + # [untyped] the expression to be checked + attr_reader :predicate + + # [Statements] the expressions to be executed + attr_reader :statements + + # [nil, Elsif, Else] the next clause in the chain + attr_reader :consequent + + # [Location] the location of this node + attr_reader :location + + def initialize(predicate:, statements:, consequent:, location:) + @predicate = predicate + @statements = statements + @consequent = consequent + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('if') + + q.breakable + q.pp(predicate) + + q.breakable + q.pp(statements) + + if consequent + q.breakable + q.pp(consequent) + end + end + end + + def to_json(*opts) + { + type: :if, + pred: predicate, + stmts: statements, + cons: consequent, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_if: ( + # untyped predicate, + # Statements statements, + # (nil | Elsif | Else) consequent + # ) -> If + def on_if(predicate, statements, consequent) + beginning = find_token(Kw, 'if') + ending = consequent || find_token(Kw, 'end') + + statements.bind(predicate.location.end_char, ending.location.start_char) + + If.new( + predicate: predicate, + statements: statements, + consequent: consequent, + location: beginning.location.to(ending.location) + ) + end + + # IfOp represents a ternary clause. + # + # predicate ? truthy : falsy + # + class IfOp + # [untyped] the expression to be checked + attr_reader :predicate + + # [untyped] the expression to be executed if the predicate is truthy + attr_reader :truthy + + # [untyped] the expression to be executed if the predicate is falsy + attr_reader :falsy + + # [Location] the location of this node + attr_reader :location + + def initialize(predicate:, truthy:, falsy:, location:) + @predicate = predicate + @truthy = truthy + @falsy = falsy + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('ifop') + + q.breakable + q.pp(predicate) + + q.breakable + q.pp(truthy) + + q.breakable + q.pp(falsy) + end + end + + def to_json(*opts) + { + type: :ifop, + pred: predicate, + tthy: truthy, + flsy: falsy, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_ifop: (untyped predicate, untyped truthy, untyped falsy) -> IfOp + def on_ifop(predicate, truthy, falsy) + IfOp.new( + predicate: predicate, + truthy: truthy, + falsy: falsy, + location: predicate.location.to(falsy.location) + ) + end + + # IfMod represents the modifier form of an +if+ statement. + # + # expression if predicate + # + class IfMod + # [untyped] the expression to be executed + attr_reader :statement + + # [untyped] the expression to be checked + attr_reader :predicate + + # [Location] the location of this node + attr_reader :location + + def initialize(statement:, predicate:, location:) + @statement = statement + @predicate = predicate + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('if_mod') + + q.breakable + q.pp(statement) + + q.breakable + q.pp(predicate) + end + end + + def to_json(*opts) + { + type: :if_mod, + stmt: statement, + pred: predicate, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_if_mod: (untyped predicate, untyped statement) -> IfMod + def on_if_mod(predicate, statement) + find_token(Kw, 'if') + + IfMod.new( + statement: statement, + predicate: predicate, + location: statement.location.to(predicate.location) + ) + end + + # def on_ignored_nl(value) + # value + # end + + # def on_ignored_sp(value) + # value + # end + + # Imaginary represents an imaginary number literal. + # + # 1i + # + class Imaginary + # [String] the value of the imaginary number literal + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('imaginary') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :imaginary, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_imaginary: (String value) -> Imaginary + def on_imaginary(value) + node = + Imaginary.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # In represents using the +in+ keyword within the Ruby 2.7+ pattern matching + # syntax. + # + # case value + # in pattern + # end + # + class In + # [untyped] the pattern to check against + attr_reader :pattern + + # [Statements] the expressions to execute if the pattern matched + attr_reader :statements + + # [nil | In | Else] the next clause in the chain + attr_reader :consequent + + # [Location] the location of this node + attr_reader :location + + def initialize(pattern:, statements:, consequent:, location:) + @pattern = pattern + @statements = statements + @consequent = consequent + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('in') + + q.breakable + q.pp(pattern) + + q.breakable + q.pp(statements) + + if consequent + q.breakable + q.pp(consequent) + end + end + end + + def to_json(*opts) + { + type: :in, + pattern: pattern, + stmts: statements, + cons: consequent, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_in: (RAssign pattern, nil statements, nil consequent) -> RAssign + # | ( + # untyped pattern, + # Statements statements, + # (nil | In | Else) consequent + # ) -> In + def on_in(pattern, statements, consequent) + # Here we have a rightward assignment + return pattern unless statements + + beginning = find_token(Kw, 'in') + ending = consequent || find_token(Kw, 'end') + + statements.bind(beginning.location.end_char, ending.location.start_char) + + In.new( + pattern: pattern, + statements: statements, + consequent: consequent, + location: beginning.location.to(ending.location) + ) + end + + # Int represents an integer number literal. + # + # 1 + # + class Int + # [String] the value of the integer + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('int') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :int, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_int: (String value) -> Int + def on_int(value) + node = + Int.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # IVar represents an instance variable literal. + # + # @variable + # + class IVar + # [String] the name of the instance variable + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('ivar') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :ivar, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_ivar: (String value) -> IVar + def on_ivar(value) + node = + IVar.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # Kw represents the use of a keyword. It can be almost anywhere in the syntax + # tree, so you end up seeing it quite a lot. + # + # if value + # end + # + # In the above example, there would be two Kw nodes: one for the if and one + # for the end. Note that anything that matches the list of keywords in Ruby + # will use a Kw, so if you use a keyword in a symbol literal for instance: + # + # :if + # + # then the contents of the symbol node will contain a Kw node. + class Kw + # [String] the value of the keyword + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('kw') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :kw, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_kw: (String value) -> Kw + def on_kw(value) + node = + Kw.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # KwRestParam represents defining a parameter in a method definition that + # accepts all remaining keyword parameters. + # + # def method(**kwargs) end + # + class KwRestParam + # [nil | Ident] the name of the parameter + attr_reader :name + + # [Location] the location of this node + attr_reader :location + + def initialize(name:, location:) + @name = name + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('kwrest_param') + + q.breakable + q.pp(name) + end + end + + def to_json(*opts) + { type: :kwrest_param, name: name, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_kwrest_param: ((nil | Ident) name) -> KwRestParam + def on_kwrest_param(name) + location = find_token(Op, '**').location + location = location.to(name.location) if name + + KwRestParam.new(name: name, location: location) + end + + # Label represents the use of an identifier to associate with an object. You + # can find it in a hash key, as in: + # + # { key: value } + # + # In this case "key:" would be the body of the label. You can also find it in + # pattern matching, as in: + # + # case value + # in key: + # end + # + # In this case "key:" would be the body of the label. + class Label + # [String] the value of the label + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('label') + + q.breakable + q.text(':') + q.text(value[0...-1]) + end + end + + def to_json(*opts) + { type: :label, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_label: (String value) -> Label + def on_label(value) + node = + Label.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # LabelEnd represents the end of a dynamic symbol. + # + # { "key": value } + # + # In the example above, LabelEnd represents the "\":" token at the end of the + # hash key. This node is important for determining the type of quote being + # used by the label. + class LabelEnd + # [String] the end of the label + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_label_end: (String value) -> LabelEnd + def on_label_end(value) + node = + LabelEnd.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # Lambda represents using a lambda literal (not the lambda method call). + # + # ->(value) { value * 2 } + # + class Lambda + # [Params | Paren] the parameter declaration for this lambda + attr_reader :params + + # [BodyStmt | Statements] the expressions to be executed in this lambda + attr_reader :statements + + # [Location] the location of this node + attr_reader :location + + def initialize(params:, statements:, location:) + @params = params + @statements = statements + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('lambda') + + q.breakable + q.pp(params) + + q.breakable + q.pp(statements) + end + end + + def to_json(*opts) + { + type: :lambda, + params: params, + stmts: statements, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_lambda: ( + # (Params | Paren) params, + # (BodyStmt | Statements) statements + # ) -> Lambda + def on_lambda(params, statements) + beginning = find_token(TLambda) + + if token = find_token(TLamBeg, consume: false) + opening = tokens.delete(token) + closing = find_token(RBrace) + else + opening = find_token(Kw, 'do') + closing = find_token(Kw, 'end') + end + + statements.bind(opening.location.end_char, closing.location.start_char) + + Lambda.new( + params: params, + statements: statements, + location: beginning.location.to(closing.location) + ) + end + + # LBrace represents the use of a left brace, i.e., {. + class LBrace + # [String] the left brace + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('lbrace') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :lbrace, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_lbrace: (String value) -> LBrace + def on_lbrace(value) + node = + LBrace.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # LBracket represents the use of a left bracket, i.e., [. + class LBracket + # [String] the left bracket + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_lbracket: (String value) -> LBracket + def on_lbracket(value) + node = + LBracket.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # LParen represents the use of a left parenthesis, i.e., (. + class LParen + # [String] the left parenthesis + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('lparen') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :lparen, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_lparen: (String value) -> LParen + def on_lparen(value) + node = + LParen.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # def on_magic_comment(key, value) + # [key, value] + # end + + # MAssign is a parent node of any kind of multiple assignment. This includes + # splitting out variables on the left like: + # + # first, second, third = value + # + # as well as splitting out variables on the right, as in: + # + # value = first, second, third + # + # Both sides support splats, as well as variables following them. There's also + # destructuring behavior that you can achieve with the following: + # + # first, = value + # + class MAssign + # [Mlhs | MlhsAddPost | MlhsAddStar | MlhsParen] the target of the multiple + # assignment + attr_reader :target + + # [untyped] the value being assigned + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(target:, value:, location:) + @target = target + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('massign') + + q.breakable + q.pp(target) + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :massign, target: target, value: value, loc: location }.to_json( + *opts + ) + end + end + + # :call-seq: + # on_massign: ( + # (Mlhs | MlhsAddPost | MlhsAddStar | MlhsParen) target, + # untyped value + # ) -> MAssign + def on_massign(target, value) + comma_range = target.location.end_char...value.location.start_char + target.comma = true if source[comma_range].strip.start_with?(',') + + MAssign.new( + target: target, + value: value, + location: target.location.to(value.location) + ) + end + + # MethodAddArg represents a method call with arguments and parentheses. + # + # method(argument) + # + # MethodAddArg can also represent with a method on an object, as in: + # + # object.method(argument) + # + # Finally, MethodAddArg can represent calling a method with no receiver that + # ends in a ?. In this case, the parser knows it's a method call and not a + # local variable, so it uses a MethodAddArg node as opposed to a VCall node, + # as in: + # + # method? + # + class MethodAddArg + # [Call | FCall] the method call + attr_reader :call + + # [ArgParen | Args | ArgsAddBlock] the arguments to the method call + attr_reader :arguments + + # [Location] the location of this node + attr_reader :location + + def initialize(call:, arguments:, location:) + @call = call + @arguments = arguments + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('method_add_arg') + + q.breakable + q.pp(call) + + q.breakable + q.pp(arguments) + end + end + + def to_json(*opts) + { + type: :method_add_arg, + call: call, + args: arguments, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_method_add_arg: ( + # (Call | FCall) call, + # (ArgParen | Args | ArgsAddBlock) arguments + # ) -> MethodAddArg + def on_method_add_arg(call, arguments) + location = call.location + + location = location.to(arguments.location) unless arguments.is_a?(Args) + + MethodAddArg.new(call: call, arguments: arguments, location: location) + end + + # MethodAddBlock represents a method call with a block argument. + # + # method {} + # + class MethodAddBlock + # [Call | Command | CommandCall | FCall | MethodAddArg] the method call + attr_reader :call + + # [BraceBlock | DoBlock] the block being sent with the method call + attr_reader :block + + # [Location] the location of this node + attr_reader :location + + def initialize(call:, block:, location:) + @call = call + @block = block + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('method_add_block') + + q.breakable + q.pp(call) + + q.breakable + q.pp(block) + end + end + + def to_json(*opts) + { + type: :method_add_block, + call: call, + block: block, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_method_add_block: ( + # (Call | Command | CommandCall | FCall | MethodAddArg) call, + # (BraceBlock | DoBlock) block + # ) -> MethodAddBlock + def on_method_add_block(call, block) + MethodAddBlock.new( + call: call, + block: block, + location: call.location.to(block.location) + ) + end + + # MLHS represents a list of values being destructured on the left-hand side + # of a multiple assignment. + # + # first, second, third = value + # + class MLHS + # Array[ARefField | Field | Ident | MlhsParen | VarField] the parts of + # the left-hand side of a multiple assignment + attr_reader :parts + + # [boolean] whether or not there is a trailing comma at the end of this + # list, which impacts destructuring. It's an attr_accessor so that while + # the syntax tree is being built it can be set by its parent node + attr_accessor :comma + + # [Location] the location of this node + attr_reader :location + + def initialize(parts:, comma: false, location:) + @parts = parts + @comma = comma + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('mlhs') + + q.breakable + q.group(2, '(', ')') { q.seplist(parts) { |part| q.pp(part) } } + end + end + + def to_json(*opts) + { type: :mlhs, parts: parts, comma: comma, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_mlhs_add: ( + # MLHS mlhs, + # (ARefField | Field | Ident | MlhsParen | VarField) part + # ) -> MLHS + def on_mlhs_add(mlhs, part) + if mlhs.parts.empty? + MLHS.new(parts: [part], location: part.location) + else + MLHS.new( + parts: mlhs.parts << part, + location: mlhs.location.to(part.location) + ) + end + end + + # MLHSAddPost represents adding another set of variables onto a list of + # assignments after a splat variable within a multiple assignment. + # + # left, *middle, right = values + # + class MLHSAddPost + # [MlhsAddStar] the value being starred + attr_reader :star + + # [Mlhs] the values after the star + attr_reader :mlhs + + # [Location] the location of this node + attr_reader :location + + def initialize(star:, mlhs:, location:) + @star = star + @mlhs = mlhs + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('mlhs_add_post') + + q.breakable + q.pp(star) + + q.breakable + q.pp(mlhs) + end + end + + def to_json(*opts) + { type: :mlhs_add_post, star: star, mlhs: mlhs, loc: location }.to_json( + *opts + ) + end + end + + # :call-seq: + # on_mlhs_add_post: (MLHSAddStar star, MLHS mlhs) -> MLHSAddPost + def on_mlhs_add_post(star, mlhs) + MLHSAddPost.new( + star: star, + mlhs: mlhs, + location: star.location.to(mlhs.location) + ) + end + + # MLHSAddStar represents a splatted variable inside of a multiple assignment + # on the left hand side. + # + # first, *rest = values + # + class MLHSAddStar + # [MLHS] the values before the starred expression + attr_reader :mlhs + + # [nil | ARefField | Field | Ident | VarField] the expression being + # splatted + attr_reader :star + + # [Location] the location of this node + attr_reader :location + + def initialize(mlhs:, star:, location:) + @mlhs = mlhs + @star = star + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('mlhs_add_star') + + q.breakable + q.pp(mlhs) + + q.breakable + q.pp(star) + end + end + + def to_json(*opts) + { type: :mlhs_add_star, mlhs: mlhs, star: star, loc: location }.to_json( + *opts + ) + end + end + + # :call-seq: + # on_mlhs_add_star: ( + # MLHS mlhs, + # (nil | ARefField | Field | Ident | VarField) part + # ) -> MLHSAddStar + def on_mlhs_add_star(mlhs, part) + beginning = find_token(Op, '*') + ending = part || beginning + + MLHSAddStar.new( + mlhs: mlhs, + star: part, + location: beginning.location.to(ending.location) + ) + end + + # :call-seq: + # on_mlhs_new: () -> MLHS + def on_mlhs_new + MLHS.new(parts: [], location: Location.fixed(line: lineno, char: char_pos)) + end + + # MLHSParen represents parentheses being used to destruct values in a multiple + # assignment on the left hand side. + # + # (left, right) = value + # + class MLHSParen + # [Mlhs | MlhsAddPost | MlhsAddStar | MlhsParen] the contents inside of the + # parentheses + attr_reader :contents + + # [Location] the location of this node + attr_reader :location + + def initialize(contents:, location:) + @contents = contents + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('mlhs_paren') + + q.breakable + q.pp(contents) + end + end + + def to_json(*opts) + { type: :mlhs_paren, cnts: contents, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_mlhs_paren: ( + # (Mlhs | MlhsAddPost | MlhsAddStar | MlhsParen) contents + # ) -> MLHSParen + def on_mlhs_paren(contents) + lparen = find_token(LParen) + rparen = find_token(RParen) + + comma_range = lparen.location.end_char...rparen.location.start_char + contents.comma = true if source[comma_range].strip.end_with?(',') + + MLHSParen.new( + contents: contents, + location: lparen.location.to(rparen.location) + ) + end + + # ModuleDeclaration represents defining a module using the +module+ keyword. + # + # module Namespace + # end + # + class ModuleDeclaration + # [ConstPathRef | ConstRef | TopConstRef] the name of the module + attr_reader :constant + + # [BodyStmt] the expressions to be executed in the context of the module + attr_reader :bodystmt + + # [Location] the location of this node + attr_reader :location + + def initialize(constant:, bodystmt:, location:) + @constant = constant + @bodystmt = bodystmt + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('module') + + q.breakable + q.pp(constant) + + q.breakable + q.pp(bodystmt) + end + end + + def to_json(*opts) + { + type: :module, + constant: constant, + bodystmt: bodystmt, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_module: ( + # (ConstPathRef | ConstRef | TopConstRef) constant, + # BodyStmt bodystmt + # ) -> ModuleDeclaration + def on_module(constant, bodystmt) + beginning = find_token(Kw, 'module') + ending = find_token(Kw, 'end') + + bodystmt.bind( + find_next_statement_start(constant.location.end_char), + ending.location.start_char + ) + + ModuleDeclaration.new( + constant: constant, + bodystmt: bodystmt, + location: beginning.location.to(ending.location) + ) + end + + # MRHS represents the values that are being assigned on the right-hand side of + # a multiple assignment. + # + # values = first, second, third + # + class MRHS + # Array[untyped] the parts that are being assigned + attr_reader :parts + + # [Location] the location of this node + attr_reader :location + + def initialize(parts:, location:) + @parts = parts + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('mrhs') + + q.breakable + q.group(2, '(', ')') { q.seplist(parts) { |part| q.pp(part) } } + end + end + + def to_json(*opts) + { type: :mrhs, parts: parts, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_mrhs_new: () -> MRHS + def on_mrhs_new + MRHS.new(parts: [], location: Location.fixed(line: lineno, char: char_pos)) + end + + # :call-seq: + # on_mrhs_add: (MRHS mrhs, untyped part) -> MRHS + def on_mrhs_add(mrhs, part) + if mrhs.is_a?(MRHSNewFromArgs) + MRHS.new( + parts: [*mrhs.arguments.parts, part], + location: mrhs.location.to(part.location) + ) + elsif mrhs.parts.empty? + MRHS.new(parts: [part], location: mrhs.location) + else + MRHS.new(parts: mrhs.parts << part, loc: mrhs.location.to(part.location)) + end + end + + # MRHSAddStar represents using the splat operator to expand out a value on the + # right hand side of a multiple assignment. + # + # values = first, *rest + # + class MRHSAddStar + # [MRHS | MRHSNewFromArgs] the values before the splatted expression + attr_reader :mrhs + + # [untyped] the splatted expression + attr_reader :star + + # [Location] the location of this node + attr_reader :location + + def initialize(mrhs:, star:, location:) + @mrhs = mrhs + @star = star + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('mrhs_add_star') + + q.breakable + q.pp(mrhs) + + q.breakable + q.pp(star) + end + end + + def to_json(*opts) + { type: :mrhs_add_star, mrhs: mrhs, star: star, loc: location }.to_json( + *opts + ) + end + end + + # :call-seq: + # on_mrhs_add_star: ( + # (MRHS | MRHSNewFromArgs) mrhs, + # untyped star + # ) -> MRHSAddStar + def on_mrhs_add_star(mrhs, star) + beginning = find_token(Op, '*') + ending = star || beginning + + MRHSAddStar.new( + mrhs: mrhs, + star: star, + location: beginning.location.to(ending.location) + ) + end + + # MRHSNewFromArgs represents the shorthand of a multiple assignment that + # allows you to assign values using just commas as opposed to assigning from + # an array. + # + # values = first, second, third + # + class MRHSNewFromArgs + # [Args] the arguments being used in the assignment + attr_reader :arguments + + # [Location] the location of this node + attr_reader :location + + def initialize(arguments:, location:) + @arguments = arguments + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('mrhs_new_from_args') + + q.breakable + q.pp(arguments) + end + end + + def to_json(*opts) + { type: :mrhs_new_from_args, args: arguments, loc: location }.to_json( + *opts + ) + end + end + + # :call-seq: + # on_mrhs_new_from_args: (Args arguments) -> MRHSNewFromArgs + def on_mrhs_new_from_args(arguments) + MRHSNewFromArgs.new(arguments: arguments, location: arguments.location) + end + + # Next represents using the +next+ keyword. + # + # next + # + # The +next+ keyword can also optionally be called with an argument: + # + # next value + # + # +next+ can even be called with multiple arguments, but only if parentheses + # are omitted, as in: + # + # next first, second, third + # + # If a single value is being given, parentheses can be used, as in: + # + # next(value) + # + class Next + # [Args | ArgsAddBlock] the arguments passed to the next keyword + attr_reader :arguments + + # [Location] the location of this node + attr_reader :location + + def initialize(arguments:, location:) + @arguments = arguments + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('next') + + q.breakable + q.pp(arguments) + end + end + + def to_json(*opts) + { type: :next, args: arguments, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_next: ((Args | ArgsAddBlock) arguments) -> Next + def on_next(arguments) + keyword = find_token(Kw, 'next') + + location = keyword.location + location = location.to(arguments.location) unless arguments.is_a?(Args) + + Next.new(arguments: arguments, location: location) + end + + # def on_nl(value) + # value + # end + + # def on_nokw_param(value) + # value + # end + + # Op represents an operator literal in the source. + # + # 1 + 2 + # + # In the example above, the Op node represents the + operator. + class Op + # [String] the operator + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('op') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :op, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_op: (String value) -> Op + def on_op(value) + node = + Op.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # OpAssign represents assigning a value to a variable or constant using an + # operator like += or ||=. + # + # variable += value + # + class OpAssign + # [ARefField | ConstPathField | Field | TopConstField | VarField] the target + # to assign the result of the expression to + attr_reader :target + + # [Op] the operator being used for the assignment + attr_reader :operator + + # [untyped] the expression to be assigned + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(target:, operator:, value:, location:) + @target = target + @operator = operator + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('opassign') + + q.breakable + q.pp(target) + + q.breakable + q.pp(operator) + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { + type: :opassign, + target: target, + op: operator, + value: value, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_opassign: ( + # (ARefField | ConstPathField | Field | TopConstField | VarField) target, + # Op operator, + # untyped value + # ) -> OpAssign + def on_opassign(target, operator, value) + OpAssign.new( + target: target, + operator: operator, + value: value, + location: target.location.to(value.location) + ) + end + + # def on_operator_ambiguous(value) + # value + # end + + # Params represents defining parameters on a method or lambda. + # + # def method(param) end + # + class Params + # [Array[ Ident ]] any required parameters + attr_reader :requireds + + # [Array[ [ Ident, untyped ] ]] any optional parameters and their default + # values + attr_reader :optionals + + # [nil | ArgsForward | ExcessedComma | RestParam] the optional rest + # parameter + attr_reader :rest + + # [Array[ Ident ]] any positional parameters that exist after a rest + # parameter + attr_reader :posts + + # [Array[ [ Ident, nil | untyped ] ]] any keyword parameters and their + # optional default values + attr_reader :keywords + + # [nil | :nil | KwRestParam] the optional keyword rest parameter + attr_reader :keyword_rest + + # [nil | BlockArg] the optional block parameter + attr_reader :block + + # [Location] the location of this node + attr_reader :location + + def initialize( + requireds: [], + optionals: [], + rest: nil, + posts: [], + keywords: [], + keyword_rest: nil, + block: nil, + location: + ) + @requireds = requireds + @optionals = optionals + @rest = rest + @posts = posts + @keywords = keywords + @keyword_rest = keyword_rest + @block = block + @location = location + end + + # Params nodes are the most complicated in the tree. Occasionally you want + # to know if they are "empty", which means not having any parameters + # declared. This logic accesses every kind of parameter and determines if + # it's missing. + def empty? + requireds.empty? && optionals.empty? && !rest && posts.empty? && + keywords.empty? && !keyword_rest && !block + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('params') + + if requireds.any? + q.breakable + q.group(2, '(', ')') { q.seplist(requireds) { |name| q.pp(name) } } + end + + if optionals.any? + q.breakable + q.group(2, '(', ')') do + q.seplist(optionals) do |(name, default)| + q.pp(name) + q.text('=') + q.group(2) do + q.breakable('') + q.pp(default) + end + end + end + end + + if rest + q.breakable + q.pp(rest) + end + + if posts.any? + q.breakable + q.group(2, '(', ')') { q.seplist(posts) { |value| q.pp(value) } } + end + + if keywords.any? + q.breakable + q.group(2, '(', ')') do + q.seplist(keywords) do |(name, default)| + q.pp(name) + + if default + q.text('=') + q.group(2) do + q.breakable('') + q.pp(default) + end + end + end + end + end + + if keyword_rest + q.breakable + q.pp(keyword_rest) + end + + if block + q.breakable + q.pp(block) + end + end + end + + def to_json(*opts) + { + type: :params, + reqs: requireds, + opts: optionals, + rest: rest, + posts: posts, + keywords: keywords, + kwrest: keyword_rest, + block: block, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_params: ( + # (nil | Array[Ident]) requireds, + # (nil | Array[[Ident, untyped]]) optionals, + # (nil | ArgsForward | ExcessedComma | RestParam) rest, + # (nil | Array[Ident]) posts, + # (nil | Array[[Ident, nil | untyped]]) keywords, + # (nil | :nil | KwRestParam) keyword_rest, + # (nil | BlockArg) block + # ) -> Params + def on_params( + requireds, + optionals, + rest, + posts, + keywords, + keyword_rest, + block + ) + parts = [ + *requireds, + *optionals&.flatten(1), + rest, + *posts, + *keywords&.flat_map { |(key, value)| [key, value || nil] }, + (keyword_rest if keyword_rest != :nil), + block + ].compact + + location = + if parts.any? + parts[0].location.to(parts[-1].location) + else + Location.fixed(line: lineno, char: char_pos) + end + + Params.new( + requireds: requireds || [], + optionals: optionals || [], + rest: rest, + posts: posts || [], + keywords: keywords || [], + keyword_rest: keyword_rest, + block: block, + location: location + ) + end + + # Paren represents using balanced parentheses in a couple places in a Ruby + # program. In general parentheses can be used anywhere a Ruby expression can + # be used. + # + # (1 + 2) + # + class Paren + # [LParen] the left parenthesis that opened this statement + attr_reader :lparen + + # [untyped] the expression inside the parentheses + attr_reader :contents + + # [Location] the location of this node + attr_reader :location + + def initialize(lparen:, contents:, location:) + @lparen = lparen + @contents = contents + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('paren') + + q.breakable + q.pp(contents) + end + end + + def to_json(*opts) + { type: :paren, lparen: lparen, cnts: contents, loc: location }.to_json( + *opts + ) + end + end + + # :call-seq: + # on_paren: (untyped contents) -> Paren + def on_paren(contents) + lparen = find_token(LParen) + rparen = find_token(RParen) + + if contents && contents.is_a?(Params) + location = contents.location + location = + Location.new( + start_line: location.start_line, + start_char: find_next_statement_start(lparen.location.end_char), + end_line: location.end_line, + end_char: rparen.location.start_char + ) + + contents = + Params.new( + requireds: contents.requireds, + optionals: contents.optionals, + rest: contents.rest, + posts: contents.posts, + keywords: contents.keywords, + keyword_rest: contents.keyword_rest, + block: contents.block, + location: location + ) + end + + Paren.new( + lparen: lparen, + contents: contents, + location: lparen.location.to(rparen.location) + ) + end + + # If we encounter a parse error, just immediately bail out so that our runner + # can catch it. + def on_parse_error(error, *) + raise ParseError.new(error, lineno, column) + end + alias on_alias_error on_parse_error + alias on_assign_error on_parse_error + alias on_class_name_error on_parse_error + alias on_param_error on_parse_error + + # Period represents the use of the +.+ operator. It is usually found in method + # calls. + class Period + # [String] the period + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('period') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :period, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_period: (String value) -> Period + def on_period(value) + Period.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + end + + # Program represents the overall syntax tree. + class Program + # [Statements] the top-level expressions of the program + attr_reader :statements + + # [Array[ Comment | EmbDoc ]] the comments inside the program + attr_reader :comments + + # [Location] the location of this node + attr_reader :location + + def initialize(statements:, comments:, location:) + @statements = statements + @comments = comments + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('program') + + q.breakable + q.pp(statements) + end + end + + def to_json(*opts) + { + type: :program, + stmts: statements, + comments: comments, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_program: (Statements statements) -> Program + def on_program(statements) + location = + Location.new( + start_line: 1, + start_char: 0, + end_line: lines.length, + end_char: source.length + ) + + statements.body << @__end__ if @__end__ + statements.bind(0, source.length) + + Program.new(statements: statements, comments: @comments, location: location) + end + + # QSymbols represents a symbol literal array without interpolation. + # + # %i[one two three] + # + class QSymbols + # [Array[ TStringContent ]] the elements of the array + attr_reader :elements + + # [Location] the location of this node + attr_reader :location + + def initialize(elements:, location:) + @elements = elements + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('qsymbols') + + q.breakable + q.group(2, '(', ')') { q.seplist(elements) { |element| q.pp(element) } } + end + end + + def to_json(*opts) + { type: :qsymbols, elems: elements, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_qsymbols_add: (QSymbols qsymbols, TStringContent element) -> QSymbols + def on_qsymbols_add(qsymbols, element) + QSymbols.new( + elements: qsymbols.elements << element, + location: qsymbols.location.to(element.location) + ) + end + + # QSymbolsBeg represents the beginning of a symbol literal array. + # + # %i[one two three] + # + # In the snippet above, QSymbolsBeg represents the "%i[" token. Note that + # these kinds of arrays can start with a lot of different delimiter types + # (e.g., %i| or %i<). + class QSymbolsBeg + # [String] the beginning of the array literal + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_qsymbols_beg: (String value) -> QSymbolsBeg + def on_qsymbols_beg(value) + node = + QSymbolsBeg.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # :call-seq: + # on_qsymbols_new: () -> QSymbols + def on_qsymbols_new + qsymbols_beg = find_token(QSymbolsBeg) + + QSymbols.new(elements: [], location: qsymbols_beg.location) + end + + # QWords represents a string literal array without interpolation. + # + # %w[one two three] + # + class QWords + # [Array[ TStringContent ]] the elements of the array + attr_reader :elements + + # [Location] the location of this node + attr_reader :location + + def initialize(elements:, location:) + @elements = elements + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('qwords') + + q.breakable + q.group(2, '(', ')') { q.seplist(elements) { |element| q.pp(element) } } + end + end + + def to_json(*opts) + { type: :qwords, elems: elements, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_qwords_add: (QWords qwords, TStringContent element) -> QWords + def on_qwords_add(qwords, element) + QWords.new( + elements: qwords.elements << element, + location: qwords.location.to(element.location) + ) + end + + # QWordsBeg represents the beginning of a string literal array. + # + # %w[one two three] + # + # In the snippet above, QWordsBeg represents the "%w[" token. Note that these + # kinds of arrays can start with a lot of different delimiter types (e.g., + # %w| or %w<). + class QWordsBeg + # [String] the beginning of the array literal + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_qwords_beg: (String value) -> QWordsBeg + def on_qwords_beg(value) + node = + QWordsBeg.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # :call-seq: + # on_qwords_new: () -> QWords + def on_qwords_new + qwords_beg = find_token(QWordsBeg) + + QWords.new(elements: [], location: qwords_beg.location) + end + + # RationalLiteral represents the use of a rational number literal. + # + # 1r + # + class RationalLiteral + # [String] the rational number literal + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('rational') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :rational, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_rational: (String value) -> RationalLiteral + def on_rational(value) + node = + RationalLiteral.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # RBrace represents the use of a right brace, i.e., +++. + class RBrace + # [String] the right brace + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_rbrace: (String value) -> RBrace + def on_rbrace(value) + node = + RBrace.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # RBracket represents the use of a right bracket, i.e., +]+. + class RBracket + # [String] the right bracket + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_rbracket: (String value) -> RBracket + def on_rbracket(value) + node = + RBracket.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # Redo represents the use of the +redo+ keyword. + # + # redo + # + class Redo + # [String] the value of the keyword + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('redo') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :redo, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_redo: () -> Redo + def on_redo + keyword = find_token(Kw, 'redo') + + Redo.new(value: keyword.value, location: keyword.location) + end + + # RegexpContent represents the body of a regular expression. + # + # /.+ #{pattern} .+/ + # + # In the example above, a RegexpContent node represents everything contained + # within the forward slashes. + class RegexpContent + # [String] the opening of the regular expression + attr_reader :beginning + + # [Array[ StringDVar | StringEmbExpr | TStringContent ]] the parts of the + # regular expression + attr_reader :parts + + # [Location] the location of this node + attr_reader :location + + def initialize(beginning:, parts:, location:) + @beginning = beginning + @parts = parts + @location = location + end + end + + # :call-seq: + # on_regexp_add: ( + # RegexpContent regexp_content, + # (StringDVar | StringEmbExpr | TStringContent) part + # ) -> RegexpContent + def on_regexp_add(regexp_content, part) + RegexpContent.new( + beginning: regexp_content.beginning, + parts: regexp_content.parts << part, + location: regexp_content.location.to(part.location) + ) + end + + # RegexpBeg represents the start of a regular expression literal. + # + # /.+/ + # + # In the example above, RegexpBeg represents the first / token. Regular + # expression literals can also be declared using the %r syntax, as in: + # + # %r{.+} + # + class RegexpBeg + # [String] the beginning of the regular expression + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_regexp_beg: (String value) -> RegexpBeg + def on_regexp_beg(value) + node = + RegexpBeg.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # RegexpEnd represents the end of a regular expression literal. + # + # /.+/m + # + # In the example above, the RegexpEnd event represents the /m at the end of + # the regular expression literal. You can also declare regular expression + # literals using %r, as in: + # + # %r{.+}m + # + class RegexpEnd + # [String] the end of the regular expression + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_regexp_end: (String value) -> RegexpEnd + def on_regexp_end(value) + RegexpEnd.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + end + + # RegexpLiteral represents a regular expression literal. + # + # /.+/ + # + class RegexpLiteral + # [String] the beginning of the regular expression literal + attr_reader :beginning + + # [String] the ending of the regular expression literal + attr_reader :ending + + # [Array[ StringEmbExpr | StringDVar | TStringContent ]] the parts of the + # regular expression literal + attr_reader :parts + + # [Locatione] the location of this node + attr_reader :location + + def initialize(beginning:, ending:, parts:, location:) + @beginning = beginning + @ending = ending + @parts = parts + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('regexp_literal') + + q.breakable + q.group(2, '(', ')') { q.seplist(parts) { |part| q.pp(part) } } + end + end + + def to_json(*opts) + { + type: :regexp_literal, + beging: beginning, + ending: ending, + parts: parts, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_regexp_literal: ( + # RegexpContent regexp_content, + # RegexpEnd ending + # ) -> RegexpLiteral + def on_regexp_literal(regexp_content, ending) + RegexpLiteral.new( + beginning: regexp_content.beginning, + ending: ending.value, + parts: regexp_content.parts, + location: regexp_content.location.to(ending.location) + ) + end + + # :call-seq: + # on_regexp_new: () -> RegexpContent + def on_regexp_new + regexp_beg = find_token(RegexpBeg) + + RegexpContent.new( + beginning: regexp_beg.value, + parts: [], + location: regexp_beg.location + ) + end + + # RescueEx represents the list of exceptions being rescued in a rescue clause. + # + # begin + # rescue Exception => exception + # end + # + class RescueEx + # [untyped] the list of exceptions being rescued + attr_reader :exceptions + + # [nil | Field | VarField] the expression being used to capture the raised + # exception + attr_reader :variable + + # [Location] the location of this node + attr_reader :location + + def initialize(exceptions:, variable:, location:) + @exceptions = exceptions + @variable = variable + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('rescue_ex') + + q.breakable + q.pp(exceptions) + + q.breakable + q.pp(variable) + end + end + + def to_json(*opts) + { + type: :rescue_ex, + extns: exceptions, + var: variable, + loc: location + }.to_json(*opts) + end + end + + # Rescue represents the use of the rescue keyword inside of a BodyStmt node. + # + # begin + # rescue + # end + # + class Rescue + # [RescueEx] the exceptions being rescued + attr_reader :exception + + # [Statements] the expressions to evaluate when an error is rescued + attr_reader :statements + + # [nil | Rescue] the optional next clause in the chain + attr_reader :consequent + + # [Location] the location of this node + attr_reader :location + + def initialize(exception:, statements:, consequent:, location:) + @exception = exception + @statements = statements + @consequent = consequent + @location = location + end + + def bind_end(end_char) + @location = + Location.new( + start_line: location.start_line, + start_char: location.start_char, + end_line: location.end_line, + end_char: end_char + ) + + if consequent + consequent.bind_end(end_char) + statements.bind_end(consequent.location.start_char) + else + statements.bind_end(end_char) + end + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('rescue') + + if exception + q.breakable + q.pp(exception) + end + + q.breakable + q.pp(statements) + + if consequent + q.breakable + q.pp(consequent) + end + end + end + + def to_json(*opts) + { + type: :rescue, + extn: exception, + stmts: statements, + cons: consequent, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_rescue: ( + # (nil | [untyped] | MRHS | MRHSAddStar) exceptions, + # (nil | Field | VarField) variable, + # Statements statements, + # (nil | Rescue) consequent + # ) -> Rescue + def on_rescue(exceptions, variable, statements, consequent) + keyword = find_token(Kw, 'rescue') + exceptions = exceptions[0] if exceptions.is_a?(Array) + + last_node = variable || exceptions || keyword + statements.bind( + find_next_statement_start(last_node.location.end_char), + char_pos + ) + + # We add an additional inner node here that ripper doesn't provide so that + # we have a nice place to attach inline comments. But we only need it if we + # have an exception or a variable that we're rescuing. + rescue_ex = + if exceptions || variable + RescueEx.new( + exceptions: exceptions, + variable: variable, + location: + Location.new( + start_line: keyword.location.start_line, + start_char: keyword.location.end_char + 1, + end_line: last_node.location.end_line, + end_char: last_node.location.end_char + ) + ) + end + + Rescue.new( + exception: rescue_ex, + statements: statements, + consequent: consequent, + location: + Location.new( + start_line: keyword.location.start_line, + start_char: keyword.location.start_char, + end_line: lineno, + end_char: char_pos + ) + ) + end + + # RescueMod represents the use of the modifier form of a +rescue+ clause. + # + # expression rescue value + # + class RescueMod + # [untyped] the expression to execute + attr_reader :statement + + # [untyped] the value to use if the executed expression raises an error + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(statement:, value:, location:) + @statement = statement + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('rescue_mod') + + q.breakable + q.pp(statement) + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { + type: :rescue_mod, + stmt: statement, + value: value, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_rescue_mod: (untyped statement, untyped value) -> RescueMod + def on_rescue_mod(statement, value) + find_token(Kw, 'rescue') + + RescueMod.new( + statement: statement, + value: value, + location: statement.location.to(value.location) + ) + end + + # RestParam represents defining a parameter in a method definition that + # accepts all remaining positional parameters. + # + # def method(*rest) end + # + class RestParam + # [nil | Ident] the name of the parameter + attr_reader :name + + # [Location] the location of this node + attr_reader :location + + def initialize(name:, location:) + @name = name + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('rest_param') + + q.breakable + q.pp(name) + end + end + + def to_json(*opts) + { type: :rest_param, name: name, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_rest_param: ((nil | Ident) name) -> RestParam + def on_rest_param(name) + location = find_token(Op, '*').location + location = location.to(name.location) if name + + RestParam.new(name: name, location: location) + end + + # Retry represents the use of the +retry+ keyword. + # + # retry + # + class Retry + # [String] the value of the keyword + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('retry') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :retry, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_retry: () -> Retry + def on_retry + keyword = find_token(Kw, 'retry') + + Retry.new(value: keyword.value, location: keyword.location) + end + + # Return represents using the +return+ keyword with arguments. + # + # return value + # + class Return + # [Args | ArgsAddBlock] the arguments being passed to the keyword + attr_reader :arguments + + # [Location] the location of this node + attr_reader :location + + def initialize(arguments:, location:) + @arguments = arguments + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('return') + + q.breakable + q.pp(arguments) + end + end + + def to_json(*opts) + { type: :return, args: arguments, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_return: ((Args | ArgsAddBlock) arguments) -> Return + def on_return(arguments) + keyword = find_token(Kw, 'return') + + Return.new( + arguments: arguments, + location: keyword.location.to(arguments.location) + ) + end + + # Return0 represents the bare +return+ keyword with no arguments. + # + # return + # + class Return0 + # [String] the value of the keyword + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('return0') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :return0, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_return0: () -> Return0 + def on_return0 + keyword = find_token(Kw, 'return') + + Return0.new(value: keyword.value, location: keyword.location) + end + + # RParen represents the use of a right parenthesis, i.e., +)+. + class RParen + # [String] the parenthesis + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_rparen: (String value) -> RParen + def on_rparen(value) + node = + RParen.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # SClass represents a block of statements that should be evaluated within the + # context of the singleton class of an object. It's frequently used to define + # singleton methods. + # + # class << self + # end + # + class SClass + # [untyped] the target of the singleton class to enter + attr_reader :target + + # [BodyStmt] the expressions to be executed + attr_reader :bodystmt + + # [Location] the location of this node + attr_reader :location + + def initialize(target:, bodystmt:, location:) + @target = target + @bodystmt = bodystmt + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('sclass') + + q.breakable + q.pp(target) + + q.breakable + q.pp(bodystmt) + end + end + + def to_json(*opts) + { + type: :sclass, + target: target, + bodystmt: bodystmt, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_sclass: (untyped target, BodyStmt bodystmt) -> SClass + def on_sclass(target, bodystmt) + beginning = find_token(Kw, 'class') + ending = find_token(Kw, 'end') + + bodystmt.bind( + find_next_statement_start(target.location.end_char), + ending.location.start_char + ) + + SClass.new( + target: target, + bodystmt: bodystmt, + location: beginning.location.to(ending.location) + ) + end + + # def on_semicolon(value) + # value + # end + + # def on_sp(value) + # value + # end + + # stmts_add is a parser event that represents a single statement inside a + # list of statements within any lexical block. It accepts as arguments the + # parent stmts node as well as an stmt which can be any expression in + # Ruby. + def on_stmts_add(statements, statement) + statements << statement + end + + # Everything that has a block of code inside of it has a list of statements. + # Normally we would just track those as a node that has an array body, but we + # have some special handling in order to handle empty statement lists. They + # need to have the right location information, so all of the parent node of + # stmts nodes will report back down the location information. We then + # propagate that onto void_stmt nodes inside the stmts in order to make sure + # all comments get printed appropriately. + class Statements + # [SyntaxTree] the parser that created this node + attr_reader :parser + + # [Array[ untyped ]] the list of expressions contained within this node + attr_reader :body + + # [Location] the location of this node + attr_reader :location + + def initialize(parser:, body:, location:) + @parser = parser + @body = body + @location = location + end + + def bind(start_char, end_char) + @location = + Location.new( + start_line: location.start_line, + start_char: start_char, + end_line: location.end_line, + end_char: end_char + ) + + if body[0].is_a?(VoidStmt) + location = body[0].location + location = + Location.new( + start_line: location.start_line, + start_char: start_char, + end_line: location.end_line, + end_char: start_char + ) + + body[0] = VoidStmt.new(location: location) + end + + attach_comments(start_char, end_char) + end + + def bind_end(end_char) + @location = + Location.new( + start_line: location.start_line, + start_char: location.start_char, + end_line: location.end_line, + end_char: end_char + ) + end + + def <<(statement) + @location = + body.any? ? location.to(statement.location) : statement.location + + body << statement + self + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('statements') + + q.breakable + q.seplist(body) { |statement| q.pp(statement) } + end + end + + def to_json(*opts) + { type: :statements, body: body, loc: location }.to_json(*opts) + end + + private + + def attach_comments(start_char, end_char) + attachable = + parser.comments.select do |comment| + !comment.inline? && start_char <= comment.location.start_char && + end_char >= comment.location.end_char && + !comment.value.include?('prettier-ignore') + end + + return if attachable.empty? + + parser.comments -= attachable + @body = (body + attachable).sort_by! { |node| node.location.start_char } + end + end + + # :call-seq: + # on_stmts_new: () -> Statements + def on_stmts_new + Statements.new( + parser: self, + body: [], + location: Location.fixed(line: lineno, char: char_pos) + ) + end + + # StringContent represents the contents of a string-like value. + # + # "string" + # + class StringContent + # [Array[ StringEmbExpr | StringDVar | TStringContent ]] the parts of the + # string + attr_reader :parts + + # [Location] the location of this node + attr_reader :location + + def initialize(parts:, location:) + @parts = parts + @location = location + end + end + + # :call-seq: + # on_string_add: ( + # String string, + # (StringEmbExpr | StringDVar | TStringContent) part + # ) -> StringContent + def on_string_add(string, part) + location = + string.parts.any? ? string.location.to(part.location) : part.location + + StringContent.new(parts: string.parts << part, location: location) + end + + # StringConcat represents concatenating two strings together using a backward + # slash. + # + # "first" \ + # "second" + # + class StringConcat + # [StringConcat | StringLiteral] the left side of the concatenation + attr_reader :left + + # [StringLiteral] the right side of the concatenation + attr_reader :right + + # [Location] the location of this node + attr_reader :location + + def initialize(left:, right:, location:) + @left = left + @right = right + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('string_concat') + + q.breakable + q.pp(left) + + q.breakable + q.pp(right) + end + end + + def to_json(*opts) + { type: :string_concat, left: left, right: right, loc: location }.to_json( + *opts + ) + end + end + + # :call-seq: + # on_string_concat: ( + # (StringConcat | StringLiteral) left, + # StringLiteral right + # ) -> StringConcat + def on_string_concat(left, right) + StringConcat.new( + left: left, + right: right, + location: left.location.to(right.location) + ) + end + + # :call-seq: + # on_string_content: () -> StringContent + def on_string_content + StringContent.new( + parts: [], + location: Location.fixed(line: lineno, char: char_pos) + ) + end + + # StringDVar represents shorthand interpolation of a variable into a string. + # It allows you to take an instance variable, class variable, or global + # variable and omit the braces when interpolating. + # + # "#@variable" + # + class StringDVar + # [Backref | VarRef] the variable being interpolated + attr_reader :variable + + # [Location] the location of this node + attr_reader :location + + def initialize(variable:, location:) + @variable = variable + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('string_dvar') + + q.breakable + q.pp(variable) + end + end + + def to_json(*opts) + { type: :string_dvar, var: variable, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_string_dvar: ((Backref | VarRef) variable) -> StringDVar + def on_string_dvar(variable) + embvar = find_token(EmbVar) + + StringDVar.new( + variable: variable, + location: embvar.location.to(variable.location) + ) + end + + # StringEmbExpr represents interpolated content. It can be contained within a + # couple of different parent nodes, including regular expressions, strings, + # and dynamic symbols. + # + # "string #{expression}" + # + class StringEmbExpr + # [Statements] the expressions to be interpolated + attr_reader :statements + + # [Location] the location of this node + attr_reader :location + + def initialize(statements:, location:) + @statements = statements + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('string_embexpr') + + q.breakable + q.pp(statements) + end + end + + def to_json(*opts) + { type: :string_embexpr, stmts: statements, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_string_embexpr: (Statements statements) -> StringEmbExpr + def on_string_embexpr(statements) + embexpr_beg = find_token(EmbExprBeg) + embexpr_end = find_token(EmbExprEnd) + + statements.bind( + embexpr_beg.location.end_char, + embexpr_end.location.start_char + ) + + StringEmbExpr.new( + statements: statements, + location: embexpr_beg.location.to(embexpr_end.location) + ) + end + + # StringLiteral represents a string literal. + # + # "string" + # + class StringLiteral + # [Array[ StringEmbExpr | StringDVar | TStringContent ]] the parts of the + # string literal + attr_reader :parts + + # [String] which quote was used by the string literal + attr_reader :quote + + # [Location] the location of this node + attr_reader :location + + def initialize(parts:, quote:, location:) + @parts = parts + @quote = quote + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('string_literal') + + q.breakable + q.group(2, '(', ')') { q.seplist(parts) { |part| q.pp(part) } } + end + end + + def to_json(*opts) + { + type: :string_literal, + parts: parts, + quote: quote, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_string_literal: (String string) -> Heredoc | StringLiteral + def on_string_literal(string) + heredoc = @heredocs[-1] + + if heredoc && heredoc.ending + heredoc = @heredocs.pop + + Heredoc.new( + beginning: heredoc.beginning, + ending: heredoc.ending, + parts: string.parts, + location: heredoc.location + ) + else + tstring_beg = find_token(TStringBeg) + tstring_end = find_token(TStringEnd) + + StringLiteral.new( + parts: string.parts, + quote: tstring_beg.value, + location: tstring_beg.location.to(tstring_end.location) + ) + end + end + + # Super represents using the +super+ keyword with arguments. It can optionally + # use parentheses. + # + # super(value) + # + class Super + # [ArgParen | Args | ArgsAddBlock] the arguments to the keyword + attr_reader :arguments + + # [Location] the location of this node + attr_reader :location + + def initialize(arguments:, location:) + @arguments = arguments + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('super') + + q.breakable + q.pp(arguments) + end + end + + def to_json(*opts) + { type: :super, args: arguments, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_super: ((ArgParen | Args | ArgsAddBlock) arguments) -> Super + def on_super(arguments) + keyword = find_token(Kw, 'super') + + Super.new( + arguments: arguments, + location: keyword.location.to(arguments.location) + ) + end + + # SymBeg represents the beginning of a symbol literal. + # + # :symbol + # + # SymBeg is also used for dynamic symbols, as in: + # + # :"symbol" + # + # Finally, SymBeg is also used for symbols using the %s syntax, as in: + # + # %s[symbol] + # + # The value of this node is a string. In most cases (as in the first example + # above) it will contain just ":". In the case of dynamic symbols it will + # contain ":'" or ":\"". In the case of %s symbols, it will contain the start + # of the symbol including the %s and the delimiter. + class SymBeg + # [String] the beginning of the symbol + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # symbeg is a token that represents the beginning of a symbol literal. + # In most cases it will contain just ":" as in the value, but if its a dynamic + # symbol being defined it will contain ":'" or ":\"". + def on_symbeg(value) + node = + SymBeg.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # SymbolContent represents symbol contents and is always the child of a + # SymbolLiteral node. + # + # :symbol + # + class SymbolContent + # [Backtick | Const | CVar | GVar | Ident | IVar | Kw | Op] the value of the + # symbol + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_symbol: ( + # (Backtick | Const | CVar | GVar | Ident | IVar | Kw | Op) value + # ) -> SymbolContent + def on_symbol(value) + tokens.pop + + SymbolContent.new(value: value, location: value.location) + end + + # SymbolLiteral represents a symbol in the system with no interpolation + # (as opposed to a DynaSymbol which has interpolation). + # + # :symbol + # + class SymbolLiteral + # [Backtick | Const | CVar | GVar | Ident | IVar | Kw | Op] the value of the + # symbol + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('symbol_literal') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :symbol_literal, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_symbol_literal: ( + # ( + # Backtick | Const | CVar | GVar | Ident | + # IVar | Kw | Op | SymbolContent + # ) value + # ) -> SymbolLiteral + def on_symbol_literal(value) + if tokens[-1] == value + SymbolLiteral.new(value: tokens.pop, location: value.location) + else + symbeg = find_token(SymBeg) + + SymbolLiteral.new( + value: value.value, + location: symbeg.location.to(value.location) + ) + end + end + + # Symbols represents a symbol array literal with interpolation. + # + # %I[one two three] + # + class Symbols + # [Array[ Word ]] the words in the symbol array literal + attr_reader :elements + + # [Location] the location of this node + attr_reader :location + + def initialize(elements:, location:) + @elements = elements + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('symbols') + + q.breakable + q.group(2, '(', ')') { q.seplist(elements) { |element| q.pp(element) } } + end + end + + def to_json(*opts) + { type: :symbols, elems: elements, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_symbols_add: (Symbols symbols, Word word) -> Symbols + def on_symbols_add(symbols, word) + Symbols.new( + elements: symbols.elements << word, + location: symbols.location.to(word.location) + ) + end + + # SymbolsBeg represents the start of a symbol array literal with + # interpolation. + # + # %I[one two three] + # + # In the snippet above, SymbolsBeg represents the "%I[" token. Note that these + # kinds of arrays can start with a lot of different delimiter types + # (e.g., %I| or %I<). + class SymbolsBeg + # [String] the beginning of the symbol literal array + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_symbols_beg: (String value) -> SymbolsBeg + def on_symbols_beg(value) + node = + SymbolsBeg.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # :call-seq: + # on_symbols_new: () -> Symbols + def on_symbols_new + symbols_beg = find_token(SymbolsBeg) + + Symbols.new(elements: [], location: symbols_beg.location) + end + + # TLambda represents the beginning of a lambda literal. + # + # -> { value } + # + # In the example above the TLambda represents the +->+ operator. + class TLambda + # [String] the beginning of the lambda literal + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_tlambda: (String value) -> TLambda + def on_tlambda(value) + node = + TLambda.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # TLamBeg represents the beginning of the body of a lambda literal using + # braces. + # + # -> { value } + # + # In the example above the TLamBeg represents the +{+ operator. + class TLamBeg + # [String] the beginning of the body of the lambda literal + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_tlambeg: (String value) -> TLamBeg + def on_tlambeg(value) + node = + TLamBeg.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # TopConstField is always the child node of some kind of assignment. It + # represents when you're assigning to a constant that is being referenced at + # the top level. + # + # ::Constant = value + # + class TopConstField + # [Const] the constant being assigned + attr_reader :constant + + # [Location] the location of this node + attr_reader :location + + def initialize(constant:, location:) + @constant = constant + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('top_const_field') + + q.breakable + q.pp(constant) + end + end + + def to_json(*opts) + { type: :top_const_field, constant: constant, loc: location }.to_json( + *opts + ) + end + end + + # :call-seq: + # on_top_const_field: (Const constant) -> TopConstRef + def on_top_const_field(constant) + operator = find_colon2_before(constant) + + TopConstField.new( + constant: constant, + location: operator.location.to(constant.location) + ) + end + + # TopConstRef is very similar to TopConstField except that it is not involved + # in an assignment. + # + # ::Constant + # + class TopConstRef + # [Const] the constant being referenced + attr_reader :constant + + # [Location] the location of this node + attr_reader :location + + def initialize(constant:, location:) + @constant = constant + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('top_const_ref') + + q.breakable + q.pp(constant) + end + end + + def to_json(*opts) + { type: :top_const_ref, constant: constant, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_top_const_ref: (Const constant) -> TopConstRef + def on_top_const_ref(constant) + operator = find_colon2_before(constant) + + TopConstRef.new( + constant: constant, + location: operator.location.to(constant.location) + ) + end + + # TStringBeg represents the beginning of a string literal. + # + # "string" + # + # In the example above, TStringBeg represents the first set of quotes. Strings + # can also use single quotes. They can also be declared using the +%q+ and + # +%Q+ syntax, as in: + # + # %q{string} + # + class TStringBeg + # [String] the beginning of the string + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_tstring_beg: (String value) -> TStringBeg + def on_tstring_beg(value) + node = + TStringBeg.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # TStringContent represents plain characters inside of an entity that accepts + # string content like a string, heredoc, command string, or regular + # expression. + # + # "string" + # + # In the example above, TStringContent represents the +string+ token contained + # within the string. + class TStringContent + # [String] the content of the string + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('tstring_content') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { + type: :tstring_content, + value: value.force_encoding('UTF-8'), + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_tstring_content: (String value) -> TStringContent + def on_tstring_content(value) + TStringContent.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + end + + # TStringEnd represents the end of a string literal. + # + # "string" + # + # In the example above, TStringEnd represents the second set of quotes. + # Strings can also use single quotes. They can also be declared using the +%q+ + # and +%Q+ syntax, as in: + # + # %q{string} + # + class TStringEnd + # [String] the end of the string + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_tstring_end: (String value) -> TStringEnd + def on_tstring_end(value) + node = + TStringEnd.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # Not represents the unary +not+ method being called on an expression. + # + # not value + # + class Not + # [untyped] the statement on which to operate + attr_reader :statement + + # [boolean] whether or not parentheses were used + attr_reader :parentheses + + # [Location] the location of this node + attr_reader :location + + def initialize(statement:, parentheses:, location:) + @statement = statement + @parentheses = parentheses + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('not') + + q.breakable + q.pp(statement) + end + end + + def to_json(*opts) + { + type: :not, + value: statement, + paren: parentheses, + loc: location + }.to_json(*opts) + end + end + + # Unary represents a unary method being called on an expression, as in +!+ or + # +~+. + # + # !value + # + class Unary + # [String] the operator being used + attr_reader :operator + + # [untyped] the statement on which to operate + attr_reader :statement + + # [Location] the location of this node + attr_reader :location + + def initialize(operator:, statement:, location:) + @operator = operator + @statement = statement + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('unary') + + q.breakable + q.pp(operator) + + q.breakable + q.pp(statement) + end + end + + def to_json(*opts) + { type: :unary, op: operator, value: statement, loc: location }.to_json( + *opts + ) + end + end + + # :call-seq: + # on_unary: (:not operator, untyped statement) -> Not + # | (Symbol operator, untyped statement) -> Unary + def on_unary(operator, statement) + if operator == :not + # We have somewhat special handling of the not operator since if it has + # parentheses they don't get reported as a paren node for some reason. + + beginning = find_token(Kw, 'not') + ending = statement + + range = beginning.location.end_char...statement.location.start_char + paren = source[range].include?('(') + + if paren + find_token(LParen) + ending = find_token(RParen) + end + + Not.new( + statement: statement, + parentheses: paren, + location: beginning.location.to(ending.location) + ) + else + # Special case instead of using find_token here. It turns out that + # if you have a range that goes from a negative number to a negative + # number then you can end up with a .. or a ... that's higher in the + # stack. So we need to explicitly disallow those operators. + index = + tokens.rindex do |token| + token.is_a?(Op) && + token.location.start_char < statement.location.start_char && + !%w[.. ...].include?(token.value) + end + + beginning = tokens.delete_at(index) + + Unary.new( + operator: operator[0], # :+@ -> "+" + statement: statement, + location: beginning.location.to(statement.location) + ) + end + end + + # Undef represents the use of the +undef+ keyword. + # + # undef method + # + class Undef + # [Array[ DynaSymbol | SymbolLiteral ]] the symbols to undefine + attr_reader :symbols + + # [Location] the location of this node + attr_reader :location + + def initialize(symbols:, location:) + @symbols = symbols + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('undef') + + q.breakable + q.group(2, '(', ')') { q.seplist(symbols) { |symbol| q.pp(symbol) } } + end + end + + def to_json(*opts) + { type: :undef, syms: symbols, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_undef: (Array[DynaSymbol | SymbolLiteral] symbols) -> Undef + def on_undef(symbols) + keyword = find_token(Kw, 'undef') + + Undef.new( + symbols: symbols, + location: keyword.location.to(symbols.last.location) + ) + end + + # Unless represents the first clause in an +unless+ chain. + # + # unless predicate + # end + # + class Unless + # [untyped] the expression to be checked + attr_reader :predicate + + # [Statements] the expressions to be executed + attr_reader :statements + + # [nil, Elsif, Else] the next clause in the chain + attr_reader :consequent + + # [Location] the location of this node + attr_reader :location + + def initialize(predicate:, statements:, consequent:, location:) + @predicate = predicate + @statements = statements + @consequent = consequent + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('unless') + + q.breakable + q.pp(predicate) + + q.breakable + q.pp(statements) + + if consequent + q.breakable + q.pp(consequent) + end + end + end + + def to_json(*opts) + { + type: :unless, + pred: predicate, + stmts: statements, + cons: consequent, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_unless: ( + # untyped predicate, + # Statements statements, + # ((nil | Elsif | Else) consequent) + # ) -> Unless + def on_unless(predicate, statements, consequent) + beginning = find_token(Kw, 'unless') + ending = consequent || find_token(Kw, 'end') + + statements.bind(predicate.location.end_char, ending.location.start_char) + + Unless.new( + predicate: predicate, + statements: statements, + consequent: consequent, + location: beginning.location.to(ending.location) + ) + end + + # UnlessMod represents the modifier form of an +unless+ statement. + # + # expression unless predicate + # + class UnlessMod + # [untyped] the expression to be executed + attr_reader :statement + + # [untyped] the expression to be checked + attr_reader :predicate + + # [Location] the location of this node + attr_reader :location + + def initialize(statement:, predicate:, location:) + @statement = statement + @predicate = predicate + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('unless_mod') + + q.breakable + q.pp(statement) + + q.breakable + q.pp(predicate) + end + end + + def to_json(*opts) + { + type: :unless_mod, + stmt: statement, + pred: predicate, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_unless_mod: (untyped predicate, untyped statement) -> UnlessMod + def on_unless_mod(predicate, statement) + find_token(Kw, 'unless') + + UnlessMod.new( + statement: statement, + predicate: predicate, + location: statement.location.to(predicate.location) + ) + end + + # Until represents an +until+ loop. + # + # until predicate + # end + # + class Until + # [untyped] the expression to be checked + attr_reader :predicate + + # [Statements] the expressions to be executed + attr_reader :statements + + # [Location] the location of this node + attr_reader :location + + def initialize(predicate:, statements:, location:) + @predicate = predicate + @statements = statements + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('until') + + q.breakable + q.pp(predicate) + + q.breakable + q.pp(statements) + end + end + + def to_json(*opts) + { + type: :until, + pred: predicate, + stmts: statements, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_until: (untyped predicate, Statements statements) -> Until + def on_until(predicate, statements) + beginning = find_token(Kw, 'until') + ending = find_token(Kw, 'end') + + # Consume the do keyword if it exists so that it doesn't get confused for + # some other block + keyword = find_token(Kw, 'do', consume: false) + if keyword && keyword.location.start_char > predicate.location.end_char && + keyword.location.end_char < ending.location.start_char + tokens.delete(keyword) + end + + # Update the Statements location information + statements.bind(predicate.location.end_char, ending.location.start_char) + + Until.new( + predicate: predicate, + statements: statements, + location: beginning.location.to(ending.location) + ) + end + + # UntilMod represents the modifier form of a +until+ loop. + # + # expression until predicate + # + class UntilMod + # [untyped] the expression to be executed + attr_reader :statement + + # [untyped] the expression to be checked + attr_reader :predicate + + # [Location] the location of this node + attr_reader :location + + def initialize(statement:, predicate:, location:) + @statement = statement + @predicate = predicate + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('until_mod') + + q.breakable + q.pp(statement) + + q.breakable + q.pp(predicate) + end + end + + def to_json(*opts) + { + type: :until_mod, + stmt: statement, + pred: predicate, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_until_mod: (untyped predicate, untyped statement) -> UntilMod + def on_until_mod(predicate, statement) + find_token(Kw, 'until') + + UntilMod.new( + statement: statement, + predicate: predicate, + location: statement.location.to(predicate.location) + ) + end + + # VarAlias represents when you're using the +alias+ keyword with global + # variable arguments. + # + # alias $new $old + # + class VarAlias + # [GVar] the new alias of the variable + attr_reader :left + + # [Backref | GVar] the current name of the variable to be aliased + attr_reader :right + + # [Location] the location of this node + attr_reader :location + + def initialize(left:, right:, location:) + @left = left + @right = right + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('var_alias') + + q.breakable + q.pp(left) + + q.breakable + q.pp(right) + end + end + + def to_json(*opts) + { type: :var_alias, left: left, right: right, loc: location }.to_json( + *opts + ) + end + end + + # :call-seq: + # on_var_alias: (GVar left, (Backref | GVar) right) -> VarAlias + def on_var_alias(left, right) + keyword = find_token(Kw, 'alias') + + VarAlias.new( + left: left, + right: right, + location: keyword.location.to(right.location) + ) + end + + # VarField represents a variable that is being assigned a value. As such, it + # is always a child of an assignment type node. + # + # variable = value + # + # In the example above, the VarField node represents the +variable+ token. + class VarField + # [nil | Const | CVar | GVar | Ident | IVar] the target of this node + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('var_field') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :var_field, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_var_field: ( + # (nil | Const | CVar | GVar | Ident | IVar) value + # ) -> VarField + def on_var_field(value) + location = + if value + value.location + else + # You can hit this pattern if you're assigning to a splat using pattern + # matching syntax in Ruby 2.7+ + Location.fixed(line: lineno, char: char_pos) + end + + VarField.new(value: value, location: location) + end + + # VarRef represents a variable reference. + # + # true + # + # This can be a plain local variable like the example above. It can also be a + # constant, a class variable, a global variable, an instance variable, a + # keyword (like +self+, +nil+, +true+, or +false+), or a numbered block + # variable. + class VarRef + # [Const | CVar | GVar | Ident | IVar | Kw] the value of this node + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('var_ref') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :var_ref, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_var_ref: ((Const | CVar | GVar | Ident | IVar | Kw) value) -> VarRef + def on_var_ref(value) + VarRef.new(value: value, location: value.location) + end + + # AccessCtrl represents a call to a method visibility control, i.e., +public+, + # +protected+, or +private+. + # + # private + # + class AccessCtrl + # [Ident] the value of this expression + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('access_ctrl') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :access_ctrl, value: value, loc: location }.to_json(*opts) + end + end + + # VCall represent any plain named object with Ruby that could be either a + # local variable or a method call. + # + # variable + # + class VCall + # [Ident] the value of this expression + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('vcall') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :vcall, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_vcall: (Ident ident) -> AccessCtrl | VCall + def on_vcall(ident) + @controls ||= %w[private protected public].freeze + + if @controls.include?(ident.value) && ident.value == lines[lineno - 1].strip + # Access controls like private, protected, and public are reported as + # vcall nodes since they're technically method calls. We want to be able + # add new lines around them as necessary, so here we're going to + # explicitly track those as a different node type. + AccessCtrl.new(value: ident, location: ident.location) + else + VCall.new(value: ident, location: ident.location) + end + end + + # VoidStmt represents an empty lexical block of code. + # + # ;; + # + class VoidStmt + # [Location] the location of this node + attr_reader :location + + def initialize(location:) + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') { q.text('void_stmt') } + end + + def to_json(*opts) + { type: :void_stmt, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_void_stmt: () -> VoidStmt + def on_void_stmt + VoidStmt.new(location: Location.fixed(line: lineno, char: char_pos)) + end + + # When represents a +when+ clause in a +case+ chain. + # + # case value + # when predicate + # end + # + class When + # [untyped] the arguments to the when clause + attr_reader :arguments + + # [Statements] the expressions to be executed + attr_reader :statements + + # [nil | Else | When] the next clause in the chain + attr_reader :consequent + + # [Location] the location of this node + attr_reader :location + + def initialize(arguments:, statements:, consequent:, location:) + @arguments = arguments + @statements = statements + @consequent = consequent + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('when') + + q.breakable + q.pp(arguments) + + q.breakable + q.pp(statements) + + if consequent + q.breakable + q.pp(consequent) + end + end + end + + def to_json(*opts) + { + type: :when, + args: arguments, + stmts: statements, + cons: consequent, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_when: ( + # untyped arguments, + # Statements statements, + # (nil | Else | When) consequent + # ) -> When + def on_when(arguments, statements, consequent) + beginning = find_token(Kw, 'when') + ending = consequent || find_token(Kw, 'end') + + statements.bind(arguments.location.end_char, ending.location.start_char) + + When.new( + arguments: arguments, + statements: statements, + consequent: consequent, + location: beginning.location.to(ending.location) + ) + end + + # While represents a +while+ loop. + # + # while predicate + # end + # + class While + # [untyped] the expression to be checked + attr_reader :predicate + + # [Statements] the expressions to be executed + attr_reader :statements + + # [Location] the location of this node + attr_reader :location + + def initialize(predicate:, statements:, location:) + @predicate = predicate + @statements = statements + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('while') + + q.breakable + q.pp(predicate) + + q.breakable + q.pp(statements) + end + end + + def to_json(*opts) + { + type: :while, + pred: predicate, + stmts: statements, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_while: (untyped predicate, Statements statements) -> While + def on_while(predicate, statements) + beginning = find_token(Kw, 'while') + ending = find_token(Kw, 'end') + + # Consume the do keyword if it exists so that it doesn't get confused for + # some other block + keyword = find_token(Kw, 'do', consume: false) + if keyword && keyword.location.start_char > predicate.location.end_char && + keyword.location.end_char < ending.location.start_char + tokens.delete(keyword) + end + + # Update the Statements location information + statements.bind(predicate.location.end_char, ending.location.start_char) + + While.new( + predicate: predicate, + statements: statements, + location: beginning.location.to(ending.location) + ) + end + + # WhileMod represents the modifier form of a +while+ loop. + # + # expression while predicate + # + class WhileMod + # [untyped] the expression to be executed + attr_reader :statement + + # [untyped] the expression to be checked + attr_reader :predicate + + # [Location] the location of this node + attr_reader :location + + def initialize(statement:, predicate:, location:) + @statement = statement + @predicate = predicate + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('while_mod') + + q.breakable + q.pp(statement) + + q.breakable + q.pp(predicate) + end + end + + def to_json(*opts) + { + type: :while_mod, + stmt: statement, + pred: predicate, + loc: location + }.to_json(*opts) + end + end + + # :call-seq: + # on_while_mod: (untyped predicate, untyped statement) -> WhileMod + def on_while_mod(predicate, statement) + find_token(Kw, 'while') + + WhileMod.new( + statement: statement, + predicate: predicate, + location: statement.location.to(predicate.location) + ) + end + + # Word represents an element within a special array literal that accepts + # interpolation. + # + # %W[a#{b}c xyz] + # + # In the example above, there would be two Word nodes within a parent Words + # node. + class Word + # [Array[ StringEmbExpr | StringDVar | TStringContent ]] the parts of the + # word + attr_reader :parts + + # [Location] the location of this node + attr_reader :location + + def initialize(parts:, location:) + @parts = parts + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('word') + + q.breakable + q.group(2, '(', ')') { q.seplist(parts) { |part| q.pp(part) } } + end + end + + def to_json(*opts) + { type: :word, parts: parts, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_word_add: ( + # Word word, + # (StringEmbExpr | StringDVar | TStringContent) part + # ) -> Word + def on_word_add(word, part) + location = + word.parts.empty? ? part.location : word.location.to(part.location) + + Word.new(parts: word.parts << part, location: location) + end + + # :call-seq: + # on_word_new: () -> Word + def on_word_new + Word.new(parts: [], location: Location.fixed(line: lineno, char: char_pos)) + end + + # Words represents a string literal array with interpolation. + # + # %W[one two three] + # + class Words + # [Array[ Word ]] the elements of this array + attr_reader :elements + + # [Location] the location of this node + attr_reader :location + + def initialize(elements:, location:) + @elements = elements + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('words') + + q.breakable + q.group(2, '(', ')') { q.seplist(elements) { |element| q.pp(element) } } + end + end + + def to_json(*opts) + { type: :words, elems: elements, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_words_add: (Words words, Word word) -> Words + def on_words_add(words, word) + Words.new( + elements: words.elements << word, + location: words.location.to(word.location) + ) + end + + # WordsBeg represents the beginning of a string literal array with + # interpolation. + # + # %W[one two three] + # + # In the snippet above, a WordsBeg would be created with the value of "%W[". + # Note that these kinds of arrays can start with a lot of different delimiter + # types (e.g., %W| or %W<). + class WordsBeg + # [String] the start of the word literal array + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + end + + # :call-seq: + # on_words_beg: (String value) -> WordsBeg + def on_words_beg(value) + node = + WordsBeg.new( + value: value, + location: Location.token(line: lineno, char: char_pos, size: value.size) + ) + + tokens << node + node + end + + # :call-seq: + # on_words_new: () -> Words + def on_words_new + words_beg = find_token(WordsBeg) + + Words.new(elements: [], location: words_beg.location) + end + + # def on_words_sep(value) + # value + # end + + # XString represents the contents of an XStringLiteral. + # + # `ls` + # + class XString + # [Array[ StringEmbExpr | StringDVar | TStringContent ]] the parts of the + # xstring + attr_reader :parts + + # [Location] the location of this node + attr_reader :location + + def initialize(parts:, location:) + @parts = parts + @location = location + end + end + + # :call-seq: + # on_xstring_add: ( + # XString xstring, + # (StringEmbExpr | StringDVar | TStringContent) part + # ) -> XString + def on_xstring_add(xstring, part) + XString.new( + parts: xstring.parts << part, + location: xstring.location.to(part.location) + ) + end + + # :call-seq: + # on_xstring_new: () -> XString + def on_xstring_new + heredoc = @heredocs[-1] + + location = + if heredoc && heredoc.beginning.value.include?('`') + heredoc.location + else + find_token(Backtick).location + end + + XString.new(parts: [], location: location) + end + + # XStringLiteral represents a string that gets executed. + # + # `ls` + # + class XStringLiteral + # [Array[ StringEmbExpr | StringDVar | TStringContent ]] the parts of the + # xstring + attr_reader :parts + + # [Location] the location of this node + attr_reader :location + + def initialize(parts:, location:) + @parts = parts + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('xstring_literal') + + q.breakable + q.group(2, '(', ')') { q.seplist(parts) { |part| q.pp(part) } } + end + end + + def to_json(*opts) + { type: :xstring_literal, parts: parts, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_xstring_literal: (XString xstring) -> Heredoc | XStringLiteral + def on_xstring_literal(xstring) + heredoc = @heredocs[-1] + + if heredoc && heredoc.beginning.value.include?('`') + Heredoc.new( + beginning: heredoc.beginning, + ending: heredoc.ending, + parts: xstring.parts, + location: heredoc.location + ) + else + ending = find_token(TStringEnd) + + XStringLiteral.new( + parts: xstring.parts, + location: xstring.location.to(ending.location) + ) + end + end + + # Yield represents using the +yield+ keyword with arguments. + # + # yield value + # + class Yield + # [ArgsAddBlock | Paren] the arguments passed to the yield + attr_reader :arguments + + # [Location] the location of this node + attr_reader :location + + def initialize(arguments:, location:) + @arguments = arguments + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('yield') + + q.breakable + q.pp(arguments) + end + end + + def to_json(*opts) + { type: :yield, args: arguments, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_yield: ((ArgsAddBlock | Paren) arguments) -> Yield + def on_yield(arguments) + keyword = find_token(Kw, 'yield') + + Yield.new( + arguments: arguments, + location: keyword.location.to(arguments.location) + ) + end + + # Yield0 represents the bare +yield+ keyword with no arguments. + # + # yield + # + class Yield0 + # [String] the value of the keyword + attr_reader :value + + # [Location] the location of this node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('yield0') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :yield0, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_yield0: () -> Yield0 + def on_yield0 + keyword = find_token(Kw, 'yield') + + Yield0.new(value: keyword.value, location: keyword.location) + end + + # ZSuper represents the bare +super+ keyword with no arguments. + # + # super + # + class ZSuper + # [String] the value of the keyword + attr_reader :value + + # [Location] the location of the node + attr_reader :location + + def initialize(value:, location:) + @value = value + @location = location + end + + def pretty_print(q) + q.group(2, '(', ')') do + q.text('zsuper') + + q.breakable + q.pp(value) + end + end + + def to_json(*opts) + { type: :zsuper, value: value, loc: location }.to_json(*opts) + end + end + + # :call-seq: + # on_zsuper: () -> ZSuper + def on_zsuper + keyword = find_token(Kw, 'super') + + ZSuper.new(value: keyword.value, location: keyword.location) + end +end diff --git a/spec/syntax_suggest/fixtures/this_project_extra_def.rb.txt b/spec/syntax_suggest/fixtures/this_project_extra_def.rb.txt new file mode 100644 index 0000000000..e62fd3fa66 --- /dev/null +++ b/spec/syntax_suggest/fixtures/this_project_extra_def.rb.txt @@ -0,0 +1,64 @@ +module SyntaxErrorSearch + # Used for formatting invalid blocks + class DisplayInvalidBlocks + attr_reader :filename + + def initialize(block_array, io: $stderr, filename: nil) + @filename = filename + @io = io + @blocks = block_array + @lines = @blocks.map(&:lines).flatten + @digit_count = @lines.last.line_number.to_s.length + @code_lines = @blocks.first.code_lines + + @invalid_line_hash = @lines.each_with_object({}) {|line, h| h[line] = true} + end + + def call + @io.puts <<~EOM + + SyntaxSuggest: A syntax error was detected + + This code has an unmatched `end` this is caused by either + missing a syntax keyword (`def`, `do`, etc.) or inclusion + of an extra `end` line: + EOM + + @io.puts(<<~EOM) if filename + file: #{filename} + EOM + + @io.puts <<~EOM + #{code_with_filename} + EOM + end + + def filename + + def code_with_filename + string = String.new("") + string << "```\n" + string << "#".rjust(@digit_count) + " filename: #{filename}\n\n" if filename + string << code_with_lines + string << "```\n" + string + end + + def code_with_lines + @code_lines.map do |line| + next if line.hidden? + number = line.line_number.to_s.rjust(@digit_count) + if line.empty? + "#{number.to_s}#{line}" + else + string = String.new + string << "\e[1;3m" if @invalid_line_hash[line] # Bold, italics + string << "#{number.to_s} " + string << line.to_s + string << "\e[0m" + string + end + end.join + end + end +end diff --git a/spec/syntax_suggest/fixtures/webmock.rb.txt b/spec/syntax_suggest/fixtures/webmock.rb.txt new file mode 100644 index 0000000000..16da0d2ac0 --- /dev/null +++ b/spec/syntax_suggest/fixtures/webmock.rb.txt @@ -0,0 +1,35 @@ +describe "webmock tests" do + before(:each) do + WebMock.enable! + end + + after(:each) do + WebMock.disable! + end + + it "port" do + port = rand(1000...9999) + stub_request(:any, "localhost:#{port}") + + query = Cutlass::FunctionQuery.new( + port: port + ).call + + expect(WebMock).to have_requested(:post, "localhost:#{port}"). + with(body: "{}") + end + + it "body" do + body = { lol: "hi" } + port = 8080 + stub_request(:any, "localhost:#{port}") + + query = Cutlass::FunctionQuery.new( + port: port + body: body + ).call + + expect(WebMock).to have_requested(:post, "localhost:#{port}"). + with(body: body.to_json) + end +end diff --git a/spec/syntax_suggest/integration/exe_cli_spec.rb b/spec/syntax_suggest/integration/exe_cli_spec.rb new file mode 100644 index 0000000000..b9a3173715 --- /dev/null +++ b/spec/syntax_suggest/integration/exe_cli_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe "exe" do + def exe_path + if ruby_core? + root_dir.join("../libexec").join("syntax_suggest") + else + root_dir.join("exe").join("syntax_suggest") + end + end + + def exe(cmd) + ruby = ENV.fetch("RUBY", "ruby") + out = run!("#{ruby} #{exe_path} #{cmd}", raise_on_nonzero_exit: false) + puts out if ENV["SYNTAX_SUGGEST_DEBUG"] + out + end + + it "prints the version" do + out = exe("-v") + expect(out.strip).to include(SyntaxSuggest::VERSION) + end + end +end diff --git a/spec/syntax_suggest/integration/ruby_command_line_spec.rb b/spec/syntax_suggest/integration/ruby_command_line_spec.rb new file mode 100644 index 0000000000..c1ec4be54e --- /dev/null +++ b/spec/syntax_suggest/integration/ruby_command_line_spec.rb @@ -0,0 +1,193 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + ruby = ENV.fetch("RUBY", "ruby") + RSpec.describe "Requires with ruby cli" do + it "namespaces all monkeypatched methods" do + Dir.mktmpdir do |dir| + tmpdir = Pathname(dir) + script = tmpdir.join("script.rb") + script.write <<~EOM + puts Kernel.private_methods + EOM + + syntax_suggest_methods_file = tmpdir.join("syntax_suggest_methods.txt") + api_only_methods_file = tmpdir.join("api_only_methods.txt") + kernel_methods_file = tmpdir.join("kernel_methods.txt") + + d_pid = Process.spawn("#{ruby} -I#{lib_dir} -rsyntax_suggest #{script} 2>&1 > #{syntax_suggest_methods_file}") + k_pid = Process.spawn("#{ruby} #{script} 2>&1 >> #{kernel_methods_file}") + r_pid = Process.spawn("#{ruby} -I#{lib_dir} -rsyntax_suggest/api #{script} 2>&1 > #{api_only_methods_file}") + + Process.wait(k_pid) + Process.wait(d_pid) + Process.wait(r_pid) + + kernel_methods_array = kernel_methods_file.read.strip.lines.map(&:strip) + syntax_suggest_methods_array = syntax_suggest_methods_file.read.strip.lines.map(&:strip) + api_only_methods_array = api_only_methods_file.read.strip.lines.map(&:strip) + + # In ruby 3.1.0-preview1 the `timeout` file is already required + # we can remove it if it exists to normalize the output for + # all ruby versions + [syntax_suggest_methods_array, kernel_methods_array, api_only_methods_array].each do |array| + array.delete("timeout") + end + + methods = (syntax_suggest_methods_array - kernel_methods_array).sort + if methods.any? + expect(methods).to eq(["syntax_suggest_original_load", "syntax_suggest_original_require", "syntax_suggest_original_require_relative"]) + end + + methods = (api_only_methods_array - kernel_methods_array).sort + expect(methods).to eq([]) + end + end + + # Since Ruby 3.2 includes syntax_suggest as a default gem, we might accidentally + # be requiring the default gem instead of this library under test. Assert that's + # not the case + it "tests current version of syntax_suggest" do + Dir.mktmpdir do |dir| + tmpdir = Pathname(dir) + script = tmpdir.join("script.rb") + contents = <<~'EOM' + puts "suggest_version is #{SyntaxSuggest::VERSION}" + EOM + script.write(contents) + + out = `#{ruby} -I#{lib_dir} -rsyntax_suggest/version #{script} 2>&1` + + expect(out).to include("suggest_version is #{SyntaxSuggest::VERSION}").once + end + end + + it "detects require error and adds a message with auto mode" do + Dir.mktmpdir do |dir| + tmpdir = Pathname(dir) + script = tmpdir.join("script.rb") + script.write <<~EOM + describe "things" do + it "blerg" do + end + + it "flerg" + end + + it "zlerg" do + end + end + EOM + + require_rb = tmpdir.join("require.rb") + require_rb.write <<~EOM + load "#{script.expand_path}" + EOM + + out = `#{ruby} -I#{lib_dir} -rsyntax_suggest #{require_rb} 2>&1` + + expect($?.success?).to be_falsey + expect(out).to include('> 5 it "flerg"').once + end + end + + it "gem can be tested when executing on Ruby with default gem included" do + skip if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.2") + + out = `#{ruby} -I#{lib_dir} -rsyntax_suggest -e "puts SyntaxError.instance_method(:detailed_message).source_location" 2>&1` + + expect($?.success?).to be_truthy + expect(out).to include(lib_dir.join("syntax_suggest").join("core_ext.rb").to_s).once + end + + it "annotates a syntax error in Ruby 3.2+ when require is not used" do + skip if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.2") + + Dir.mktmpdir do |dir| + tmpdir = Pathname(dir) + script = tmpdir.join("script.rb") + script.write <<~EOM + describe "things" do + it "blerg" do + end + + it "flerg" + end + + it "zlerg" do + end + end + EOM + + out = `#{ruby} -I#{lib_dir} -rsyntax_suggest #{script} 2>&1` + + expect($?.success?).to be_falsey + expect(out).to include('> 5 it "flerg"').once + end + end + + it "does not load internals into memory if no syntax error" do + Dir.mktmpdir do |dir| + tmpdir = Pathname(dir) + script = tmpdir.join("script.rb") + script.write <<~EOM + class Dog + end + + if defined?(SyntaxSuggest::DEFAULT_VALUE) + puts "SyntaxSuggest is loaded" + else + puts "SyntaxSuggest is NOT loaded" + end + EOM + + require_rb = tmpdir.join("require.rb") + require_rb.write <<~EOM + load "#{script.expand_path}" + EOM + + out = `#{ruby} -I#{lib_dir} -rsyntax_suggest #{require_rb} 2>&1` + + expect($?.success?).to be_truthy + expect(out).to include("SyntaxSuggest is NOT loaded").once + end + end + + it "ignores eval" do + Dir.mktmpdir do |dir| + tmpdir = Pathname(dir) + script = tmpdir.join("script.rb") + script.write <<~EOM + $stderr = STDOUT + eval("def lol") + EOM + + out = `#{ruby} -I#{lib_dir} -rsyntax_suggest #{script} 2>&1` + + expect($?.success?).to be_falsey + expect(out).to match(/\(eval.*\):1/) + + expect(out).to_not include("SyntaxSuggest") + expect(out).to_not include("Could not find filename") + end + end + + it "does not say 'syntax ok' when a syntax error fires" do + Dir.mktmpdir do |dir| + tmpdir = Pathname(dir) + script = tmpdir.join("script.rb") + script.write <<~EOM + break + EOM + + out = `#{ruby} -I#{lib_dir} -rsyntax_suggest -e "require_relative '#{script}'" 2>&1` + + expect($?.success?).to be_falsey + expect(out.downcase).to_not include("syntax ok") + expect(out).to include("Invalid break") + end + end + end +end diff --git a/spec/syntax_suggest/integration/syntax_suggest_spec.rb b/spec/syntax_suggest/integration/syntax_suggest_spec.rb new file mode 100644 index 0000000000..9071d37c1b --- /dev/null +++ b/spec/syntax_suggest/integration/syntax_suggest_spec.rb @@ -0,0 +1,239 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe "Integration tests that don't spawn a process (like using the cli)" do + it "does not timeout on massive files" do + next unless ENV["SYNTAX_SUGGEST_TIMEOUT"] + + file = fixtures_dir.join("syntax_tree.rb.txt") + lines = file.read.lines + lines.delete_at(768 - 1) + + io = StringIO.new + + benchmark = Benchmark.measure do + debug_perf do + SyntaxSuggest.call( + io: io, + source: lines.join, + filename: file + ) + end + end + + debug_display(io.string) + debug_display(benchmark) + + expect(io.string).to include(<<~EOM) + 6 class SyntaxTree < Ripper + 170 def self.parse(source) + 174 end + > 754 def on_args_add(arguments, argument) + > 776 class ArgsAddBlock + > 810 end + 9233 end + EOM + end + + it "re-checks all block code, not just what's visible issues/95" do + file = fixtures_dir.join("ruby_buildpack.rb.txt") + io = StringIO.new + + debug_perf do + benchmark = Benchmark.measure do + SyntaxSuggest.call( + io: io, + source: file.read, + filename: file + ) + end + debug_display(io.string) + debug_display(benchmark) + end + + expect(io.string).to_not include("def ruby_install_binstub_path") + expect(io.string).to include(<<~EOM) + > 1067 def add_yarn_binary + > 1068 return [] if yarn_preinstalled? + > 1069 | + > 1075 end + EOM + end + + it "returns good results on routes.rb" do + source = fixtures_dir.join("routes.rb.txt").read + + io = StringIO.new + SyntaxSuggest.call( + io: io, + source: source + ) + debug_display(io.string) + + expect(io.string).to include(<<~EOM) + 1 Rails.application.routes.draw do + > 113 namespace :admin do + > 116 match "/foobar(*path)", via: :all, to: redirect { |_params, req| + > 120 } + 121 end + EOM + end + + it "handles multi-line-methods issues/64" do + source = fixtures_dir.join("webmock.rb.txt").read + + io = StringIO.new + SyntaxSuggest.call( + io: io, + source: source + ) + debug_display(io.string) + + expect(io.string).to include(<<~EOM) + 1 describe "webmock tests" do + 22 it "body" do + 27 query = Cutlass::FunctionQuery.new( + > 28 port: port + > 29 body: body + 30 ).call + 34 end + 35 end + EOM + end + + it "handles derailed output issues/50" do + source = fixtures_dir.join("derailed_require_tree.rb.txt").read + + io = StringIO.new + SyntaxSuggest.call( + io: io, + source: source + ) + debug_display(io.string) + + expect(io.string).to include(<<~EOM) + 5 module DerailedBenchmarks + 6 class RequireTree + > 13 def initialize(name) + > 18 def self.reset! + > 25 end + 73 end + 74 end + EOM + end + + it "handles heredocs" do + lines = fixtures_dir.join("rexe.rb.txt").read.lines + lines.delete_at(85 - 1) + io = StringIO.new + SyntaxSuggest.call( + io: io, + source: lines.join + ) + + out = io.string + debug_display(out) + + expect(out).to include(<<~EOM) + 16 class Rexe + > 77 class Lookups + > 78 def input_modes + > 148 end + 551 end + EOM + end + + it "rexe" do + lines = fixtures_dir.join("rexe.rb.txt").read.lines + lines.delete_at(148 - 1) + source = lines.join + + io = StringIO.new + SyntaxSuggest.call( + io: io, + source: source + ) + out = io.string + expect(out).to include(<<~EOM) + 16 class Rexe + > 77 class Lookups + > 140 def format_requires + > 148 end + 551 end + EOM + end + + it "ambiguous end" do + source = <<~EOM + def call # 0 + print "lol" # 1 + end # one # 2 + end # two # 3 + EOM + io = StringIO.new + SyntaxSuggest.call( + io: io, + source: source + ) + out = io.string + expect(out).to include(<<~EOM) + > 1 def call # 0 + > 3 end # one # 2 + > 4 end # two # 3 + EOM + end + + it "simple regression" do + source = <<~EOM + class Dog + def bark + puts "woof" + end + EOM + io = StringIO.new + SyntaxSuggest.call( + io: io, + source: source + ) + out = io.string + expect(out).to include(<<~EOM) + > 1 class Dog + > 2 def bark + > 4 end + EOM + end + + it "empty else" do + source = <<~EOM + class Foo + def foo + if cond? + foo + else + + end + end + + # ... + + def bar + if @recv + end_is_missing_here + end + end + EOM + + io = StringIO.new + SyntaxSuggest.call( + io: io, + source: source + ) + out = io.string + expect(out).to include(<<~EOM) + end_is_missing_here + EOM + end + end +end diff --git a/spec/syntax_suggest/spec_helper.rb b/spec/syntax_suggest/spec_helper.rb new file mode 100644 index 0000000000..89bc9f4ab1 --- /dev/null +++ b/spec/syntax_suggest/spec_helper.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: true + +require "bundler/setup" +require "syntax_suggest/api" + +require "benchmark" +require "tempfile" + +RSpec.configure do |config| + # Enable flags like --only-failures and --next-failure + config.example_status_persistence_file_path = ".rspec_status" + + # Disable RSpec exposing methods globally on `Module` and `main` + config.disable_monkey_patching! + + config.expect_with :rspec do |c| + c.syntax = :expect + end + + if config.color_mode == :automatic + if config.color_enabled? && ((ENV["TERM"] == "dumb") || ENV["NO_COLOR"]&.slice(0)) + config.color_mode = :off + end + end +end + +# Used for debugging modifications to +# display output +def debug_display(output) + return unless ENV["DEBUG_DISPLAY"] + puts + puts output + puts +end + +def spec_dir + Pathname(__dir__) +end + +def lib_dir + if ruby_core? + root_dir.join("../lib") + else + root_dir.join("lib") + end +end + +def root_dir + spec_dir.join("..") +end + +def fixtures_dir + spec_dir.join("fixtures") +end + +def ruby_core? + !root_dir.join("syntax_suggest.gemspec").exist? +end + +def code_line_array(source) + SyntaxSuggest::CleanDocument.new(source: source).call.lines +end + +autoload :RubyProf, "ruby-prof" + +def debug_perf + raise "No block given" unless block_given? + + if ENV["DEBUG_PERF"] + out = nil + result = RubyProf.profile do + out = yield + end + + dir = SyntaxSuggest.record_dir("tmp") + printer = RubyProf::MultiPrinter.new(result, [:flat, :graph, :graph_html, :tree, :call_tree, :stack, :dot]) + printer.print(path: dir, profile: "profile") + + out + else + yield + end +end + +def run!(cmd, raise_on_nonzero_exit: true) + out = `#{cmd} 2>&1` + raise "Command: #{cmd} failed: #{out}" if !$?.success? && raise_on_nonzero_exit + out +end + +# Allows us to write cleaner tests since <<~EOM block quotes +# strip off all leading indentation and we need it to be preserved +# sometimes. +class String + def indent(number) + lines.map do |line| + if line.chomp.empty? + line + else + " " * number + line + end + end.join + end +end diff --git a/spec/syntax_suggest/unit/api_spec.rb b/spec/syntax_suggest/unit/api_spec.rb new file mode 100644 index 0000000000..e900b9e10b --- /dev/null +++ b/spec/syntax_suggest/unit/api_spec.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" +begin + require "ruby-prof" +rescue LoadError +end + +module SyntaxSuggest + RSpec.describe "Top level SyntaxSuggest api" do + it "doesn't load prism if env var is set" do + skip("SYNTAX_SUGGEST_DISABLE_PRISM not set") unless ENV["SYNTAX_SUGGEST_DISABLE_PRISM"] + + expect(SyntaxSuggest.use_prism_parser?).to be_falsey + end + + it "has a `handle_error` interface" do + fake_error = Object.new + def fake_error.message + "#{__FILE__}:216: unterminated string meets end of file " + end + + def fake_error.is_a?(v) + true + end + + io = StringIO.new + SyntaxSuggest.handle_error( + fake_error, + re_raise: false, + io: io + ) + + expect(io.string.strip).to eq("") + end + + it "raises original error with warning if a non-syntax error is passed" do + error = NameError.new("blerg") + io = StringIO.new + expect { + SyntaxSuggest.handle_error( + error, + re_raise: false, + io: io + ) + }.to raise_error { |e| + expect(io.string).to include("Must pass a SyntaxError") + expect(e).to eq(error) + } + end + + it "raises original error with warning if file is not found" do + fake_error = SyntaxError.new + def fake_error.message + "#does/not/exist/lol/doesnotexist:216: unterminated string meets end of file " + end + + io = StringIO.new + expect { + SyntaxSuggest.handle_error( + fake_error, + re_raise: false, + io: io + ) + }.to raise_error { |e| + expect(io.string).to include("Could not find filename") + expect(e).to eq(fake_error) + } + end + + it "respects highlight API" do + skip if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.2") + + core_ext_file = lib_dir.join("syntax_suggest").join("core_ext.rb") + require_relative core_ext_file + + error_klass = Class.new do + def path + fixtures_dir.join("this_project_extra_def.rb.txt") + end + + def detailed_message(**kwargs) + "error" + end + end + error_klass.prepend(SyntaxSuggest.module_for_detailed_message) + error = error_klass.new + + expect(error.detailed_message(highlight: true)).to include(SyntaxSuggest::DisplayCodeWithLineNumbers::TERMINAL_HIGHLIGHT) + expect(error.detailed_message(highlight: false)).to_not include(SyntaxSuggest::DisplayCodeWithLineNumbers::TERMINAL_HIGHLIGHT) + end + + it "can be disabled via falsey kwarg" do + skip if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.2") + + core_ext_file = lib_dir.join("syntax_suggest").join("core_ext.rb") + require_relative core_ext_file + + error_klass = Class.new do + def path + fixtures_dir.join("this_project_extra_def.rb.txt") + end + + def detailed_message(**kwargs) + "error" + end + end + error_klass.prepend(SyntaxSuggest.module_for_detailed_message) + error = error_klass.new + + expect(error.detailed_message(syntax_suggest: true)).to_not eq(error.detailed_message(syntax_suggest: false)) + end + end +end diff --git a/spec/syntax_suggest/unit/around_block_scan_spec.rb b/spec/syntax_suggest/unit/around_block_scan_spec.rb new file mode 100644 index 0000000000..6c940a5919 --- /dev/null +++ b/spec/syntax_suggest/unit/around_block_scan_spec.rb @@ -0,0 +1,165 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe AroundBlockScan do + it "continues scan from last location even if scan is false" do + source = <<~EOM + print 'omg' + print 'lol' + print 'haha' + EOM + code_lines = CodeLine.from_source(source) + block = CodeBlock.new(lines: code_lines[1]) + expand = AroundBlockScan.new(code_lines: code_lines, block: block) + .scan_neighbors_not_empty + + expect(expand.code_block.to_s).to eq(source) + expand.scan_while { |line| false } + + expect(expand.code_block.to_s).to eq(source) + end + + it "scan_adjacent_indent works on first or last line" do + source_string = <<~EOM + def foo + if [options.output_format_tty, options.output_format_block].include?(nil) + raise("Bad output mode '\#{v}'; each must be one of \#{lookups.output_formats.keys}.") + end + end + EOM + + code_lines = code_line_array(source_string) + block = CodeBlock.new(lines: code_lines[4]) + expand = AroundBlockScan.new(code_lines: code_lines, block: block) + .scan_adjacent_indent + + expect(expand.code_block.to_s).to eq(<<~EOM) + def foo + if [options.output_format_tty, options.output_format_block].include?(nil) + raise("Bad output mode '\#{v}'; each must be one of \#{lookups.output_formats.keys}.") + end + end + EOM + end + + it "expands indentation" do + source_string = <<~EOM + def foo + if [options.output_format_tty, options.output_format_block].include?(nil) + raise("Bad output mode '\#{v}'; each must be one of \#{lookups.output_formats.keys}.") + end + end + EOM + + code_lines = code_line_array(source_string) + block = CodeBlock.new(lines: code_lines[2]) + expand = AroundBlockScan.new(code_lines: code_lines, block: block) + .stop_after_kw + .scan_adjacent_indent + + expect(expand.code_block.to_s).to eq(<<~EOM.indent(2)) + if [options.output_format_tty, options.output_format_block].include?(nil) + raise("Bad output mode '\#{v}'; each must be one of \#{lookups.output_formats.keys}.") + end + EOM + end + + it "can stop before hitting another end" do + source_string = <<~EOM + def lol + end + def foo + puts "lol" + end + EOM + + code_lines = code_line_array(source_string) + block = CodeBlock.new(lines: code_lines[3]) + expand = AroundBlockScan.new(code_lines: code_lines, block: block) + expand.stop_after_kw + expand.scan_while { true } + + expect(expand.code_block.to_s).to eq(<<~EOM) + def foo + puts "lol" + end + EOM + end + + it "captures multiple empty and hidden lines" do + source_string = <<~EOM + def foo + Foo.call + + puts "lol" + + end + end + EOM + + code_lines = code_line_array(source_string) + block = CodeBlock.new(lines: code_lines[3]) + expand = AroundBlockScan.new(code_lines: code_lines, block: block) + expand.scan_while { true } + + expect(expand.lines.first.index).to eq(0) + expect(expand.lines.last.index).to eq(6) + expect(expand.code_block.to_s).to eq(source_string) + end + + it "only takes what you ask" do + source_string = <<~EOM + def foo + Foo.call + + puts "lol" + + end + end + EOM + + code_lines = code_line_array(source_string) + block = CodeBlock.new(lines: code_lines[3]) + expand = AroundBlockScan.new(code_lines: code_lines, block: block) + expand.scan_while { |line| line.not_empty? } + + expect(expand.code_block.to_s).to eq(<<~EOM.indent(4)) + puts "lol" + EOM + end + + it "skips what you want" do + source_string = <<~EOM + def foo + Foo.call + + puts "haha" + # hide me + + puts "lol" + + end + end + EOM + + code_lines = code_line_array(source_string) + code_lines[4].mark_invisible + + block = CodeBlock.new(lines: code_lines[3]) + expand = AroundBlockScan.new(code_lines: code_lines, block: block) + expand.force_add_empty + expand.force_add_hidden + expand.scan_neighbors_not_empty + + expect(expand.code_block.to_s).to eq(<<~EOM.indent(4)) + + puts "haha" + + puts "lol" + + EOM + end + end +end diff --git a/spec/syntax_suggest/unit/block_expand_spec.rb b/spec/syntax_suggest/unit/block_expand_spec.rb new file mode 100644 index 0000000000..fde0360775 --- /dev/null +++ b/spec/syntax_suggest/unit/block_expand_spec.rb @@ -0,0 +1,230 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe BlockExpand do + it "empty line in methods" do + source_string = <<~EOM + class Dog # index 0 + def bark # index 1 + + end # index 3 + + def sit # index 5 + print "sit" # index 6 + end # index 7 + end # index 8 + end # extra end + EOM + + code_lines = code_line_array(source_string) + + sit = code_lines[4..7] + sit.each(&:mark_invisible) + + block = CodeBlock.new(lines: sit) + expansion = BlockExpand.new(code_lines: code_lines) + block = expansion.expand_neighbors(block) + + expect(block.to_s).to eq(<<~EOM.indent(2)) + def bark # index 1 + + end # index 3 + EOM + end + + it "captures multiple empty and hidden lines" do + source_string = <<~EOM + def foo + Foo.call + + + puts "lol" + + # hidden + end + end + EOM + + code_lines = code_line_array(source_string) + + code_lines[6].mark_invisible + + block = CodeBlock.new(lines: [code_lines[3]]) + expansion = BlockExpand.new(code_lines: code_lines) + block = expansion.call(block) + + expect(block.to_s).to eq(<<~EOM.indent(4)) + + + puts "lol" + + EOM + end + + it "captures multiple empty lines" do + source_string = <<~EOM + def foo + Foo.call + + + puts "lol" + + end + end + EOM + + code_lines = code_line_array(source_string) + block = CodeBlock.new(lines: [code_lines[3]]) + expansion = BlockExpand.new(code_lines: code_lines) + block = expansion.call(block) + + expect(block.to_s).to eq(<<~EOM.indent(4)) + + + puts "lol" + + EOM + end + + it "expands neighbors then indentation" do + source_string = <<~EOM + def foo + Foo.call + puts "hey" + puts "lol" + puts "sup" + end + end + EOM + + code_lines = code_line_array(source_string) + block = CodeBlock.new(lines: [code_lines[3]]) + expansion = BlockExpand.new(code_lines: code_lines) + block = expansion.call(block) + + expect(block.to_s).to eq(<<~EOM.indent(4)) + puts "hey" + puts "lol" + puts "sup" + EOM + + block = expansion.call(block) + + expect(block.to_s).to eq(<<~EOM.indent(2)) + Foo.call + puts "hey" + puts "lol" + puts "sup" + end + EOM + end + + it "handles else code" do + source_string = <<~EOM + Foo.call + if blerg + puts "lol" + else + puts "haha" + end + end + EOM + + code_lines = code_line_array(source_string) + block = CodeBlock.new(lines: [code_lines[2]]) + expansion = BlockExpand.new(code_lines: code_lines) + block = expansion.call(block) + + expect(block.to_s).to eq(<<~EOM.indent(2)) + if blerg + puts "lol" + else + puts "haha" + end + EOM + end + + it "expand until next boundary (indentation)" do + source_string = <<~EOM + describe "what" do + Foo.call + end + + describe "hi" + Bar.call do + Foo.call + end + end + + it "blerg" do + end + EOM + + code_lines = code_line_array(source_string) + + block = CodeBlock.new( + lines: code_lines[6] + ) + + expansion = BlockExpand.new(code_lines: code_lines) + block = expansion.call(block) + + expect(block.to_s).to eq(<<~EOM.indent(2)) + Bar.call do + Foo.call + end + EOM + + block = expansion.call(block) + + expect(block.to_s).to eq(<<~EOM) + describe "hi" + Bar.call do + Foo.call + end + end + EOM + end + + it "expand until next boundary (empty lines)" do + source_string = <<~EOM + describe "what" do + end + + describe "hi" + end + + it "blerg" do + end + EOM + + code_lines = code_line_array(source_string) + expansion = BlockExpand.new(code_lines: code_lines) + + block = CodeBlock.new(lines: code_lines[3]) + block = expansion.call(block) + + expect(block.to_s).to eq(<<~EOM) + + describe "hi" + end + + EOM + + block = expansion.call(block) + + expect(block.to_s).to eq(<<~EOM) + describe "what" do + end + + describe "hi" + end + + it "blerg" do + end + EOM + end + end +end diff --git a/spec/syntax_suggest/unit/capture/before_after_keyword_ends_spec.rb b/spec/syntax_suggest/unit/capture/before_after_keyword_ends_spec.rb new file mode 100644 index 0000000000..09f8d90d33 --- /dev/null +++ b/spec/syntax_suggest/unit/capture/before_after_keyword_ends_spec.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require_relative "../../spec_helper" + +module SyntaxSuggest + RSpec.describe Capture::BeforeAfterKeywordEnds do + it "before after keyword ends" do + source = <<~EOM + def nope + print 'not me' + end + + def lol + print 'lol' + end + + def hello # 8 + + def yolo + print 'haha' + end + + def nada + print 'nope' + end + EOM + + code_lines = CleanDocument.new(source: source).call.lines + block = CodeBlock.new(lines: code_lines[8]) + + expect(block.to_s).to include("def hello") + + lines = Capture::BeforeAfterKeywordEnds.new( + block: block, + code_lines: code_lines + ).call + lines.sort! + + expect(lines.join).to include(<<~EOM) + def lol + end + def yolo + end + EOM + end + end +end diff --git a/spec/syntax_suggest/unit/capture/falling_indent_lines_spec.rb b/spec/syntax_suggest/unit/capture/falling_indent_lines_spec.rb new file mode 100644 index 0000000000..ed2265539a --- /dev/null +++ b/spec/syntax_suggest/unit/capture/falling_indent_lines_spec.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require_relative "../../spec_helper" + +module SyntaxSuggest + RSpec.describe Capture::FallingIndentLines do + it "on_falling_indent" do + source = <<~EOM + class OH + def lol + print 'lol + end + + def hello + it "foo" do + end + + def yolo + print 'haha' + end + end + EOM + + code_lines = CleanDocument.new(source: source).call.lines + block = CodeBlock.new(lines: code_lines[6]) + + lines = [] + Capture::FallingIndentLines.new( + block: block, + code_lines: code_lines + ).call do |line| + lines << line + end + lines.sort! + + expect(lines.join).to eq(<<~EOM) + class OH + def hello + end + end + EOM + end + end +end diff --git a/spec/syntax_suggest/unit/capture_code_context_spec.rb b/spec/syntax_suggest/unit/capture_code_context_spec.rb new file mode 100644 index 0000000000..d9379d0ce7 --- /dev/null +++ b/spec/syntax_suggest/unit/capture_code_context_spec.rb @@ -0,0 +1,229 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe CaptureCodeContext do + it "capture_before_after_kws two" do + source = <<~EOM + class OH + + def hello + + def hai + end + end + EOM + + code_lines = CleanDocument.new(source: source).call.lines + block = CodeBlock.new(lines: code_lines[2]) + + display = CaptureCodeContext.new( + blocks: [block], + code_lines: code_lines + ) + display.capture_before_after_kws(block) + expect(display.sorted_lines.join).to eq(<<~EOM.indent(2)) + def hello + def hai + end + EOM + end + + it "capture_before_after_kws" do + source = <<~EOM + def sit + end + + def bark + + def eat + end + EOM + + code_lines = CleanDocument.new(source: source).call.lines + block = CodeBlock.new(lines: code_lines[3]) + + display = CaptureCodeContext.new( + blocks: [block], + code_lines: code_lines + ) + + lines = display.capture_before_after_kws(block).sort + expect(lines.join).to eq(<<~EOM) + def sit + end + def bark + def eat + end + EOM + end + + it "handles ambiguous end" do + source = <<~EOM + def call # 0 + print "lol" # 1 + end # one # 2 + end # two # 3 + EOM + + code_lines = CleanDocument.new(source: source).call.lines + code_lines[0..2].each(&:mark_invisible) + block = CodeBlock.new(lines: code_lines) + + display = CaptureCodeContext.new( + blocks: [block], + code_lines: code_lines + ) + lines = display.call + + lines = lines.sort.map(&:original) + + expect(lines.join).to eq(<<~EOM) + def call # 0 + end # one # 2 + end # two # 3 + EOM + end + + it "shows ends of captured block" do + lines = fixtures_dir.join("rexe.rb.txt").read.lines + lines.delete_at(148 - 1) + source = lines.join + + code_lines = CleanDocument.new(source: source).call.lines + + code_lines[0..75].each(&:mark_invisible) + code_lines[77..].each(&:mark_invisible) + expect(code_lines.join.strip).to eq("class Lookups") + + block = CodeBlock.new(lines: code_lines[76..149]) + + display = CaptureCodeContext.new( + blocks: [block], + code_lines: code_lines + ) + lines = display.call + + lines = lines.sort.map(&:original) + expect(lines.join).to include(<<~EOM.indent(2)) + class Lookups + def format_requires + end + EOM + end + + it "shows ends of captured block" do + source = <<~EOM + class Dog + def bark + puts "woof" + end + EOM + + code_lines = CleanDocument.new(source: source).call.lines + block = CodeBlock.new(lines: code_lines) + code_lines[1..].each(&:mark_invisible) + + expect(block.to_s.strip).to eq("class Dog") + + display = CaptureCodeContext.new( + blocks: [block], + code_lines: code_lines + ) + lines = display.call.sort.map(&:original) + expect(lines.join).to eq(<<~EOM) + class Dog + def bark + end + EOM + end + + it "captures surrounding context on falling indent" do + source = <<~EOM + class Blerg + end + + class OH + + def hello + it "foo" do + end + end + + class Zerg + end + EOM + code_lines = CleanDocument.new(source: source).call.lines + block = CodeBlock.new(lines: code_lines[6]) + + expect(block.to_s.strip).to eq('it "foo" do') + + display = CaptureCodeContext.new( + blocks: [block], + code_lines: code_lines + ) + lines = display.call.sort.map(&:original) + expect(lines.join).to eq(<<~EOM) + class OH + def hello + it "foo" do + end + end + EOM + end + + it "captures surrounding context on same indent" do + source = <<~EOM + class Blerg + end + class OH + + def nope + end + + def lol + end + + end # here + + def haha + end + + def nope + end + end + + class Zerg + end + EOM + + code_lines = CleanDocument.new(source: source).call.lines + block = CodeBlock.new(lines: code_lines[7..10]) + expect(block.to_s).to eq(<<~EOM.indent(2)) + def lol + end + + end # here + EOM + + code_context = CaptureCodeContext.new( + blocks: [block], + code_lines: code_lines + ) + + lines = code_context.call + out = DisplayCodeWithLineNumbers.new( + lines: lines + ).call + + expect(out).to eq(<<~EOM.indent(2)) + 3 class OH + 8 def lol + 9 end + 11 end # here + 18 end + EOM + end + end +end diff --git a/spec/syntax_suggest/unit/clean_document_spec.rb b/spec/syntax_suggest/unit/clean_document_spec.rb new file mode 100644 index 0000000000..5b5ca04cfd --- /dev/null +++ b/spec/syntax_suggest/unit/clean_document_spec.rb @@ -0,0 +1,260 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe CleanDocument do + it "heredocs" do + source = fixtures_dir.join("this_project_extra_def.rb.txt").read + code_lines = CleanDocument.new(source: source).call.lines + + expect(code_lines[18 - 1].to_s).to eq(<<-EOL) + @io.puts <<~EOM + + SyntaxSuggest: A syntax error was detected + + This code has an unmatched `end` this is caused by either + missing a syntax keyword (`def`, `do`, etc.) or inclusion + of an extra `end` line: + EOM + EOL + expect(code_lines[18].to_s).to eq("") + + expect(code_lines[27 - 1].to_s).to eq(<<-'EOL') + @io.puts(<<~EOM) if filename + file: #{filename} + EOM + EOL + expect(code_lines[27].to_s).to eq("") + + expect(code_lines[31 - 1].to_s).to eq(<<-'EOL') + @io.puts <<~EOM + #{code_with_filename} + EOM + EOL + expect(code_lines[31].to_s).to eq("") + end + + it "joins: multi line methods" do + source = <<~EOM + User + .where(name: 'schneems') + .first + EOM + + doc = CleanDocument.new(source: source).join_consecutive! + + expect(doc.lines[0].to_s).to eq(source) + expect(doc.lines[1].to_s).to eq("") + expect(doc.lines[2].to_s).to eq("") + expect(doc.lines[3]).to eq(nil) + + lines = doc.lines + expect( + DisplayCodeWithLineNumbers.new( + lines: lines + ).call + ).to eq(<<~EOM.indent(2)) + 1 User + 2 .where(name: 'schneems') + 3 .first + EOM + + expect( + DisplayCodeWithLineNumbers.new( + lines: lines, + highlight_lines: lines[0] + ).call + ).to eq(<<~EOM) + > 1 User + > 2 .where(name: 'schneems') + > 3 .first + EOM + end + + it "joins multi-line chained methods when separated by comments" do + source = <<~EOM + User. + # comment + where(name: 'schneems'). + # another comment + first + EOM + + doc = CleanDocument.new(source: source).join_consecutive! + code_lines = doc.lines + + expect(code_lines[0].to_s.count($/)).to eq(5) + code_lines[1..].each do |line| + expect(line.to_s.strip.length).to eq(0) + end + end + + it "helper method: take_while_including" do + source = <<~EOM + User + .where(name: 'schneems') + .first + EOM + + doc = CleanDocument.new(source: source) + + lines = doc.take_while_including { |line| !line.to_s.include?("where") } + expect(lines.count).to eq(2) + end + + it "comments: removes comments" do + source = <<~EOM + # lol + puts "what" + # yolo + EOM + + lines = CleanDocument.new(source: source).lines + expect(lines[0].to_s).to eq($/) + expect(lines[1].to_s).to eq('puts "what"' + $/) + expect(lines[2].to_s).to eq($/) + end + + it "trailing slash: does not join trailing do" do + # Some keywords and syntaxes trigger the "ignored line" + # lex output, we ignore them by filtering by BEG + # + # The `do` keyword is one of these: + # https://gist.github.com/schneems/6a7d7f988d3329fb3bd4b5be3e2efc0c + source = <<~EOM + foo do + puts "lol" + end + EOM + + doc = CleanDocument.new(source: source).join_consecutive! + + expect(doc.lines[0].to_s).to eq(source.lines[0]) + expect(doc.lines[1].to_s).to eq(source.lines[1]) + expect(doc.lines[2].to_s).to eq(source.lines[2]) + end + + it "trailing slash: formats output" do + source = <<~'EOM' + context "timezones workaround" do + it "should receive a time in UTC format and return the time with the"\ + "office's UTC offset subtracted from it" do + travel_to DateTime.new(2020, 10, 1, 10, 0, 0) do + office = build(:office) + end + end + end + EOM + + code_lines = CleanDocument.new(source: source).call.lines + expect( + DisplayCodeWithLineNumbers.new( + lines: code_lines.select(&:visible?) + ).call + ).to eq(<<~'EOM'.indent(2)) + 1 context "timezones workaround" do + 2 it "should receive a time in UTC format and return the time with the"\ + 3 "office's UTC offset subtracted from it" do + 4 travel_to DateTime.new(2020, 10, 1, 10, 0, 0) do + 5 office = build(:office) + 6 end + 7 end + 8 end + EOM + + expect( + DisplayCodeWithLineNumbers.new( + lines: code_lines.select(&:visible?), + highlight_lines: code_lines[1] + ).call + ).to eq(<<~'EOM') + 1 context "timezones workaround" do + > 2 it "should receive a time in UTC format and return the time with the"\ + > 3 "office's UTC offset subtracted from it" do + 4 travel_to DateTime.new(2020, 10, 1, 10, 0, 0) do + 5 office = build(:office) + 6 end + 7 end + 8 end + EOM + end + + it "trailing slash: basic detection" do + source = <<~'EOM' + it "trailing s" \ + "lash" do + EOM + + code_lines = CleanDocument.new(source: source).call.lines + + expect(code_lines[0]).to_not be_hidden + expect(code_lines[1]).to be_hidden + + expect( + code_lines.join + ).to eq(code_lines.map(&:original).join) + end + + it "trailing slash: joins multiple lines" do + source = <<~'EOM' + it "should " \ + "keep " \ + "going " do + end + EOM + + doc = CleanDocument.new(source: source).join_trailing_slash! + expect(doc.lines[0].to_s).to eq(source.lines[0..2].join) + expect(doc.lines[1].to_s).to eq("") + expect(doc.lines[2].to_s).to eq("") + expect(doc.lines[3].to_s).to eq(source.lines[3]) + + lines = doc.lines + expect( + DisplayCodeWithLineNumbers.new( + lines: lines + ).call + ).to eq(<<~'EOM'.indent(2)) + 1 it "should " \ + 2 "keep " \ + 3 "going " do + 4 end + EOM + + expect( + DisplayCodeWithLineNumbers.new( + lines: lines, + highlight_lines: lines[0] + ).call + ).to eq(<<~'EOM') + > 1 it "should " \ + > 2 "keep " \ + > 3 "going " do + 4 end + EOM + end + + it "trailing slash: no false positives" do + source = <<~'EOM' + def formatters + @formatters ||= { + amazing_print: ->(obj) { obj.ai + "\n" }, + inspect: ->(obj) { obj.inspect + "\n" }, + json: ->(obj) { obj.to_json }, + marshal: ->(obj) { Marshal.dump(obj) }, + none: ->(_obj) { nil }, + pretty_json: ->(obj) { JSON.pretty_generate(obj) }, + pretty_print: ->(obj) { obj.pretty_inspect }, + puts: ->(obj) { require 'stringio'; sio = StringIO.new; sio.puts(obj); sio.string }, + to_s: ->(obj) { obj.to_s + "\n" }, + yaml: ->(obj) { obj.to_yaml }, + } + end + EOM + + code_lines = CleanDocument.new(source: source).call.lines + expect(code_lines.join).to eq(code_lines.join) + end + end +end diff --git a/spec/syntax_suggest/unit/cli_spec.rb b/spec/syntax_suggest/unit/cli_spec.rb new file mode 100644 index 0000000000..23412f0193 --- /dev/null +++ b/spec/syntax_suggest/unit/cli_spec.rb @@ -0,0 +1,224 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + class FakeExit + def initialize + @called = false + @value = nil + end + + def exit(value = nil) + @called = true + @value = value + end + + def called? + @called + end + + attr_reader :value + end + + RSpec.describe Cli do + it "parses valid code" do + Dir.mktmpdir do |dir| + dir = Pathname(dir) + file = dir.join("script.rb") + file.write("puts 'lol'") + + io = StringIO.new + exit_obj = FakeExit.new + Cli.new( + io: io, + argv: [file.to_s], + exit_obj: exit_obj + ).call + + expect(exit_obj.called?).to be_truthy + expect(exit_obj.value).to eq(0) + expect(io.string.strip).to eq("Syntax OK") + end + end + + it "parses invalid code" do + file = fixtures_dir.join("this_project_extra_def.rb.txt") + + io = StringIO.new + exit_obj = FakeExit.new + Cli.new( + io: io, + argv: [file.to_s], + exit_obj: exit_obj + ).call + + out = io.string + debug_display(out) + + expect(exit_obj.called?).to be_truthy + expect(exit_obj.value).to eq(1) + expect(out.strip).to include("> 36 def filename") + end + + it "parses valid code with flags" do + Dir.mktmpdir do |dir| + dir = Pathname(dir) + file = dir.join("script.rb") + file.write("puts 'lol'") + + io = StringIO.new + exit_obj = FakeExit.new + cli = Cli.new( + io: io, + argv: ["--terminal", file.to_s], + exit_obj: exit_obj + ) + cli.call + + expect(exit_obj.called?).to be_truthy + expect(exit_obj.value).to eq(0) + expect(cli.options[:terminal]).to be_truthy + expect(io.string.strip).to eq("Syntax OK") + end + end + + it "errors when no file given" do + io = StringIO.new + exit_obj = FakeExit.new + cli = Cli.new( + io: io, + argv: ["--terminal"], + exit_obj: exit_obj + ) + cli.call + + expect(exit_obj.called?).to be_truthy + expect(exit_obj.value).to eq(1) + expect(io.string.strip).to eq("No file given") + end + + it "errors when file does not exist" do + io = StringIO.new + exit_obj = FakeExit.new + cli = Cli.new( + io: io, + argv: ["lol-i-d-o-not-ex-ist-yololo.txtblerglol"], + exit_obj: exit_obj + ) + cli.call + + expect(exit_obj.called?).to be_truthy + expect(exit_obj.value).to eq(1) + expect(io.string.strip).to include("file not found:") + end + + # We cannot execute the parser here + # because it calls `exit` and it will exit + # our tests, however we can assert that the + # parser has the right value for version + it "-v version" do + io = StringIO.new + exit_obj = FakeExit.new + parser = Cli.new( + io: io, + argv: ["-v"], + exit_obj: exit_obj + ).parser + + expect(parser.version).to include(SyntaxSuggest::VERSION.to_s) + end + + it "SYNTAX_SUGGEST_RECORD_DIR" do + io = StringIO.new + exit_obj = FakeExit.new + cli = Cli.new( + io: io, + argv: [], + env: {"SYNTAX_SUGGEST_RECORD_DIR" => "hahaha"}, + exit_obj: exit_obj + ).parse + + expect(exit_obj.called?).to be_falsey + expect(cli.options[:record_dir]).to eq("hahaha") + end + + it "--record-dir=<dir>" do + io = StringIO.new + exit_obj = FakeExit.new + cli = Cli.new( + io: io, + argv: ["--record=lol"], + exit_obj: exit_obj + ).parse + + expect(exit_obj.called?).to be_falsey + expect(cli.options[:record_dir]).to eq("lol") + end + + it "terminal default to respecting TTY" do + io = StringIO.new + exit_obj = FakeExit.new + cli = Cli.new( + io: io, + argv: [], + exit_obj: exit_obj + ).parse + + expect(exit_obj.called?).to be_falsey + expect(cli.options[:terminal]).to eq(SyntaxSuggest::DEFAULT_VALUE) + end + + it "--terminal" do + io = StringIO.new + exit_obj = FakeExit.new + cli = Cli.new( + io: io, + argv: ["--terminal"], + exit_obj: exit_obj + ).parse + + expect(exit_obj.called?).to be_falsey + expect(cli.options[:terminal]).to be_truthy + end + + it "--no-terminal" do + io = StringIO.new + exit_obj = FakeExit.new + cli = Cli.new( + io: io, + argv: ["--no-terminal"], + exit_obj: exit_obj + ).parse + + expect(exit_obj.called?).to be_falsey + expect(cli.options[:terminal]).to be_falsey + end + + it "--help outputs help" do + io = StringIO.new + exit_obj = FakeExit.new + Cli.new( + io: io, + argv: ["--help"], + exit_obj: exit_obj + ).call + + expect(exit_obj.called?).to be_truthy + expect(io.string).to include("Usage: syntax_suggest <file> [options]") + end + + it "<empty args> outputs help" do + io = StringIO.new + exit_obj = FakeExit.new + Cli.new( + io: io, + argv: [], + exit_obj: exit_obj + ).call + + expect(exit_obj.called?).to be_truthy + expect(io.string).to include("Usage: syntax_suggest <file> [options]") + end + end +end diff --git a/spec/syntax_suggest/unit/code_block_spec.rb b/spec/syntax_suggest/unit/code_block_spec.rb new file mode 100644 index 0000000000..3ab2751b27 --- /dev/null +++ b/spec/syntax_suggest/unit/code_block_spec.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe CodeBlock do + it "can detect if it's valid or not" do + code_lines = code_line_array(<<~EOM) + def foo + puts 'lol' + end + EOM + + block = CodeBlock.new(lines: code_lines[1]) + expect(block.valid?).to be_truthy + end + + it "can be sorted in indentation order" do + code_lines = code_line_array(<<~EOM) + def foo + puts 'lol' + end + EOM + + block_0 = CodeBlock.new(lines: code_lines[0]) + block_1 = CodeBlock.new(lines: code_lines[1]) + block_2 = CodeBlock.new(lines: code_lines[2]) + + expect(block_0 <=> block_0.dup).to eq(0) + expect(block_1 <=> block_0).to eq(1) + expect(block_1 <=> block_2).to eq(-1) + + array = [block_2, block_1, block_0].sort + expect(array.last).to eq(block_2) + + block = CodeBlock.new(lines: CodeLine.new(line: " " * 8 + "foo", index: 4, lex: [])) + array.prepend(block) + expect(array.max).to eq(block) + end + + it "knows it's current indentation level" do + code_lines = code_line_array(<<~EOM) + def foo + puts 'lol' + end + EOM + + block = CodeBlock.new(lines: code_lines[1]) + expect(block.current_indent).to eq(2) + + block = CodeBlock.new(lines: code_lines[0]) + expect(block.current_indent).to eq(0) + end + + it "knows it's current indentation level when mismatched indents" do + code_lines = code_line_array(<<~EOM) + def foo + puts 'lol' + end + EOM + + block = CodeBlock.new(lines: [code_lines[1], code_lines[2]]) + expect(block.current_indent).to eq(1) + end + + it "before lines and after lines" do + code_lines = code_line_array(<<~EOM) + def foo + bar; end + end + EOM + + block = CodeBlock.new(lines: code_lines[1]) + expect(block.valid?).to be_falsey + end + end +end diff --git a/spec/syntax_suggest/unit/code_frontier_spec.rb b/spec/syntax_suggest/unit/code_frontier_spec.rb new file mode 100644 index 0000000000..c9aba7c8d8 --- /dev/null +++ b/spec/syntax_suggest/unit/code_frontier_spec.rb @@ -0,0 +1,135 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe CodeFrontier do + it "detect_bad_blocks" do + code_lines = code_line_array(<<~EOM) + describe "lol" do + end + end + + it "lol" do + end + end + EOM + + frontier = CodeFrontier.new(code_lines: code_lines) + blocks = [] + blocks << CodeBlock.new(lines: code_lines[1]) + blocks << CodeBlock.new(lines: code_lines[5]) + blocks.each do |b| + frontier << b + end + + expect(frontier.detect_invalid_blocks.sort).to eq(blocks.sort) + end + + it "self.combination" do + expect( + CodeFrontier.combination([:a, :b, :c, :d]) + ).to eq( + [ + [:a], [:b], [:c], [:d], + [:a, :b], + [:a, :c], + [:a, :d], + [:b, :c], + [:b, :d], + [:c, :d], + [:a, :b, :c], + [:a, :b, :d], + [:a, :c, :d], + [:b, :c, :d], + [:a, :b, :c, :d] + ] + ) + end + + it "doesn't duplicate blocks" do + code_lines = code_line_array(<<~EOM) + def foo + puts "lol" + puts "lol" + puts "lol" + end + EOM + + frontier = CodeFrontier.new(code_lines: code_lines) + frontier << CodeBlock.new(lines: [code_lines[2]]) + expect(frontier.count).to eq(1) + + frontier << CodeBlock.new(lines: [code_lines[1], code_lines[2], code_lines[3]]) + # expect(frontier.count).to eq(1) + expect(frontier.pop.to_s).to eq(<<~EOM.indent(2)) + puts "lol" + puts "lol" + puts "lol" + EOM + + expect(frontier.pop).to be_nil + + code_lines = code_line_array(<<~EOM) + def foo + puts "lol" + puts "lol" + puts "lol" + end + EOM + + frontier = CodeFrontier.new(code_lines: code_lines) + frontier << CodeBlock.new(lines: [code_lines[2]]) + expect(frontier.count).to eq(1) + + frontier << CodeBlock.new(lines: [code_lines[3]]) + expect(frontier.count).to eq(2) + expect(frontier.pop.to_s).to eq(<<~EOM.indent(2)) + puts "lol" + EOM + end + + it "detects if multiple syntax errors are found" do + code_lines = code_line_array(<<~EOM) + def foo + end + end + EOM + + frontier = CodeFrontier.new(code_lines: code_lines) + + frontier << CodeBlock.new(lines: code_lines[1]) + block = frontier.pop + expect(block.to_s).to eq(<<~EOM.indent(2)) + end + EOM + frontier << block + + expect(frontier.holds_all_syntax_errors?).to be_truthy + end + + it "detects if it has not captured all syntax errors" do + code_lines = code_line_array(<<~EOM) + def foo + puts "lol" + end + + describe "lol" + end + + it "lol" + end + EOM + + frontier = CodeFrontier.new(code_lines: code_lines) + frontier << CodeBlock.new(lines: [code_lines[1]]) + block = frontier.pop + expect(block.to_s).to eq(<<~EOM.indent(2)) + puts "lol" + EOM + frontier << block + + expect(frontier.holds_all_syntax_errors?).to be_falsey + end + end +end diff --git a/spec/syntax_suggest/unit/code_line_spec.rb b/spec/syntax_suggest/unit/code_line_spec.rb new file mode 100644 index 0000000000..5b62cc2757 --- /dev/null +++ b/spec/syntax_suggest/unit/code_line_spec.rb @@ -0,0 +1,165 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe CodeLine do + it "bug in keyword detection" do + lines = CodeLine.from_source(<<~EOM) + def to_json(*opts) + { + type: :module, + }.to_json(*opts) + end + EOM + expect(lines.count(&:is_kw?)).to eq(1) + expect(lines.count(&:is_end?)).to eq(1) + end + + it "supports endless method definitions" do + skip("Unsupported ruby version") unless Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3") + + line = CodeLine.from_source(<<~EOM).first + def square(x) = x * x + EOM + + expect(line.is_kw?).to be_falsey + expect(line.is_end?).to be_falsey + end + + it "retains original line value, after being marked invisible" do + line = CodeLine.from_source(<<~EOM).first + puts "lol" + EOM + expect(line.line).to match('puts "lol"') + line.mark_invisible + expect(line.line).to eq("") + expect(line.original).to match('puts "lol"') + end + + it "knows which lines can be joined" do + code_lines = CodeLine.from_source(<<~EOM) + user = User. + where(name: 'schneems'). + first + puts user.name + EOM + + # Indicates line 1 can join 2, 2 can join 3, but 3 won't join it's next line + expect(code_lines.map(&:ignore_newline_not_beg?)).to eq([true, true, false, false]) + end + + it "trailing if" do + code_lines = CodeLine.from_source(<<~EOM) + puts "lol" if foo + if foo + end + EOM + + expect(code_lines.map(&:is_kw?)).to eq([false, true, false]) + end + + it "trailing unless" do + code_lines = CodeLine.from_source(<<~EOM) + puts "lol" unless foo + unless foo + end + EOM + + expect(code_lines.map(&:is_kw?)).to eq([false, true, false]) + end + + it "trailing slash" do + code_lines = CodeLine.from_source(<<~'EOM') + it "trailing s" \ + "lash" do + EOM + + expect(code_lines.map(&:trailing_slash?)).to eq([true, false]) + + code_lines = CodeLine.from_source(<<~'EOM') + amazing_print: ->(obj) { obj.ai + "\n" }, + EOM + expect(code_lines.map(&:trailing_slash?)).to eq([false]) + end + + it "knows it's got an end" do + line = CodeLine.from_source(" end").first + + expect(line.is_end?).to be_truthy + expect(line.is_kw?).to be_falsey + end + + it "knows it's got a keyword" do + line = CodeLine.from_source(" if").first + + expect(line.is_end?).to be_falsey + expect(line.is_kw?).to be_truthy + end + + it "ignores marked lines" do + code_lines = CodeLine.from_source(<<~EOM) + def foo + Array(value) |x| + end + end + EOM + + expect(SyntaxSuggest.valid?(code_lines)).to be_falsey + expect(code_lines.join).to eq(<<~EOM) + def foo + Array(value) |x| + end + end + EOM + + expect(code_lines[0].visible?).to be_truthy + expect(code_lines[3].visible?).to be_truthy + + code_lines[0].mark_invisible + code_lines[3].mark_invisible + + expect(code_lines[0].visible?).to be_falsey + expect(code_lines[3].visible?).to be_falsey + + expect(code_lines.join).to eq(<<~EOM.indent(2)) + Array(value) |x| + end + EOM + expect(SyntaxSuggest.valid?(code_lines)).to be_falsey + end + + it "knows empty lines" do + code_lines = CodeLine.from_source(<<~EOM) + # Not empty + + # Not empty + EOM + + expect(code_lines.map(&:empty?)).to eq([false, true, false]) + expect(code_lines.map(&:not_empty?)).to eq([true, false, true]) + expect(code_lines.map { |l| SyntaxSuggest.valid?(l) }).to eq([true, true, true]) + end + + it "counts indentations" do + code_lines = CodeLine.from_source(<<~EOM) + def foo + Array(value) |x| + puts 'lol' + end + end + EOM + + expect(code_lines.map(&:indent)).to eq([0, 2, 4, 2, 0]) + end + + it "doesn't count empty lines as having an indentation" do + code_lines = CodeLine.from_source(<<~EOM) + + + EOM + + expect(code_lines.map(&:indent)).to eq([0, 0]) + end + end +end diff --git a/spec/syntax_suggest/unit/code_search_spec.rb b/spec/syntax_suggest/unit/code_search_spec.rb new file mode 100644 index 0000000000..502de14d7f --- /dev/null +++ b/spec/syntax_suggest/unit/code_search_spec.rb @@ -0,0 +1,505 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe CodeSearch do + it "rexe regression" do + lines = fixtures_dir.join("rexe.rb.txt").read.lines + lines.delete_at(148 - 1) + source = lines.join + + search = CodeSearch.new(source) + search.call + + expect(search.invalid_blocks.join.strip).to eq(<<~EOM.strip) + class Lookups + EOM + end + + it "squished do regression" do + source = <<~EOM + def call + trydo + + @options = CommandLineParser.new.parse + + options.requires.each { |r| require!(r) } + load_global_config_if_exists + options.loads.each { |file| load(file) } + + @user_source_code = ARGV.join(' ') + @user_source_code = 'self' if @user_source_code == '' + + @callable = create_callable + + init_rexe_context + init_parser_and_formatters + + # This is where the user's source code will be executed; the action will in turn call `execute`. + lookup_action(options.input_mode).call unless options.noop + + output_log_entry + end # one + end # two + EOM + + search = CodeSearch.new(source) + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM.indent(2)) + trydo + end # one + EOM + end + + it "regression test ambiguous end" do + source = <<~EOM + def call # 0 + print "lol" # 1 + end # one # 2 + end # two # 3 + EOM + + search = CodeSearch.new(source) + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM) + end # two # 3 + EOM + end + + it "regression dog test" do + source = <<~EOM + class Dog + def bark + puts "woof" + end + EOM + search = CodeSearch.new(source) + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM) + class Dog + EOM + expect(search.invalid_blocks.first.lines.length).to eq(4) + end + + it "handles mismatched |" do + source = <<~EOM + class Blerg + Foo.call do |a + end # one + + puts lol + class Foo + end # two + end # three + EOM + search = CodeSearch.new(source) + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM.indent(2)) + Foo.call do |a + end # one + EOM + end + + it "handles mismatched }" do + source = <<~EOM + class Blerg + Foo.call do { + + puts lol + class Foo + end # two + end # three + EOM + search = CodeSearch.new(source) + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM.indent(2)) + Foo.call do { + EOM + end + + it "handles no spaces between blocks and trailing slash" do + source = <<~'EOM' + require "rails_helper" + RSpec.describe Foo, type: :model do + describe "#bar" do + context "context" do + it "foos the bar with a foo and then bazes the foo with a bar to"\ + "fooify the barred bar" do + travel_to DateTime.new(2020, 10, 1, 10, 0, 0) do + foo = build(:foo) + end + end + end + end + describe "#baz?" do + context "baz has barred the foo" do + it "returns true" do # <== HERE + end + end + end + EOM + + search = CodeSearch.new(source) + search.call + + expect(search.invalid_blocks.join.strip).to eq('it "returns true" do # <== HERE') + end + + it "handles no spaces between blocks" do + source = <<~EOM + context "foo bar" do + it "bars the foo" do + travel_to DateTime.new(2020, 10, 1, 10, 0, 0) do + end + end + end + context "test" do + it "should" do + end + EOM + search = CodeSearch.new(source) + search.call + + expect(search.invalid_blocks.join.strip).to eq('it "should" do') + end + + it "records debugging steps to a directory" do + Dir.mktmpdir do |dir| + dir = Pathname(dir) + search = CodeSearch.new(<<~EOM, record_dir: dir) + class OH + def hello + def hai + end + end + EOM + search.call + + expect(search.record_dir.entries.map(&:to_s)).to include("1-add-1-(3__4).txt") + expect(search.record_dir.join("1-add-1-(3__4).txt").read).to include(<<~EOM) + 1 class OH + 2 def hello + > 3 def hai + > 4 end + 5 end + EOM + end + end + + it "def with missing end" do + search = CodeSearch.new(<<~EOM) + class OH + def hello + + def hai + puts "lol" + end + end + EOM + search.call + + expect(search.invalid_blocks.join.strip).to eq("def hello") + + search = CodeSearch.new(<<~EOM) + class OH + def hello + + def hai + end + end + EOM + search.call + + expect(search.invalid_blocks.join.strip).to eq("def hello") + + search = CodeSearch.new(<<~EOM) + class OH + def hello + def hai + end + end + EOM + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM.indent(2)) + def hello + EOM + end + + describe "real world cases" do + it "finds hanging def in this project" do + source_string = fixtures_dir.join("this_project_extra_def.rb.txt").read + search = CodeSearch.new(source_string) + search.call + + document = DisplayCodeWithLineNumbers.new( + lines: search.code_lines.select(&:visible?), + terminal: false, + highlight_lines: search.invalid_blocks.flat_map(&:lines) + ).call + + expect(document).to include(<<~EOM) + > 36 def filename + EOM + end + + it "Format Code blocks real world example" do + search = CodeSearch.new(<<~EOM) + require 'rails_helper' + + RSpec.describe AclassNameHere, type: :worker do + describe "thing" do + context "when" do + let(:thing) { stuff } + let(:another_thing) { moarstuff } + subject { foo.new.perform(foo.id, true) } + + it "stuff" do + subject + + expect(foo.foo.foo).to eq(true) + end + end + end # line 16 accidental end, but valid block + + context "stuff" do + let(:thing) { create(:foo, foo: stuff) } + let(:another_thing) { create(:stuff) } + + subject { described_class.new.perform(foo.id, false) } + + it "more stuff" do + subject + + expect(foo.foo.foo).to eq(false) + end + end + end # mismatched due to 16 + end + EOM + search.call + + document = DisplayCodeWithLineNumbers.new( + lines: search.code_lines.select(&:visible?), + terminal: false, + highlight_lines: search.invalid_blocks.flat_map(&:lines) + ).call + + expect(document).to include(<<~EOM) + 1 require 'rails_helper' + 2 + 3 RSpec.describe AclassNameHere, type: :worker do + > 4 describe "thing" do + > 16 end # line 16 accidental end, but valid block + > 30 end # mismatched due to 16 + 31 end + EOM + end + end + + # For code that's not perfectly formatted, we ideally want to do our best + # These examples represent the results that exist today, but I would like to improve upon them + describe "needs improvement" do + describe "mis-matched-indentation" do + it "extra space before end" do + search = CodeSearch.new(<<~EOM) + Foo.call + def foo + puts "lol" + puts "lol" + end # one + end # two + EOM + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM) + Foo.call + end # two + EOM + end + + it "stacked ends 2" do + search = CodeSearch.new(<<~EOM) + def cat + blerg + end + + Foo.call do + end # one + end # two + + def dog + end + EOM + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM) + Foo.call do + end # one + end # two + + EOM + end + + it "stacked ends " do + search = CodeSearch.new(<<~EOM) + Foo.call + def foo + puts "lol" + puts "lol" + end + end + EOM + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM) + Foo.call + end + EOM + end + + it "missing space before end" do + search = CodeSearch.new(<<~EOM) + Foo.call + + def foo + puts "lol" + puts "lol" + end + end + EOM + search.call + + # expand-1 and expand-2 seem to be broken? + expect(search.invalid_blocks.join).to eq(<<~EOM) + Foo.call + end + EOM + end + end + end + + it "returns syntax error in outer block without inner block" do + search = CodeSearch.new(<<~EOM) + Foo.call + def foo + puts "lol" + puts "lol" + end # one + end # two + EOM + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM) + Foo.call + end # two + EOM + end + + it "doesn't just return an empty `end`" do + search = CodeSearch.new(<<~EOM) + Foo.call + end + EOM + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM) + Foo.call + end + EOM + end + + it "finds multiple syntax errors" do + search = CodeSearch.new(<<~EOM) + describe "hi" do + Foo.call + end + end + + it "blerg" do + Bar.call + end + end + EOM + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM.indent(2)) + Foo.call + end + Bar.call + end + EOM + end + + it "finds a typo def" do + search = CodeSearch.new(<<~EOM) + defzfoo + puts "lol" + end + EOM + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM) + defzfoo + end + EOM + end + + it "finds a mis-matched def" do + search = CodeSearch.new(<<~EOM) + def foo + def blerg + end + EOM + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM.indent(2)) + def blerg + EOM + end + + it "finds a naked end" do + search = CodeSearch.new(<<~EOM) + def foo + end # one + end # two + EOM + search.call + + expect(search.invalid_blocks.join).to eq(<<~EOM.indent(2)) + end # one + EOM + end + + it "returns when no invalid blocks are found" do + search = CodeSearch.new(<<~EOM) + def foo + puts 'lol' + end + EOM + search.call + + expect(search.invalid_blocks).to eq([]) + end + + it "expands frontier by eliminating valid lines" do + search = CodeSearch.new(<<~EOM) + def foo + puts 'lol' + end + EOM + search.create_blocks_from_untracked_lines + + expect(search.code_lines.join).to eq(<<~EOM) + def foo + end + EOM + end + end +end diff --git a/spec/syntax_suggest/unit/core_ext_spec.rb b/spec/syntax_suggest/unit/core_ext_spec.rb new file mode 100644 index 0000000000..499c38a240 --- /dev/null +++ b/spec/syntax_suggest/unit/core_ext_spec.rb @@ -0,0 +1,34 @@ +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe "Core extension" do + it "SyntaxError monkepatch ensures there is a newline to the end of the file" do + skip if Gem::Version.new(RUBY_VERSION) < Gem::Version.new("3.2") + + Dir.mktmpdir do |dir| + tmpdir = Pathname(dir) + file = tmpdir.join("file.rb") + file.write(<<~EOM.strip) + print 'no newline + EOM + + core_ext_file = lib_dir.join("syntax_suggest").join("core_ext") + require_relative core_ext_file + + original_message = "blerg" + error = SyntaxError.new(original_message) + def error.set_tmp_path_for_testing=(path) + @tmp_path_for_testing = path + end + error.set_tmp_path_for_testing = file + def error.path + @tmp_path_for_testing + end + + detailed = error.detailed_message(highlight: false, syntax_suggest: true) + expect(detailed).to include("'no newline\n#{original_message}") + expect(detailed).to_not include("print 'no newline#{original_message}") + end + end + end +end diff --git a/spec/syntax_suggest/unit/display_invalid_blocks_spec.rb b/spec/syntax_suggest/unit/display_invalid_blocks_spec.rb new file mode 100644 index 0000000000..b11d7d242e --- /dev/null +++ b/spec/syntax_suggest/unit/display_invalid_blocks_spec.rb @@ -0,0 +1,174 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe DisplayInvalidBlocks do + it "works with valid code" do + syntax_string = <<~EOM + class OH + def hello + end + def hai + end + end + EOM + + search = CodeSearch.new(syntax_string) + search.call + + io = StringIO.new + display = DisplayInvalidBlocks.new( + io: io, + blocks: search.invalid_blocks, + terminal: false, + code_lines: search.code_lines + ) + display.call + expect(io.string).to include("") + end + + it "selectively prints to terminal if input is a tty by default" do + source = <<~EOM + class OH + def hello + def hai + end + end + EOM + + code_lines = CleanDocument.new(source: source).call.lines + + io = StringIO.new + def io.isatty + true + end + + block = CodeBlock.new(lines: code_lines[1]) + display = DisplayInvalidBlocks.new( + io: io, + blocks: block, + code_lines: code_lines + ) + display.call + expect(io.string).to include([ + "> 2 ", + DisplayCodeWithLineNumbers::TERMINAL_HIGHLIGHT, + " def hello" + ].join) + + io = StringIO.new + def io.isatty + false + end + + block = CodeBlock.new(lines: code_lines[1]) + display = DisplayInvalidBlocks.new( + io: io, + blocks: block, + code_lines: code_lines + ) + display.call + expect(io.string).to include("> 2 def hello") + end + + it "outputs to io when using `call`" do + source = <<~EOM + class OH + def hello + def hai + end + end + EOM + + code_lines = CleanDocument.new(source: source).call.lines + + io = StringIO.new + block = CodeBlock.new(lines: code_lines[1]) + display = DisplayInvalidBlocks.new( + io: io, + blocks: block, + terminal: false, + code_lines: code_lines + ) + display.call + expect(io.string).to include("> 2 def hello") + end + + it " wraps code with github style codeblocks" do + source = <<~EOM + class OH + def hello + + def hai + end + end + EOM + + code_lines = CleanDocument.new(source: source).call.lines + block = CodeBlock.new(lines: code_lines[1]) + io = StringIO.new + DisplayInvalidBlocks.new( + io: io, + blocks: block, + terminal: false, + code_lines: code_lines + ).call + expect(io.string).to include(<<~EOM) + 1 class OH + > 2 def hello + 4 def hai + 5 end + 6 end + EOM + end + + it "shows terminal characters" do + code_lines = code_line_array(<<~EOM) + class OH + def hello + def hai + end + end + EOM + + io = StringIO.new + block = CodeBlock.new(lines: code_lines[1]) + DisplayInvalidBlocks.new( + io: io, + blocks: block, + terminal: false, + code_lines: code_lines + ).call + + expect(io.string).to include([ + " 1 class OH", + "> 2 def hello", + " 3 def hai", + " 4 end", + " 5 end", + "" + ].join($/)) + + block = CodeBlock.new(lines: code_lines[1]) + io = StringIO.new + DisplayInvalidBlocks.new( + io: io, + blocks: block, + terminal: true, + code_lines: code_lines + ).call + + expect(io.string).to include( + [ + " 1 class OH", + ["> 2 ", DisplayCodeWithLineNumbers::TERMINAL_HIGHLIGHT, " def hello"].join, + " 3 def hai", + " 4 end", + " 5 end", + "" + ].join($/ + DisplayCodeWithLineNumbers::TERMINAL_END) + ) + end + end +end diff --git a/spec/syntax_suggest/unit/explain_syntax_spec.rb b/spec/syntax_suggest/unit/explain_syntax_spec.rb new file mode 100644 index 0000000000..c62a42b925 --- /dev/null +++ b/spec/syntax_suggest/unit/explain_syntax_spec.rb @@ -0,0 +1,255 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe "ExplainSyntax" do + it "handles shorthand syntaxes with non-bracket characters" do + source = <<~EOM + %Q* lol + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq([]) + expect(explain.errors.join.strip).to_not be_empty + end + + it "handles %w[]" do + source = <<~EOM + node.is_a?(Op) && %w[| ||].include?(node.value) && + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq([]) + end + + it "doesn't falsely identify strings or symbols as critical chars" do + source = <<~EOM + a = ['(', '{', '[', '|'] + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq([]) + + source = <<~EOM + a = [:'(', :'{', :'[', :'|'] + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq([]) + end + + it "finds missing |" do + source = <<~EOM + Foo.call do | + end + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq(["|"]) + expect(explain.errors).to eq([explain.why("|")]) + end + + it "finds missing {" do + source = <<~EOM + class Cat + lol = { + end + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq(["}"]) + expect(explain.errors).to eq([explain.why("}")]) + end + + it "finds missing }" do + source = <<~EOM + def foo + lol = "foo" => :bar } + end + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq(["{"]) + expect(explain.errors).to eq([explain.why("{")]) + end + + it "finds missing [" do + source = <<~EOM + class Cat + lol = [ + end + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq(["]"]) + expect(explain.errors).to eq([explain.why("]")]) + end + + it "finds missing ]" do + source = <<~EOM + def foo + lol = ] + end + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq(["["]) + expect(explain.errors).to eq([explain.why("[")]) + end + + it "finds missing (" do + source = "def initialize; ); end" + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq(["("]) + expect(explain.errors).to eq([explain.why("(")]) + end + + it "finds missing )" do + source = "def initialize; (; end" + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq([")"]) + expect(explain.errors).to eq([explain.why(")")]) + end + + it "finds missing keyword" do + source = <<~EOM + class Cat + end + end + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq(["keyword"]) + expect(explain.errors).to eq([explain.why("keyword")]) + end + + it "finds missing end" do + source = <<~EOM + class Cat + def meow + end + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq(["end"]) + expect(explain.errors).to eq([explain.why("end")]) + end + + it "falls back to ripper on unknown errors" do + source = <<~EOM + class Cat + def meow + 1 * + end + end + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq([]) + expect(explain.errors).to eq(GetParseErrors.errors(source)) + end + + it "handles an unexpected rescue" do + source = <<~EOM + def foo + if bar + "baz" + else + "foo" + rescue FooBar + nil + end + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq(["end"]) + end + + # String embeds are `"#{foo} <-- here` + # + # We need to count a `#{` as a `{` + # otherwise it will report that we are + # missing a curly when we are using valid + # string embed syntax + it "is not confused by valid string embed" do + source = <<~'EOM' + foo = "#{hello}" + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + expect(explain.missing).to eq([]) + end + + # Missing string embed beginnings are not a + # syntax error. i.e. `"foo}"` or `"{foo}` or "#foo}" + # would just be strings with extra characters. + # + # However missing the end curly will trigger + # an error: i.e. `"#{foo` + # + # String embed beginning is a `#{` rather than + # a `{`, make sure we handle that case and + # report the correct missing `}` diagnosis + it "finds missing string embed end" do + source = <<~'EOM' + "#{foo + EOM + + explain = ExplainSyntax.new( + code_lines: CodeLine.from_source(source) + ).call + + expect(explain.missing).to eq(["}"]) + end + end +end diff --git a/spec/syntax_suggest/unit/lex_all_spec.rb b/spec/syntax_suggest/unit/lex_all_spec.rb new file mode 100644 index 0000000000..9621c9ecec --- /dev/null +++ b/spec/syntax_suggest/unit/lex_all_spec.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe "EndBlockParse" do + it "finds blocks based on `end` keyword" do + source = <<~EOM + describe "cat" # 1 + Cat.call do # 2 + end # 3 + end # 4 + # 5 + it "dog" do # 6 + Dog.call do # 7 + end # 8 + end # 9 + EOM + + lex = LexAll.new(source: source) + expect(lex.map(&:token).to_s).to include("dog") + expect(lex.first.line).to eq(1) + expect(lex.last.line).to eq(9) + end + end +end diff --git a/spec/syntax_suggest/unit/pathname_from_message_spec.rb b/spec/syntax_suggest/unit/pathname_from_message_spec.rb new file mode 100644 index 0000000000..de58acebaa --- /dev/null +++ b/spec/syntax_suggest/unit/pathname_from_message_spec.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe "PathnameFromMessage" do + it "handles filenames with colons in them" do + Dir.mktmpdir do |dir| + dir = Pathname(dir) + + file = dir.join("scr:atch.rb").tap { |p| FileUtils.touch(p) } + + message = "#{file}:2:in `require_relative': /private/tmp/bad.rb:1: syntax error, unexpected `end' (SyntaxError)" + file = PathnameFromMessage.new(message).call.name + + expect(file).to be_truthy + end + end + + it "checks if the file exists" do + Dir.mktmpdir do |dir| + dir = Pathname(dir) + + file = dir.join("scratch.rb") + # No touch, file does not exist + expect(file.exist?).to be_falsey + + message = "#{file}:2:in `require_relative': /private/tmp/bad.rb:1: syntax error, unexpected `end' (SyntaxError)" + io = StringIO.new + file = PathnameFromMessage.new(message, io: io).call.name + + expect(io.string).to include(file.to_s) + expect(file).to be_falsey + end + end + + it "does not output error message on syntax error inside of an (eval)" do + message = "(eval):1: invalid multibyte char (UTF-8) (SyntaxError)\n" + io = StringIO.new + file = PathnameFromMessage.new(message, io: io).call.name + + expect(io.string).to eq("") + expect(file).to be_falsey + end + + it "does not output error message on syntax error inside of an (eval at __FILE__:__LINE__)" do + message = "(eval at #{__FILE__}:#{__LINE__}):1: invalid multibyte char (UTF-8) (SyntaxError)\n" + io = StringIO.new + file = PathnameFromMessage.new(message, io: io).call.name + + expect(io.string).to eq("") + expect(file).to be_falsey + end + + it "does not output error message on syntax error inside of streamed code" do + # An example of streamed code is: $ echo "def foo" | ruby + message = "-:1: syntax error, unexpected end-of-input\n" + io = StringIO.new + file = PathnameFromMessage.new(message, io: io).call.name + + expect(io.string).to eq("") + expect(file).to be_falsey + end + end +end diff --git a/spec/syntax_suggest/unit/priority_queue_spec.rb b/spec/syntax_suggest/unit/priority_queue_spec.rb new file mode 100644 index 0000000000..17361833e5 --- /dev/null +++ b/spec/syntax_suggest/unit/priority_queue_spec.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + class CurrentIndex + attr_reader :current_indent + + def initialize(value) + @current_indent = value + end + + def <=>(other) + @current_indent <=> other.current_indent + end + + def inspect + @current_indent + end + end + + RSpec.describe CodeFrontier do + it "works" do + q = PriorityQueue.new + q << 1 + q << 2 + expect(q.elements).to eq([2, 1]) + + q << 3 + expect(q.elements).to eq([3, 1, 2]) + + expect(q.pop).to eq(3) + expect(q.pop).to eq(2) + expect(q.pop).to eq(1) + expect(q.pop).to eq(nil) + + array = [] + q = PriorityQueue.new + array.reverse_each do |v| + q << v + end + expect(q.elements).to eq(array) + + array = [100, 36, 17, 19, 25, 0, 3, 1, 7, 2] + array.reverse_each do |v| + q << v + end + + expect(q.pop).to eq(100) + expect(q.elements).to eq([36, 25, 19, 17, 0, 1, 7, 2, 3]) + + # expected [36, 25, 19, 17, 0, 1, 7, 2, 3] + expect(q.pop).to eq(36) + expect(q.pop).to eq(25) + expect(q.pop).to eq(19) + expect(q.pop).to eq(17) + expect(q.pop).to eq(7) + expect(q.pop).to eq(3) + expect(q.pop).to eq(2) + expect(q.pop).to eq(1) + expect(q.pop).to eq(0) + expect(q.pop).to eq(nil) + end + + it "priority queue" do + frontier = PriorityQueue.new + frontier << CurrentIndex.new(0) + frontier << CurrentIndex.new(1) + + expect(frontier.sorted.map(&:current_indent)).to eq([0, 1]) + + frontier << CurrentIndex.new(1) + expect(frontier.sorted.map(&:current_indent)).to eq([0, 1, 1]) + + frontier << CurrentIndex.new(0) + expect(frontier.sorted.map(&:current_indent)).to eq([0, 0, 1, 1]) + + frontier << CurrentIndex.new(10) + expect(frontier.sorted.map(&:current_indent)).to eq([0, 0, 1, 1, 10]) + + frontier << CurrentIndex.new(2) + expect(frontier.sorted.map(&:current_indent)).to eq([0, 0, 1, 1, 2, 10]) + + frontier = PriorityQueue.new + values = [18, 18, 0, 18, 0, 18, 18, 18, 18, 16, 18, 8, 18, 8, 8, 8, 16, 6, 0, 0, 16, 16, 4, 14, 14, 12, 12, 12, 10, 12, 12, 12, 12, 8, 10, 10, 8, 8, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 8, 10, 6, 6, 6, 6, 6, 6, 8, 10, 8, 8, 10, 8, 10, 8, 10, 8, 6, 8, 8, 6, 8, 6, 6, 8, 0, 8, 0, 0, 8, 8, 0, 8, 0, 8, 8, 0, 8, 8, 8, 0, 8, 0, 8, 8, 8, 8, 8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8, 8, 8, 6, 8, 6, 6, 6, 6, 8, 6, 8, 6, 6, 4, 4, 6, 6, 4, 6, 4, 6, 6, 4, 6, 4, 4, 6, 6, 6, 6, 4, 4, 4, 2, 4, 4, 4, 4, 4, 4, 6, 6, 0, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 0, 0, 6, 6, 2] + + values.each do |v| + value = CurrentIndex.new(v) + frontier << value # CurrentIndex.new(v) + end + + expect(frontier.sorted.map(&:current_indent)).to eq(values.sort) + end + end +end diff --git a/spec/syntax_suggest/unit/scan_history_spec.rb b/spec/syntax_suggest/unit/scan_history_spec.rb new file mode 100644 index 0000000000..d8b0a54ba6 --- /dev/null +++ b/spec/syntax_suggest/unit/scan_history_spec.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require_relative "../spec_helper" + +module SyntaxSuggest + RSpec.describe ScanHistory do + it "retains commits" do + source = <<~EOM + class OH # 0 + def lol # 1 + print 'lol # 2 + end # 3 + + def hello # 5 + it "foo" do # 6 + end # 7 + + def yolo # 8 + print 'haha' # 9 + end # 10 + end + EOM + + code_lines = CleanDocument.new(source: source).call.lines + block = CodeBlock.new(lines: code_lines[6]) + + scanner = ScanHistory.new(code_lines: code_lines, block: block) + scanner.scan(up: ->(_, _, _) { true }, down: ->(_, _, _) { true }) + + expect(scanner.changed?).to be_truthy + scanner.commit_if_changed + expect(scanner.changed?).to be_falsey + + expect(scanner.lines).to eq(code_lines) + + scanner.stash_changes # Assert does nothing if changes are already committed + expect(scanner.lines).to eq(code_lines) + + scanner.revert_last_commit + + expect(scanner.lines.join).to eq(code_lines[6].to_s) + end + + it "is stashable" do + source = <<~EOM + class OH # 0 + def lol # 1 + print 'lol # 2 + end # 3 + + def hello # 5 + it "foo" do # 6 + end # 7 + + def yolo # 8 + print 'haha' # 9 + end # 10 + end + EOM + + code_lines = CleanDocument.new(source: source).call.lines + block = CodeBlock.new(lines: code_lines[6]) + + scanner = ScanHistory.new(code_lines: code_lines, block: block) + scanner.scan(up: ->(_, _, _) { true }, down: ->(_, _, _) { true }) + + expect(scanner.lines).to eq(code_lines) + expect(scanner.changed?).to be_truthy + expect(scanner.next_up).to be_falsey + expect(scanner.next_down).to be_falsey + + scanner.stash_changes + + expect(scanner.changed?).to be_falsey + + expect(scanner.next_up).to eq(code_lines[5]) + expect(scanner.lines.join).to eq(code_lines[6].to_s) + expect(scanner.next_down).to eq(code_lines[7]) + end + + it "doesnt change if you dont't change it" do + source = <<~EOM + class OH # 0 + def lol # 1 + print 'lol # 2 + end # 3 + + def hello # 5 + it "foo" do # 6 + end # 7 + + def yolo # 8 + print 'haha' # 9 + end # 10 + end + EOM + + code_lines = CleanDocument.new(source: source).call.lines + block = CodeBlock.new(lines: code_lines[6]) + + scanner = ScanHistory.new(code_lines: code_lines, block: block) + + lines = scanner.lines + expect(scanner.changed?).to be_falsey + expect(scanner.next_up).to eq(code_lines[5]) + expect(scanner.next_down).to eq(code_lines[7]) + + expect(scanner.stash_changes.lines).to eq(lines) + expect(scanner.revert_last_commit.lines).to eq(lines) + + expect(scanner.scan(up: ->(_, _, _) { false }, down: ->(_, _, _) { false }).lines).to eq(lines) + end + end +end |