diff options
Diffstat (limited to 'lib')
744 files changed, 46703 insertions, 30486 deletions
diff --git a/lib/English.gemspec b/lib/English.gemspec index 5d0a396575..5f4eb420c2 100644 --- a/lib/English.gemspec +++ b/lib/English.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |spec| spec.name = "english" - spec.version = "0.7.2" + spec.version = "0.8.0" spec.authors = ["Yukihiro Matsumoto"] spec.email = ["matz@ruby-lang.org"] diff --git a/lib/English.rb b/lib/English.rb index da50dfdc29..03fe721991 100644 --- a/lib/English.rb +++ b/lib/English.rb @@ -39,7 +39,6 @@ # $PROCESS_ID:: $$ # $CHILD_STATUS:: $? # $LAST_MATCH_INFO:: $~ -# $IGNORECASE:: $= # $ARGV:: $* # $MATCH:: $& # $PREMATCH:: $` @@ -58,44 +57,32 @@ alias $ERROR_POSITION $@ # The default separator pattern used by String#split. May be set from # the command line using the <code>-F</code> flag. alias $FS $; - -# The default separator pattern used by String#split. May be set from -# the command line using the <code>-F</code> flag. alias $FIELD_SEPARATOR $; # The separator string output between the parameters to methods such # as Kernel#print and Array#join. Defaults to +nil+, which adds no # text. -alias $OFS $, # The separator string output between the parameters to methods such # as Kernel#print and Array#join. Defaults to +nil+, which adds no # text. +alias $OFS $, alias $OUTPUT_FIELD_SEPARATOR $, # The input record separator (newline by default). This is the value # that routines such as Kernel#gets use to determine record # boundaries. If set to +nil+, +gets+ will read the entire file. alias $RS $/ - -# The input record separator (newline by default). This is the value -# that routines such as Kernel#gets use to determine record -# boundaries. If set to +nil+, +gets+ will read the entire file. alias $INPUT_RECORD_SEPARATOR $/ # The string appended to the output of every call to methods such as # Kernel#print and IO#write. The default value is +nil+. alias $ORS $\ - -# The string appended to the output of every call to methods such as -# Kernel#print and IO#write. The default value is +nil+. alias $OUTPUT_RECORD_SEPARATOR $\ # The number of the last line read from the current input file. -alias $INPUT_LINE_NUMBER $. - -# The number of the last line read from the current input file. alias $NR $. +alias $INPUT_LINE_NUMBER $. # The last line read by Kernel#gets or # Kernel#readline. Many string-related functions in the @@ -135,8 +122,6 @@ alias $DEFAULT_INPUT $< # The process number of the program being executed. Read only. alias $PID $$ - -# The process number of the program being executed. Read only. alias $PROCESS_ID $$ # The exit status of the last child process to terminate. Read @@ -151,9 +136,6 @@ alias $CHILD_STATUS $? # scope. alias $LAST_MATCH_INFO $~ -# This variable is no longer effective. Deprecated. -alias $IGNORECASE $= - # An array of strings containing the command-line # options from the invocation of the program. Options # used by the Ruby interpreter will have been diff --git a/lib/abbrev.gemspec b/lib/abbrev.gemspec deleted file mode 100644 index 50c500bbc7..0000000000 --- a/lib/abbrev.gemspec +++ /dev/null @@ -1,29 +0,0 @@ -name = File.basename(__FILE__, ".gemspec") -version = ["lib", Array.new(name.count("-")+1, ".").join("/")].find do |dir| - break File.foreach(File.join(__dir__, dir, "#{name.tr('-', '/')}.rb")) do |line| - /^\s*VERSION\s*=\s*"(.*)"/ =~ line and break $1 - end rescue nil -end - -Gem::Specification.new do |spec| - spec.name = name - spec.version = version - spec.authors = ["Akinori MUSHA"] - spec.email = ["knu@idaemons.org"] - - spec.summary = %q{Calculates a set of unique abbreviations for a given set of strings} - spec.description = %q{Calculates a set of unique abbreviations for a given set of strings} - spec.homepage = "https://github.com/ruby/abbrev" - spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0") - spec.licenses = ["Ruby", "BSD-2-Clause"] - - spec.metadata["homepage_uri"] = spec.homepage - spec.metadata["source_code_uri"] = spec.homepage - - spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do - `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } - end - spec.bindir = "exe" - spec.executables = [] - spec.require_paths = ["lib"] -end diff --git a/lib/abbrev.rb b/lib/abbrev.rb deleted file mode 100644 index ee86f06874..0000000000 --- a/lib/abbrev.rb +++ /dev/null @@ -1,133 +0,0 @@ -# frozen_string_literal: true -#-- -# Copyright (c) 2001,2003 Akinori MUSHA <knu@iDaemons.org> -# -# All rights reserved. You can redistribute and/or modify it under -# the same terms as Ruby. -# -# $Idaemons: /home/cvs/rb/abbrev.rb,v 1.2 2001/05/30 09:37:45 knu Exp $ -# $RoughId: abbrev.rb,v 1.4 2003/10/14 19:45:42 knu Exp $ -# $Id$ -#++ - -## -# Calculates the set of unambiguous abbreviations for a given set of strings. -# -# require 'abbrev' -# require 'pp' -# -# pp Abbrev.abbrev(['ruby']) -# #=> {"ruby"=>"ruby", "rub"=>"ruby", "ru"=>"ruby", "r"=>"ruby"} -# -# pp Abbrev.abbrev(%w{ ruby rules }) -# -# _Generates:_ -# { "ruby" => "ruby", -# "rub" => "ruby", -# "rules" => "rules", -# "rule" => "rules", -# "rul" => "rules" } -# -# It also provides an array core extension, Array#abbrev. -# -# pp %w{ summer winter }.abbrev -# -# _Generates:_ -# { "summer" => "summer", -# "summe" => "summer", -# "summ" => "summer", -# "sum" => "summer", -# "su" => "summer", -# "s" => "summer", -# "winter" => "winter", -# "winte" => "winter", -# "wint" => "winter", -# "win" => "winter", -# "wi" => "winter", -# "w" => "winter" } - -module Abbrev - VERSION = "0.1.1" - - # Given a set of strings, calculate the set of unambiguous abbreviations for - # those strings, and return a hash where the keys are all the possible - # abbreviations and the values are the full strings. - # - # Thus, given +words+ is "car" and "cone", the keys pointing to "car" would - # be "ca" and "car", while those pointing to "cone" would be "co", "con", and - # "cone". - # - # require 'abbrev' - # - # Abbrev.abbrev(%w{ car cone }) - # #=> {"ca"=>"car", "con"=>"cone", "co"=>"cone", "car"=>"car", "cone"=>"cone"} - # - # The optional +pattern+ parameter is a pattern or a string. Only input - # strings that match the pattern or start with the string are included in the - # output hash. - # - # Abbrev.abbrev(%w{car box cone crab}, /b/) - # #=> {"box"=>"box", "bo"=>"box", "b"=>"box", "crab" => "crab"} - # - # Abbrev.abbrev(%w{car box cone}, 'ca') - # #=> {"car"=>"car", "ca"=>"car"} - def abbrev(words, pattern = nil) - table = {} - seen = Hash.new(0) - - if pattern.is_a?(String) - pattern = /\A#{Regexp.quote(pattern)}/ # regard as a prefix - end - - words.each do |word| - next if word.empty? - word.size.downto(1) { |len| - abbrev = word[0...len] - - next if pattern && pattern !~ abbrev - - case seen[abbrev] += 1 - when 1 - table[abbrev] = word - when 2 - table.delete(abbrev) - else - break - end - } - end - - words.each do |word| - next if pattern && pattern !~ word - - table[word] = word - end - - table - end - - module_function :abbrev -end - -class Array - # Calculates the set of unambiguous abbreviations for the strings in +self+. - # - # require 'abbrev' - # %w{ car cone }.abbrev - # #=> {"car"=>"car", "ca"=>"car", "cone"=>"cone", "con"=>"cone", "co"=>"cone"} - # - # The optional +pattern+ parameter is a pattern or a string. Only input - # strings that match the pattern or start with the string are included in the - # output hash. - # - # %w{ fast boat day }.abbrev(/^.a/) - # #=> {"fast"=>"fast", "fas"=>"fast", "fa"=>"fast", "day"=>"day", "da"=>"day"} - # - # Abbrev.abbrev(%w{car box cone}, "ca") - # #=> {"car"=>"car", "ca"=>"car"} - # - # See also Abbrev.abbrev - def abbrev(pattern = nil) - Abbrev::abbrev(self, pattern) - end -end diff --git a/lib/base64.gemspec b/lib/base64.gemspec deleted file mode 100644 index 44a2ce1fd1..0000000000 --- a/lib/base64.gemspec +++ /dev/null @@ -1,27 +0,0 @@ -name = File.basename(__FILE__, ".gemspec") -version = ["lib", Array.new(name.count("-")+1).join("/")].find do |dir| - break File.foreach(File.join(__dir__, dir, "#{name.tr('-', '/')}.rb")) do |line| - /^\s*VERSION\s*=\s*"(.*)"/ =~ line and break $1 - end rescue nil -end - -Gem::Specification.new do |spec| - spec.name = name - spec.version = version - spec.authors = ["Yusuke Endoh"] - spec.email = ["mame@ruby-lang.org"] - - spec.summary = %q{Support for encoding and decoding binary data using a Base64 representation.} - spec.description = %q{Support for encoding and decoding binary data using a Base64 representation.} - spec.homepage = "https://github.com/ruby/base64" - spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0") - spec.licenses = ["Ruby", "BSD-2-Clause"] - - spec.metadata["homepage_uri"] = spec.homepage - spec.metadata["source_code_uri"] = spec.homepage - - spec.files = ["README.md", "LICENSE.txt", "lib/base64.rb"] - spec.bindir = "exe" - spec.executables = [] - spec.require_paths = ["lib"] -end diff --git a/lib/base64.rb b/lib/base64.rb deleted file mode 100644 index 15c0b297ce..0000000000 --- a/lib/base64.rb +++ /dev/null @@ -1,113 +0,0 @@ -# frozen_string_literal: true -# -# = base64.rb: methods for base64-encoding and -decoding strings -# - -# The Base64 module provides for the encoding (#encode64, #strict_encode64, -# #urlsafe_encode64) and decoding (#decode64, #strict_decode64, -# #urlsafe_decode64) of binary data using a Base64 representation. -# -# == Example -# -# A simple encoding and decoding. -# -# require "base64" -# -# enc = Base64.encode64('Send reinforcements') -# # -> "U2VuZCByZWluZm9yY2VtZW50cw==\n" -# plain = Base64.decode64(enc) -# # -> "Send reinforcements" -# -# The purpose of using base64 to encode data is that it translates any -# binary data into purely printable characters. - -module Base64 - - VERSION = "0.1.1" - - module_function - - # Returns the Base64-encoded version of +bin+. - # This method complies with RFC 2045. - # Line feeds are added to every 60 encoded characters. - # - # require 'base64' - # Base64.encode64("Now is the time for all good coders\nto learn Ruby") - # - # <i>Generates:</i> - # - # Tm93IGlzIHRoZSB0aW1lIGZvciBhbGwgZ29vZCBjb2RlcnMKdG8gbGVhcm4g - # UnVieQ== - def encode64(bin) - [bin].pack("m") - end - - # Returns the Base64-decoded version of +str+. - # This method complies with RFC 2045. - # Characters outside the base alphabet are ignored. - # - # require 'base64' - # str = 'VGhpcyBpcyBsaW5lIG9uZQpUaGlzIG' + - # 'lzIGxpbmUgdHdvClRoaXMgaXMgbGlu' + - # 'ZSB0aHJlZQpBbmQgc28gb24uLi4K' - # puts Base64.decode64(str) - # - # <i>Generates:</i> - # - # This is line one - # This is line two - # This is line three - # And so on... - def decode64(str) - str.unpack1("m") - end - - # Returns the Base64-encoded version of +bin+. - # This method complies with RFC 4648. - # No line feeds are added. - def strict_encode64(bin) - [bin].pack("m0") - end - - # Returns the Base64-decoded version of +str+. - # This method complies with RFC 4648. - # ArgumentError is raised if +str+ is incorrectly padded or contains - # non-alphabet characters. Note that CR or LF are also rejected. - def strict_decode64(str) - str.unpack1("m0") - end - - # Returns the Base64-encoded version of +bin+. - # This method complies with ``Base 64 Encoding with URL and Filename Safe - # Alphabet'' in RFC 4648. - # The alphabet uses '-' instead of '+' and '_' instead of '/'. - # Note that the result can still contain '='. - # You can remove the padding by setting +padding+ as false. - def urlsafe_encode64(bin, padding: true) - str = strict_encode64(bin) - str.chomp!("==") or str.chomp!("=") unless padding - str.tr!("+/", "-_") - str - end - - # Returns the Base64-decoded version of +str+. - # This method complies with ``Base 64 Encoding with URL and Filename Safe - # Alphabet'' in RFC 4648. - # The alphabet uses '-' instead of '+' and '_' instead of '/'. - # - # The padding character is optional. - # This method accepts both correctly-padded and unpadded input. - # Note that it still rejects incorrectly-padded input. - def urlsafe_decode64(str) - # NOTE: RFC 4648 does say nothing about unpadded input, but says that - # "the excess pad characters MAY also be ignored", so it is inferred that - # unpadded input is also acceptable. - if !str.end_with?("=") && str.length % 4 != 0 - str = str.ljust((str.length + 3) & ~3, "=") - str.tr!("-_", "+/") - else - str = str.tr("-_", "+/") - end - strict_decode64(str) - end -end diff --git a/lib/benchmark.rb b/lib/benchmark.rb index d9f89239fc..9f43255e42 100644 --- a/lib/benchmark.rb +++ b/lib/benchmark.rb @@ -121,7 +121,7 @@ module Benchmark - VERSION = "0.2.1" + VERSION = "0.3.0" BENCHMARK_VERSION = "2002-04-25" # :nodoc: diff --git a/lib/bundled_gems.rb b/lib/bundled_gems.rb new file mode 100644 index 0000000000..c933ad0471 --- /dev/null +++ b/lib/bundled_gems.rb @@ -0,0 +1,191 @@ +# -*- frozen-string-literal: true -*- + +# :stopdoc: + +module Gem::BUNDLED_GEMS + SINCE = { + "rexml" => "3.0.0", + "rss" => "3.0.0", + "webrick" => "3.0.0", + "matrix" => "3.1.0", + "net-ftp" => "3.1.0", + "net-imap" => "3.1.0", + "net-pop" => "3.1.0", + "net-smtp" => "3.1.0", + "prime" => "3.1.0", + "racc" => "3.3.0", + "abbrev" => "3.4.0", + "base64" => "3.4.0", + "bigdecimal" => "3.4.0", + "csv" => "3.4.0", + "drb" => "3.4.0", + "getoptlong" => "3.4.0", + "mutex_m" => "3.4.0", + "nkf" => "3.4.0", + "observer" => "3.4.0", + "resolv-replace" => "3.4.0", + "rinda" => "3.4.0", + "syslog" => "3.4.0", + "ostruct" => "3.5.0", + "pstore" => "3.5.0", + "rdoc" => "3.5.0", + }.freeze + + EXACT = { + "abbrev" => true, + "base64" => true, + "bigdecimal" => true, + "csv" => true, + "drb" => true, + "getoptlong" => true, + "mutex_m" => true, + "nkf" => true, "kconv" => "nkf", + "observer" => true, + "resolv-replace" => true, + "rinda" => true, + "syslog" => true, + "ostruct" => true, + "pstore" => true, + "rdoc" => true, + }.freeze + + PREFIXED = { + "bigdecimal" => true, + "csv" => true, + "drb" => true, + "rinda" => true, + "syslog" => true, + }.freeze + + WARNED = {} # unfrozen + + conf = ::RbConfig::CONFIG + LIBDIR = (conf["rubylibdir"] + "/").freeze + ARCHDIR = (conf["rubyarchdir"] + "/").freeze + dlext = [conf["DLEXT"], "so"].uniq + DLEXT = /\.#{Regexp.union(dlext)}\z/ + LIBEXT = /\.#{Regexp.union("rb", *dlext)}\z/ + + def self.replace_require(specs) + return if [::Kernel.singleton_class, ::Kernel].any? {|klass| klass.respond_to?(:no_warning_require) } + + spec_names = specs.to_a.each_with_object({}) {|spec, h| h[spec.name] = true } + + [::Kernel.singleton_class, ::Kernel].each do |kernel_class| + kernel_class.send(:alias_method, :no_warning_require, :require) + kernel_class.send(:define_method, :require) do |name| + if message = ::Gem::BUNDLED_GEMS.warning?(name, specs: spec_names) # rubocop:disable Style/HashSyntax + warn message, :uplevel => 1 + end + kernel_class.send(:no_warning_require, name) + end + if kernel_class == ::Kernel + kernel_class.send(:private, :require) + else + kernel_class.send(:public, :require) + end + end + end + + def self.find_gem(path) + if !path + return + elsif path.start_with?(ARCHDIR) + n = path.delete_prefix(ARCHDIR).sub(DLEXT, "") + elsif path.start_with?(LIBDIR) + n = path.delete_prefix(LIBDIR).chomp(".rb") + else + return + end + EXACT[n] or PREFIXED[n = n[%r[\A[^/]+(?=/)]]] && n + end + + def self.warning?(name, specs: nil) + # name can be a feature name or a file path with String or Pathname + feature = File.path(name) + # bootsnap expands `require "csv"` to `require "#{LIBDIR}/csv.rb"`, + # and `require "syslog"` to `require "#{ARCHDIR}/syslog.so"`. + name = feature.delete_prefix(ARCHDIR) + name.delete_prefix!(LIBDIR) + name.tr!("/", "-") + name.sub!(LIBEXT, "") + return if specs.include?(name) + _t, path = $:.resolve_feature_path(feature) + if gem = find_gem(path) + return if specs.include?(gem) + caller = caller_locations(3, 3)&.find {|c| c&.absolute_path} + return if find_gem(caller&.absolute_path) + elsif SINCE[name] && !path + gem = true + else + return + end + + return if WARNED[name] + WARNED[name] = true + if gem == true + gem = name + "#{feature} was loaded from the standard library, but" + elsif gem + return if WARNED[gem] + WARNED[gem] = true + "#{feature} is found in #{gem}, which" + else + return + end + build_message(gem) + end + + def self.build_message(gem) + msg = " #{RUBY_VERSION < SINCE[gem] ? "will no longer be" : "is not"} part of the default gems since Ruby #{SINCE[gem]}." + + if defined?(Bundler) + msg += " Add #{gem} to your Gemfile or gemspec." + + # We detect the gem name from caller_locations. We need to skip 2 frames like: + # lib/ruby/3.3.0+0/bundled_gems.rb:90:in `warning?'", + # lib/ruby/3.3.0+0/bundler/rubygems_integration.rb:247:in `block (2 levels) in replace_require'", + # + # Additionally, we need to skip Bootsnap and Zeitwerk if present, these + # gems decorate Kernel#require, so they are not really the ones issuing + # the require call users should be warned about. Those are upwards. + location = Thread.each_caller_location(2) do |cl| + break cl.path unless cl.base_label == "require" + end + + if location && File.file?(location) && !location.start_with?(Gem::BUNDLED_GEMS::LIBDIR) + caller_gem = nil + Gem.path.each do |path| + if location =~ %r{#{path}/gems/([\w\-\.]+)} + caller_gem = $1 + break + end + end + if caller_gem + msg += " Also contact author of #{caller_gem} to add #{gem} into its gemspec." + end + end + else + msg += " Install #{gem} from RubyGems." + end + + msg + end + + freeze +end + +# for RubyGems without Bundler environment. +# If loading library is not part of the default gems and the bundled gems, warn it. +class LoadError + def message + return super unless path + + name = path.tr("/", "-") + if !defined?(Bundler) && Gem::BUNDLED_GEMS::SINCE[name] && !Gem::BUNDLED_GEMS::WARNED[name] + warn name + Gem::BUNDLED_GEMS.build_message(name) + end + super + end +end + +# :startdoc: diff --git a/lib/bundler.rb b/lib/bundler.rb index f83268e9cd..adf0a8e450 100644 --- a/lib/bundler.rb +++ b/lib/bundler.rb @@ -17,7 +17,7 @@ require_relative "bundler/build_metadata" # Bundler provides a consistent environment for Ruby projects by # tracking and installing the exact gems and versions that are needed. # -# Since Ruby 2.6, Bundler is a part of Ruby's standard library. +# Bundler is a part of Ruby's standard library. # # Bundler is used by creating _gemfiles_ listing all the project dependencies # and (optionally) their versions and then using @@ -39,6 +39,10 @@ module Bundler environment_preserver.replace_with_backup SUDO_MUTEX = Thread::Mutex.new + autoload :Checksum, File.expand_path("bundler/checksum", __dir__) + autoload :CLI, File.expand_path("bundler/cli", __dir__) + autoload :CIDetector, File.expand_path("bundler/ci_detector", __dir__) + autoload :CompactIndexClient, File.expand_path("bundler/compact_index_client", __dir__) autoload :Definition, File.expand_path("bundler/definition", __dir__) autoload :Dependency, File.expand_path("bundler/dependency", __dir__) autoload :Deprecate, File.expand_path("bundler/deprecate", __dir__) @@ -98,9 +102,7 @@ module Bundler end def create_bundle_path - SharedHelpers.filesystem_access(bundle_path.to_s) do |p| - mkdir_p(p) - end unless bundle_path.exist? + mkdir_p(bundle_path) unless bundle_path.exist? @bundle_path = bundle_path.realpath rescue Errno::EEXIST @@ -117,7 +119,7 @@ module Bundler @bin_path ||= begin path = settings[:bin] || "bin" path = Pathname.new(path).expand_path(root).expand_path - SharedHelpers.filesystem_access(path) {|p| FileUtils.mkdir_p(p) } + mkdir_p(path) path end end @@ -165,6 +167,25 @@ module Bundler end end + # Automatically install dependencies if Bundler.settings[:auto_install] exists. + # This is set through config cmd `bundle config set --global auto_install 1`. + # + # Note that this method `nil`s out the global Definition object, so it + # should be called first, before you instantiate anything like an + # `Installer` that'll keep a reference to the old one instead. + def auto_install + return unless settings[:auto_install] + + begin + definition.specs + rescue GemNotFound, GitError + ui.info "Automatically installing missing gems." + reset! + CLI::Install.new({}).run + reset! + end + end + # Setups Bundler environment (see Bundler.setup) if it is not already set, # and loads all gems from groups specified. Unlike ::setup, can be called # multiple times with different groups (if they were allowed by setup). @@ -184,6 +205,7 @@ module Bundler # Bundler.require(:test) # requires second_gem # def require(*groups) + load_plugins setup(*groups).require(*groups) end @@ -192,7 +214,7 @@ module Bundler end def environment - SharedHelpers.major_deprecation 2, "Bundler.environment has been removed in favor of Bundler.load", :print_caller_location => true + SharedHelpers.major_deprecation 2, "Bundler.environment has been removed in favor of Bundler.load", print_caller_location: true load end @@ -200,12 +222,13 @@ module Bundler # # @param unlock [Hash, Boolean, nil] Gems that have been requested # to be updated or true if all gems should be updated + # @param lockfile [Pathname] Path to Gemfile.lock # @return [Bundler::Definition] - def definition(unlock = nil) + def definition(unlock = nil, lockfile = default_lockfile) @definition = nil if unlock @definition ||= begin configure - Definition.build(default_gemfile, default_lockfile, unlock) + Definition.build(default_gemfile, lockfile, unlock) end end @@ -330,14 +353,6 @@ module Bundler def rm_rf(path) FileUtils.remove_entry_secure(path) if path && File.exist?(path) - rescue ArgumentError - message = <<EOF -It is a security vulnerability to allow your home directory to be world-writable, and bundler cannot continue. -You should probably consider fixing this issue by running `chmod o-w ~` on *nix. -Please refer to https://ruby-doc.org/stdlib-3.1.2/libdoc/fileutils/rdoc/FileUtils.html#method-c-remove_entry_secure for details. -EOF - File.world_writable?(path) ? Bundler.ui.warn(message) : raise - raise PathError, "Please fix the world-writable issue with your #{path} directory" end def settings @@ -353,13 +368,13 @@ EOF # @deprecated Use `unbundled_env` instead def clean_env - Bundler::SharedHelpers.major_deprecation( - 2, + message = "`Bundler.clean_env` has been deprecated in favor of `Bundler.unbundled_env`. " \ - "If you instead want the environment before bundler was originally loaded, use `Bundler.original_env`", - :print_caller_location => true - ) - + "If you instead want the environment before bundler was originally loaded, use `Bundler.original_env`" + removed_message = + "`Bundler.clean_env` has been removed in favor of `Bundler.unbundled_env`. " \ + "If you instead want the environment before bundler was originally loaded, use `Bundler.original_env`" + Bundler::SharedHelpers.major_deprecation(2, message, removed_message: removed_message, print_caller_location: true) unbundled_env end @@ -396,13 +411,13 @@ EOF # @deprecated Use `with_unbundled_env` instead def with_clean_env - Bundler::SharedHelpers.major_deprecation( - 2, + message = "`Bundler.with_clean_env` has been deprecated in favor of `Bundler.with_unbundled_env`. " \ - "If you instead want the environment before bundler was originally loaded, use `Bundler.with_original_env`", - :print_caller_location => true - ) - + "If you instead want the environment before bundler was originally loaded, use `Bundler.with_original_env`" + removed_message = + "`Bundler.with_clean_env` has been removed in favor of `Bundler.with_unbundled_env`. " \ + "If you instead want the environment before bundler was originally loaded, use `Bundler.with_original_env`" + Bundler::SharedHelpers.major_deprecation(2, message, removed_message: removed_message, print_caller_location: true) with_env(unbundled_env) { yield } end @@ -418,13 +433,13 @@ EOF # @deprecated Use `unbundled_system` instead def clean_system(*args) - Bundler::SharedHelpers.major_deprecation( - 2, + message = "`Bundler.clean_system` has been deprecated in favor of `Bundler.unbundled_system`. " \ - "If you instead want to run the command in the environment before bundler was originally loaded, use `Bundler.original_system`", - :print_caller_location => true - ) - + "If you instead want to run the command in the environment before bundler was originally loaded, use `Bundler.original_system`" + removed_message = + "`Bundler.clean_system` has been removed in favor of `Bundler.unbundled_system`. " \ + "If you instead want to run the command in the environment before bundler was originally loaded, use `Bundler.original_system`" + Bundler::SharedHelpers.major_deprecation(2, message, removed_message: removed_message, print_caller_location: true) with_env(unbundled_env) { Kernel.system(*args) } end @@ -440,13 +455,13 @@ EOF # @deprecated Use `unbundled_exec` instead def clean_exec(*args) - Bundler::SharedHelpers.major_deprecation( - 2, + message = "`Bundler.clean_exec` has been deprecated in favor of `Bundler.unbundled_exec`. " \ - "If you instead want to exec to a command in the environment before bundler was originally loaded, use `Bundler.original_exec`", - :print_caller_location => true - ) - + "If you instead want to exec to a command in the environment before bundler was originally loaded, use `Bundler.original_exec`" + removed_message = + "`Bundler.clean_exec` has been removed in favor of `Bundler.unbundled_exec`. " \ + "If you instead want to exec to a command in the environment before bundler was originally loaded, use `Bundler.original_exec`" + Bundler::SharedHelpers.major_deprecation(2, message, removed_message: removed_message, print_caller_location: true) with_env(unbundled_env) { Kernel.exec(*args) } end @@ -489,7 +504,7 @@ EOF configured_bundle_path.use_system_gems? end - def mkdir_p(path, options = {}) + def mkdir_p(path) SharedHelpers.filesystem_access(path, :write) do |p| FileUtils.mkdir_p(p) end @@ -515,7 +530,16 @@ EOF end def safe_load_marshal(data) - load_marshal(data, :marshal_proc => SafeMarshal.proc) + if Gem.respond_to?(:load_safe_marshal) + Gem.load_safe_marshal + begin + Gem::SafeMarshal.safe_load(data) + rescue Gem::SafeMarshal::Reader::Error, Gem::SafeMarshal::Visitors::ToRuby::Error => e + raise MarshalError, "#{e.class}: #{e.message}" + end + else + load_marshal(data, marshal_proc: SafeMarshal.proc) + end end def load_gemspec(file, validate = false) @@ -558,6 +582,23 @@ EOF @feature_flag ||= FeatureFlag.new(VERSION) end + def load_plugins(definition = Bundler.definition) + return if defined?(@load_plugins_ran) + + Bundler.rubygems.load_plugins + + requested_path_gems = definition.requested_specs.select {|s| s.source.is_a?(Source::Path) } + path_plugin_files = requested_path_gems.map do |spec| + Bundler.rubygems.spec_matches_for_glob(spec, "rubygems_plugin#{Bundler.rubygems.suffix_pattern}") + rescue TypeError + error_message = "#{spec.name} #{spec.version} has an invalid gemspec" + raise Gem::InvalidSpecificationException, error_message + end.flatten + Bundler.rubygems.load_plugin_files(path_plugin_files) + Bundler.rubygems.load_env_plugins + @load_plugins_ran = true + end + def reset! reset_paths! Plugin.reset! diff --git a/lib/bundler/build_metadata.rb b/lib/bundler/build_metadata.rb index 8bffb2fae7..5d2a8b53bb 100644 --- a/lib/bundler/build_metadata.rb +++ b/lib/bundler/build_metadata.rb @@ -29,7 +29,7 @@ module Bundler # commit instance variable then we can't determine its commits SHA. git_dir = File.expand_path("../../../.git", __dir__) if File.directory?(git_dir) - return @git_commit_sha = Dir.chdir(git_dir) { `git rev-parse --short HEAD`.strip.freeze } + return @git_commit_sha = IO.popen(%w[git rev-parse --short HEAD], { chdir: git_dir }, &:read).strip.freeze end @git_commit_sha ||= "unknown" diff --git a/lib/bundler/bundler.gemspec b/lib/bundler/bundler.gemspec index da50b46225..2d6269fae1 100644 --- a/lib/bundler/bundler.gemspec +++ b/lib/bundler/bundler.gemspec @@ -29,15 +29,17 @@ Gem::Specification.new do |s| "source_code_uri" => "https://github.com/rubygems/rubygems/tree/master/bundler", } - s.required_ruby_version = ">= 2.6.0" - s.required_rubygems_version = ">= 3.0.1" + s.required_ruby_version = ">= 3.0.0" + + # It should match the RubyGems version shipped with `required_ruby_version` above + s.required_rubygems_version = ">= 3.2.3" s.files = Dir.glob("lib/bundler{.rb,/**/*}", File::FNM_DOTMATCH).reject {|f| File.directory?(f) } # include the gemspec itself because warbler breaks w/o it s.files += %w[lib/bundler/bundler.gemspec] - s.bindir = "libexec" + s.bindir = "exe" s.executables = %w[bundle bundler] s.require_paths = ["lib"] end diff --git a/lib/bundler/capistrano.rb b/lib/bundler/capistrano.rb index 1f3712d48e..705840143f 100644 --- a/lib/bundler/capistrano.rb +++ b/lib/bundler/capistrano.rb @@ -17,6 +17,6 @@ end Capistrano::Configuration.instance(:must_exist).load do before "deploy:finalize_update", "bundle:install" - Bundler::Deployment.define_task(self, :task, :except => { :no_release => true }) + Bundler::Deployment.define_task(self, :task, except: { no_release: true }) set :rake, lambda { "#{fetch(:bundle_cmd, "bundle")} exec rake" } end diff --git a/lib/bundler/checksum.rb b/lib/bundler/checksum.rb new file mode 100644 index 0000000000..60ba93417c --- /dev/null +++ b/lib/bundler/checksum.rb @@ -0,0 +1,254 @@ +# frozen_string_literal: true + +module Bundler + class Checksum + ALGO_SEPARATOR = "=" + DEFAULT_ALGORITHM = "sha256" + private_constant :DEFAULT_ALGORITHM + DEFAULT_BLOCK_SIZE = 16_384 + private_constant :DEFAULT_BLOCK_SIZE + + class << self + def from_gem_package(gem_package, algo = DEFAULT_ALGORITHM) + return if Bundler.settings[:disable_checksum_validation] + return unless source = gem_package.instance_variable_get(:@gem) + return unless source.respond_to?(:with_read_io) + + source.with_read_io do |io| + from_gem(io, source.path) + ensure + io.rewind + end + end + + def from_gem(io, pathname, algo = DEFAULT_ALGORITHM) + digest = Bundler::SharedHelpers.digest(algo.upcase).new + buf = String.new(capacity: DEFAULT_BLOCK_SIZE) + digest << io.readpartial(DEFAULT_BLOCK_SIZE, buf) until io.eof? + Checksum.new(algo, digest.hexdigest!, Source.new(:gem, pathname)) + end + + def from_api(digest, source_uri, algo = DEFAULT_ALGORITHM) + return if Bundler.settings[:disable_checksum_validation] + + Checksum.new(algo, to_hexdigest(digest, algo), Source.new(:api, source_uri)) + end + + def from_lock(lock_checksum, lockfile_location) + algo, digest = lock_checksum.strip.split(ALGO_SEPARATOR, 2) + Checksum.new(algo, to_hexdigest(digest, algo), Source.new(:lock, lockfile_location)) + end + + def to_hexdigest(digest, algo = DEFAULT_ALGORITHM) + return digest unless algo == DEFAULT_ALGORITHM + return digest if digest.match?(/\A[0-9a-f]{64}\z/i) + + if digest.match?(%r{\A[-0-9a-z_+/]{43}={0,2}\z}i) + digest = digest.tr("-_", "+/") # fix urlsafe base64 + digest.unpack1("m0").unpack1("H*") + else + raise ArgumentError, "#{digest.inspect} is not a valid SHA256 hex or base64 digest" + end + end + end + + attr_reader :algo, :digest, :sources + + def initialize(algo, digest, source) + @algo = algo + @digest = digest + @sources = [source] + end + + def ==(other) + match?(other) && other.sources == sources + end + + alias_method :eql?, :== + + def same_source?(other) + sources.include?(other.sources.first) + end + + def match?(other) + other.is_a?(self.class) && other.digest == digest && other.algo == algo + end + + def hash + digest.hash + end + + def to_s + "#{to_lock} (from #{sources.first}#{", ..." if sources.size > 1})" + end + + def to_lock + "#{algo}#{ALGO_SEPARATOR}#{digest}" + end + + def merge!(other) + return nil unless match?(other) + + @sources.concat(other.sources).uniq! + self + end + + def formatted_sources + sources.join("\n and ").concat("\n") + end + + def removable? + sources.all?(&:removable?) + end + + def removal_instructions + msg = +"" + i = 1 + sources.each do |source| + msg << " #{i}. #{source.removal}\n" + i += 1 + end + msg << " #{i}. run `bundle install`\n" + end + + def inspect + abbr = "#{algo}#{ALGO_SEPARATOR}#{digest[0, 8]}" + from = "from #{sources.join(" and ")}" + "#<#{self.class}:#{object_id} #{abbr} #{from}>" + end + + class Source + attr_reader :type, :location + + def initialize(type, location) + @type = type + @location = location + end + + def removable? + type == :lock || type == :gem + end + + def ==(other) + other.is_a?(self.class) && other.type == type && other.location == location + end + + # phrased so that the usual string format is grammatically correct + # rake (10.3.2) sha256=abc123 from #{to_s} + def to_s + case type + when :lock + "the lockfile CHECKSUMS at #{location}" + when :gem + "the gem at #{location}" + when :api + "the API at #{location}" + else + "#{location} (#{type})" + end + end + + # A full sentence describing how to remove the checksum + def removal + case type + when :lock + "remove the matching checksum in #{location}" + when :gem + "remove the gem at #{location}" + when :api + "checksums from #{location} cannot be locally modified, you may need to update your sources" + else + "remove #{location} (#{type})" + end + end + end + + class Store + attr_reader :store + protected :store + + def initialize + @store = {} + @store_mutex = Mutex.new + end + + def inspect + "#<#{self.class}:#{object_id} size=#{store.size}>" + end + + # Replace when the new checksum is from the same source. + # The primary purpose is registering checksums from gems where there are + # duplicates of the same gem (according to full_name) in the index. + # + # In particular, this is when 2 gems have two similar platforms, e.g. + # "darwin20" and "darwin-20", both of which resolve to darwin-20. + # In the Index, the later gem replaces the former, so we do that here. + # + # However, if the new checksum is from a different source, we register like normal. + # This ensures a mismatch error where there are multiple top level sources + # that contain the same gem with different checksums. + def replace(spec, checksum) + return unless checksum + + lock_name = spec.name_tuple.lock_name + @store_mutex.synchronize do + existing = fetch_checksum(lock_name, checksum.algo) + if !existing || existing.same_source?(checksum) + store_checksum(lock_name, checksum) + else + merge_checksum(lock_name, checksum, existing) + end + end + end + + def register(spec, checksum) + return unless checksum + + register_checksum(spec.name_tuple.lock_name, checksum) + end + + def merge!(other) + other.store.each do |lock_name, checksums| + checksums.each do |_algo, checksum| + register_checksum(lock_name, checksum) + end + end + end + + def to_lock(spec) + lock_name = spec.name_tuple.lock_name + checksums = @store[lock_name] + if checksums + "#{lock_name} #{checksums.values.map(&:to_lock).sort.join(",")}" + else + lock_name + end + end + + private + + def register_checksum(lock_name, checksum) + @store_mutex.synchronize do + existing = fetch_checksum(lock_name, checksum.algo) + if existing + merge_checksum(lock_name, checksum, existing) + else + store_checksum(lock_name, checksum) + end + end + end + + def merge_checksum(lock_name, checksum, existing) + existing.merge!(checksum) || raise(ChecksumMismatchError.new(lock_name, existing, checksum)) + end + + def store_checksum(lock_name, checksum) + (@store[lock_name] ||= {})[checksum.algo] = checksum + end + + def fetch_checksum(lock_name, algo) + @store[lock_name]&.fetch(algo, nil) + end + end + end +end diff --git a/lib/bundler/ci_detector.rb b/lib/bundler/ci_detector.rb new file mode 100644 index 0000000000..e5fedbdea8 --- /dev/null +++ b/lib/bundler/ci_detector.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Bundler + module CIDetector + # NOTE: Any changes made here will need to be made to both lib/rubygems/ci_detector.rb and + # bundler/lib/bundler/ci_detector.rb (which are enforced duplicates). + # TODO: Drop that duplication once bundler drops support for RubyGems 3.4 + # + # ## Recognized CI providers, their signifiers, and the relevant docs ## + # + # Travis CI - CI, TRAVIS https://docs.travis-ci.com/user/environment-variables/#default-environment-variables + # Cirrus CI - CI, CIRRUS_CI https://cirrus-ci.org/guide/writing-tasks/#environment-variables + # Circle CI - CI, CIRCLECI https://circleci.com/docs/variables/#built-in-environment-variables + # Gitlab CI - CI, GITLAB_CI https://docs.gitlab.com/ee/ci/variables/ + # AppVeyor - CI, APPVEYOR https://www.appveyor.com/docs/environment-variables/ + # CodeShip - CI_NAME https://docs.cloudbees.com/docs/cloudbees-codeship/latest/pro-builds-and-configuration/environment-variables#_default_environment_variables + # dsari - CI, DSARI https://github.com/rfinnie/dsari#running + # Jenkins - BUILD_NUMBER https://www.jenkins.io/doc/book/pipeline/jenkinsfile/#using-environment-variables + # TeamCity - TEAMCITY_VERSION https://www.jetbrains.com/help/teamcity/predefined-build-parameters.html#Predefined+Server+Build+Parameters + # Appflow - CI_BUILD_ID https://ionic.io/docs/appflow/automation/environments#predefined-environments + # TaskCluster - TASKCLUSTER_ROOT_URL https://docs.taskcluster.net/docs/manual/design/env-vars + # Semaphore - CI, SEMAPHORE https://docs.semaphoreci.com/ci-cd-environment/environment-variables/ + # BuildKite - CI, BUILDKITE https://buildkite.com/docs/pipelines/environment-variables + # GoCD - GO_SERVER_URL https://docs.gocd.org/current/faq/dev_use_current_revision_in_build.html + # GH Actions - CI, GITHUB_ACTIONS https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables + # + # ### Some "standard" ENVs that multiple providers may set ### + # + # * CI - this is set by _most_ (but not all) CI providers now; it's approaching a standard. + # * CI_NAME - Not as frequently used, but some providers set this to specify their own name + + # Any of these being set is a reasonably reliable indicator that we are + # executing in a CI environment. + ENV_INDICATORS = [ + "CI", + "CI_NAME", + "CONTINUOUS_INTEGRATION", + "BUILD_NUMBER", + "CI_APP_ID", + "CI_BUILD_ID", + "CI_BUILD_NUMBER", + "RUN_ID", + "TASKCLUSTER_ROOT_URL", + ].freeze + + # For each CI, this env suffices to indicate that we're on _that_ CI's + # containers. (A few of them only supply a CI_NAME variable, which is also + # nice). And if they set "CI" but we can't tell which one they are, we also + # want to know that - a bare "ci" without another token tells us as much. + ENV_DESCRIPTORS = { + "TRAVIS" => "travis", + "CIRCLECI" => "circle", + "CIRRUS_CI" => "cirrus", + "DSARI" => "dsari", + "SEMAPHORE" => "semaphore", + "JENKINS_URL" => "jenkins", + "BUILDKITE" => "buildkite", + "GO_SERVER_URL" => "go", + "GITLAB_CI" => "gitlab", + "GITHUB_ACTIONS" => "github", + "TASKCLUSTER_ROOT_URL" => "taskcluster", + "CI" => "ci", + }.freeze + + def self.ci? + ENV_INDICATORS.any? {|var| ENV.include?(var) } + end + + def self.ci_strings + matching_names = ENV_DESCRIPTORS.select {|env, _| ENV[env] }.values + matching_names << ENV["CI_NAME"].downcase if ENV["CI_NAME"] + matching_names.reject(&:empty?).sort.uniq + end + end +end diff --git a/lib/bundler/cli.rb b/lib/bundler/cli.rb index dd91038d64..eb67668cd2 100644 --- a/lib/bundler/cli.rb +++ b/lib/bundler/cli.rb @@ -5,6 +5,7 @@ require_relative "vendored_thor" module Bundler class CLI < Thor require_relative "cli/common" + require_relative "cli/install" package_name "Bundler" @@ -69,7 +70,7 @@ module Bundler Bundler.settings.set_command_option_if_given :retry, options[:retry] current_cmd = args.last[:current_command].name - auto_install if AUTO_INSTALL_CMDS.include?(current_cmd) + Bundler.auto_install if AUTO_INSTALL_CMDS.include?(current_cmd) rescue UnknownArgumentError => e raise InvalidOption, e.message ensure @@ -80,10 +81,10 @@ module Bundler unprinted_warnings.each {|w| Bundler.ui.warn(w) } end - check_unknown_options!(:except => [:config, :exec]) + check_unknown_options!(except: [:config, :exec]) stop_on_unknown_option! :exec - desc "cli_help", "Prints a summary of bundler commands", :hide => true + desc "cli_help", "Prints a summary of bundler commands", hide: true def cli_help version Bundler.ui.info "\n" @@ -99,21 +100,23 @@ module Bundler shell.say "Bundler commands:\n\n" shell.say " Primary commands:\n" - shell.print_table(primary_commands, :indent => 4, :truncate => true) + shell.print_table(primary_commands, indent: 4, truncate: true) shell.say shell.say " Utilities:\n" - shell.print_table(utilities, :indent => 4, :truncate => true) + shell.print_table(utilities, indent: 4, truncate: true) shell.say self.class.send(:class_options_help, shell) end default_task(Bundler.feature_flag.default_cli_command) - class_option "no-color", :type => :boolean, :desc => "Disable colorization in output" - class_option "retry", :type => :numeric, :aliases => "-r", :banner => "NUM", - :desc => "Specify the number of times you wish to attempt network commands" - class_option "verbose", :type => :boolean, :desc => "Enable verbose output mode", :aliases => "-V" + class_option "no-color", type: :boolean, desc: "Disable colorization in output" + class_option "retry", type: :numeric, aliases: "-r", banner: "NUM", + desc: "Specify the number of times you wish to attempt network commands" + class_option "verbose", type: :boolean, desc: "Enable verbose output mode", aliases: "-V" def help(cli = nil) + cli = self.class.all_aliases[cli] if self.class.all_aliases[cli] + case cli when "gemfile" then command = "gemfile" when nil then command = "bundle" @@ -127,8 +130,8 @@ module Bundler if man_pages.include?(command) man_page = man_pages[command] - if Bundler.which("man") && man_path !~ %r{^file:/.+!/META-INF/jruby.home/.+} - Kernel.exec "man #{man_page}" + if Bundler.which("man") && !man_path.match?(%r{^file:/.+!/META-INF/jruby.home/.+}) + Kernel.exec("man", man_page) else puts File.read("#{man_path}/#{File.basename(man_page)}.ronn") end @@ -155,8 +158,8 @@ module Bundler Gemfile to a gem with a gemspec, the --gemspec option will automatically add each dependency listed in the gemspec file to the newly created Gemfile. D - method_option "gemspec", :type => :string, :banner => "Use the specified .gemspec to create the Gemfile" - method_option "gemfile", :type => :string, :banner => "Use the specified name for the gemfile instead of 'Gemfile'" + method_option "gemspec", type: :string, banner: "Use the specified .gemspec to create the Gemfile" + method_option "gemfile", type: :string, banner: "Use the specified name for the gemfile instead of 'Gemfile'" def init require_relative "cli/init" Init.new(options.dup).run @@ -168,12 +171,9 @@ module Bundler all gems are found, Bundler prints a success message and exits with a status of 0. If not, the first missing gem is listed and Bundler exits status 1. D - method_option "dry-run", :type => :boolean, :default => false, :banner => - "Lock the Gemfile" - method_option "gemfile", :type => :string, :banner => - "Use the specified gemfile instead of Gemfile" - method_option "path", :type => :string, :banner => - "Specify a different path than the system default ($BUNDLE_PATH or $GEM_HOME).#{" Bundler will remember this value for future installs on this machine" unless Bundler.feature_flag.forget_cli_options?}" + method_option "dry-run", type: :boolean, default: false, banner: "Lock the Gemfile" + method_option "gemfile", type: :string, banner: "Use the specified gemfile instead of Gemfile" + method_option "path", type: :string, banner: "Specify a different path than the system default ($BUNDLE_PATH or $GEM_HOME).#{" Bundler will remember this value for future installs on this machine" unless Bundler.feature_flag.forget_cli_options?}" def check remembered_flag_deprecation("path") @@ -187,10 +187,14 @@ module Bundler long_desc <<-D Removes the given gems from the Gemfile while ensuring that the resulting Gemfile is still valid. If the gem is not found, Bundler prints a error message and if gem could not be removed due to any reason Bundler will display a warning. D - method_option "install", :type => :boolean, :banner => - "Runs 'bundle install' after removing the gems from the Gemfile" + method_option "install", type: :boolean, banner: "Runs 'bundle install' after removing the gems from the Gemfile" def remove(*gems) - SharedHelpers.major_deprecation(2, "The `--install` flag has been deprecated. `bundle install` is triggered by default.") if ARGV.include?("--install") + if ARGV.include?("--install") + message = "The `--install` flag has been deprecated. `bundle install` is triggered by default." + removed_message = "The `--install` flag has been removed. `bundle install` is triggered by default." + SharedHelpers.major_deprecation(2, message, removed_message: removed_message) + end + require_relative "cli/remove" Remove.new(gems, options).run end @@ -206,58 +210,40 @@ module Bundler If the bundle has already been installed, bundler will tell you so and then exit. D - method_option "binstubs", :type => :string, :lazy_default => "bin", :banner => - "Generate bin stubs for bundled gems to ./bin" - method_option "clean", :type => :boolean, :banner => - "Run bundle clean automatically after install" - method_option "deployment", :type => :boolean, :banner => - "Install using defaults tuned for deployment environments" - method_option "frozen", :type => :boolean, :banner => - "Do not allow the Gemfile.lock to be updated after this install" - method_option "full-index", :type => :boolean, :banner => - "Fall back to using the single-file index of all gems" - method_option "gemfile", :type => :string, :banner => - "Use the specified gemfile instead of Gemfile" - method_option "jobs", :aliases => "-j", :type => :numeric, :banner => - "Specify the number of jobs to run in parallel" - method_option "local", :type => :boolean, :banner => - "Do not attempt to fetch gems remotely and use the gem cache instead" - method_option "prefer-local", :type => :boolean, :banner => - "Only attempt to fetch gems remotely if not present locally, even if newer versions are available remotely" - method_option "no-cache", :type => :boolean, :banner => - "Don't update the existing gem cache." - method_option "redownload", :type => :boolean, :aliases => "--force", :banner => - "Force downloading every gem." - method_option "no-prune", :type => :boolean, :banner => - "Don't remove stale gems from the cache." - method_option "path", :type => :string, :banner => - "Specify a different path than the system default ($BUNDLE_PATH or $GEM_HOME).#{" Bundler will remember this value for future installs on this machine" unless Bundler.feature_flag.forget_cli_options?}" - method_option "quiet", :type => :boolean, :banner => - "Only output warnings and errors." - method_option "shebang", :type => :string, :banner => - "Specify a different shebang executable name than the default (usually 'ruby')" - method_option "standalone", :type => :array, :lazy_default => [], :banner => - "Make a bundle that can work without the Bundler runtime" - method_option "system", :type => :boolean, :banner => - "Install to the system location ($BUNDLE_PATH or $GEM_HOME) even if the bundle was previously installed somewhere else for this application" - method_option "trust-policy", :alias => "P", :type => :string, :banner => - "Gem trust policy (like gem install -P). Must be one of " + - Bundler.rubygems.security_policy_keys.join("|") - method_option "without", :type => :array, :banner => - "Exclude gems that are part of the specified named group." - method_option "with", :type => :array, :banner => - "Include gems that are part of the specified named group." + method_option "binstubs", type: :string, lazy_default: "bin", banner: "Generate bin stubs for bundled gems to ./bin" + method_option "clean", type: :boolean, banner: "Run bundle clean automatically after install" + method_option "deployment", type: :boolean, banner: "Install using defaults tuned for deployment environments" + method_option "frozen", type: :boolean, banner: "Do not allow the Gemfile.lock to be updated after this install" + method_option "full-index", type: :boolean, banner: "Fall back to using the single-file index of all gems" + method_option "gemfile", type: :string, banner: "Use the specified gemfile instead of Gemfile" + method_option "jobs", aliases: "-j", type: :numeric, banner: "Specify the number of jobs to run in parallel" + method_option "local", type: :boolean, banner: "Do not attempt to fetch gems remotely and use the gem cache instead" + method_option "prefer-local", type: :boolean, banner: "Only attempt to fetch gems remotely if not present locally, even if newer versions are available remotely" + method_option "no-cache", type: :boolean, banner: "Don't update the existing gem cache." + method_option "redownload", type: :boolean, aliases: "--force", banner: "Force downloading every gem." + method_option "no-prune", type: :boolean, banner: "Don't remove stale gems from the cache." + method_option "path", type: :string, banner: "Specify a different path than the system default ($BUNDLE_PATH or $GEM_HOME).#{" Bundler will remember this value for future installs on this machine" unless Bundler.feature_flag.forget_cli_options?}" + method_option "quiet", type: :boolean, banner: "Only output warnings and errors." + method_option "shebang", type: :string, banner: "Specify a different shebang executable name than the default (usually 'ruby')" + method_option "standalone", type: :array, lazy_default: [], banner: "Make a bundle that can work without the Bundler runtime" + method_option "system", type: :boolean, banner: "Install to the system location ($BUNDLE_PATH or $GEM_HOME) even if the bundle was previously installed somewhere else for this application" + method_option "trust-policy", alias: "P", type: :string, banner: "Gem trust policy (like gem install -P). Must be one of " + + Bundler.rubygems.security_policy_keys.join("|") + method_option "without", type: :array, banner: "Exclude gems that are part of the specified named group." + method_option "with", type: :array, banner: "Include gems that are part of the specified named group." def install SharedHelpers.major_deprecation(2, "The `--force` option has been renamed to `--redownload`") if ARGV.include?("--force") - %w[clean deployment frozen no-prune path shebang system without with].each do |option| + %w[clean deployment frozen no-prune path shebang without with].each do |option| remembered_flag_deprecation(option) end + print_remembered_flag_deprecation("--system", "path.system", "true") if ARGV.include?("--system") + remembered_negative_flag_deprecation("no-deployment") require_relative "cli/install" - Bundler.settings.temporary(:no_install => false) do + Bundler.settings.temporary(no_install: false) do Install.new(options.dup).run end end @@ -270,44 +256,27 @@ module Bundler update when you have changed the Gemfile, or if you want to get the newest possible versions of the gems in the bundle. D - method_option "full-index", :type => :boolean, :banner => - "Fall back to using the single-file index of all gems" - method_option "gemfile", :type => :string, :banner => - "Use the specified gemfile instead of Gemfile" - method_option "group", :aliases => "-g", :type => :array, :banner => - "Update a specific group" - method_option "jobs", :aliases => "-j", :type => :numeric, :banner => - "Specify the number of jobs to run in parallel" - method_option "local", :type => :boolean, :banner => - "Do not attempt to fetch gems remotely and use the gem cache instead" - method_option "quiet", :type => :boolean, :banner => - "Only output warnings and errors." - method_option "source", :type => :array, :banner => - "Update a specific source (and all gems associated with it)" - method_option "redownload", :type => :boolean, :aliases => "--force", :banner => - "Force downloading every gem." - method_option "ruby", :type => :boolean, :banner => - "Update ruby specified in Gemfile.lock" - method_option "bundler", :type => :string, :lazy_default => "> 0.a", :banner => - "Update the locked version of bundler" - method_option "patch", :type => :boolean, :banner => - "Prefer updating only to next patch version" - method_option "minor", :type => :boolean, :banner => - "Prefer updating only to next minor version" - method_option "major", :type => :boolean, :banner => - "Prefer updating to next major version (default)" - method_option "pre", :type => :boolean, :banner => - "Always choose the highest allowed version when updating gems, regardless of prerelease status" - method_option "strict", :type => :boolean, :banner => - "Do not allow any gem to be updated past latest --patch | --minor | --major" - method_option "conservative", :type => :boolean, :banner => - "Use bundle install conservative update behavior and do not allow shared dependencies to be updated." - method_option "all", :type => :boolean, :banner => - "Update everything." + method_option "full-index", type: :boolean, banner: "Fall back to using the single-file index of all gems" + method_option "gemfile", type: :string, banner: "Use the specified gemfile instead of Gemfile" + method_option "group", aliases: "-g", type: :array, banner: "Update a specific group" + method_option "jobs", aliases: "-j", type: :numeric, banner: "Specify the number of jobs to run in parallel" + method_option "local", type: :boolean, banner: "Do not attempt to fetch gems remotely and use the gem cache instead" + method_option "quiet", type: :boolean, banner: "Only output warnings and errors." + method_option "source", type: :array, banner: "Update a specific source (and all gems associated with it)" + method_option "redownload", type: :boolean, aliases: "--force", banner: "Force downloading every gem." + method_option "ruby", type: :boolean, banner: "Update ruby specified in Gemfile.lock" + method_option "bundler", type: :string, lazy_default: "> 0.a", banner: "Update the locked version of bundler" + method_option "patch", type: :boolean, banner: "Prefer updating only to next patch version" + method_option "minor", type: :boolean, banner: "Prefer updating only to next minor version" + method_option "major", type: :boolean, banner: "Prefer updating to next major version (default)" + method_option "pre", type: :boolean, banner: "Always choose the highest allowed version when updating gems, regardless of prerelease status" + method_option "strict", type: :boolean, banner: "Do not allow any gem to be updated past latest --patch | --minor | --major" + method_option "conservative", type: :boolean, banner: "Use bundle install conservative update behavior and do not allow shared dependencies to be updated." + method_option "all", type: :boolean, banner: "Update everything." def update(*gems) SharedHelpers.major_deprecation(2, "The `--force` option has been renamed to `--redownload`") if ARGV.include?("--force") require_relative "cli/update" - Bundler.settings.temporary(:no_install => false) do + Bundler.settings.temporary(no_install: false) do Update.new(options, gems).run end end @@ -317,21 +286,25 @@ module Bundler Show lists the names and versions of all gems that are required by your Gemfile. Calling show with [GEM] will list the exact location of that gem on your machine. D - method_option "paths", :type => :boolean, - :banner => "List the paths of all gems that are required by your Gemfile." - method_option "outdated", :type => :boolean, - :banner => "Show verbose output including whether gems are outdated." + method_option "paths", type: :boolean, + banner: "List the paths of all gems that are required by your Gemfile." + method_option "outdated", type: :boolean, + banner: "Show verbose output including whether gems are outdated." def show(gem_name = nil) - SharedHelpers.major_deprecation(2, "the `--outdated` flag to `bundle show` was undocumented and will be removed without replacement") if ARGV.include?("--outdated") + if ARGV.include?("--outdated") + message = "the `--outdated` flag to `bundle show` was undocumented and will be removed without replacement" + removed_message = "the `--outdated` flag to `bundle show` was undocumented and has been removed without replacement" + SharedHelpers.major_deprecation(2, message, removed_message: removed_message) + end require_relative "cli/show" Show.new(options, gem_name).run end desc "list", "List all gems in the bundle" - method_option "name-only", :type => :boolean, :banner => "print only the gem names" - method_option "only-group", :type => :array, :default => [], :banner => "print gems from a given set of groups" - method_option "without-group", :type => :array, :default => [], :banner => "print all gems except from a given set of groups" - method_option "paths", :type => :boolean, :banner => "print the path to each gem in the bundle" + method_option "name-only", type: :boolean, banner: "print only the gem names" + method_option "only-group", type: :array, default: [], banner: "print gems from a given set of groups" + method_option "without-group", type: :array, default: [], banner: "print all gems except from a given set of groups" + method_option "paths", type: :boolean, banner: "print the path to each gem in the bundle" def list require_relative "cli/list" List.new(options).run @@ -340,8 +313,8 @@ module Bundler map aliases_for("list") desc "info GEM [OPTIONS]", "Show information for the given gem" - method_option "path", :type => :boolean, :banner => "Print full path to gem" - method_option "version", :type => :boolean, :banner => "Print gem version" + method_option "path", type: :boolean, banner: "Print full path to gem" + method_option "version", type: :boolean, banner: "Print gem version" def info(gem_name) require_relative "cli/info" Info.new(options, gem_name).run @@ -353,18 +326,12 @@ module Bundler or the --binstubs directory if one has been set. Calling binstubs with [GEM [GEM]] will create binstubs for all given gems. D - method_option "force", :type => :boolean, :default => false, :banner => - "Overwrite existing binstubs if they exist" - method_option "path", :type => :string, :lazy_default => "bin", :banner => - "Binstub destination directory (default bin)" - method_option "shebang", :type => :string, :banner => - "Specify a different shebang executable name than the default (usually 'ruby')" - method_option "standalone", :type => :boolean, :banner => - "Make binstubs that can work without the Bundler runtime" - method_option "all", :type => :boolean, :banner => - "Install binstubs for all gems" - method_option "all-platforms", :type => :boolean, :default => false, :banner => - "Install binstubs for all platforms" + method_option "force", type: :boolean, default: false, banner: "Overwrite existing binstubs if they exist" + method_option "path", type: :string, lazy_default: "bin", banner: "Binstub destination directory (default bin)" + method_option "shebang", type: :string, banner: "Specify a different shebang executable name than the default (usually 'ruby')" + method_option "standalone", type: :boolean, banner: "Make binstubs that can work without the Bundler runtime" + method_option "all", type: :boolean, banner: "Install binstubs for all gems" + method_option "all-platforms", type: :boolean, default: false, banner: "Install binstubs for all platforms" def binstubs(*gems) require_relative "cli/binstubs" Binstubs.new(options, gems).run @@ -374,19 +341,19 @@ module Bundler long_desc <<-D Adds the specified gem to Gemfile (if valid) and run 'bundle install' in one step. D - method_option "version", :aliases => "-v", :type => :string - method_option "group", :aliases => "-g", :type => :string - method_option "source", :aliases => "-s", :type => :string - method_option "require", :aliases => "-r", :type => :string, :banner => "Adds require path to gem. Provide false, or a path as a string." - method_option "path", :type => :string - method_option "git", :type => :string - method_option "github", :type => :string - method_option "branch", :type => :string - method_option "ref", :type => :string - method_option "skip-install", :type => :boolean, :banner => - "Adds gem to the Gemfile but does not install it" - method_option "optimistic", :type => :boolean, :banner => "Adds optimistic declaration of version to gem" - method_option "strict", :type => :boolean, :banner => "Adds strict declaration of version to gem" + method_option "version", aliases: "-v", type: :string + method_option "group", aliases: "-g", type: :string + method_option "source", aliases: "-s", type: :string + method_option "require", aliases: "-r", type: :string, banner: "Adds require path to gem. Provide false, or a path as a string." + method_option "path", type: :string + method_option "git", type: :string + method_option "github", type: :string + method_option "branch", type: :string + method_option "ref", type: :string + method_option "glob", type: :string, banner: "The location of a dependency's .gemspec, expanded within Ruby (single quotes recommended)" + method_option "skip-install", type: :boolean, banner: "Adds gem to the Gemfile but does not install it" + method_option "optimistic", type: :boolean, banner: "Adds optimistic declaration of version to gem" + method_option "strict", type: :boolean, banner: "Adds strict declaration of version to gem" def add(*gems) require_relative "cli/add" Add.new(options.dup, gems).run @@ -402,54 +369,45 @@ module Bundler For more information on patch level options (--major, --minor, --patch, --strict) see documentation on the same options on the update command. D - method_option "group", :type => :string, :banner => "List gems from a specific group" - method_option "groups", :type => :boolean, :banner => "List gems organized by groups" - method_option "local", :type => :boolean, :banner => - "Do not attempt to fetch gems remotely and use the gem cache instead" - method_option "pre", :type => :boolean, :banner => "Check for newer pre-release gems" - method_option "source", :type => :array, :banner => "Check against a specific source" - method_option "filter-strict", :type => :boolean, :aliases => "--strict", :banner => - "Only list newer versions allowed by your Gemfile requirements" - method_option "update-strict", :type => :boolean, :banner => - "Strict conservative resolution, do not allow any gem to be updated past latest --patch | --minor | --major" - method_option "minor", :type => :boolean, :banner => "Prefer updating only to next minor version" - method_option "major", :type => :boolean, :banner => "Prefer updating to next major version (default)" - method_option "patch", :type => :boolean, :banner => "Prefer updating only to next patch version" - method_option "filter-major", :type => :boolean, :banner => "Only list major newer versions" - method_option "filter-minor", :type => :boolean, :banner => "Only list minor newer versions" - method_option "filter-patch", :type => :boolean, :banner => "Only list patch newer versions" - method_option "parseable", :aliases => "--porcelain", :type => :boolean, :banner => - "Use minimal formatting for more parseable output" - method_option "only-explicit", :type => :boolean, :banner => - "Only list gems specified in your Gemfile, not their dependencies" + method_option "group", type: :string, banner: "List gems from a specific group" + method_option "groups", type: :boolean, banner: "List gems organized by groups" + method_option "local", type: :boolean, banner: "Do not attempt to fetch gems remotely and use the gem cache instead" + method_option "pre", type: :boolean, banner: "Check for newer pre-release gems" + method_option "source", type: :array, banner: "Check against a specific source" + method_option "filter-strict", type: :boolean, aliases: "--strict", banner: "Only list newer versions allowed by your Gemfile requirements" + method_option "update-strict", type: :boolean, banner: "Strict conservative resolution, do not allow any gem to be updated past latest --patch | --minor | --major" + method_option "minor", type: :boolean, banner: "Prefer updating only to next minor version" + method_option "major", type: :boolean, banner: "Prefer updating to next major version (default)" + method_option "patch", type: :boolean, banner: "Prefer updating only to next patch version" + method_option "filter-major", type: :boolean, banner: "Only list major newer versions" + method_option "filter-minor", type: :boolean, banner: "Only list minor newer versions" + method_option "filter-patch", type: :boolean, banner: "Only list patch newer versions" + method_option "parseable", aliases: "--porcelain", type: :boolean, banner: "Use minimal formatting for more parseable output" + method_option "only-explicit", type: :boolean, banner: "Only list gems specified in your Gemfile, not their dependencies" def outdated(*gems) require_relative "cli/outdated" Outdated.new(options, gems).run end desc "fund [OPTIONS]", "Lists information about gems seeking funding assistance" - method_option "group", :aliases => "-g", :type => :array, :banner => - "Fetch funding information for a specific group" + method_option "group", aliases: "-g", type: :array, banner: "Fetch funding information for a specific group" def fund require_relative "cli/fund" Fund.new(options).run end desc "cache [OPTIONS]", "Locks and then caches all of the gems into vendor/cache" - method_option "all", :type => :boolean, - :default => Bundler.feature_flag.cache_all?, - :banner => "Include all sources (including path and git)." - method_option "all-platforms", :type => :boolean, :banner => "Include gems for all platforms present in the lockfile, not only the current one" - method_option "cache-path", :type => :string, :banner => - "Specify a different cache path than the default (vendor/cache)." - method_option "gemfile", :type => :string, :banner => "Use the specified gemfile instead of Gemfile" - method_option "no-install", :type => :boolean, :banner => "Don't install the gems, only update the cache." - method_option "no-prune", :type => :boolean, :banner => "Don't remove stale gems from the cache." - method_option "path", :type => :string, :banner => - "Specify a different path than the system default ($BUNDLE_PATH or $GEM_HOME).#{" Bundler will remember this value for future installs on this machine" unless Bundler.feature_flag.forget_cli_options?}" - method_option "quiet", :type => :boolean, :banner => "Only output warnings and errors." - method_option "frozen", :type => :boolean, :banner => - "Do not allow the Gemfile.lock to be updated after this bundle cache operation's install" + method_option "all", type: :boolean, + default: Bundler.feature_flag.cache_all?, + banner: "Include all sources (including path and git)." + method_option "all-platforms", type: :boolean, banner: "Include gems for all platforms present in the lockfile, not only the current one" + method_option "cache-path", type: :string, banner: "Specify a different cache path than the default (vendor/cache)." + method_option "gemfile", type: :string, banner: "Use the specified gemfile instead of Gemfile" + method_option "no-install", type: :boolean, banner: "Don't install the gems, only update the cache." + method_option "no-prune", type: :boolean, banner: "Don't remove stale gems from the cache." + method_option "path", type: :string, banner: "Specify a different path than the system default ($BUNDLE_PATH or $GEM_HOME).#{" Bundler will remember this value for future installs on this machine" unless Bundler.feature_flag.forget_cli_options?}" + method_option "quiet", type: :boolean, banner: "Only output warnings and errors." + method_option "frozen", type: :boolean, banner: "Do not allow the Gemfile.lock to be updated after this bundle cache operation's install" long_desc <<-D The cache command will copy the .gem files for every gem in the bundle into the directory ./vendor/cache. If you then check that directory into your source @@ -457,17 +415,20 @@ module Bundler bundle without having to download any additional gems. D def cache - SharedHelpers.major_deprecation 2, - "The `--all` flag is deprecated because it relies on being " \ - "remembered across bundler invocations, which bundler will no longer " \ - "do in future versions. Instead please use `bundle config set cache_all true`, " \ - "and stop using this flag" if ARGV.include?("--all") - - SharedHelpers.major_deprecation 2, - "The `--path` flag is deprecated because its semantics are unclear. " \ - "Use `bundle config cache_path` to configure the path of your cache of gems, " \ - "and `bundle config path` to configure the path where your gems are installed, " \ - "and stop using this flag" if ARGV.include?("--path") + print_remembered_flag_deprecation("--all", "cache_all", "true") if ARGV.include?("--all") + + if ARGV.include?("--path") + message = + "The `--path` flag is deprecated because its semantics are unclear. " \ + "Use `bundle config cache_path` to configure the path of your cache of gems, " \ + "and `bundle config path` to configure the path where your gems are installed, " \ + "and stop using this flag" + removed_message = + "The `--path` flag has been removed because its semantics were unclear. " \ + "Use `bundle config cache_path` to configure the path of your cache of gems, " \ + "and `bundle config path` to configure the path where your gems are installed." + SharedHelpers.major_deprecation 2, message, removed_message: removed_message + end require_relative "cli/cache" Cache.new(options).run @@ -476,8 +437,8 @@ module Bundler map aliases_for("cache") desc "exec [OPTIONS]", "Run the command in context of the bundle" - method_option :keep_file_descriptors, :type => :boolean, :default => true - method_option :gemfile, :type => :string, :required => false + method_option :keep_file_descriptors, type: :boolean, default: true + method_option :gemfile, type: :string, required: false long_desc <<-D Exec runs a command, providing it access to the gems in the bundle. While using bundle exec you can require and call the bundled gems as if they were installed @@ -485,7 +446,9 @@ module Bundler D def exec(*args) if ARGV.include?("--no-keep-file-descriptors") - SharedHelpers.major_deprecation(2, "The `--no-keep-file-descriptors` has been deprecated. `bundle exec` no longer mess with your file descriptors. Close them in the exec'd script if you need to") + message = "The `--no-keep-file-descriptors` has been deprecated. `bundle exec` no longer mess with your file descriptors. Close them in the exec'd script if you need to" + removed_message = "The `--no-keep-file-descriptors` has been removed. `bundle exec` no longer mess with your file descriptors. Close them in the exec'd script if you need to" + SharedHelpers.major_deprecation(2, message, removed_message: removed_message) end require_relative "cli/exec" @@ -510,7 +473,7 @@ module Bundler subcommand "config", Config desc "open GEM", "Opens the source directory of the given bundled gem" - method_option "path", :type => :string, :lazy_default => "", :banner => "Open relative path of the gem source." + method_option "path", type: :string, lazy_default: "", banner: "Open relative path of the gem source." def open(name) require_relative "cli/open" Open.new(options, name).run @@ -555,17 +518,17 @@ module Bundler end unless Bundler.feature_flag.bundler_3_mode? - desc "viz [OPTIONS]", "Generates a visual dependency graph", :hide => true + desc "viz [OPTIONS]", "Generates a visual dependency graph", hide: true long_desc <<-D Viz generates a PNG file of the current Gemfile as a dependency graph. Viz requires the ruby-graphviz gem (and its dependencies). The associated gems must also be installed via 'bundle install'. D - method_option :file, :type => :string, :default => "gem_graph", :aliases => "-f", :desc => "The name to use for the generated file. see format option" - method_option :format, :type => :string, :default => "png", :aliases => "-F", :desc => "This is output format option. Supported format is png, jpg, svg, dot ..." - method_option :requirements, :type => :boolean, :default => false, :aliases => "-R", :desc => "Set to show the version of each required dependency." - method_option :version, :type => :boolean, :default => false, :aliases => "-v", :desc => "Set to show each gem version." - method_option :without, :type => :array, :default => [], :aliases => "-W", :banner => "GROUP[ GROUP...]", :desc => "Exclude gems that are part of the specified named group." + method_option :file, type: :string, default: "gem_graph", aliases: "-f", desc: "The name to use for the generated file. see format option" + method_option :format, type: :string, default: "png", aliases: "-F", desc: "This is output format option. Supported format is png, jpg, svg, dot ..." + method_option :requirements, type: :boolean, default: false, aliases: "-R", desc: "Set to show the version of each required dependency." + method_option :version, type: :boolean, default: false, aliases: "-v", desc: "Set to show each gem version." + method_option :without, type: :array, default: [], aliases: "-W", banner: "GROUP[ GROUP...]", desc: "Exclude gems that are part of the specified named group." def viz SharedHelpers.major_deprecation 2, "The `viz` command has been renamed to `graph` and moved to a plugin. See https://github.com/rubygems/bundler-graph" require_relative "cli/viz" @@ -576,23 +539,23 @@ module Bundler old_gem = instance_method(:gem) desc "gem NAME [OPTIONS]", "Creates a skeleton for creating a rubygem" - method_option :exe, :type => :boolean, :default => false, :aliases => ["--bin", "-b"], :desc => "Generate a binary executable for your library." - method_option :coc, :type => :boolean, :desc => "Generate a code of conduct file. Set a default with `bundle config set --global gem.coc true`." - method_option :edit, :type => :string, :aliases => "-e", :required => false, :banner => "EDITOR", - :lazy_default => [ENV["BUNDLER_EDITOR"], ENV["VISUAL"], ENV["EDITOR"]].find {|e| !e.nil? && !e.empty? }, - :desc => "Open generated gemspec in the specified editor (defaults to $EDITOR or $BUNDLER_EDITOR)" - method_option :ext, :type => :string, :desc => "Generate the boilerplate for C extension code.", :enum => EXTENSIONS - method_option :git, :type => :boolean, :default => true, :desc => "Initialize a git repo inside your library." - method_option :mit, :type => :boolean, :desc => "Generate an MIT license file. Set a default with `bundle config set --global gem.mit true`." - method_option :rubocop, :type => :boolean, :desc => "Add rubocop to the generated Rakefile and gemspec. Set a default with `bundle config set --global gem.rubocop true`." - method_option :changelog, :type => :boolean, :desc => "Generate changelog file. Set a default with `bundle config set --global gem.changelog true`." - method_option :test, :type => :string, :lazy_default => Bundler.settings["gem.test"] || "", :aliases => "-t", :banner => "Use the specified test framework for your library", - :desc => "Generate a test directory for your library, either rspec, minitest or test-unit. Set a default with `bundle config set --global gem.test (rspec|minitest|test-unit)`." - method_option :ci, :type => :string, :lazy_default => Bundler.settings["gem.ci"] || "", - :desc => "Generate CI configuration, either GitHub Actions, GitLab CI or CircleCI. Set a default with `bundle config set --global gem.ci (github|gitlab|circle)`" - method_option :linter, :type => :string, :lazy_default => Bundler.settings["gem.linter"] || "", - :desc => "Add a linter and code formatter, either RuboCop or Standard. Set a default with `bundle config set --global gem.linter (rubocop|standard)`" - method_option :github_username, :type => :string, :default => Bundler.settings["gem.github_username"], :banner => "Set your username on GitHub", :desc => "Fill in GitHub username on README so that you don't have to do it manually. Set a default with `bundle config set --global gem.github_username <your_username>`." + method_option :exe, type: :boolean, default: false, aliases: ["--bin", "-b"], desc: "Generate a binary executable for your library." + method_option :coc, type: :boolean, desc: "Generate a code of conduct file. Set a default with `bundle config set --global gem.coc true`." + method_option :edit, type: :string, aliases: "-e", required: false, banner: "EDITOR", + lazy_default: [ENV["BUNDLER_EDITOR"], ENV["VISUAL"], ENV["EDITOR"]].find {|e| !e.nil? && !e.empty? }, + desc: "Open generated gemspec in the specified editor (defaults to $EDITOR or $BUNDLER_EDITOR)" + method_option :ext, type: :string, desc: "Generate the boilerplate for C extension code.", enum: EXTENSIONS + method_option :git, type: :boolean, default: true, desc: "Initialize a git repo inside your library." + method_option :mit, type: :boolean, desc: "Generate an MIT license file. Set a default with `bundle config set --global gem.mit true`." + method_option :rubocop, type: :boolean, desc: "Add rubocop to the generated Rakefile and gemspec. Set a default with `bundle config set --global gem.rubocop true`." + method_option :changelog, type: :boolean, desc: "Generate changelog file. Set a default with `bundle config set --global gem.changelog true`." + method_option :test, type: :string, lazy_default: Bundler.settings["gem.test"] || "", aliases: "-t", banner: "Use the specified test framework for your library", + desc: "Generate a test directory for your library, either rspec, minitest or test-unit. Set a default with `bundle config set --global gem.test (rspec|minitest|test-unit)`." + method_option :ci, type: :string, lazy_default: Bundler.settings["gem.ci"] || "", + desc: "Generate CI configuration, either GitHub Actions, GitLab CI or CircleCI. Set a default with `bundle config set --global gem.ci (github|gitlab|circle)`" + method_option :linter, type: :string, lazy_default: Bundler.settings["gem.linter"] || "", + desc: "Add a linter and code formatter, either RuboCop or Standard. Set a default with `bundle config set --global gem.linter (rubocop|standard)`" + method_option :github_username, type: :string, default: Bundler.settings["gem.github_username"], banner: "Set your username on GitHub", desc: "Fill in GitHub username on README so that you don't have to do it manually. Set a default with `bundle config set --global gem.github_username <your_username>`." def gem(name) end @@ -623,29 +586,24 @@ module Bundler File.expand_path("templates", __dir__) end - desc "clean [OPTIONS]", "Cleans up unused gems in your bundler directory", :hide => true - method_option "dry-run", :type => :boolean, :default => false, :banner => - "Only print out changes, do not clean gems" - method_option "force", :type => :boolean, :default => false, :banner => - "Forces cleaning up unused gems even if Bundler is configured to use globally installed gems. As a consequence, removes all system gems except for the ones in the current application." + desc "clean [OPTIONS]", "Cleans up unused gems in your bundler directory", hide: true + method_option "dry-run", type: :boolean, default: false, banner: "Only print out changes, do not clean gems" + method_option "force", type: :boolean, default: false, banner: "Forces cleaning up unused gems even if Bundler is configured to use globally installed gems. As a consequence, removes all system gems except for the ones in the current application." def clean require_relative "cli/clean" Clean.new(options.dup).run end desc "platform [OPTIONS]", "Displays platform compatibility information" - method_option "ruby", :type => :boolean, :default => false, :banner => - "only display ruby related platform information" + method_option "ruby", type: :boolean, default: false, banner: "only display ruby related platform information" def platform require_relative "cli/platform" Platform.new(options).run end - desc "inject GEM VERSION", "Add the named gem, with version requirements, to the resolved Gemfile", :hide => true - method_option "source", :type => :string, :banner => - "Install gem from the given source" - method_option "group", :type => :string, :banner => - "Install gem into a bundler group" + desc "inject GEM VERSION", "Add the named gem, with version requirements, to the resolved Gemfile", hide: true + method_option "source", type: :string, banner: "Install gem from the given source" + method_option "group", type: :string, banner: "Install gem into a bundler group" def inject(name, version) SharedHelpers.major_deprecation 2, "The `inject` command has been replaced by the `add` command" require_relative "cli/inject" @@ -653,36 +611,21 @@ module Bundler end desc "lock", "Creates a lockfile without installing" - method_option "update", :type => :array, :lazy_default => true, :banner => - "ignore the existing lockfile, update all gems by default, or update list of given gems" - method_option "local", :type => :boolean, :default => false, :banner => - "do not attempt to fetch remote gemspecs and use the local gem cache only" - method_option "print", :type => :boolean, :default => false, :banner => - "print the lockfile to STDOUT instead of writing to the file system" - method_option "gemfile", :type => :string, :banner => - "Use the specified gemfile instead of Gemfile" - method_option "lockfile", :type => :string, :default => nil, :banner => - "the path the lockfile should be written to" - method_option "full-index", :type => :boolean, :default => false, :banner => - "Fall back to using the single-file index of all gems" - method_option "add-platform", :type => :array, :default => [], :banner => - "Add a new platform to the lockfile" - method_option "remove-platform", :type => :array, :default => [], :banner => - "Remove a platform from the lockfile" - method_option "patch", :type => :boolean, :banner => - "If updating, prefer updating only to next patch version" - method_option "minor", :type => :boolean, :banner => - "If updating, prefer updating only to next minor version" - method_option "major", :type => :boolean, :banner => - "If updating, prefer updating to next major version (default)" - method_option "pre", :type => :boolean, :banner => - "If updating, always choose the highest allowed version, regardless of prerelease status" - method_option "strict", :type => :boolean, :banner => - "If updating, do not allow any gem to be updated past latest --patch | --minor | --major" - method_option "conservative", :type => :boolean, :banner => - "If updating, use bundle install conservative update behavior and do not allow shared dependencies to be updated" - method_option "bundler", :type => :string, :lazy_default => "> 0.a", :banner => - "Update the locked version of bundler" + method_option "update", type: :array, lazy_default: true, banner: "ignore the existing lockfile, update all gems by default, or update list of given gems" + method_option "local", type: :boolean, default: false, banner: "do not attempt to fetch remote gemspecs and use the local gem cache only" + method_option "print", type: :boolean, default: false, banner: "print the lockfile to STDOUT instead of writing to the file system" + method_option "gemfile", type: :string, banner: "Use the specified gemfile instead of Gemfile" + method_option "lockfile", type: :string, default: nil, banner: "the path the lockfile should be written to" + method_option "full-index", type: :boolean, default: false, banner: "Fall back to using the single-file index of all gems" + method_option "add-platform", type: :array, default: [], banner: "Add a new platform to the lockfile" + method_option "remove-platform", type: :array, default: [], banner: "Remove a platform from the lockfile" + method_option "patch", type: :boolean, banner: "If updating, prefer updating only to next patch version" + method_option "minor", type: :boolean, banner: "If updating, prefer updating only to next minor version" + method_option "major", type: :boolean, banner: "If updating, prefer updating to next major version (default)" + method_option "pre", type: :boolean, banner: "If updating, always choose the highest allowed version, regardless of prerelease status" + method_option "strict", type: :boolean, banner: "If updating, do not allow any gem to be updated past latest --patch | --minor | --major" + method_option "conservative", type: :boolean, banner: "If updating, use bundle install conservative update behavior and do not allow shared dependencies to be updated" + method_option "bundler", type: :string, lazy_default: "> 0.a", banner: "Update the locked version of bundler" def lock require_relative "cli/lock" Lock.new(options).run @@ -699,10 +642,8 @@ module Bundler missing dependencies are detected, Bundler prints them and exits status 1. Otherwise, Bundler prints a success message and exits with a status of 0. D - method_option "gemfile", :type => :string, :banner => - "Use the specified gemfile instead of Gemfile" - method_option "quiet", :type => :boolean, :banner => - "Only output warnings and errors." + method_option "gemfile", type: :string, banner: "Use the specified gemfile instead of Gemfile" + method_option "quiet", type: :boolean, banner: "Only output warnings and errors." def doctor require_relative "cli/doctor" Doctor.new(options).run @@ -722,7 +663,9 @@ module Bundler D def pristine(*gems) require_relative "cli/pristine" - Pristine.new(gems).run + Bundler.settings.temporary(no_install: false) do + Pristine.new(gems).run + end end if Bundler.feature_flag.plugins? @@ -743,7 +686,6 @@ module Bundler exec_used = args.index {|a| exec_commands.include? a } command = args.find {|a| bundler_commands.include? a } - command = all_aliases[command] if all_aliases[command] if exec_used && help_used if exec_used + help_used == 1 @@ -764,7 +706,9 @@ module Bundler # when deprecated version of `--ext` is called # print out deprecation warning and pretend `--ext=c` was provided if deprecated_ext_value?(arguments) - SharedHelpers.major_deprecation 2, "Extensions can now be generated using C or Rust, so `--ext` with no arguments has been deprecated. Please select a language, e.g. `--ext=rust` to generate a Rust extension. This gem will now be generated as if `--ext=c` was used." + message = "Extensions can now be generated using C or Rust, so `--ext` with no arguments has been deprecated. Please select a language, e.g. `--ext=rust` to generate a Rust extension. This gem will now be generated as if `--ext=c` was used." + removed_message = "Extensions can now be generated using C or Rust, so `--ext` with no arguments has been removed. Please select a language, e.g. `--ext=rust` to generate a Rust extension." + SharedHelpers.major_deprecation 2, message, removed_message: removed_message arguments[arguments.index("--ext")] = "--ext=c" end end @@ -794,26 +738,6 @@ module Bundler private - # Automatically invoke `bundle install` and resume if - # Bundler.settings[:auto_install] exists. This is set through config cmd - # `bundle config set --global auto_install 1`. - # - # Note that this method `nil`s out the global Definition object, so it - # should be called first, before you instantiate anything like an - # `Installer` that'll keep a reference to the old one instead. - def auto_install - return unless Bundler.settings[:auto_install] - - begin - Bundler.definition.specs - rescue GemNotFound - Bundler.ui.info "Automatically installing missing gems." - Bundler.reset! - invoke :install, [] - Bundler.reset! - end - end - def current_command _, _, config = @_initializer config[:current_command] @@ -843,13 +767,10 @@ module Bundler return unless SharedHelpers.md5_available? - latest = Fetcher::CompactIndex. - new(nil, Source::Rubygems::Remote.new(Bundler::URI("https://rubygems.org")), nil). - send(:compact_index_client). - instance_variable_get(:@cache). - dependencies("bundler"). - map {|d| Gem::Version.new(d.first) }. - max + require_relative "vendored_uri" + remote = Source::Rubygems::Remote.new(Gem::URI("https://rubygems.org")) + cache_path = Bundler.user_cache.join("compact_index", remote.cache_slug) + latest = Bundler::CompactIndexClient.new(cache_path).latest_version("bundler") return unless latest current = Gem::Version.new(VERSION) @@ -883,12 +804,23 @@ module Bundler value = options[name] value = value.join(" ").to_s if option.type == :array + value = "'#{value}'" unless option.type == :boolean - Bundler::SharedHelpers.major_deprecation 2, + print_remembered_flag_deprecation(flag_name, name.tr("-", "_"), value) + end + + def print_remembered_flag_deprecation(flag_name, option_name, option_value) + message = "The `#{flag_name}` flag is deprecated because it relies on being " \ "remembered across bundler invocations, which bundler will no longer " \ - "do in future versions. Instead please use `bundle config set --local #{name.tr("-", "_")} " \ - "'#{value}'`, and stop using this flag" + "do in future versions. Instead please use `bundle config set #{option_name} " \ + "#{option_value}`, and stop using this flag" + removed_message = + "The `#{flag_name}` flag has been removed because it relied on being " \ + "remembered across bundler invocations, which bundler will no longer " \ + "do. Instead please use `bundle config set #{option_name} " \ + "#{option_value}`, and stop using this flag" + Bundler::SharedHelpers.major_deprecation 2, message, removed_message: removed_message end end end diff --git a/lib/bundler/cli/add.rb b/lib/bundler/cli/add.rb index 08fa6547fb..002d9e1d33 100644 --- a/lib/bundler/cli/add.rb +++ b/lib/bundler/cli/add.rb @@ -28,9 +28,9 @@ module Bundler dependencies = gems.map {|g| Bundler::Dependency.new(g, version, options) } Injector.inject(dependencies, - :conservative_versioning => options[:version].nil?, # Perform conservative versioning only when version is not specified - :optimistic => options[:optimistic], - :strict => options[:strict]) + conservative_versioning: options[:version].nil?, # Perform conservative versioning only when version is not specified + optimistic: options[:optimistic], + strict: options[:strict]) end def validate_options! diff --git a/lib/bundler/cli/binstubs.rb b/lib/bundler/cli/binstubs.rb index fc2fad47a5..8ce138df96 100644 --- a/lib/bundler/cli/binstubs.rb +++ b/lib/bundler/cli/binstubs.rb @@ -17,9 +17,9 @@ module Bundler installer = Installer.new(Bundler.root, Bundler.definition) installer_opts = { - :force => options[:force], - :binstubs_cmd => true, - :all_platforms => options["all-platforms"], + force: options[:force], + binstubs_cmd: true, + all_platforms: options["all-platforms"], } if options[:all] @@ -45,7 +45,7 @@ module Bundler next end - Bundler.settings.temporary(:path => (Bundler.settings[:path] || Bundler.root)) do + Bundler.settings.temporary(path: Bundler.settings[:path] || Bundler.root) do installer.generate_standalone_bundler_executable_stubs(spec, installer_opts) end else diff --git a/lib/bundler/cli/cache.rb b/lib/bundler/cli/cache.rb index c8698ed7e3..2e63a16ec3 100644 --- a/lib/bundler/cli/cache.rb +++ b/lib/bundler/cli/cache.rb @@ -19,7 +19,7 @@ module Bundler # TODO: move cache contents here now that all bundles are locked custom_path = Bundler.settings[:path] if options[:path] - Bundler.settings.temporary(:cache_all_platforms => options["all-platforms"]) do + Bundler.settings.temporary(cache_all_platforms: options["all-platforms"]) do Bundler.load.cache(custom_path) end end diff --git a/lib/bundler/cli/check.rb b/lib/bundler/cli/check.rb index cc1f37f0c3..33d31cdd27 100644 --- a/lib/bundler/cli/check.rb +++ b/lib/bundler/cli/check.rb @@ -29,10 +29,10 @@ module Bundler Bundler.ui.warn "Install missing gems with `bundle install`" exit 1 elsif !Bundler.default_lockfile.file? && Bundler.frozen_bundle? - Bundler.ui.error "This bundle has been frozen, but there is no #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)} present" + Bundler.ui.error "This bundle has been frozen, but there is no #{SharedHelpers.relative_lockfile_path} present" exit 1 else - Bundler.load.lock(:preserve_unknown_sections => true) unless options[:"dry-run"] + Bundler.load.lock(preserve_unknown_sections: true) unless options[:"dry-run"] Bundler.ui.info "The Gemfile's dependencies are satisfied" end end diff --git a/lib/bundler/cli/common.rb b/lib/bundler/cli/common.rb index d654406f65..7ef6deb2cf 100644 --- a/lib/bundler/cli/common.rb +++ b/lib/bundler/cli/common.rb @@ -54,9 +54,12 @@ module Bundler Bundler.definition.specs.each do |spec| return spec if spec.name == name - specs << spec if regexp && spec.name =~ regexp + specs << spec if regexp && spec.name.match?(regexp) end + default_spec = default_gem_spec(name) + specs << default_spec if default_spec + case specs.count when 0 dep_in_other_group = Bundler.definition.current_dependencies.find {|dep|dep.name == name } @@ -75,6 +78,11 @@ module Bundler raise GemNotFound, gem_not_found_message(name, Bundler.definition.dependencies) end + def self.default_gem_spec(name) + gem_spec = Gem::Specification.find_all_by_name(name).last + gem_spec if gem_spec&.default_gem? + end + def self.ask_for_spec_from(specs) specs.each_with_index do |spec, index| Bundler.ui.info "#{index.succ} : #{spec.name}", true diff --git a/lib/bundler/cli/config.rb b/lib/bundler/cli/config.rb index e1222c75dd..77b502fe60 100644 --- a/lib/bundler/cli/config.rb +++ b/lib/bundler/cli/config.rb @@ -2,17 +2,17 @@ module Bundler class CLI::Config < Thor - class_option :parseable, :type => :boolean, :banner => "Use minimal formatting for more parseable output" + class_option :parseable, type: :boolean, banner: "Use minimal formatting for more parseable output" def self.scope_options - method_option :global, :type => :boolean, :banner => "Only change the global config" - method_option :local, :type => :boolean, :banner => "Only change the local config" + method_option :global, type: :boolean, banner: "Only change the global config" + method_option :local, type: :boolean, banner: "Only change the local config" end private_class_method :scope_options - desc "base NAME [VALUE]", "The Bundler 1 config interface", :hide => true + desc "base NAME [VALUE]", "The Bundler 1 config interface", hide: true scope_options - method_option :delete, :type => :boolean, :banner => "delete" + method_option :delete, type: :boolean, banner: "delete" def base(name = nil, *value) new_args = if ARGV.size == 1 @@ -25,8 +25,9 @@ module Bundler ["config", "get", ARGV[1]] end - SharedHelpers.major_deprecation 3, - "Using the `config` command without a subcommand [list, get, set, unset] is deprecated and will be removed in the future. Use `bundle #{new_args.join(" ")}` instead." + message = "Using the `config` command without a subcommand [list, get, set, unset] is deprecated and will be removed in the future. Use `bundle #{new_args.join(" ")}` instead." + removed_message = "Using the `config` command without a subcommand [list, get, set, unset] is has been removed. Use `bundle #{new_args.join(" ")}` instead." + SharedHelpers.major_deprecation 3, message, removed_message: removed_message Base.new(options, name, value, self).run end diff --git a/lib/bundler/cli/console.rb b/lib/bundler/cli/console.rb index 1eb8ea8254..840cf14fd7 100644 --- a/lib/bundler/cli/console.rb +++ b/lib/bundler/cli/console.rb @@ -9,8 +9,9 @@ module Bundler end def run - Bundler::SharedHelpers.major_deprecation 2, "bundle console will be replaced " \ - "by `bin/console` generated by `bundle gem <name>`" + message = "bundle console will be replaced by `bin/console` generated by `bundle gem <name>`" + removed_message = "bundle console has been replaced by `bin/console` generated by `bundle gem <name>`" + Bundler::SharedHelpers.major_deprecation 2, message, removed_message: removed_message group ? Bundler.require(:default, *group.split(" ").map!(&:to_sym)) : Bundler.require ARGV.clear diff --git a/lib/bundler/cli/doctor.rb b/lib/bundler/cli/doctor.rb index e299a5a8c2..1f6fc93c16 100644 --- a/lib/bundler/cli/doctor.rb +++ b/lib/bundler/cli/doctor.rb @@ -6,8 +6,8 @@ require "fiddle" module Bundler class CLI::Doctor - DARWIN_REGEX = /\s+(.+) \(compatibility /.freeze - LDD_REGEX = /\t\S+ => (\S+) \(\S+\)/.freeze + DARWIN_REGEX = /\s+(.+) \(compatibility / + LDD_REGEX = /\t\S+ => (\S+) \(\S+\)/ attr_reader :options diff --git a/lib/bundler/cli/exec.rb b/lib/bundler/cli/exec.rb index 42b602a055..f81cd5d2c4 100644 --- a/lib/bundler/cli/exec.rb +++ b/lib/bundler/cli/exec.rb @@ -12,7 +12,7 @@ module Bundler @options = options @cmd = args.shift @args = args - @args << { :close_others => !options.keep_file_descriptors? } unless Bundler.current_ruby.jruby? + @args << { close_others: !options.keep_file_descriptors? } unless Bundler.current_ruby.jruby? end def run diff --git a/lib/bundler/cli/gem.rb b/lib/bundler/cli/gem.rb index 7f1200f4a0..b6571d0e86 100644 --- a/lib/bundler/cli/gem.rb +++ b/lib/bundler/cli/gem.rb @@ -11,7 +11,7 @@ module Bundler class CLI::Gem TEST_FRAMEWORK_VERSIONS = { "rspec" => "3.0", - "minitest" => "5.0", + "minitest" => "5.16", "test-unit" => "3.0", }.freeze @@ -32,7 +32,6 @@ module Bundler validate_ext_name if @extension validate_rust_builder_rubygems_version if @extension == "rust" - travis_removal_info end def run @@ -59,23 +58,23 @@ module Bundler end config = { - :name => name, - :underscored_name => underscored_name, - :namespaced_path => namespaced_path, - :makefile_path => "#{underscored_name}/#{underscored_name}", - :constant_name => constant_name, - :constant_array => constant_array, - :author => git_author_name.empty? ? "TODO: Write your name" : git_author_name, - :email => git_user_email.empty? ? "TODO: Write your email address" : git_user_email, - :test => options[:test], - :ext => extension, - :exe => options[:exe], - :bundler_version => bundler_dependency_version, - :git => use_git, - :github_username => github_username.empty? ? "[USERNAME]" : github_username, - :required_ruby_version => required_ruby_version, - :rust_builder_required_rubygems_version => rust_builder_required_rubygems_version, - :minitest_constant_name => minitest_constant_name, + name: name, + underscored_name: underscored_name, + namespaced_path: namespaced_path, + makefile_path: "#{underscored_name}/#{underscored_name}", + constant_name: constant_name, + constant_array: constant_array, + author: git_author_name.empty? ? "TODO: Write your name" : git_author_name, + email: git_user_email.empty? ? "TODO: Write your email address" : git_user_email, + test: options[:test], + ext: extension, + exe: options[:exe], + bundler_version: bundler_dependency_version, + git: use_git, + github_username: github_username.empty? ? "[USERNAME]" : github_username, + required_ruby_version: required_ruby_version, + rust_builder_required_rubygems_version: rust_builder_required_rubygems_version, + minitest_constant_name: minitest_constant_name, } ensure_safe_gem_name(name, constant_array) @@ -137,10 +136,13 @@ module Bundler case config[:ci] when "github" templates.merge!("github/workflows/main.yml.tt" => ".github/workflows/main.yml") + config[:ci_config_path] = ".github " when "gitlab" templates.merge!("gitlab-ci.yml.tt" => ".gitlab-ci.yml") + config[:ci_config_path] = ".gitlab-ci.yml " when "circle" templates.merge!("circleci/config.yml.tt" => ".circleci/config.yml") + config[:ci_config_path] = ".circleci " end if ask_and_set(:mit, "Do you want to license your code permissively under the MIT license?", @@ -233,9 +235,7 @@ module Bundler end if use_git - Dir.chdir(target) do - `git add .` - end + IO.popen(%w[git add .], { chdir: target }, &:read) end # Open gemspec in editor @@ -348,7 +348,7 @@ module Bundler Bundler.ui.confirm "Do you want to add a code linter and formatter to your gem? " \ "Supported Linters:\n" \ "* RuboCop: https://rubocop.org\n" \ - "* Standard: https://github.com/testdouble/standard\n" \ + "* Standard: https://github.com/standardrb/standard\n" \ "\n" Bundler.ui.info hint_text("linter") @@ -379,15 +379,20 @@ module Bundler def deprecated_rubocop_option if !options[:rubocop].nil? if options[:rubocop] - Bundler::SharedHelpers.major_deprecation 2, "--rubocop is deprecated, use --linter=rubocop" + Bundler::SharedHelpers.major_deprecation 2, + "--rubocop is deprecated, use --linter=rubocop", + removed_message: "--rubocop has been removed, use --linter=rubocop" "rubocop" else - Bundler::SharedHelpers.major_deprecation 2, "--no-rubocop is deprecated, use --linter" + Bundler::SharedHelpers.major_deprecation 2, + "--no-rubocop is deprecated, use --linter", + removed_message: "--no-rubocop has been removed, use --linter" false end elsif !Bundler.settings["gem.rubocop"].nil? Bundler::SharedHelpers.major_deprecation 2, - "config gem.rubocop is deprecated; we've updated your config to use gem.linter instead" + "config gem.rubocop is deprecated; we've updated your config to use gem.linter instead", + removed_message: "config gem.rubocop has been removed; we've updated your config to use gem.linter instead" Bundler.settings["gem.rubocop"] ? "rubocop" : false end end @@ -431,7 +436,7 @@ module Bundler end def required_ruby_version - "2.6.0" + "3.0.0" end def rubocop_version @@ -442,19 +447,6 @@ module Bundler "1.3" end - # TODO: remove at next minor release - def travis_removal_info - if options[:ci] == "travis" - Bundler.ui.error "Support for Travis CI was removed from gem skeleton generator." - exit 1 - end - - if Bundler.settings["gem.ci"] == "travis" - Bundler.ui.error "Support for Travis CI was removed from gem skeleton generator, but it is present in bundle config. Please configure another provider using `bundle config set gem.ci SERVICE` (where SERVICE is one of github/gitlab/circle) or unset configuration using `bundle config unset gem.ci`." - exit 1 - end - end - def validate_rust_builder_rubygems_version if Gem::Version.new(rust_builder_required_rubygems_version) > Gem.rubygems_version Bundler.ui.error "Your RubyGems version (#{Gem.rubygems_version}) is too old to build Rust extension. Please update your RubyGems using `gem update --system` or any other way and try again." diff --git a/lib/bundler/cli/info.rb b/lib/bundler/cli/info.rb index 36c7a58f12..8f34956aca 100644 --- a/lib/bundler/cli/info.rb +++ b/lib/bundler/cli/info.rb @@ -25,19 +25,8 @@ module Bundler private - def spec_for_gem(gem_name) - spec = Bundler.definition.specs.find {|s| s.name == gem_name } - spec || default_gem_spec(gem_name) || Bundler::CLI::Common.select_spec(gem_name, :regex_match) - end - - def default_gem_spec(gem_name) - return unless Gem::Specification.respond_to?(:find_all_by_name) - gem_spec = Gem::Specification.find_all_by_name(gem_name).last - return gem_spec if gem_spec&.default_gem? - end - - def spec_not_found(gem_name) - raise GemNotFound, Bundler::CLI::Common.gem_not_found_message(gem_name, Bundler.definition.dependencies) + def spec_for_gem(name) + Bundler::CLI::Common.select_spec(name, :regex_match) end def print_gem_version(spec) diff --git a/lib/bundler/cli/install.rb b/lib/bundler/cli/install.rb index c71bcf159f..a233d5d2e5 100644 --- a/lib/bundler/cli/install.rb +++ b/lib/bundler/cli/install.rb @@ -14,7 +14,7 @@ module Bundler Bundler.self_manager.install_locked_bundler_and_restart_with_it_if_needed - Bundler::SharedHelpers.set_env "RB_USER_INSTALL", "1" if Bundler::FREEBSD + Bundler::SharedHelpers.set_env "RB_USER_INSTALL", "1" if Gem.freebsd_platform? # Disable color in deployment mode Bundler.ui.shell = Thor::Shell::Basic.new if options[:deployment] @@ -28,8 +28,8 @@ module Bundler flag = "--deployment flag" if options[:deployment] flag ||= "--frozen flag" if options[:frozen] flag ||= "deployment setting" - raise ProductionError, "The #{flag} requires a #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)}. Please make " \ - "sure you have checked your #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)} into version control " \ + raise ProductionError, "The #{flag} requires a lockfile. Please make " \ + "sure you have checked your #{SharedHelpers.relative_lockfile_path} into version control " \ "before deploying." end @@ -51,7 +51,8 @@ module Bundler if options["binstubs"] Bundler::SharedHelpers.major_deprecation 2, - "The --binstubs option will be removed in favor of `bundle binstubs --all`" + "The --binstubs option will be removed in favor of `bundle binstubs --all`", + removed_message: "The --binstubs option have been removed in favor of `bundle binstubs --all`" end Plugin.gemfile_install(Bundler.default_gemfile) if Bundler.feature_flag.plugins? @@ -61,7 +62,7 @@ module Bundler installer = Installer.install(Bundler.root, definition, options) - Bundler.settings.temporary(:cache_all_platforms => options[:local] ? false : Bundler.settings[:cache_all_platforms]) do + Bundler.settings.temporary(cache_all_platforms: options[:local] ? false : Bundler.settings[:cache_all_platforms]) do Bundler.load.cache(nil, options[:local]) if Bundler.app_cache.exist? && !options["no-cache"] && !Bundler.frozen_bundle? end @@ -95,7 +96,7 @@ module Bundler def warn_if_root return if Bundler.settings[:silence_root_warning] || Gem.win_platform? || !Process.uid.zero? Bundler.ui.warn "Don't run Bundler as root. Installing your bundle as root " \ - "will break this application for all non-root users on this machine.", :wrap => true + "will break this application for all non-root users on this machine.", wrap: true end def dependencies_count_for(definition) @@ -148,7 +149,7 @@ module Bundler Bundler.settings.set_command_option_if_given :path, options[:path] if options["standalone"] && Bundler.settings[:path].nil? && !options["local"] - Bundler.settings.temporary(:path_relative_to_cwd => false) do + Bundler.settings.temporary(path_relative_to_cwd: false) do Bundler.settings.set_command_option :path, "bundle" end end diff --git a/lib/bundler/cli/lock.rb b/lib/bundler/cli/lock.rb index cb3ed27138..dac3d2a09a 100644 --- a/lib/bundler/cli/lock.rb +++ b/lib/bundler/cli/lock.rb @@ -26,42 +26,46 @@ module Bundler if update.is_a?(Array) # unlocking specific gems Bundler::CLI::Common.ensure_all_gems_in_lockfile!(update) - update = { :gems => update, :conservative => conservative } + update = { gems: update, conservative: conservative } elsif update && conservative - update = { :conservative => conservative } + update = { conservative: conservative } elsif update && bundler - update = { :bundler => bundler } + update = { bundler: bundler } end - definition = Bundler.definition(update) - Bundler::CLI::Common.configure_gem_version_promoter(Bundler.definition, options) if options[:update] + file = options[:lockfile] + file = file ? Pathname.new(file).expand_path : Bundler.default_lockfile - options["remove-platform"].each do |platform| - definition.remove_platform(platform) - end + Bundler.settings.temporary(frozen: false) do + definition = Bundler.definition(update, file) + + Bundler::CLI::Common.configure_gem_version_promoter(definition, options) if options[:update] - options["add-platform"].each do |platform_string| - platform = Gem::Platform.new(platform_string) - if platform.to_s == "unknown" - Bundler.ui.warn "The platform `#{platform_string}` is unknown to RubyGems " \ - "and adding it will likely lead to resolution errors" + options["remove-platform"].each do |platform| + definition.remove_platform(platform) end - definition.add_platform(platform) - end - if definition.platforms.empty? - raise InvalidOption, "Removing all platforms from the bundle is not allowed" - end + options["add-platform"].each do |platform_string| + platform = Gem::Platform.new(platform_string) + if platform.to_s == "unknown" + Bundler.ui.warn "The platform `#{platform_string}` is unknown to RubyGems " \ + "and adding it will likely lead to resolution errors" + end + definition.add_platform(platform) + end + + if definition.platforms.empty? + raise InvalidOption, "Removing all platforms from the bundle is not allowed" + end - definition.resolve_remotely! unless options[:local] + definition.resolve_remotely! unless options[:local] - if print - puts definition.to_lock - else - file = options[:lockfile] - file = file ? File.expand_path(file) : Bundler.default_lockfile - puts "Writing lockfile to #{file}" - definition.lock(file) + if print + puts definition.to_lock + else + puts "Writing lockfile to #{file}" + definition.lock + end end Bundler.ui.level = previous_ui_level diff --git a/lib/bundler/cli/open.rb b/lib/bundler/cli/open.rb index 8522ec92d6..f24693b843 100644 --- a/lib/bundler/cli/open.rb +++ b/lib/bundler/cli/open.rb @@ -18,13 +18,11 @@ module Bundler Bundler.ui.info "Unable to open #{name} because it's a default gem, so the directory it would normally be installed to does not exist." else root_path = spec.full_gem_path - Dir.chdir(root_path) do - require "shellwords" - command = Shellwords.split(editor) << File.join([root_path, path].compact) - Bundler.with_original_env do - system(*command) - end || Bundler.ui.info("Could not run '#{command.join(" ")}'") - end + require "shellwords" + command = Shellwords.split(editor) << File.join([root_path, path].compact) + Bundler.with_original_env do + system(*command, { chdir: root_path }) + end || Bundler.ui.info("Could not run '#{command.join(" ")}'") end end end diff --git a/lib/bundler/cli/outdated.rb b/lib/bundler/cli/outdated.rb index 68c701aefb..ec42e631bb 100644 --- a/lib/bundler/cli/outdated.rb +++ b/lib/bundler/cli/outdated.rb @@ -41,12 +41,12 @@ module Bundler # We're doing a full update Bundler.definition(true) else - Bundler.definition(:gems => gems, :sources => sources) + Bundler.definition(gems: gems, sources: sources) end Bundler::CLI::Common.configure_gem_version_promoter( Bundler.definition, - options.merge(:strict => @strict) + options.merge(strict: @strict) ) definition_resolution = proc do @@ -90,10 +90,10 @@ module Bundler end outdated_gems << { - :active_spec => active_spec, - :current_spec => current_spec, - :dependency => dependency, - :groups => groups, + active_spec: active_spec, + current_spec: current_spec, + dependency: dependency, + groups: groups, } end diff --git a/lib/bundler/cli/plugin.rb b/lib/bundler/cli/plugin.rb index fe3f4412fa..fd61ef0d95 100644 --- a/lib/bundler/cli/plugin.rb +++ b/lib/bundler/cli/plugin.rb @@ -5,20 +5,15 @@ module Bundler class CLI::Plugin < Thor desc "install PLUGINS", "Install the plugin from the source" long_desc <<-D - Install plugins either from the rubygems source provided (with --source option) or from a git source provided with --git (for remote repos) or --local_git (for local repos). If no sources are provided, it uses Gem.sources + Install plugins either from the rubygems source provided (with --source option), from a git source provided with --git, or a local path provided with --path. If no sources are provided, it uses Gem.sources D - method_option "source", :type => :string, :default => nil, :banner => - "URL of the RubyGems source to fetch the plugin from" - method_option "version", :type => :string, :default => nil, :banner => - "The version of the plugin to fetch" - method_option "git", :type => :string, :default => nil, :banner => - "URL of the git repo to fetch from" - method_option "local_git", :type => :string, :default => nil, :banner => - "Path of the local git repo to fetch from" - method_option "branch", :type => :string, :default => nil, :banner => - "The git branch to checkout" - method_option "ref", :type => :string, :default => nil, :banner => - "The git revision to check out" + method_option "source", type: :string, default: nil, banner: "URL of the RubyGems source to fetch the plugin from" + method_option "version", type: :string, default: nil, banner: "The version of the plugin to fetch" + method_option "git", type: :string, default: nil, banner: "URL of the git repo to fetch from" + method_option "local_git", type: :string, default: nil, banner: "Path of the local git repo to fetch from (deprecated)" + method_option "branch", type: :string, default: nil, banner: "The git branch to checkout" + method_option "ref", type: :string, default: nil, banner: "The git revision to check out" + method_option "path", type: :string, default: nil, banner: "Path of a local gem to directly use" def install(*plugins) Bundler::Plugin.install(plugins, options) end @@ -27,8 +22,7 @@ module Bundler long_desc <<-D Uninstall given list of plugins. To uninstall all the plugins, use -all option. D - method_option "all", :type => :boolean, :default => nil, :banner => - "Uninstall all the installed plugins. If no plugin is installed, then it does nothing." + method_option "all", type: :boolean, default: nil, banner: "Uninstall all the installed plugins. If no plugin is installed, then it does nothing." def uninstall(*plugins) Bundler::Plugin.uninstall(plugins, options) end diff --git a/lib/bundler/cli/pristine.rb b/lib/bundler/cli/pristine.rb index d6654f8053..e0d7452c44 100644 --- a/lib/bundler/cli/pristine.rb +++ b/lib/bundler/cli/pristine.rb @@ -12,40 +12,48 @@ module Bundler definition.validate_runtime! installer = Bundler::Installer.new(Bundler.root, definition) - Bundler.load.specs.each do |spec| - next if spec.name == "bundler" # Source::Rubygems doesn't install bundler - next if !@gems.empty? && !@gems.include?(spec.name) - - gem_name = "#{spec.name} (#{spec.version}#{spec.git_version})" - gem_name += " (#{spec.platform})" if !spec.platform.nil? && spec.platform != Gem::Platform::RUBY - - case source = spec.source - when Source::Rubygems - cached_gem = spec.cache_file - unless File.exist?(cached_gem) - Bundler.ui.error("Failed to pristine #{gem_name}. Cached gem #{cached_gem} does not exist.") + ProcessLock.lock do + installed_specs = definition.specs.reject do |spec| + next if spec.name == "bundler" # Source::Rubygems doesn't install bundler + next if !@gems.empty? && !@gems.include?(spec.name) + + gem_name = "#{spec.name} (#{spec.version}#{spec.git_version})" + gem_name += " (#{spec.platform})" if !spec.platform.nil? && spec.platform != Gem::Platform::RUBY + + case source = spec.source + when Source::Rubygems + cached_gem = spec.cache_file + unless File.exist?(cached_gem) + Bundler.ui.error("Failed to pristine #{gem_name}. Cached gem #{cached_gem} does not exist.") + next + end + + FileUtils.rm_rf spec.full_gem_path + when Source::Git + if source.local? + Bundler.ui.warn("Cannot pristine #{gem_name}. Gem is locally overridden.") + next + end + + source.remote! + if extension_cache_path = source.extension_cache_path(spec) + FileUtils.rm_rf extension_cache_path + end + FileUtils.rm_rf spec.extension_dir + FileUtils.rm_rf spec.full_gem_path + else + Bundler.ui.warn("Cannot pristine #{gem_name}. Gem is sourced from local path.") next end - FileUtils.rm_rf spec.full_gem_path - when Source::Git - if source.local? - Bundler.ui.warn("Cannot pristine #{gem_name}. Gem is locally overridden.") - next - end + true + end.map(&:name) - source.remote! - if extension_cache_path = source.extension_cache_path(spec) - FileUtils.rm_rf extension_cache_path - end - FileUtils.rm_rf spec.extension_dir - FileUtils.rm_rf spec.full_gem_path - else - Bundler.ui.warn("Cannot pristine #{gem_name}. Gem is sourced from local path.") - next - end - - Bundler::GemInstaller.new(spec, installer, false, 0, true).install_from_spec + jobs = installer.send(:installation_parallelization, {}) + pristine_count = definition.specs.count - installed_specs.count + # allow a pristining a single gem to skip the parallel worker + jobs = [jobs, pristine_count].min + ParallelInstaller.call(installer, definition.specs, jobs, false, true, skip: installed_specs) end end end diff --git a/lib/bundler/cli/update.rb b/lib/bundler/cli/update.rb index b49182655b..985e8db051 100644 --- a/lib/bundler/cli/update.rb +++ b/lib/bundler/cli/update.rb @@ -35,7 +35,7 @@ module Bundler if full_update if conservative - Bundler.definition(:conservative => conservative) + Bundler.definition(conservative: conservative) else Bundler.definition(true) end @@ -51,9 +51,9 @@ module Bundler gems.concat(deps.map(&:name)) end - Bundler.definition(:gems => gems, :sources => sources, :ruby => options[:ruby], - :conservative => conservative, - :bundler => update_bundler) + Bundler.definition(gems: gems, sources: sources, ruby: options[:ruby], + conservative: conservative, + bundler: update_bundler) end Bundler::CLI::Common.configure_gem_version_promoter(Bundler.definition, options) @@ -63,6 +63,7 @@ module Bundler opts = options.dup opts["update"] = true opts["local"] = options[:local] + opts["force"] = options[:redownload] Bundler.settings.set_command_option_if_given :jobs, opts["jobs"] @@ -70,7 +71,7 @@ module Bundler if locked_gems = Bundler.definition.locked_gems previous_locked_info = locked_gems.specs.reduce({}) do |h, s| - h[s.name] = { :spec => s, :version => s.version, :source => s.source.identifier } + h[s.name] = { spec: s, version: s.version, source: s.source.identifier } h end end diff --git a/lib/bundler/compact_index_client.rb b/lib/bundler/compact_index_client.rb index 127a50e810..2bef0a1afc 100644 --- a/lib/bundler/compact_index_client.rb +++ b/lib/bundler/compact_index_client.rb @@ -5,7 +5,13 @@ require "set" module Bundler class CompactIndexClient + # NOTE: MD5 is here not because we expect a server to respond with it, but + # because we use it to generate the etag on first request during the upgrade + # to the compact index client that uses opaque etags saved to files. + # Remove once 2.5.0 has been out for a while. + SUPPORTED_DIGESTS = { "sha-256" => :SHA256, "md5" => :MD5 }.freeze DEBUG_MUTEX = Thread::Mutex.new + def self.debug return unless ENV["DEBUG_COMPACT_INDEX"] DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") } @@ -14,24 +20,18 @@ module Bundler class Error < StandardError; end require_relative "compact_index_client/cache" + require_relative "compact_index_client/cache_file" + require_relative "compact_index_client/parser" require_relative "compact_index_client/updater" - attr_reader :directory - - def initialize(directory, fetcher) - @directory = Pathname.new(directory) - @updater = Updater.new(fetcher) - @cache = Cache.new(@directory) - @endpoints = Set.new - @info_checksums_by_name = {} - @parsed_checksums = false - @mutex = Thread::Mutex.new + def initialize(directory, fetcher = nil) + @cache = Cache.new(directory, fetcher) + @parser = Parser.new(@cache) end def execution_mode=(block) Bundler::CompactIndexClient.debug { "execution_mode=" } - @endpoints = Set.new - + @cache.reset! @execution_mode = block end @@ -53,67 +53,28 @@ module Bundler end def names - Bundler::CompactIndexClient.debug { "/names" } - update(@cache.names_path, "names") - @cache.names + Bundler::CompactIndexClient.debug { "names" } + @parser.names end def versions - Bundler::CompactIndexClient.debug { "/versions" } - update(@cache.versions_path, "versions") - versions, @info_checksums_by_name = @cache.versions - versions + Bundler::CompactIndexClient.debug { "versions" } + @parser.versions end def dependencies(names) Bundler::CompactIndexClient.debug { "dependencies(#{names})" } - execution_mode.call(names) do |name| - update_info(name) - @cache.dependencies(name).map {|d| d.unshift(name) } - end.flatten(1) - end - - def update_and_parse_checksums! - Bundler::CompactIndexClient.debug { "update_and_parse_checksums!" } - return @info_checksums_by_name if @parsed_checksums - update(@cache.versions_path, "versions") - @info_checksums_by_name = @cache.checksums - @parsed_checksums = true - end - - private - - def update(local_path, remote_path) - Bundler::CompactIndexClient.debug { "update(#{local_path}, #{remote_path})" } - unless synchronize { @endpoints.add?(remote_path) } - Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" } - return - end - @updater.update(local_path, url(remote_path)) - end - - def update_info(name) - Bundler::CompactIndexClient.debug { "update_info(#{name})" } - path = @cache.info_path(name) - checksum = @updater.checksum_for_file(path) - unless existing = @info_checksums_by_name[name] - Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since it is missing from versions" } - return - end - if checksum == existing - Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since the versions checksum matches the local checksum" } - return - end - Bundler::CompactIndexClient.debug { "updating info for #{name} since the versions checksum #{existing} != the local checksum #{checksum}" } - update(path, "info/#{name}") + execution_mode.call(names) {|name| @parser.info(name) }.flatten(1) end - def url(path) - path + def latest_version(name) + Bundler::CompactIndexClient.debug { "latest_version(#{name})" } + @parser.info(name).map {|d| Gem::Version.new(d[1]) }.max end - def synchronize - @mutex.synchronize { yield } + def available? + Bundler::CompactIndexClient.debug { "available?" } + @parser.available? end end end diff --git a/lib/bundler/compact_index_client/cache.rb b/lib/bundler/compact_index_client/cache.rb index 0b43581c11..bedd7f8028 100644 --- a/lib/bundler/compact_index_client/cache.rb +++ b/lib/bundler/compact_index_client/cache.rb @@ -7,94 +7,89 @@ module Bundler class Cache attr_reader :directory - def initialize(directory) + def initialize(directory, fetcher = nil) @directory = Pathname.new(directory).expand_path - info_roots.each do |dir| - SharedHelpers.filesystem_access(dir) do - FileUtils.mkdir_p(dir) - end - end - end + @updater = Updater.new(fetcher) if fetcher + @mutex = Thread::Mutex.new + @endpoints = Set.new - def names - lines(names_path) + @info_root = mkdir("info") + @special_characters_info_root = mkdir("info-special-characters") + @info_etag_root = mkdir("info-etags") end - def names_path - directory.join("names") + def names + fetch("names", names_path, names_etag_path) end def versions - versions_by_name = Hash.new {|hash, key| hash[key] = [] } - info_checksums_by_name = {} - - lines(versions_path).each do |line| - name, versions_string, info_checksum = line.split(" ", 3) - info_checksums_by_name[name] = info_checksum || "" - versions_string.split(",").each do |version| - if version.start_with?("-") - version = version[1..-1].split("-", 2).unshift(name) - versions_by_name[name].delete(version) - else - version = version.split("-", 2).unshift(name) - versions_by_name[name] << version - end - end - end - - [versions_by_name, info_checksums_by_name] - end - - def versions_path - directory.join("versions") + fetch("versions", versions_path, versions_etag_path) end - def checksums - checksums = {} + def info(name, remote_checksum = nil) + path = info_path(name) - lines(versions_path).each do |line| - name, _, checksum = line.split(" ", 3) - checksums[name] = checksum + if remote_checksum && remote_checksum != SharedHelpers.checksum_for_file(path, :MD5) + fetch("info/#{name}", path, info_etag_path(name)) + else + Bundler::CompactIndexClient.debug { "update skipped info/#{name} (#{remote_checksum ? "versions index checksum is nil" : "versions index checksum matches local"})" } + read(path) end - - checksums end - def dependencies(name) - lines(info_path(name)).map do |line| - parse_gem(line) - end + def reset! + @mutex.synchronize { @endpoints.clear } end + private + + def names_path = directory.join("names") + def names_etag_path = directory.join("names.etag") + def versions_path = directory.join("versions") + def versions_etag_path = directory.join("versions.etag") + def info_path(name) name = name.to_s + # TODO: converge this into the info_root by hashing all filenames like info_etag_path if /[^a-z0-9_-]/.match?(name) name += "-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}" - info_roots.last.join(name) + @special_characters_info_root.join(name) else - info_roots.first.join(name) + @info_root.join(name) end end - private + def info_etag_path(name) + name = name.to_s + @info_etag_root.join("#{name}-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}") + end + + def mkdir(name) + directory.join(name).tap do |dir| + SharedHelpers.filesystem_access(dir) do + FileUtils.mkdir_p(dir) + end + end + end + + def fetch(remote_path, path, etag_path) + if already_fetched?(remote_path) + Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" } + else + Bundler::CompactIndexClient.debug { "fetching #{remote_path}" } + @updater&.update(remote_path, path, etag_path) + end - def lines(path) - return [] unless path.file? - lines = SharedHelpers.filesystem_access(path, :read, &:read).split("\n") - header = lines.index("---") - header ? lines[header + 1..-1] : lines + read(path) end - def parse_gem(line) - @dependency_parser ||= GemParser.new - @dependency_parser.parse(line) + def already_fetched?(remote_path) + @mutex.synchronize { !@endpoints.add?(remote_path) } end - def info_roots - [ - directory.join("info"), - directory.join("info-special-characters"), - ] + def read(path) + return unless path.file? + SharedHelpers.filesystem_access(path, :read, &:read) end end end diff --git a/lib/bundler/compact_index_client/cache_file.rb b/lib/bundler/compact_index_client/cache_file.rb new file mode 100644 index 0000000000..299d683438 --- /dev/null +++ b/lib/bundler/compact_index_client/cache_file.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +require_relative "../vendored_fileutils" +require "rubygems/package" + +module Bundler + class CompactIndexClient + # write cache files in a way that is robust to concurrent modifications + # if digests are given, the checksums will be verified + class CacheFile + DEFAULT_FILE_MODE = 0o644 + private_constant :DEFAULT_FILE_MODE + + class Error < RuntimeError; end + class ClosedError < Error; end + + class DigestMismatchError < Error + def initialize(digests, expected_digests) + super "Calculated checksums #{digests.inspect} did not match expected #{expected_digests.inspect}." + end + end + + # Initialize with a copy of the original file, then yield the instance. + def self.copy(path, &block) + new(path) do |file| + file.initialize_digests + + SharedHelpers.filesystem_access(path, :read) do + path.open("rb") do |s| + file.open {|f| IO.copy_stream(s, f) } + end + end + + yield file + end + end + + # Write data to a temp file, then replace the original file with it verifying the digests if given. + def self.write(path, data, digests = nil) + return unless data + new(path) do |file| + file.digests = digests + file.write(data) + end + end + + attr_reader :original_path, :path + + def initialize(original_path, &block) + @original_path = original_path + @perm = original_path.file? ? original_path.stat.mode : DEFAULT_FILE_MODE + @path = original_path.sub(/$/, ".#{$$}.tmp") + return unless block_given? + begin + yield self + ensure + close + end + end + + def size + path.size + end + + # initialize the digests using CompactIndexClient::SUPPORTED_DIGESTS, or a subset based on keys. + def initialize_digests(keys = nil) + @digests = keys ? SUPPORTED_DIGESTS.slice(*keys) : SUPPORTED_DIGESTS.dup + @digests.transform_values! {|algo_class| SharedHelpers.digest(algo_class).new } + end + + # reset the digests so they don't contain any previously read data + def reset_digests + @digests&.each_value(&:reset) + end + + # set the digests that will be verified at the end + def digests=(expected_digests) + @expected_digests = expected_digests + + if @expected_digests.nil? + @digests = nil + elsif @digests + @digests = @digests.slice(*@expected_digests.keys) + else + initialize_digests(@expected_digests.keys) + end + end + + def digests? + @digests&.any? + end + + # Open the temp file for writing, reusing original permissions, yielding the IO object. + def open(write_mode = "wb", perm = @perm, &block) + raise ClosedError, "Cannot reopen closed file" if @closed + SharedHelpers.filesystem_access(path, :write) do + path.open(write_mode, perm) do |f| + yield digests? ? Gem::Package::DigestIO.new(f, @digests) : f + end + end + end + + # Returns false without appending when no digests since appending is too error prone to do without digests. + def append(data) + return false unless digests? + open("a") {|f| f.write data } + verify && commit + end + + def write(data) + reset_digests + open {|f| f.write data } + commit! + end + + def commit! + verify || raise(DigestMismatchError.new(@base64digests, @expected_digests)) + commit + end + + # Verify the digests, returning true on match, false on mismatch. + def verify + return true unless @expected_digests && digests? + @base64digests = @digests.transform_values!(&:base64digest) + @digests = nil + @base64digests.all? {|algo, digest| @expected_digests[algo] == digest } + end + + # Replace the original file with the temp file without verifying digests. + # The file is permanently closed. + def commit + raise ClosedError, "Cannot commit closed file" if @closed + SharedHelpers.filesystem_access(original_path, :write) do + FileUtils.mv(path, original_path) + end + @closed = true + end + + # Remove the temp file without replacing the original file. + # The file is permanently closed. + def close + return if @closed + FileUtils.remove_file(path) if @path&.file? + @closed = true + end + end + end +end diff --git a/lib/bundler/compact_index_client/gem_parser.rb b/lib/bundler/compact_index_client/gem_parser.rb index e7bf4c6001..60a1817607 100644 --- a/lib/bundler/compact_index_client/gem_parser.rb +++ b/lib/bundler/compact_index_client/gem_parser.rb @@ -6,12 +6,15 @@ module Bundler GemParser = Gem::Resolver::APISet::GemParser else class GemParser + EMPTY_ARRAY = [].freeze + private_constant :EMPTY_ARRAY + def parse(line) version_and_platform, rest = line.split(" ", 2) version, platform = version_and_platform.split("-", 2) - dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest - dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : [] - requirements = requirements ? requirements.map {|d| parse_dependency(d) } : [] + dependencies, requirements = rest.split("|", 2).map! {|s| s.split(",") } if rest + dependencies = dependencies ? dependencies.map! {|d| parse_dependency(d) } : EMPTY_ARRAY + requirements = requirements ? requirements.map! {|d| parse_dependency(d) } : EMPTY_ARRAY [version, platform, dependencies, requirements] end @@ -20,6 +23,7 @@ module Bundler def parse_dependency(string) dependency = string.split(":") dependency[-1] = dependency[-1].split("&") if dependency.size > 1 + dependency[0] = -dependency[0] dependency end end diff --git a/lib/bundler/compact_index_client/parser.rb b/lib/bundler/compact_index_client/parser.rb new file mode 100644 index 0000000000..a227bc2cfd --- /dev/null +++ b/lib/bundler/compact_index_client/parser.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module Bundler + class CompactIndexClient + class Parser + # `compact_index` - an object responding to #names, #versions, #info(name, checksum), + # returning the file contents as a string + def initialize(compact_index) + @compact_index = compact_index + @info_checksums = nil + @versions_by_name = nil + @available = nil + end + + def names + lines(@compact_index.names) + end + + def versions + @versions_by_name ||= Hash.new {|hash, key| hash[key] = [] } + @info_checksums = {} + + lines(@compact_index.versions).each do |line| + name, versions_string, checksum = line.split(" ", 3) + @info_checksums[name] = checksum || "" + versions_string.split(",") do |version| + delete = version.delete_prefix!("-") + version = version.split("-", 2).unshift(name) + if delete + @versions_by_name[name].delete(version) + else + @versions_by_name[name] << version + end + end + end + + @versions_by_name + end + + def info(name) + data = @compact_index.info(name, info_checksums[name]) + lines(data).map {|line| gem_parser.parse(line).unshift(name) } + end + + def available? + return @available unless @available.nil? + @available = !info_checksums.empty? + end + + private + + def info_checksums + @info_checksums ||= lines(@compact_index.versions).each_with_object({}) do |line, checksums| + parse_version_checksum(line, checksums) + end + end + + def lines(data) + return [] if data.nil? || data.empty? + lines = data.split("\n") + header = lines.index("---") + header ? lines[header + 1..-1] : lines + end + + def gem_parser + @gem_parser ||= GemParser.new + end + + # This is mostly the same as `split(" ", 3)` but it avoids allocating extra objects. + # This method gets called at least once for every gem when parsing versions. + def parse_version_checksum(line, checksums) + line.freeze # allows slicing into the string to not allocate a copy of the line + name_end = line.index(" ") + checksum_start = line.index(" ", name_end + 1) + 1 + checksum_end = line.size - checksum_start + # freeze name since it is used as a hash key + # pre-freezing means a frozen copy isn't created + name = line[0, name_end].freeze + checksum = line[checksum_start, checksum_end] + checksums[name] = checksum + end + end + end +end diff --git a/lib/bundler/compact_index_client/updater.rb b/lib/bundler/compact_index_client/updater.rb index 0f7bf9bb50..88c7146900 100644 --- a/lib/bundler/compact_index_client/updater.rb +++ b/lib/bundler/compact_index_client/updater.rb @@ -1,20 +1,11 @@ # frozen_string_literal: true -require_relative "../vendored_fileutils" - module Bundler class CompactIndexClient class Updater - class MisMatchedChecksumError < Error - def initialize(path, server_checksum, local_checksum) - @path = path - @server_checksum = server_checksum - @local_checksum = local_checksum - end - - def message - "The checksum of /#{@path} does not match the checksum provided by the server! Something is wrong " \ - "(local checksum is #{@local_checksum.inspect}, was expecting #{@server_checksum.inspect})." + class MismatchedChecksumError < Error + def initialize(path, message) + super "The checksum of /#{path} does not match the checksum provided by the server! Something is wrong. #{message}" end end @@ -22,95 +13,91 @@ module Bundler @fetcher = fetcher end - def update(local_path, remote_path, retrying = nil) - headers = {} - - local_temp_path = local_path.sub(/$/, ".#{$$}") - local_temp_path = local_temp_path.sub(/$/, ".retrying") if retrying - local_temp_path = local_temp_path.sub(/$/, ".tmp") - - # first try to fetch any new bytes on the existing file - if retrying.nil? && local_path.file? - copy_file local_path, local_temp_path + def update(remote_path, local_path, etag_path) + append(remote_path, local_path, etag_path) || replace(remote_path, local_path, etag_path) + rescue CacheFile::DigestMismatchError => e + raise MismatchedChecksumError.new(remote_path, e.message) + rescue Zlib::GzipFile::Error + raise Bundler::HTTPError + end - headers["If-None-Match"] = etag_for(local_temp_path) - headers["Range"] = - if local_temp_path.size.nonzero? - # Subtract a byte to ensure the range won't be empty. - # Avoids 416 (Range Not Satisfiable) responses. - "bytes=#{local_temp_path.size - 1}-" - else - "bytes=#{local_temp_path.size}-" - end - end + private - response = @fetcher.call(remote_path, headers) - return nil if response.is_a?(Net::HTTPNotModified) + def append(remote_path, local_path, etag_path) + return false unless local_path.file? && local_path.size.nonzero? - content = response.body + CacheFile.copy(local_path) do |file| + etag = etag_path.read.tap(&:chomp!) if etag_path.file? - etag = (response["ETag"] || "").gsub(%r{\AW/}, "") - correct_response = SharedHelpers.filesystem_access(local_temp_path) do - if response.is_a?(Net::HTTPPartialContent) && local_temp_path.size.nonzero? - local_temp_path.open("a") {|f| f << slice_body(content, 1..-1) } + # Subtract a byte to ensure the range won't be empty. + # Avoids 416 (Range Not Satisfiable) responses. + response = @fetcher.call(remote_path, request_headers(etag, file.size - 1)) + break true if response.is_a?(Gem::Net::HTTPNotModified) - etag_for(local_temp_path) == etag + file.digests = parse_digests(response) + # server may ignore Range and return the full response + if response.is_a?(Gem::Net::HTTPPartialContent) + break false unless file.append(response.body.byteslice(1..-1)) else - local_temp_path.open("wb") {|f| f << content } - - etag.length.zero? || etag_for(local_temp_path) == etag + file.write(response.body) end + CacheFile.write(etag_path, etag_from_response(response)) + true end + end - if correct_response - SharedHelpers.filesystem_access(local_path) do - FileUtils.mv(local_temp_path, local_path) - end - return nil - end - - if retrying - raise MisMatchedChecksumError.new(remote_path, etag, etag_for(local_temp_path)) - end + # request without range header to get the full file or a 304 Not Modified + def replace(remote_path, local_path, etag_path) + etag = etag_path.read.tap(&:chomp!) if etag_path.file? + response = @fetcher.call(remote_path, request_headers(etag)) + return true if response.is_a?(Gem::Net::HTTPNotModified) + CacheFile.write(local_path, response.body, parse_digests(response)) + CacheFile.write(etag_path, etag_from_response(response)) + end - update(local_path, remote_path, :retrying) - rescue Zlib::GzipFile::Error - raise Bundler::HTTPError - ensure - FileUtils.remove_file(local_temp_path) if File.exist?(local_temp_path) + def request_headers(etag, range_start = nil) + headers = {} + headers["Range"] = "bytes=#{range_start}-" if range_start + headers["If-None-Match"] = %("#{etag}") if etag + headers end - def etag_for(path) - sum = checksum_for_file(path) - sum ? %("#{sum}") : nil + def etag_for_request(etag_path) + etag_path.read.tap(&:chomp!) if etag_path.file? end - def slice_body(body, range) - body.byteslice(range) + def etag_from_response(response) + return unless response["ETag"] + etag = response["ETag"].delete_prefix("W/") + return if etag.delete_prefix!('"') && !etag.delete_suffix!('"') + etag end - def checksum_for_file(path) - return nil unless path.file? - # This must use File.read instead of Digest.file().hexdigest - # because we need to preserve \n line endings on windows when calculating - # the checksum - SharedHelpers.filesystem_access(path, :read) do - SharedHelpers.digest(:MD5).hexdigest(File.read(path)) + # Unwraps and returns a Hash of digest algorithms and base64 values + # according to RFC 8941 Structured Field Values for HTTP. + # https://www.rfc-editor.org/rfc/rfc8941#name-parsing-a-byte-sequence + # Ignores unsupported algorithms. + def parse_digests(response) + return unless header = response["Repr-Digest"] || response["Digest"] + digests = {} + header.split(",") do |param| + algorithm, value = param.split("=", 2) + algorithm.strip! + algorithm.downcase! + next unless SUPPORTED_DIGESTS.key?(algorithm) + next unless value = byte_sequence(value) + digests[algorithm] = value end + digests.empty? ? nil : digests end - private - - def copy_file(source, dest) - SharedHelpers.filesystem_access(source, :read) do - File.open(source, "r") do |s| - SharedHelpers.filesystem_access(dest, :write) do - File.open(dest, "wb", s.stat.mode) do |f| - IO.copy_stream(s, f) - end - end - end - end + # Unwrap surrounding colons (byte sequence) + # The wrapping characters must be matched or we return nil. + # Also handles quotes because right now rubygems.org sends them. + def byte_sequence(value) + return if value.delete_prefix!(":") && !value.delete_suffix!(":") + return if value.delete_prefix!('"') && !value.delete_suffix!('"') + value end end end diff --git a/lib/bundler/constants.rb b/lib/bundler/constants.rb index 8dd8a53815..9564771e78 100644 --- a/lib/bundler/constants.rb +++ b/lib/bundler/constants.rb @@ -1,7 +1,14 @@ # frozen_string_literal: true +require "rbconfig" + module Bundler WINDOWS = RbConfig::CONFIG["host_os"] =~ /(msdos|mswin|djgpp|mingw)/ + deprecate_constant :WINDOWS + FREEBSD = RbConfig::CONFIG["host_os"].to_s.include?("bsd") - NULL = WINDOWS ? "NUL" : "/dev/null" + deprecate_constant :FREEBSD + + NULL = File::NULL + deprecate_constant :NULL end diff --git a/lib/bundler/current_ruby.rb b/lib/bundler/current_ruby.rb index 1d2d8ff2fb..93e0c401c0 100644 --- a/lib/bundler/current_ruby.rb +++ b/lib/bundler/current_ruby.rb @@ -43,7 +43,7 @@ module Bundler ].freeze def ruby? - return true if Bundler::GemHelpers.generic_local_platform == Gem::Platform::RUBY + return true if Bundler::GemHelpers.generic_local_platform_is_ruby? !windows? && (RUBY_ENGINE == "ruby" || RUBY_ENGINE == "rbx" || RUBY_ENGINE == "maglev" || RUBY_ENGINE == "truffleruby") end diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb index 1444cc2b0a..6cf1f9a255 100644 --- a/lib/bundler/definition.rb +++ b/lib/bundler/definition.rb @@ -18,7 +18,8 @@ module Bundler :platforms, :ruby_version, :lockfile, - :gemfiles + :gemfiles, + :locked_checksums ) # Given a gemfile and lockfile creates a Bundler definition @@ -68,7 +69,6 @@ module Bundler @sources = sources @unlock = unlock @optional_groups = optional_groups - @remote = false @prefer_local = false @specs = nil @ruby_version = ruby_version @@ -76,22 +76,27 @@ module Bundler @lockfile = lockfile @lockfile_contents = String.new + @locked_bundler_version = nil - @locked_ruby_version = nil + @resolved_bundler_version = nil + + @locked_ruby_version = nil @new_platform = nil @removed_platform = nil - if lockfile && File.exist?(lockfile) + if lockfile_exists? @lockfile_contents = Bundler.read_file(lockfile) @locked_gems = LockfileParser.new(@lockfile_contents) @locked_platforms = @locked_gems.platforms @platforms = @locked_platforms.dup @locked_bundler_version = @locked_gems.bundler_version @locked_ruby_version = @locked_gems.ruby_version + @originally_locked_deps = @locked_gems.dependencies @originally_locked_specs = SpecSet.new(@locked_gems.specs) + @locked_checksums = @locked_gems.checksums if unlock != true - @locked_deps = @locked_gems.dependencies + @locked_deps = @originally_locked_deps @locked_specs = @originally_locked_specs @locked_sources = @locked_gems.sources else @@ -106,9 +111,11 @@ module Bundler @locked_gems = nil @locked_deps = {} @locked_specs = SpecSet.new([]) + @originally_locked_deps = {} @originally_locked_specs = @locked_specs @locked_sources = [] @locked_platforms = [] + @locked_checksums = nil end locked_gem_sources = @locked_sources.select {|s| s.is_a?(Source::Rubygems) } @@ -124,7 +131,7 @@ module Bundler @sources.merged_gem_lockfile_sections!(locked_gem_sources.first) end - @unlock[:sources] ||= [] + @sources_to_unlock = @unlock.delete(:sources) || [] @unlock[:ruby] ||= if @ruby_version && locked_ruby_version_object @ruby_version.diff(locked_ruby_version_object) end @@ -136,17 +143,19 @@ module Bundler @path_changes = converge_paths @source_changes = converge_sources + @explicit_unlocks = @unlock.delete(:gems) || [] + if @unlock[:conservative] - @unlock[:gems] ||= @dependencies.map(&:name) + @gems_to_unlock = @explicit_unlocks.any? ? @explicit_unlocks : @dependencies.map(&:name) else - eager_unlock = (@unlock[:gems] || []).map {|name| Dependency.new(name, ">= 0") } - @unlock[:gems] = @locked_specs.for(eager_unlock, false, platforms).map(&:name).uniq + eager_unlock = @explicit_unlocks.map {|name| Dependency.new(name, ">= 0") } + @gems_to_unlock = @locked_specs.for(eager_unlock, false, platforms).map(&:name).uniq end @dependency_changes = converge_dependencies @local_changes = converge_locals - @incomplete_lockfile = check_missing_lockfile_specs + check_lockfile end def gem_version_promoter @@ -154,37 +163,24 @@ module Bundler end def resolve_only_locally! - @remote = false sources.local_only! resolve end def resolve_with_cache! + sources.local! sources.cached! resolve end def resolve_remotely! - @remote = true + sources.cached! sources.remote! resolve end - def resolution_mode=(options) - if options["local"] - @remote = false - else - @remote = true - @prefer_local = options["prefer-local"] - end - end - - def setup_sources_for_resolve - if @remote == false - sources.cached! - else - sources.remote! - end + def prefer_local! + @prefer_local = true end # For given dependency list returns a SpecSet with Gemspec of all the required @@ -217,8 +213,8 @@ module Bundler rescue BundlerError => e @resolve = nil @resolver = nil + @resolution_packages = nil @specs = nil - @gem_version_promoter = nil Bundler.ui.debug "The definition is missing dependencies, failed to resolve & materialize locally (#{e})" true @@ -233,8 +229,17 @@ module Bundler end def current_dependencies + filter_relevant(dependencies) + end + + def current_locked_dependencies + filter_relevant(locked_dependencies) + end + + def filter_relevant(dependencies) + platforms_array = [generic_local_platform].freeze dependencies.select do |d| - d.should_include? && !d.gem_platforms([generic_local_platform]).empty? + d.should_include? && !d.gem_platforms(platforms_array).empty? end end @@ -258,9 +263,15 @@ module Bundler def dependencies_for(groups) groups.map!(&:to_sym) - current_dependencies.reject do |d| - (d.groups & groups).empty? + deps = current_dependencies # always returns a new array + deps.select! do |d| + if RUBY_VERSION >= "3.1" + d.groups.intersect?(groups) + else + !(d.groups & groups).empty? + end end + deps end # Resolve all the dependencies specified in Gemfile. It ensures that @@ -272,7 +283,7 @@ module Bundler @resolve ||= if Bundler.frozen_bundle? Bundler.ui.debug "Frozen, using resolution from the lockfile" @locked_specs - elsif !unlocking? && nothing_changed? + elsif no_resolve_needed? if deleted_deps.any? Bundler.ui.debug "Some dependencies were deleted, using a subset of the resolution from the lockfile" SpecSet.new(filter_specs(@locked_specs, @dependencies - deleted_deps)) @@ -285,7 +296,12 @@ module Bundler end end else - Bundler.ui.debug "Found changes from the lockfile, re-resolving dependencies because #{change_reason}" + if lockfile_exists? + Bundler.ui.debug "Found changes from the lockfile, re-resolving dependencies because #{change_reason}" + else + Bundler.ui.debug "Resolving dependencies because there's no lockfile" + end + start_resolution end end @@ -298,34 +314,26 @@ module Bundler dependencies.map(&:groups).flatten.uniq end - def lock(file, preserve_unknown_sections = false) - return if Definition.no_lock - - contents = to_lock - - # Convert to \r\n if the existing lock has them - # i.e., Windows with `git config core.autocrlf=true` - contents.gsub!(/\n/, "\r\n") if @lockfile_contents.match?("\r\n") - - if @locked_bundler_version - locked_major = @locked_bundler_version.segments.first - current_major = Bundler.gem_version.segments.first - - updating_major = locked_major < current_major - end + def lock(file_or_preserve_unknown_sections = false, preserve_unknown_sections_or_unused = false) + if [true, false, nil].include?(file_or_preserve_unknown_sections) + target_lockfile = lockfile || Bundler.default_lockfile + preserve_unknown_sections = file_or_preserve_unknown_sections + else + target_lockfile = file_or_preserve_unknown_sections + preserve_unknown_sections = preserve_unknown_sections_or_unused - preserve_unknown_sections ||= !updating_major && (Bundler.frozen_bundle? || !(unlocking? || @unlocking_bundler)) + suggestion = if target_lockfile == lockfile + "To fix this warning, remove it from the `Definition#lock` call." + else + "Instead, instantiate a new definition passing `#{target_lockfile}`, and call `lock` without a file argument on that definition" + end - return if file && File.exist?(file) && lockfiles_equal?(@lockfile_contents, contents, preserve_unknown_sections) + msg = "`Definition#lock` was passed a target file argument. #{suggestion}" - if Bundler.frozen_bundle? - Bundler.ui.error "Cannot write a changed lockfile while frozen." - return + Bundler::SharedHelpers.major_deprecation 2, msg end - SharedHelpers.filesystem_access(file) do |p| - File.open(p, "wb") {|f| f.puts(contents) } - end + write_lock(target_lockfile, preserve_unknown_sections) end def locked_ruby_version @@ -349,25 +357,16 @@ module Bundler end end + def bundler_version_to_lock + @resolved_bundler_version || Bundler.gem_version + end + def to_lock require_relative "lockfile_generator" LockfileGenerator.generate(self) end def ensure_equivalent_gemfile_and_lockfile(explicit_flag = false) - msg = String.new - msg << "You are trying to install in deployment mode after changing\n" \ - "your Gemfile. Run `bundle install` elsewhere and add the\n" \ - "updated #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)} to version control." - - unless explicit_flag - suggested_command = unless Bundler.settings.locations("frozen").keys.include?(:env) - "bundle config set frozen false" - end - msg << "\n\nIf this is a development machine, remove the #{Bundler.default_gemfile} " \ - "freeze \nby running `#{suggested_command}`." if suggested_command - end - added = [] deleted = [] changed = [] @@ -381,32 +380,36 @@ module Bundler deleted.concat deleted_deps.map {|d| "* #{pretty_dep(d)}" } if deleted_deps.any? both_sources = Hash.new {|h, k| h[k] = [] } - @dependencies.each {|d| both_sources[d.name][0] = d } - - locked_dependencies.each do |d| - next if !Bundler.feature_flag.bundler_3_mode? && @locked_specs[d.name].empty? - - both_sources[d.name][1] = d - end + current_dependencies.each {|d| both_sources[d.name][0] = d } + current_locked_dependencies.each {|d| both_sources[d.name][1] = d } both_sources.each do |name, (dep, lock_dep)| next if dep.nil? || lock_dep.nil? - gemfile_source = dep.source || sources.default_source - lock_source = lock_dep.source || sources.default_source + gemfile_source = dep.source || default_source + lock_source = lock_dep.source || default_source next if lock_source.include?(gemfile_source) - gemfile_source_name = dep.source ? gemfile_source.identifier : "no specified source" - lockfile_source_name = lock_dep.source ? lock_source.identifier : "no specified source" + gemfile_source_name = dep.source ? gemfile_source.to_gemfile : "no specified source" + lockfile_source_name = lock_dep.source ? lock_source.to_gemfile : "no specified source" changed << "* #{name} from `#{lockfile_source_name}` to `#{gemfile_source_name}`" end reason = change_reason - msg << "\n\n#{reason.split(", ").map(&:capitalize).join("\n")}" unless reason.strip.empty? + msg = String.new + msg << "#{reason.capitalize.strip}, but the lockfile can't be updated because frozen mode is set" msg << "\n\nYou have added to the Gemfile:\n" << added.join("\n") if added.any? msg << "\n\nYou have deleted from the Gemfile:\n" << deleted.join("\n") if deleted.any? msg << "\n\nYou have changed in the Gemfile:\n" << changed.join("\n") if changed.any? - msg << "\n" + msg << "\n\nRun `bundle install` elsewhere and add the updated #{SharedHelpers.relative_gemfile_path} to version control.\n" + + unless explicit_flag + suggested_command = unless Bundler.settings.locations("frozen").keys.include?(:env) + "bundle config set frozen false" + end + msg << "If this is a development machine, remove the #{SharedHelpers.relative_lockfile_path} " \ + "freeze by running `#{suggested_command}`." if suggested_command + end raise ProductionError, msg if added.any? || deleted.any? || changed.any? || !nothing_changed? end @@ -445,8 +448,8 @@ module Bundler return if current_platform_locked? raise ProductionError, "Your bundle only supports platforms #{@platforms.map(&:to_s)} " \ - "but your local platform is #{Bundler.local_platform}. " \ - "Add the current platform to the lockfile with\n`bundle lock --add-platform #{Bundler.local_platform}` and try again." + "but your local platform is #{local_platform}. " \ + "Add the current platform to the lockfile with\n`bundle lock --add-platform #{local_platform}` and try again." end def add_platform(platform) @@ -471,7 +474,21 @@ module Bundler private :sources def nothing_changed? - !@source_changes && !@dependency_changes && !@new_platform && !@path_changes && !@local_changes && !@incomplete_lockfile + return false unless lockfile_exists? + + !@source_changes && + !@dependency_changes && + !@new_platform && + !@path_changes && + !@local_changes && + !@missing_lockfile_dep && + !@unlocking_bundler && + !@locked_spec_with_missing_deps && + !@locked_spec_with_invalid_deps + end + + def no_resolve_needed? + !unlocking? && nothing_changed? end def unlocking? @@ -480,20 +497,75 @@ module Bundler private + def should_add_extra_platforms? + !lockfile_exists? && generic_local_platform_is_ruby? && !Bundler.settings[:force_ruby_platform] + end + + def lockfile_exists? + file_exists?(lockfile) + end + + def file_exists?(file) + file && File.exist?(file) + end + + def write_lock(file, preserve_unknown_sections) + return if Definition.no_lock + + contents = to_lock + + # Convert to \r\n if the existing lock has them + # i.e., Windows with `git config core.autocrlf=true` + contents.gsub!(/\n/, "\r\n") if @lockfile_contents.match?("\r\n") + + if @locked_bundler_version + locked_major = @locked_bundler_version.segments.first + current_major = bundler_version_to_lock.segments.first + + updating_major = locked_major < current_major + end + + preserve_unknown_sections ||= !updating_major && (Bundler.frozen_bundle? || !(unlocking? || @unlocking_bundler)) + + if file_exists?(file) && lockfiles_equal?(@lockfile_contents, contents, preserve_unknown_sections) + return if Bundler.frozen_bundle? + SharedHelpers.filesystem_access(file) { FileUtils.touch(file) } + return + end + + if Bundler.frozen_bundle? + Bundler.ui.error "Cannot write a changed lockfile while frozen." + return + end + + SharedHelpers.filesystem_access(file) do |p| + File.open(p, "wb") {|f| f.puts(contents) } + end + end + def resolver @resolver ||= Resolver.new(resolution_packages, gem_version_promoter) end def expanded_dependencies - dependencies + metadata_dependencies + dependencies_with_bundler + metadata_dependencies + end + + def dependencies_with_bundler + return dependencies unless @unlocking_bundler + return dependencies if dependencies.map(&:name).include?("bundler") + + [Dependency.new("bundler", @unlocking_bundler)] + dependencies end def resolution_packages @resolution_packages ||= begin last_resolve = converge_locked_specs - remove_ruby_from_platforms_if_necessary!(current_dependencies) - packages = Resolver::Base.new(source_requirements, expanded_dependencies, last_resolve, @platforms, :locked_specs => @originally_locked_specs, :unlock => @unlock[:gems], :prerelease => gem_version_promoter.pre?) - additional_base_requirements_for_resolve(packages, last_resolve) + remove_invalid_platforms!(current_dependencies) + packages = Resolver::Base.new(source_requirements, expanded_dependencies, last_resolve, @platforms, locked_specs: @originally_locked_specs, unlock: @gems_to_unlock, prerelease: gem_version_promoter.pre?) + packages = additional_base_requirements_to_prevent_downgrades(packages, last_resolve) + packages = additional_base_requirements_to_force_updates(packages) + packages end end @@ -508,7 +580,7 @@ module Bundler if missing_specs.any? missing_specs.each do |s| locked_gem = @locked_specs[s.name].last - next if locked_gem.nil? || locked_gem.version != s.version || !@remote + next if locked_gem.nil? || locked_gem.version != s.version || sources.local_mode? raise GemNotFound, "Your bundle is locked to #{locked_gem} from #{locked_gem.source}, but that version can " \ "no longer be found in that source. That means the author of #{locked_gem} has removed it. " \ "You'll need to update your bundle to a version other than #{locked_gem} that hasn't been " \ @@ -527,7 +599,7 @@ module Bundler break if incomplete_specs.empty? Bundler.ui.debug("The lockfile does not have all gems needed for the current platform though, Bundler will still re-resolve dependencies") - setup_sources_for_resolve + sources.remote! resolution_packages.delete(incomplete_specs) @resolve = start_resolution specs = resolve.materialize(dependencies) @@ -549,9 +621,14 @@ module Bundler end def start_resolution - result = resolver.start + result = SpecSet.new(resolver.start) + + @resolved_bundler_version = result.find {|spec| spec.name == "bundler" }&.version + @platforms = result.add_extra_platforms!(platforms) if should_add_extra_platforms? - SpecSet.new(SpecSet.new(result).for(dependencies, false, @platforms)) + result.complete_platforms!(platforms) + + SpecSet.new(result.for(dependencies, false, @platforms)) end def precompute_source_requirements_for_indirect_dependencies? @@ -572,7 +649,7 @@ module Bundler end def current_ruby_platform_locked? - return false unless generic_local_platform == Gem::Platform::RUBY + return false unless generic_local_platform_is_ruby? return false if Bundler.settings[:force_ruby_platform] && !@platforms.include?(Gem::Platform::RUBY) current_platform_locked? @@ -580,7 +657,7 @@ module Bundler def current_platform_locked? @platforms.any? do |bundle_platform| - MatchPlatform.platforms_match?(bundle_platform, Bundler.local_platform) + MatchPlatform.platforms_match?(bundle_platform, local_platform) end end @@ -592,14 +669,18 @@ module Bundler def change_reason if unlocking? - unlock_reason = @unlock.reject {|_k, v| Array(v).empty? }.map do |k, v| - if v == true - k.to_s - else - v = Array(v) - "#{k}: (#{v.join(", ")})" - end - end.join(", ") + unlock_targets = if @gems_to_unlock.any? + ["gems", @gems_to_unlock] + elsif @sources_to_unlock.any? + ["sources", @sources_to_unlock] + end + + unlock_reason = if unlock_targets + "#{unlock_targets.first}: (#{unlock_targets.last.join(", ")})" + else + @unlock[:ruby] ? "ruby" : "" + end + return "bundler is unlocking #{unlock_reason}" end [ @@ -608,7 +689,10 @@ module Bundler [@new_platform, "you added a new platform to your gemfile"], [@path_changes, "the gemspecs for path gems changed"], [@local_changes, "the gemspecs for git local gems changed"], - [@incomplete_lockfile, "your lock file is missing some gems"], + [@missing_lockfile_dep, "your lock file is missing \"#{@missing_lockfile_dep}\""], + [@unlocking_bundler, "an update to the version of Bundler itself was requested"], + [@locked_spec_with_missing_deps, "your lock file includes \"#{@locked_spec_with_missing_deps}\" but not some of its dependencies"], + [@locked_spec_with_invalid_deps, "your lockfile does not satisfy dependencies of \"#{@locked_spec_with_invalid_deps}\""], ].select(&:first).map(&:last).join(", ") end @@ -635,8 +719,7 @@ module Bundler locked_index = Index.new locked_index.use(@locked_specs.select {|s| source.can_lock?(s) }) - # order here matters, since Index#== is checking source.specs.include?(locked_index) - locked_index != source.specs + !locked_index.subset?(source.specs) rescue PathError, GitError => e Bundler.ui.debug "Assuming that #{source} has not changed since fetching its specs errored (#{e})" false @@ -652,7 +735,7 @@ module Bundler spec = @dependencies.find {|s| s.name == k } source = spec&.source if source&.respond_to?(:local_override!) - source.unlock! if @unlock[:gems].include?(spec.name) + source.unlock! if @gems_to_unlock.include?(spec.name) locals << [source, source.local_override!(v)] end end @@ -660,22 +743,39 @@ module Bundler sources_with_changes = locals.select do |source, changed| changed || specs_changed?(source) end.map(&:first) - !sources_with_changes.each {|source| @unlock[:sources] << source.name }.empty? + !sources_with_changes.each {|source| @sources_to_unlock << source.name }.empty? end - def check_missing_lockfile_specs - all_locked_specs = @locked_specs.map(&:name) << "bundler" + def check_lockfile + @missing_lockfile_dep = nil + + @locked_spec_with_invalid_deps = nil + @locked_spec_with_missing_deps = nil - missing = @locked_specs.select do |s| - s.dependencies.any? {|dep| !all_locked_specs.include?(dep.name) } + missing = [] + invalid = [] + + @locked_specs.each do |s| + validation = @locked_specs.validate_deps(s) + + missing << s if validation == :missing + invalid << s if validation == :invalid end if missing.any? @locked_specs.delete(missing) - true - else - false + @locked_spec_with_missing_deps = missing.first.name + elsif !@dependency_changes + @missing_lockfile_dep = current_dependencies.find do |d| + @locked_specs[d.name].empty? && d.name != "bundler" + end&.name + end + + if invalid.any? + @locked_specs.delete(invalid) + + @locked_spec_with_invalid_deps = invalid.first.name end end @@ -710,12 +810,17 @@ module Bundler changes = sources.replace_sources!(@locked_sources) sources.all_sources.each do |source| + # has to be done separately, because we want to keep the locked checksum + # store for a source, even when doing a full update + if @locked_checksums && @locked_gems && locked_source = @locked_gems.sources.find {|s| s == source && !s.equal?(source) } + source.checksum_store.merge!(locked_source.checksum_store) + end # If the source is unlockable and the current command allows an unlock of # the source (for example, you are doing a `bundle update <foo>` of a git-pinned # gem), unlock it. For git sources, this means to unlock the revision, which # will cause the `ref` used to be the most recent for the branch (or master) if # an explicit `ref` is not used. - if source.respond_to?(:unlock!) && @unlock[:sources].include?(source.name) + if source.respond_to?(:unlock!) && @sources_to_unlock.include?(source.name) source.unlock! changes = true end @@ -732,9 +837,7 @@ module Bundler dep.source = sources.get(dep.source) end - next if unlocking? - - unless locked_dep = @locked_deps[dep.name] + unless locked_dep = @originally_locked_deps[dep.name] changes = true next end @@ -761,7 +864,7 @@ module Bundler def converge_locked_specs converged = converge_specs(@locked_specs) - resolve = SpecSet.new(converged.reject {|s| @unlock[:gems].include?(s.name) }) + resolve = SpecSet.new(converged.reject {|s| @gems_to_unlock.include?(s.name) }) diff = nil @@ -780,37 +883,38 @@ module Bundler def converge_specs(specs) converged = [] - - deps = @dependencies.select do |dep| - specs[dep].any? {|s| s.satisfies?(dep) && (!dep.source || s.source.include?(dep.source)) } - end + deps = [] @specs_that_changed_sources = [] specs.each do |s| + name = s.name dep = @dependencies.find {|d| s.satisfies?(d) } + lockfile_source = s.source - # Replace the locked dependency's source with the equivalent source from the Gemfile - s.source = if dep&.source - gemfile_source = dep.source - lockfile_source = s.source + if dep + gemfile_source = dep.source || default_source @specs_that_changed_sources << s if gemfile_source != lockfile_source + deps << dep if !dep.source || lockfile_source.include?(dep.source) + @gems_to_unlock << name if lockfile_source.include?(dep.source) && lockfile_source != gemfile_source - gemfile_source + # Replace the locked dependency's source with the equivalent source from the Gemfile + s.source = gemfile_source else - sources.get_with_fallback(s.source) + # Replace the locked dependency's source with the default source, if the locked source is no longer in the Gemfile + s.source = default_source unless sources.get(lockfile_source) end - next if @unlock[:sources].include?(s.source.name) + next if @sources_to_unlock.include?(s.source.name) # Path sources have special logic if s.source.instance_of?(Source::Path) || s.source.instance_of?(Source::Gemspec) new_specs = begin s.source.specs - rescue PathError, GitError + rescue PathError # if we won't need the source (according to the lockfile), - # don't error if the path/git source isn't available + # don't error if the path source isn't available next if specs. for(requested_dependencies, false). none? {|locked_spec| locked_spec.source == s.source } @@ -824,12 +928,12 @@ module Bundler else # If the spec is no longer in the path source, unlock it. This # commonly happens if the version changed in the gemspec - @unlock[:gems] << s.name + @gems_to_unlock << name end end - if dep.nil? && requested_dependencies.find {|d| s.name == d.name } - @unlock[:gems] << s.name + if dep.nil? && requested_dependencies.find {|d| name == d.name } + @gems_to_unlock << s.name else converged << s end @@ -840,7 +944,7 @@ module Bundler def metadata_dependencies @metadata_dependencies ||= [ - Dependency.new("Ruby\0", Gem.ruby_version), + Dependency.new("Ruby\0", Bundler::RubyVersion.system.gem_version), Dependency.new("RubyGems\0", Gem::VERSION), ] end @@ -852,20 +956,32 @@ module Bundler source_requirements = if precompute_source_requirements_for_indirect_dependencies? all_requirements = source_map.all_requirements all_requirements = pin_locally_available_names(all_requirements) if @prefer_local - { :default => sources.default_source }.merge(all_requirements) + { default: default_source }.merge(all_requirements) else - { :default => Source::RubygemsAggregate.new(sources, source_map) }.merge(source_map.direct_requirements) + { default: Source::RubygemsAggregate.new(sources, source_map) }.merge(source_map.direct_requirements) end - source_requirements.merge!(source_map.locked_requirements) unless @remote + source_requirements.merge!(source_map.locked_requirements) if nothing_changed? metadata_dependencies.each do |dep| source_requirements[dep.name] = sources.metadata_source end - source_requirements[:default_bundler] = source_requirements["bundler"] || sources.default_source - source_requirements["bundler"] = sources.metadata_source # needs to come last to override + + default_bundler_source = source_requirements["bundler"] || default_source + + if @unlocking_bundler + default_bundler_source.add_dependency_names("bundler") + else + source_requirements[:default_bundler] = default_bundler_source + source_requirements["bundler"] = sources.metadata_source # needs to come last to override + end + verify_changed_sources! source_requirements end + def default_source + sources.default_source + end + def verify_changed_sources! @specs_that_changed_sources.each do |s| if s.source.specs.search(s.name).empty? @@ -894,7 +1010,7 @@ module Bundler current == proposed end - def additional_base_requirements_for_resolve(resolution_packages, last_resolve) + def additional_base_requirements_to_prevent_downgrades(resolution_packages, last_resolve) return resolution_packages unless @locked_gems && !sources.expired_sources?(@locked_gems.sources) converge_specs(@originally_locked_specs - last_resolve).each do |locked_spec| next if locked_spec.source.is_a?(Source::Path) @@ -903,16 +1019,39 @@ module Bundler resolution_packages end - def remove_ruby_from_platforms_if_necessary!(dependencies) - return if Bundler.frozen_bundle? || - Bundler.local_platform == Gem::Platform::RUBY || - !platforms.include?(Gem::Platform::RUBY) || - (@new_platform && platforms.last == Gem::Platform::RUBY) || + def additional_base_requirements_to_force_updates(resolution_packages) + return resolution_packages if @explicit_unlocks.empty? + full_update = dup_for_full_unlock.resolve + @explicit_unlocks.each do |name| + version = full_update[name].first&.version + resolution_packages.base_requirements[name] = Gem::Requirement.new("= #{version}") if version + end + resolution_packages + end + + def dup_for_full_unlock + unlocked_definition = self.class.new(@lockfile, @dependencies, @sources, true, @ruby_version, @optional_groups, @gemfiles) + unlocked_definition.gem_version_promoter.tap do |gvp| + gvp.level = gem_version_promoter.level + gvp.strict = gem_version_promoter.strict + gvp.pre = gem_version_promoter.pre + end + unlocked_definition + end + + def remove_invalid_platforms!(dependencies) + return if Bundler.frozen_bundle? + + platforms.reverse_each do |platform| + next if local_platform == platform || + (@new_platform && platforms.last == platform) || + @path_changes || @dependency_changes || - !@originally_locked_specs.incomplete_ruby_specs?(dependencies) + !@originally_locked_specs.incomplete_for_platform?(dependencies, platform) - remove_platform(Gem::Platform::RUBY) - add_current_platform + remove_platform(platform) + add_current_platform if platform == Gem::Platform::RUBY + end end def source_map diff --git a/lib/bundler/dependency.rb b/lib/bundler/dependency.rb index 21a4564dcc..2a4f72fe55 100644 --- a/lib/bundler/dependency.rb +++ b/lib/bundler/dependency.rb @@ -7,21 +7,21 @@ require_relative "rubygems_ext" module Bundler class Dependency < Gem::Dependency attr_reader :autorequire - attr_reader :groups, :platforms, :gemfile, :path, :git, :github, :branch, :ref + attr_reader :groups, :platforms, :gemfile, :path, :git, :github, :branch, :ref, :glob - ALL_RUBY_VERSIONS = ((18..27).to_a + (30..33).to_a).freeze + ALL_RUBY_VERSIONS = (18..27).to_a.concat((30..34).to_a).freeze PLATFORM_MAP = { - :ruby => [Gem::Platform::RUBY, ALL_RUBY_VERSIONS], - :mri => [Gem::Platform::RUBY, ALL_RUBY_VERSIONS], - :rbx => [Gem::Platform::RUBY], - :truffleruby => [Gem::Platform::RUBY], - :jruby => [Gem::Platform::JAVA, [18, 19]], - :windows => [Gem::Platform::WINDOWS, ALL_RUBY_VERSIONS], + ruby: [Gem::Platform::RUBY, ALL_RUBY_VERSIONS], + mri: [Gem::Platform::RUBY, ALL_RUBY_VERSIONS], + rbx: [Gem::Platform::RUBY], + truffleruby: [Gem::Platform::RUBY], + jruby: [Gem::Platform::JAVA, [18, 19]], + windows: [Gem::Platform::WINDOWS, ALL_RUBY_VERSIONS], # deprecated - :mswin => [Gem::Platform::MSWIN, ALL_RUBY_VERSIONS], - :mswin64 => [Gem::Platform::MSWIN64, ALL_RUBY_VERSIONS - [18]], - :mingw => [Gem::Platform::MINGW, ALL_RUBY_VERSIONS], - :x64_mingw => [Gem::Platform::X64_MINGW, ALL_RUBY_VERSIONS - [18, 19]], + mswin: [Gem::Platform::MSWIN, ALL_RUBY_VERSIONS], + mswin64: [Gem::Platform::MSWIN64, ALL_RUBY_VERSIONS - [18]], + mingw: [Gem::Platform::MINGW, ALL_RUBY_VERSIONS], + x64_mingw: [Gem::Platform::X64_MINGW, ALL_RUBY_VERSIONS - [18, 19]], }.each_with_object({}) do |(platform, spec), hash| hash[platform] = spec[0] spec[1]&.each {|version| hash[:"#{platform}_#{version}"] = spec[0] } @@ -39,6 +39,7 @@ module Bundler @github = options["github"] @branch = options["branch"] @ref = options["ref"] + @glob = options["glob"] @platforms = Array(options["platforms"]) @env = options["env"] @should_include = options.fetch("should_include", true) @@ -48,10 +49,13 @@ module Bundler @autorequire = Array(options["require"] || []) if options.key?("require") end + RUBY_PLATFORM_ARRAY = [Gem::Platform::RUBY].freeze + private_constant :RUBY_PLATFORM_ARRAY + # Returns the platforms this dependency is valid for, in the same order as # passed in the `valid_platforms` parameter def gem_platforms(valid_platforms) - return [Gem::Platform::RUBY] if force_ruby_platform + return RUBY_PLATFORM_ARRAY if force_ruby_platform return valid_platforms if @platforms.empty? valid_platforms.select {|p| expanded_platforms.include?(GemHelpers.generic(p)) } @@ -65,6 +69,10 @@ module Bundler @should_include && current_env? && current_platform? end + def gemspec_dev_dep? + type == :development + end + def current_env? return true unless @env if @env.is_a?(Hash) diff --git a/lib/bundler/digest.rb b/lib/bundler/digest.rb index 148e9f7788..2c6d971f1b 100644 --- a/lib/bundler/digest.rb +++ b/lib/bundler/digest.rb @@ -50,7 +50,7 @@ module Bundler words.map!.with_index {|word, index| SHA1_MASK & (word + mutated[index]) } end - words.pack("N*").unpack("H*").first + words.pack("N*").unpack1("H*") end private diff --git a/lib/bundler/dsl.rb b/lib/bundler/dsl.rb index 03c80a408c..6af80fb31f 100644 --- a/lib/bundler/dsl.rb +++ b/lib/bundler/dsl.rb @@ -18,9 +18,10 @@ module Bundler VALID_KEYS = %w[group groups git path glob name branch ref tag require submodules platform platforms type source install_if gemfile force_ruby_platform].freeze - GITHUB_PULL_REQUEST_URL = %r{\Ahttps://github\.com/([A-Za-z0-9_\-\.]+/[A-Za-z0-9_\-\.]+)/pull/(\d+)\z}.freeze + GITHUB_PULL_REQUEST_URL = %r{\Ahttps://github\.com/([A-Za-z0-9_\-\.]+/[A-Za-z0-9_\-\.]+)/pull/(\d+)\z} + GITLAB_MERGE_REQUEST_URL = %r{\Ahttps://gitlab\.com/([A-Za-z0-9_\-\./]+)/-/merge_requests/(\d+)\z} - attr_reader :gemspecs + attr_reader :gemspecs, :gemfile attr_accessor :dependencies def initialize @@ -46,7 +47,7 @@ module Bundler @gemfile = expanded_gemfile_path @gemfiles << expanded_gemfile_path contents ||= Bundler.read_file(@gemfile.to_s) - instance_eval(contents.dup.tap {|x| x.untaint if RUBY_VERSION < "2.7" }, gemfile.to_s, 1) + instance_eval(contents, @gemfile.to_s, 1) rescue Exception => e # rubocop:disable Lint/RescueException message = "There was an error " \ "#{e.is_a?(GemfileEvalError) ? "evaluating" : "parsing"} " \ @@ -76,11 +77,11 @@ module Bundler @gemspecs << spec - gem spec.name, :name => spec.name, :path => path, :glob => glob + gem spec.name, name: spec.name, path: path, glob: glob group(development_group) do spec.development_dependencies.each do |dep| - gem dep.name, *(dep.requirement.as_list + [:type => :development]) + gem dep.name, *(dep.requirement.as_list + [type: :development]) end end when 0 @@ -102,39 +103,51 @@ module Bundler # if there's already a dependency with this name we try to prefer one if current = @dependencies.find {|d| d.name == dep.name } - deleted_dep = @dependencies.delete(current) if current.type == :development + if current.requirement != dep.requirement + current_requirement_open = current.requirements_list.include?(">= 0") - unless deleted_dep - if current.requirement != dep.requirement - return if dep.type == :development + gemspec_dep = [dep, current].find(&:gemspec_dev_dep?) + if gemspec_dep + gemfile_dep = [dep, current].find(&:runtime?) + unless current_requirement_open + Bundler.ui.warn "A gemspec development dependency (#{gemspec_dep.name}, #{gemspec_dep.requirement}) is being overridden by a Gemfile dependency (#{gemfile_dep.name}, #{gemfile_dep.requirement}).\n" \ + "This behaviour may change in the future. Please remove either of them, or make sure they both have the same requirement\n" + end + else update_prompt = "" if File.basename(@gemfile) == Injector::INJECTED_GEMS - if dep.requirements_list.include?(">= 0") && !current.requirements_list.include?(">= 0") + if dep.requirements_list.include?(">= 0") && !current_requirement_open update_prompt = ". Gem already added" else update_prompt = ". If you want to update the gem version, run `bundle update #{current.name}`" - update_prompt += ". You may also need to change the version requirement specified in the Gemfile if it's too restrictive." unless current.requirements_list.include?(">= 0") + update_prompt += ". You may also need to change the version requirement specified in the Gemfile if it's too restrictive." unless current_requirement_open end end raise GemfileError, "You cannot specify the same gem twice with different version requirements.\n" \ - "You specified: #{current.name} (#{current.requirement}) and #{dep.name} (#{dep.requirement})" \ - "#{update_prompt}" - elsif current.source != dep.source - return if dep.type == :development - raise GemfileError, "You cannot specify the same gem twice coming from different sources.\n" \ - "You specified that #{dep.name} (#{dep.requirement}) should come from " \ - "#{current.source || "an unspecified source"} and #{dep.source}\n" - else - Bundler.ui.warn "Your Gemfile lists the gem #{current.name} (#{current.requirement}) more than once.\n" \ - "You should probably keep only one of them.\n" \ - "Remove any duplicate entries and specify the gem only once.\n" \ - "While it's not a problem now, it could cause errors if you change the version of one of them later." + "You specified: #{current.name} (#{current.requirement}) and #{dep.name} (#{dep.requirement})" \ + "#{update_prompt}" end end + + # Always prefer the dependency from the Gemfile + if current.gemspec_dev_dep? + @dependencies.delete(current) + elsif dep.gemspec_dev_dep? + return + elsif current.source != dep.source + raise GemfileError, "You cannot specify the same gem twice coming from different sources.\n" \ + "You specified that #{dep.name} (#{dep.requirement}) should come from " \ + "#{current.source || "an unspecified source"} and #{dep.source}\n" + else + Bundler.ui.warn "Your Gemfile lists the gem #{current.name} (#{current.requirement}) more than once.\n" \ + "You should probably keep only one of them.\n" \ + "Remove any duplicate entries and specify the gem only once.\n" \ + "While it's not a problem now, it could cause errors if you change the version of one of them later." + end end @dependencies << dep @@ -296,6 +309,20 @@ module Bundler repo_name ||= user_name "https://#{user_name}@bitbucket.org/#{user_name}/#{repo_name}.git" end + + git_source(:gitlab) do |repo_name| + if repo_name =~ GITLAB_MERGE_REQUEST_URL + { + "git" => "https://gitlab.com/#{$1}.git", + "branch" => nil, + "ref" => "refs/merge-requests/#{$2}/head", + "tag" => nil, + } + else + repo_name = "#{repo_name}/#{repo_name}" unless repo_name.include?("/") + "https://gitlab.com/#{repo_name}.git" + end + end end def with_source(source) @@ -397,13 +424,11 @@ module Bundler end def validate_keys(command, opts, valid_keys) - invalid_keys = opts.keys - valid_keys - - git_source = opts.keys & @git_sources.keys.map(&:to_s) - if opts["branch"] && !(opts["git"] || opts["github"] || git_source.any?) + if opts["branch"] && !(opts["git"] || opts["github"] || (opts.keys & @git_sources.keys.map(&:to_s)).any?) raise GemfileError, %(The `branch` option for `#{command}` is not allowed. Only gems with a git source can specify a branch) end + invalid_keys = opts.keys - valid_keys return true unless invalid_keys.any? message = String.new @@ -422,9 +447,13 @@ module Bundler def normalize_source(source) case source when :gemcutter, :rubygems, :rubyforge - Bundler::SharedHelpers.major_deprecation 2, "The source :#{source} is deprecated because HTTP " \ - "requests are insecure.\nPlease change your source to 'https://" \ - "rubygems.org' if possible, or 'http://rubygems.org' if not." + message = + "The source :#{source} is deprecated because HTTP requests are insecure.\n" \ + "Please change your source to 'https://rubygems.org' if possible, or 'http://rubygems.org' if not." + removed_message = + "The source :#{source} is disallowed because HTTP requests are insecure.\n" \ + "Please change your source to 'https://rubygems.org' if possible, or 'http://rubygems.org' if not." + Bundler::SharedHelpers.major_deprecation 2, message, removed_message: removed_message "http://rubygems.org" when String source @@ -469,10 +498,17 @@ module Bundler "should come from that source" raise GemfileEvalError, msg else - Bundler::SharedHelpers.major_deprecation 2, "Your Gemfile contains multiple global sources. " \ + message = + "Your Gemfile contains multiple global sources. " \ "Using `source` more than once without a block is a security risk, and " \ "may result in installing unexpected gems. To resolve this warning, use " \ "a block to indicate which gems should come from the secondary source." + removed_message = + "Your Gemfile contains multiple global sources. " \ + "Using `source` more than once without a block is a security risk, and " \ + "may result in installing unexpected gems. To resolve this error, use " \ + "a block to indicate which gems should come from the secondary source." + Bundler::SharedHelpers.major_deprecation 2, message, removed_message: removed_message end end diff --git a/lib/bundler/endpoint_specification.rb b/lib/bundler/endpoint_specification.rb index 863544b1f9..87cb352efa 100644 --- a/lib/bundler/endpoint_specification.rb +++ b/lib/bundler/endpoint_specification.rb @@ -94,7 +94,7 @@ module Bundler def _local_specification return unless @loaded_from && File.exist?(local_specification_path) - eval(File.read(local_specification_path)).tap do |spec| + eval(File.read(local_specification_path), nil, local_specification_path).tap do |spec| spec.loaded_from = @loaded_from end end @@ -125,7 +125,11 @@ module Bundler next unless v case k.to_s when "checksum" - @checksum = v.last + begin + @checksum = Checksum.from_api(v.last, @spec_fetcher.uri) + rescue ArgumentError => e + raise ArgumentError, "Invalid checksum for #{full_name}: #{e.message}" + end when "rubygems" @required_rubygems_version = Gem::Requirement.new(v) when "ruby" diff --git a/lib/bundler/env.rb b/lib/bundler/env.rb index 4f8b6f605d..f6cb198e38 100644 --- a/lib/bundler/env.rb +++ b/lib/bundler/env.rb @@ -40,11 +40,11 @@ module Bundler out << "\n## Gemfile\n" gemfiles.each do |gemfile| - out << "\n### #{Pathname.new(gemfile).relative_path_from(SharedHelpers.pwd)}\n\n" + out << "\n### #{SharedHelpers.relative_path_to(gemfile)}\n\n" out << "```ruby\n" << read_file(gemfile).chomp << "\n```\n" end - out << "\n### #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)}\n\n" + out << "\n### #{SharedHelpers.relative_path_to(Bundler.default_lockfile)}\n\n" out << "```\n" << read_file(Bundler.default_lockfile).chomp << "\n```\n" end diff --git a/lib/bundler/environment_preserver.rb b/lib/bundler/environment_preserver.rb index 57013f5d50..444ab6fd37 100644 --- a/lib/bundler/environment_preserver.rb +++ b/lib/bundler/environment_preserver.rb @@ -19,14 +19,7 @@ module Bundler BUNDLER_PREFIX = "BUNDLER_ORIG_" def self.from_env - new(env_to_hash(ENV), BUNDLER_KEYS) - end - - def self.env_to_hash(env) - to_hash = env.to_hash - return to_hash unless Gem.win_platform? - - to_hash.each_with_object({}) {|(k,v), a| a[k.upcase] = v } + new(ENV.to_hash, BUNDLER_KEYS) end # @param env [Hash] @@ -39,18 +32,7 @@ module Bundler # Replaces `ENV` with the bundler environment variables backed up def replace_with_backup - unless Gem.win_platform? - ENV.replace(backup) - return - end - - # Fallback logic for Windows below to workaround - # https://bugs.ruby-lang.org/issues/16798. Can be dropped once all - # supported rubies include the fix for that. - - ENV.clear - - backup.each {|k, v| ENV[k] = v } + ENV.replace(backup) end # @return [Hash] @@ -58,9 +40,9 @@ module Bundler env = @original.clone @keys.each do |key| value = env[key] - if !value.nil? && !value.empty? + if !value.nil? env[@prefix + key] ||= value - elsif value.nil? + else env[@prefix + key] ||= INTENTIONALLY_NIL end end @@ -72,7 +54,7 @@ module Bundler env = @original.clone @keys.each do |key| value_original = env[@prefix + key] - next if value_original.nil? || value_original.empty? + next if value_original.nil? if value_original == INTENTIONALLY_NIL env.delete(key) else diff --git a/lib/bundler/errors.rb b/lib/bundler/errors.rb index 5839fc6a73..c29b1bfed8 100644 --- a/lib/bundler/errors.rb +++ b/lib/bundler/errors.rb @@ -52,6 +52,49 @@ module Bundler class GemfileEvalError < GemfileError; end class MarshalError < StandardError; end + class ChecksumMismatchError < SecurityError + def initialize(lock_name, existing, checksum) + @lock_name = lock_name + @existing = existing + @checksum = checksum + end + + def message + <<~MESSAGE + Bundler found mismatched checksums. This is a potential security risk. + #{@lock_name} #{@existing.to_lock} + from #{@existing.sources.join("\n and ")} + #{@lock_name} #{@checksum.to_lock} + from #{@checksum.sources.join("\n and ")} + + #{mismatch_resolution_instructions} + To ignore checksum security warnings, disable checksum validation with + `bundle config set --local disable_checksum_validation true` + MESSAGE + end + + def mismatch_resolution_instructions + removable, remote = [@existing, @checksum].partition(&:removable?) + case removable.size + when 0 + msg = +"Mismatched checksums each have an authoritative source:\n" + msg << " 1. #{@existing.sources.reject(&:removable?).map(&:to_s).join(" and ")}\n" + msg << " 2. #{@checksum.sources.reject(&:removable?).map(&:to_s).join(" and ")}\n" + msg << "You may need to alter your Gemfile sources to resolve this issue.\n" + when 1 + msg = +"If you trust #{remote.first.sources.first}, to resolve this issue you can:\n" + msg << removable.first.removal_instructions + when 2 + msg = +"To resolve this issue you can either:\n" + msg << @checksum.removal_instructions + msg << "or if you are sure that the new checksum from #{@checksum.sources.first} is correct:\n" + msg << @existing.removal_instructions + end + end + + status_code(37) + end + class PermissionError < BundlerError def initialize(path, permission_type = :write) @path = path @@ -172,4 +215,33 @@ module Bundler status_code(36) end + + class InsecureInstallPathError < BundlerError + def initialize(path) + @path = path + end + + def message + "The installation path is insecure. Bundler cannot continue.\n" \ + "#{@path} is world-writable (without sticky bit).\n" \ + "Bundler cannot safely replace gems in world-writeable directories due to potential vulnerabilities.\n" \ + "Please change the permissions of this directory or choose a different install path." + end + + status_code(38) + end + + class CorruptBundlerInstallError < BundlerError + def initialize(loaded_spec) + @loaded_spec = loaded_spec + end + + def message + "The running version of Bundler (#{Bundler::VERSION}) does not match the version of the specification installed for it (#{@loaded_spec.version}). " \ + "This can be caused by reinstalling Ruby without removing previous installation, leaving around an upgraded default version of Bundler. " \ + "Reinstalling Ruby from scratch should fix the problem." + end + + status_code(39) + end end diff --git a/lib/bundler/feature_flag.rb b/lib/bundler/feature_flag.rb index 983de3137c..ab2189f7f0 100644 --- a/lib/bundler/feature_flag.rb +++ b/lib/bundler/feature_flag.rb @@ -37,7 +37,6 @@ module Bundler settings_flag(:plugins) { @bundler_version >= Gem::Version.new("1.14") } settings_flag(:print_only_version_number) { bundler_3_mode? } settings_flag(:setup_makes_kernel_gem_public) { !bundler_3_mode? } - settings_flag(:suppress_install_using_messages) { bundler_3_mode? } settings_flag(:update_requires_all_flag) { bundler_4_mode? } settings_option(:default_cli_command) { bundler_3_mode? ? :cli_help : :install } diff --git a/lib/bundler/fetcher.rb b/lib/bundler/fetcher.rb index e12c15af8a..6288b22dcd 100644 --- a/lib/bundler/fetcher.rb +++ b/lib/bundler/fetcher.rb @@ -1,14 +1,15 @@ # frozen_string_literal: true require_relative "vendored_persistent" +require_relative "vendored_timeout" require "cgi" require "securerandom" require "zlib" -require "rubygems/request" module Bundler # Handles all the fetching with the rubygems server class Fetcher + autoload :Base, File.expand_path("fetcher/base", __dir__) autoload :CompactIndex, File.expand_path("fetcher/compact_index", __dir__) autoload :Downloader, File.expand_path("fetcher/downloader", __dir__) autoload :Dependency, File.expand_path("fetcher/dependency", __dir__) @@ -61,6 +62,16 @@ module Bundler end end + # This error is raised if HTTP authentication is correct, but lacks + # necessary permissions. + class AuthenticationForbiddenError < HTTPError + def initialize(remote_uri) + remote_uri = filter_uri(remote_uri) + super "Access token could not be authenticated for #{remote_uri}.\n" \ + "Make sure it's valid and has the necessary scopes configured." + end + end + # Exceptions classes that should bypass retry attempts. If your password didn't work the # first time, it's not going to the third time. NET_ERRORS = [:HTTPBadGateway, :HTTPBadRequest, :HTTPFailedDependency, @@ -70,9 +81,9 @@ module Bundler :HTTPRequestURITooLong, :HTTPUnauthorized, :HTTPUnprocessableEntity, :HTTPUnsupportedMediaType, :HTTPVersionNotSupported].freeze FAIL_ERRORS = begin - fail_errors = [AuthenticationRequiredError, BadAuthenticationError, FallbackError] + fail_errors = [AuthenticationRequiredError, BadAuthenticationError, AuthenticationForbiddenError, FallbackError, SecurityError] fail_errors << Gem::Requirement::BadRequirementError - fail_errors.concat(NET_ERRORS.map {|e| Net.const_get(e) }) + fail_errors.concat(NET_ERRORS.map {|e| Gem::Net.const_get(e) }) end.freeze class << self @@ -84,6 +95,7 @@ module Bundler self.max_retries = Bundler.settings[:retry] # How many retries for the API call def initialize(remote) + @cis = nil @remote = remote Socket.do_not_reverse_lookup = true @@ -99,15 +111,17 @@ module Bundler spec -= [nil, "ruby", ""] spec_file_name = "#{spec.join "-"}.gemspec" - uri = Bundler::URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz") - if uri.scheme == "file" - path = Bundler.rubygems.correct_for_windows_path(uri.path) + uri = Gem::URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz") + spec = if uri.scheme == "file" + path = Gem::Util.correct_for_windows_path(uri.path) Bundler.safe_load_marshal Bundler.rubygems.inflate(Gem.read_binary(path)) elsif cached_spec_path = gemspec_cached_path(spec_file_name) Bundler.load_gemspec(cached_spec_path) else Bundler.safe_load_marshal Bundler.rubygems.inflate(downloader.fetch(uri).body) end + raise MarshalError, "is #{spec.inspect}" unless spec.is_a?(Gem::Specification) + spec rescue MarshalError raise HTTPError, "Gemspec #{spec} contained invalid data.\n" \ "Your network or your gem server is probably having issues right now." @@ -124,20 +138,11 @@ module Bundler def specs(gem_names, source) index = Bundler::Index.new - if Bundler::Fetcher.disable_endpoint - @use_api = false - specs = fetchers.last.specs(gem_names) - else - specs = [] - @fetchers = fetchers.drop_while do |f| - !f.available? || (f.api_fetcher? && !gem_names) || !specs = f.specs(gem_names) - end - @use_api = false if fetchers.none?(&:api_fetcher?) - end - - specs.each do |name, version, platform, dependencies, metadata| + fetch_specs(gem_names).each do |name, version, platform, dependencies, metadata| spec = if dependencies - EndpointSpecification.new(name, version, platform, self, dependencies, metadata) + EndpointSpecification.new(name, version, platform, self, dependencies, metadata).tap do |es| + source.checksum_store.replace(es, es.checksum) + end else RemoteSpecification.new(name, version, platform, self) end @@ -148,22 +153,10 @@ module Bundler index rescue CertificateFailureError - Bundler.ui.info "" if gem_names && use_api # newline after dots + Bundler.ui.info "" if gem_names && api_fetcher? # newline after dots raise end - def use_api - return @use_api if defined?(@use_api) - - fetchers.shift until fetchers.first.available? - - @use_api = if remote_uri.scheme == "file" || Bundler::Fetcher.disable_endpoint - false - else - fetchers.first.api_fetcher? - end - end - def user_agent @user_agent ||= begin ruby = Bundler::RubyVersion.system @@ -199,10 +192,6 @@ module Bundler end end - def fetchers - @fetchers ||= FETCHERS.map {|f| f.new(downloader, @remote, uri) } - end - def http_proxy return unless uri = connection.proxy_uri uri.to_s @@ -212,25 +201,49 @@ module Bundler "#<#{self.class}:0x#{object_id} uri=#{uri}>" end + def api_fetcher? + fetchers.first.api_fetcher? + end + + def gem_remote_fetcher + @gem_remote_fetcher ||= begin + require_relative "fetcher/gem_remote_fetcher" + fetcher = GemRemoteFetcher.new Gem.configuration[:http_proxy] + fetcher.headers["User-Agent"] = user_agent + fetcher.headers["X-Gemfile-Source"] = @remote.original_uri.to_s if @remote.original_uri + fetcher + end + end + private - FETCHERS = [CompactIndex, Dependency, Index].freeze + def available_fetchers + if Bundler::Fetcher.disable_endpoint + [Index] + elsif remote_uri.scheme == "file" + Bundler.ui.debug("Using a local server, bundler won't use the CompactIndex API") + [Index] + else + [CompactIndex, Dependency, Index] + end + end + + def fetchers + @fetchers ||= available_fetchers.map {|f| f.new(downloader, @remote, uri, gem_remote_fetcher) }.drop_while {|f| !f.available? } + end + + def fetch_specs(gem_names) + fetchers.reject!(&:api_fetcher?) unless gem_names + fetchers.reject! do |f| + specs = f.specs(gem_names) + return specs if specs + true + end + [] + end def cis - env_cis = { - "TRAVIS" => "travis", - "CIRCLECI" => "circle", - "SEMAPHORE" => "semaphore", - "JENKINS_URL" => "jenkins", - "BUILDBOX" => "buildbox", - "GO_SERVER_URL" => "go", - "SNAP_CI" => "snap", - "GITLAB_CI" => "gitlab", - "GITHUB_ACTIONS" => "github", - "CI_NAME" => ENV["CI_NAME"], - "CI" => "ci", - } - env_cis.find_all {|env, _| ENV[env] }.map {|_, ci| ci } + @cis ||= Bundler::CIDetector.ci_strings end def connection @@ -240,9 +253,9 @@ module Bundler Bundler.settings[:ssl_client_cert] raise SSLError if needs_ssl && !defined?(OpenSSL::SSL) - con = PersistentHTTP.new :name => "bundler", :proxy => :ENV + con = Gem::Net::HTTP::Persistent.new name: "bundler", proxy: :ENV if gem_proxy = Gem.configuration[:http_proxy] - con.proxy = Bundler::URI.parse(gem_proxy) if gem_proxy != :no_proxy + con.proxy = Gem::URI.parse(gem_proxy) if gem_proxy != :no_proxy end if remote_uri.scheme == "https" @@ -275,10 +288,10 @@ module Bundler end HTTP_ERRORS = [ - Timeout::Error, EOFError, SocketError, Errno::ENETDOWN, Errno::ENETUNREACH, + Gem::Timeout::Error, EOFError, SocketError, Errno::ENETDOWN, Errno::ENETUNREACH, Errno::EINVAL, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EAGAIN, - Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, - PersistentHTTP::Error, Zlib::BufError, Errno::EHOSTUNREACH + Gem::Net::HTTPBadResponse, Gem::Net::HTTPHeaderSyntaxError, Gem::Net::ProtocolError, + Gem::Net::HTTP::Persistent::Error, Zlib::BufError, Errno::EHOSTUNREACH ].freeze def bundler_cert_store @@ -294,6 +307,7 @@ module Bundler end else store.set_default_paths + require "rubygems/request" Gem::Request.get_cert_files.each {|c| store.add_file c } end store diff --git a/lib/bundler/fetcher/base.rb b/lib/bundler/fetcher/base.rb index 62cc75add8..cfec2f8e94 100644 --- a/lib/bundler/fetcher/base.rb +++ b/lib/bundler/fetcher/base.rb @@ -6,12 +6,14 @@ module Bundler attr_reader :downloader attr_reader :display_uri attr_reader :remote + attr_reader :gem_remote_fetcher - def initialize(downloader, remote, display_uri) + def initialize(downloader, remote, display_uri, gem_remote_fetcher) raise "Abstract class" if self.class == Base @downloader = downloader @remote = remote @display_uri = display_uri + @gem_remote_fetcher = gem_remote_fetcher end def remote_uri @@ -38,9 +40,9 @@ module Bundler private - def log_specs(debug_msg) + def log_specs(&block) if Bundler.ui.debug? - Bundler.ui.debug debug_msg + Bundler.ui.debug yield else Bundler.ui.info ".", false end diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb index 674d2b49f1..b92482d888 100644 --- a/lib/bundler/fetcher/compact_index.rb +++ b/lib/bundler/fetcher/compact_index.rb @@ -4,8 +4,6 @@ require_relative "base" require_relative "../worker" module Bundler - autoload :CompactIndexClient, File.expand_path("../compact_index_client", __dir__) - class Fetcher class CompactIndex < Base def self.compact_index_request(method_name) @@ -13,9 +11,9 @@ module Bundler undef_method(method_name) define_method(method_name) do |*args, &blk| method.bind(self).call(*args, &blk) - rescue NetworkDownError, CompactIndexClient::Updater::MisMatchedChecksumError => e + rescue NetworkDownError, CompactIndexClient::Updater::MismatchedChecksumError => e raise HTTPError, e.message - rescue AuthenticationRequiredError + rescue AuthenticationRequiredError, BadAuthenticationError # Fail since we got a 401 from the server. raise rescue HTTPError => e @@ -35,7 +33,7 @@ module Bundler remaining_gems = gem_names.dup until remaining_gems.empty? - log_specs "Looking up gems #{remaining_gems.inspect}" + log_specs { "Looking up gems #{remaining_gems.inspect}" } deps = begin parallel_compact_index_client.dependencies(remaining_gems) @@ -44,7 +42,7 @@ module Bundler @bundle_worker = nil # reset it. Not sure if necessary serial_compact_index_client.dependencies(remaining_gems) end - next_gems = deps.map {|d| d[3].map(&:first).flatten(1) }.flatten(1).uniq + next_gems = deps.flat_map {|d| d[3].flat_map(&:first) }.uniq deps.each {|dep| gem_info << dep } complete_gems.concat(deps.map(&:first)).uniq! remaining_gems = next_gems - complete_gems @@ -60,13 +58,9 @@ module Bundler Bundler.ui.debug("FIPS mode is enabled, bundler can't use the CompactIndex API") return nil end - if fetch_uri.scheme == "file" - Bundler.ui.debug("Using a local server, bundler won't use the CompactIndex API") - return false - end # Read info file checksums out of /versions, so we can know if gems are up to date - compact_index_client.update_and_parse_checksums! - rescue CompactIndexClient::Updater::MisMatchedChecksumError => e + compact_index_client.available? + rescue CompactIndexClient::Updater::MismatchedChecksumError => e Bundler.ui.debug(e.message) nil end @@ -125,7 +119,7 @@ module Bundler rescue NetworkDownError => e raise unless Bundler.feature_flag.allow_offline_install? && headers["If-None-Match"] ui.warn "Using the cached data for the new index because of a network error: #{e}" - Net::HTTPNotModified.new(nil, nil, nil) + Gem::Net::HTTPNotModified.new(nil, nil, nil) end end end diff --git a/lib/bundler/fetcher/dependency.rb b/lib/bundler/fetcher/dependency.rb index 18b606abb6..0b807c9a4b 100644 --- a/lib/bundler/fetcher/dependency.rb +++ b/lib/bundler/fetcher/dependency.rb @@ -24,7 +24,7 @@ module Bundler def specs(gem_names, full_dependency_list = [], last_spec_list = []) query_list = gem_names.uniq - full_dependency_list - log_specs "Query List: #{query_list.inspect}" + log_specs { "Query List: #{query_list.inspect}" } return last_spec_list if query_list.empty? diff --git a/lib/bundler/fetcher/downloader.rb b/lib/bundler/fetcher/downloader.rb index 28d33f1aed..868b39b959 100644 --- a/lib/bundler/fetcher/downloader.rb +++ b/lib/bundler/fetcher/downloader.rb @@ -20,31 +20,35 @@ module Bundler Bundler.ui.debug("HTTP #{response.code} #{response.message} #{filtered_uri}") case response - when Net::HTTPSuccess, Net::HTTPNotModified + when Gem::Net::HTTPSuccess, Gem::Net::HTTPNotModified response - when Net::HTTPRedirection - new_uri = Bundler::URI.parse(response["location"]) + when Gem::Net::HTTPRedirection + new_uri = Gem::URI.parse(response["location"]) if new_uri.host == uri.host new_uri.user = uri.user new_uri.password = uri.password end fetch(new_uri, headers, counter + 1) - when Net::HTTPRequestedRangeNotSatisfiable + when Gem::Net::HTTPRequestedRangeNotSatisfiable new_headers = headers.dup new_headers.delete("Range") new_headers["Accept-Encoding"] = "gzip" fetch(uri, new_headers) - when Net::HTTPRequestEntityTooLarge + when Gem::Net::HTTPRequestEntityTooLarge raise FallbackError, response.body - when Net::HTTPTooManyRequests + when Gem::Net::HTTPTooManyRequests raise TooManyRequestsError, response.body - when Net::HTTPUnauthorized + when Gem::Net::HTTPUnauthorized raise BadAuthenticationError, uri.host if uri.userinfo raise AuthenticationRequiredError, uri.host - when Net::HTTPNotFound - raise FallbackError, "Net::HTTPNotFound: #{filtered_uri}" + when Gem::Net::HTTPForbidden + raise AuthenticationForbiddenError, uri.host + when Gem::Net::HTTPNotFound + raise FallbackError, "Gem::Net::HTTPNotFound: #{filtered_uri}" else - raise HTTPError, "#{response.class}#{": #{response.body}" unless response.body.empty?}" + message = "Gem::#{response.class.name.gsub(/\AGem::/, "")}" + message += ": #{response.body}" unless response.body.empty? + raise HTTPError, message end end @@ -54,7 +58,7 @@ module Bundler filtered_uri = URICredentialsFilter.credential_filtered_uri(uri) Bundler.ui.debug "HTTP GET #{filtered_uri}" - req = Net::HTTP::Get.new uri.request_uri, headers + req = Gem::Net::HTTP::Get.new uri.request_uri, headers if uri.user user = CGI.unescape(uri.user) password = uri.password ? CGI.unescape(uri.password) : nil diff --git a/lib/bundler/fetcher/gem_remote_fetcher.rb b/lib/bundler/fetcher/gem_remote_fetcher.rb new file mode 100644 index 0000000000..3fc7b68263 --- /dev/null +++ b/lib/bundler/fetcher/gem_remote_fetcher.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require "rubygems/remote_fetcher" + +module Bundler + class Fetcher + class GemRemoteFetcher < Gem::RemoteFetcher + def request(*args) + super do |req| + req.delete("User-Agent") if headers["User-Agent"] + yield req if block_given? + end + end + end + end +end diff --git a/lib/bundler/fetcher/index.rb b/lib/bundler/fetcher/index.rb index 6bb9fcc193..6e37e1e5d1 100644 --- a/lib/bundler/fetcher/index.rb +++ b/lib/bundler/fetcher/index.rb @@ -6,7 +6,7 @@ module Bundler class Fetcher class Index < Base def specs(_gem_names) - Bundler.rubygems.fetch_all_remote_specs(remote) + Bundler.rubygems.fetch_all_remote_specs(remote, gem_remote_fetcher) rescue Gem::RemoteFetcher::FetchError => e case e.message when /certificate verify failed/ @@ -15,8 +15,7 @@ module Bundler raise BadAuthenticationError, remote_uri if remote_uri.userinfo raise AuthenticationRequiredError, remote_uri when /403/ - raise BadAuthenticationError, remote_uri if remote_uri.userinfo - raise AuthenticationRequiredError, remote_uri + raise AuthenticationForbiddenError, remote_uri else raise HTTPError, "Could not fetch specs from #{display_uri} due to underlying error <#{e.message}>" end diff --git a/lib/bundler/friendly_errors.rb b/lib/bundler/friendly_errors.rb index b66a046d37..e61ed64450 100644 --- a/lib/bundler/friendly_errors.rb +++ b/lib/bundler/friendly_errors.rb @@ -32,7 +32,7 @@ module Bundler if Bundler.ui.debug? Bundler.ui.trace error else - Bundler.ui.error error.message, :wrap => true + Bundler.ui.error error.message, wrap: true end when Thor::Error Bundler.ui.error error.message @@ -40,7 +40,7 @@ module Bundler Bundler.ui.error "\nQuitting..." Bundler.ui.trace error when Gem::InvalidSpecificationException - Bundler.ui.error error.message, :wrap => true + Bundler.ui.error error.message, wrap: true when SystemExit when *[defined?(Java::JavaLang::OutOfMemoryError) && Java::JavaLang::OutOfMemoryError].compact Bundler.ui.error "\nYour JVM has run out of memory, and Bundler cannot continue. " \ diff --git a/lib/bundler/gem_helper.rb b/lib/bundler/gem_helper.rb index dcf759cded..5ce0ef6280 100644 --- a/lib/bundler/gem_helper.rb +++ b/lib/bundler/gem_helper.rb @@ -47,7 +47,7 @@ module Bundler built_gem_path = build_gem end - desc "Generate SHA512 checksum if #{name}-#{version}.gem into the checksums directory." + desc "Generate SHA512 checksum of #{name}-#{version}.gem into the checksums directory." task "build:checksum" => "build" do build_checksum(built_gem_path) end @@ -215,7 +215,7 @@ module Bundler def sh_with_status(cmd, &block) Bundler.ui.debug(cmd) SharedHelpers.chdir(base) do - outbuf = IO.popen(cmd, :err => [:child, :out], &:read) + outbuf = IO.popen(cmd, err: [:child, :out], &:read) status = $? block&.call(outbuf) if status.success? [outbuf, status] diff --git a/lib/bundler/gem_helpers.rb b/lib/bundler/gem_helpers.rb index 2e6d788f9c..de007523ec 100644 --- a/lib/bundler/gem_helpers.rb +++ b/lib/bundler/gem_helpers.rb @@ -34,6 +34,11 @@ module Bundler end module_function :local_platform + def generic_local_platform_is_ruby? + generic_local_platform == Gem::Platform::RUBY + end + module_function :generic_local_platform_is_ruby? + def platform_specificity_match(spec_platform, user_platform) spec_platform = Gem::Platform.new(spec_platform) @@ -48,6 +53,13 @@ module Bundler end module_function :select_best_platform_match + def force_ruby_platform(specs) + matching = specs.select {|spec| spec.match_platform(Gem::Platform::RUBY) && spec.force_ruby_platform! } + + sort_best_platform_match(matching, Gem::Platform::RUBY) + end + module_function :force_ruby_platform + def sort_best_platform_match(matching, platform) exact = matching.select {|spec| spec.platform == platform } return exact if exact.any? @@ -107,8 +119,6 @@ module Bundler def same_deps(spec, exemplary_spec) same_runtime_deps = spec.dependencies.sort == exemplary_spec.dependencies.sort - return same_runtime_deps unless spec.is_a?(Gem::Specification) && exemplary_spec.is_a?(Gem::Specification) - same_metadata_deps = spec.required_ruby_version == exemplary_spec.required_ruby_version && spec.required_rubygems_version == exemplary_spec.required_rubygems_version same_runtime_deps && same_metadata_deps end diff --git a/lib/bundler/gem_version_promoter.rb b/lib/bundler/gem_version_promoter.rb index d281f46eeb..ecc65b4956 100644 --- a/lib/bundler/gem_version_promoter.rb +++ b/lib/bundler/gem_version_promoter.rb @@ -45,17 +45,37 @@ module Bundler # Given a Resolver::Package and an Array of Specifications of available # versions for a gem, this method will return the Array of Specifications - # sorted (and possibly truncated if strict is true) in an order to give - # preference to the current level (:major, :minor or :patch) when resolution - # is deciding what versions best resolve all dependencies in the bundle. + # sorted in an order to give preference to the current level (:major, :minor + # or :patch) when resolution is deciding what versions best resolve all + # dependencies in the bundle. # @param package [Resolver::Package] The package being resolved. # @param specs [Specification] An array of Specifications for the package. - # @return [Specification] A new instance of the Specification Array sorted and - # possibly filtered. + # @return [Specification] A new instance of the Specification Array sorted. def sort_versions(package, specs) - specs = filter_dep_specs(specs, package) if strict + locked_version = package.locked_version + + result = specs.sort do |a, b| + unless package.prerelease_specified? || pre? + a_pre = a.prerelease? + b_pre = b.prerelease? - sort_dep_specs(specs, package) + next 1 if a_pre && !b_pre + next -1 if b_pre && !a_pre + end + + if major? || locked_version.nil? + b <=> a + elsif either_version_older_than_locked?(a, b, locked_version) + b <=> a + elsif segments_do_not_match?(a, b, :major) + a <=> b + elsif !minor? && segments_do_not_match?(a, b, :minor) + a <=> b + else + b <=> a + end + end + post_sort(result, package.unlock?, locked_version) end # @return [bool] Convenience method for testing value of level variable. @@ -73,9 +93,18 @@ module Bundler pre == true end - private + # Given a Resolver::Package and an Array of Specifications of available + # versions for a gem, this method will truncate the Array if strict + # is true. That means filtering out downgrades from the version currently + # locked, and filtering out upgrades that go past the selected level (major, + # minor, or patch). + # @param package [Resolver::Package] The package being resolved. + # @param specs [Specification] An array of Specifications for the package. + # @return [Specification] A new instance of the Specification Array + # truncated. + def filter_versions(package, specs) + return specs unless strict - def filter_dep_specs(specs, package) locked_version = package.locked_version return specs if locked_version.nil? || major? @@ -89,35 +118,10 @@ module Bundler end end - def sort_dep_specs(specs, package) - locked_version = package.locked_version - - result = specs.sort do |a, b| - unless package.prerelease_specified? || pre? - a_pre = a.prerelease? - b_pre = b.prerelease? - - next -1 if a_pre && !b_pre - next 1 if b_pre && !a_pre - end - - if major? - a <=> b - elsif either_version_older_than_locked?(a, b, locked_version) - a <=> b - elsif segments_do_not_match?(a, b, :major) - b <=> a - elsif !minor? && segments_do_not_match?(a, b, :minor) - b <=> a - else - a <=> b - end - end - post_sort(result, package.unlock?, locked_version) - end + private def either_version_older_than_locked?(a, b, locked_version) - locked_version && (a.version < locked_version || b.version < locked_version) + a.version < locked_version || b.version < locked_version end def segments_do_not_match?(a, b, level) @@ -133,13 +137,13 @@ module Bundler if unlock || locked_version.nil? result else - move_version_to_end(result, locked_version) + move_version_to_beginning(result, locked_version) end end - def move_version_to_end(result, version) + def move_version_to_beginning(result, version) move, keep = result.partition {|s| s.version.to_s == version.to_s } - keep.concat(move) + move.concat(keep) end end end diff --git a/lib/bundler/graph.rb b/lib/bundler/graph.rb index 3c008e63e3..b22b17a453 100644 --- a/lib/bundler/graph.rb +++ b/lib/bundler/graph.rb @@ -84,7 +84,7 @@ module Bundler else raise ArgumentError, "2nd argument is invalid" end - label.nil? ? {} : { :label => label } + label.nil? ? {} : { label: label } end def spec_for_dependency(dependency) @@ -103,7 +103,7 @@ module Bundler end def g - @g ||= ::GraphViz.digraph(@graph_name, :concentrate => true, :normalize => true, :nodesep => 0.55) do |g| + @g ||= ::GraphViz.digraph(@graph_name, concentrate: true, normalize: true, nodesep: 0.55) do |g| g.edge[:weight] = 2 g.edge[:fontname] = g.node[:fontname] = "Arial, Helvetica, SansSerif" g.edge[:fontsize] = 12 @@ -114,10 +114,10 @@ module Bundler @groups.each do |group| g.add_nodes( group, { - :style => "filled", - :fillcolor => "#B9B9D5", - :shape => "box3d", - :fontsize => 16, + style: "filled", + fillcolor: "#B9B9D5", + shape: "box3d", + fontsize: 16, }.merge(@node_options[group]) ) end @@ -125,8 +125,8 @@ module Bundler @relations.each do |parent, children| children.each do |child| if @groups.include?(parent) - g.add_nodes(child, { :style => "filled", :fillcolor => "#B9B9D5" }.merge(@node_options[child])) - g.add_edges(parent, child, { :constraint => false }.merge(@edge_options["#{parent}_#{child}"])) + g.add_nodes(child, { style: "filled", fillcolor: "#B9B9D5" }.merge(@node_options[child])) + g.add_edges(parent, child, { constraint: false }.merge(@edge_options["#{parent}_#{child}"])) else g.add_nodes(child, @node_options[child]) g.add_edges(parent, child, @edge_options["#{parent}_#{child}"]) @@ -135,7 +135,7 @@ module Bundler end if @output_format.to_s == "debug" - $stdout.puts g.output :none => String + $stdout.puts g.output none: String Bundler.ui.info "debugging bundle viz..." else begin diff --git a/lib/bundler/index.rb b/lib/bundler/index.rb index b8c599f63a..df46facc88 100644 --- a/lib/bundler/index.rb +++ b/lib/bundler/index.rb @@ -10,8 +10,8 @@ module Bundler i end - attr_reader :specs, :all_specs, :sources - protected :specs, :all_specs + attr_reader :specs, :duplicates, :sources + protected :specs, :duplicates RUBY = "ruby" NULL = "\0" @@ -19,21 +19,21 @@ module Bundler def initialize @sources = [] @cache = {} - @specs = Hash.new {|h, k| h[k] = {} } - @all_specs = Hash.new {|h, k| h[k] = EMPTY_SEARCH } + @specs = {} + @duplicates = {} end def initialize_copy(o) @sources = o.sources.dup @cache = {} - @specs = Hash.new {|h, k| h[k] = {} } - @all_specs = Hash.new {|h, k| h[k] = EMPTY_SEARCH } + @specs = {} + @duplicates = {} o.specs.each do |name, hash| @specs[name] = hash.dup end - o.all_specs.each do |name, array| - @all_specs[name] = array.dup + o.duplicates.each do |name, array| + @duplicates[name] = array.dup end end @@ -46,12 +46,11 @@ module Bundler true end - def search_all(name) - all_matches = local_search(name) + @all_specs[name] - @sources.each do |source| - all_matches.concat(source.search_all(name)) - end - all_matches + def search_all(name, &blk) + return enum_for(:search_all, name) unless blk + specs_by_name(name).each(&blk) + @duplicates[name]&.each(&blk) + @sources.each {|source| source.search_all(name, &blk) } end # Search this index's specs, and any source indexes that this index knows @@ -61,11 +60,14 @@ module Bundler return results unless @sources.any? @sources.each do |source| - results.concat(source.search(query)) + results = safe_concat(results, source.search(query)) end - results.uniq(&:full_name) + results.uniq!(&:full_name) unless results.empty? # avoid modifying frozen EMPTY_SEARCH + results end + alias_method :[], :search + def local_search(query) case query when Gem::Specification, RemoteSpecification, LazySpecification, EndpointSpecification then search_by_spec(query) @@ -76,12 +78,10 @@ module Bundler end end - alias_method :[], :search - - def <<(spec) - @specs[spec.name][spec.full_name] = spec - spec + def add(spec) + (@specs[spec.name] ||= {}).store(spec.full_name, spec) end + alias_method :<<, :add def each(&blk) return enum_for(:each) unless blk @@ -115,15 +115,25 @@ module Bundler names.uniq end - def use(other, override_dupes = false) + # Combines indexes proritizing existing specs, like `Hash#reverse_merge!` + # Duplicate specs found in `other` are stored in `@duplicates`. + def use(other) return unless other - other.each do |s| - if (dupes = search_by_spec(s)) && !dupes.empty? - # safe to << since it's a new array when it has contents - @all_specs[s.name] = dupes << s - next unless override_dupes + other.each do |spec| + exist?(spec) ? add_duplicate(spec) : add(spec) + end + self + end + + # Combines indexes proritizing specs from `other`, like `Hash#merge!` + # Duplicate specs found in `self` are saved in `@duplicates`. + def merge!(other) + return unless other + other.each do |spec| + if existing = find_by_spec(spec) + add_duplicate(existing) end - self << s + add spec end self end @@ -135,8 +145,7 @@ module Bundler end # Whether all the specs in self are in other - # TODO: rename to #include? - def ==(other) + def subset?(other) all? do |spec| other_spec = other[spec].first other_spec && dependencies_eql?(spec, other_spec) && spec.source == other_spec.source @@ -157,19 +166,40 @@ module Bundler private + def safe_concat(a, b) + return a if b.empty? + return b if a.empty? + a.concat(b) + end + + def add_duplicate(spec) + (@duplicates[spec.name] ||= []) << spec + end + def specs_by_name_and_version(name, version) - specs_by_name(name).select {|spec| spec.version == version } + results = @specs[name]&.values + return EMPTY_SEARCH unless results + results.select! {|spec| spec.version == version } + results end def specs_by_name(name) - @specs[name].values + @specs[name]&.values || EMPTY_SEARCH end EMPTY_SEARCH = [].freeze def search_by_spec(spec) - spec = @specs[spec.name][spec.full_name] + spec = find_by_spec(spec) spec ? [spec] : EMPTY_SEARCH end + + def find_by_spec(spec) + @specs[spec.name]&.fetch(spec.full_name, nil) + end + + def exist?(spec) + @specs[spec.name]&.key?(spec.full_name) + end end end diff --git a/lib/bundler/injector.rb b/lib/bundler/injector.rb index cb644a7f69..879b481339 100644 --- a/lib/bundler/injector.rb +++ b/lib/bundler/injector.rb @@ -29,7 +29,7 @@ module Bundler end # temporarily unfreeze - Bundler.settings.temporary(:deployment => false, :frozen => false) do + Bundler.settings.temporary(deployment: false, frozen: false) do # evaluate the Gemfile we have now builder = Dsl.new builder.eval_gemfile(gemfile_path) @@ -50,7 +50,7 @@ module Bundler append_to(gemfile_path, build_gem_lines(@options[:conservative_versioning])) if @deps.any? # since we resolved successfully, write out the lockfile - @definition.lock(Bundler.default_lockfile) + @definition.lock # invalidate the cached Bundler.definition Bundler.reset_paths! @@ -86,7 +86,7 @@ module Bundler segments = version.segments seg_end_index = version >= Gem::Version.new("1.0") ? 1 : 2 - prerelease_suffix = version.to_s.gsub(version.release.to_s, "") if version.prerelease? + prerelease_suffix = version.to_s.delete_prefix(version.release.to_s) if version.prerelease? "#{version_prefix}#{segments[0..seg_end_index].join(".")}#{prerelease_suffix}" end @@ -120,9 +120,10 @@ module Bundler github = ", :github => \"#{d.github}\"" unless d.github.nil? branch = ", :branch => \"#{d.branch}\"" unless d.branch.nil? ref = ", :ref => \"#{d.ref}\"" unless d.ref.nil? + glob = ", :glob => \"#{d.glob}\"" unless d.glob.nil? require_path = ", :require => #{convert_autorequire(d.autorequire)}" unless d.autorequire.nil? - %(gem #{name}#{requirement}#{group}#{source}#{path}#{git}#{github}#{branch}#{ref}#{require_path}) + %(gem #{name}#{requirement}#{group}#{source}#{path}#{git}#{github}#{branch}#{ref}#{glob}#{require_path}) end.join("\n") end diff --git a/lib/bundler/inline.rb b/lib/bundler/inline.rb index 5c184f67a1..ae4ccf2138 100644 --- a/lib/bundler/inline.rb +++ b/lib/bundler/inline.rb @@ -48,14 +48,14 @@ def gemfile(install = false, options = {}, &gemfile) builder.instance_eval(&gemfile) builder.check_primary_source_safety - Bundler.settings.temporary(:deployment => false, :frozen => false) do + Bundler.settings.temporary(deployment: false, frozen: false) do definition = builder.to_definition(nil, true) def definition.lock(*); end definition.validate_runtime! if install || definition.missing_specs? - Bundler.settings.temporary(:inline => true, :no_install => false) do - installer = Bundler::Installer.install(Bundler.root, definition, :system => true) + Bundler.settings.temporary(inline: true, no_install: false) do + installer = Bundler::Installer.install(Bundler.root, definition, system: true) installer.post_install_messages.each do |name, message| Bundler.ui.info "Post-install message from #{name}:\n#{message}" end diff --git a/lib/bundler/installer.rb b/lib/bundler/installer.rb index 59b6a6ad22..256f0be348 100644 --- a/lib/bundler/installer.rb +++ b/lib/bundler/installer.rb @@ -81,7 +81,7 @@ module Bundler if resolve_if_needed(options) ensure_specs_are_compatible! - load_plugins + Bundler.load_plugins(@definition) options.delete(:jobs) else options[:jobs] = 1 # to avoid the overhead of Bundler::Worker @@ -136,12 +136,12 @@ module Bundler mode = Gem.win_platform? ? "wb:UTF-8" : "w" require "erb" - content = ERB.new(template, :trim_mode => "-").result(binding) + content = ERB.new(template, trim_mode: "-").result(binding) - File.write(binstub_path, content, :mode => mode, :perm => 0o777 & ~File.umask) + File.write(binstub_path, content, mode: mode, perm: 0o777 & ~File.umask) if Gem.win_platform? || options[:all_platforms] prefix = "@ruby -x \"%~f0\" %*\n@exit /b %ERRORLEVEL%\n\n" - File.write("#{binstub_path}.cmd", prefix + content, :mode => mode) + File.write("#{binstub_path}.cmd", prefix + content, mode: mode) end end @@ -179,12 +179,12 @@ module Bundler mode = Gem.win_platform? ? "wb:UTF-8" : "w" require "erb" - content = ERB.new(template, :trim_mode => "-").result(binding) + content = ERB.new(template, trim_mode: "-").result(binding) - File.write("#{bin_path}/#{executable}", content, :mode => mode, :perm => 0o755) + File.write("#{bin_path}/#{executable}", content, mode: mode, perm: 0o755) if Gem.win_platform? || options[:all_platforms] prefix = "@ruby -x \"%~f0\" %*\n@exit /b %ERRORLEVEL%\n\n" - File.write("#{bin_path}/#{executable}.cmd", prefix + content, :mode => mode) + File.write("#{bin_path}/#{executable}.cmd", prefix + content, mode: mode) end end end @@ -213,20 +213,6 @@ module Bundler Bundler.settings.processor_count end - def load_plugins - Bundler.rubygems.load_plugins - - requested_path_gems = @definition.requested_specs.select {|s| s.source.is_a?(Source::Path) } - path_plugin_files = requested_path_gems.map do |spec| - Bundler.rubygems.spec_matches_for_glob(spec, "rubygems_plugin#{Bundler.rubygems.suffix_pattern}") - rescue TypeError - error_message = "#{spec.name} #{spec.version} has an invalid gemspec" - raise Gem::InvalidSpecificationException, error_message - end.flatten - Bundler.rubygems.load_plugin_files(path_plugin_files) - Bundler.rubygems.load_env_plugins - end - def ensure_specs_are_compatible! @definition.specs.each do |spec| unless spec.matches_current_ruby? @@ -249,19 +235,19 @@ module Bundler # returns whether or not a re-resolve was needed def resolve_if_needed(options) - @definition.resolution_mode = options - - if !@definition.unlocking? && !options["force"] && !Bundler.settings[:inline] && Bundler.default_lockfile.file? - return false if @definition.nothing_changed? && !@definition.missing_specs? + @definition.prefer_local! if options["prefer-local"] + + if options["local"] || (@definition.no_resolve_needed? && !@definition.missing_specs?) + @definition.resolve_with_cache! + false + else + @definition.resolve_remotely! + true end - - @definition.setup_sources_for_resolve - - true end - def lock(opts = {}) - @definition.lock(Bundler.default_lockfile, opts[:preserve_unknown_sections]) + def lock + @definition.lock end end end diff --git a/lib/bundler/installer/gem_installer.rb b/lib/bundler/installer/gem_installer.rb index c855c7afa0..d3bbcc90f5 100644 --- a/lib/bundler/installer/gem_installer.rb +++ b/lib/bundler/installer/gem_installer.rb @@ -17,11 +17,11 @@ module Bundler Bundler.ui.debug "#{worker}: #{spec.name} (#{spec.version}) from #{spec.loaded_from}" generate_executable_stubs [true, post_install_message] - rescue Bundler::InstallHookError, Bundler::SecurityError, Bundler::APIResponseMismatchError + rescue Bundler::InstallHookError, Bundler::SecurityError, Bundler::APIResponseMismatchError, Bundler::InsecureInstallPathError raise rescue Errno::ENOSPC [false, out_of_space_message] - rescue Bundler::BundlerError, Gem::InstallError, Bundler::APIResponseInvalidDependenciesError => e + rescue Bundler::BundlerError, Gem::InstallError => e [false, specific_failure_message(e)] end @@ -53,10 +53,10 @@ module Bundler def install spec.source.install( spec, - :force => force, - :ensure_builtin_gems_cached => standalone, - :build_args => Array(spec_settings), - :previous_spec => previous_spec, + force: force, + ensure_builtin_gems_cached: standalone, + build_args: Array(spec_settings), + previous_spec: previous_spec, ) end @@ -77,7 +77,7 @@ module Bundler if Bundler.settings[:bin] && standalone installer.generate_standalone_bundler_executable_stubs(spec) elsif Bundler.settings[:bin] - installer.generate_bundler_executable_stubs(spec, :force => true) + installer.generate_bundler_executable_stubs(spec, force: true) end end end diff --git a/lib/bundler/installer/parallel_installer.rb b/lib/bundler/installer/parallel_installer.rb index 83a381f592..e745088f81 100644 --- a/lib/bundler/installer/parallel_installer.rb +++ b/lib/bundler/installer/parallel_installer.rb @@ -42,8 +42,7 @@ module Bundler # Checks installed dependencies against spec's dependencies to make # sure needed dependencies have been installed. - def dependencies_installed?(all_specs) - installed_specs = all_specs.select(&:installed?).map(&:name) + def dependencies_installed?(installed_specs) dependencies.all? {|d| installed_specs.include? d.name } end @@ -63,20 +62,23 @@ module Bundler end end - def self.call(*args) - new(*args).call + def self.call(*args, **kwargs) + new(*args, **kwargs).call end attr_reader :size - def initialize(installer, all_specs, size, standalone, force) + def initialize(installer, all_specs, size, standalone, force, skip: nil) @installer = installer @size = size @standalone = standalone @force = force @specs = all_specs.map {|s| SpecInstallation.new(s) } + @specs.each do |spec_install| + spec_install.state = :installed if skip.include?(spec_install.name) + end if skip @spec_set = all_specs - @rake = @specs.find {|s| s.name == "rake" } + @rake = @specs.find {|s| s.name == "rake" unless s.installed? } end def call @@ -91,38 +93,12 @@ module Bundler install_serially end - check_for_unmet_dependencies - handle_error if failed_specs.any? @specs ensure worker_pool&.stop end - def check_for_unmet_dependencies - unmet_dependencies = @specs.map do |s| - [ - s, - s.dependencies.reject {|dep| @specs.any? {|spec| dep.matches_spec?(spec.spec) } }, - ] - end.reject {|a| a.last.empty? } - return if unmet_dependencies.empty? - - warning = [] - warning << "Your lockfile doesn't include a valid resolution." - warning << "You can fix this by regenerating your lockfile or manually editing the bad locked gems to a version that satisfies all dependencies." - warning << "The unmet dependencies are:" - - unmet_dependencies.each do |spec, unmet_spec_dependencies| - unmet_spec_dependencies.each do |unmet_spec_dependency| - found = @specs.find {|s| s.name == unmet_spec_dependency.name && !unmet_spec_dependency.matches_spec?(s.spec) } - warning << "* #{unmet_spec_dependency}, dependency of #{spec.full_name}, unsatisfied by #{found.full_name}" - end - end - - Bundler.ui.warn(warning.join("\n")) - end - private def failed_specs @@ -209,8 +185,14 @@ module Bundler # previously installed specifications. We continue until all specs # are installed. def enqueue_specs - @specs.select(&:ready_to_enqueue?).each do |spec| - if spec.dependencies_installed? @specs + installed_specs = {} + @specs.each do |spec| + next unless spec.installed? + installed_specs[spec.name] = true + end + + @specs.each do |spec| + if spec.ready_to_enqueue? && spec.dependencies_installed?(installed_specs) spec.state = :enqueued worker_pool.enq spec end diff --git a/lib/bundler/installer/standalone.rb b/lib/bundler/installer/standalone.rb index 2a8c9a432d..5331df2e95 100644 --- a/lib/bundler/installer/standalone.rb +++ b/lib/bundler/installer/standalone.rb @@ -12,6 +12,7 @@ module Bundler end File.open File.join(bundler_path, "setup.rb"), "w" do |file| file.puts "require 'rbconfig'" + file.puts prevent_gem_activation file.puts define_path_helpers file.puts reverse_rubygems_kernel_mixin paths.each do |path| @@ -55,13 +56,26 @@ module Bundler if spec.source.instance_of?(Source::Path) && spec.source.path.absolute? full_path else - Pathname.new(full_path).relative_path_from(Bundler.root.join(bundler_path)).to_s + SharedHelpers.relative_path_to(full_path, from: Bundler.root.join(bundler_path)) end rescue TypeError error_message = "#{spec.name} #{spec.version} has an invalid gemspec" raise Gem::InvalidSpecificationException.new(error_message) end + def prevent_gem_activation + <<~'END' + module Kernel + remove_method(:gem) if private_method_defined?(:gem) + + def gem(*) + end + + private :gem + end + END + end + def define_path_helpers <<~'END' unless defined?(Gem) @@ -87,8 +101,7 @@ module Bundler if Gem.respond_to?(:discover_gems_on_require=) Gem.discover_gems_on_require = false else - kernel = (class << ::Kernel; self; end) - [kernel, ::Kernel].each do |k| + [::Kernel.singleton_class, ::Kernel].each do |k| if k.private_method_defined?(:gem_original_require) private_require = k.private_method_defined?(:require) k.send(:remove_method, :require) diff --git a/lib/bundler/lazy_specification.rb b/lib/bundler/lazy_specification.rb index c9b161dc0e..8669e021c2 100644 --- a/lib/bundler/lazy_specification.rb +++ b/lib/bundler/lazy_specification.rb @@ -4,16 +4,29 @@ require_relative "force_platform" module Bundler class LazySpecification + include MatchMetadata include MatchPlatform include ForcePlatform - attr_reader :name, :version, :dependencies, :platform - attr_accessor :source, :remote, :force_ruby_platform + attr_reader :name, :version, :platform + attr_accessor :source, :remote, :force_ruby_platform, :dependencies, :required_ruby_version, :required_rubygems_version + + alias_method :runtime_dependencies, :dependencies + + def self.from_spec(s) + lazy_spec = new(s.name, s.version, s.platform, s.source) + lazy_spec.dependencies = s.dependencies + lazy_spec.required_ruby_version = s.required_ruby_version + lazy_spec.required_rubygems_version = s.required_rubygems_version + lazy_spec + end def initialize(name, version, platform, source = nil) @name = name @version = version @dependencies = [] + @required_ruby_version = Gem::Requirement.default + @required_rubygems_version = Gem::Requirement.default @platform = platform || Gem::Platform::RUBY @source = source @force_ruby_platform = default_force_ruby_platform @@ -27,6 +40,14 @@ module Bundler end end + def lock_name + @lock_name ||= name_tuple.lock_name + end + + def name_tuple + Gem::NameTuple.new(@name, @version, @platform) + end + def ==(other) full_name == other.full_name end @@ -61,12 +82,7 @@ module Bundler def to_lock out = String.new - - if platform == Gem::Platform::RUBY - out << " #{name} (#{version})\n" - else - out << " #{name} (#{version}-#{platform})\n" - end + out << " #{lock_name}\n" dependencies.sort_by(&:to_s).uniq.each do |dep| next if dep.type == :development @@ -89,7 +105,7 @@ module Bundler installable_candidates = GemHelpers.select_best_platform_match(matching_specs, target_platform) - specification = __materialize__(installable_candidates, :fallback_to_non_installable => false) + specification = __materialize__(installable_candidates, fallback_to_non_installable: false) return specification unless specification.nil? if target_platform != platform @@ -109,9 +125,7 @@ module Bundler # bad gem. def __materialize__(candidates, fallback_to_non_installable: Bundler.frozen_bundle?) search = candidates.reverse.find do |spec| - spec.is_a?(StubSpecification) || - (spec.matches_current_ruby? && - spec.matches_current_rubygems?) + spec.is_a?(StubSpecification) || spec.matches_current_metadata? end if search.nil? && fallback_to_non_installable search = candidates.last @@ -122,11 +136,7 @@ module Bundler end def to_s - @to_s ||= if platform == Gem::Platform::RUBY - "#{name} (#{version})" - else - "#{name} (#{version}-#{platform})" - end + lock_name end def git_version @@ -134,6 +144,10 @@ module Bundler " #{source.revision[0..6]}" end + def force_ruby_platform! + @force_ruby_platform = true + end + private def use_exact_resolved_specifications? diff --git a/lib/bundler/lockfile_generator.rb b/lib/bundler/lockfile_generator.rb index a7ee026f67..a646d00ee1 100644 --- a/lib/bundler/lockfile_generator.rb +++ b/lib/bundler/lockfile_generator.rb @@ -19,6 +19,7 @@ module Bundler add_sources add_platforms add_dependencies + add_checksums add_locked_ruby_version add_bundled_with @@ -65,13 +66,21 @@ module Bundler end end + def add_checksums + return unless definition.locked_checksums + checksums = definition.resolve.map do |spec| + spec.source.checksum_store.to_lock(spec) + end + add_section("CHECKSUMS", checksums) + end + def add_locked_ruby_version return unless locked_ruby_version = definition.locked_ruby_version add_section("RUBY VERSION", locked_ruby_version.to_s) end def add_bundled_with - add_section("BUNDLED WITH", Bundler::VERSION) + add_section("BUNDLED WITH", definition.bundler_version_to_lock.to_s) end def add_section(name, value) diff --git a/lib/bundler/lockfile_parser.rb b/lib/bundler/lockfile_parser.rb index 7360a36752..1e11621e55 100644 --- a/lib/bundler/lockfile_parser.rb +++ b/lib/bundler/lockfile_parser.rb @@ -2,10 +2,41 @@ module Bundler class LockfileParser - attr_reader :sources, :dependencies, :specs, :platforms, :bundler_version, :ruby_version + class Position + attr_reader :line, :column + def initialize(line, column) + @line = line + @column = column + end + + def advance!(string) + lines = string.count("\n") + if lines > 0 + @line += lines + @column = string.length - string.rindex("\n") + else + @column += string.length + end + end + + def to_s + "#{line}:#{column}" + end + end + + attr_reader( + :sources, + :dependencies, + :specs, + :platforms, + :bundler_version, + :ruby_version, + :checksums, + ) BUNDLED = "BUNDLED WITH" DEPENDENCIES = "DEPENDENCIES" + CHECKSUMS = "CHECKSUMS" PLATFORMS = "PLATFORMS" RUBY = "RUBY VERSION" GIT = "GIT" @@ -13,7 +44,7 @@ module Bundler PATH = "PATH" PLUGIN = "PLUGIN SOURCE" SPECS = " specs:" - OPTIONS = /^ ([a-z]+): (.*)$/i.freeze + OPTIONS = /^ ([a-z]+): (.*)$/i SOURCE = [GIT, GEM, PATH, PLUGIN].freeze SECTIONS_BY_VERSION_INTRODUCED = { @@ -21,15 +52,18 @@ module Bundler Gem::Version.create("1.10") => [BUNDLED].freeze, Gem::Version.create("1.12") => [RUBY].freeze, Gem::Version.create("1.13") => [PLUGIN].freeze, + Gem::Version.create("2.5.0") => [CHECKSUMS].freeze, }.freeze - KNOWN_SECTIONS = SECTIONS_BY_VERSION_INTRODUCED.values.flatten.freeze + KNOWN_SECTIONS = SECTIONS_BY_VERSION_INTRODUCED.values.flatten!.freeze ENVIRONMENT_VERSION_SECTIONS = [BUNDLED, RUBY].freeze deprecate_constant(:ENVIRONMENT_VERSION_SECTIONS) def self.sections_in_lockfile(lockfile_contents) - lockfile_contents.scan(/^\w[\w ]*$/).uniq + sections = lockfile_contents.scan(/^\w[\w ]*$/) + sections.uniq! + sections end def self.unknown_sections_in_lockfile(lockfile_contents) @@ -38,7 +72,7 @@ module Bundler def self.sections_to_ignore(base_version = nil) base_version &&= base_version.release - base_version ||= Gem::Version.create("1.0".dup) + base_version ||= Gem::Version.create("1.0") attributes = [] SECTIONS_BY_VERSION_INTRODUCED.each do |version, introduced| next if version <= base_version @@ -61,37 +95,53 @@ module Bundler @platforms = [] @sources = [] @dependencies = {} - @state = nil + @parse_method = nil @specs = {} + @lockfile_path = begin + SharedHelpers.relative_lockfile_path + rescue GemfileNotFound + "Gemfile.lock" + end + @pos = Position.new(1, 1) if lockfile.match?(/<<<<<<<|=======|>>>>>>>|\|\|\|\|\|\|\|/) - raise LockfileError, "Your #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)} contains merge conflicts.\n" \ - "Run `git checkout HEAD -- #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)}` first to get a clean lock." + raise LockfileError, "Your #{@lockfile_path} contains merge conflicts.\n" \ + "Run `git checkout HEAD -- #{@lockfile_path}` first to get a clean lock." end - lockfile.split(/(?:\r?\n)+/).each do |line| + lockfile.split(/((?:\r?\n)+)/) do |line| + # split alternates between the line and the following whitespace + next @pos.advance!(line) if line.match?(/^\s*$/) + if SOURCE.include?(line) - @state = :source + @parse_method = :parse_source parse_source(line) elsif line == DEPENDENCIES - @state = :dependency + @parse_method = :parse_dependency + elsif line == CHECKSUMS + # This is a temporary solution to make this feature disabled by default + # for all gemfiles that don't already explicitly include the feature. + @checksums = true + @parse_method = :parse_checksum elsif line == PLATFORMS - @state = :platform + @parse_method = :parse_platform elsif line == RUBY - @state = :ruby + @parse_method = :parse_ruby elsif line == BUNDLED - @state = :bundled_with + @parse_method = :parse_bundled_with elsif /^[^\s]/.match?(line) - @state = nil - elsif @state - send("parse_#{@state}", line) + @parse_method = nil + elsif @parse_method + send(@parse_method, line) end + @pos.advance!(line) end - @specs = @specs.values.sort_by(&:full_name) + @specs = @specs.values.sort_by!(&:full_name) rescue ArgumentError => e Bundler.ui.debug(e) - raise LockfileError, "Your lockfile is unreadable. Run `rm #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)}` " \ - "and then `bundle install` to generate a new lockfile." + raise LockfileError, "Your lockfile is unreadable. Run `rm #{@lockfile_path}` " \ + "and then `bundle install` to generate a new lockfile. The error occurred while " \ + "evaluating #{@lockfile_path}:#{@pos}" end def may_include_redundant_platform_specific_gems? @@ -110,21 +160,9 @@ module Bundler def parse_source(line) case line when SPECS - case @type - when PATH - @current_source = TYPES[@type].from_lock(@opts) - @sources << @current_source - when GIT - @current_source = TYPES[@type].from_lock(@opts) - @sources << @current_source - when GEM - @opts["remotes"] = Array(@opts.delete("remote")).reverse - @current_source = TYPES[@type].from_lock(@opts) - @sources << @current_source - when PLUGIN - @current_source = Plugin.source_from_lock(@opts) - @sources << @current_source - end + return unless TYPES.key?(@type) + @current_source = TYPES[@type].from_lock(@opts) + @sources << @current_source when OPTIONS value = $2 value = true if value == "true" @@ -154,18 +192,19 @@ module Bundler (?:#{space}\(([^-]*) # Space, followed by version (?:-(.*))?\))? # Optional platform (!)? # Optional pinned marker + (?:#{space}([^ ]+))? # Optional checksum $ # Line end - /xo.freeze + /xo def parse_dependency(line) return unless line =~ NAME_VERSION spaces = $1 return unless spaces.size == 2 - name = $2 + name = -$2 version = $3 pinned = $5 - version = version.split(",").map(&:strip) if version + version = version.split(",").each(&:strip!) if version dep = Bundler::Dependency.new(name, version) @@ -186,23 +225,47 @@ module Bundler @dependencies[dep.name] = dep end - def parse_spec(line) + def parse_checksum(line) return unless line =~ NAME_VERSION + spaces = $1 + return unless spaces.size == 2 + checksums = $6 + return unless checksums name = $2 version = $3 platform = $4 + version = Gem::Version.new(version) + platform = platform ? Gem::Platform.new(platform) : Gem::Platform::RUBY + full_name = Gem::NameTuple.new(name, version, platform).full_name + return unless spec = @specs[full_name] + + checksums.split(",") do |lock_checksum| + column = line.index(lock_checksum) + 1 + checksum = Checksum.from_lock(lock_checksum, "#{@lockfile_path}:#{@pos.line}:#{column}") + spec.source.checksum_store.register(spec, checksum) + end + end + + def parse_spec(line) + return unless line =~ NAME_VERSION + spaces = $1 + name = -$2 + version = $3 + if spaces.size == 4 + # only load platform for non-dependency (spec) line + platform = $4 + version = Gem::Version.new(version) platform = platform ? Gem::Platform.new(platform) : Gem::Platform::RUBY - @current_spec = LazySpecification.new(name, version, platform) - @current_spec.source = @current_source + @current_spec = LazySpecification.new(name, version, platform, @current_source) @current_source.add_dependency_names(name) @specs[@current_spec.full_name] = @current_spec elsif spaces.size == 6 - version = version.split(",").map(&:strip) if version + version = version.split(",").each(&:strip!) if version dep = Gem::Dependency.new(name, version) @current_spec.dependencies << dep end @@ -213,13 +276,14 @@ module Bundler end def parse_bundled_with(line) - line = line.strip + line.strip! return unless Gem::Version.correct?(line) @bundler_version = Gem::Version.create(line) end def parse_ruby(line) - @ruby_version = line.strip + line.strip! + @ruby_version = line end end end diff --git a/lib/bundler/man/bundle-add.1 b/lib/bundler/man/bundle-add.1 index 0e21c75506..56a3b6f85c 100644 --- a/lib/bundler/man/bundle-add.1 +++ b/lib/bundler/man/bundle-add.1 @@ -1,81 +1,58 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-ADD" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-ADD" "1" "May 2024" "" .SH "NAME" \fBbundle\-add\fR \- Add gem to the Gemfile and run bundle install -. .SH "SYNOPSIS" \fBbundle add\fR \fIGEM_NAME\fR [\-\-group=GROUP] [\-\-version=VERSION] [\-\-source=SOURCE] [\-\-path=PATH] [\-\-git=GIT] [\-\-github=GITHUB] [\-\-branch=BRANCH] [\-\-ref=REF] [\-\-skip\-install] [\-\-strict] [\-\-optimistic] -. .SH "DESCRIPTION" Adds the named gem to the Gemfile and run \fBbundle install\fR\. \fBbundle install\fR can be avoided by using the flag \fB\-\-skip\-install\fR\. -. .P Example: -. .P bundle add rails -. .P bundle add rails \-\-version "< 3\.0, > 1\.1" -. .P bundle add rails \-\-version "~> 5\.0\.0" \-\-source "https://gems\.example\.com" \-\-group "development" -. .P bundle add rails \-\-skip\-install -. .P bundle add rails \-\-group "development, test" -. .SH "OPTIONS" -. .TP \fB\-\-version\fR, \fB\-v\fR Specify version requirements(s) for the added gem\. -. .TP \fB\-\-group\fR, \fB\-g\fR Specify the group(s) for the added gem\. Multiple groups should be separated by commas\. -. .TP \fB\-\-source\fR, \fB\-s\fR Specify the source for the added gem\. -. .TP \fB\-\-require\fR, \fB\-r\fR Adds require path to gem\. Provide false, or a path as a string\. -. .TP \fB\-\-path\fR Specify the file system path for the added gem\. -. .TP \fB\-\-git\fR Specify the git source for the added gem\. -. .TP \fB\-\-github\fR Specify the github source for the added gem\. -. .TP \fB\-\-branch\fR Specify the git branch for the added gem\. -. .TP \fB\-\-ref\fR Specify the git ref for the added gem\. -. .TP \fB\-\-skip\-install\fR Adds the gem to the Gemfile but does not install it\. -. .TP \fB\-\-optimistic\fR Adds optimistic declaration of version\. -. .TP \fB\-\-strict\fR Adds strict declaration of version\. diff --git a/lib/bundler/man/bundle-binstubs.1 b/lib/bundler/man/bundle-binstubs.1 index 2774e9d28a..4ec301951f 100644 --- a/lib/bundler/man/bundle-binstubs.1 +++ b/lib/bundler/man/bundle-binstubs.1 @@ -1,41 +1,29 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-BINSTUBS" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-BINSTUBS" "1" "May 2024" "" .SH "NAME" \fBbundle\-binstubs\fR \- Install the binstubs of the listed gems -. .SH "SYNOPSIS" \fBbundle binstubs\fR \fIGEM_NAME\fR [\-\-force] [\-\-path PATH] [\-\-standalone] -. .SH "DESCRIPTION" Binstubs are scripts that wrap around executables\. Bundler creates a small Ruby file (a binstub) that loads Bundler, runs the command, and puts it into \fBbin/\fR\. Binstubs are a shortcut\-or alternative\- to always using \fBbundle exec\fR\. This gives you a file that can be run directly, and one that will always run the correct gem version used by the application\. -. .P For example, if you run \fBbundle binstubs rspec\-core\fR, Bundler will create the file \fBbin/rspec\fR\. That file will contain enough code to load Bundler, tell it to load the bundled gems, and then run rspec\. -. .P This command generates binstubs for executables in \fBGEM_NAME\fR\. Binstubs are put into \fBbin\fR, or the \fB\-\-path\fR directory if one has been set\. Calling binstubs with [GEM [GEM]] will create binstubs for all given gems\. -. .SH "OPTIONS" -. .TP \fB\-\-force\fR Overwrite existing binstubs if they exist\. -. .TP \fB\-\-path\fR The location to install the specified binstubs to\. This defaults to \fBbin\fR\. -. .TP \fB\-\-standalone\fR Makes binstubs that can work without depending on Rubygems or Bundler at runtime\. -. .TP \fB\-\-shebang\fR -Specify a different shebang executable name than the default (default \'ruby\') -. +Specify a different shebang executable name than the default (default 'ruby') .TP \fB\-\-all\fR Create binstubs for all gems in the bundle\. diff --git a/lib/bundler/man/bundle-cache.1 b/lib/bundler/man/bundle-cache.1 index 67a8fdd14d..e2da1269e6 100644 --- a/lib/bundler/man/bundle-cache.1 +++ b/lib/bundler/man/bundle-cache.1 @@ -1,61 +1,40 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-CACHE" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-CACHE" "1" "May 2024" "" .SH "NAME" \fBbundle\-cache\fR \- Package your needed \fB\.gem\fR files into your application -. .SH "SYNOPSIS" \fBbundle cache\fR -. .P alias: \fBpackage\fR, \fBpack\fR -. .SH "DESCRIPTION" Copy all of the \fB\.gem\fR files needed to run the application into the \fBvendor/cache\fR directory\. In the future, when running \fBbundle install(1)\fR \fIbundle\-install\.1\.html\fR, use the gems in the cache in preference to the ones on \fBrubygems\.org\fR\. -. .SH "GIT AND PATH GEMS" The \fBbundle cache\fR command can also package \fB:git\fR and \fB:path\fR dependencies besides \.gem files\. This needs to be explicitly enabled via the \fB\-\-all\fR option\. Once used, the \fB\-\-all\fR option will be remembered\. -. .SH "SUPPORT FOR MULTIPLE PLATFORMS" When using gems that have different packages for different platforms, Bundler supports caching of gems for other platforms where the Gemfile has been resolved (i\.e\. present in the lockfile) in \fBvendor/cache\fR\. This needs to be enabled via the \fB\-\-all\-platforms\fR option\. This setting will be remembered in your local bundler configuration\. -. .SH "REMOTE FETCHING" By default, if you run \fBbundle install(1)\fR \fIbundle\-install\.1\.html\fR after running bundle cache(1) \fIbundle\-cache\.1\.html\fR, bundler will still connect to \fBrubygems\.org\fR to check whether a platform\-specific gem exists for any of the gems in \fBvendor/cache\fR\. -. .P For instance, consider this Gemfile(5): -. .IP "" 4 -. .nf - source "https://rubygems\.org" gem "nokogiri" -. .fi -. .IP "" 0 -. .P If you run \fBbundle cache\fR under C Ruby, bundler will retrieve the version of \fBnokogiri\fR for the \fB"ruby"\fR platform\. If you deploy to JRuby and run \fBbundle install\fR, bundler is forced to check to see whether a \fB"java"\fR platformed \fBnokogiri\fR exists\. -. .P Even though the \fBnokogiri\fR gem for the Ruby platform is \fItechnically\fR acceptable on JRuby, it has a C extension that does not run on JRuby\. As a result, bundler will, by default, still connect to \fBrubygems\.org\fR to check whether it has a version of one of your gems more specific to your platform\. -. .P This problem is also not limited to the \fB"java"\fR platform\. A similar (common) problem can happen when developing on Windows and deploying to Linux, or even when developing on OSX and deploying to Linux\. -. .P If you know for sure that the gems packaged in \fBvendor/cache\fR are appropriate for the platform you are on, you can run \fBbundle install \-\-local\fR to skip checking for more appropriate gems, and use the ones in \fBvendor/cache\fR\. -. .P One way to be sure that you have the right platformed versions of all your gems is to run \fBbundle cache\fR on an identical machine and check in the gems\. For instance, you can run \fBbundle cache\fR on an identical staging box during your staging process, and check in the \fBvendor/cache\fR before deploying to production\. -. .P By default, bundle cache(1) \fIbundle\-cache\.1\.html\fR fetches and also installs the gems to the default location\. To package the dependencies to \fBvendor/cache\fR without installing them to the local install location, you can run \fBbundle cache \-\-no\-install\fR\. -. .SH "HISTORY" In Bundler 2\.1, \fBcache\fR took in the functionalities of \fBpackage\fR and now \fBpackage\fR and \fBpack\fR are aliases of \fBcache\fR\. diff --git a/lib/bundler/man/bundle-check.1 b/lib/bundler/man/bundle-check.1 index 7679945c48..dee1af1326 100644 --- a/lib/bundler/man/bundle-check.1 +++ b/lib/bundler/man/bundle-check.1 @@ -1,30 +1,23 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-CHECK" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-CHECK" "1" "May 2024" "" .SH "NAME" \fBbundle\-check\fR \- Verifies if dependencies are satisfied by installed gems -. .SH "SYNOPSIS" \fBbundle check\fR [\-\-dry\-run] [\-\-gemfile=FILE] [\-\-path=PATH] -. .SH "DESCRIPTION" \fBcheck\fR searches the local machine for each of the gems requested in the Gemfile\. If all gems are found, Bundler prints a success message and exits with a status of 0\. -. .P If not, the first missing gem is listed and Bundler exits status 1\. -. +.P +If the lockfile needs to be updated then it will be resolved using the gems installed on the local machine, if they satisfy the requirements\. .SH "OPTIONS" -. .TP \fB\-\-dry\-run\fR Locks the [\fBGemfile(5)\fR][Gemfile(5)] before running the command\. -. .TP \fB\-\-gemfile\fR Use the specified gemfile instead of the [\fBGemfile(5)\fR][Gemfile(5)]\. -. .TP \fB\-\-path\fR Specify a different path than the system default (\fB$BUNDLE_PATH\fR or \fB$GEM_HOME\fR)\. Bundler will remember this value for future installs on this machine\. diff --git a/lib/bundler/man/bundle-check.1.ronn b/lib/bundler/man/bundle-check.1.ronn index f2846b8ff2..eb3ff1daf9 100644 --- a/lib/bundler/man/bundle-check.1.ronn +++ b/lib/bundler/man/bundle-check.1.ronn @@ -15,6 +15,9 @@ a status of 0. If not, the first missing gem is listed and Bundler exits status 1. +If the lockfile needs to be updated then it will be resolved using the gems +installed on the local machine, if they satisfy the requirements. + ## OPTIONS * `--dry-run`: diff --git a/lib/bundler/man/bundle-clean.1 b/lib/bundler/man/bundle-clean.1 index 2eb745698c..7c7f9b5c77 100644 --- a/lib/bundler/man/bundle-clean.1 +++ b/lib/bundler/man/bundle-clean.1 @@ -1,23 +1,16 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-CLEAN" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-CLEAN" "1" "May 2024" "" .SH "NAME" \fBbundle\-clean\fR \- Cleans up unused gems in your bundler directory -. .SH "SYNOPSIS" \fBbundle clean\fR [\-\-dry\-run] [\-\-force] -. .SH "DESCRIPTION" This command will remove all unused gems in your bundler directory\. This is useful when you have made many changes to your gem dependencies\. -. .SH "OPTIONS" -. .TP \fB\-\-dry\-run\fR Print the changes, but do not clean the unused gems\. -. .TP \fB\-\-force\fR Forces cleaning up unused gems even if Bundler is configured to use globally installed gems\. As a consequence, removes all system gems except for the ones in the current application\. diff --git a/lib/bundler/man/bundle-config.1 b/lib/bundler/man/bundle-config.1 index 493b57e1de..a207dbbcaa 100644 --- a/lib/bundler/man/bundle-config.1 +++ b/lib/bundler/man/bundle-config.1 @@ -1,515 +1,319 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-CONFIG" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-CONFIG" "1" "May 2024" "" .SH "NAME" \fBbundle\-config\fR \- Set bundler configuration options -. .SH "SYNOPSIS" \fBbundle config\fR list -. .br \fBbundle config\fR [get] NAME -. .br \fBbundle config\fR [set] NAME VALUE -. .br \fBbundle config\fR unset NAME -. .SH "DESCRIPTION" -This command allows you to interact with Bundler\'s configuration system\. -. +This command allows you to interact with Bundler's configuration system\. .P Bundler loads configuration settings in this order: -. .IP "1." 4 Local config (\fB<project_root>/\.bundle/config\fR or \fB$BUNDLE_APP_CONFIG/config\fR) -. .IP "2." 4 Environmental variables (\fBENV\fR) -. .IP "3." 4 Global config (\fB~/\.bundle/config\fR) -. .IP "4." 4 Bundler default config -. .IP "" 0 -. .P Executing \fBbundle config list\fR will print a list of all bundler configuration for the current bundle, and where that configuration was set\. -. .P Executing \fBbundle config get <name>\fR will print the value of that configuration setting, and where it was set\. -. .P Executing \fBbundle config set <name> <value>\fR defaults to setting \fBlocal\fR configuration if executing from within a local application, otherwise it will set \fBglobal\fR configuration\. See \fB\-\-local\fR and \fB\-\-global\fR options below\. -. .P Executing \fBbundle config set \-\-local <name> <value>\fR will set that configuration in the directory for the local application\. The configuration will be stored in \fB<project_root>/\.bundle/config\fR\. If \fBBUNDLE_APP_CONFIG\fR is set, the configuration will be stored in \fB$BUNDLE_APP_CONFIG/config\fR\. -. .P Executing \fBbundle config set \-\-global <name> <value>\fR will set that configuration to the value specified for all bundles executed as the current user\. The configuration will be stored in \fB~/\.bundle/config\fR\. If \fIname\fR already is set, \fIname\fR will be overridden and user will be warned\. -. .P Executing \fBbundle config unset <name>\fR will delete the configuration in both local and global sources\. -. .P Executing \fBbundle config unset \-\-global <name>\fR will delete the configuration only from the user configuration\. -. .P Executing \fBbundle config unset \-\-local <name>\fR will delete the configuration only from the local application\. -. .P Executing bundle with the \fBBUNDLE_IGNORE_CONFIG\fR environment variable set will cause it to ignore all configuration\. -. .SH "REMEMBERING OPTIONS" -Flags passed to \fBbundle install\fR or the Bundler runtime, such as \fB\-\-path foo\fR or \fB\-\-without production\fR, are remembered between commands and saved to your local application\'s configuration (normally, \fB\./\.bundle/config\fR)\. -. +Flags passed to \fBbundle install\fR or the Bundler runtime, such as \fB\-\-path foo\fR or \fB\-\-without production\fR, are remembered between commands and saved to your local application's configuration (normally, \fB\./\.bundle/config\fR)\. .P -However, this will be changed in bundler 3, so it\'s better not to rely on this behavior\. If these options must be remembered, it\'s better to set them using \fBbundle config\fR (e\.g\., \fBbundle config set \-\-local path foo\fR)\. -. +However, this will be changed in bundler 3, so it's better not to rely on this behavior\. If these options must be remembered, it's better to set them using \fBbundle config\fR (e\.g\., \fBbundle config set \-\-local path foo\fR)\. .P The options that can be configured are: -. .TP \fBbin\fR -Creates a directory (defaults to \fB~/bin\fR) and place any executables from the gem there\. These executables run in Bundler\'s context\. If used, you might add this directory to your environment\'s \fBPATH\fR variable\. For instance, if the \fBrails\fR gem comes with a \fBrails\fR executable, this flag will create a \fBbin/rails\fR executable that ensures that all referred dependencies will be resolved using the bundled gems\. -. +Creates a directory (defaults to \fB~/bin\fR) and place any executables from the gem there\. These executables run in Bundler's context\. If used, you might add this directory to your environment's \fBPATH\fR variable\. For instance, if the \fBrails\fR gem comes with a \fBrails\fR executable, this flag will create a \fBbin/rails\fR executable that ensures that all referred dependencies will be resolved using the bundled gems\. .TP \fBdeployment\fR -In deployment mode, Bundler will \'roll\-out\' the bundle for \fBproduction\fR use\. Please check carefully if you want to have this option enabled in \fBdevelopment\fR or \fBtest\fR environments\. -. +In deployment mode, Bundler will 'roll\-out' the bundle for \fBproduction\fR use\. Please check carefully if you want to have this option enabled in \fBdevelopment\fR or \fBtest\fR environments\. .TP \fBonly\fR A space\-separated list of groups to install only gems of the specified groups\. -. .TP \fBpath\fR -The location to install the specified gems to\. This defaults to Rubygems\' setting\. Bundler shares this location with Rubygems, \fBgem install \.\.\.\fR will have gem installed there, too\. Therefore, gems installed without a \fB\-\-path \.\.\.\fR setting will show up by calling \fBgem list\fR\. Accordingly, gems installed to other locations will not get listed\. -. +The location to install the specified gems to\. This defaults to Rubygems' setting\. Bundler shares this location with Rubygems, \fBgem install \|\.\|\.\|\.\fR will have gem installed there, too\. Therefore, gems installed without a \fB\-\-path \|\.\|\.\|\.\fR setting will show up by calling \fBgem list\fR\. Accordingly, gems installed to other locations will not get listed\. .TP \fBwithout\fR A space\-separated list of groups referencing gems to skip during installation\. -. .TP \fBwith\fR A space\-separated list of \fBoptional\fR groups referencing gems to include during installation\. -. .SH "BUILD OPTIONS" You can use \fBbundle config\fR to give Bundler the flags to pass to the gem installer every time bundler tries to install a particular gem\. -. .P A very common example, the \fBmysql\fR gem, requires Snow Leopard users to pass configuration flags to \fBgem install\fR to specify where to find the \fBmysql_config\fR executable\. -. .IP "" 4 -. .nf - gem install mysql \-\- \-\-with\-mysql\-config=/usr/local/mysql/bin/mysql_config -. .fi -. .IP "" 0 -. .P Since the specific location of that executable can change from machine to machine, you can specify these flags on a per\-machine basis\. -. .IP "" 4 -. .nf - bundle config set \-\-global build\.mysql \-\-with\-mysql\-config=/usr/local/mysql/bin/mysql_config -. .fi -. .IP "" 0 -. .P After running this command, every time bundler needs to install the \fBmysql\fR gem, it will pass along the flags you specified\. -. .SH "CONFIGURATION KEYS" Configuration keys in bundler have two forms: the canonical form and the environment variable form\. -. .P -For instance, passing the \fB\-\-without\fR flag to bundle install(1) \fIbundle\-install\.1\.html\fR prevents Bundler from installing certain groups specified in the Gemfile(5)\. Bundler persists this value in \fBapp/\.bundle/config\fR so that calls to \fBBundler\.setup\fR do not try to find gems from the \fBGemfile\fR that you didn\'t install\. Additionally, subsequent calls to bundle install(1) \fIbundle\-install\.1\.html\fR remember this setting and skip those groups\. -. +For instance, passing the \fB\-\-without\fR flag to bundle install(1) \fIbundle\-install\.1\.html\fR prevents Bundler from installing certain groups specified in the Gemfile(5)\. Bundler persists this value in \fBapp/\.bundle/config\fR so that calls to \fBBundler\.setup\fR do not try to find gems from the \fBGemfile\fR that you didn't install\. Additionally, subsequent calls to bundle install(1) \fIbundle\-install\.1\.html\fR remember this setting and skip those groups\. .P The canonical form of this configuration is \fB"without"\fR\. To convert the canonical form to the environment variable form, capitalize it, and prepend \fBBUNDLE_\fR\. The environment variable form of \fB"without"\fR is \fBBUNDLE_WITHOUT\fR\. -. .P Any periods in the configuration keys must be replaced with two underscores when setting it via environment variables\. The configuration key \fBlocal\.rack\fR becomes the environment variable \fBBUNDLE_LOCAL__RACK\fR\. -. .SH "LIST OF AVAILABLE KEYS" The following is a list of all configuration keys and their purpose\. You can learn more about their operation in bundle install(1) \fIbundle\-install\.1\.html\fR\. -. -.IP "\(bu" 4 -\fBallow_deployment_source_credential_changes\fR (\fBBUNDLE_ALLOW_DEPLOYMENT_SOURCE_CREDENTIAL_CHANGES\fR): When in deployment mode, allow changing the credentials to a gem\'s source\. Ex: \fBhttps://some\.host\.com/gems/path/\fR \-> \fBhttps://user_name:password@some\.host\.com/gems/path\fR -. .IP "\(bu" 4 \fBallow_offline_install\fR (\fBBUNDLE_ALLOW_OFFLINE_INSTALL\fR): Allow Bundler to use cached data when installing without network access\. -. .IP "\(bu" 4 \fBauto_clean_without_path\fR (\fBBUNDLE_AUTO_CLEAN_WITHOUT_PATH\fR): Automatically run \fBbundle clean\fR after installing when an explicit \fBpath\fR has not been set and Bundler is not installing into the system gems\. -. .IP "\(bu" 4 \fBauto_install\fR (\fBBUNDLE_AUTO_INSTALL\fR): Automatically run \fBbundle install\fR when gems are missing\. -. .IP "\(bu" 4 \fBbin\fR (\fBBUNDLE_BIN\fR): Install executables from gems in the bundle to the specified directory\. Defaults to \fBfalse\fR\. -. .IP "\(bu" 4 \fBcache_all\fR (\fBBUNDLE_CACHE_ALL\fR): Cache all gems, including path and git gems\. This needs to be explicitly configured on bundler 1 and bundler 2, but will be the default on bundler 3\. -. .IP "\(bu" 4 \fBcache_all_platforms\fR (\fBBUNDLE_CACHE_ALL_PLATFORMS\fR): Cache gems for all platforms\. -. .IP "\(bu" 4 \fBcache_path\fR (\fBBUNDLE_CACHE_PATH\fR): The directory that bundler will place cached gems in when running \fBbundle package\fR, and that bundler will look in when installing gems\. Defaults to \fBvendor/cache\fR\. -. .IP "\(bu" 4 \fBclean\fR (\fBBUNDLE_CLEAN\fR): Whether Bundler should run \fBbundle clean\fR automatically after \fBbundle install\fR\. -. .IP "\(bu" 4 \fBconsole\fR (\fBBUNDLE_CONSOLE\fR): The console that \fBbundle console\fR starts\. Defaults to \fBirb\fR\. -. .IP "\(bu" 4 \fBdefault_install_uses_path\fR (\fBBUNDLE_DEFAULT_INSTALL_USES_PATH\fR): Whether a \fBbundle install\fR without an explicit \fB\-\-path\fR argument defaults to installing gems in \fB\.bundle\fR\. -. .IP "\(bu" 4 \fBdeployment\fR (\fBBUNDLE_DEPLOYMENT\fR): Disallow changes to the \fBGemfile\fR\. When the \fBGemfile\fR is changed and the lockfile has not been updated, running Bundler commands will be blocked\. -. .IP "\(bu" 4 \fBdisable_checksum_validation\fR (\fBBUNDLE_DISABLE_CHECKSUM_VALIDATION\fR): Allow installing gems even if they do not match the checksum provided by RubyGems\. -. .IP "\(bu" 4 \fBdisable_exec_load\fR (\fBBUNDLE_DISABLE_EXEC_LOAD\fR): Stop Bundler from using \fBload\fR to launch an executable in\-process in \fBbundle exec\fR\. -. .IP "\(bu" 4 \fBdisable_local_branch_check\fR (\fBBUNDLE_DISABLE_LOCAL_BRANCH_CHECK\fR): Allow Bundler to use a local git override without a branch specified in the Gemfile\. -. .IP "\(bu" 4 \fBdisable_local_revision_check\fR (\fBBUNDLE_DISABLE_LOCAL_REVISION_CHECK\fR): Allow Bundler to use a local git override without checking if the revision present in the lockfile is present in the repository\. -. .IP "\(bu" 4 -\fBdisable_shared_gems\fR (\fBBUNDLE_DISABLE_SHARED_GEMS\fR): Stop Bundler from accessing gems installed to RubyGems\' normal location\. -. +\fBdisable_shared_gems\fR (\fBBUNDLE_DISABLE_SHARED_GEMS\fR): Stop Bundler from accessing gems installed to RubyGems' normal location\. .IP "\(bu" 4 \fBdisable_version_check\fR (\fBBUNDLE_DISABLE_VERSION_CHECK\fR): Stop Bundler from checking if a newer Bundler version is available on rubygems\.org\. -. .IP "\(bu" 4 -\fBforce_ruby_platform\fR (\fBBUNDLE_FORCE_RUBY_PLATFORM\fR): Ignore the current machine\'s platform and install only \fBruby\fR platform gems\. As a result, gems with native extensions will be compiled from source\. -. +\fBforce_ruby_platform\fR (\fBBUNDLE_FORCE_RUBY_PLATFORM\fR): Ignore the current machine's platform and install only \fBruby\fR platform gems\. As a result, gems with native extensions will be compiled from source\. .IP "\(bu" 4 \fBfrozen\fR (\fBBUNDLE_FROZEN\fR): Disallow changes to the \fBGemfile\fR\. When the \fBGemfile\fR is changed and the lockfile has not been updated, running Bundler commands will be blocked\. Defaults to \fBtrue\fR when \fB\-\-deployment\fR is used\. -. .IP "\(bu" 4 \fBgem\.github_username\fR (\fBBUNDLE_GEM__GITHUB_USERNAME\fR): Sets a GitHub username or organization to be used in \fBREADME\fR file when you create a new gem via \fBbundle gem\fR command\. It can be overridden by passing an explicit \fB\-\-github\-username\fR flag to \fBbundle gem\fR\. -. .IP "\(bu" 4 \fBgem\.push_key\fR (\fBBUNDLE_GEM__PUSH_KEY\fR): Sets the \fB\-\-key\fR parameter for \fBgem push\fR when using the \fBrake release\fR command with a private gemstash server\. -. .IP "\(bu" 4 \fBgemfile\fR (\fBBUNDLE_GEMFILE\fR): The name of the file that bundler should use as the \fBGemfile\fR\. This location of this file also sets the root of the project, which is used to resolve relative paths in the \fBGemfile\fR, among other things\. By default, bundler will search up from the current working directory until it finds a \fBGemfile\fR\. -. .IP "\(bu" 4 \fBglobal_gem_cache\fR (\fBBUNDLE_GLOBAL_GEM_CACHE\fR): Whether Bundler should cache all gems globally, rather than locally to the installing Ruby installation\. -. .IP "\(bu" 4 \fBignore_funding_requests\fR (\fBBUNDLE_IGNORE_FUNDING_REQUESTS\fR): When set, no funding requests will be printed\. -. .IP "\(bu" 4 \fBignore_messages\fR (\fBBUNDLE_IGNORE_MESSAGES\fR): When set, no post install messages will be printed\. To silence a single gem, use dot notation like \fBignore_messages\.httparty true\fR\. -. .IP "\(bu" 4 \fBinit_gems_rb\fR (\fBBUNDLE_INIT_GEMS_RB\fR): Generate a \fBgems\.rb\fR instead of a \fBGemfile\fR when running \fBbundle init\fR\. -. .IP "\(bu" 4 \fBjobs\fR (\fBBUNDLE_JOBS\fR): The number of gems Bundler can install in parallel\. Defaults to the number of available processors\. -. .IP "\(bu" 4 \fBno_install\fR (\fBBUNDLE_NO_INSTALL\fR): Whether \fBbundle package\fR should skip installing gems\. -. .IP "\(bu" 4 \fBno_prune\fR (\fBBUNDLE_NO_PRUNE\fR): Whether Bundler should leave outdated gems unpruned when caching\. -. .IP "\(bu" 4 \fBonly\fR (\fBBUNDLE_ONLY\fR): A space\-separated list of groups to install only gems of the specified groups\. -. .IP "\(bu" 4 \fBpath\fR (\fBBUNDLE_PATH\fR): The location on disk where all gems in your bundle will be located regardless of \fB$GEM_HOME\fR or \fB$GEM_PATH\fR values\. Bundle gems not found in this location will be installed by \fBbundle install\fR\. Defaults to \fBGem\.dir\fR\. When \-\-deployment is used, defaults to vendor/bundle\. -. .IP "\(bu" 4 \fBpath\.system\fR (\fBBUNDLE_PATH__SYSTEM\fR): Whether Bundler will install gems into the default system path (\fBGem\.dir\fR)\. -. .IP "\(bu" 4 \fBpath_relative_to_cwd\fR (\fBBUNDLE_PATH_RELATIVE_TO_CWD\fR) Makes \fB\-\-path\fR relative to the CWD instead of the \fBGemfile\fR\. -. .IP "\(bu" 4 -\fBplugins\fR (\fBBUNDLE_PLUGINS\fR): Enable Bundler\'s experimental plugin system\. -. +\fBplugins\fR (\fBBUNDLE_PLUGINS\fR): Enable Bundler's experimental plugin system\. .IP "\(bu" 4 \fBprefer_patch\fR (BUNDLE_PREFER_PATCH): Prefer updating only to next patch version during updates\. Makes \fBbundle update\fR calls equivalent to \fBbundler update \-\-patch\fR\. -. .IP "\(bu" 4 \fBprint_only_version_number\fR (\fBBUNDLE_PRINT_ONLY_VERSION_NUMBER\fR): Print only version number from \fBbundler \-\-version\fR\. -. .IP "\(bu" 4 \fBredirect\fR (\fBBUNDLE_REDIRECT\fR): The number of redirects allowed for network requests\. Defaults to \fB5\fR\. -. .IP "\(bu" 4 \fBretry\fR (\fBBUNDLE_RETRY\fR): The number of times to retry failed network requests\. Defaults to \fB3\fR\. -. .IP "\(bu" 4 \fBsetup_makes_kernel_gem_public\fR (\fBBUNDLE_SETUP_MAKES_KERNEL_GEM_PUBLIC\fR): Have \fBBundler\.setup\fR make the \fBKernel#gem\fR method public, even though RubyGems declares it as private\. -. .IP "\(bu" 4 \fBshebang\fR (\fBBUNDLE_SHEBANG\fR): The program name that should be invoked for generated binstubs\. Defaults to the ruby install name used to generate the binstub\. -. .IP "\(bu" 4 \fBsilence_deprecations\fR (\fBBUNDLE_SILENCE_DEPRECATIONS\fR): Whether Bundler should silence deprecation warnings for behavior that will be changed in the next major version\. -. .IP "\(bu" 4 \fBsilence_root_warning\fR (\fBBUNDLE_SILENCE_ROOT_WARNING\fR): Silence the warning Bundler prints when installing gems as root\. -. .IP "\(bu" 4 \fBssl_ca_cert\fR (\fBBUNDLE_SSL_CA_CERT\fR): Path to a designated CA certificate file or folder containing multiple certificates for trusted CAs in PEM format\. -. .IP "\(bu" 4 \fBssl_client_cert\fR (\fBBUNDLE_SSL_CLIENT_CERT\fR): Path to a designated file containing a X\.509 client certificate and key in PEM format\. -. .IP "\(bu" 4 \fBssl_verify_mode\fR (\fBBUNDLE_SSL_VERIFY_MODE\fR): The SSL verification mode Bundler uses when making HTTPS requests\. Defaults to verify peer\. -. -.IP "\(bu" 4 -\fBsuppress_install_using_messages\fR (\fBBUNDLE_SUPPRESS_INSTALL_USING_MESSAGES\fR): Avoid printing \fBUsing \.\.\.\fR messages during installation when the version of a gem has not changed\. -. .IP "\(bu" 4 \fBsystem_bindir\fR (\fBBUNDLE_SYSTEM_BINDIR\fR): The location where RubyGems installs binstubs\. Defaults to \fBGem\.bindir\fR\. -. .IP "\(bu" 4 \fBtimeout\fR (\fBBUNDLE_TIMEOUT\fR): The seconds allowed before timing out for network requests\. Defaults to \fB10\fR\. -. .IP "\(bu" 4 \fBupdate_requires_all_flag\fR (\fBBUNDLE_UPDATE_REQUIRES_ALL_FLAG\fR): Require passing \fB\-\-all\fR to \fBbundle update\fR when everything should be updated, and disallow passing no options to \fBbundle update\fR\. -. .IP "\(bu" 4 \fBuser_agent\fR (\fBBUNDLE_USER_AGENT\fR): The custom user agent fragment Bundler includes in API requests\. -. +.IP "\(bu" 4 +\fBversion\fR (\fBBUNDLE_VERSION\fR): The version of Bundler to use when running under Bundler environment\. Defaults to \fBlockfile\fR\. You can also specify \fBsystem\fR or \fBx\.y\.z\fR\. \fBlockfile\fR will use the Bundler version specified in the \fBGemfile\.lock\fR, \fBsystem\fR will use the system version of Bundler, and \fBx\.y\.z\fR will use the specified version of Bundler\. .IP "\(bu" 4 \fBwith\fR (\fBBUNDLE_WITH\fR): A \fB:\fR\-separated list of groups whose gems bundler should install\. -. .IP "\(bu" 4 \fBwithout\fR (\fBBUNDLE_WITHOUT\fR): A \fB:\fR\-separated list of groups whose gems bundler should not install\. -. .IP "" 0 -. .P In general, you should set these settings per\-application by using the applicable flag to the bundle install(1) \fIbundle\-install\.1\.html\fR or bundle cache(1) \fIbundle\-cache\.1\.html\fR command\. -. .P You can set them globally either via environment variables or \fBbundle config\fR, whichever is preferable for your setup\. If you use both, environment variables will take preference over global settings\. -. .SH "LOCAL GIT REPOS" Bundler also allows you to work against a git repository locally instead of using the remote version\. This can be achieved by setting up a local override: -. .IP "" 4 -. .nf - bundle config set \-\-local local\.GEM_NAME /path/to/local/git/repository -. .fi -. .IP "" 0 -. .P For example, in order to use a local Rack repository, a developer could call: -. .IP "" 4 -. .nf - bundle config set \-\-local local\.rack ~/Work/git/rack -. .fi -. .IP "" 0 -. .P -Now instead of checking out the remote git repository, the local override will be used\. Similar to a path source, every time the local git repository change, changes will be automatically picked up by Bundler\. This means a commit in the local git repo will update the revision in the \fBGemfile\.lock\fR to the local git repo revision\. This requires the same attention as git submodules\. Before pushing to the remote, you need to ensure the local override was pushed, otherwise you may point to a commit that only exists in your local machine\. You\'ll also need to CGI escape your usernames and passwords as well\. -. +Now instead of checking out the remote git repository, the local override will be used\. Similar to a path source, every time the local git repository change, changes will be automatically picked up by Bundler\. This means a commit in the local git repo will update the revision in the \fBGemfile\.lock\fR to the local git repo revision\. This requires the same attention as git submodules\. Before pushing to the remote, you need to ensure the local override was pushed, otherwise you may point to a commit that only exists in your local machine\. You'll also need to CGI escape your usernames and passwords as well\. .P -Bundler does many checks to ensure a developer won\'t work with invalid references\. Particularly, we force a developer to specify a branch in the \fBGemfile\fR in order to use this feature\. If the branch specified in the \fBGemfile\fR and the current branch in the local git repository do not match, Bundler will abort\. This ensures that a developer is always working against the correct branches, and prevents accidental locking to a different branch\. -. +Bundler does many checks to ensure a developer won't work with invalid references\. Particularly, we force a developer to specify a branch in the \fBGemfile\fR in order to use this feature\. If the branch specified in the \fBGemfile\fR and the current branch in the local git repository do not match, Bundler will abort\. This ensures that a developer is always working against the correct branches, and prevents accidental locking to a different branch\. .P Finally, Bundler also ensures that the current revision in the \fBGemfile\.lock\fR exists in the local git repository\. By doing this, Bundler forces you to fetch the latest changes in the remotes\. -. .SH "MIRRORS OF GEM SOURCES" Bundler supports overriding gem sources with mirrors\. This allows you to configure rubygems\.org as the gem source in your Gemfile while still using your mirror to fetch gems\. -. .IP "" 4 -. .nf - bundle config set \-\-global mirror\.SOURCE_URL MIRROR_URL -. .fi -. .IP "" 0 -. .P For example, to use a mirror of https://rubygems\.org hosted at https://example\.org: -. .IP "" 4 -. .nf - bundle config set \-\-global mirror\.https://rubygems\.org https://example\.org -. .fi -. .IP "" 0 -. .P Each mirror also provides a fallback timeout setting\. If the mirror does not respond within the fallback timeout, Bundler will try to use the original server instead of the mirror\. -. .IP "" 4 -. .nf - bundle config set \-\-global mirror\.SOURCE_URL\.fallback_timeout TIMEOUT -. .fi -. .IP "" 0 -. .P For example, to fall back to rubygems\.org after 3 seconds: -. .IP "" 4 -. .nf - bundle config set \-\-global mirror\.https://rubygems\.org\.fallback_timeout 3 -. .fi -. .IP "" 0 -. .P The default fallback timeout is 0\.1 seconds, but the setting can currently only accept whole seconds (for example, 1, 15, or 30)\. -. .SH "CREDENTIALS FOR GEM SOURCES" Bundler allows you to configure credentials for any gem source, which allows you to avoid putting secrets into your Gemfile\. -. .IP "" 4 -. .nf - bundle config set \-\-global SOURCE_HOSTNAME USERNAME:PASSWORD -. .fi -. .IP "" 0 -. .P For example, to save the credentials of user \fBclaudette\fR for the gem source at \fBgems\.longerous\.com\fR, you would run: -. .IP "" 4 -. .nf - bundle config set \-\-global gems\.longerous\.com claudette:s00pers3krit -. .fi -. .IP "" 0 -. .P Or you can set the credentials as an environment variable like this: -. .IP "" 4 -. .nf - export BUNDLE_GEMS__LONGEROUS__COM="claudette:s00pers3krit" -. .fi -. .IP "" 0 -. .P For gems with a git source with HTTP(S) URL you can specify credentials like so: -. .IP "" 4 -. .nf - bundle config set \-\-global https://github\.com/rubygems/rubygems\.git username:password -. .fi -. .IP "" 0 -. .P Or you can set the credentials as an environment variable like so: -. .IP "" 4 -. .nf - export BUNDLE_GITHUB__COM=username:password -. .fi -. .IP "" 0 -. .P This is especially useful for private repositories on hosts such as GitHub, where you can use personal OAuth tokens: -. .IP "" 4 -. .nf - export BUNDLE_GITHUB__COM=abcd0123generatedtoken:x\-oauth\-basic -. .fi -. .IP "" 0 -. .P Note that any configured credentials will be redacted by informative commands such as \fBbundle config list\fR or \fBbundle config get\fR, unless you use the \fB\-\-parseable\fR flag\. This is to avoid unintentionally leaking credentials when copy\-pasting bundler output\. -. .P Also note that to guarantee a sane mapping between valid environment variable names and valid host names, bundler makes the following transformations: -. .IP "\(bu" 4 -Any \fB\-\fR characters in a host name are mapped to a triple dash (\fB___\fR) in the corresponding environment variable\. -. +Any \fB\-\fR characters in a host name are mapped to a triple underscore (\fB___\fR) in the corresponding environment variable\. .IP "\(bu" 4 -Any \fB\.\fR characters in a host name are mapped to a double dash (\fB__\fR) in the corresponding environment variable\. -. +Any \fB\.\fR characters in a host name are mapped to a double underscore (\fB__\fR) in the corresponding environment variable\. .IP "" 0 -. .P -This means that if you have a gem server named \fBmy\.gem\-host\.com\fR, you\'ll need to use the \fBBUNDLE_MY__GEM___HOST__COM\fR variable to configure credentials for it through ENV\. -. +This means that if you have a gem server named \fBmy\.gem\-host\.com\fR, you'll need to use the \fBBUNDLE_MY__GEM___HOST__COM\fR variable to configure credentials for it through ENV\. .SH "CONFIGURE BUNDLER DIRECTORIES" -Bundler\'s home, config, cache and plugin directories are able to be configured through environment variables\. The default location for Bundler\'s home directory is \fB~/\.bundle\fR, which all directories inherit from by default\. The following outlines the available environment variables and their default values -. +Bundler's home, config, cache and plugin directories are able to be configured through environment variables\. The default location for Bundler's home directory is \fB~/\.bundle\fR, which all directories inherit from by default\. The following outlines the available environment variables and their default values .IP "" 4 -. .nf - BUNDLE_USER_HOME : $HOME/\.bundle BUNDLE_USER_CACHE : $BUNDLE_USER_HOME/cache BUNDLE_USER_CONFIG : $BUNDLE_USER_HOME/config BUNDLE_USER_PLUGIN : $BUNDLE_USER_HOME/plugin -. .fi -. .IP "" 0 diff --git a/lib/bundler/man/bundle-config.1.ronn b/lib/bundler/man/bundle-config.1.ronn index bc8b27cf89..7e5f458fb2 100644 --- a/lib/bundler/man/bundle-config.1.ronn +++ b/lib/bundler/man/bundle-config.1.ronn @@ -137,9 +137,6 @@ the environment variable `BUNDLE_LOCAL__RACK`. The following is a list of all configuration keys and their purpose. You can learn more about their operation in [bundle install(1)](bundle-install.1.html). -* `allow_deployment_source_credential_changes` (`BUNDLE_ALLOW_DEPLOYMENT_SOURCE_CREDENTIAL_CHANGES`): - When in deployment mode, allow changing the credentials to a gem's source. - Ex: `https://some.host.com/gems/path/` -> `https://user_name:password@some.host.com/gems/path` * `allow_offline_install` (`BUNDLE_ALLOW_OFFLINE_INSTALL`): Allow Bundler to use cached data when installing without network access. * `auto_clean_without_path` (`BUNDLE_AUTO_CLEAN_WITHOUT_PATH`): @@ -265,9 +262,6 @@ learn more about their operation in [bundle install(1)](bundle-install.1.html). * `ssl_verify_mode` (`BUNDLE_SSL_VERIFY_MODE`): The SSL verification mode Bundler uses when making HTTPS requests. Defaults to verify peer. -* `suppress_install_using_messages` (`BUNDLE_SUPPRESS_INSTALL_USING_MESSAGES`): - Avoid printing `Using ...` messages during installation when the version of - a gem has not changed. * `system_bindir` (`BUNDLE_SYSTEM_BINDIR`): The location where RubyGems installs binstubs. Defaults to `Gem.bindir`. * `timeout` (`BUNDLE_TIMEOUT`): @@ -277,6 +271,12 @@ learn more about their operation in [bundle install(1)](bundle-install.1.html). and disallow passing no options to `bundle update`. * `user_agent` (`BUNDLE_USER_AGENT`): The custom user agent fragment Bundler includes in API requests. +* `version` (`BUNDLE_VERSION`): + The version of Bundler to use when running under Bundler environment. + Defaults to `lockfile`. You can also specify `system` or `x.y.z`. + `lockfile` will use the Bundler version specified in the `Gemfile.lock`, + `system` will use the system version of Bundler, and `x.y.z` will use + the specified version of Bundler. * `with` (`BUNDLE_WITH`): A `:`-separated list of groups whose gems bundler should install. * `without` (`BUNDLE_WITHOUT`): @@ -385,10 +385,10 @@ copy-pasting bundler output. Also note that to guarantee a sane mapping between valid environment variable names and valid host names, bundler makes the following transformations: -* Any `-` characters in a host name are mapped to a triple dash (`___`) in the +* Any `-` characters in a host name are mapped to a triple underscore (`___`) in the corresponding environment variable. -* Any `.` characters in a host name are mapped to a double dash (`__`) in the +* Any `.` characters in a host name are mapped to a double underscore (`__`) in the corresponding environment variable. This means that if you have a gem server named `my.gem-host.com`, you'll need to diff --git a/lib/bundler/man/bundle-console.1 b/lib/bundler/man/bundle-console.1 index ff239004bf..dca18ec43d 100644 --- a/lib/bundler/man/bundle-console.1 +++ b/lib/bundler/man/bundle-console.1 @@ -1,53 +1,35 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-CONSOLE" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-CONSOLE" "1" "May 2024" "" .SH "NAME" \fBbundle\-console\fR \- Deprecated way to open an IRB session with the bundle pre\-loaded -. .SH "SYNOPSIS" \fBbundle console\fR [GROUP] -. .SH "DESCRIPTION" Starts an interactive Ruby console session in the context of the current bundle\. -. .P If no \fBGROUP\fR is specified, all gems in the \fBdefault\fR group in the Gemfile(5) \fIhttps://bundler\.io/man/gemfile\.5\.html\fR are preliminarily loaded\. -. .P If \fBGROUP\fR is specified, all gems in the given group in the Gemfile in addition to the gems in \fBdefault\fR group are loaded\. Even if the given group does not exist in the Gemfile, IRB console starts without any warning or error\. -. .P The environment variable \fBBUNDLE_CONSOLE\fR or \fBbundle config set console\fR can be used to change the shell from the following: -. .IP "\(bu" 4 \fBirb\fR (default) -. .IP "\(bu" 4 \fBpry\fR (https://github\.com/pry/pry) -. .IP "\(bu" 4 \fBripl\fR (https://github\.com/cldwalker/ripl) -. .IP "" 0 -. .P \fBbundle console\fR uses irb by default\. An alternative Pry or Ripl can be used with \fBbundle console\fR by adjusting the \fBconsole\fR Bundler setting\. Also make sure that \fBpry\fR or \fBripl\fR is in your Gemfile\. -. .SH "EXAMPLE" -. .nf - $ bundle config set console pry $ bundle console -Resolving dependencies\.\.\. +Resolving dependencies\|\.\|\.\|\. [1] pry(main)> -. .fi -. .SH "NOTES" This command was deprecated in Bundler 2\.1 and will be removed in 3\.0\. Use \fBbin/console\fR script, which can be generated by \fBbundle gem <NAME>\fR\. -. .SH "SEE ALSO" Gemfile(5) \fIhttps://bundler\.io/man/gemfile\.5\.html\fR diff --git a/lib/bundler/man/bundle-doctor.1 b/lib/bundler/man/bundle-doctor.1 index e463b67477..6489cc07f7 100644 --- a/lib/bundler/man/bundle-doctor.1 +++ b/lib/bundler/man/bundle-doctor.1 @@ -1,44 +1,30 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-DOCTOR" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-DOCTOR" "1" "May 2024" "" .SH "NAME" \fBbundle\-doctor\fR \- Checks the bundle for common problems -. .SH "SYNOPSIS" \fBbundle doctor\fR [\-\-quiet] [\-\-gemfile=GEMFILE] -. .SH "DESCRIPTION" Checks your Gemfile and gem environment for common problems\. If issues are detected, Bundler prints them and exits status 1\. Otherwise, Bundler prints a success message and exits status 0\. -. .P Examples of common problems caught by bundle\-doctor include: -. .IP "\(bu" 4 Invalid Bundler settings -. .IP "\(bu" 4 Mismatched Ruby versions -. .IP "\(bu" 4 Mismatched platforms -. .IP "\(bu" 4 Uninstalled gems -. .IP "\(bu" 4 Missing dependencies -. .IP "" 0 -. .SH "OPTIONS" -. .TP \fB\-\-quiet\fR Only output warnings and errors\. -. .TP \fB\-\-gemfile=<gemfile>\fR -The location of the Gemfile(5) which Bundler should use\. This defaults to a Gemfile(5) in the current working directory\. In general, Bundler will assume that the location of the Gemfile(5) is also the project\'s root and will try to find \fBGemfile\.lock\fR and \fBvendor/cache\fR relative to this location\. +The location of the Gemfile(5) which Bundler should use\. This defaults to a Gemfile(5) in the current working directory\. In general, Bundler will assume that the location of the Gemfile(5) is also the project's root and will try to find \fBGemfile\.lock\fR and \fBvendor/cache\fR relative to this location\. diff --git a/lib/bundler/man/bundle-exec.1 b/lib/bundler/man/bundle-exec.1 index 9e9efe8b6d..1548d29670 100644 --- a/lib/bundler/man/bundle-exec.1 +++ b/lib/bundler/man/bundle-exec.1 @@ -1,165 +1,104 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-EXEC" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-EXEC" "1" "May 2024" "" .SH "NAME" \fBbundle\-exec\fR \- Execute a command in the context of the bundle -. .SH "SYNOPSIS" \fBbundle exec\fR [\-\-keep\-file\-descriptors] \fIcommand\fR -. .SH "DESCRIPTION" This command executes the command, making all gems specified in the [\fBGemfile(5)\fR][Gemfile(5)] available to \fBrequire\fR in Ruby programs\. -. .P Essentially, if you would normally have run something like \fBrspec spec/my_spec\.rb\fR, and you want to use the gems specified in the [\fBGemfile(5)\fR][Gemfile(5)] and installed via bundle install(1) \fIbundle\-install\.1\.html\fR, you should run \fBbundle exec rspec spec/my_spec\.rb\fR\. -. .P -Note that \fBbundle exec\fR does not require that an executable is available on your shell\'s \fB$PATH\fR\. -. +Note that \fBbundle exec\fR does not require that an executable is available on your shell's \fB$PATH\fR\. .SH "OPTIONS" -. .TP \fB\-\-keep\-file\-descriptors\fR -Exec in Ruby 2\.0 began discarding non\-standard file descriptors\. When this flag is passed, exec will revert to the 1\.9 behaviour of passing all file descriptors to the new process\. -. +Passes all file descriptors to the new processes\. Default is true from bundler version 2\.2\.26\. Setting it to false is now deprecated\. .SH "BUNDLE INSTALL \-\-BINSTUBS" If you use the \fB\-\-binstubs\fR flag in bundle install(1) \fIbundle\-install\.1\.html\fR, Bundler will automatically create a directory (which defaults to \fBapp_root/bin\fR) containing all of the executables available from gems in the bundle\. -. .P After using \fB\-\-binstubs\fR, \fBbin/rspec spec/my_spec\.rb\fR is identical to \fBbundle exec rspec spec/my_spec\.rb\fR\. -. .SH "ENVIRONMENT MODIFICATIONS" \fBbundle exec\fR makes a number of changes to the shell environment, then executes the command you specify in full\. -. .IP "\(bu" 4 -make sure that it\'s still possible to shell out to \fBbundle\fR from inside a command invoked by \fBbundle exec\fR (using \fB$BUNDLE_BIN_PATH\fR) -. +make sure that it's still possible to shell out to \fBbundle\fR from inside a command invoked by \fBbundle exec\fR (using \fB$BUNDLE_BIN_PATH\fR) .IP "\(bu" 4 put the directory containing executables (like \fBrails\fR, \fBrspec\fR, \fBrackup\fR) for your bundle on \fB$PATH\fR -. .IP "\(bu" 4 make sure that if bundler is invoked in the subshell, it uses the same \fBGemfile\fR (by setting \fBBUNDLE_GEMFILE\fR) -. .IP "\(bu" 4 add \fB\-rbundler/setup\fR to \fB$RUBYOPT\fR, which makes sure that Ruby programs invoked in the subshell can see the gems in the bundle -. .IP "" 0 -. .P It also modifies Rubygems: -. .IP "\(bu" 4 disallow loading additional gems not in the bundle -. .IP "\(bu" 4 -modify the \fBgem\fR method to be a no\-op if a gem matching the requirements is in the bundle, and to raise a \fBGem::LoadError\fR if it\'s not -. +modify the \fBgem\fR method to be a no\-op if a gem matching the requirements is in the bundle, and to raise a \fBGem::LoadError\fR if it's not .IP "\(bu" 4 Define \fBGem\.refresh\fR to be a no\-op, since the source index is always frozen when using bundler, and to prevent gems from the system leaking into the environment -. .IP "\(bu" 4 Override \fBGem\.bin_path\fR to use the gems in the bundle, making system executables work -. .IP "\(bu" 4 Add all gems in the bundle into Gem\.loaded_specs -. .IP "" 0 -. .P -Finally, \fBbundle exec\fR also implicitly modifies \fBGemfile\.lock\fR if the lockfile and the Gemfile do not match\. Bundler needs the Gemfile to determine things such as a gem\'s groups, \fBautorequire\fR, and platforms, etc\., and that information isn\'t stored in the lockfile\. The Gemfile and lockfile must be synced in order to \fBbundle exec\fR successfully, so \fBbundle exec\fR updates the lockfile beforehand\. -. +Finally, \fBbundle exec\fR also implicitly modifies \fBGemfile\.lock\fR if the lockfile and the Gemfile do not match\. Bundler needs the Gemfile to determine things such as a gem's groups, \fBautorequire\fR, and platforms, etc\., and that information isn't stored in the lockfile\. The Gemfile and lockfile must be synced in order to \fBbundle exec\fR successfully, so \fBbundle exec\fR updates the lockfile beforehand\. .SS "Loading" By default, when attempting to \fBbundle exec\fR to a file with a ruby shebang, Bundler will \fBKernel\.load\fR that file instead of using \fBKernel\.exec\fR\. For the vast majority of cases, this is a performance improvement\. In a rare few cases, this could cause some subtle side\-effects (such as dependence on the exact contents of \fB$0\fR or \fB__FILE__\fR) and the optimization can be disabled by enabling the \fBdisable_exec_load\fR setting\. -. .SS "Shelling out" -Any Ruby code that opens a subshell (like \fBsystem\fR, backticks, or \fB%x{}\fR) will automatically use the current Bundler environment\. If you need to shell out to a Ruby command that is not part of your current bundle, use the \fBwith_unbundled_env\fR method with a block\. Any subshells created inside the block will be given the environment present before Bundler was activated\. For example, Homebrew commands run Ruby, but don\'t work inside a bundle: -. +Any Ruby code that opens a subshell (like \fBsystem\fR, backticks, or \fB%x{}\fR) will automatically use the current Bundler environment\. If you need to shell out to a Ruby command that is not part of your current bundle, use the \fBwith_unbundled_env\fR method with a block\. Any subshells created inside the block will be given the environment present before Bundler was activated\. For example, Homebrew commands run Ruby, but don't work inside a bundle: .IP "" 4 -. .nf - Bundler\.with_unbundled_env do `brew install wget` end -. .fi -. .IP "" 0 -. .P Using \fBwith_unbundled_env\fR is also necessary if you are shelling out to a different bundle\. Any Bundler commands run in a subshell will inherit the current Gemfile, so commands that need to run in the context of a different bundle also need to use \fBwith_unbundled_env\fR\. -. .IP "" 4 -. .nf - Bundler\.with_unbundled_env do Dir\.chdir "/other/bundler/project" do `bundle exec \./script` end end -. .fi -. .IP "" 0 -. .P Bundler provides convenience helpers that wrap \fBsystem\fR and \fBexec\fR, and they can be used like this: -. .IP "" 4 -. .nf - -Bundler\.clean_system(\'brew install wget\') -Bundler\.clean_exec(\'brew install wget\') -. +Bundler\.clean_system('brew install wget') +Bundler\.clean_exec('brew install wget') .fi -. .IP "" 0 -. .SH "RUBYGEMS PLUGINS" At present, the Rubygems plugin system requires all files named \fBrubygems_plugin\.rb\fR on the load path of \fIany\fR installed gem when any Ruby code requires \fBrubygems\.rb\fR\. This includes executables installed into the system, like \fBrails\fR, \fBrackup\fR, and \fBrspec\fR\. -. .P Since Rubygems plugins can contain arbitrary Ruby code, they commonly end up activating themselves or their dependencies\. -. .P For instance, the \fBgemcutter 0\.5\fR gem depended on \fBjson_pure\fR\. If you had that version of gemcutter installed (even if you \fIalso\fR had a newer version without this problem), Rubygems would activate \fBgemcutter 0\.5\fR and \fBjson_pure <latest>\fR\. -. .P If your Gemfile(5) also contained \fBjson_pure\fR (or a gem with a dependency on \fBjson_pure\fR), the latest version on your system might conflict with the version in your Gemfile(5), or the snapshot version in your \fBGemfile\.lock\fR\. -. .P If this happens, bundler will say: -. .IP "" 4 -. .nf - You have already activated json_pure 1\.4\.6 but your Gemfile requires json_pure 1\.4\.3\. Consider using bundle exec\. -. .fi -. .IP "" 0 -. .P In this situation, you almost certainly want to remove the underlying gem with the problematic gem plugin\. In general, the authors of these plugins (in this case, the \fBgemcutter\fR gem) have released newer versions that are more careful in their plugins\. -. .P You can find a list of all the gems containing gem plugins by running -. .IP "" 4 -. .nf - -ruby \-e "puts Gem\.find_files(\'rubygems_plugin\.rb\')" -. +ruby \-e "puts Gem\.find_files('rubygems_plugin\.rb')" .fi -. .IP "" 0 -. .P -At the very least, you should remove all but the newest version of each gem plugin, and also remove all gem plugins that you aren\'t using (\fBgem uninstall gem_name\fR)\. +At the very least, you should remove all but the newest version of each gem plugin, and also remove all gem plugins that you aren't using (\fBgem uninstall gem_name\fR)\. diff --git a/lib/bundler/man/bundle-exec.1.ronn b/lib/bundler/man/bundle-exec.1.ronn index 05948095e2..9d5b559f26 100644 --- a/lib/bundler/man/bundle-exec.1.ronn +++ b/lib/bundler/man/bundle-exec.1.ronn @@ -21,9 +21,8 @@ available on your shell's `$PATH`. ## OPTIONS * `--keep-file-descriptors`: - Exec in Ruby 2.0 began discarding non-standard file descriptors. When this - flag is passed, exec will revert to the 1.9 behaviour of passing all file - descriptors to the new process. + Passes all file descriptors to the new processes. Default is true from + bundler version 2.2.26. Setting it to false is now deprecated. ## BUNDLE INSTALL --BINSTUBS diff --git a/lib/bundler/man/bundle-gem.1 b/lib/bundler/man/bundle-gem.1 index ea64871fb6..5df7b0ef2f 100644 --- a/lib/bundler/man/bundle-gem.1 +++ b/lib/bundler/man/bundle-gem.1 @@ -1,105 +1,69 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-GEM" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-GEM" "1" "May 2024" "" .SH "NAME" \fBbundle\-gem\fR \- Generate a project skeleton for creating a rubygem -. .SH "SYNOPSIS" \fBbundle gem\fR \fIGEM_NAME\fR \fIOPTIONS\fR -. .SH "DESCRIPTION" Generates a directory named \fBGEM_NAME\fR with a \fBRakefile\fR, \fBGEM_NAME\.gemspec\fR, and other supporting files and directories that can be used to develop a rubygem with that name\. -. .P Run \fBrake \-T\fR in the resulting project for a list of Rake tasks that can be used to test and publish the gem to rubygems\.org\. -. .P -The generated project skeleton can be customized with OPTIONS, as explained below\. Note that these options can also be specified via Bundler\'s global configuration file using the following names: -. +The generated project skeleton can be customized with OPTIONS, as explained below\. Note that these options can also be specified via Bundler's global configuration file using the following names: .IP "\(bu" 4 \fBgem\.coc\fR -. .IP "\(bu" 4 \fBgem\.mit\fR -. .IP "\(bu" 4 \fBgem\.test\fR -. .IP "" 0 -. .SH "OPTIONS" -. .IP "\(bu" 4 \fB\-\-exe\fR or \fB\-b\fR or \fB\-\-bin\fR: Specify that Bundler should create a binary executable (as \fBexe/GEM_NAME\fR) in the generated rubygem project\. This binary will also be added to the \fBGEM_NAME\.gemspec\fR manifest\. This behavior is disabled by default\. -. .IP "\(bu" 4 \fB\-\-no\-exe\fR: Do not create a binary (overrides \fB\-\-exe\fR specified in the global config)\. -. .IP "\(bu" 4 -\fB\-\-coc\fR: Add a \fBCODE_OF_CONDUCT\.md\fR file to the root of the generated project\. If this option is unspecified, an interactive prompt will be displayed and the answer will be saved in Bundler\'s global config for future \fBbundle gem\fR use\. -. +\fB\-\-coc\fR: Add a \fBCODE_OF_CONDUCT\.md\fR file to the root of the generated project\. If this option is unspecified, an interactive prompt will be displayed and the answer will be saved in Bundler's global config for future \fBbundle gem\fR use\. .IP "\(bu" 4 \fB\-\-no\-coc\fR: Do not create a \fBCODE_OF_CONDUCT\.md\fR (overrides \fB\-\-coc\fR specified in the global config)\. -. .IP "\(bu" 4 \fB\-\-ext=c\fR, \fB\-\-ext=rust\fR Add boilerplate for C or Rust (currently magnus \fIhttps://docs\.rs/magnus\fR based) extension code to the generated project\. This behavior is disabled by default\. -. .IP "\(bu" 4 \fB\-\-no\-ext\fR: Do not add extension code (overrides \fB\-\-ext\fR specified in the global config)\. -. .IP "\(bu" 4 -\fB\-\-mit\fR: Add an MIT license to a \fBLICENSE\.txt\fR file in the root of the generated project\. Your name from the global git config is used for the copyright statement\. If this option is unspecified, an interactive prompt will be displayed and the answer will be saved in Bundler\'s global config for future \fBbundle gem\fR use\. -. +\fB\-\-mit\fR: Add an MIT license to a \fBLICENSE\.txt\fR file in the root of the generated project\. Your name from the global git config is used for the copyright statement\. If this option is unspecified, an interactive prompt will be displayed and the answer will be saved in Bundler's global config for future \fBbundle gem\fR use\. .IP "\(bu" 4 \fB\-\-no\-mit\fR: Do not create a \fBLICENSE\.txt\fR (overrides \fB\-\-mit\fR specified in the global config)\. -. .IP "\(bu" 4 \fB\-t\fR, \fB\-\-test=minitest\fR, \fB\-\-test=rspec\fR, \fB\-\-test=test\-unit\fR: Specify the test framework that Bundler should use when generating the project\. Acceptable values are \fBminitest\fR, \fBrspec\fR and \fBtest\-unit\fR\. The \fBGEM_NAME\.gemspec\fR will be configured and a skeleton test/spec directory will be created based on this option\. Given no option is specified: -. .IP -When Bundler is configured to generate tests, this defaults to Bundler\'s global config setting \fBgem\.test\fR\. -. +When Bundler is configured to generate tests, this defaults to Bundler's global config setting \fBgem\.test\fR\. .IP When Bundler is configured to not generate tests, an interactive prompt will be displayed and the answer will be used for the current rubygem project\. -. .IP -When Bundler is unconfigured, an interactive prompt will be displayed and the answer will be saved in Bundler\'s global config for future \fBbundle gem\fR use\. -. +When Bundler is unconfigured, an interactive prompt will be displayed and the answer will be saved in Bundler's global config for future \fBbundle gem\fR use\. .IP "\(bu" 4 \fB\-\-ci\fR, \fB\-\-ci=github\fR, \fB\-\-ci=gitlab\fR, \fB\-\-ci=circle\fR: Specify the continuous integration service that Bundler should use when generating the project\. Acceptable values are \fBgithub\fR, \fBgitlab\fR and \fBcircle\fR\. A configuration file will be generated in the project directory\. Given no option is specified: -. .IP -When Bundler is configured to generate CI files, this defaults to Bundler\'s global config setting \fBgem\.ci\fR\. -. +When Bundler is configured to generate CI files, this defaults to Bundler's global config setting \fBgem\.ci\fR\. .IP When Bundler is configured to not generate CI files, an interactive prompt will be displayed and the answer will be used for the current rubygem project\. -. .IP -When Bundler is unconfigured, an interactive prompt will be displayed and the answer will be saved in Bundler\'s global config for future \fBbundle gem\fR use\. -. +When Bundler is unconfigured, an interactive prompt will be displayed and the answer will be saved in Bundler's global config for future \fBbundle gem\fR use\. .IP "\(bu" 4 -\fB\-\-linter\fR, \fB\-\-linter=rubocop\fR, \fB\-\-linter=standard\fR: Specify the linter and code formatter that Bundler should add to the project\'s development dependencies\. Acceptable values are \fBrubocop\fR and \fBstandard\fR\. A configuration file will be generated in the project directory\. Given no option is specified: -. +\fB\-\-linter\fR, \fB\-\-linter=rubocop\fR, \fB\-\-linter=standard\fR: Specify the linter and code formatter that Bundler should add to the project's development dependencies\. Acceptable values are \fBrubocop\fR and \fBstandard\fR\. A configuration file will be generated in the project directory\. Given no option is specified: .IP -When Bundler is configured to add a linter, this defaults to Bundler\'s global config setting \fBgem\.linter\fR\. -. +When Bundler is configured to add a linter, this defaults to Bundler's global config setting \fBgem\.linter\fR\. .IP When Bundler is configured not to add a linter, an interactive prompt will be displayed and the answer will be used for the current rubygem project\. -. .IP -When Bundler is unconfigured, an interactive prompt will be displayed and the answer will be saved in Bundler\'s global config for future \fBbundle gem\fR use\. -. +When Bundler is unconfigured, an interactive prompt will be displayed and the answer will be saved in Bundler's global config for future \fBbundle gem\fR use\. .IP "\(bu" 4 \fB\-e\fR, \fB\-\-edit[=EDITOR]\fR: Open the resulting GEM_NAME\.gemspec in EDITOR, or the default editor if not specified\. The default is \fB$BUNDLER_EDITOR\fR, \fB$VISUAL\fR, or \fB$EDITOR\fR\. -. .IP "" 0 -. .SH "SEE ALSO" -. .IP "\(bu" 4 bundle config(1) \fIbundle\-config\.1\.html\fR -. .IP "" 0 diff --git a/lib/bundler/man/bundle-help.1 b/lib/bundler/man/bundle-help.1 index a3b059ea07..a3e7c7770d 100644 --- a/lib/bundler/man/bundle-help.1 +++ b/lib/bundler/man/bundle-help.1 @@ -1,13 +1,9 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-HELP" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-HELP" "1" "May 2024" "" .SH "NAME" \fBbundle\-help\fR \- Displays detailed help for each subcommand -. .SH "SYNOPSIS" \fBbundle help\fR [COMMAND] -. .SH "DESCRIPTION" Displays detailed help for the given subcommand\. You can specify a single \fBCOMMAND\fR at the same time\. When \fBCOMMAND\fR is omitted, help for \fBhelp\fR command will be displayed\. diff --git a/lib/bundler/man/bundle-info.1 b/lib/bundler/man/bundle-info.1 index 5af60c6a77..a3d7ff0988 100644 --- a/lib/bundler/man/bundle-info.1 +++ b/lib/bundler/man/bundle-info.1 @@ -1,19 +1,13 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-INFO" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-INFO" "1" "May 2024" "" .SH "NAME" \fBbundle\-info\fR \- Show information for the given gem in your bundle -. .SH "SYNOPSIS" -\fBbundle info\fR [GEM] [\-\-path] -. +\fBbundle info\fR [GEM_NAME] [\-\-path] .SH "DESCRIPTION" -Print the basic information about the provided GEM such as homepage, version, path and summary\. -. +Given a gem name present in your bundle, print the basic information about it such as homepage, version, path and summary\. .SH "OPTIONS" -. .TP \fB\-\-path\fR Print the path of the given gem diff --git a/lib/bundler/man/bundle-info.1.ronn b/lib/bundler/man/bundle-info.1.ronn index 47e457aa3c..cecdeb564f 100644 --- a/lib/bundler/man/bundle-info.1.ronn +++ b/lib/bundler/man/bundle-info.1.ronn @@ -3,13 +3,13 @@ bundle-info(1) -- Show information for the given gem in your bundle ## SYNOPSIS -`bundle info` [GEM] +`bundle info` [GEM_NAME] [--path] ## DESCRIPTION -Print the basic information about the provided GEM such as homepage, version, -path and summary. +Given a gem name present in your bundle, print the basic information about it + such as homepage, version, path and summary. ## OPTIONS diff --git a/lib/bundler/man/bundle-init.1 b/lib/bundler/man/bundle-init.1 index e93b4fd5e9..a0edaaa18f 100644 --- a/lib/bundler/man/bundle-init.1 +++ b/lib/bundler/man/bundle-init.1 @@ -1,29 +1,20 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-INIT" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-INIT" "1" "May 2024" "" .SH "NAME" \fBbundle\-init\fR \- Generates a Gemfile into the current working directory -. .SH "SYNOPSIS" \fBbundle init\fR [\-\-gemspec=FILE] -. .SH "DESCRIPTION" Init generates a default [\fBGemfile(5)\fR][Gemfile(5)] in the current working directory\. When adding a [\fBGemfile(5)\fR][Gemfile(5)] to a gem with a gemspec, the \fB\-\-gemspec\fR option will automatically add each dependency listed in the gemspec file to the newly created [\fBGemfile(5)\fR][Gemfile(5)]\. -. .SH "OPTIONS" -. .TP \fB\-\-gemspec\fR Use the specified \.gemspec to create the [\fBGemfile(5)\fR][Gemfile(5)] -. .TP \fB\-\-gemfile\fR Use the specified name for the gemfile instead of \fBGemfile\fR -. .SH "FILES" Included in the default [\fBGemfile(5)\fR][Gemfile(5)] generated is the line \fB# frozen_string_literal: true\fR\. This is a magic comment supported for the first time in Ruby 2\.3\. The presence of this line results in all string literals in the file being implicitly frozen\. -. .SH "SEE ALSO" Gemfile(5) \fIhttps://bundler\.io/man/gemfile\.5\.html\fR diff --git a/lib/bundler/man/bundle-inject.1 b/lib/bundler/man/bundle-inject.1 index 657d5ef4a7..7a1038206e 100644 --- a/lib/bundler/man/bundle-inject.1 +++ b/lib/bundler/man/bundle-inject.1 @@ -1,36 +1,23 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-INJECT" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-INJECT" "1" "May 2024" "" .SH "NAME" \fBbundle\-inject\fR \- Add named gem(s) with version requirements to Gemfile -. .SH "SYNOPSIS" \fBbundle inject\fR [GEM] [VERSION] -. .SH "DESCRIPTION" Adds the named gem(s) with their version requirements to the resolved [\fBGemfile(5)\fR][Gemfile(5)]\. -. .P -This command will add the gem to both your [\fBGemfile(5)\fR][Gemfile(5)] and Gemfile\.lock if it isn\'t listed yet\. -. +This command will add the gem to both your [\fBGemfile(5)\fR][Gemfile(5)] and Gemfile\.lock if it isn't listed yet\. .P Example: -. .IP "" 4 -. .nf - bundle install -bundle inject \'rack\' \'> 0\' -. +bundle inject 'rack' '> 0' .fi -. .IP "" 0 -. .P -This will inject the \'rack\' gem with a version greater than 0 in your [\fBGemfile(5)\fR][Gemfile(5)] and Gemfile\.lock\. -. +This will inject the 'rack' gem with a version greater than 0 in your [\fBGemfile(5)\fR][Gemfile(5)] and Gemfile\.lock\. .P The \fBbundle inject\fR command was deprecated in Bundler 2\.1 and will be removed in Bundler 3\.0\. diff --git a/lib/bundler/man/bundle-install.1 b/lib/bundler/man/bundle-install.1 index 8f144692f3..cc46a03b7f 100644 --- a/lib/bundler/man/bundle-install.1 +++ b/lib/bundler/man/bundle-install.1 @@ -1,313 +1,215 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-INSTALL" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-INSTALL" "1" "May 2024" "" .SH "NAME" \fBbundle\-install\fR \- Install the dependencies specified in your Gemfile -. .SH "SYNOPSIS" -\fBbundle install\fR [\-\-binstubs[=DIRECTORY]] [\-\-clean] [\-\-deployment] [\-\-frozen] [\-\-full\-index] [\-\-gemfile=GEMFILE] [\-\-jobs=NUMBER] [\-\-local] [\-\-no\-cache] [\-\-no\-prune] [\-\-path PATH] [\-\-quiet] [\-\-redownload] [\-\-retry=NUMBER] [\-\-shebang] [\-\-standalone[=GROUP[ GROUP\.\.\.]]] [\-\-system] [\-\-trust\-policy=POLICY] [\-\-with=GROUP[ GROUP\.\.\.]] [\-\-without=GROUP[ GROUP\.\.\.]] -. +\fBbundle install\fR [\-\-binstubs[=DIRECTORY]] [\-\-clean] [\-\-deployment] [\-\-frozen] [\-\-full\-index] [\-\-gemfile=GEMFILE] [\-\-jobs=NUMBER] [\-\-local] [\-\-no\-cache] [\-\-no\-prune] [\-\-path PATH] [\-\-prefer\-local] [\-\-quiet] [\-\-redownload] [\-\-retry=NUMBER] [\-\-shebang] [\-\-standalone[=GROUP[ GROUP\|\.\|\.\|\.]]] [\-\-system] [\-\-trust\-policy=POLICY] [\-\-with=GROUP[ GROUP\|\.\|\.\|\.]] [\-\-without=GROUP[ GROUP\|\.\|\.\|\.]] .SH "DESCRIPTION" Install the gems specified in your Gemfile(5)\. If this is the first time you run bundle install (and a \fBGemfile\.lock\fR does not exist), Bundler will fetch all remote sources, resolve dependencies and install all needed gems\. -. .P If a \fBGemfile\.lock\fR does exist, and you have not updated your Gemfile(5), Bundler will fetch all remote sources, but use the dependencies specified in the \fBGemfile\.lock\fR instead of resolving dependencies\. -. .P If a \fBGemfile\.lock\fR does exist, and you have updated your Gemfile(5), Bundler will use the dependencies in the \fBGemfile\.lock\fR for all gems that you did not update, but will re\-resolve the dependencies of gems that you did update\. You can find more information about this update process below under \fICONSERVATIVE UPDATING\fR\. -. .SH "OPTIONS" The \fB\-\-clean\fR, \fB\-\-deployment\fR, \fB\-\-frozen\fR, \fB\-\-no\-prune\fR, \fB\-\-path\fR, \fB\-\-shebang\fR, \fB\-\-system\fR, \fB\-\-without\fR and \fB\-\-with\fR options are deprecated because they only make sense if they are applied to every subsequent \fBbundle install\fR run automatically and that requires \fBbundler\fR to silently remember them\. Since \fBbundler\fR will no longer remember CLI flags in future versions, \fBbundle config\fR (see bundle\-config(1)) should be used to apply them permanently\. -. .TP \fB\-\-binstubs[=<directory>]\fR Binstubs are scripts that wrap around executables\. Bundler creates a small Ruby file (a binstub) that loads Bundler, runs the command, and puts it in \fBbin/\fR\. This lets you link the binstub inside of an application to the exact gem version the application needs\. -. .IP -Creates a directory (defaults to \fB~/bin\fR) and places any executables from the gem there\. These executables run in Bundler\'s context\. If used, you might add this directory to your environment\'s \fBPATH\fR variable\. For instance, if the \fBrails\fR gem comes with a \fBrails\fR executable, this flag will create a \fBbin/rails\fR executable that ensures that all referred dependencies will be resolved using the bundled gems\. -. +Creates a directory (defaults to \fB~/bin\fR) and places any executables from the gem there\. These executables run in Bundler's context\. If used, you might add this directory to your environment's \fBPATH\fR variable\. For instance, if the \fBrails\fR gem comes with a \fBrails\fR executable, this flag will create a \fBbin/rails\fR executable that ensures that all referred dependencies will be resolved using the bundled gems\. .TP \fB\-\-clean\fR -On finishing the installation Bundler is going to remove any gems not present in the current Gemfile(5)\. Don\'t worry, gems currently in use will not be removed\. -. +On finishing the installation Bundler is going to remove any gems not present in the current Gemfile(5)\. Don't worry, gems currently in use will not be removed\. .IP This option is deprecated in favor of the \fBclean\fR setting\. -. .TP \fB\-\-deployment\fR -In \fIdeployment mode\fR, Bundler will \'roll\-out\' the bundle for production or CI use\. Please check carefully if you want to have this option enabled in your development environment\. -. +In \fIdeployment mode\fR, Bundler will 'roll\-out' the bundle for production or CI use\. Please check carefully if you want to have this option enabled in your development environment\. .IP This option is deprecated in favor of the \fBdeployment\fR setting\. -. .TP \fB\-\-redownload\fR Force download every gem, even if the required versions are already available locally\. -. .TP \fB\-\-frozen\fR Do not allow the Gemfile\.lock to be updated after this install\. Exits non\-zero if there are going to be changes to the Gemfile\.lock\. -. .IP This option is deprecated in favor of the \fBfrozen\fR setting\. -. .TP \fB\-\-full\-index\fR -Bundler will not call Rubygems\' API endpoint (default) but download and cache a (currently big) index file of all gems\. Performance can be improved for large bundles that seldom change by enabling this option\. -. +Bundler will not call Rubygems' API endpoint (default) but download and cache a (currently big) index file of all gems\. Performance can be improved for large bundles that seldom change by enabling this option\. .TP \fB\-\-gemfile=<gemfile>\fR -The location of the Gemfile(5) which Bundler should use\. This defaults to a Gemfile(5) in the current working directory\. In general, Bundler will assume that the location of the Gemfile(5) is also the project\'s root and will try to find \fBGemfile\.lock\fR and \fBvendor/cache\fR relative to this location\. -. +The location of the Gemfile(5) which Bundler should use\. This defaults to a Gemfile(5) in the current working directory\. In general, Bundler will assume that the location of the Gemfile(5) is also the project's root and will try to find \fBGemfile\.lock\fR and \fBvendor/cache\fR relative to this location\. .TP \fB\-\-jobs=[<number>]\fR, \fB\-j[<number>]\fR The maximum number of parallel download and install jobs\. The default is the number of available processors\. -. .TP \fB\-\-local\fR -Do not attempt to connect to \fBrubygems\.org\fR\. Instead, Bundler will use the gems already present in Rubygems\' cache or in \fBvendor/cache\fR\. Note that if an appropriate platform\-specific gem exists on \fBrubygems\.org\fR it will not be found\. -. +Do not attempt to connect to \fBrubygems\.org\fR\. Instead, Bundler will use the gems already present in Rubygems' cache or in \fBvendor/cache\fR\. Note that if an appropriate platform\-specific gem exists on \fBrubygems\.org\fR it will not be found\. .TP \fB\-\-prefer\-local\fR -Force using locally installed gems, or gems already present in Rubygems\' cache or in \fBvendor/cache\fR, when resolving, even if newer versions are available remotely\. Only attempt to connect to \fBrubygems\.org\fR for gems that are not present locally\. -. +Force using locally installed gems, or gems already present in Rubygems' cache or in \fBvendor/cache\fR, when resolving, even if newer versions are available remotely\. Only attempt to connect to \fBrubygems\.org\fR for gems that are not present locally\. .TP \fB\-\-no\-cache\fR Do not update the cache in \fBvendor/cache\fR with the newly bundled gems\. This does not remove any gems in the cache but keeps the newly bundled gems from being cached during the install\. -. .TP \fB\-\-no\-prune\fR -Don\'t remove stale gems from the cache when the installation finishes\. -. +Don't remove stale gems from the cache when the installation finishes\. .IP This option is deprecated in favor of the \fBno_prune\fR setting\. -. .TP \fB\-\-path=<path>\fR -The location to install the specified gems to\. This defaults to Rubygems\' setting\. Bundler shares this location with Rubygems, \fBgem install \.\.\.\fR will have gem installed there, too\. Therefore, gems installed without a \fB\-\-path \.\.\.\fR setting will show up by calling \fBgem list\fR\. Accordingly, gems installed to other locations will not get listed\. -. +The location to install the specified gems to\. This defaults to Rubygems' setting\. Bundler shares this location with Rubygems, \fBgem install \|\.\|\.\|\.\fR will have gem installed there, too\. Therefore, gems installed without a \fB\-\-path \|\.\|\.\|\.\fR setting will show up by calling \fBgem list\fR\. Accordingly, gems installed to other locations will not get listed\. .IP This option is deprecated in favor of the \fBpath\fR setting\. -. .TP \fB\-\-quiet\fR Do not print progress information to the standard output\. Instead, Bundler will exit using a status code (\fB$?\fR)\. -. .TP \fB\-\-retry=[<number>]\fR Retry failed network or git requests for \fInumber\fR times\. -. .TP \fB\-\-shebang=<ruby\-executable>\fR Uses the specified ruby executable (usually \fBruby\fR) to execute the scripts created with \fB\-\-binstubs\fR\. In addition, if you use \fB\-\-binstubs\fR together with \fB\-\-shebang jruby\fR these executables will be changed to execute \fBjruby\fR instead\. -. .IP This option is deprecated in favor of the \fBshebang\fR setting\. -. .TP \fB\-\-standalone[=<list>]\fR -Makes a bundle that can work without depending on Rubygems or Bundler at runtime\. A space separated list of groups to install has to be specified\. Bundler creates a directory named \fBbundle\fR and installs the bundle there\. It also generates a \fBbundle/bundler/setup\.rb\fR file to replace Bundler\'s own setup in the manner required\. Using this option implicitly sets \fBpath\fR, which is a [remembered option][REMEMBERED OPTIONS]\. -. +Makes a bundle that can work without depending on Rubygems or Bundler at runtime\. A space separated list of groups to install has to be specified\. Bundler creates a directory named \fBbundle\fR and installs the bundle there\. It also generates a \fBbundle/bundler/setup\.rb\fR file to replace Bundler's own setup in the manner required\. Using this option implicitly sets \fBpath\fR, which is a [remembered option][REMEMBERED OPTIONS]\. .TP \fB\-\-system\fR -Installs the gems specified in the bundle to the system\'s Rubygems location\. This overrides any previous configuration of \fB\-\-path\fR\. -. +Installs the gems specified in the bundle to the system's Rubygems location\. This overrides any previous configuration of \fB\-\-path\fR\. .IP This option is deprecated in favor of the \fBsystem\fR setting\. -. .TP \fB\-\-trust\-policy=[<policy>]\fR Apply the Rubygems security policy \fIpolicy\fR, where policy is one of \fBHighSecurity\fR, \fBMediumSecurity\fR, \fBLowSecurity\fR, \fBAlmostNoSecurity\fR, or \fBNoSecurity\fR\. For more details, please see the Rubygems signing documentation linked below in \fISEE ALSO\fR\. -. .TP \fB\-\-with=<list>\fR A space\-separated list of groups referencing gems to install\. If an optional group is given it is installed\. If a group is given that is in the remembered list of groups given to \-\-without, it is removed from that list\. -. .IP This option is deprecated in favor of the \fBwith\fR setting\. -. .TP \fB\-\-without=<list>\fR A space\-separated list of groups referencing gems to skip during installation\. If a group is given that is in the remembered list of groups given to \-\-with, it is removed from that list\. -. .IP This option is deprecated in favor of the \fBwithout\fR setting\. -. .SH "DEPLOYMENT MODE" -Bundler\'s defaults are optimized for development\. To switch to defaults optimized for deployment and for CI, use the \fB\-\-deployment\fR flag\. Do not activate deployment mode on development machines, as it will cause an error when the Gemfile(5) is modified\. -. +Bundler's defaults are optimized for development\. To switch to defaults optimized for deployment and for CI, use the \fB\-\-deployment\fR flag\. Do not activate deployment mode on development machines, as it will cause an error when the Gemfile(5) is modified\. .IP "1." 4 A \fBGemfile\.lock\fR is required\. -. .IP To ensure that the same versions of the gems you developed with and tested with are also used in deployments, a \fBGemfile\.lock\fR is required\. -. .IP This is mainly to ensure that you remember to check your \fBGemfile\.lock\fR into version control\. -. .IP "2." 4 The \fBGemfile\.lock\fR must be up to date -. .IP In development, you can modify your Gemfile(5) and re\-run \fBbundle install\fR to \fIconservatively update\fR your \fBGemfile\.lock\fR snapshot\. -. .IP In deployment, your \fBGemfile\.lock\fR should be up\-to\-date with changes made in your Gemfile(5)\. -. .IP "3." 4 Gems are installed to \fBvendor/bundle\fR not your default system location -. .IP -In development, it\'s convenient to share the gems used in your application with other applications and other scripts that run on the system\. -. +In development, it's convenient to share the gems used in your application with other applications and other scripts that run on the system\. .IP In deployment, isolation is a more important default\. In addition, the user deploying the application may not have permission to install gems to the system, or the web server may not have permission to read them\. -. .IP As a result, \fBbundle install \-\-deployment\fR installs gems to the \fBvendor/bundle\fR directory in the application\. This may be overridden using the \fB\-\-path\fR option\. -. .IP "" 0 -. .SH "INSTALLING GROUPS" By default, \fBbundle install\fR will install all gems in all groups in your Gemfile(5), except those declared for a different platform\. -. .P However, you can explicitly tell Bundler to skip installing certain groups with the \fB\-\-without\fR option\. This option takes a space\-separated list of groups\. -. .P While the \fB\-\-without\fR option will skip \fIinstalling\fR the gems in the specified groups, it will still \fIdownload\fR those gems and use them to resolve the dependencies of every gem in your Gemfile(5)\. -. .P This is so that installing a different set of groups on another machine (such as a production server) will not change the gems and versions that you have already developed and tested against\. -. .P \fBBundler offers a rock\-solid guarantee that the third\-party code you are running in development and testing is also the third\-party code you are running in production\. You can choose to exclude some of that code in different environments, but you will never be caught flat\-footed by different versions of third\-party code being used in different environments\.\fR -. .P For a simple illustration, consider the following Gemfile(5): -. .IP "" 4 -. .nf +source 'https://rubygems\.org' -source \'https://rubygems\.org\' - -gem \'sinatra\' +gem 'sinatra' group :production do - gem \'rack\-perftools\-profiler\' + gem 'rack\-perftools\-profiler' end -. .fi -. .IP "" 0 -. .P In this case, \fBsinatra\fR depends on any version of Rack (\fB>= 1\.0\fR), while \fBrack\-perftools\-profiler\fR depends on 1\.x (\fB~> 1\.0\fR)\. -. .P When you run \fBbundle install \-\-without production\fR in development, we look at the dependencies of \fBrack\-perftools\-profiler\fR as well\. That way, you do not spend all your time developing against Rack 2\.0, using new APIs unavailable in Rack 1\.x, only to have Bundler switch to Rack 1\.2 when the \fBproduction\fR group \fIis\fR used\. -. .P This should not cause any problems in practice, because we do not attempt to \fBinstall\fR the gems in the excluded groups, and only evaluate as part of the dependency resolution process\. -. .P This also means that you cannot include different versions of the same gem in different groups, because doing so would result in different sets of dependencies used in development and production\. Because of the vagaries of the dependency resolution process, this usually affects more than the gems you list in your Gemfile(5), and can (surprisingly) radically change the gems you are using\. -. .SH "THE GEMFILE\.LOCK" When you run \fBbundle install\fR, Bundler will persist the full names and versions of all gems that you used (including dependencies of the gems specified in the Gemfile(5)) into a file called \fBGemfile\.lock\fR\. -. .P Bundler uses this file in all subsequent calls to \fBbundle install\fR, which guarantees that you always use the same exact code, even as your application moves across machines\. -. .P Because of the way dependency resolution works, even a seemingly small change (for instance, an update to a point\-release of a dependency of a gem in your Gemfile(5)) can result in radically different gems being needed to satisfy all dependencies\. -. .P As a result, you \fBSHOULD\fR check your \fBGemfile\.lock\fR into version control, in both applications and gems\. If you do not, every machine that checks out your repository (including your production server) will resolve all dependencies again, which will result in different versions of third\-party code being used if \fBany\fR of the gems in the Gemfile(5) or any of their dependencies have been updated\. -. .P When Bundler first shipped, the \fBGemfile\.lock\fR was included in the \fB\.gitignore\fR file included with generated gems\. Over time, however, it became clear that this practice forces the pain of broken dependencies onto new contributors, while leaving existing contributors potentially unaware of the problem\. Since \fBbundle install\fR is usually the first step towards a contribution, the pain of broken dependencies would discourage new contributors from contributing\. As a result, we have revised our guidance for gem authors to now recommend checking in the lock for gems\. -. .SH "CONSERVATIVE UPDATING" When you make a change to the Gemfile(5) and then run \fBbundle install\fR, Bundler will update only the gems that you modified\. -. .P In other words, if a gem that you \fBdid not modify\fR worked before you called \fBbundle install\fR, it will continue to use the exact same versions of all dependencies as it used before the update\. -. .P -Let\'s take a look at an example\. Here\'s your original Gemfile(5): -. +Let's take a look at an example\. Here's your original Gemfile(5): .IP "" 4 -. .nf +source 'https://rubygems\.org' -source \'https://rubygems\.org\' - -gem \'actionpack\', \'2\.3\.8\' -gem \'activemerchant\' -. +gem 'actionpack', '2\.3\.8' +gem 'activemerchant' .fi -. .IP "" 0 -. .P In this case, both \fBactionpack\fR and \fBactivemerchant\fR depend on \fBactivesupport\fR\. The \fBactionpack\fR gem depends on \fBactivesupport 2\.3\.8\fR and \fBrack ~> 1\.1\.0\fR, while the \fBactivemerchant\fR gem depends on \fBactivesupport >= 2\.3\.2\fR, \fBbraintree >= 2\.0\.0\fR, and \fBbuilder >= 2\.0\.0\fR\. -. .P When the dependencies are first resolved, Bundler will select \fBactivesupport 2\.3\.8\fR, which satisfies the requirements of both gems in your Gemfile(5)\. -. .P Next, you modify your Gemfile(5) to: -. .IP "" 4 -. .nf +source 'https://rubygems\.org' -source \'https://rubygems\.org\' - -gem \'actionpack\', \'3\.0\.0\.rc\' -gem \'activemerchant\' -. +gem 'actionpack', '3\.0\.0\.rc' +gem 'activemerchant' .fi -. .IP "" 0 -. .P The \fBactionpack 3\.0\.0\.rc\fR gem has a number of new dependencies, and updates the \fBactivesupport\fR dependency to \fB= 3\.0\.0\.rc\fR and the \fBrack\fR dependency to \fB~> 1\.2\.1\fR\. -. .P When you run \fBbundle install\fR, Bundler notices that you changed the \fBactionpack\fR gem, but not the \fBactivemerchant\fR gem\. It evaluates the gems currently being used to satisfy its requirements: -. .TP \fBactivesupport 2\.3\.8\fR also used to satisfy a dependency in \fBactivemerchant\fR, which is not being updated -. .TP \fBrack ~> 1\.1\.0\fR not currently being used to satisfy another dependency -. .P Because you did not explicitly ask to update \fBactivemerchant\fR, you would not expect it to suddenly stop working after updating \fBactionpack\fR\. However, satisfying the new \fBactivesupport 3\.0\.0\.rc\fR dependency of actionpack requires updating one of its dependencies\. -. .P Even though \fBactivemerchant\fR declares a very loose dependency that theoretically matches \fBactivesupport 3\.0\.0\.rc\fR, Bundler treats gems in your Gemfile(5) that have not changed as an atomic unit together with their dependencies\. In this case, the \fBactivemerchant\fR dependency is treated as \fBactivemerchant 1\.7\.1 + activesupport 2\.3\.8\fR, so \fBbundle install\fR will report that it cannot update \fBactionpack\fR\. -. .P To explicitly update \fBactionpack\fR, including its dependencies which other gems in the Gemfile(5) still depend on, run \fBbundle update actionpack\fR (see \fBbundle update(1)\fR)\. -. .P \fBSummary\fR: In general, after making a change to the Gemfile(5) , you should first try to run \fBbundle install\fR, which will guarantee that no other gem in the Gemfile(5) is impacted by the change\. If that does not work, run bundle update(1) \fIbundle\-update\.1\.html\fR\. -. .SH "SEE ALSO" -. .IP "\(bu" 4 -Gem install docs \fIhttp://guides\.rubygems\.org/rubygems\-basics/#installing\-gems\fR -. +Gem install docs \fIhttps://guides\.rubygems\.org/rubygems\-basics/#installing\-gems\fR .IP "\(bu" 4 -Rubygems signing docs \fIhttp://guides\.rubygems\.org/security/\fR -. +Rubygems signing docs \fIhttps://guides\.rubygems\.org/security/\fR .IP "" 0 diff --git a/lib/bundler/man/bundle-install.1.ronn b/lib/bundler/man/bundle-install.1.ronn index be9ed0f974..ed8169de05 100644 --- a/lib/bundler/man/bundle-install.1.ronn +++ b/lib/bundler/man/bundle-install.1.ronn @@ -14,6 +14,7 @@ bundle-install(1) -- Install the dependencies specified in your Gemfile [--no-cache] [--no-prune] [--path PATH] + [--prefer-local] [--quiet] [--redownload] [--retry=NUMBER] @@ -378,5 +379,5 @@ does not work, run [bundle update(1)](bundle-update.1.html). ## SEE ALSO -* [Gem install docs](http://guides.rubygems.org/rubygems-basics/#installing-gems) -* [Rubygems signing docs](http://guides.rubygems.org/security/) +* [Gem install docs](https://guides.rubygems.org/rubygems-basics/#installing-gems) +* [Rubygems signing docs](https://guides.rubygems.org/security/) diff --git a/lib/bundler/man/bundle-list.1 b/lib/bundler/man/bundle-list.1 index f24f62dd38..21723608cc 100644 --- a/lib/bundler/man/bundle-list.1 +++ b/lib/bundler/man/bundle-list.1 @@ -1,49 +1,34 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-LIST" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-LIST" "1" "May 2024" "" .SH "NAME" \fBbundle\-list\fR \- List all the gems in the bundle -. .SH "SYNOPSIS" -\fBbundle list\fR [\-\-name\-only] [\-\-paths] [\-\-without\-group=GROUP[ GROUP\.\.\.]] [\-\-only\-group=GROUP[ GROUP\.\.\.]] -. +\fBbundle list\fR [\-\-name\-only] [\-\-paths] [\-\-without\-group=GROUP[ GROUP\|\.\|\.\|\.]] [\-\-only\-group=GROUP[ GROUP\|\.\|\.\|\.]] .SH "DESCRIPTION" Prints a list of all the gems in the bundle including their version\. -. .P Example: -. .P bundle list \-\-name\-only -. .P bundle list \-\-paths -. .P bundle list \-\-without\-group test -. .P bundle list \-\-only\-group dev -. .P bundle list \-\-only\-group dev test \-\-paths -. .SH "OPTIONS" -. .TP \fB\-\-name\-only\fR Print only the name of each gem\. -. .TP \fB\-\-paths\fR Print the path to each gem in the bundle\. -. .TP \fB\-\-without\-group=<list>\fR A space\-separated list of groups of gems to skip during printing\. -. .TP \fB\-\-only\-group=<list>\fR A space\-separated list of groups of gems to print\. diff --git a/lib/bundler/man/bundle-lock.1 b/lib/bundler/man/bundle-lock.1 index 55d1035d77..8b81b7732f 100644 --- a/lib/bundler/man/bundle-lock.1 +++ b/lib/bundler/man/bundle-lock.1 @@ -1,84 +1,60 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-LOCK" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-LOCK" "1" "May 2024" "" .SH "NAME" \fBbundle\-lock\fR \- Creates / Updates a lockfile without installing -. .SH "SYNOPSIS" \fBbundle lock\fR [\-\-update] [\-\-local] [\-\-print] [\-\-lockfile=PATH] [\-\-full\-index] [\-\-add\-platform] [\-\-remove\-platform] [\-\-patch] [\-\-minor] [\-\-major] [\-\-strict] [\-\-conservative] -. .SH "DESCRIPTION" Lock the gems specified in Gemfile\. -. .SH "OPTIONS" -. .TP \fB\-\-update=<*gems>\fR Ignores the existing lockfile\. Resolve then updates lockfile\. Taking a list of gems or updating all gems if no list is given\. -. .TP \fB\-\-local\fR -Do not attempt to connect to \fBrubygems\.org\fR\. Instead, Bundler will use the gems already present in Rubygems\' cache or in \fBvendor/cache\fR\. Note that if a appropriate platform\-specific gem exists on \fBrubygems\.org\fR it will not be found\. -. +Do not attempt to connect to \fBrubygems\.org\fR\. Instead, Bundler will use the gems already present in Rubygems' cache or in \fBvendor/cache\fR\. Note that if a appropriate platform\-specific gem exists on \fBrubygems\.org\fR it will not be found\. .TP \fB\-\-print\fR Prints the lockfile to STDOUT instead of writing to the file system\. -. .TP \fB\-\-lockfile=<path>\fR The path where the lockfile should be written to\. -. .TP \fB\-\-full\-index\fR Fall back to using the single\-file index of all gems\. -. .TP \fB\-\-add\-platform\fR Add a new platform to the lockfile, re\-resolving for the addition of that platform\. -. .TP \fB\-\-remove\-platform\fR Remove a platform from the lockfile\. -. .TP \fB\-\-patch\fR If updating, prefer updating only to next patch version\. -. .TP \fB\-\-minor\fR If updating, prefer updating only to next minor version\. -. .TP \fB\-\-major\fR If updating, prefer updating to next major version (default)\. -. .TP \fB\-\-strict\fR If updating, do not allow any gem to be updated past latest \-\-patch | \-\-minor | \-\-major\. -. .TP \fB\-\-conservative\fR If updating, use bundle install conservative update behavior and do not allow shared dependencies to be updated\. -. .SH "UPDATING ALL GEMS" If you run \fBbundle lock\fR with \fB\-\-update\fR option without list of gems, bundler will ignore any previously installed gems and resolve all dependencies again based on the latest versions of all gems available in the sources\. -. .SH "UPDATING A LIST OF GEMS" Sometimes, you want to update a single gem in the Gemfile(5), and leave the rest of the gems that you specified locked to the versions in the \fBGemfile\.lock\fR\. -. .P For instance, you only want to update \fBnokogiri\fR, run \fBbundle lock \-\-update nokogiri\fR\. -. .P Bundler will update \fBnokogiri\fR and any of its dependencies, but leave the rest of the gems that you specified locked to the versions in the \fBGemfile\.lock\fR\. -. .SH "SUPPORTING OTHER PLATFORMS" -If you want your bundle to support platforms other than the one you\'re running locally, you can run \fBbundle lock \-\-add\-platform PLATFORM\fR to add PLATFORM to the lockfile, force bundler to re\-resolve and consider the new platform when picking gems, all without needing to have a machine that matches PLATFORM handy to install those platform\-specific gems on\. -. +If you want your bundle to support platforms other than the one you're running locally, you can run \fBbundle lock \-\-add\-platform PLATFORM\fR to add PLATFORM to the lockfile, force bundler to re\-resolve and consider the new platform when picking gems, all without needing to have a machine that matches PLATFORM handy to install those platform\-specific gems on\. .P For a full explanation of gem platforms, see \fBgem help platform\fR\. -. .SH "PATCH LEVEL OPTIONS" See bundle update(1) \fIbundle\-update\.1\.html\fR for details\. diff --git a/lib/bundler/man/bundle-open.1 b/lib/bundler/man/bundle-open.1 index ff44d1224f..41a8804a09 100644 --- a/lib/bundler/man/bundle-open.1 +++ b/lib/bundler/man/bundle-open.1 @@ -1,51 +1,31 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-OPEN" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-OPEN" "1" "May 2024" "" .SH "NAME" \fBbundle\-open\fR \- Opens the source directory for a gem in your bundle -. .SH "SYNOPSIS" \fBbundle open\fR [GEM] [\-\-path=PATH] -. .SH "DESCRIPTION" Opens the source directory of the provided GEM in your editor\. -. .P For this to work the \fBEDITOR\fR or \fBBUNDLER_EDITOR\fR environment variable has to be set\. -. .P Example: -. .IP "" 4 -. .nf - -bundle open \'rack\' -. +bundle open 'rack' .fi -. .IP "" 0 -. .P -Will open the source directory for the \'rack\' gem in your bundle\. -. +Will open the source directory for the 'rack' gem in your bundle\. .IP "" 4 -. .nf - -bundle open \'rack\' \-\-path \'README\.md\' -. +bundle open 'rack' \-\-path 'README\.md' .fi -. .IP "" 0 -. .P -Will open the README\.md file of the \'rack\' gem source in your bundle\. -. +Will open the README\.md file of the 'rack' gem source in your bundle\. .SH "OPTIONS" -. .TP \fB\-\-path\fR Specify GEM source relative path to open\. diff --git a/lib/bundler/man/bundle-outdated.1 b/lib/bundler/man/bundle-outdated.1 index 8455b71b45..838fce45cd 100644 --- a/lib/bundler/man/bundle-outdated.1 +++ b/lib/bundler/man/bundle-outdated.1 @@ -1,148 +1,100 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-OUTDATED" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-OUTDATED" "1" "May 2024" "" .SH "NAME" \fBbundle\-outdated\fR \- List installed gems with newer versions available -. .SH "SYNOPSIS" \fBbundle outdated\fR [GEM] [\-\-local] [\-\-pre] [\-\-source] [\-\-strict] [\-\-parseable | \-\-porcelain] [\-\-group=GROUP] [\-\-groups] [\-\-patch|\-\-minor|\-\-major] [\-\-filter\-major] [\-\-filter\-minor] [\-\-filter\-patch] [\-\-only\-explicit] -. .SH "DESCRIPTION" Outdated lists the names and versions of gems that have a newer version available in the given source\. Calling outdated with [GEM [GEM]] will only check for newer versions of the given gems\. Prerelease gems are ignored by default\. If your gems are up to date, Bundler will exit with a status of 0\. Otherwise, it will exit 1\. -. .SH "OPTIONS" -. .TP \fB\-\-local\fR Do not attempt to fetch gems remotely and use the gem cache instead\. -. .TP \fB\-\-pre\fR Check for newer pre\-release gems\. -. .TP \fB\-\-source\fR Check against a specific source\. -. .TP \fB\-\-strict\fR Only list newer versions allowed by your Gemfile requirements, also respecting conservative update flags (\-\-patch, \-\-minor, \-\-major)\. -. .TP \fB\-\-parseable\fR, \fB\-\-porcelain\fR Use minimal formatting for more parseable output\. -. .TP \fB\-\-group\fR List gems from a specific group\. -. .TP \fB\-\-groups\fR List gems organized by groups\. -. .TP \fB\-\-minor\fR Prefer updating only to next minor version\. -. .TP \fB\-\-major\fR Prefer updating to next major version (default)\. -. .TP \fB\-\-patch\fR Prefer updating only to next patch version\. -. .TP \fB\-\-filter\-major\fR Only list major newer versions\. -. .TP \fB\-\-filter\-minor\fR Only list minor newer versions\. -. .TP \fB\-\-filter\-patch\fR Only list patch newer versions\. -. .TP \fB\-\-only\-explicit\fR Only list gems specified in your Gemfile, not their dependencies\. -. .SH "PATCH LEVEL OPTIONS" See bundle update(1) \fIbundle\-update\.1\.html\fR for details\. -. .SH "FILTERING OUTPUT" The 3 filtering options do not affect the resolution of versions, merely what versions are shown in the output\. -. .P If the regular output shows the following: -. .IP "" 4 -. .nf - -* faker (newest 1\.6\.6, installed 1\.6\.5, requested ~> 1\.4) in groups "development, test" -* hashie (newest 3\.4\.6, installed 1\.2\.0, requested = 1\.2\.0) in groups "default" -* headless (newest 2\.3\.1, installed 2\.2\.3) in groups "test" -. +* Gem Current Latest Requested Groups +* faker 1\.6\.5 1\.6\.6 ~> 1\.4 development, test +* hashie 1\.2\.0 3\.4\.6 = 1\.2\.0 default +* headless 2\.2\.3 2\.3\.1 = 2\.2\.3 test .fi -. .IP "" 0 -. .P \fB\-\-filter\-major\fR would only show: -. .IP "" 4 -. .nf - -* hashie (newest 3\.4\.6, installed 1\.2\.0, requested = 1\.2\.0) in groups "default" -. +* Gem Current Latest Requested Groups +* hashie 1\.2\.0 3\.4\.6 = 1\.2\.0 default .fi -. .IP "" 0 -. .P \fB\-\-filter\-minor\fR would only show: -. .IP "" 4 -. .nf - -* headless (newest 2\.3\.1, installed 2\.2\.3) in groups "test" -. +* Gem Current Latest Requested Groups +* headless 2\.2\.3 2\.3\.1 = 2\.2\.3 test .fi -. .IP "" 0 -. .P \fB\-\-filter\-patch\fR would only show: -. .IP "" 4 -. .nf - -* faker (newest 1\.6\.6, installed 1\.6\.5, requested ~> 1\.4) in groups "development, test" -. +* Gem Current Latest Requested Groups +* faker 1\.6\.5 1\.6\.6 ~> 1\.4 development, test .fi -. .IP "" 0 -. .P Filter options can be combined\. \fB\-\-filter\-minor\fR and \fB\-\-filter\-patch\fR would show: -. .IP "" 4 -. .nf - -* faker (newest 1\.6\.6, installed 1\.6\.5, requested ~> 1\.4) in groups "development, test" -* headless (newest 2\.3\.1, installed 2\.2\.3) in groups "test" -. +* Gem Current Latest Requested Groups +* faker 1\.6\.5 1\.6\.6 ~> 1\.4 development, test .fi -. .IP "" 0 -. .P Combining all three \fBfilter\fR options would be the same result as providing none of them\. diff --git a/lib/bundler/man/bundle-outdated.1.ronn b/lib/bundler/man/bundle-outdated.1.ronn index 04096ffbb6..4ac65d0532 100644 --- a/lib/bundler/man/bundle-outdated.1.ronn +++ b/lib/bundler/man/bundle-outdated.1.ronn @@ -78,25 +78,29 @@ in the output. If the regular output shows the following: - * faker (newest 1.6.6, installed 1.6.5, requested ~> 1.4) in groups "development, test" - * hashie (newest 3.4.6, installed 1.2.0, requested = 1.2.0) in groups "default" - * headless (newest 2.3.1, installed 2.2.3) in groups "test" + * Gem Current Latest Requested Groups + * faker 1.6.5 1.6.6 ~> 1.4 development, test + * hashie 1.2.0 3.4.6 = 1.2.0 default + * headless 2.2.3 2.3.1 = 2.2.3 test `--filter-major` would only show: - * hashie (newest 3.4.6, installed 1.2.0, requested = 1.2.0) in groups "default" + * Gem Current Latest Requested Groups + * hashie 1.2.0 3.4.6 = 1.2.0 default `--filter-minor` would only show: - * headless (newest 2.3.1, installed 2.2.3) in groups "test" + * Gem Current Latest Requested Groups + * headless 2.2.3 2.3.1 = 2.2.3 test `--filter-patch` would only show: - * faker (newest 1.6.6, installed 1.6.5, requested ~> 1.4) in groups "development, test" + * Gem Current Latest Requested Groups + * faker 1.6.5 1.6.6 ~> 1.4 development, test Filter options can be combined. `--filter-minor` and `--filter-patch` would show: - * faker (newest 1.6.6, installed 1.6.5, requested ~> 1.4) in groups "development, test" - * headless (newest 2.3.1, installed 2.2.3) in groups "test" + * Gem Current Latest Requested Groups + * faker 1.6.5 1.6.6 ~> 1.4 development, test Combining all three `filter` options would be the same result as providing none of them. diff --git a/lib/bundler/man/bundle-platform.1 b/lib/bundler/man/bundle-platform.1 index 2794878719..0dbce7a7a4 100644 --- a/lib/bundler/man/bundle-platform.1 +++ b/lib/bundler/man/bundle-platform.1 @@ -1,41 +1,27 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-PLATFORM" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-PLATFORM" "1" "May 2024" "" .SH "NAME" \fBbundle\-platform\fR \- Displays platform compatibility information -. .SH "SYNOPSIS" \fBbundle platform\fR [\-\-ruby] -. .SH "DESCRIPTION" \fBplatform\fR displays information from your Gemfile, Gemfile\.lock, and Ruby VM about your platform\. -. .P For instance, using this Gemfile(5): -. .IP "" 4 -. .nf - source "https://rubygems\.org" ruby "3\.1\.2" gem "rack" -. .fi -. .IP "" 0 -. .P If you run \fBbundle platform\fR on Ruby 3\.1\.2, it displays the following output: -. .IP "" 4 -. .nf - Your platform is: x86_64\-linux Your app has gems that work on these platforms: @@ -48,24 +34,16 @@ Your Gemfile specifies a Ruby version requirement: * ruby 3\.1\.2 Your current platform satisfies the Ruby version requirement\. -. .fi -. .IP "" 0 -. .P -\fBplatform\fR lists all the platforms in your \fBGemfile\.lock\fR as well as the \fBruby\fR directive if applicable from your Gemfile(5)\. It also lets you know if the \fBruby\fR directive requirement has been met\. If \fBruby\fR directive doesn\'t match the running Ruby VM, it tells you what part does not\. -. +\fBplatform\fR lists all the platforms in your \fBGemfile\.lock\fR as well as the \fBruby\fR directive if applicable from your Gemfile(5)\. It also lets you know if the \fBruby\fR directive requirement has been met\. If \fBruby\fR directive doesn't match the running Ruby VM, it tells you what part does not\. .SH "OPTIONS" -. .TP \fB\-\-ruby\fR -It will display the ruby directive information, so you don\'t have to parse it from the Gemfile(5)\. -. +It will display the ruby directive information, so you don't have to parse it from the Gemfile(5)\. .SH "SEE ALSO" -. .IP "\(bu" 4 bundle\-lock(1) \fIbundle\-lock\.1\.html\fR -. .IP "" 0 diff --git a/lib/bundler/man/bundle-plugin.1 b/lib/bundler/man/bundle-plugin.1 index 39d3dfa04e..1290abb3ba 100644 --- a/lib/bundler/man/bundle-plugin.1 +++ b/lib/bundler/man/bundle-plugin.1 @@ -1,81 +1,58 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-PLUGIN" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-PLUGIN" "1" "May 2024" "" .SH "NAME" \fBbundle\-plugin\fR \- Manage Bundler plugins -. .SH "SYNOPSIS" -\fBbundle plugin\fR install PLUGINS [\-\-source=\fISOURCE\fR] [\-\-version=\fIversion\fR] [\-\-git|\-\-local_git=\fIgit\-url\fR] [\-\-branch=\fIbranch\fR|\-\-ref=\fIrev\fR] -. +\fBbundle plugin\fR install PLUGINS [\-\-source=\fISOURCE\fR] [\-\-version=\fIversion\fR] [\-\-git=\fIgit\-url\fR] [\-\-branch=\fIbranch\fR|\-\-ref=\fIrev\fR] [\-\-path=\fIpath\fR] .br \fBbundle plugin\fR uninstall PLUGINS -. .br \fBbundle plugin\fR list -. .br \fBbundle plugin\fR help [COMMAND] -. .SH "DESCRIPTION" You can install, uninstall, and list plugin(s) with this command to extend functionalities of Bundler\. -. .SH "SUB\-COMMANDS" -. .SS "install" Install the given plugin(s)\. -. -.IP "\(bu" 4 -\fBbundle plugin install bundler\-graph\fR: Install bundler\-graph gem from RubyGems\.org\. The global source, specified in source in Gemfile is ignored\. -. -.IP "\(bu" 4 -\fBbundle plugin install bundler\-graph \-\-source https://example\.com\fR: Install bundler\-graph gem from example\.com\. The global source, specified in source in Gemfile is not considered\. -. -.IP "\(bu" 4 -\fBbundle plugin install bundler\-graph \-\-version 0\.2\.1\fR: You can specify the version of the gem via \fB\-\-version\fR\. -. -.IP "\(bu" 4 -\fBbundle plugin install bundler\-graph \-\-git https://github\.com/rubygems/bundler\-graph\fR: Install bundler\-graph gem from Git repository\. \fB\-\-git\fR can be replaced with \fB\-\-local\-git\fR\. You cannot use both \fB\-\-git\fR and \fB\-\-local\-git\fR\. You can use standard Git URLs like: -. -.IP "\(bu" 4 +.TP +\fBbundle plugin install bundler\-graph\fR +Install bundler\-graph gem from globally configured sources (defaults to RubyGems\.org)\. The global source, specified in source in Gemfile is ignored\. +.TP +\fBbundle plugin install bundler\-graph \-\-source https://example\.com\fR +Install bundler\-graph gem from example\.com\. The global source, specified in source in Gemfile is not considered\. +.TP +\fBbundle plugin install bundler\-graph \-\-version 0\.2\.1\fR +You can specify the version of the gem via \fB\-\-version\fR\. +.TP +\fBbundle plugin install bundler\-graph \-\-git https://github\.com/rubygems/bundler\-graph\fR +Install bundler\-graph gem from Git repository\. You can use standard Git URLs like: +.IP \fBssh://[user@]host\.xz[:port]/path/to/repo\.git\fR -. -.IP "\(bu" 4 +.br \fBhttp[s]://host\.xz[:port]/path/to/repo\.git\fR -. -.IP "\(bu" 4 +.br \fB/path/to/repo\fR -. -.IP "\(bu" 4 +.br \fBfile:///path/to/repo\fR -. -.IP "" 0 -. .IP -When you specify \fB\-\-git\fR/\fB\-\-local\-git\fR, you can use \fB\-\-branch\fR or \fB\-\-ref\fR to specify any branch, tag, or commit hash (revision) to use\. When you specify both, only the latter is used\. -. -.IP "" 0 -. +When you specify \fB\-\-git\fR, you can use \fB\-\-branch\fR or \fB\-\-ref\fR to specify any branch, tag, or commit hash (revision) to use\. +.TP +\fBbundle plugin install bundler\-graph \-\-path \.\./bundler\-graph\fR +Install bundler\-graph gem from a local path\. .SS "uninstall" Uninstall the plugin(s) specified in PLUGINS\. -. .SS "list" List the installed plugins and available commands\. -. .P No options\. -. .SS "help" Describe subcommands or one specific subcommand\. -. .P No options\. -. .SH "SEE ALSO" -. .IP "\(bu" 4 How to write a Bundler plugin \fIhttps://bundler\.io/guides/bundler_plugins\.html\fR -. .IP "" 0 diff --git a/lib/bundler/man/bundle-plugin.1.ronn b/lib/bundler/man/bundle-plugin.1.ronn index 4f234eeba7..b0a34660ea 100644 --- a/lib/bundler/man/bundle-plugin.1.ronn +++ b/lib/bundler/man/bundle-plugin.1.ronn @@ -4,7 +4,8 @@ bundle-plugin(1) -- Manage Bundler plugins ## SYNOPSIS `bundle plugin` install PLUGINS [--source=<SOURCE>] [--version=<version>] - [--git|--local_git=<git-url>] [--branch=<branch>|--ref=<rev>]<br> + [--git=<git-url>] [--branch=<branch>|--ref=<rev>] + [--path=<path>]<br> `bundle plugin` uninstall PLUGINS<br> `bundle plugin` list<br> `bundle plugin` help [COMMAND] @@ -20,7 +21,7 @@ You can install, uninstall, and list plugin(s) with this command to extend funct Install the given plugin(s). * `bundle plugin install bundler-graph`: - Install bundler-graph gem from RubyGems.org. The global source, specified in source in Gemfile is ignored. + Install bundler-graph gem from globally configured sources (defaults to RubyGems.org). The global source, specified in source in Gemfile is ignored. * `bundle plugin install bundler-graph --source https://example.com`: Install bundler-graph gem from example.com. The global source, specified in source in Gemfile is not considered. @@ -29,14 +30,17 @@ Install the given plugin(s). You can specify the version of the gem via `--version`. * `bundle plugin install bundler-graph --git https://github.com/rubygems/bundler-graph`: - Install bundler-graph gem from Git repository. `--git` can be replaced with `--local-git`. You cannot use both `--git` and `--local-git`. You can use standard Git URLs like: + Install bundler-graph gem from Git repository. You can use standard Git URLs like: - * `ssh://[user@]host.xz[:port]/path/to/repo.git` - * `http[s]://host.xz[:port]/path/to/repo.git` - * `/path/to/repo` - * `file:///path/to/repo` + `ssh://[user@]host.xz[:port]/path/to/repo.git`<br> + `http[s]://host.xz[:port]/path/to/repo.git`<br> + `/path/to/repo`<br> + `file:///path/to/repo` - When you specify `--git`/`--local-git`, you can use `--branch` or `--ref` to specify any branch, tag, or commit hash (revision) to use. When you specify both, only the latter is used. + When you specify `--git`, you can use `--branch` or `--ref` to specify any branch, tag, or commit hash (revision) to use. + +* `bundle plugin install bundler-graph --path ../bundler-graph`: + Install bundler-graph gem from a local path. ### uninstall diff --git a/lib/bundler/man/bundle-pristine.1 b/lib/bundler/man/bundle-pristine.1 index f42c7ce156..bf4a7cd323 100644 --- a/lib/bundler/man/bundle-pristine.1 +++ b/lib/bundler/man/bundle-pristine.1 @@ -1,34 +1,23 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-PRISTINE" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-PRISTINE" "1" "May 2024" "" .SH "NAME" \fBbundle\-pristine\fR \- Restores installed gems to their pristine condition -. .SH "SYNOPSIS" \fBbundle pristine\fR -. .SH "DESCRIPTION" \fBpristine\fR restores the installed gems in the bundle to their pristine condition using the local gem cache from RubyGems\. For git gems, a forced checkout will be performed\. -. .P -For further explanation, \fBbundle pristine\fR ignores unpacked files on disk\. In other words, this command utilizes the local \fB\.gem\fR cache or the gem\'s git repository as if one were installing from scratch\. -. +For further explanation, \fBbundle pristine\fR ignores unpacked files on disk\. In other words, this command utilizes the local \fB\.gem\fR cache or the gem's git repository as if one were installing from scratch\. .P -Note: the Bundler gem cannot be restored to its original state with \fBpristine\fR\. One also cannot use \fBbundle pristine\fR on gems with a \'path\' option in the Gemfile, because bundler has no original copy it can restore from\. -. +Note: the Bundler gem cannot be restored to its original state with \fBpristine\fR\. One also cannot use \fBbundle pristine\fR on gems with a 'path' option in the Gemfile, because bundler has no original copy it can restore from\. .P When is it practical to use \fBbundle pristine\fR? -. .P It comes in handy when a developer is debugging a gem\. \fBbundle pristine\fR is a great way to get rid of experimental changes to a gem that one may not want\. -. .P Why use \fBbundle pristine\fR over \fBgem pristine \-\-all\fR? -. .P Both commands are very similar\. For context: \fBbundle pristine\fR, without arguments, cleans all gems from the lockfile\. Meanwhile, \fBgem pristine \-\-all\fR cleans all installed gems for that Ruby version\. -. .P If a developer forgets which gems in their project they might have been debugging, the Rubygems \fBgem pristine [GEMNAME]\fR command may be inconvenient\. One can avoid waiting for \fBgem pristine \-\-all\fR, and instead run \fBbundle pristine\fR\. diff --git a/lib/bundler/man/bundle-remove.1 b/lib/bundler/man/bundle-remove.1 index b18d80554d..c3c96b416d 100644 --- a/lib/bundler/man/bundle-remove.1 +++ b/lib/bundler/man/bundle-remove.1 @@ -1,31 +1,21 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-REMOVE" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-REMOVE" "1" "May 2024" "" .SH "NAME" \fBbundle\-remove\fR \- Removes gems from the Gemfile -. .SH "SYNOPSIS" -\fBbundle remove [GEM [GEM \.\.\.]] [\-\-install]\fR -. +\fBbundle remove [GEM [GEM \|\.\|\.\|\.]] [\-\-install]\fR .SH "DESCRIPTION" Removes the given gems from the Gemfile while ensuring that the resulting Gemfile is still valid\. If a gem cannot be removed, a warning is printed\. If a gem is already absent from the Gemfile, and error is raised\. -. .SH "OPTIONS" -. .TP \fB\-\-install\fR Runs \fBbundle install\fR after the given gems have been removed from the Gemfile, which ensures that both the lockfile and the installed gems on disk are also updated to remove the given gem(s)\. -. .P Example: -. .P bundle remove rails -. .P bundle remove rails rack -. .P bundle remove rails rack \-\-install diff --git a/lib/bundler/man/bundle-show.1 b/lib/bundler/man/bundle-show.1 index efd9ccb0e0..c054875efd 100644 --- a/lib/bundler/man/bundle-show.1 +++ b/lib/bundler/man/bundle-show.1 @@ -1,22 +1,15 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-SHOW" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-SHOW" "1" "May 2024" "" .SH "NAME" \fBbundle\-show\fR \- Shows all the gems in your bundle, or the path to a gem -. .SH "SYNOPSIS" \fBbundle show\fR [GEM] [\-\-paths] -. .SH "DESCRIPTION" Without the [GEM] option, \fBshow\fR will print a list of the names and versions of all gems that are required by your [\fBGemfile(5)\fR][Gemfile(5)], sorted by name\. -. .P Calling show with [GEM] will list the exact location of that gem on your machine\. -. .SH "OPTIONS" -. .TP \fB\-\-paths\fR List the paths of all gems that are required by your [\fBGemfile(5)\fR][Gemfile(5)], sorted by gem name\. diff --git a/lib/bundler/man/bundle-update.1 b/lib/bundler/man/bundle-update.1 index c67c44ff86..acd80ec75c 100644 --- a/lib/bundler/man/bundle-update.1 +++ b/lib/bundler/man/bundle-update.1 @@ -1,114 +1,81 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-UPDATE" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-UPDATE" "1" "May 2024" "" .SH "NAME" \fBbundle\-update\fR \- Update your gems to the latest available versions -. .SH "SYNOPSIS" \fBbundle update\fR \fI*gems\fR [\-\-all] [\-\-group=NAME] [\-\-source=NAME] [\-\-local] [\-\-ruby] [\-\-bundler[=VERSION]] [\-\-full\-index] [\-\-jobs=JOBS] [\-\-quiet] [\-\-patch|\-\-minor|\-\-major] [\-\-redownload] [\-\-strict] [\-\-conservative] -. .SH "DESCRIPTION" Update the gems specified (all gems, if \fB\-\-all\fR flag is used), ignoring the previously installed gems specified in the \fBGemfile\.lock\fR\. In general, you should use bundle install(1) \fIbundle\-install\.1\.html\fR to install the same exact gems and versions across machines\. -. .P You would use \fBbundle update\fR to explicitly update the version of a gem\. -. .SH "OPTIONS" -. .TP \fB\-\-all\fR Update all gems specified in Gemfile\. -. .TP \fB\-\-group=<name>\fR, \fB\-g=[<name>]\fR Only update the gems in the specified group\. For instance, you can update all gems in the development group with \fBbundle update \-\-group development\fR\. You can also call \fBbundle update rails \-\-group test\fR to update the rails gem and all gems in the test group, for example\. -. .TP \fB\-\-source=<name>\fR The name of a \fB:git\fR or \fB:path\fR source used in the Gemfile(5)\. For instance, with a \fB:git\fR source of \fBhttp://github\.com/rails/rails\.git\fR, you would call \fBbundle update \-\-source rails\fR -. .TP \fB\-\-local\fR Do not attempt to fetch gems remotely and use the gem cache instead\. -. .TP \fB\-\-ruby\fR Update the locked version of Ruby to the current version of Ruby\. -. .TP \fB\-\-bundler\fR Update the locked version of bundler to the invoked bundler version\. -. .TP \fB\-\-full\-index\fR Fall back to using the single\-file index of all gems\. -. .TP \fB\-\-jobs=[<number>]\fR, \fB\-j[<number>]\fR Specify the number of jobs to run in parallel\. The default is the number of available processors\. -. .TP \fB\-\-retry=[<number>]\fR Retry failed network or git requests for \fInumber\fR times\. -. .TP \fB\-\-quiet\fR Only output warnings and errors\. -. .TP \fB\-\-redownload\fR Force downloading every gem\. -. .TP \fB\-\-patch\fR Prefer updating only to next patch version\. -. .TP \fB\-\-minor\fR Prefer updating only to next minor version\. -. .TP \fB\-\-major\fR Prefer updating to next major version (default)\. -. .TP \fB\-\-strict\fR Do not allow any gem to be updated past latest \fB\-\-patch\fR | \fB\-\-minor\fR | \fB\-\-major\fR\. -. .TP \fB\-\-conservative\fR Use bundle install conservative update behavior and do not allow indirect dependencies to be updated\. -. .SH "UPDATING ALL GEMS" If you run \fBbundle update \-\-all\fR, bundler will ignore any previously installed gems and resolve all dependencies again based on the latest versions of all gems available in the sources\. -. .P Consider the following Gemfile(5): -. .IP "" 4 -. .nf - source "https://rubygems\.org" gem "rails", "3\.0\.0\.rc" gem "nokogiri" -. .fi -. .IP "" 0 -. .P When you run bundle install(1) \fIbundle\-install\.1\.html\fR the first time, bundler will resolve all of the dependencies, all the way down, and install what you need: -. .IP "" 4 -. .nf - -Fetching gem metadata from https://rubygems\.org/\.\.\.\.\.\.\.\.\. -Resolving dependencies\.\.\. +Fetching gem metadata from https://rubygems\.org/\|\.\|\.\|\.\|\.\|\.\|\.\|\.\|\.\|\. +Resolving dependencies\|\.\|\.\|\. Installing builder 2\.1\.2 Installing abstract 1\.0\.0 Installing rack 1\.2\.8 @@ -138,55 +105,36 @@ Installing nokogiri 1\.6\.5 Bundle complete! 2 Gemfile dependencies, 26 gems total\. Use `bundle show [gemname]` to see where a bundled gem is installed\. -. .fi -. .IP "" 0 -. .P As you can see, even though you have two gems in the Gemfile(5), your application needs 26 different gems in order to run\. Bundler remembers the exact versions it installed in \fBGemfile\.lock\fR\. The next time you run bundle install(1) \fIbundle\-install\.1\.html\fR, bundler skips the dependency resolution and installs the same gems as it installed last time\. -. .P -After checking in the \fBGemfile\.lock\fR into version control and cloning it on another machine, running bundle install(1) \fIbundle\-install\.1\.html\fR will \fIstill\fR install the gems that you installed last time\. You don\'t need to worry that a new release of \fBerubis\fR or \fBmail\fR changes the gems you use\. -. +After checking in the \fBGemfile\.lock\fR into version control and cloning it on another machine, running bundle install(1) \fIbundle\-install\.1\.html\fR will \fIstill\fR install the gems that you installed last time\. You don't need to worry that a new release of \fBerubis\fR or \fBmail\fR changes the gems you use\. .P However, from time to time, you might want to update the gems you are using to the newest versions that still match the gems in your Gemfile(5)\. -. .P To do this, run \fBbundle update \-\-all\fR, which will ignore the \fBGemfile\.lock\fR, and resolve all the dependencies again\. Keep in mind that this process can result in a significantly different set of the 25 gems, based on the requirements of new gems that the gem authors released since the last time you ran \fBbundle update \-\-all\fR\. -. .SH "UPDATING A LIST OF GEMS" Sometimes, you want to update a single gem in the Gemfile(5), and leave the rest of the gems that you specified locked to the versions in the \fBGemfile\.lock\fR\. -. .P For instance, in the scenario above, imagine that \fBnokogiri\fR releases version \fB1\.4\.4\fR, and you want to update it \fIwithout\fR updating Rails and all of its dependencies\. To do this, run \fBbundle update nokogiri\fR\. -. .P Bundler will update \fBnokogiri\fR and any of its dependencies, but leave alone Rails and its dependencies\. -. .SH "OVERLAPPING DEPENDENCIES" Sometimes, multiple gems declared in your Gemfile(5) are satisfied by the same second\-level dependency\. For instance, consider the case of \fBthin\fR and \fBrack\-perftools\-profiler\fR\. -. .IP "" 4 -. .nf - source "https://rubygems\.org" gem "thin" gem "rack\-perftools\-profiler" -. .fi -. .IP "" 0 -. .P The \fBthin\fR gem depends on \fBrack >= 1\.0\fR, while \fBrack\-perftools\-profiler\fR depends on \fBrack ~> 1\.0\fR\. If you run bundle install, you get: -. .IP "" 4 -. .nf - Fetching source index for https://rubygems\.org/ Installing daemons (1\.1\.0) Installing eventmachine (0\.12\.10) with native extensions @@ -196,199 +144,132 @@ Installing rack (1\.2\.1) Installing rack\-perftools_profiler (0\.0\.2) Installing thin (1\.2\.7) with native extensions Using bundler (1\.0\.0\.rc\.3) -. .fi -. .IP "" 0 -. .P -In this case, the two gems have their own set of dependencies, but they share \fBrack\fR in common\. If you run \fBbundle update thin\fR, bundler will update \fBdaemons\fR, \fBeventmachine\fR and \fBrack\fR, which are dependencies of \fBthin\fR, but not \fBopen4\fR or \fBperftools\.rb\fR, which are dependencies of \fBrack\-perftools_profiler\fR\. Note that \fBbundle update thin\fR will update \fBrack\fR even though it\'s \fIalso\fR a dependency of \fBrack\-perftools_profiler\fR\. -. +In this case, the two gems have their own set of dependencies, but they share \fBrack\fR in common\. If you run \fBbundle update thin\fR, bundler will update \fBdaemons\fR, \fBeventmachine\fR and \fBrack\fR, which are dependencies of \fBthin\fR, but not \fBopen4\fR or \fBperftools\.rb\fR, which are dependencies of \fBrack\-perftools_profiler\fR\. Note that \fBbundle update thin\fR will update \fBrack\fR even though it's \fIalso\fR a dependency of \fBrack\-perftools_profiler\fR\. .P In short, by default, when you update a gem using \fBbundle update\fR, bundler will update all dependencies of that gem, including those that are also dependencies of another gem\. -. .P To prevent updating indirect dependencies, prior to version 1\.14 the only option was the \fBCONSERVATIVE UPDATING\fR behavior in bundle install(1) \fIbundle\-install\.1\.html\fR: -. .P In this scenario, updating the \fBthin\fR version manually in the Gemfile(5), and then running bundle install(1) \fIbundle\-install\.1\.html\fR will only update \fBdaemons\fR and \fBeventmachine\fR, but not \fBrack\fR\. For more information, see the \fBCONSERVATIVE UPDATING\fR section of bundle install(1) \fIbundle\-install\.1\.html\fR\. -. .P Starting with 1\.14, specifying the \fB\-\-conservative\fR option will also prevent indirect dependencies from being updated\. -. .SH "PATCH LEVEL OPTIONS" Version 1\.14 introduced 4 patch\-level options that will influence how gem versions are resolved\. One of the following options can be used: \fB\-\-patch\fR, \fB\-\-minor\fR or \fB\-\-major\fR\. \fB\-\-strict\fR can be added to further influence resolution\. -. .TP \fB\-\-patch\fR Prefer updating only to next patch version\. -. .TP \fB\-\-minor\fR Prefer updating only to next minor version\. -. .TP \fB\-\-major\fR Prefer updating to next major version (default)\. -. .TP \fB\-\-strict\fR Do not allow any gem to be updated past latest \fB\-\-patch\fR | \fB\-\-minor\fR | \fB\-\-major\fR\. -. .P -When Bundler is resolving what versions to use to satisfy declared requirements in the Gemfile or in parent gems, it looks up all available versions, filters out any versions that don\'t satisfy the requirement, and then, by default, sorts them from newest to oldest, considering them in that order\. -. +When Bundler is resolving what versions to use to satisfy declared requirements in the Gemfile or in parent gems, it looks up all available versions, filters out any versions that don't satisfy the requirement, and then, by default, sorts them from newest to oldest, considering them in that order\. .P Providing one of the patch level options (e\.g\. \fB\-\-patch\fR) changes the sort order of the satisfying versions, causing Bundler to consider the latest \fB\-\-patch\fR or \fB\-\-minor\fR version available before other versions\. Note that versions outside the stated patch level could still be resolved to if necessary to find a suitable dependency graph\. -. .P -For example, if gem \'foo\' is locked at 1\.0\.2, with no gem requirement defined in the Gemfile, and versions 1\.0\.3, 1\.0\.4, 1\.1\.0, 1\.1\.1, 2\.0\.0 all exist, the default order of preference by default (\fB\-\-major\fR) will be "2\.0\.0, 1\.1\.1, 1\.1\.0, 1\.0\.4, 1\.0\.3, 1\.0\.2"\. -. +For example, if gem 'foo' is locked at 1\.0\.2, with no gem requirement defined in the Gemfile, and versions 1\.0\.3, 1\.0\.4, 1\.1\.0, 1\.1\.1, 2\.0\.0 all exist, the default order of preference by default (\fB\-\-major\fR) will be "2\.0\.0, 1\.1\.1, 1\.1\.0, 1\.0\.4, 1\.0\.3, 1\.0\.2"\. .P If the \fB\-\-patch\fR option is used, the order of preference will change to "1\.0\.4, 1\.0\.3, 1\.0\.2, 1\.1\.1, 1\.1\.0, 2\.0\.0"\. -. .P If the \fB\-\-minor\fR option is used, the order of preference will change to "1\.1\.1, 1\.1\.0, 1\.0\.4, 1\.0\.3, 1\.0\.2, 2\.0\.0"\. -. .P Combining the \fB\-\-strict\fR option with any of the patch level options will remove any versions beyond the scope of the patch level option, to ensure that no gem is updated that far\. -. .P To continue the previous example, if both \fB\-\-patch\fR and \fB\-\-strict\fR options are used, the available versions for resolution would be "1\.0\.4, 1\.0\.3, 1\.0\.2"\. If \fB\-\-minor\fR and \fB\-\-strict\fR are used, it would be "1\.1\.1, 1\.1\.0, 1\.0\.4, 1\.0\.3, 1\.0\.2"\. -. .P -Gem requirements as defined in the Gemfile will still be the first determining factor for what versions are available\. If the gem requirement for \fBfoo\fR in the Gemfile is \'~> 1\.0\', that will accomplish the same thing as providing the \fB\-\-minor\fR and \fB\-\-strict\fR options\. -. +Gem requirements as defined in the Gemfile will still be the first determining factor for what versions are available\. If the gem requirement for \fBfoo\fR in the Gemfile is '~> 1\.0', that will accomplish the same thing as providing the \fB\-\-minor\fR and \fB\-\-strict\fR options\. .SH "PATCH LEVEL EXAMPLES" Given the following gem specifications: -. .IP "" 4 -. .nf - foo 1\.4\.3, requires: ~> bar 2\.0 foo 1\.4\.4, requires: ~> bar 2\.0 foo 1\.4\.5, requires: ~> bar 2\.1 foo 1\.5\.0, requires: ~> bar 2\.1 foo 1\.5\.1, requires: ~> bar 3\.0 bar with versions 2\.0\.3, 2\.0\.4, 2\.1\.0, 2\.1\.1, 3\.0\.0 -. .fi -. .IP "" 0 -. .P Gemfile: -. .IP "" 4 -. .nf - -gem \'foo\' -. +gem 'foo' .fi -. .IP "" 0 -. .P Gemfile\.lock: -. .IP "" 4 -. .nf - foo (1\.4\.3) bar (~> 2\.0) bar (2\.0\.3) -. .fi -. .IP "" 0 -. .P Cases: -. .IP "" 4 -. .nf - # Command Line Result \-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\-\- -1 bundle update \-\-patch \'foo 1\.4\.5\', \'bar 2\.1\.1\' -2 bundle update \-\-patch foo \'foo 1\.4\.5\', \'bar 2\.1\.1\' -3 bundle update \-\-minor \'foo 1\.5\.1\', \'bar 3\.0\.0\' -4 bundle update \-\-minor \-\-strict \'foo 1\.5\.0\', \'bar 2\.1\.1\' -5 bundle update \-\-patch \-\-strict \'foo 1\.4\.4\', \'bar 2\.0\.4\' -. +1 bundle update \-\-patch 'foo 1\.4\.5', 'bar 2\.1\.1' +2 bundle update \-\-patch foo 'foo 1\.4\.5', 'bar 2\.1\.1' +3 bundle update \-\-minor 'foo 1\.5\.1', 'bar 3\.0\.0' +4 bundle update \-\-minor \-\-strict 'foo 1\.5\.0', 'bar 2\.1\.1' +5 bundle update \-\-patch \-\-strict 'foo 1\.4\.4', 'bar 2\.0\.4' .fi -. .IP "" 0 -. .P In case 1, bar is upgraded to 2\.1\.1, a minor version increase, because the dependency from foo 1\.4\.5 required it\. -. .P -In case 2, only foo is requested to be unlocked, but bar is also allowed to move because it\'s not a declared dependency in the Gemfile\. -. +In case 2, only foo is requested to be unlocked, but bar is also allowed to move because it's not a declared dependency in the Gemfile\. .P In case 3, bar goes up a whole major release, because a minor increase is preferred now for foo, and when it goes to 1\.5\.1, it requires 3\.0\.0 of bar\. -. .P -In case 4, foo is preferred up to a minor version, but 1\.5\.1 won\'t work because the \-\-strict flag removes bar 3\.0\.0 from consideration since it\'s a major increment\. -. +In case 4, foo is preferred up to a minor version, but 1\.5\.1 won't work because the \-\-strict flag removes bar 3\.0\.0 from consideration since it's a major increment\. .P In case 5, both foo and bar have any minor or major increments removed from consideration because of the \-\-strict flag, so the most they can move is up to 1\.4\.4 and 2\.0\.4\. -. .SH "RECOMMENDED WORKFLOW" In general, when working with an application managed with bundler, you should use the following workflow: -. .IP "\(bu" 4 After you create your Gemfile(5) for the first time, run -. .IP $ bundle install -. .IP "\(bu" 4 Check the resulting \fBGemfile\.lock\fR into version control -. .IP $ git add Gemfile\.lock -. .IP "\(bu" 4 When checking out this repository on another development machine, run -. .IP $ bundle install -. .IP "\(bu" 4 When checking out this repository on a deployment machine, run -. .IP $ bundle install \-\-deployment -. .IP "\(bu" 4 After changing the Gemfile(5) to reflect a new or update dependency, run -. .IP $ bundle install -. .IP "\(bu" 4 Make sure to check the updated \fBGemfile\.lock\fR into version control -. .IP $ git add Gemfile\.lock -. .IP "\(bu" 4 If bundle install(1) \fIbundle\-install\.1\.html\fR reports a conflict, manually update the specific gems that you changed in the Gemfile(5) -. .IP $ bundle update rails thin -. .IP "\(bu" 4 If you want to update all the gems to the latest possible versions that still match the gems listed in the Gemfile(5), run -. .IP $ bundle update \-\-all -. .IP "" 0 diff --git a/lib/bundler/man/bundle-version.1 b/lib/bundler/man/bundle-version.1 index 9a3820f3e6..44eaf92224 100644 --- a/lib/bundler/man/bundle-version.1 +++ b/lib/bundler/man/bundle-version.1 @@ -1,35 +1,22 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-VERSION" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-VERSION" "1" "May 2024" "" .SH "NAME" \fBbundle\-version\fR \- Prints Bundler version information -. .SH "SYNOPSIS" \fBbundle version\fR -. .SH "DESCRIPTION" Prints Bundler version information\. -. .SH "OPTIONS" No options\. -. .SH "EXAMPLE" Print the version of Bundler with build date and commit hash of the in the Git source\. -. .IP "" 4 -. .nf - bundle version -. .fi -. .IP "" 0 -. .P shows \fBBundler version 2\.3\.21 (2022\-08\-24 commit d54be5fdd8)\fR for example\. -. .P cf\. \fBbundle \-\-version\fR shows \fBBundler version 2\.3\.21\fR\. diff --git a/lib/bundler/man/bundle-viz.1 b/lib/bundler/man/bundle-viz.1 index 3a07010309..77b902214c 100644 --- a/lib/bundler/man/bundle-viz.1 +++ b/lib/bundler/man/bundle-viz.1 @@ -1,41 +1,29 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE\-VIZ" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE\-VIZ" "1" "May 2024" "" .SH "NAME" \fBbundle\-viz\fR \- Generates a visual dependency graph for your Gemfile -. .SH "SYNOPSIS" \fBbundle viz\fR [\-\-file=FILE] [\-\-format=FORMAT] [\-\-requirements] [\-\-version] [\-\-without=GROUP GROUP] -. .SH "DESCRIPTION" \fBviz\fR generates a PNG file of the current \fBGemfile(5)\fR as a dependency graph\. \fBviz\fR requires the ruby\-graphviz gem (and its dependencies)\. -. .P The associated gems must also be installed via \fBbundle install(1)\fR \fIbundle\-install\.1\.html\fR\. -. .P \fBviz\fR command was deprecated in Bundler 2\.2\. Use bundler\-graph plugin \fIhttps://github\.com/rubygems/bundler\-graph\fR instead\. -. .SH "OPTIONS" -. .TP \fB\-\-file\fR, \fB\-f\fR The name to use for the generated file\. See \fB\-\-format\fR option -. .TP \fB\-\-format\fR, \fB\-F\fR -This is output format option\. Supported format is png, jpg, svg, dot \.\.\. -. +This is output format option\. Supported format is png, jpg, svg, dot \|\.\|\.\|\. .TP \fB\-\-requirements\fR, \fB\-R\fR Set to show the version of each required dependency\. -. .TP \fB\-\-version\fR, \fB\-v\fR Set to show each gem version\. -. .TP \fB\-\-without\fR, \fB\-W\fR Exclude gems that are part of the specified named group\. diff --git a/lib/bundler/man/bundle.1 b/lib/bundler/man/bundle.1 index 873ba566b1..199d1ce9fd 100644 --- a/lib/bundler/man/bundle.1 +++ b/lib/bundler/man/bundle.1 @@ -1,141 +1,102 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "BUNDLE" "1" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "BUNDLE" "1" "May 2024" "" .SH "NAME" \fBbundle\fR \- Ruby Dependency Management -. .SH "SYNOPSIS" \fBbundle\fR COMMAND [\-\-no\-color] [\-\-verbose] [ARGS] -. .SH "DESCRIPTION" -Bundler manages an \fBapplication\'s dependencies\fR through its entire life across many machines systematically and repeatably\. -. +Bundler manages an \fBapplication's dependencies\fR through its entire life across many machines systematically and repeatably\. .P See the bundler website \fIhttps://bundler\.io\fR for information on getting started, and Gemfile(5) for more information on the \fBGemfile\fR format\. -. .SH "OPTIONS" -. .TP \fB\-\-no\-color\fR Print all output without color -. .TP \fB\-\-retry\fR, \fB\-r\fR Specify the number of times you wish to attempt network commands -. .TP \fB\-\-verbose\fR, \fB\-V\fR Print out additional logging information -. .SH "BUNDLE COMMANDS" We divide \fBbundle\fR subcommands into primary commands and utilities: -. .SH "PRIMARY COMMANDS" -. .TP \fBbundle install(1)\fR \fIbundle\-install\.1\.html\fR Install the gems specified by the \fBGemfile\fR or \fBGemfile\.lock\fR -. .TP \fBbundle update(1)\fR \fIbundle\-update\.1\.html\fR Update dependencies to their latest versions -. .TP \fBbundle cache(1)\fR \fIbundle\-cache\.1\.html\fR Package the \.gem files required by your application into the \fBvendor/cache\fR directory (aliases: \fBbundle package\fR, \fBbundle pack\fR) -. .TP \fBbundle exec(1)\fR \fIbundle\-exec\.1\.html\fR Execute a script in the current bundle -. .TP \fBbundle config(1)\fR \fIbundle\-config\.1\.html\fR Specify and read configuration options for Bundler -. .TP \fBbundle help(1)\fR \fIbundle\-help\.1\.html\fR Display detailed help for each subcommand -. .SH "UTILITIES" -. .TP \fBbundle add(1)\fR \fIbundle\-add\.1\.html\fR Add the named gem to the Gemfile and run \fBbundle install\fR -. .TP \fBbundle binstubs(1)\fR \fIbundle\-binstubs\.1\.html\fR Generate binstubs for executables in a gem -. .TP \fBbundle check(1)\fR \fIbundle\-check\.1\.html\fR Determine whether the requirements for your application are installed and available to Bundler -. .TP \fBbundle show(1)\fR \fIbundle\-show\.1\.html\fR Show the source location of a particular gem in the bundle -. .TP \fBbundle outdated(1)\fR \fIbundle\-outdated\.1\.html\fR Show all of the outdated gems in the current bundle -. .TP \fBbundle console(1)\fR (deprecated) Start an IRB session in the current bundle -. .TP \fBbundle open(1)\fR \fIbundle\-open\.1\.html\fR Open an installed gem in the editor -. .TP \fBbundle lock(1)\fR \fIbundle\-lock\.1\.html\fR Generate a lockfile for your dependencies -. .TP \fBbundle viz(1)\fR \fIbundle\-viz\.1\.html\fR (deprecated) Generate a visual representation of your dependencies -. .TP \fBbundle init(1)\fR \fIbundle\-init\.1\.html\fR Generate a simple \fBGemfile\fR, placed in the current directory -. .TP \fBbundle gem(1)\fR \fIbundle\-gem\.1\.html\fR Create a simple gem, suitable for development with Bundler -. .TP \fBbundle platform(1)\fR \fIbundle\-platform\.1\.html\fR Display platform compatibility information -. .TP \fBbundle clean(1)\fR \fIbundle\-clean\.1\.html\fR Clean up unused gems in your Bundler directory -. .TP \fBbundle doctor(1)\fR \fIbundle\-doctor\.1\.html\fR Display warnings about common problems -. .TP \fBbundle remove(1)\fR \fIbundle\-remove\.1\.html\fR Removes gems from the Gemfile -. .TP \fBbundle plugin(1)\fR \fIbundle\-plugin\.1\.html\fR Manage Bundler plugins -. .TP \fBbundle version(1)\fR \fIbundle\-version\.1\.html\fR Prints Bundler version information -. .SH "PLUGINS" -When running a command that isn\'t listed in PRIMARY COMMANDS or UTILITIES, Bundler will try to find an executable on your path named \fBbundler\-<command>\fR and execute it, passing down any extra arguments to it\. -. +When running a command that isn't listed in PRIMARY COMMANDS or UTILITIES, Bundler will try to find an executable on your path named \fBbundler\-<command>\fR and execute it, passing down any extra arguments to it\. .SH "OBSOLETE" These commands are obsolete and should no longer be used: -. .IP "\(bu" 4 \fBbundle inject(1)\fR -. .IP "" 0 diff --git a/lib/bundler/man/gemfile.5 b/lib/bundler/man/gemfile.5 index 8e56506c88..fa9e31f615 100644 --- a/lib/bundler/man/gemfile.5 +++ b/lib/bundler/man/gemfile.5 @@ -1,215 +1,143 @@ -.\" generated with Ronn/v0.7.3 -.\" http://github.com/rtomayko/ronn/tree/0.7.3 -. -.TH "GEMFILE" "5" "February 2023" "" "" -. +.\" generated with nRonn/v0.11.1 +.\" https://github.com/n-ronn/nronn/tree/0.11.1 +.TH "GEMFILE" "5" "May 2024" "" .SH "NAME" \fBGemfile\fR \- A format for describing gem dependencies for Ruby programs -. .SH "SYNOPSIS" A \fBGemfile\fR describes the gem dependencies required to execute associated Ruby code\. -. .P Place the \fBGemfile\fR in the root of the directory containing the associated code\. For instance, in a Rails application, place the \fBGemfile\fR in the same directory as the \fBRakefile\fR\. -. .SH "SYNTAX" A \fBGemfile\fR is evaluated as Ruby code, in a context which makes available a number of methods used to describe the gem requirements\. -. .SH "GLOBAL SOURCE" At the top of the \fBGemfile\fR, add a single line for the \fBRubyGems\fR source that contains the gems listed in the \fBGemfile\fR\. -. .IP "" 4 -. .nf - source "https://rubygems\.org" -. .fi -. .IP "" 0 -. .P You can add only one global source\. In Bundler 1\.13, adding multiple global sources was deprecated\. The \fBsource\fR \fBMUST\fR be a valid RubyGems repository\. -. .P To use more than one source of RubyGems, you should use \fI\fBsource\fR block\fR\. -. .P A source is checked for gems following the heuristics described in \fISOURCE PRIORITY\fR\. -. .P \fBNote about a behavior of the feature deprecated in Bundler 1\.13\fR: If a gem is found in more than one global source, Bundler will print a warning after installing the gem indicating which source was used, and listing the other sources where the gem is available\. A specific source can be selected for gems that need to use a non\-standard repository, suppressing this warning, by using the \fI\fB:source\fR option\fR or \fBsource\fR block\. -. .SS "CREDENTIALS" Some gem sources require a username and password\. Use bundle config(1) \fIbundle\-config\.1\.html\fR to set the username and password for any of the sources that need it\. The command must be run once on each computer that will install the Gemfile, but this keeps the credentials from being stored in plain text in version control\. -. .IP "" 4 -. .nf - bundle config gems\.example\.com user:password -. .fi -. .IP "" 0 -. .P For some sources, like a company Gemfury account, it may be easier to include the credentials in the Gemfile as part of the source URL\. -. .IP "" 4 -. .nf - source "https://user:password@gems\.example\.com" -. .fi -. .IP "" 0 -. .P Credentials in the source URL will take precedence over credentials set using \fBconfig\fR\. -. .SH "RUBY" If your application requires a specific Ruby version or engine, specify your requirements using the \fBruby\fR method, with the following arguments\. All parameters are \fBOPTIONAL\fR unless otherwise specified\. -. .SS "VERSION (required)" The version of Ruby that your application requires\. If your application requires an alternate Ruby engine, such as JRuby, TruffleRuby, etc\., this should be the Ruby version that the engine is compatible with\. -. .IP "" 4 -. .nf - ruby "3\.1\.2" -. .fi -. .IP "" 0 -. +.P +If you wish to derive your Ruby version from a version file (ie \.ruby\-version), you can use the \fBfile\fR option instead\. +.IP "" 4 +.nf +ruby file: "\.ruby\-version" +.fi +.IP "" 0 +.P +The version file should conform to any of the following formats: +.IP "\(bu" 4 +\fB3\.1\.2\fR (\.ruby\-version) +.IP "\(bu" 4 +\fBruby 3\.1\.2\fR (\.tool\-versions, read: https://asdf\-vm\.com/manage/configuration\.html#tool\-versions) +.IP "" 0 .SS "ENGINE" Each application \fImay\fR specify a Ruby engine\. If an engine is specified, an engine version \fImust\fR also be specified\. -. .P -What exactly is an Engine? \- A Ruby engine is an implementation of the Ruby language\. -. +What exactly is an Engine? +.IP "\(bu" 4 +A Ruby engine is an implementation of the Ruby language\. .IP "\(bu" 4 -For background: the reference or original implementation of the Ruby programming language is called Matz\'s Ruby Interpreter \fIhttps://en\.wikipedia\.org/wiki/Ruby_MRI\fR, or MRI for short\. This is named after Ruby creator Yukihiro Matsumoto, also known as Matz\. MRI is also known as CRuby, because it is written in C\. MRI is the most widely used Ruby engine\. -. +For background: the reference or original implementation of the Ruby programming language is called Matz's Ruby Interpreter \fIhttps://en\.wikipedia\.org/wiki/Ruby_MRI\fR, or MRI for short\. This is named after Ruby creator Yukihiro Matsumoto, also known as Matz\. MRI is also known as CRuby, because it is written in C\. MRI is the most widely used Ruby engine\. .IP "\(bu" 4 -Other implementations \fIhttps://www\.ruby\-lang\.org/en/about/\fR of Ruby exist\. Some of the more well\-known implementations include JRuby \fIhttp://jruby\.org/\fR and TruffleRuby \fIhttps://www\.graalvm\.org/ruby/\fR\. Rubinius is an alternative implementation of Ruby written in Ruby\. JRuby is an implementation of Ruby on the JVM, short for Java Virtual Machine\. TruffleRuby is a Ruby implementation on the GraalVM, a language toolkit built on the JVM\. -. +Other implementations \fIhttps://www\.ruby\-lang\.org/en/about/\fR of Ruby exist\. Some of the more well\-known implementations include JRuby \fIhttps://www\.jruby\.org/\fR and TruffleRuby \fIhttps://www\.graalvm\.org/ruby/\fR\. Rubinius is an alternative implementation of Ruby written in Ruby\. JRuby is an implementation of Ruby on the JVM, short for Java Virtual Machine\. TruffleRuby is a Ruby implementation on the GraalVM, a language toolkit built on the JVM\. .IP "" 0 -. .SS "ENGINE VERSION" Each application \fImay\fR specify a Ruby engine version\. If an engine version is specified, an engine \fImust\fR also be specified\. If the engine is "ruby" the engine version specified \fImust\fR match the Ruby version\. -. .IP "" 4 -. .nf - ruby "2\.6\.8", engine: "jruby", engine_version: "9\.3\.8\.0" -. .fi -. .IP "" 0 -. .SS "PATCHLEVEL" Each application \fImay\fR specify a Ruby patchlevel\. Specifying the patchlevel has been meaningless since Ruby 2\.1\.0 was released as the patchlevel is now uniquely determined by a combination of major, minor, and teeny version numbers\. -. .P This option was implemented in Bundler 1\.4\.0 for Ruby 2\.0 or earlier\. -. .IP "" 4 -. .nf - ruby "3\.1\.2", patchlevel: "20" -. .fi -. .IP "" 0 -. .SH "GEMS" Specify gem requirements using the \fBgem\fR method, with the following arguments\. All parameters are \fBOPTIONAL\fR unless otherwise specified\. -. .SS "NAME (required)" For each gem requirement, list a single \fIgem\fR line\. -. .IP "" 4 -. .nf - gem "nokogiri" -. .fi -. .IP "" 0 -. .SS "VERSION" Each \fIgem\fR \fBMAY\fR have one or more version specifiers\. -. .IP "" 4 -. .nf - gem "nokogiri", ">= 1\.4\.2" gem "RedCloth", ">= 4\.1\.0", "< 4\.2\.0" -. .fi -. .IP "" 0 -. .SS "REQUIRE AS" Each \fIgem\fR \fBMAY\fR specify files that should be used when autorequiring via \fBBundler\.require\fR\. You may pass an array with multiple files or \fBtrue\fR if the file you want \fBrequired\fR has the same name as \fIgem\fR or \fBfalse\fR to prevent any file from being autorequired\. -. .IP "" 4 -. .nf - gem "redis", require: ["redis/connection/hiredis", "redis"] gem "webmock", require: false gem "byebug", require: true -. .fi -. .IP "" 0 -. .P The argument defaults to the name of the gem\. For example, these are identical: -. .IP "" 4 -. .nf - gem "nokogiri" gem "nokogiri", require: "nokogiri" gem "nokogiri", require: true -. .fi -. .IP "" 0 -. .SS "GROUPS" Each \fIgem\fR \fBMAY\fR specify membership in one or more groups\. Any \fIgem\fR that does not specify membership in any group is placed in the \fBdefault\fR group\. -. .IP "" 4 -. .nf - gem "rspec", group: :test gem "wirble", groups: [:development, :test] -. .fi -. .IP "" 0 -. .P The Bundler runtime allows its two main methods, \fBBundler\.setup\fR and \fBBundler\.require\fR, to limit their impact to particular groups\. -. .IP "" 4 -. .nf - -# setup adds gems to Ruby\'s load path +# setup adds gems to Ruby's load path Bundler\.setup # defaults to all groups require "bundler/setup" # same as Bundler\.setup Bundler\.setup(:default) # only set up the _default_ group @@ -221,437 +149,269 @@ Bundler\.require # defaults to the _default_ group Bundler\.require(:default) # identical Bundler\.require(:default, :test) # requires the _default_ and _test_ groups Bundler\.require(:test) # requires the _test_ group -. .fi -. .IP "" 0 -. .P The Bundler CLI allows you to specify a list of groups whose gems \fBbundle install\fR should not install with the \fBwithout\fR configuration\. -. .P To specify multiple groups to ignore, specify a list of groups separated by spaces\. -. .IP "" 4 -. .nf - bundle config set \-\-local without test bundle config set \-\-local without development test -. .fi -. .IP "" 0 -. .P Also, calling \fBBundler\.setup\fR with no parameters, or calling \fBrequire "bundler/setup"\fR will setup all groups except for the ones you excluded via \fB\-\-without\fR (since they are not available)\. -. .P Note that on \fBbundle install\fR, bundler downloads and evaluates all gems, in order to create a single canonical list of all of the required gems and their dependencies\. This means that you cannot list different versions of the same gems in different groups\. For more details, see Understanding Bundler \fIhttps://bundler\.io/rationale\.html\fR\. -. .SS "PLATFORMS" If a gem should only be used in a particular platform or set of platforms, you can specify them\. Platforms are essentially identical to groups, except that you do not need to use the \fB\-\-without\fR install\-time flag to exclude groups of gems for other platforms\. -. .P There are a number of \fBGemfile\fR platforms: -. .TP \fBruby\fR C Ruby (MRI), Rubinius, or TruffleRuby, but not Windows -. .TP \fBmri\fR C Ruby (MRI) only, but not Windows -. .TP \fBwindows\fR Windows C Ruby (MRI), including RubyInstaller 32\-bit and 64\-bit versions -. .TP \fBmswin\fR Windows C Ruby (MRI), including RubyInstaller 32\-bit versions -. .TP \fBmswin64\fR Windows C Ruby (MRI), including RubyInstaller 64\-bit versions -. .TP \fBrbx\fR Rubinius -. .TP \fBjruby\fR JRuby -. .TP \fBtruffleruby\fR TruffleRuby -. .P On platforms \fBruby\fR, \fBmri\fR, \fBmswin\fR, \fBmswin64\fR, and \fBwindows\fR, you may additionally specify a version by appending the major and minor version numbers without a delimiter\. For example, to specify that a gem should only be used on platform \fBruby\fR version 3\.1, use: -. .IP "" 4 -. .nf - ruby_31 -. .fi -. .IP "" 0 -. .P As with groups (above), you may specify one or more platforms: -. .IP "" 4 -. .nf - gem "weakling", platforms: :jruby gem "ruby\-debug", platforms: :mri_31 gem "nokogiri", platforms: [:windows_31, :jruby] -. .fi -. .IP "" 0 -. .P All operations involving groups (\fBbundle install\fR \fIbundle\-install\.1\.html\fR, \fBBundler\.setup\fR, \fBBundler\.require\fR) behave exactly the same as if any groups not matching the current platform were explicitly excluded\. -. .P The following platform values are deprecated and should be replaced with \fBwindows\fR: -. .IP "\(bu" 4 \fBmswin\fR, \fBmswin64\fR, \fBmingw32\fR, \fBx64_mingw\fR -. .IP "" 0 -. .SS "FORCE_RUBY_PLATFORM" If you always want the pure ruby variant of a gem to be chosen over platform specific variants, you can use the \fBforce_ruby_platform\fR option: -. .IP "" 4 -. .nf - gem "ffi", force_ruby_platform: true -. .fi -. .IP "" 0 -. .P This can be handy (assuming the pure ruby variant works fine) when: -. .IP "\(bu" 4 -You\'re having issues with the platform specific variant\. -. +You're having issues with the platform specific variant\. .IP "\(bu" 4 The platform specific variant does not yet support a newer ruby (and thus has a \fBrequired_ruby_version\fR upper bound), but you still want your Gemfile{\.lock} files to resolve under that ruby\. -. .IP "" 0 -. .SS "SOURCE" -You can select an alternate RubyGems repository for a gem using the \':source\' option\. -. +You can select an alternate RubyGems repository for a gem using the ':source' option\. .IP "" 4 -. .nf - gem "some_internal_gem", source: "https://gems\.example\.com" -. .fi -. .IP "" 0 -. .P This forces the gem to be loaded from this source and ignores the global source declared at the top level of the file\. If the gem does not exist in this source, it will not be installed\. -. .P Bundler will search for child dependencies of this gem by first looking in the source selected for the parent, but if they are not found there, it will fall back on the global source\. -. .P \fBNote about a behavior of the feature deprecated in Bundler 1\.13\fR: Selecting a specific source repository this way also suppresses the ambiguous gem warning described above in \fIGLOBAL SOURCE\fR\. -. .P Using the \fB:source\fR option for an individual gem will also make that source available as a possible global source for any other gems which do not specify explicit sources\. Thus, when adding gems with explicit sources, it is recommended that you also ensure all other gems in the Gemfile are using explicit sources\. -. .SS "GIT" If necessary, you can specify that a gem is located at a particular git repository using the \fB:git\fR parameter\. The repository can be accessed via several protocols: -. .TP \fBHTTP(S)\fR gem "rails", git: "https://github\.com/rails/rails\.git" -. .TP \fBSSH\fR gem "rails", git: "git@github\.com:rails/rails\.git" -. .TP \fBgit\fR gem "rails", git: "git://github\.com/rails/rails\.git" -. .P If using SSH, the user that you use to run \fBbundle install\fR \fBMUST\fR have the appropriate keys available in their \fB$HOME/\.ssh\fR\. -. .P \fBNOTE\fR: \fBhttp://\fR and \fBgit://\fR URLs should be avoided if at all possible\. These protocols are unauthenticated, so a man\-in\-the\-middle attacker can deliver malicious code and compromise your system\. HTTPS and SSH are strongly preferred\. -. .P The \fBgroup\fR, \fBplatforms\fR, and \fBrequire\fR options are available and behave exactly the same as they would for a normal gem\. -. .P A git repository \fBSHOULD\fR have at least one file, at the root of the directory containing the gem, with the extension \fB\.gemspec\fR\. This file \fBMUST\fR contain a valid gem specification, as expected by the \fBgem build\fR command\. -. .P If a git repository does not have a \fB\.gemspec\fR, bundler will attempt to create one, but it will not contain any dependencies, executables, or C extension compilation instructions\. As a result, it may fail to properly integrate into your application\. -. .P If a git repository does have a \fB\.gemspec\fR for the gem you attached it to, a version specifier, if provided, means that the git repository is only valid if the \fB\.gemspec\fR specifies a version matching the version specifier\. If not, bundler will print a warning\. -. .IP "" 4 -. .nf - gem "rails", "2\.3\.8", git: "https://github\.com/rails/rails\.git" # bundle install will fail, because the \.gemspec in the rails -# repository\'s master branch specifies version 3\.0\.0 -. +# repository's master branch specifies version 3\.0\.0 .fi -. .IP "" 0 -. .P If a git repository does \fBnot\fR have a \fB\.gemspec\fR for the gem you attached it to, a version specifier \fBMUST\fR be provided\. Bundler will use this version in the simple \fB\.gemspec\fR it creates\. -. .P Git repositories support a number of additional options\. -. .TP \fBbranch\fR, \fBtag\fR, and \fBref\fR You \fBMUST\fR only specify at most one of these options\. The default is \fBbranch: "master"\fR\. For example: -. .IP gem "rails", git: "https://github\.com/rails/rails\.git", branch: "5\-0\-stable" -. .IP gem "rails", git: "https://github\.com/rails/rails\.git", tag: "v5\.0\.0" -. .IP gem "rails", git: "https://github\.com/rails/rails\.git", ref: "4aded" -. .TP \fBsubmodules\fR For reference, a git submodule \fIhttps://git\-scm\.com/book/en/v2/Git\-Tools\-Submodules\fR lets you have another git repository within a subfolder of your repository\. Specify \fBsubmodules: true\fR to cause bundler to expand any submodules included in the git repository -. .P If a git repository contains multiple \fB\.gemspecs\fR, each \fB\.gemspec\fR represents a gem located at the same place in the file system as the \fB\.gemspec\fR\. -. .IP "" 4 -. .nf - |~rails [git root] | |\-rails\.gemspec [rails gem located here] |~actionpack | |\-actionpack\.gemspec [actionpack gem located here] |~activesupport | |\-activesupport\.gemspec [activesupport gem located here] -|\.\.\. -. +|\|\.\|\.\|\. .fi -. .IP "" 0 -. .P To install a gem located in a git repository, bundler changes to the directory containing the gemspec, runs \fBgem build name\.gemspec\fR and then installs the resulting gem\. The \fBgem build\fR command, which comes standard with Rubygems, evaluates the \fB\.gemspec\fR in the context of the directory in which it is located\. -. .SS "GIT SOURCE" -A custom git source can be defined via the \fBgit_source\fR method\. Provide the source\'s name as an argument, and a block which receives a single argument and interpolates it into a string to return the full repo address: -. +A custom git source can be defined via the \fBgit_source\fR method\. Provide the source's name as an argument, and a block which receives a single argument and interpolates it into a string to return the full repo address: .IP "" 4 -. .nf - git_source(:stash){ |repo_name| "https://stash\.corp\.acme\.pl/#{repo_name}\.git" } -gem \'rails\', stash: \'forks/rails\' -. +gem 'rails', stash: 'forks/rails' .fi -. .IP "" 0 -. .P In addition, if you wish to choose a specific branch: -. .IP "" 4 -. .nf - gem "rails", stash: "forks/rails", branch: "branch_name" -. .fi -. .IP "" 0 -. .SS "GITHUB" \fBNOTE\fR: This shorthand should be avoided until Bundler 2\.0, since it currently expands to an insecure \fBgit://\fR URL\. This allows a man\-in\-the\-middle attacker to compromise your system\. -. .P If the git repository you want to use is hosted on GitHub and is public, you can use the :github shorthand to specify the github username and repository name (without the trailing "\.git"), separated by a slash\. If both the username and repository name are the same, you can omit one\. -. .IP "" 4 -. .nf - gem "rails", github: "rails/rails" gem "rails", github: "rails" -. .fi -. .IP "" 0 -. .P Are both equivalent to -. .IP "" 4 -. .nf - gem "rails", git: "https://github\.com/rails/rails\.git" -. .fi -. .IP "" 0 -. .P Since the \fBgithub\fR method is a specialization of \fBgit_source\fR, it accepts a \fB:branch\fR named argument\. -. .P You can also directly pass a pull request URL: -. .IP "" 4 -. .nf - gem "rails", github: "https://github\.com/rails/rails/pull/43753" -. .fi -. .IP "" 0 -. .P Which is equivalent to: -. .IP "" 4 -. .nf - gem "rails", github: "rails/rails", branch: "refs/pull/43753/head" -. .fi -. .IP "" 0 -. .SS "GIST" If the git repository you want to use is hosted as a GitHub Gist and is public, you can use the :gist shorthand to specify the gist identifier (without the trailing "\.git")\. -. .IP "" 4 -. .nf - gem "the_hatch", gist: "4815162342" -. .fi -. .IP "" 0 -. .P Is equivalent to: -. .IP "" 4 -. .nf - gem "the_hatch", git: "https://gist\.github\.com/4815162342\.git" -. .fi -. .IP "" 0 -. .P Since the \fBgist\fR method is a specialization of \fBgit_source\fR, it accepts a \fB:branch\fR named argument\. -. .SS "BITBUCKET" If the git repository you want to use is hosted on Bitbucket and is public, you can use the :bitbucket shorthand to specify the bitbucket username and repository name (without the trailing "\.git"), separated by a slash\. If both the username and repository name are the same, you can omit one\. -. .IP "" 4 -. .nf - gem "rails", bitbucket: "rails/rails" gem "rails", bitbucket: "rails" -. .fi -. .IP "" 0 -. .P Are both equivalent to -. .IP "" 4 -. .nf - gem "rails", git: "https://rails@bitbucket\.org/rails/rails\.git" -. .fi -. .IP "" 0 -. .P Since the \fBbitbucket\fR method is a specialization of \fBgit_source\fR, it accepts a \fB:branch\fR named argument\. -. .SS "PATH" You can specify that a gem is located in a particular location on the file system\. Relative paths are resolved relative to the directory containing the \fBGemfile\fR\. -. .P Similar to the semantics of the \fB:git\fR option, the \fB:path\fR option requires that the directory in question either contains a \fB\.gemspec\fR for the gem, or that you specify an explicit version that bundler should use\. -. .P Unlike \fB:git\fR, bundler does not compile C extensions for gems specified as paths\. -. .IP "" 4 -. .nf - gem "rails", path: "vendor/rails" -. .fi -. .IP "" 0 -. .P -If you would like to use multiple local gems directly from the filesystem, you can set a global \fBpath\fR option to the path containing the gem\'s files\. This will automatically load gemspec files from subdirectories\. -. +If you would like to use multiple local gems directly from the filesystem, you can set a global \fBpath\fR option to the path containing the gem's files\. This will automatically load gemspec files from subdirectories\. .IP "" 4 -. .nf - -path \'components\' do - gem \'admin_ui\' - gem \'public_ui\' +path 'components' do + gem 'admin_ui' + gem 'public_ui' end -. .fi -. .IP "" 0 -. .SH "BLOCK FORM OF SOURCE, GIT, PATH, GROUP and PLATFORMS" The \fB:source\fR, \fB:git\fR, \fB:path\fR, \fB:group\fR, and \fB:platforms\fR options may be applied to a group of gems by using block form\. -. .IP "" 4 -. .nf - source "https://gems\.example\.com" do gem "some_internal_gem" gem "another_internal_gem" @@ -671,61 +431,40 @@ group :development, optional: true do gem "wirble" gem "faker" end -. .fi -. .IP "" 0 -. .P In the case of the group block form the :optional option can be given to prevent a group from being installed unless listed in the \fB\-\-with\fR option given to the \fBbundle install\fR command\. -. .P In the case of the \fBgit\fR block form, the \fB:ref\fR, \fB:branch\fR, \fB:tag\fR, and \fB:submodules\fR options may be passed to the \fBgit\fR method, and all gems in the block will inherit those options\. -. .P The presence of a \fBsource\fR block in a Gemfile also makes that source available as a possible global source for any other gems which do not specify explicit sources\. Thus, when defining source blocks, it is recommended that you also ensure all other gems in the Gemfile are using explicit sources, either via source blocks or \fB:source\fR directives on individual gems\. -. .SH "INSTALL_IF" The \fBinstall_if\fR method allows gems to be installed based on a proc or lambda\. This is especially useful for optional gems that can only be used if certain software is installed or some other conditions are met\. -. .IP "" 4 -. .nf - install_if \-> { RUBY_PLATFORM =~ /darwin/ } do gem "pasteboard" end -. .fi -. .IP "" 0 -. .SH "GEMSPEC" -The \fB\.gemspec\fR \fIhttp://guides\.rubygems\.org/specification\-reference/\fR file is where you provide metadata about your gem to Rubygems\. Some required Gemspec attributes include the name, description, and homepage of your gem\. This is also where you specify the dependencies your gem needs to run\. -. +The \fB\.gemspec\fR \fIhttps://guides\.rubygems\.org/specification\-reference/\fR file is where you provide metadata about your gem to Rubygems\. Some required Gemspec attributes include the name, description, and homepage of your gem\. This is also where you specify the dependencies your gem needs to run\. .P If you wish to use Bundler to help install dependencies for a gem while it is being developed, use the \fBgemspec\fR method to pull in the dependencies listed in the \fB\.gemspec\fR file\. -. .P -The \fBgemspec\fR method adds any runtime dependencies as gem requirements in the default group\. It also adds development dependencies as gem requirements in the \fBdevelopment\fR group\. Finally, it adds a gem requirement on your project (\fBpath: \'\.\'\fR)\. In conjunction with \fBBundler\.setup\fR, this allows you to require project files in your test code as you would if the project were installed as a gem; you need not manipulate the load path manually or require project files via relative paths\. -. +The \fBgemspec\fR method adds any runtime dependencies as gem requirements in the default group\. It also adds development dependencies as gem requirements in the \fBdevelopment\fR group\. Finally, it adds a gem requirement on your project (\fBpath: '\.'\fR)\. In conjunction with \fBBundler\.setup\fR, this allows you to require project files in your test code as you would if the project were installed as a gem; you need not manipulate the load path manually or require project files via relative paths\. .P -The \fBgemspec\fR method supports optional \fB:path\fR, \fB:glob\fR, \fB:name\fR, and \fB:development_group\fR options, which control where bundler looks for the \fB\.gemspec\fR, the glob it uses to look for the gemspec (defaults to: "{,\fI,\fR/*}\.gemspec"), what named \fB\.gemspec\fR it uses (if more than one is present), and which group development dependencies are included in\. -. +The \fBgemspec\fR method supports optional \fB:path\fR, \fB:glob\fR, \fB:name\fR, and \fB:development_group\fR options, which control where bundler looks for the \fB\.gemspec\fR, the glob it uses to look for the gemspec (defaults to: \fB{,*,*/*}\.gemspec\fR), what named \fB\.gemspec\fR it uses (if more than one is present), and which group development dependencies are included in\. .P When a \fBgemspec\fR dependency encounters version conflicts during resolution, the local version under development will always be selected \-\- even if there are remote versions that better match other requirements for the \fBgemspec\fR gem\. -. .SH "SOURCE PRIORITY" When attempting to locate a gem to satisfy a gem requirement, bundler uses the following priority order: -. .IP "1." 4 The source explicitly attached to the gem (using \fB:source\fR, \fB:path\fR, or \fB:git\fR) -. .IP "2." 4 For implicit gems (dependencies of explicit gems), any source, git, or path repository declared on the parent\. This results in bundler prioritizing the ActiveSupport gem from the Rails git repository over ones from \fBrubygems\.org\fR -. .IP "3." 4 If neither of the above conditions are met, the global source will be used\. If multiple global sources are specified, they will be prioritized from last to first, but this is deprecated since Bundler 1\.13, so Bundler prints a warning and will abort with an error in the future\. -. .IP "" 0 diff --git a/lib/bundler/man/gemfile.5.ronn b/lib/bundler/man/gemfile.5.ronn index a5d2288b96..7c1e00d13a 100644 --- a/lib/bundler/man/gemfile.5.ronn +++ b/lib/bundler/man/gemfile.5.ronn @@ -69,6 +69,16 @@ should be the Ruby version that the engine is compatible with. ruby "3.1.2" +If you wish to derive your Ruby version from a version file (ie .ruby-version), +you can use the `file` option instead. + + ruby file: ".ruby-version" + +The version file should conform to any of the following formats: + + - `3.1.2` (.ruby-version) + - `ruby 3.1.2` (.tool-versions, read: https://asdf-vm.com/manage/configuration.html#tool-versions) + ### ENGINE Each application _may_ specify a Ruby engine. If an engine is specified, an @@ -86,7 +96,7 @@ What exactly is an Engine? - [Other implementations](https://www.ruby-lang.org/en/about/) of Ruby exist. Some of the more well-known implementations include - [JRuby](http://jruby.org/) and [TruffleRuby](https://www.graalvm.org/ruby/). + [JRuby](https://www.jruby.org/) and [TruffleRuby](https://www.graalvm.org/ruby/). Rubinius is an alternative implementation of Ruby written in Ruby. JRuby is an implementation of Ruby on the JVM, short for Java Virtual Machine. TruffleRuby is a Ruby implementation on the GraalVM, a language toolkit built on the JVM. @@ -499,7 +509,7 @@ software is installed or some other conditions are met. ## GEMSPEC -The [`.gemspec`](http://guides.rubygems.org/specification-reference/) file is where +The [`.gemspec`](https://guides.rubygems.org/specification-reference/) file is where you provide metadata about your gem to Rubygems. Some required Gemspec attributes include the name, description, and homepage of your gem. This is also where you specify the dependencies your gem needs to run. @@ -518,7 +528,7 @@ paths. The `gemspec` method supports optional `:path`, `:glob`, `:name`, and `:development_group` options, which control where bundler looks for the `.gemspec`, the glob it uses to look -for the gemspec (defaults to: "{,*,*/*}.gemspec"), what named `.gemspec` it uses +for the gemspec (defaults to: `{,*,*/*}.gemspec`), what named `.gemspec` it uses (if more than one is present), and which group development dependencies are included in. When a `gemspec` dependency encounters version conflicts during resolution, the diff --git a/lib/bundler/match_metadata.rb b/lib/bundler/match_metadata.rb index 499036ca93..f6cc27df32 100644 --- a/lib/bundler/match_metadata.rb +++ b/lib/bundler/match_metadata.rb @@ -2,6 +2,10 @@ module Bundler module MatchMetadata + def matches_current_metadata? + matches_current_ruby? && matches_current_rubygems? + end + def matches_current_ruby? @required_ruby_version.satisfied_by?(Gem.ruby_version) end diff --git a/lib/bundler/match_platform.rb b/lib/bundler/match_platform.rb index 7f7e8227f9..ece9fb8679 100644 --- a/lib/bundler/match_platform.rb +++ b/lib/bundler/match_platform.rb @@ -12,7 +12,7 @@ module Bundler def self.platforms_match?(gemspec_platform, local_platform) return true if gemspec_platform.nil? - return true if Gem::Platform::RUBY == gemspec_platform + return true if gemspec_platform == Gem::Platform::RUBY return true if local_platform == gemspec_platform gemspec_platform = Gem::Platform.new(gemspec_platform) return true if gemspec_platform === local_platform diff --git a/lib/bundler/mirror.rb b/lib/bundler/mirror.rb index 9d437a0951..494a6d6aef 100644 --- a/lib/bundler/mirror.rb +++ b/lib/bundler/mirror.rb @@ -47,7 +47,7 @@ module Bundler def fetch_valid_mirror_for(uri) downcased = uri.to_s.downcase - mirror = @mirrors[downcased] || @mirrors[Bundler::URI(downcased).host] || Mirror.new(uri) + mirror = @mirrors[downcased] || @mirrors[Gem::URI(downcased).host] || Mirror.new(uri) mirror.validate!(@prober) mirror = Mirror.new(uri) unless mirror.valid? mirror @@ -74,7 +74,7 @@ module Bundler @uri = if uri.nil? nil else - Bundler::URI(uri.to_s) + Gem::URI(uri.to_s) end @valid = nil end @@ -126,7 +126,7 @@ module Bundler if uri == "all" @all = true else - @uri = Bundler::URI(uri).absolute? ? Settings.normalize_uri(uri) : uri + @uri = Gem::URI(uri).absolute? ? Settings.normalize_uri(uri) : uri end @value = value end diff --git a/lib/bundler/plugin.rb b/lib/bundler/plugin.rb index f3caff8963..588fa79be8 100644 --- a/lib/bundler/plugin.rb +++ b/lib/bundler/plugin.rb @@ -62,7 +62,8 @@ module Bundler if names.any? names.each do |name| if index.installed?(name) - Bundler.rm_rf(index.plugin_path(name)) + path = index.plugin_path(name).to_s + Bundler.rm_rf(path) if index.installed_in_plugin_root?(name) index.unregister_plugin(name) Bundler.ui.info "Uninstalled plugin #{name}" else @@ -100,7 +101,7 @@ module Bundler # @param [Pathname] gemfile path # @param [Proc] block that can be evaluated for (inline) Gemfile def gemfile_install(gemfile = nil, &inline) - Bundler.settings.temporary(:frozen => false, :deployment => false) do + Bundler.settings.temporary(frozen: false, deployment: false) do builder = DSL.new if block_given? builder.instance_eval(&inline) @@ -197,7 +198,7 @@ module Bundler # @param [Hash] The options that are present in the lock file # @return [API::Source] the instance of the class that handles the source # type passed in locked_opts - def source_from_lock(locked_opts) + def from_lock(locked_opts) src = source(locked_opts["type"]) src.new(locked_opts.merge("uri" => locked_opts["remote"])) @@ -227,7 +228,7 @@ module Bundler plugins = index.hook_plugins(event) return unless plugins.any? - (plugins - @loaded_plugin_names).each {|name| load_plugin(name) } + plugins.each {|name| load_plugin(name) } @hooks_by_event[event].each {|blk| blk.call(*args, &arg_blk) } end @@ -239,6 +240,11 @@ module Bundler Index.new.installed?(plugin) end + # @return [true, false] whether the plugin is loaded + def loaded?(plugin) + @loaded_plugin_names.include?(plugin) + end + # Post installation processing and registering with index # # @param [Array<String>] plugins list to be installed @@ -301,7 +307,7 @@ module Bundler @hooks_by_event = Hash.new {|h, k| h[k] = [] } load_paths = spec.load_paths - Bundler.rubygems.add_to_load_path(load_paths) + Gem.add_to_load_path(*load_paths) path = Pathname.new spec.full_gem_path begin @@ -329,13 +335,14 @@ module Bundler # @param [String] name of the plugin def load_plugin(name) return unless name && !name.empty? + return if loaded?(name) # Need to ensure before this that plugin root where the rest of gems # are installed to be on load path to support plugin deps. Currently not # done to avoid conflicts path = index.plugin_path(name) - Bundler.rubygems.add_to_load_path(index.load_paths(name)) + Gem.add_to_load_path(*index.load_paths(name)) load path.join(PLUGIN_FILE_NAME) diff --git a/lib/bundler/plugin/api/source.rb b/lib/bundler/plugin/api/source.rb index 67c45bd204..8563ee358a 100644 --- a/lib/bundler/plugin/api/source.rb +++ b/lib/bundler/plugin/api/source.rb @@ -39,7 +39,7 @@ module Bundler # is present to be compatible with `Definition` and is used by # rubygems source. module Source - attr_reader :uri, :options, :name + attr_reader :uri, :options, :name, :checksum_store attr_accessor :dependency_names def initialize(opts) @@ -48,6 +48,7 @@ module Bundler @uri = opts["uri"] @type = opts["type"] @name = opts["name"] || "#{@type} at #{@uri}" + @checksum_store = Checksum::Store.new end # This is used by the default `spec` method to constructs the @@ -95,7 +96,7 @@ module Bundler # # Note: Do not override if you don't know what you are doing. def post_install(spec, disable_exts = false) - opts = { :env_shebang => false, :disable_extensions => disable_exts } + opts = { env_shebang: false, disable_extensions: disable_exts } installer = Bundler::Source::Path::Installer.new(spec, opts) installer.post_install end @@ -106,7 +107,7 @@ module Bundler def install_path @install_path ||= begin - base_name = File.basename(Bundler::URI.parse(uri).normalize.path) + base_name = File.basename(Gem::URI.parse(uri).normalize.path) gem_install_dir.join("#{base_name}-#{uri_hash[0..11]}") end @@ -175,7 +176,7 @@ module Bundler # # This is used by `app_cache_path` def app_cache_dirname - base_name = File.basename(Bundler::URI.parse(uri).normalize.path) + base_name = File.basename(Gem::URI.parse(uri).normalize.path) "#{base_name}-#{uri_hash}" end diff --git a/lib/bundler/plugin/events.rb b/lib/bundler/plugin/events.rb index bc037d1af5..29c05098ae 100644 --- a/lib/bundler/plugin/events.rb +++ b/lib/bundler/plugin/events.rb @@ -56,6 +56,30 @@ module Bundler # Includes an Array of Bundler::Dependency objects # GEM_AFTER_INSTALL_ALL = "after-install-all" define :GEM_AFTER_INSTALL_ALL, "after-install-all" + + # @!parse + # A hook called before each individual gem is required + # Includes a Bundler::Dependency. + # GEM_BEFORE_REQUIRE = "before-require" + define :GEM_BEFORE_REQUIRE, "before-require" + + # @!parse + # A hook called after each individual gem is required + # Includes a Bundler::Dependency. + # GEM_AFTER_REQUIRE = "after-require" + define :GEM_AFTER_REQUIRE, "after-require" + + # @!parse + # A hook called before any gems require + # Includes an Array of Bundler::Dependency objects. + # GEM_BEFORE_REQUIRE_ALL = "before-require-all" + define :GEM_BEFORE_REQUIRE_ALL, "before-require-all" + + # @!parse + # A hook called after all gems required + # Includes an Array of Bundler::Dependency objects. + # GEM_AFTER_REQUIRE_ALL = "after-require-all" + define :GEM_AFTER_REQUIRE_ALL, "after-require-all" end end end diff --git a/lib/bundler/plugin/index.rb b/lib/bundler/plugin/index.rb index a2d5eaa38a..c2ab8f90da 100644 --- a/lib/bundler/plugin/index.rb +++ b/lib/bundler/plugin/index.rb @@ -136,6 +136,14 @@ module Bundler @hooks[event] || [] end + # This plugin is installed inside the .bundle/plugin directory, + # and thus is managed solely by Bundler + def installed_in_plugin_root?(name) + return false unless (path = installed?(name)) + + path.start_with?("#{Plugin.root}/") + end + private # Reads the index file from the directory and initializes the instance diff --git a/lib/bundler/plugin/installer.rb b/lib/bundler/plugin/installer.rb index c9ff12ce4b..4f60862bb4 100644 --- a/lib/bundler/plugin/installer.rb +++ b/lib/bundler/plugin/installer.rb @@ -10,6 +10,7 @@ module Bundler class Installer autoload :Rubygems, File.expand_path("installer/rubygems", __dir__) autoload :Git, File.expand_path("installer/git", __dir__) + autoload :Path, File.expand_path("installer/path", __dir__) def install(names, options) check_sources_consistency!(options) @@ -18,8 +19,8 @@ module Bundler if options[:git] install_git(names, version, options) - elsif options[:local_git] - install_local_git(names, version, options) + elsif options[:path] + install_path(names, version, options[:path]) else sources = options[:source] || Gem.sources install_rubygems(names, version, sources) @@ -45,20 +46,40 @@ module Bundler if options.key?(:git) && options.key?(:local_git) raise InvalidOption, "Remote and local plugin git sources can't be both specified" end + + # back-compat; local_git is an alias for git + if options.key?(:local_git) + Bundler::SharedHelpers.major_deprecation(2, "--local_git is deprecated, use --git") + options[:git] = options.delete(:local_git) + end + + if (options.keys & [:source, :git, :path]).length > 1 + raise InvalidOption, "Only one of --source, --git, or --path may be specified" + end + + if (options.key?(:branch) || options.key?(:ref)) && !options.key?(:git) + raise InvalidOption, "--#{options.key?(:branch) ? "branch" : "ref"} can only be used with git sources" + end + + if options.key?(:branch) && options.key?(:ref) + raise InvalidOption, "--branch and --ref can't be both specified" + end end def install_git(names, version, options) - uri = options.delete(:git) - options["uri"] = uri + source_list = SourceList.new + source = source_list.add_git_source({ "uri" => options[:git], + "branch" => options[:branch], + "ref" => options[:ref] }) - install_all_sources(names, version, options, options[:source]) + install_all_sources(names, version, source_list, source) end - def install_local_git(names, version, options) - uri = options.delete(:local_git) - options["uri"] = uri + def install_path(names, version, path) + source_list = SourceList.new + source = source_list.add_path_source({ "path" => path, "root_path" => SharedHelpers.pwd }) - install_all_sources(names, version, options, options[:source]) + install_all_sources(names, version, source_list, source) end # Installs the plugin from rubygems source and returns the path where the @@ -70,20 +91,19 @@ module Bundler # # @return [Hash] map of names to the specs of plugins installed def install_rubygems(names, version, sources) - install_all_sources(names, version, nil, sources) - end - - def install_all_sources(names, version, git_source_options, rubygems_source) source_list = SourceList.new - source_list.add_git_source(git_source_options) if git_source_options - Array(rubygems_source).each {|remote| source_list.add_global_rubygems_remote(remote) } if rubygems_source + Array(sources).each {|remote| source_list.add_global_rubygems_remote(remote) } + + install_all_sources(names, version, source_list) + end - deps = names.map {|name| Dependency.new name, version } + def install_all_sources(names, version, source_list, source = nil) + deps = names.map {|name| Dependency.new(name, version, { "source" => source }) } Bundler.configure_gem_home_and_path(Plugin.root) - Bundler.settings.temporary(:deployment => false, :frozen => false) do + Bundler.settings.temporary(deployment: false, frozen: false) do definition = Definition.new(nil, deps, source_list, true) install_definition(definition) diff --git a/lib/bundler/plugin/installer/path.rb b/lib/bundler/plugin/installer/path.rb new file mode 100644 index 0000000000..58a8fa7426 --- /dev/null +++ b/lib/bundler/plugin/installer/path.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Bundler + module Plugin + class Installer + class Path < Bundler::Source::Path + def root + SharedHelpers.in_bundle? ? Bundler.root : Plugin.root + end + + def generate_bin(spec, disable_extensions = false) + # Need to find a way without code duplication + # For now, we can ignore this + end + end + end + end +end diff --git a/lib/bundler/plugin/source_list.rb b/lib/bundler/plugin/source_list.rb index 547661cf2f..746996de55 100644 --- a/lib/bundler/plugin/source_list.rb +++ b/lib/bundler/plugin/source_list.rb @@ -9,6 +9,10 @@ module Bundler add_source_to_list Plugin::Installer::Git.new(options), git_sources end + def add_path_source(options = {}) + add_source_to_list Plugin::Installer::Path.new(options), path_sources + end + def add_rubygems_source(options = {}) add_source_to_list Plugin::Installer::Rubygems.new(options), @rubygems_sources end @@ -17,10 +21,6 @@ module Bundler path_sources + git_sources + rubygems_sources + [metadata_source] end - def default_source - git_sources.first || global_rubygems_source - end - private def rubygems_aggregate_class diff --git a/lib/bundler/remote_specification.rb b/lib/bundler/remote_specification.rb index f626a3218e..9d237f3fa0 100644 --- a/lib/bundler/remote_specification.rb +++ b/lib/bundler/remote_specification.rb @@ -88,6 +88,10 @@ module Bundler end end + def runtime_dependencies + dependencies.select(&:runtime?) + end + def git_version return unless loaded_from && source.is_a?(Bundler::Source::Git) " #{source.revision[0..6]}" diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index c8cc88a3ee..1a6711ea6f 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -29,7 +29,7 @@ module Bundler Bundler.ui.info "Resolving dependencies...", true - solve_versions(:root => root, :logger => logger) + solve_versions(root: root, logger: logger) end def setup_solver @@ -37,31 +37,39 @@ module Bundler root_version = Resolver::Candidate.new(0) @all_specs = Hash.new do |specs, name| - specs[name] = source_for(name).specs.search(name).reject do |s| - s.dependencies.any? {|d| d.name == name && !d.requirement.satisfied_by?(s.version) } # ignore versions that depend on themselves incorrectly - end.sort_by {|s| [s.version, s.platform.to_s] } + source = source_for(name) + matches = source.specs.search(name) + + # Don't bother to check for circular deps when no dependency API are + # available, since it's too slow to be usable. That edge case won't work + # but resolution other than that should work fine and reasonably fast. + if source.respond_to?(:dependency_api_available?) && source.dependency_api_available? + matches = filter_invalid_self_dependencies(matches, name) + end + + specs[name] = matches.sort_by {|s| [s.version, s.platform.to_s] } + end + + @all_versions = Hash.new do |candidates, package| + candidates[package] = all_versions_for(package) end @sorted_versions = Hash.new do |candidates, package| - candidates[package] = if package.root? - [root_version] - else - all_versions_for(package).sort - end + candidates[package] = filtered_versions_for(package).sort end + @sorted_versions[root] = [root_version] + root_dependencies = prepare_dependencies(@requirements, @packages) @cached_dependencies = Hash.new do |dependencies, package| - dependencies[package] = if package.root? - { root_version => root_dependencies } - else - Hash.new do |versions, version| - versions[version] = to_dependency_hash(version.dependencies.reject {|d| d.name == package.name }, @packages) - end + dependencies[package] = Hash.new do |versions, version| + versions[version] = to_dependency_hash(version.dependencies.reject {|d| d.name == package.name }, @packages) end end + @cached_dependencies[root] = { root_version => root_dependencies } + logger = Bundler::UI::Shell.new logger.level = debug? ? "debug" : "warn" @@ -69,7 +77,7 @@ module Bundler end def solve_versions(root:, logger:) - solver = PubGrub::VersionSolver.new(:source => self, :root => root, :logger => logger) + solver = PubGrub::VersionSolver.new(source: self, root: root, logger: logger) result = solver.solve result.map {|package, version| version.to_specs(package) }.flatten.uniq rescue PubGrub::SolveFailure => e @@ -123,7 +131,7 @@ module Bundler if base_requirements[name] names_to_unlock << name - elsif package.ignores_prereleases? + elsif package.ignores_prereleases? && @all_specs[name].any? {|s| s.version.prerelease? } names_to_allow_prereleases_for << name end @@ -144,13 +152,19 @@ module Bundler requirement_to_range(dependency) end - PubGrub::VersionConstraint.new(package, :range => range) + PubGrub::VersionConstraint.new(package, range: range) end def versions_for(package, range=VersionRange.any) - versions = range.select_versions(@sorted_versions[package]) + versions = select_sorted_versions(package, range) - sort_versions(package, versions) + # Conditional avoids (among other things) calling + # sort_versions_by_preferred with the root package + if versions.size > 1 + sort_versions_by_preferred(package, versions) + else + versions + end end def no_versions_incompatibility_for(package, unsatisfied_term) @@ -160,7 +174,7 @@ module Bundler constraint_string = constraint.constraint_string requirements = constraint_string.split(" OR ").map {|req| Gem::Requirement.new(req.split(",")) } - if name == "bundler" + if name == "bundler" && bundler_pinned_to_current_version? custom_explanation = "the current Bundler version (#{Bundler::VERSION}) does not satisfy #{constraint}" extended_explanation = bundler_not_found_message(requirements) else @@ -173,7 +187,7 @@ module Bundler extended_explanation = other_specs_matching_message(specs_matching_other_platforms, label) if specs_matching_other_platforms.any? end - Incompatibility.new([unsatisfied_term], :cause => cause, :custom_explanation => custom_explanation, :extended_explanation => extended_explanation) + Incompatibility.new([unsatisfied_term], cause: cause, custom_explanation: custom_explanation, extended_explanation: extended_explanation) end def debug? @@ -212,9 +226,9 @@ module Bundler sorted_versions[high] end - range = PubGrub::VersionRange.new(:min => low, :max => high, :include_min => true) + range = PubGrub::VersionRange.new(min: low, max: high, include_min: true) - self_constraint = PubGrub::VersionConstraint.new(package, :range => range) + self_constraint = PubGrub::VersionConstraint.new(package, range: range) dep_term = PubGrub::Term.new(dep_constraint, false) self_term = PubGrub::Term.new(self_constraint, true) @@ -223,37 +237,63 @@ module Bundler "current #{dep_package} version is #{dep_constraint.constraint_string}" end - PubGrub::Incompatibility.new([self_term, dep_term], :cause => :dependency, :custom_explanation => custom_explanation) + PubGrub::Incompatibility.new([self_term, dep_term], cause: :dependency, custom_explanation: custom_explanation) end end def all_versions_for(package) name = package.name results = (@base[name] + filter_prereleases(@all_specs[name], package)).uniq {|spec| [spec.version.hash, spec.platform] } + + if name == "bundler" && !bundler_pinned_to_current_version? + bundler_spec = Gem.loaded_specs["bundler"] + results << bundler_spec if bundler_spec + end + locked_requirement = base_requirements[name] results = filter_matching_specs(results, locked_requirement) if locked_requirement - versions = results.group_by(&:version).reduce([]) do |groups, (version, specs)| - platform_specs = package.platforms.flat_map {|platform| select_best_platform_match(specs, platform) } - next groups if platform_specs.empty? + results.group_by(&:version).reduce([]) do |groups, (version, specs)| + platform_specs = package.platforms.map {|platform| select_best_platform_match(specs, platform) } + + # If package is a top-level dependency, + # candidate is only valid if there are matching versions for all resolution platforms. + # + # If package is not a top-level deependency, + # then it's not necessary that it has matching versions for all platforms, since it may have been introduced only as + # a dependency for a platform specific variant, so it will only need to have a valid version for that platform. + # + if package.top_level? + next groups if platform_specs.any?(&:empty?) + else + next groups if platform_specs.all?(&:empty?) + end + + platform_specs.flatten! ruby_specs = select_best_platform_match(specs, Gem::Platform::RUBY) - groups << Resolver::Candidate.new(version, :specs => ruby_specs) if ruby_specs.any? + groups << Resolver::Candidate.new(version, specs: ruby_specs) if ruby_specs.any? next groups if platform_specs == ruby_specs || package.force_ruby_platform? - groups << Resolver::Candidate.new(version, :specs => platform_specs) + groups << Resolver::Candidate.new(version, specs: platform_specs) groups end - - sort_versions(package, versions) end def source_for(name) @source_requirements[name] || @source_requirements[:default] end + def default_bundler_source + @source_requirements[:default_bundler] + end + + def bundler_pinned_to_current_version? + !default_bundler_source.nil? + end + def name_for_explicit_dependency_source Bundler.default_gemfile.basename.to_s rescue StandardError @@ -273,15 +313,21 @@ module Bundler end specs_matching_requirement = filter_matching_specs(specs, package.dependency.requirement) - if specs_matching_requirement.any? + not_found_message = if specs_matching_requirement.any? specs = specs_matching_requirement matching_part = requirement_label platforms = package.platforms - platform_label = platforms.size == 1 ? "platform '#{platforms.first}" : "platforms '#{platforms.join("', '")}" - requirement_label = "#{requirement_label}' with #{platform_label}" + + if platforms.size == 1 + "Could not find gem '#{requirement_label}' with platform '#{platforms.first}'" + else + "Could not find gems matching '#{requirement_label}' valid for all resolution platforms (#{platforms.join(", ")})" + end + else + "Could not find gem '#{requirement_label}'" end - message = String.new("Could not find gem '#{requirement_label}' in #{source}#{cache_message}.\n") + message = String.new("#{not_found_message} in #{source}#{cache_message}.\n") if specs.any? message << "\n#{other_specs_matching_message(specs, matching_part)}" @@ -292,6 +338,21 @@ module Bundler private + def filtered_versions_for(package) + @gem_version_promoter.filter_versions(package, @all_versions[package]) + end + + def raise_all_versions_filtered_out!(package) + level = @gem_version_promoter.level + name = package.name + locked_version = package.locked_version + requirement = package.dependency + + raise GemNotFound, + "#{name} is locked to #{locked_version}, while Gemfile is requesting #{requirement}. " \ + "--strict --#{level} was specified, but there are no #{level} level upgrades from #{locked_version} satisfying #{requirement}, so version solving has failed" + end + def filter_matching_specs(specs, requirements) Array(requirements).flat_map do |requirement| specs.select {| spec| requirement_satisfied_by?(requirement, spec) } @@ -304,16 +365,19 @@ module Bundler specs.reject {|s| s.version.prerelease? } end + # Ignore versions that depend on themselves incorrectly + def filter_invalid_self_dependencies(specs, name) + specs.reject do |s| + s.dependencies.any? {|d| d.name == name && !d.requirement.satisfied_by?(s.version) } + end + end + def requirement_satisfied_by?(requirement, spec) requirement.satisfied_by?(spec.version) || spec.source.is_a?(Source::Gemspec) end - def sort_versions(package, versions) - if versions.size > 1 - @gem_version_promoter.sort_versions(package, versions).reverse - else - versions - end + def sort_versions_by_preferred(package, versions) + @gem_version_promoter.sort_versions(package, versions) end def repository_for(package) @@ -330,12 +394,19 @@ module Bundler next [dep_package, dep_constraint] if name == "bundler" - versions = versions_for(dep_package, dep_constraint.range) + dep_range = dep_constraint.range + versions = select_sorted_versions(dep_package, dep_range) if versions.empty? && dep_package.ignores_prereleases? + @all_versions.delete(dep_package) @sorted_versions.delete(dep_package) dep_package.consider_prereleases! - versions = versions_for(dep_package, dep_constraint.range) + versions = select_sorted_versions(dep_package, dep_range) + end + + if versions.empty? && select_all_versions(dep_package, dep_range).any? + raise_all_versions_filtered_out!(dep_package) end + next [dep_package, dep_constraint] unless versions.empty? next unless dep_package.current_platform? @@ -344,6 +415,14 @@ module Bundler end.compact.to_h end + def select_sorted_versions(package, range) + range.select_versions(@sorted_versions[package]) + end + + def select_all_versions(package, range) + range.select_versions(@all_versions[package]) + end + def other_specs_matching_message(specs, requirement) message = String.new("The source contains the following gems matching '#{requirement}':\n") message << specs.map {|s| " * #{s.full_name}" }.join("\n") @@ -359,19 +438,19 @@ module Bundler when "~>" name = "~> #{ver}" bump = Resolver::Candidate.new(version.bump.to_s + ".A") - PubGrub::VersionRange.new(:name => name, :min => ver, :max => bump, :include_min => true) + PubGrub::VersionRange.new(name: name, min: ver, max: bump, include_min: true) when ">" - PubGrub::VersionRange.new(:min => platform_ver) + PubGrub::VersionRange.new(min: platform_ver) when ">=" - PubGrub::VersionRange.new(:min => ver, :include_min => true) + PubGrub::VersionRange.new(min: ver, include_min: true) when "<" - PubGrub::VersionRange.new(:max => ver) + PubGrub::VersionRange.new(max: ver) when "<=" - PubGrub::VersionRange.new(:max => platform_ver, :include_max => true) + PubGrub::VersionRange.new(max: platform_ver, include_max: true) when "=" - PubGrub::VersionRange.new(:min => ver, :max => platform_ver, :include_min => true, :include_max => true) + PubGrub::VersionRange.new(min: ver, max: platform_ver, include_min: true, include_max: true) when "!=" - PubGrub::VersionRange.new(:min => ver, :max => platform_ver, :include_min => true, :include_max => true).invert + PubGrub::VersionRange.new(min: ver, max: platform_ver, include_min: true, include_max: true).invert else raise "bad version specifier: #{op}" end @@ -398,7 +477,7 @@ module Bundler end def bundler_not_found_message(conflict_dependencies) - candidate_specs = filter_matching_specs(source_for(:default_bundler).specs.search("bundler"), conflict_dependencies) + candidate_specs = filter_matching_specs(default_bundler_source.specs.search("bundler"), conflict_dependencies) if candidate_specs.any? target_version = candidate_specs.last.version diff --git a/lib/bundler/resolver/base.rb b/lib/bundler/resolver/base.rb index e5c3763c3f..ad19eeb3f4 100644 --- a/lib/bundler/resolver/base.rb +++ b/lib/bundler/resolver/base.rb @@ -24,7 +24,7 @@ module Bundler name = dep.name - @packages[name] = Package.new(name, dep_platforms, **options.merge(:dependency => dep)) + @packages[name] = Package.new(name, dep_platforms, **options.merge(dependency: dep)) dep end.compact diff --git a/lib/bundler/resolver/candidate.rb b/lib/bundler/resolver/candidate.rb index e695ef08ee..9e8b913335 100644 --- a/lib/bundler/resolver/candidate.rb +++ b/lib/bundler/resolver/candidate.rb @@ -15,7 +15,7 @@ module Bundler # considered separately. # # Some candidates may also keep some information explicitly about the - # package the refer to. These candidates are referred to as "canonical" and + # package they refer to. These candidates are referred to as "canonical" and # are used when materializing resolution results back into RubyGems # specifications that can be installed, written to lock files, and so on. # diff --git a/lib/bundler/resolver/incompatibility.rb b/lib/bundler/resolver/incompatibility.rb index c61151fbeb..4ac1b2e1ea 100644 --- a/lib/bundler/resolver/incompatibility.rb +++ b/lib/bundler/resolver/incompatibility.rb @@ -8,7 +8,7 @@ module Bundler def initialize(terms, cause:, custom_explanation: nil, extended_explanation: nil) @extended_explanation = extended_explanation - super(terms, :cause => cause, :custom_explanation => custom_explanation) + super(terms, cause: cause, custom_explanation: custom_explanation) end end end diff --git a/lib/bundler/resolver/package.rb b/lib/bundler/resolver/package.rb index 7499a75006..0461328683 100644 --- a/lib/bundler/resolver/package.rb +++ b/lib/bundler/resolver/package.rb @@ -21,6 +21,7 @@ module Bundler @locked_version = locked_specs[name].first&.version @unlock = unlock @dependency = dependency || Dependency.new(name, @locked_version) + @top_level = !dependency.nil? @prerelease = @dependency.prerelease? || @locked_version&.prerelease? || prerelease ? :consider_first : :ignore end @@ -32,6 +33,10 @@ module Bundler false end + def top_level? + @top_level + end + def meta? @name.end_with?("\0") end diff --git a/lib/bundler/resolver/spec_group.rb b/lib/bundler/resolver/spec_group.rb index b44c19a73f..5cee444e5e 100644 --- a/lib/bundler/resolver/spec_group.rb +++ b/lib/bundler/resolver/spec_group.rb @@ -25,9 +25,8 @@ module Bundler def to_specs(force_ruby_platform) @specs.map do |s| - lazy_spec = LazySpecification.new(name, version, s.platform, source) + lazy_spec = LazySpecification.from_spec(s) lazy_spec.force_ruby_platform = force_ruby_platform - lazy_spec.dependencies.replace s.dependencies lazy_spec end end @@ -64,8 +63,6 @@ module Bundler end def metadata_dependencies(spec) - return [] if spec.is_a?(LazySpecification) - [ metadata_dependency("Ruby", spec.required_ruby_version), metadata_dependency("RubyGems", spec.required_rubygems_version), diff --git a/lib/bundler/retry.rb b/lib/bundler/retry.rb index 2415ade200..b95c42c361 100644 --- a/lib/bundler/retry.rb +++ b/lib/bundler/retry.rb @@ -56,7 +56,7 @@ module Bundler def keep_trying? return true if current_run.zero? return false if last_attempt? - return true if @failed + true if @failed end def last_attempt? diff --git a/lib/bundler/ruby_dsl.rb b/lib/bundler/ruby_dsl.rb index 3b3a0583a5..fb4b79c4df 100644 --- a/lib/bundler/ruby_dsl.rb +++ b/lib/bundler/ruby_dsl.rb @@ -3,16 +3,51 @@ module Bundler module RubyDsl def ruby(*ruby_version) - options = ruby_version.last.is_a?(Hash) ? ruby_version.pop : {} + options = ruby_version.pop if ruby_version.last.is_a?(Hash) ruby_version.flatten! - raise GemfileError, "Please define :engine_version" if options[:engine] && options[:engine_version].nil? - raise GemfileError, "Please define :engine" if options[:engine_version] && options[:engine].nil? - if options[:engine] == "ruby" && options[:engine_version] && - ruby_version != Array(options[:engine_version]) - raise GemfileEvalError, "ruby_version must match the :engine_version for MRI" + if options + patchlevel = options[:patchlevel] + engine = options[:engine] + engine_version = options[:engine_version] + + raise GemfileError, "Please define :engine_version" if engine && engine_version.nil? + raise GemfileError, "Please define :engine" if engine_version && engine.nil? + + if options[:file] + raise GemfileError, "Do not pass version argument when using :file option" unless ruby_version.empty? + ruby_version << normalize_ruby_file(options[:file]) + end + + if engine == "ruby" && engine_version && ruby_version != Array(engine_version) + raise GemfileEvalError, "ruby_version must match the :engine_version for MRI" + end + end + + @ruby_version = RubyVersion.new(ruby_version, patchlevel, engine, engine_version) + end + + # Support the various file formats found in .ruby-version files. + # + # 3.2.2 + # ruby-3.2.2 + # + # Also supports .tool-versions files for asdf. Lines not starting with "ruby" are ignored. + # + # ruby 2.5.1 # comment is ignored + # ruby 2.5.1# close comment and extra spaces doesn't confuse + # + # Intentionally does not support `3.2.1@gemset` since rvm recommends using .ruby-gemset instead + # + # Loads the file relative to the dirname of the Gemfile itself. + def normalize_ruby_file(filename) + file_content = Bundler.read_file(gemfile.dirname.join(filename)) + # match "ruby-3.2.2" or "ruby 3.2.2" capturing version string up to the first space or comment + if /^ruby(-|\s+)([^\s#]+)/.match(file_content) + $2 + else + file_content.strip end - @ruby_version = RubyVersion.new(ruby_version, options[:patchlevel], options[:engine], options[:engine_version]) end end end diff --git a/lib/bundler/ruby_version.rb b/lib/bundler/ruby_version.rb index b5396abb6e..7e9e072b83 100644 --- a/lib/bundler/ruby_version.rb +++ b/lib/bundler/ruby_version.rb @@ -23,7 +23,7 @@ module Bundler # specified must match the version. @versions = Array(versions).map do |v| - op, v = Gem::Requirement.parse(v) + op, v = Gem::Requirement.parse(normalize_version(v)) op == "=" ? v.to_s : "#{op} #{v}" end @@ -49,7 +49,7 @@ module Bundler (\d+\.\d+\.\d+(?:\.\S+)?) # ruby version (?:p(-?\d+))? # optional patchlevel (?:\s\((\S+)\s(.+)\))? # optional engine info - /xo.freeze + /xo # Returns a RubyVersion from the given string. # @param [String] the version string to match. @@ -112,6 +112,13 @@ module Bundler private + # Ruby's official preview version format uses a `-`: Example: 3.3.0-preview2 + # However, RubyGems recognizes preview version format with a `.`: Example: 3.3.0.preview2 + # Returns version string after replacing `-` with `.` + def normalize_version(version) + version.tr("-", ".") + end + def matches?(requirements, version) # Handles RUBY_PATCHLEVEL of -1 for instances like ruby-head return requirements == version if requirements.to_s == "-1" || version.to_s == "-1" diff --git a/lib/bundler/rubygems_ext.rb b/lib/bundler/rubygems_ext.rb index b96edd5e2d..18180a81a1 100644 --- a/lib/bundler/rubygems_ext.rb +++ b/lib/bundler/rubygems_ext.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true -require "pathname" - -require "rubygems/specification" +require "rubygems" unless defined?(Gem) # We can't let `Gem::Source` be autoloaded in the `Gem::Specification#source` # redefinition below, so we need to load it upfront. The reason is that if @@ -15,10 +13,6 @@ require "rubygems/specification" # `Gem::Source` from the redefined `Gem::Specification#source`. require "rubygems/source" -require_relative "match_metadata" -require_relative "force_platform" -require_relative "match_platform" - # Cherry-pick fixes to `Gem.ruby_version` to be useful for modern Bundler # versions and ignore patchlevels # (https://github.com/rubygems/rubygems/pull/5472, @@ -29,7 +23,19 @@ unless Gem.ruby_version.to_s == RUBY_VERSION || RUBY_PATCHLEVEL == -1 end module Gem + # Can be removed once RubyGems 3.5.11 support is dropped + unless Gem.respond_to?(:freebsd_platform?) + def self.freebsd_platform? + RbConfig::CONFIG["host_os"].to_s.include?("bsd") + end + end + + require "rubygems/specification" + class Specification + require_relative "match_metadata" + require_relative "match_platform" + include ::Bundler::MatchMetadata include ::Bundler::MatchPlatform @@ -46,7 +52,7 @@ module Gem def full_gem_path if source.respond_to?(:root) - Pathname.new(loaded_from).dirname.expand_path(source.root).to_s.tap {|x| x.untaint if RUBY_VERSION < "2.7" } + File.expand_path(File.dirname(loaded_from), source.root) else rg_full_gem_path end @@ -76,25 +82,7 @@ module Gem end end - alias_method :rg_missing_extensions?, :missing_extensions? - def missing_extensions? - # When we use this methods with local gemspec, we don't handle - # build status of extension correctly. So We need to find extension - # files in require_paths. - # TODO: Gem::Specification couldn't access extension name from extconf.rb - # so we find them with heuristic way. We should improve it. - if source.respond_to?(:root) - return false if raw_require_paths.any? do |path| - ext_dir = File.join(full_gem_path, path) - File.exist?(File.join(ext_dir, "#{name}.#{RbConfig::CONFIG["DLEXT"]}")) || - !Dir.glob(File.join(ext_dir, name, "*.#{RbConfig::CONFIG["DLEXT"]}")).empty? - end - end - - rg_missing_extensions? - end - - remove_method :gem_dir if instance_methods(false).include?(:gem_dir) + remove_method :gem_dir def gem_dir full_gem_path end @@ -135,17 +123,6 @@ module Gem gemfile end - # Backfill missing YAML require when not defined. Fixed since 3.1.0.pre1. - module YamlBackfiller - def to_yaml(opts = {}) - Gem.load_yaml unless defined?(::YAML) - - super(opts) - end - end - - prepend YamlBackfiller - def nondevelopment_dependencies dependencies - development_dependencies end @@ -173,7 +150,23 @@ module Gem end end + module BetterPermissionError + def data + super + rescue Errno::EACCES + raise Bundler::PermissionError.new(loaded_from, :read) + end + end + + require "rubygems/stub_specification" + + class StubSpecification + prepend BetterPermissionError + end + class Dependency + require_relative "force_platform" + include ::Bundler::ForcePlatform attr_accessor :source, :groups @@ -206,37 +199,7 @@ module Gem end end - # comparison is done order independently since rubygems 3.2.0.rc.2 - unless Gem::Requirement.new("> 1", "< 2") == Gem::Requirement.new("< 2", "> 1") - class Requirement - module OrderIndependentComparison - def ==(other) - return unless Gem::Requirement === other - - if _requirements_sorted? && other._requirements_sorted? - super - else - _with_sorted_requirements == other._with_sorted_requirements - end - end - - protected - - def _requirements_sorted? - return @_requirements_sorted if defined?(@_requirements_sorted) - strings = as_list - @_requirements_sorted = strings == strings.sort - end - - def _with_sorted_requirements - @_with_sorted_requirements ||= _requirements_sorted? ? self : self.class.new(as_list.sort) - end - end - - prepend OrderIndependentComparison - end - end - + # Requirements using lambda operator differentiate trailing zeros since rubygems 3.2.6 if Gem::Requirement.new("~> 2.0").hash == Gem::Requirement.new("~> 2.0.0").hash class Requirement module CorrectHashForLambdaOperator @@ -338,7 +301,7 @@ module Gem end # On universal Rubies, resolve the "universal" arch to the real CPU arch, without changing the extension directory. - class Specification + class BasicSpecification if /^universal\.(?<arch>.*?)-/ =~ (CROSS_COMPILING || RUBY_PLATFORM) local_platform = Platform.local if local_platform.cpu == "universal" @@ -351,23 +314,35 @@ module Gem end def extensions_dir - Gem.default_ext_dir_for(base_dir) || - File.join(base_dir, "extensions", ORIGINAL_LOCAL_PLATFORM, - Gem.extension_api_version) + @extensions_dir ||= + Gem.default_ext_dir_for(base_dir) || File.join(base_dir, "extensions", ORIGINAL_LOCAL_PLATFORM, Gem.extension_api_version) end end end end - require "rubygems/util" + require "rubygems/name_tuple" - Util.singleton_class.module_eval do - if Util.singleton_methods.include?(:glob_files_in_dir) # since 3.0.0.beta.2 - remove_method :glob_files_in_dir + class NameTuple + # Versions of RubyGems before about 3.5.0 don't to_s the platform. + unless Gem::NameTuple.new("a", Gem::Version.new("1"), Gem::Platform.new("x86_64-linux")).platform.is_a?(String) + alias_method :initialize_with_platform, :initialize + + def initialize(name, version, platform=Gem::Platform::RUBY) + if Gem::Platform === platform + initialize_with_platform(name, version, platform.to_s) + else + initialize_with_platform(name, version, platform) + end + end end - def glob_files_in_dir(glob, base_path) - Dir.glob(glob, :base => base_path).map! {|f| File.expand_path(f, base_path) } + def lock_name + if platform == Gem::Platform::RUBY + "#{name} (#{version})" + else + "#{name} (#{version}-#{platform})" + end end end end diff --git a/lib/bundler/rubygems_gem_installer.rb b/lib/bundler/rubygems_gem_installer.rb index 38035a00ac..d563868cd2 100644 --- a/lib/bundler/rubygems_gem_installer.rb +++ b/lib/bundler/rubygems_gem_installer.rb @@ -20,7 +20,7 @@ module Bundler strict_rm_rf spec.extension_dir SharedHelpers.filesystem_access(gem_dir, :create) do - FileUtils.mkdir_p gem_dir, :mode => 0o755 + FileUtils.mkdir_p gem_dir, mode: 0o755 end extract_files @@ -45,6 +45,14 @@ module Bundler spec end + def pre_install_checks + super + rescue Gem::FilePermissionError + # Ignore permission checks in RubyGems. Instead, go on, and try to write + # for real. We properly handle permission errors when they happen. + nil + end + def generate_plugins return unless Gem::Installer.instance_methods(false).include?(:generate_plugins) @@ -60,10 +68,6 @@ module Bundler end end - def pre_install_checks - super && validate_bundler_checksum(options[:bundler_expected_checksum]) - end - def build_extensions extension_cache_path = options[:bundler_extension_cache_path] extension_dir = spec.extension_dir @@ -98,6 +102,10 @@ module Bundler end end + def gem_checksum + Checksum.from_gem_package(@package) + end + private def prepare_extension_build(extension_dir) @@ -108,64 +116,21 @@ module Bundler end def strict_rm_rf(dir) - Bundler.rm_rf dir - rescue StandardError => e - raise unless File.exist?(dir) + return unless File.exist?(dir) - raise DirectoryRemovalError.new(e, "Could not delete previous installation of `#{dir}`") - end - - def validate_bundler_checksum(checksum) - return true if Bundler.settings[:disable_checksum_validation] - return true unless checksum - return true unless source = @package.instance_variable_get(:@gem) - return true unless source.respond_to?(:with_read_io) - digest = source.with_read_io do |io| - digest = SharedHelpers.digest(:SHA256).new - digest << io.read(16_384) until io.eof? - io.rewind - send(checksum_type(checksum), digest) - end - unless digest == checksum - raise SecurityError, <<-MESSAGE - Bundler cannot continue installing #{spec.name} (#{spec.version}). - The checksum for the downloaded `#{spec.full_name}.gem` does not match \ - the checksum given by the server. This means the contents of the downloaded \ - gem is different from what was uploaded to the server, and could be a potential security issue. - - To resolve this issue: - 1. delete the downloaded gem located at: `#{spec.gem_dir}/#{spec.full_name}.gem` - 2. run `bundle install` - - If you wish to continue installing the downloaded gem, and are certain it does not pose a \ - security issue despite the mismatching checksum, do the following: - 1. run `bundle config set --local disable_checksum_validation true` to turn off checksum verification - 2. run `bundle install` - - (More info: The expected SHA256 checksum was #{checksum.inspect}, but the \ - checksum for the downloaded gem was #{digest.inspect}.) - MESSAGE - end - true - end + parent = File.dirname(dir) + parent_st = File.stat(parent) - def checksum_type(checksum) - case checksum.length - when 64 then :hexdigest! - when 44 then :base64digest! - else raise InstallError, "The given checksum for #{spec.full_name} (#{checksum.inspect}) is not a valid SHA256 hexdigest nor base64digest" + if parent_st.world_writable? && !parent_st.sticky? + raise InsecureInstallPathError.new(parent) end - end - def hexdigest!(digest) - digest.hexdigest! - end + begin + FileUtils.remove_entry_secure(dir) + rescue StandardError => e + raise unless File.exist?(dir) - def base64digest!(digest) - if digest.respond_to?(:base64digest!) - digest.base64digest! - else - [digest.digest!].pack("m0") + raise DirectoryRemovalError.new(e, "Could not delete previous installation of `#{dir}`") end end end diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb index d8b7886af7..494030eab2 100644 --- a/lib/bundler/rubygems_integration.rb +++ b/lib/bundler/rubygems_integration.rb @@ -4,17 +4,12 @@ require "rubygems" unless defined?(Gem) module Bundler class RubygemsIntegration - if defined?(Gem::Ext::Builder::CHDIR_MONITOR) - EXT_LOCK = Gem::Ext::Builder::CHDIR_MONITOR - else - require "monitor" + require "monitor" - EXT_LOCK = Monitor.new - end + EXT_LOCK = Monitor.new def initialize @replaced_methods = {} - backport_ext_builder_monitor end def version @@ -43,18 +38,6 @@ module Bundler Gem.loaded_specs[name] end - def add_to_load_path(paths) - return Gem.add_to_load_path(*paths) if Gem.respond_to?(:add_to_load_path) - - if insert_index = Gem.load_path_insert_index - # Gem directories must come after -I and ENV['RUBYLIB'] - $LOAD_PATH.insert(insert_index, *paths) - else - # We are probably testing in core, -I and RUBYLIB don't apply - $LOAD_PATH.unshift(*paths) - end - end - def mark_loaded(spec) if spec.respond_to?(:activated=) current = Gem.loaded_specs[spec.name] @@ -91,9 +74,9 @@ module Bundler def spec_matches_for_glob(spec, glob) return spec.matches_for_glob(glob) if spec.respond_to?(:matches_for_glob) - spec.load_paths.map do |lp| + spec.load_paths.flat_map do |lp| Dir["#{lp}/#{glob}#{suffix_pattern}"] - end.flatten(1) + end end def stub_set_spec(stub, spec) @@ -116,16 +99,6 @@ module Bundler Gem::Util.inflate(obj) end - def correct_for_windows_path(path) - if Gem::Util.respond_to?(:correct_for_windows_path) - Gem::Util.correct_for_windows_path(path) - elsif path[0].chr == "/" && path[1].chr =~ /[a-z]/i && path[2].chr == ":" - path[1..-1] - else - path - end - end - def gem_dir Gem.dir end @@ -161,7 +134,7 @@ module Bundler def spec_cache_dirs @spec_cache_dirs ||= begin dirs = gem_path.map {|dir| File.join(dir, "specifications") } - dirs << Gem.spec_cache_dir if Gem.respond_to?(:spec_cache_dir) # Not in RubyGems 2.0.3 or earlier + dirs << Gem.spec_cache_dir dirs.uniq.select {|dir| File.directory? dir } end end @@ -184,15 +157,15 @@ module Bundler end def load_plugins - Gem.load_plugins if Gem.respond_to?(:load_plugins) + Gem.load_plugins end - def load_plugin_files(files) - Gem.load_plugin_files(files) if Gem.respond_to?(:load_plugin_files) + def load_plugin_files(plugin_files) + Gem.load_plugin_files(plugin_files) end def load_env_plugins - Gem.load_env_plugins if Gem.respond_to?(:load_env_plugins) + Gem.load_env_plugins end def ui=(obj) @@ -230,8 +203,7 @@ module Bundler if Gem.respond_to?(:discover_gems_on_require=) Gem.discover_gems_on_require = false else - kernel = (class << ::Kernel; self; end) - [kernel, ::Kernel].each do |k| + [::Kernel.singleton_class, ::Kernel].each do |k| if k.private_method_defined?(:gem_original_require) redefine_method(k, :require, k.instance_method(:gem_original_require)) end @@ -240,12 +212,9 @@ module Bundler end def replace_gem(specs, specs_by_name) - reverse_rubygems_kernel_mixin - executables = nil - kernel = (class << ::Kernel; self; end) - [kernel, ::Kernel].each do |kernel_class| + [::Kernel.singleton_class, ::Kernel].each do |kernel_class| redefine_method(kernel_class, :gem) do |dep, *reqs| if executables&.include?(File.basename(caller.first.split(":").first)) break @@ -358,6 +327,14 @@ module Bundler def replace_entrypoints(specs) specs_by_name = add_default_gems_to(specs) + reverse_rubygems_kernel_mixin + begin + # bundled_gems only provide with Ruby 3.3 or later + require "bundled_gems" + rescue LoadError + else + Gem::BUNDLED_GEMS.replace_require(specs) if Gem::BUNDLED_GEMS.respond_to?(:replace_require) + end replace_gem(specs, specs_by_name) stub_rubygems(specs) replace_bin_path(specs_by_name) @@ -447,30 +424,28 @@ module Bundler Gem::Specification.all = specs end - def fetch_specs(remote, name) + def fetch_specs(remote, name, fetcher) require "rubygems/remote_fetcher" path = remote.uri.to_s + "#{name}.#{Gem.marshal_version}.gz" - fetcher = gem_remote_fetcher - fetcher.headers = { "X-Gemfile-Source" => remote.original_uri.to_s } if remote.original_uri string = fetcher.fetch_path(path) - Bundler.safe_load_marshal(string) + specs = Bundler.safe_load_marshal(string) + raise MarshalError, "Specs #{name} from #{remote} is expected to be an Array but was unexpected class #{specs.class}" unless specs.is_a?(Array) + specs rescue Gem::RemoteFetcher::FetchError # it's okay for prerelease to fail raise unless name == "prerelease_specs" end - def fetch_all_remote_specs(remote) - specs = fetch_specs(remote, "specs") - pres = fetch_specs(remote, "prerelease_specs") || [] + def fetch_all_remote_specs(remote, gem_remote_fetcher) + specs = fetch_specs(remote, "specs", gem_remote_fetcher) + pres = fetch_specs(remote, "prerelease_specs", gem_remote_fetcher) || [] specs.concat(pres) end - def download_gem(spec, uri, cache_dir) + def download_gem(spec, uri, cache_dir, fetcher) require "rubygems/remote_fetcher" uri = Bundler.settings.mirror_for(uri) - fetcher = gem_remote_fetcher - fetcher.headers = { "X-Gemfile-Source" => spec.remote.original_uri.to_s } if spec.remote.original_uri Bundler::Retry.new("download gem from #{uri}").attempts do gem_file_name = spec.file_name local_gem_path = File.join cache_dir, gem_file_name @@ -478,7 +453,6 @@ module Bundler begin remote_gem_path = uri + "gems/#{gem_file_name}" - remote_gem_path = remote_gem_path.to_s if provides?("< 3.2.0.rc.1") SharedHelpers.filesystem_access(local_gem_path) do fetcher.cache_update_path remote_gem_path, local_gem_path @@ -497,12 +471,6 @@ module Bundler raise Bundler::HTTPError, "Could not download gem from #{uri} due to underlying error <#{e.message}>" end - def gem_remote_fetcher - require "rubygems/remote_fetcher" - proxy = Gem.configuration[:http_proxy] - Gem::RemoteFetcher.new(proxy) - end - def build(spec, skip_validation = false) require "rubygems/package" Gem::Package.build(spec, skip_validation) @@ -518,25 +486,6 @@ module Bundler end end - def backport_ext_builder_monitor - # So we can avoid requiring "rubygems/ext" in its entirety - Gem.module_eval <<-RUBY, __FILE__, __LINE__ + 1 - module Ext - end - RUBY - - require "rubygems/ext/builder" - - Gem::Ext::Builder.class_eval do - unless const_defined?(:CHDIR_MONITOR) - const_set(:CHDIR_MONITOR, EXT_LOCK) - end - - remove_const(:CHDIR_MUTEX) if const_defined?(:CHDIR_MUTEX) - const_set(:CHDIR_MUTEX, const_get(:CHDIR_MONITOR)) - end - end - def find_bundler(version) find_name("bundler").find {|s| s.version.to_s == version } end @@ -545,14 +494,8 @@ module Bundler Gem::Specification.stubs_for(name).map(&:to_spec) end - if Gem::Specification.respond_to?(:default_stubs) - def default_stubs - Gem::Specification.default_stubs("*.gemspec") - end - else - def default_stubs - Gem::Specification.send(:default_stubs, "*.gemspec") - end + def default_stubs + Gem::Specification.default_stubs("*.gemspec") end end diff --git a/lib/bundler/runtime.rb b/lib/bundler/runtime.rb index bd38353d3c..54aa30ce0b 100644 --- a/lib/bundler/runtime.rb +++ b/lib/bundler/runtime.rb @@ -28,11 +28,11 @@ module Bundler spec.load_paths.reject {|path| $LOAD_PATH.include?(path) } end.reverse.flatten - Bundler.rubygems.add_to_load_path(load_paths) + Gem.add_to_load_path(*load_paths) setup_manpath - lock(:preserve_unknown_sections => true) + lock(preserve_unknown_sections: true) self end @@ -41,12 +41,17 @@ module Bundler groups.map!(&:to_sym) groups = [:default] if groups.empty? - @definition.dependencies.each do |dep| - # Skip the dependency if it is not in any of the requested groups, or - # not for the current platform, or doesn't match the gem constraints. - next unless (dep.groups & groups).any? && dep.should_include? + dependencies = @definition.dependencies.select do |dep| + # Select the dependency if it is in any of the requested groups, and + # for the current platform, and matches the gem constraints. + (dep.groups & groups).any? && dep.should_include? + end + + Plugin.hook(Plugin::Events::GEM_BEFORE_REQUIRE_ALL, dependencies) + dependencies.each do |dep| required_file = nil + Plugin.hook(Plugin::Events::GEM_BEFORE_REQUIRE, dep) begin # Loop through all the specified autorequires for the @@ -76,7 +81,13 @@ module Bundler end end end + + Plugin.hook(Plugin::Events::GEM_AFTER_REQUIRE, dep) end + + Plugin.hook(Plugin::Events::GEM_AFTER_REQUIRE_ALL, dependencies) + + dependencies end def self.definition_method(meth) @@ -94,8 +105,8 @@ module Bundler definition_method :requires def lock(opts = {}) - return if @definition.nothing_changed? && !@definition.unlocking? - @definition.lock(Bundler.default_lockfile, opts[:preserve_unknown_sections]) + return if @definition.no_resolve_needed? + @definition.lock(opts[:preserve_unknown_sections]) end alias_method :gems, :specs diff --git a/lib/bundler/self_manager.rb b/lib/bundler/self_manager.rb index 827f3f9222..5accda4bcb 100644 --- a/lib/bundler/self_manager.rb +++ b/lib/bundler/self_manager.rb @@ -9,17 +9,23 @@ module Bundler def restart_with_locked_bundler_if_needed return unless needs_switching? && installed? - restart_with(lockfile_version) + restart_with(restart_version) end def install_locked_bundler_and_restart_with_it_if_needed return unless needs_switching? - Bundler.ui.info \ - "Bundler #{current_version} is running, but your lockfile was generated with #{lockfile_version}. " \ - "Installing Bundler #{lockfile_version} and restarting using that version." + if restart_version == lockfile_version + Bundler.ui.info \ + "Bundler #{current_version} is running, but your lockfile was generated with #{lockfile_version}. " \ + "Installing Bundler #{lockfile_version} and restarting using that version." + else + Bundler.ui.info \ + "Bundler #{current_version} is running, but your configuration was #{restart_version}. " \ + "Installing Bundler #{restart_version} and restarting using that version." + end - install_and_restart_with(lockfile_version) + install_and_restart_with(restart_version) end def update_bundler_and_restart_with_it_if_needed(target) @@ -79,7 +85,8 @@ module Bundler autoswitching_applies? && released?(lockfile_version) && !running?(lockfile_version) && - !updating? + !updating? && + Bundler.settings[:version] != "system" end def autoswitching_applies? @@ -114,7 +121,7 @@ module Bundler source = Bundler::Source::Rubygems.new("remotes" => "https://rubygems.org") source.remote! source.add_dependency_names("bundler") - source.specs + source.specs.select(&:matches_current_metadata?) end end @@ -151,7 +158,7 @@ module Bundler def installed? Bundler.configure - Bundler.rubygems.find_bundler(lockfile_version.to_s) + Bundler.rubygems.find_bundler(restart_version.to_s) end def current_version @@ -163,6 +170,17 @@ module Bundler parsed_version = Bundler::LockfileParser.bundled_with @lockfile_version = parsed_version ? Gem::Version.new(parsed_version) : nil + rescue ArgumentError + @lockfile_version = nil + end + + def restart_version + return @restart_version if defined?(@restart_version) + # BUNDLE_VERSION=x.y.z + @restart_version = Gem::Version.new(Bundler.settings[:version]) + rescue ArgumentError + # BUNDLE_VERSION=lockfile + @restart_version = lockfile_version end end end diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb index 88dd829d66..abbaec9783 100644 --- a/lib/bundler/settings.rb +++ b/lib/bundler/settings.rb @@ -7,7 +7,6 @@ module Bundler autoload :Validator, File.expand_path("settings/validator", __dir__) BOOL_KEYS = %w[ - allow_deployment_source_credential_changes allow_offline_install auto_clean_without_path auto_install @@ -43,10 +42,23 @@ module Bundler setup_makes_kernel_gem_public silence_deprecations silence_root_warning - suppress_install_using_messages update_requires_all_flag ].freeze + REMEMBERED_KEYS = %w[ + bin + cache_all + clean + deployment + frozen + no_prune + path + shebang + path.system + without + with + ].freeze + NUMBER_KEYS = %w[ jobs redirect @@ -75,6 +87,7 @@ module Bundler shebang system_bindir trust-policy + version ].freeze DEFAULT_CONFIG = { @@ -84,25 +97,38 @@ module Bundler "BUNDLE_REDIRECT" => 5, "BUNDLE_RETRY" => 3, "BUNDLE_TIMEOUT" => 10, + "BUNDLE_VERSION" => "lockfile", }.freeze def initialize(root = nil) @root = root @local_config = load_config(local_config_file) - @env_config = ENV.to_h.select {|key, _value| key =~ /\ABUNDLE_.+/ } + + @env_config = ENV.to_h + @env_config.select! {|key, _value| key.start_with?("BUNDLE_") } + @env_config.delete("BUNDLE_") + @global_config = load_config(global_config_file) @temporary = {} + + @key_cache = {} end def [](name) key = key_for(name) - value = configs.values.map {|config| config[key] }.compact.first + + value = nil + configs.each do |_, config| + value = config[key] + next if value.nil? + break + end converted_value(value, name) end def set_command_option(key, value) - if Bundler.feature_flag.forget_cli_options? + if !is_remembered(key) || Bundler.feature_flag.forget_cli_options? temporary(key => value) value else @@ -139,17 +165,22 @@ module Bundler end def all - keys = @temporary.keys | @global_config.keys | @local_config.keys | @env_config.keys + keys = @temporary.keys.union(@global_config.keys, @local_config.keys, @env_config.keys) - keys.map do |key| - key.sub(/^BUNDLE_/, "").gsub(/___/, "-").gsub(/__/, ".").downcase - end.sort + keys.map! do |key| + key = key.delete_prefix("BUNDLE_") + key.gsub!("___", "-") + key.gsub!("__", ".") + key.downcase! + key + end.sort! + keys end def local_overrides repos = {} all.each do |k| - repos[$'] = self[k] if k =~ /^local\./ + repos[k.delete_prefix("local.")] = self[k] if k.start_with?("local.") end repos end @@ -157,7 +188,7 @@ module Bundler def mirror_for(uri) if uri.is_a?(String) require_relative "vendored_uri" - uri = Bundler::URI(uri) + uri = Gem::URI(uri) end gem_mirrors.for(uri.to_s).uri @@ -219,7 +250,6 @@ module Bundler def path configs.each do |_level, settings| path = value_for("path", settings) - path = "vendor/bundle" if value_for("deployment", settings) && path.nil? path_system = value_for("path.system", settings) disabled_shared_gems = value_for("disable_shared_gems", settings) next if path.nil? && path_system.nil? && disabled_shared_gems.nil? @@ -227,7 +257,9 @@ module Bundler return Path.new(path, system_path) end - Path.new(nil, false) + path = "vendor/bundle" if self[:deployment] + + Path.new(path, false) end Path = Struct.new(:explicit_path, :system_path) do @@ -295,18 +327,18 @@ module Bundler end def key_for(key) - self.class.key_for(key) + @key_cache[key] ||= self.class.key_for(key) end private def configs - { - :temporary => @temporary, - :local => @local_config, - :env => @env_config, - :global => @global_config, - :default => DEFAULT_CONFIG, + @configs ||= { + temporary: @temporary, + local: @local_config, + env: @env_config, + global: @global_config, + default: DEFAULT_CONFIG, } end @@ -327,16 +359,20 @@ module Bundler end def is_bool(name) - BOOL_KEYS.include?(name.to_s) || BOOL_KEYS.include?(parent_setting_for(name.to_s)) + name = self.class.key_to_s(name) + BOOL_KEYS.include?(name) || BOOL_KEYS.include?(parent_setting_for(name)) end def is_string(name) - STRING_KEYS.include?(name.to_s) || name.to_s.start_with?("local.") || name.to_s.start_with?("mirror.") || name.to_s.start_with?("build.") + name = self.class.key_to_s(name) + STRING_KEYS.include?(name) || name.start_with?("local.") || name.start_with?("mirror.") || name.start_with?("build.") end def to_bool(value) case value - when nil, /\A(false|f|no|n|0|)\z/i, false + when String + value.match?(/\A(false|f|no|n|0|)\z/i) ? false : true + when nil, false false else true @@ -344,11 +380,15 @@ module Bundler end def is_num(key) - NUMBER_KEYS.include?(key.to_s) + NUMBER_KEYS.include?(self.class.key_to_s(key)) end def is_array(key) - ARRAY_KEYS.include?(key.to_s) + ARRAY_KEYS.include?(self.class.key_to_s(key)) + end + + def is_remembered(key) + REMEMBERED_KEYS.include?(self.class.key_to_s(key)) end def is_credential(key) @@ -371,7 +411,7 @@ module Bundler end def set_key(raw_key, value, hash, file) - raw_key = raw_key.to_s + raw_key = self.class.key_to_s(raw_key) value = array_to_s(value) if is_array(raw_key) key = key_for(raw_key) @@ -391,6 +431,8 @@ module Bundler end def converted_value(value, key) + key = self.class.key_to_s(key) + if is_array(key) to_array(value) elsif value.nil? @@ -449,17 +491,19 @@ module Bundler valid_file = file.exist? && !file.size.zero? return {} unless valid_file serializer_class.load(file.read).inject({}) do |config, (k, v)| - new_k = k + unless k.start_with?("#") + if k.include?("-") + Bundler.ui.warn "Your #{file} config includes `#{k}`, which contains the dash character (`-`).\n" \ + "This is deprecated, because configuration through `ENV` should be possible, but `ENV` keys cannot include dashes.\n" \ + "Please edit #{file} and replace any dashes in configuration keys with a triple underscore (`___`)." - if k.include?("-") - Bundler.ui.warn "Your #{file} config includes `#{k}`, which contains the dash character (`-`).\n" \ - "This is deprecated, because configuration through `ENV` should be possible, but `ENV` keys cannot include dashes.\n" \ - "Please edit #{file} and replace any dashes in configuration keys with a triple underscore (`___`)." + # string hash keys are frozen + k = k.gsub("-", "___") + end - new_k = k.gsub("-", "___") + config[k] = v end - config[new_k] = v config end end @@ -485,12 +529,15 @@ module Bundler (https?.*?) # URI (\.#{Regexp.union(PER_URI_OPTIONS)})? # optional suffix key \z - /ix.freeze + /ix def self.key_for(key) key = normalize_uri(key).to_s if key.is_a?(String) && key.start_with?("http", "mirror.http") - key = key.to_s.gsub(".", "__").gsub("-", "___").upcase - "BUNDLE_#{key}" + key = key_to_s(key).gsub(".", "__") + key.gsub!("-", "___") + key.upcase! + + key.prepend("BUNDLE_") end # TODO: duplicates Rubygems#normalize_uri @@ -504,11 +551,40 @@ module Bundler end uri = URINormalizer.normalize_suffix(uri) require_relative "vendored_uri" - uri = Bundler::URI(uri) + uri = Gem::URI(uri) unless uri.absolute? raise ArgumentError, format("Gem sources must be absolute. You provided '%s'.", uri) end "#{prefix}#{uri}#{suffix}" end + + # This is a hot method, so avoid respond_to? checks on every invocation + if :read.respond_to?(:name) + def self.key_to_s(key) + case key + when String + key + when Symbol + key.name + when Gem::URI::HTTP + key.to_s + else + raise ArgumentError, "Invalid key: #{key.inspect}" + end + end + else + def self.key_to_s(key) + case key + when String + key + when Symbol + key.to_s + when Gem::URI::HTTP + key.to_s + else + raise ArgumentError, "Invalid key: #{key.inspect}" + end + end + end end end diff --git a/lib/bundler/setup.rb b/lib/bundler/setup.rb index 801fd5312a..6010d66742 100644 --- a/lib/bundler/setup.rb +++ b/lib/bundler/setup.rb @@ -5,6 +5,9 @@ require_relative "shared_helpers" if Bundler::SharedHelpers.in_bundle? require_relative "../bundler" + # try to auto_install first before we get to the `Bundler.ui.silence`, so user knows what is happening + Bundler.auto_install + if STDOUT.tty? || ENV["BUNDLER_FORCE_TTY"] begin Bundler.ui.silence { Bundler.setup } @@ -12,7 +15,10 @@ if Bundler::SharedHelpers.in_bundle? Bundler.ui.error e.message Bundler.ui.warn e.backtrace.join("\n") if ENV["DEBUG"] if e.is_a?(Bundler::GemNotFound) - suggested_cmd = "bundle install" + default_bundle = Gem.bin_path("bundler", "bundle") + current_bundle = Bundler::SharedHelpers.bundle_bin_path + suggested_bundle = default_bundle == current_bundle ? "bundle" : current_bundle + suggested_cmd = "#{suggested_bundle} install" original_gemfile = Bundler.original_env["BUNDLE_GEMFILE"] suggested_cmd += " --gemfile #{original_gemfile}" if original_gemfile Bundler.ui.warn "Run `#{suggested_cmd}` to install missing gems." diff --git a/lib/bundler/shared_helpers.rb b/lib/bundler/shared_helpers.rb index d1d4e1d07a..28f0cdff19 100644 --- a/lib/bundler/shared_helpers.rb +++ b/lib/bundler/shared_helpers.rb @@ -1,25 +1,27 @@ # frozen_string_literal: true -require "pathname" -require "rbconfig" - require_relative "version" -require_relative "constants" require_relative "rubygems_integration" require_relative "current_ruby" module Bundler + autoload :WINDOWS, File.expand_path("constants", __dir__) + autoload :FREEBSD, File.expand_path("constants", __dir__) + autoload :NULL, File.expand_path("constants", __dir__) + module SharedHelpers + autoload :Pathname, "pathname" + def root gemfile = find_gemfile raise GemfileNotFound, "Could not locate Gemfile" unless gemfile - Pathname.new(gemfile).tap {|x| x.untaint if RUBY_VERSION < "2.7" }.expand_path.parent + Pathname.new(gemfile).expand_path.parent end def default_gemfile gemfile = find_gemfile raise GemfileNotFound, "Could not locate Gemfile" unless gemfile - Pathname.new(gemfile).tap {|x| x.untaint if RUBY_VERSION < "2.7" }.expand_path + Pathname.new(gemfile).expand_path end def default_lockfile @@ -28,7 +30,7 @@ module Bundler case gemfile.basename.to_s when "gems.rb" then Pathname.new(gemfile.sub(/.rb$/, ".locked")) else Pathname.new("#{gemfile}.lock") - end.tap {|x| x.untaint if RUBY_VERSION < "2.7" } + end end def default_bundle_dir @@ -100,7 +102,7 @@ module Bundler # # @see {Bundler::PermissionError} def filesystem_access(path, action = :write, &block) - yield(path.dup.tap {|x| x.untaint if RUBY_VERSION < "2.7" }) + yield(path.dup) rescue Errno::EACCES raise PermissionError.new(path, action) rescue Errno::EAGAIN @@ -117,16 +119,18 @@ module Bundler raise GenericSystemCallError.new(e, "There was an error accessing `#{path}`.") end - def major_deprecation(major_version, message, print_caller_location: false) + def major_deprecation(major_version, message, removed_message: nil, print_caller_location: false) if print_caller_location caller_location = caller_locations(2, 2).first - message = "#{message} (called at #{caller_location.path}:#{caller_location.lineno})" + suffix = " (called at #{caller_location.path}:#{caller_location.lineno})" + message += suffix + removed_message += suffix if removed_message end bundler_major_version = Bundler.bundler_major_version if bundler_major_version > major_version require_relative "errors" - raise DeprecatedError, "[REMOVED] #{message}" + raise DeprecatedError, "[REMOVED] #{removed_message || message}" end return unless bundler_major_version >= major_version && prints_major_deprecations? @@ -193,10 +197,40 @@ module Bundler Digest(name) end + def checksum_for_file(path, digest) + return unless path.file? + # This must use File.read instead of Digest.file().hexdigest + # because we need to preserve \n line endings on windows when calculating + # the checksum + SharedHelpers.filesystem_access(path, :read) do + File.open(path, "rb") do |f| + digest = SharedHelpers.digest(digest).new + buf = String.new(capacity: 16_384, encoding: Encoding::BINARY) + digest << buf while f.read(16_384, buf) + digest.hexdigest + end + end + end + def write_to_gemfile(gemfile_path, contents) filesystem_access(gemfile_path) {|g| File.open(g, "w") {|file| file.puts contents } } end + def relative_gemfile_path + relative_path_to(Bundler.default_gemfile) + end + + def relative_lockfile_path + relative_path_to(Bundler.default_lockfile) + end + + def relative_path_to(destination, from: pwd) + Pathname.new(destination).relative_path_from(from).to_s + rescue ArgumentError + # on Windows, if source and destination are on different drivers, there's no relative path from one to the other + destination + end + private def validate_bundle_path @@ -235,7 +269,7 @@ module Bundler def search_up(*names) previous = nil - current = File.expand_path(SharedHelpers.pwd).tap {|x| x.untaint if RUBY_VERSION < "2.7" } + current = File.expand_path(SharedHelpers.pwd) until !File.directory?(current) || current == previous if ENV["BUNDLER_SPEC_RUN"] @@ -272,6 +306,13 @@ module Bundler public :set_env def set_bundle_variables + Bundler::SharedHelpers.set_env "BUNDLE_BIN_PATH", bundle_bin_path + Bundler::SharedHelpers.set_env "BUNDLE_GEMFILE", find_gemfile.to_s + Bundler::SharedHelpers.set_env "BUNDLER_VERSION", Bundler::VERSION + Bundler::SharedHelpers.set_env "BUNDLER_SETUP", File.expand_path("setup", __dir__) + end + + def bundle_bin_path # bundler exe & lib folders have same root folder, typical gem installation exe_file = File.expand_path("../../exe/bundle", __dir__) @@ -281,11 +322,9 @@ module Bundler # bundler is a default gem, exe path is separate exe_file = Bundler.rubygems.bin_path("bundler", "bundle", VERSION) unless File.exist?(exe_file) - Bundler::SharedHelpers.set_env "BUNDLE_BIN_PATH", exe_file - Bundler::SharedHelpers.set_env "BUNDLE_GEMFILE", find_gemfile.to_s - Bundler::SharedHelpers.set_env "BUNDLER_VERSION", Bundler::VERSION - Bundler::SharedHelpers.set_env "BUNDLER_SETUP", File.expand_path("setup", __dir__) unless RUBY_VERSION < "2.7" + exe_file end + public :bundle_bin_path def set_path validate_bundle_path @@ -297,7 +336,7 @@ module Bundler def set_rubyopt rubyopt = [ENV["RUBYOPT"]].compact setup_require = "-r#{File.expand_path("setup", __dir__)}" - return if !rubyopt.empty? && rubyopt.first =~ /#{setup_require}/ + return if !rubyopt.empty? && rubyopt.first.include?(setup_require) rubyopt.unshift setup_require Bundler::SharedHelpers.set_env "RUBYOPT", rubyopt.join(" ") end diff --git a/lib/bundler/source.rb b/lib/bundler/source.rb index 69804a2e63..115dbd1378 100644 --- a/lib/bundler/source.rb +++ b/lib/bundler/source.rb @@ -11,6 +11,8 @@ module Bundler attr_accessor :dependency_names + attr_reader :checksum_store + def unmet_deps specs.unmet_dependency_names end @@ -100,7 +102,7 @@ module Bundler end def print_using_message(message) - if !message.include?("(was ") && Bundler.feature_flag.suppress_install_using_messages? + if !message.include?("(was ") Bundler.ui.debug message else Bundler.ui.info message diff --git a/lib/bundler/source/git.rb b/lib/bundler/source/git.rb index 42897813b4..198e335bb6 100644 --- a/lib/bundler/source/git.rb +++ b/lib/bundler/source/git.rb @@ -11,6 +11,7 @@ module Bundler def initialize(options) @options = options + @checksum_store = Checksum::Store.new @glob = options["glob"] || DEFAULT_GLOB @allow_cached = false @@ -19,7 +20,7 @@ module Bundler # Stringify options that could be set as symbols %w[ref branch tag revision].each {|k| options[k] = options[k].to_s if options[k] } - @uri = URINormalizer.normalize_suffix(options["uri"] || "", :trailing_slash => false) + @uri = URINormalizer.normalize_suffix(options["uri"] || "", trailing_slash: false) @safe_uri = URICredentialsFilter.credential_filtered_uri(@uri) @branch = options["branch"] @ref = options["ref"] || options["branch"] || options["tag"] @@ -46,6 +47,14 @@ module Bundler out << " specs:\n" end + def to_gemfile + specifiers = %w[ref branch tag submodules glob].map do |opt| + "#{opt}: #{options[opt]}" if options[opt] + end + + uri_with_specifiers(specifiers) + end + def hash [self.class, uri, ref, branch, name, version, glob, submodules].hash end @@ -59,28 +68,32 @@ module Bundler alias_method :==, :eql? + def include?(other) + other.is_a?(Git) && uri == other.uri && + name == other.name && + glob == other.glob && + submodules == other.submodules + end + def to_s begin - at = if local? - path - elsif user_ref = options["ref"] - if /\A[a-z0-9]{4,}\z/i.match?(ref) - shortref_for_display(user_ref) - else - user_ref - end - elsif ref - ref - else - current_branch - end + at = humanized_ref || current_branch rev = "at #{at}@#{shortref_for_display(revision)}" rescue GitError "" end - specifiers = [rev, glob_for_display].compact + uri_with_specifiers([rev, glob_for_display]) + end + + def identifier + uri_with_specifiers([humanized_ref, cached_revision, glob_for_display]) + end + + def uri_with_specifiers(specifiers) + specifiers.compact! + suffix = if specifiers.any? " (#{specifiers.join(", ")})" @@ -185,7 +198,7 @@ module Bundler @copied = true end - generate_bin_options = { :disable_extensions => !Bundler.rubygems.spec_missing_extensions?(spec), :build_args => options[:build_args] } + generate_bin_options = { disable_extensions: !Bundler.rubygems.spec_missing_extensions?(spec), build_args: options[:build_args] } generate_bin(spec, generate_bin_options) requires_checkout? ? spec.post_install_message : nil @@ -243,6 +256,20 @@ module Bundler private + def humanized_ref + if local? + path + elsif user_ref = options["ref"] + if /\A[a-z0-9]{4,}\z/i.match?(ref) + shortref_for_display(user_ref) + else + user_ref + end + elsif ref + ref + end + end + def serialize_gemspecs_in(destination) destination = destination.expand_path(Bundler.root) if destination.relative? Dir["#{destination}/#{@glob}"].each do |spec_path| @@ -299,7 +326,7 @@ module Bundler if %r{^\w+://(\w+@)?}.match?(uri) # Downcase the domain component of the URI # and strip off a trailing slash, if one is present - input = Bundler::URI.parse(uri).normalize.to_s.sub(%r{/$}, "") + input = Gem::URI.parse(uri).normalize.to_s.sub(%r{/$}, "") else # If there is no URI scheme, assume it is an ssh/git URI input = uri @@ -333,7 +360,7 @@ module Bundler def load_gemspec(file) stub = Gem::StubSpecification.gemspec_stub(file, install_path.parent, install_path.parent) - stub.full_gem_path = Pathname.new(file).dirname.expand_path(root).to_s.tap {|x| x.untaint if RUBY_VERSION < "2.7" } + stub.full_gem_path = Pathname.new(file).dirname.expand_path(root).to_s StubSpecification.from_stub(stub) end diff --git a/lib/bundler/source/git/git_proxy.rb b/lib/bundler/source/git/git_proxy.rb index 926c9b8ead..2fc9c6535f 100644 --- a/lib/bundler/source/git/git_proxy.rb +++ b/lib/bundler/source/git/git_proxy.rb @@ -43,6 +43,13 @@ module Bundler end end + class AmbiguousGitReference < GitError + def initialize(options) + msg = "Specification of branch or ref with tag is ambiguous. You specified #{options.inspect}" + super msg + end + end + # The GitProxy is responsible to interact with git repositories. # All actions required by the Git source is encapsulated in this # object. @@ -53,10 +60,15 @@ module Bundler def initialize(path, uri, options = {}, revision = nil, git = nil) @path = path @uri = uri - @branch = options["branch"] @tag = options["tag"] + @branch = options["branch"] @ref = options["ref"] - @explicit_ref = branch || tag || ref + if @tag + raise AmbiguousGitReference.new(options) if @branch || @ref + @explicit_ref = @tag + else + @explicit_ref = @ref || @branch + end @revision = revision @git = git @commit_ref = nil @@ -67,15 +79,15 @@ module Bundler end def current_branch - @current_branch ||= allowed_with_path do - git("rev-parse", "--abbrev-ref", "HEAD", :dir => path).strip + @current_branch ||= with_path do + git_local("rev-parse", "--abbrev-ref", "HEAD", dir: path).strip end end def contains?(commit) allowed_with_path do - result, status = git_null("branch", "--contains", commit, :dir => path) - status.success? && result =~ /^\* (.*)$/ + result, status = git_null("branch", "--contains", commit, dir: path) + status.success? && result.match?(/^\* (.*)$/) end end @@ -84,7 +96,7 @@ module Bundler end def full_version - @full_version ||= git("--version").sub(/git version\s*/, "").strip + @full_version ||= git_local("--version").sub(/git version\s*/, "").strip end def checkout @@ -118,15 +130,20 @@ module Bundler end end - git "fetch", "--force", "--quiet", *extra_fetch_args, :dir => destination if @commit_ref + ref = @commit_ref || (locked_to_full_sha? && @revision) + if ref + git "config", "uploadpack.allowAnySHA1InWant", "true", dir: path.to_s if @commit_ref.nil? && needs_allow_any_sha1_in_want? - git "reset", "--hard", @revision, :dir => destination + git "fetch", "--force", "--quiet", *extra_fetch_args(ref), dir: destination + end + + git "reset", "--hard", @revision, dir: destination if submodules - git_retry "submodule", "update", "--init", "--recursive", :dir => destination + git_retry "submodule", "update", "--init", "--recursive", dir: destination elsif Gem::Version.create(version) >= Gem::Version.create("2.9.0") inner_command = "git -C $toplevel submodule deinit --force $sm_path" - git_retry "submodule", "foreach", "--quiet", inner_command, :dir => destination + git_retry "submodule", "foreach", "--quiet", inner_command, dir: destination end end @@ -165,6 +182,14 @@ module Bundler if err.include?("Could not find remote branch") raise MissingGitRevisionError.new(command_with_no_credentials, nil, explicit_ref, credential_filtered_uri) else + idx = command.index("--depth") + if idx + command.delete_at(idx) + command.delete_at(idx) + command_with_no_credentials = check_allowed(command) + + err += "Retrying without --depth argument." + end raise GitCommandError.new(command_with_no_credentials, path, err) end end @@ -235,38 +260,46 @@ module Bundler end def pinned_to_full_sha? - ref =~ /\A\h{40}\z/ + full_sha_revision?(ref) + end + + def locked_to_full_sha? + full_sha_revision?(@revision) + end + + def full_sha_revision?(ref) + ref&.match?(/\A\h{40}\z/) end def git_null(*command, dir: nil) check_allowed(command) - capture(command, dir, :ignore_err => true) + capture(command, dir, ignore_err: true) end def git_retry(*command, dir: nil) command_with_no_credentials = check_allowed(command) Bundler::Retry.new("`#{command_with_no_credentials}` at #{dir || SharedHelpers.pwd}").attempts do - git(*command, :dir => dir) + git(*command, dir: dir) end end def git(*command, dir: nil) - command_with_no_credentials = check_allowed(command) - - out, err, status = capture(command, dir) - - raise GitCommandError.new(command_with_no_credentials, dir || SharedHelpers.pwd, err) unless status.success? - - Bundler.ui.warn err unless err.empty? + run_command(*command, dir: dir) do |unredacted_command| + check_allowed(unredacted_command) + end + end - out + def git_local(*command, dir: nil) + run_command(*command, dir: dir) do |unredacted_command| + redact_and_check_presence(unredacted_command) + end end def has_revision_cached? return unless @revision && path.exist? - git("cat-file", "-e", @revision, :dir => path) + git("cat-file", "-e", @revision, dir: path) true rescue GitError false @@ -289,13 +322,13 @@ module Bundler end def verify(reference) - git("rev-parse", "--verify", reference, :dir => path).strip + git("rev-parse", "--verify", reference, dir: path).strip end # Adds credentials to the URI def configured_uri if /https?:/.match?(uri) - remote = Bundler::URI(uri) + remote = Gem::URI(uri) config_auth = Bundler.settings[remote.to_s] || Bundler.settings[remote.host] remote.userinfo ||= config_auth remote.to_s @@ -330,12 +363,30 @@ module Bundler end def check_allowed(command) - require "shellwords" - command_with_no_credentials = URICredentialsFilter.credential_filtered_string("git #{command.shelljoin}", uri) + command_with_no_credentials = redact_and_check_presence(command) raise GitNotAllowedError.new(command_with_no_credentials) unless allow? command_with_no_credentials end + def redact_and_check_presence(command) + raise GitNotInstalledError.new unless Bundler.git_present? + + require "shellwords" + URICredentialsFilter.credential_filtered_string("git #{command.shelljoin}", uri) + end + + def run_command(*command, dir: nil) + command_with_no_credentials = yield(command) + + out, err, status = capture(command, dir) + + raise GitCommandError.new(command_with_no_credentials, dir || SharedHelpers.pwd, err) unless status.success? + + Bundler.ui.warn err unless err.empty? + + out + end + def capture(cmd, dir, ignore_err: false) SharedHelpers.with_clean_git_env do require "open3" @@ -355,7 +406,7 @@ module Bundler if Bundler.feature_flag.bundler_3_mode? || supports_minus_c? ["git", "-C", dir.to_s, *cmd] else - ["git", *cmd, { :chdir => dir.to_s }] + ["git", *cmd, { chdir: dir.to_s }] end end @@ -381,9 +432,9 @@ module Bundler ["--depth", depth.to_s] end - def extra_fetch_args + def extra_fetch_args(ref) extra_args = [path.to_s, *depth_args] - extra_args.push(@commit_ref) + extra_args.push(ref) extra_args end @@ -395,6 +446,10 @@ module Bundler @supports_minus_c ||= Gem::Version.new(version) >= Gem::Version.new("1.8.5") end + def needs_allow_any_sha1_in_want? + @needs_allow_any_sha1_in_want ||= Gem::Version.new(version) <= Gem::Version.new("2.13.7") + end + def supports_fetching_unreachable_refs? @supports_fetching_unreachable_refs ||= Gem::Version.new(version) >= Gem::Version.new("2.5.0") end diff --git a/lib/bundler/source/metadata.rb b/lib/bundler/source/metadata.rb index 593da6d1a7..6b05e17727 100644 --- a/lib/bundler/source/metadata.rb +++ b/lib/bundler/source/metadata.rb @@ -5,27 +5,29 @@ module Bundler class Metadata < Source def specs @specs ||= Index.build do |idx| - idx << Gem::Specification.new("Ruby\0", Gem.ruby_version) + idx << Gem::Specification.new("Ruby\0", Bundler::RubyVersion.system.gem_version) idx << Gem::Specification.new("RubyGems\0", Gem::VERSION) do |s| s.required_rubygems_version = Gem::Requirement.default end - idx << Gem::Specification.new do |s| - s.name = "bundler" - s.version = VERSION - s.license = "MIT" - s.platform = Gem::Platform::RUBY - s.authors = ["bundler team"] - s.bindir = "exe" - s.homepage = "https://bundler.io" - s.summary = "The best way to manage your application's dependencies" - s.executables = %w[bundle] - # can't point to the actual gemspec or else the require paths will be wrong - s.loaded_from = __dir__ - end + if local_spec = Gem.loaded_specs["bundler"] + raise CorruptBundlerInstallError.new(local_spec) if local_spec.version.to_s != Bundler::VERSION - if local_spec = Bundler.rubygems.find_bundler(VERSION) idx << local_spec + else + idx << Gem::Specification.new do |s| + s.name = "bundler" + s.version = VERSION + s.license = "MIT" + s.platform = Gem::Platform::RUBY + s.authors = ["bundler team"] + s.bindir = "exe" + s.homepage = "https://bundler.io" + s.summary = "The best way to manage your application's dependencies" + s.executables = %w[bundle] + # can't point to the actual gemspec or else the require paths will be wrong + s.loaded_from = __dir__ + end end idx.each {|s| s.source = self } diff --git a/lib/bundler/source/path.rb b/lib/bundler/source/path.rb index bdfcf8274a..978b0b2c9f 100644 --- a/lib/bundler/source/path.rb +++ b/lib/bundler/source/path.rb @@ -14,6 +14,7 @@ module Bundler DEFAULT_GLOB = "{,*,*/*}.gemspec" def initialize(options) + @checksum_store = Checksum::Store.new @options = options.dup @glob = options["glob"] || DEFAULT_GLOB @@ -85,7 +86,7 @@ module Bundler using_message = "Using #{version_message(spec, options[:previous_spec])} from #{self}" using_message += " and installing its executables" unless spec.executables.empty? print_using_message using_message - generate_bin(spec, :disable_extensions => true) + generate_bin(spec, disable_extensions: true) nil # no post-install message end @@ -225,7 +226,7 @@ module Bundler # Some gem authors put absolute paths in their gemspec # and we have to save them from themselves spec.files = spec.files.map do |path| - next path unless /\A#{Pathname::SEPARATOR_PAT}/.match?(path) + next path unless /\A#{Pathname::SEPARATOR_PAT}/o.match?(path) next if File.directory?(path) begin Pathname.new(path).relative_path_from(gem_dir).to_s @@ -236,10 +237,10 @@ module Bundler installer = Path::Installer.new( spec, - :env_shebang => false, - :disable_extensions => options[:disable_extensions], - :build_args => options[:build_args], - :bundler_extension_cache_path => extension_cache_path(spec) + env_shebang: false, + disable_extensions: options[:disable_extensions], + build_args: options[:build_args], + bundler_extension_cache_path: extension_cache_path(spec) ) installer.post_install rescue Gem::InvalidSpecificationException => e diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index a8d2e26324..3ac1bd4ff8 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -10,7 +10,7 @@ module Bundler # Ask for X gems per API request API_REQUEST_SIZE = 50 - attr_reader :remotes, :caches + attr_accessor :remotes def initialize(options = {}) @options = options @@ -19,11 +19,15 @@ module Bundler @allow_remote = false @allow_cached = false @allow_local = options["allow_local"] || false - @caches = [cache_path, *Bundler.rubygems.gem_cache] + @checksum_store = Checksum::Store.new Array(options["remotes"]).reverse_each {|r| add_remote(r) } end + def caches + @caches ||= [cache_path, *Bundler.rubygems.gem_cache] + end + def local_only! @specs = nil @allow_local = true @@ -46,10 +50,11 @@ module Bundler end def cached! + return unless File.exist?(cache_path) + return if @allow_cached @specs = nil - @allow_local = true @allow_cached = true end @@ -85,13 +90,14 @@ module Bundler end def self.from_lock(options) + options["remotes"] = Array(options.delete("remote")).reverse new(options) end def to_lock out = String.new("GEM\n") remotes.reverse_each do |remote| - out << " remote: #{suppress_configured_credentials remote}\n" + out << " remote: #{remove_auth remote}\n" end out << " specs:\n" end @@ -120,16 +126,17 @@ module Bundler end end alias_method :name, :identifier + alias_method :to_gemfile, :identifier def specs @specs ||= begin # remote_specs usually generates a way larger Index than the other - # sources, and large_idx.use small_idx is way faster than - # small_idx.use large_idx. - idx = @allow_remote ? remote_specs.dup : Index.new - idx.use(cached_specs, :override_dupes) if @allow_cached || @allow_remote - idx.use(installed_specs, :override_dupes) if @allow_local - idx + # sources, and large_idx.merge! small_idx is way faster than + # small_idx.merge! large_idx. + index = @allow_remote ? remote_specs.dup : Index.new + index.merge!(cached_specs) if @allow_cached + index.merge!(installed_specs) if @allow_local + index end end @@ -165,15 +172,14 @@ module Bundler installer = Bundler::RubyGemsGemInstaller.at( path, - :security_policy => Bundler.rubygems.security_policies[Bundler.settings["trust-policy"]], - :install_dir => install_path.to_s, - :bin_dir => bin_path.to_s, - :ignore_dependencies => true, - :wrappers => true, - :env_shebang => true, - :build_args => options[:build_args], - :bundler_expected_checksum => spec.respond_to?(:checksum) && spec.checksum, - :bundler_extension_cache_path => extension_cache_path(spec) + security_policy: Bundler.rubygems.security_policies[Bundler.settings["trust-policy"]], + install_dir: install_path.to_s, + bin_dir: bin_path.to_s, + ignore_dependencies: true, + wrappers: true, + env_shebang: true, + build_args: options[:build_args], + bundler_extension_cache_path: extension_cache_path(spec) ) if spec.remote @@ -191,6 +197,8 @@ module Bundler spec.__swap__(s) end + spec.source.checksum_store.register(spec, installer.gem_checksum) + message = "Installing #{version_message(spec, options[:previous_spec])}" message += " with native extensions" if spec.extensions.any? Bundler.ui.confirm message @@ -233,7 +241,7 @@ module Bundler end def spec_names - if @allow_remote && dependency_api_available? + if dependency_api_available? remote_specs.spec_names else [] @@ -241,22 +249,25 @@ module Bundler end def unmet_deps - if @allow_remote && dependency_api_available? + if dependency_api_available? remote_specs.unmet_dependency_names else [] end end - def fetchers - @fetchers ||= remotes.map do |uri| + def remote_fetchers + @remote_fetchers ||= remotes.to_h do |uri| remote = Source::Rubygems::Remote.new(uri) - Bundler::Fetcher.new(remote) - end + [remote, Bundler::Fetcher.new(remote)] + end.freeze + end + + def fetchers + @fetchers ||= remote_fetchers.values.freeze end def double_check_for(unmet_dependency_names) - return unless @allow_remote return unless dependency_api_available? unmet_dependency_names = unmet_dependency_names.call @@ -271,7 +282,9 @@ module Bundler Bundler.ui.debug "Double checking for #{unmet_dependency_names || "all specs (due to the size of the request)"} in #{self}" - fetch_names(api_fetchers, unmet_dependency_names, specs, false) + fetch_names(api_fetchers, unmet_dependency_names, remote_specs) + + specs.use remote_specs end def dependency_names_to_double_check @@ -300,11 +313,7 @@ module Bundler end def credless_remotes - if Bundler.settings[:allow_deployment_source_credential_changes] - remotes.map(&method(:remove_auth)) - else - remotes.map(&method(:suppress_configured_credentials)) - end + remotes.map(&method(:remove_auth)) end def remotes_for_spec(spec) @@ -324,9 +333,9 @@ module Bundler def cached_path(spec) global_cache_path = download_cache_path(spec) - @caches << global_cache_path if global_cache_path + caches << global_cache_path if global_cache_path - possibilities = @caches.map {|p| package_path(p, spec) } + possibilities = caches.map {|p| package_path(p, spec) } possibilities.find {|p| File.exist?(p) } end @@ -337,21 +346,12 @@ module Bundler def normalize_uri(uri) uri = URINormalizer.normalize_suffix(uri.to_s) require_relative "../vendored_uri" - uri = Bundler::URI(uri) + uri = Gem::URI(uri) raise ArgumentError, "The source must be an absolute URI. For example:\n" \ - "source 'https://rubygems.org'" if !uri.absolute? || (uri.is_a?(Bundler::URI::HTTP) && uri.host.nil?) + "source 'https://rubygems.org'" if !uri.absolute? || (uri.is_a?(Gem::URI::HTTP) && uri.host.nil?) uri end - def suppress_configured_credentials(remote) - remote_nouser = remove_auth(remote) - if remote.userinfo && remote.userinfo == Bundler.settings[remote_nouser] - remote_nouser - else - remote - end - end - def remove_auth(remote) if remote.user || remote.password remote.dup.tap {|uri| uri.user = uri.password = nil }.to_s @@ -375,10 +375,9 @@ module Bundler def cached_specs @cached_specs ||= begin - idx = @allow_local ? installed_specs.dup : Index.new + idx = Index.new Dir["#{cache_path}/*.gem"].each do |gemfile| - next if /^bundler\-[\d\.]+?\.gem/.match?(gemfile) s ||= Bundler.rubygems.spec_from_gem(gemfile) s.source = self idx << s @@ -389,35 +388,30 @@ module Bundler end def api_fetchers - fetchers.select {|f| f.use_api && f.fetchers.first.api_fetcher? } + fetchers.select(&:api_fetcher?) end def remote_specs @remote_specs ||= Index.build do |idx| index_fetchers = fetchers - api_fetchers - # gather lists from non-api sites - fetch_names(index_fetchers, nil, idx, false) - - # legacy multi-remote sources need special logic to figure out - # dependency names and that logic can be very costly if one remote - # uses the dependency API but others don't. So use full indexes - # consistently in that particular case. - allow_api = !multiple_remotes? - - fetch_names(api_fetchers, allow_api && dependency_names, idx, false) + if index_fetchers.empty? + fetch_names(api_fetchers, dependency_names, idx) + else + fetch_names(fetchers, nil, idx) + end end end - def fetch_names(fetchers, dependency_names, index, override_dupes) + def fetch_names(fetchers, dependency_names, index) fetchers.each do |f| if dependency_names Bundler.ui.info "Fetching gem metadata from #{URICredentialsFilter.credential_filtered_uri(f.uri)}", Bundler.ui.debug? - index.use f.specs_with_retry(dependency_names, self), override_dupes + index.use f.specs_with_retry(dependency_names, self) Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over else Bundler.ui.info "Fetching source index from #{URICredentialsFilter.credential_filtered_uri(f.uri)}" - index.use f.specs_with_retry(nil, self), override_dupes + index.use f.specs_with_retry(nil, self) end end end @@ -478,7 +472,8 @@ module Bundler def download_gem(spec, download_cache_path, previous_spec = nil) uri = spec.remote.uri Bundler.ui.confirm("Fetching #{version_message(spec, previous_spec)}") - Bundler.rubygems.download_gem(spec, uri, download_cache_path) + gem_remote_fetcher = remote_fetchers.fetch(spec.remote).gem_remote_fetcher + Bundler.rubygems.download_gem(spec, uri, download_cache_path, gem_remote_fetcher) end # Returns the global cache path of the calling Rubygems::Source object. diff --git a/lib/bundler/source/rubygems/remote.rb b/lib/bundler/source/rubygems/remote.rb index 82c850ffbb..9c5c06de24 100644 --- a/lib/bundler/source/rubygems/remote.rb +++ b/lib/bundler/source/rubygems/remote.rb @@ -48,7 +48,7 @@ module Bundler end uri - rescue Bundler::URI::InvalidComponentError + rescue Gem::URI::InvalidComponentError error_message = "Please CGI escape your usernames and passwords before " \ "setting them for authentication." raise HTTPError.new(error_message) diff --git a/lib/bundler/source_list.rb b/lib/bundler/source_list.rb index 63798db941..5f9dd68f17 100644 --- a/lib/bundler/source_list.rb +++ b/lib/bundler/source_list.rb @@ -22,6 +22,7 @@ module Bundler @metadata_source = Source::Metadata.new @merged_gem_lockfile_sections = false + @local_mode = true end def merged_gem_lockfile_sections? @@ -73,6 +74,10 @@ module Bundler global_rubygems_source end + def local_mode? + @local_mode + end + def default_source global_path_source || global_rubygems_source end @@ -101,10 +106,6 @@ module Bundler source_list_for(source).find {|s| equivalent_source?(source, s) } end - def get_with_fallback(source) - get(source) || default_source - end - def lock_sources lock_other_sources + lock_rubygems_sources end @@ -144,11 +145,17 @@ module Bundler all_sources.each(&:local_only!) end + def local! + all_sources.each(&:local!) + end + def cached! all_sources.each(&:cached!) end def remote! + @local_mode = false + all_sources.each(&:remote!) end @@ -161,7 +168,11 @@ module Bundler end def map_sources(replacement_sources) - rubygems, git, plugin = [@rubygems_sources, @git_sources, @plugin_sources].map do |sources| + rubygems = @rubygems_sources.map do |source| + replace_rubygems_source(replacement_sources, source) || source + end + + git, plugin = [@git_sources, @plugin_sources].map do |sources| sources.map do |source| replacement_sources.find {|s| s == source } || source end @@ -175,13 +186,22 @@ module Bundler end def global_replacement_source(replacement_sources) - replacement_source = replacement_sources.find {|s| s == global_rubygems_source } + replacement_source = replace_rubygems_source(replacement_sources, global_rubygems_source) return global_rubygems_source unless replacement_source replacement_source.local! replacement_source end + def replace_rubygems_source(replacement_sources, gemfile_source) + replacement_source = replacement_sources.find {|s| s == gemfile_source } + return unless replacement_source + + # locked sources never include credentials so always prefer remotes from the gemfile + replacement_source.remotes = gemfile_source.remotes + replacement_source + end + def different_sources?(lock_sources, replacement_sources) !equivalent_sources?(lock_sources, replacement_sources) end diff --git a/lib/bundler/spec_set.rb b/lib/bundler/spec_set.rb index 21630e3a3e..2933d28450 100644 --- a/lib/bundler/spec_set.rb +++ b/lib/bundler/spec_set.rb @@ -37,7 +37,7 @@ module Bundler specs_for_dep.first.dependencies.each do |d| next if d.type == :development - incomplete = true if d.name != "bundler" && lookup[d.name].empty? + incomplete = true if d.name != "bundler" && lookup[d.name].nil? deps << [d, dep[1]] end else @@ -45,28 +45,64 @@ module Bundler end if incomplete && check - @incomplete_specs += lookup[name].any? ? lookup[name] : [LazySpecification.new(name, nil, nil)] + @incomplete_specs += lookup[name] || [LazySpecification.new(name, nil, nil)] end end specs.uniq end + def add_extra_platforms!(platforms) + return platforms.concat([Gem::Platform::RUBY]).uniq if @specs.empty? + + new_platforms = all_platforms.select do |platform| + next if platforms.include?(platform) + next unless GemHelpers.generic(platform) == Gem::Platform::RUBY + + complete_platform(platform) + end + return platforms if new_platforms.empty? + + platforms.concat(new_platforms) + + less_specific_platform = new_platforms.find {|platform| platform != Gem::Platform::RUBY && Bundler.local_platform === platform && platform === Bundler.local_platform } + platforms.delete(Bundler.local_platform) if less_specific_platform + + platforms + end + + def complete_platforms!(platforms) + platforms.each do |platform| + complete_platform(platform) + end + end + + def validate_deps(s) + s.runtime_dependencies.each do |dep| + next if dep.name == "bundler" + + return :missing unless names.include?(dep.name) + return :invalid if none? {|spec| dep.matches_spec?(spec) } + end + + :valid + end + def [](key) key = key.name if key.respond_to?(:name) - lookup[key].reverse + lookup[key]&.reverse || [] end def []=(key, value) @specs << value - @lookup = nil - @sorted = nil + + reset! end def delete(specs) specs.each {|spec| @specs.delete(spec) } - @lookup = nil - @sorted = nil + + reset! end def sort! @@ -100,12 +136,12 @@ module Bundler end end - def incomplete_ruby_specs?(deps) + def incomplete_for_platform?(deps, platform) return false if @specs.empty? @incomplete_specs = [] - self.for(deps, true, [Gem::Platform::RUBY]) + self.for(deps, true, [platform]) @incomplete_specs.any? end @@ -114,16 +150,6 @@ module Bundler @specs.select {|s| s.is_a?(LazySpecification) } end - def merge(set) - arr = sorted.dup - set.each do |set_spec| - full_name = set_spec.full_name - next if arr.any? {|spec| spec.full_name == full_name } - arr << set_spec - end - SpecSet.new(arr) - end - def -(other) SpecSet.new(to_a - other.to_a) end @@ -134,12 +160,12 @@ module Bundler def delete_by_name(name) @specs.reject! {|spec| spec.name == name } - @lookup = nil - @sorted = nil + + reset! end def what_required(spec) - unless req = find {|s| s.dependencies.any? {|d| d.type == :runtime && d.name == spec.name } } + unless req = find {|s| s.runtime_dependencies.any? {|d| d.name == spec.name } } return [spec] end what_required(req) << spec @@ -165,8 +191,52 @@ module Bundler sorted.each(&b) end + def names + lookup.keys + end + private + def reset! + @sorted = nil + @lookup = nil + end + + def complete_platform(platform) + new_specs = [] + + valid_platform = lookup.all? do |_, specs| + spec = specs.first + matching_specs = spec.source.specs.search([spec.name, spec.version]) + platform_spec = GemHelpers.select_best_platform_match(matching_specs, platform).find do |s| + s.matches_current_metadata? && valid_dependencies?(s) + end + + if platform_spec + new_specs << LazySpecification.from_spec(platform_spec) unless specs.include?(platform_spec) + true + else + false + end + end + + if valid_platform && new_specs.any? + @specs.concat(new_specs) + + reset! + end + + valid_platform + end + + def all_platforms + @specs.flat_map {|spec| spec.source.specs.search([spec.name, spec.version]).map(&:platform) }.uniq + end + + def valid_dependencies?(s) + validate_deps(s) == :valid + end + def sorted rake = @specs.find {|s| s.name == "rake" } begin @@ -185,8 +255,9 @@ module Bundler def lookup @lookup ||= begin - lookup = Hash.new {|h, k| h[k] = [] } + lookup = {} @specs.each do |s| + lookup[s.name] ||= [] lookup[s.name] << s end lookup @@ -200,8 +271,13 @@ module Bundler def specs_for_dependency(dep, platform) specs_for_name = lookup[dep.name] - target_platform = dep.force_ruby_platform ? Gem::Platform::RUBY : (platform || Bundler.local_platform) - matching_specs = GemHelpers.select_best_platform_match(specs_for_name, target_platform) + return [] unless specs_for_name + + matching_specs = if dep.force_ruby_platform + GemHelpers.force_ruby_platform(specs_for_name) + else + GemHelpers.select_best_platform_match(specs_for_name, platform || Bundler.local_platform) + end matching_specs.map!(&:materialize_for_installation).compact! if platform.nil? matching_specs end @@ -209,7 +285,11 @@ module Bundler def tsort_each_child(s) s.dependencies.sort_by(&:name).each do |d| next if d.type == :development - lookup[d.name].each {|s2| yield s2 } + + specs_for_name = lookup[d.name] + next unless specs_for_name + + specs_for_name.each {|s2| yield s2 } end end end diff --git a/lib/bundler/stub_specification.rb b/lib/bundler/stub_specification.rb index 88a4257fa4..da830cf8d4 100644 --- a/lib/bundler/stub_specification.rb +++ b/lib/bundler/stub_specification.rb @@ -9,6 +9,7 @@ module Bundler spec end + attr_reader :checksum attr_accessor :stub, :ignored def source=(source) @@ -16,7 +17,8 @@ module Bundler # Stub has no concept of source, which means that extension_dir may be wrong # This is the case for git-based gems. So, instead manually assign the extension dir return unless source.respond_to?(:extension_dir_name) - path = File.join(stub.extensions_dir, source.extension_dir_name) + unique_extension_dir = [source.extension_dir_name, File.basename(full_gem_path)].uniq.join("-") + path = File.join(stub.extensions_dir, unique_extension_dir) stub.extension_dir = File.expand_path(path) end @@ -56,7 +58,7 @@ module Bundler end def gem_build_complete_path - File.join(extension_dir, "gem.build_complete") + stub.gem_build_complete_path end def default_gem? @@ -108,6 +110,7 @@ module Bundler end rs.source = source + rs.base_dir = stub.base_dir rs end diff --git a/lib/bundler/templates/Executable.bundler b/lib/bundler/templates/Executable.bundler index e290fe91eb..caa2021701 100644 --- a/lib/bundler/templates/Executable.bundler +++ b/lib/bundler/templates/Executable.bundler @@ -27,7 +27,7 @@ m = Module.new do bundler_version = nil update_index = nil ARGV.each_with_index do |a, i| - if update_index && update_index.succ == i && a =~ Gem::Version::ANCHORED_VERSION_PATTERN + if update_index && update_index.succ == i && a.match?(Gem::Version::ANCHORED_VERSION_PATTERN) bundler_version = a end next unless a =~ /\A--bundler(?:[= ](#{Gem::Version::VERSION_PATTERN}))?\z/ diff --git a/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt b/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt index 175b821a62..6a46aba8df 100644 --- a/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt +++ b/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt @@ -2,7 +2,7 @@ ## Our Pledge -We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, religion, or sexual identity and orientation. We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. diff --git a/lib/bundler/templates/newgem/Rakefile.tt b/lib/bundler/templates/newgem/Rakefile.tt index b5a5c4e392..172183d4b4 100644 --- a/lib/bundler/templates/newgem/Rakefile.tt +++ b/lib/bundler/templates/newgem/Rakefile.tt @@ -4,13 +4,9 @@ require "bundler/gem_tasks" <% default_task_names = [config[:test_task]].compact -%> <% case config[:test] -%> <% when "minitest" -%> -require "rake/testtask" +require "minitest/test_task" -Rake::TestTask.new(:test) do |t| - t.libs << "test" - t.libs << "lib" - t.test_files = FileList["test/**/test_*.rb"] -end +Minitest::TestTask.create <% when "test-unit" -%> require "rake/testtask" @@ -46,7 +42,9 @@ require "rb_sys/extensiontask" task build: :compile -RbSys::ExtensionTask.new(<%= config[:name].inspect %>) do |ext| +GEMSPEC = Gem::Specification.load("<%= config[:underscored_name] %>.gemspec") + +RbSys::ExtensionTask.new(<%= config[:name].inspect %>, GEMSPEC) do |ext| ext.lib_dir = "lib/<%= config[:namespaced_path] %>" end <% else -%> @@ -54,7 +52,9 @@ require "rake/extensiontask" task build: :compile -Rake::ExtensionTask.new("<%= config[:underscored_name] %>") do |ext| +GEMSPEC = Gem::Specification.load("<%= config[:underscored_name] %>.gemspec") + +Rake::ExtensionTask.new("<%= config[:underscored_name] %>", GEMSPEC) do |ext| ext.lib_dir = "lib/<%= config[:namespaced_path] %>" end <% end -%> diff --git a/lib/bundler/templates/newgem/ext/newgem/Cargo.toml.tt b/lib/bundler/templates/newgem/ext/newgem/Cargo.toml.tt index 4b6e9587f7..0ebce0e4a0 100644 --- a/lib/bundler/templates/newgem/ext/newgem/Cargo.toml.tt +++ b/lib/bundler/templates/newgem/ext/newgem/Cargo.toml.tt @@ -12,4 +12,4 @@ publish = false crate-type = ["cdylib"] [dependencies] -magnus = { version = "0.4" } +magnus = { version = "0.6.2" } diff --git a/lib/bundler/templates/newgem/ext/newgem/src/lib.rs.tt b/lib/bundler/templates/newgem/ext/newgem/src/lib.rs.tt index b311283997..ba234529a3 100644 --- a/lib/bundler/templates/newgem/ext/newgem/src/lib.rs.tt +++ b/lib/bundler/templates/newgem/ext/newgem/src/lib.rs.tt @@ -1,12 +1,12 @@ -use magnus::{define_module, function, prelude::*, Error}; +use magnus::{function, prelude::*, Error, Ruby}; fn hello(subject: String) -> String { - format!("Hello from Rust, {}!", subject) + format!("Hello from Rust, {subject}!") } #[magnus::init] -fn init() -> Result<(), Error> { - let module = <%= config[:constant_array].map {|c| "define_module(#{c.dump})?"}.join(".") %>; +fn init(ruby: &Ruby) -> Result<(), Error> { + let module = ruby.<%= config[:constant_array].map {|c| "define_module(#{c.dump})?"}.join(".") %>; module.define_singleton_method("hello", function!(hello, 1))?; Ok(()) } diff --git a/lib/bundler/templates/newgem/github/workflows/main.yml.tt b/lib/bundler/templates/newgem/github/workflows/main.yml.tt index be58dd8156..32b39558d8 100644 --- a/lib/bundler/templates/newgem/github/workflows/main.yml.tt +++ b/lib/bundler/templates/newgem/github/workflows/main.yml.tt @@ -17,7 +17,7 @@ jobs: - '<%= RUBY_VERSION %>' steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 <%- if config[:ext] == 'rust' -%> - name: Set up Ruby & Rust uses: oxidize-rb/actions/setup-ruby-and-rust@v1 diff --git a/lib/bundler/templates/newgem/newgem.gemspec.tt b/lib/bundler/templates/newgem/newgem.gemspec.tt index da81f046d4..6e88f4dab1 100644 --- a/lib/bundler/templates/newgem/newgem.gemspec.tt +++ b/lib/bundler/templates/newgem/newgem.gemspec.tt @@ -27,9 +27,11 @@ Gem::Specification.new do |spec| # Specify which files should be added to the gem when it is released. # The `git ls-files -z` loads the files in the RubyGem that have been added into git. - spec.files = Dir.chdir(__dir__) do - `git ls-files -z`.split("\x0").reject do |f| - (File.expand_path(f) == __FILE__) || f.start_with?(*%w[bin/ test/ spec/ features/ .git .circleci appveyor]) + gemspec = File.basename(__FILE__) + spec.files = IO.popen(%w[git ls-files -z], chdir: __dir__, err: IO::NULL) do |ls| + ls.readlines("\x0", chomp: true).reject do |f| + (f == gemspec) || + f.start_with?(*%w[bin/ test/ spec/ features/ .git <%= config[:ci_config_path] %>appveyor Gemfile]) end end spec.bindir = "exe" diff --git a/lib/bundler/templates/newgem/rubocop.yml.tt b/lib/bundler/templates/newgem/rubocop.yml.tt index 9ecec78807..3d1c4ee7b2 100644 --- a/lib/bundler/templates/newgem/rubocop.yml.tt +++ b/lib/bundler/templates/newgem/rubocop.yml.tt @@ -2,12 +2,7 @@ AllCops: TargetRubyVersion: <%= ::Gem::Version.new(config[:required_ruby_version]).segments[0..1].join(".") %> Style/StringLiterals: - Enabled: true EnforcedStyle: double_quotes Style/StringLiteralsInInterpolation: - Enabled: true EnforcedStyle: double_quotes - -Layout/LineLength: - Max: 120 diff --git a/lib/bundler/templates/newgem/standard.yml.tt b/lib/bundler/templates/newgem/standard.yml.tt index 934b0b2c37..a0696cd2e9 100644 --- a/lib/bundler/templates/newgem/standard.yml.tt +++ b/lib/bundler/templates/newgem/standard.yml.tt @@ -1,3 +1,3 @@ # For available configuration options, see: -# https://github.com/testdouble/standard +# https://github.com/standardrb/standard ruby_version: <%= ::Gem::Version.new(config[:required_ruby_version]).segments[0..1].join(".") %> diff --git a/lib/bundler/ui/shell.rb b/lib/bundler/ui/shell.rb index 4139585c47..4555612dbb 100644 --- a/lib/bundler/ui/shell.rb +++ b/lib/bundler/ui/shell.rb @@ -130,7 +130,7 @@ module Bundler def tell_err(message, color = nil, newline = nil) return if @shell.send(:stderr).closed? - newline ||= message.to_s !~ /( |\t)\Z/ + newline ||= !message.to_s.match?(/( |\t)\Z/) message = word_wrap(message) if newline.is_a?(Hash) && newline[:wrap] color = nil if color && !$stderr.tty? @@ -147,7 +147,7 @@ module Bundler spaces ? text.gsub(/#{spaces}/, "") : text end - def word_wrap(text, line_width = @shell.terminal_width) + def word_wrap(text, line_width = Thor::Terminal.terminal_width) strip_leading_spaces(text).split("\n").collect do |line| line.length > line_width ? line.gsub(/(.{1,#{line_width}})(\s+|$)/, "\\1\n").strip : line end * "\n" diff --git a/lib/bundler/uri_credentials_filter.rb b/lib/bundler/uri_credentials_filter.rb index ccfaf0bc5d..a83f5304e2 100644 --- a/lib/bundler/uri_credentials_filter.rb +++ b/lib/bundler/uri_credentials_filter.rb @@ -11,7 +11,7 @@ module Bundler return uri if File.exist?(uri) require_relative "vendored_uri" - uri = Bundler::URI(uri) + uri = Gem::URI(uri) end if uri.userinfo @@ -25,7 +25,7 @@ module Bundler end return uri.to_s if uri_to_anonymize.is_a?(String) uri - rescue Bundler::URI::InvalidURIError # uri is not canonical uri scheme + rescue Gem::URI::InvalidURIError # uri is not canonical uri scheme uri end diff --git a/lib/rubygems/optparse/.document b/lib/bundler/vendor/connection_pool/.document index 0c43bbd6b3..0c43bbd6b3 100644 --- a/lib/rubygems/optparse/.document +++ b/lib/bundler/vendor/connection_pool/.document diff --git a/lib/bundler/vendor/connection_pool/lib/connection_pool.rb b/lib/bundler/vendor/connection_pool/lib/connection_pool.rb index 455319efe3..317088a866 100644 --- a/lib/bundler/vendor/connection_pool/lib/connection_pool.rb +++ b/lib/bundler/vendor/connection_pool/lib/connection_pool.rb @@ -1,4 +1,4 @@ -require "timeout" +require_relative "../../../vendored_timeout" require_relative "connection_pool/version" class Bundler::ConnectionPool @@ -6,7 +6,7 @@ class Bundler::ConnectionPool class PoolShuttingDownError < ::Bundler::ConnectionPool::Error; end - class TimeoutError < ::Timeout::Error; end + class TimeoutError < ::Gem::Timeout::Error; end end # Generic connection pool class for sharing a limited number of objects or network connections @@ -36,14 +36,57 @@ end # Accepts the following options: # - :size - number of connections to pool, defaults to 5 # - :timeout - amount of time to wait for a connection if none currently available, defaults to 5 seconds +# - :auto_reload_after_fork - automatically drop all connections after fork, defaults to true # class Bundler::ConnectionPool - DEFAULTS = {size: 5, timeout: 5} + DEFAULTS = {size: 5, timeout: 5, auto_reload_after_fork: true} def self.wrap(options, &block) Wrapper.new(options, &block) end + if Process.respond_to?(:fork) + INSTANCES = ObjectSpace::WeakMap.new + private_constant :INSTANCES + + def self.after_fork + INSTANCES.values.each do |pool| + next unless pool.auto_reload_after_fork + + # We're on after fork, so we know all other threads are dead. + # All we need to do is to ensure the main thread doesn't have a + # checked out connection + pool.checkin(force: true) + pool.reload do |connection| + # Unfortunately we don't know what method to call to close the connection, + # so we try the most common one. + connection.close if connection.respond_to?(:close) + end + end + nil + end + + if ::Process.respond_to?(:_fork) # MRI 3.1+ + module ForkTracker + def _fork + pid = super + if pid == 0 + Bundler::ConnectionPool.after_fork + end + pid + end + end + Process.singleton_class.prepend(ForkTracker) + end + else + INSTANCES = nil + private_constant :INSTANCES + + def self.after_fork + # noop + end + end + def initialize(options = {}, &block) raise ArgumentError, "Connection pool requires a block" unless block @@ -51,10 +94,12 @@ class Bundler::ConnectionPool @size = Integer(options.fetch(:size)) @timeout = options.fetch(:timeout) + @auto_reload_after_fork = options.fetch(:auto_reload_after_fork) @available = TimedStack.new(@size, &block) @key = :"pool-#{@available.object_id}" @key_count = :"pool-#{@available.object_id}-count" + INSTANCES[self] = self if INSTANCES end def with(options = {}) @@ -81,16 +126,16 @@ class Bundler::ConnectionPool end end - def checkin + def checkin(force: false) if ::Thread.current[@key] - if ::Thread.current[@key_count] == 1 + if ::Thread.current[@key_count] == 1 || force @available.push(::Thread.current[@key]) ::Thread.current[@key] = nil ::Thread.current[@key_count] = nil else ::Thread.current[@key_count] -= 1 end - else + elsif !force raise Bundler::ConnectionPool::Error, "no connections are checked out" end @@ -117,6 +162,8 @@ class Bundler::ConnectionPool # Size of this connection pool attr_reader :size + # Automatically drop all connections after fork + attr_reader :auto_reload_after_fork # Number of pool entries available for checkout at this instant. def available diff --git a/lib/bundler/vendor/connection_pool/lib/connection_pool/version.rb b/lib/bundler/vendor/connection_pool/lib/connection_pool/version.rb index 56ebf69902..384d6fc977 100644 --- a/lib/bundler/vendor/connection_pool/lib/connection_pool/version.rb +++ b/lib/bundler/vendor/connection_pool/lib/connection_pool/version.rb @@ -1,3 +1,3 @@ class Bundler::ConnectionPool - VERSION = "2.3.0" + VERSION = "2.4.1" end diff --git a/lib/rubygems/tsort/.document b/lib/bundler/vendor/fileutils/.document index 0c43bbd6b3..0c43bbd6b3 100644 --- a/lib/rubygems/tsort/.document +++ b/lib/bundler/vendor/fileutils/.document diff --git a/lib/bundler/vendor/fileutils/lib/fileutils.rb b/lib/bundler/vendor/fileutils/lib/fileutils.rb index 211311c069..6db19caf6f 100644 --- a/lib/bundler/vendor/fileutils/lib/fileutils.rb +++ b/lib/bundler/vendor/fileutils/lib/fileutils.rb @@ -180,7 +180,7 @@ end # - {CVE-2004-0452}[https://cve.mitre.org/cgi-bin/cvename.cgi?name=CAN-2004-0452]. # module Bundler::FileUtils - VERSION = "1.7.0" + VERSION = "1.7.2" def self.private_module_function(name) #:nodoc: module_function name @@ -192,8 +192,6 @@ module Bundler::FileUtils # # Bundler::FileUtils.pwd # => "/rdoc/fileutils" # - # Bundler::FileUtils.getwd is an alias for Bundler::FileUtils.pwd. - # # Related: Bundler::FileUtils.cd. # def pwd @@ -235,8 +233,6 @@ module Bundler::FileUtils # cd .. # cd fileutils # - # Bundler::FileUtils.chdir is an alias for Bundler::FileUtils.cd. - # # Related: Bundler::FileUtils.pwd. # def cd(dir, verbose: nil, &block) # :yield: dir @@ -515,8 +511,6 @@ module Bundler::FileUtils # Raises an exception if +dest+ is the path to an existing file # and keyword argument +force+ is not +true+. # - # Bundler::FileUtils#link is an alias for Bundler::FileUtils#ln. - # # Related: Bundler::FileUtils.link_entry (has different options). # def ln(src, dest, force: nil, noop: nil, verbose: nil) @@ -707,8 +701,6 @@ module Bundler::FileUtils # ln -sf src2.txt dest2.txt # ln -s srcdir3/src0.txt srcdir3/src1.txt destdir3 # - # Bundler::FileUtils.symlink is an alias for Bundler::FileUtils.ln_s. - # # Related: Bundler::FileUtils.ln_sf. # def ln_s(src, dest, force: nil, relative: false, target_directory: true, noop: nil, verbose: nil) @@ -876,8 +868,6 @@ module Bundler::FileUtils # # Raises an exception if +src+ is a directory. # - # Bundler::FileUtils.copy is an alias for Bundler::FileUtils.cp. - # # Related: {methods for copying}[rdoc-ref:FileUtils@Copying]. # def cp(src, dest, preserve: nil, noop: nil, verbose: nil) @@ -1164,8 +1154,6 @@ module Bundler::FileUtils # mv src0 dest0 # mv src1.txt src1 dest1 # - # Bundler::FileUtils.move is an alias for Bundler::FileUtils.mv. - # def mv(src, dest, force: nil, noop: nil, verbose: nil, secure: nil) fu_output_message "mv#{force ? ' -f' : ''} #{[src,dest].flatten.join ' '}" if verbose return if noop @@ -1223,8 +1211,6 @@ module Bundler::FileUtils # # rm src0.dat src0.txt # - # Bundler::FileUtils.remove is an alias for Bundler::FileUtils.rm. - # # Related: {methods for deleting}[rdoc-ref:FileUtils@Deleting]. # def rm(list, force: nil, noop: nil, verbose: nil) @@ -1250,8 +1236,6 @@ module Bundler::FileUtils # # See Bundler::FileUtils.rm for keyword arguments. # - # Bundler::FileUtils.safe_unlink is an alias for Bundler::FileUtils.rm_f. - # # Related: {methods for deleting}[rdoc-ref:FileUtils@Deleting]. # def rm_f(list, noop: nil, verbose: nil) @@ -1339,8 +1323,6 @@ module Bundler::FileUtils # # See Bundler::FileUtils.rm_r for keyword arguments. # - # Bundler::FileUtils.rmtree is an alias for Bundler::FileUtils.rm_rf. - # # Related: {methods for deleting}[rdoc-ref:FileUtils@Deleting]. # def rm_rf(list, noop: nil, verbose: nil, secure: nil) @@ -1642,7 +1624,13 @@ module Bundler::FileUtils st = File.stat(s) unless File.exist?(d) and compare_file(s, d) remove_file d, true - copy_file s, d + if d.end_with?('/') + mkdir_p d + copy_file s, d + File.basename(s) + else + mkdir_p File.expand_path('..', d) + copy_file s, d + end File.utime st.atime, st.mtime, d if preserve File.chmod fu_mode(mode, st), d if mode File.chown uid, gid, d if uid or gid diff --git a/lib/bundler/vendor/net-http-persistent/.document b/lib/bundler/vendor/net-http-persistent/.document new file mode 100644 index 0000000000..0c43bbd6b3 --- /dev/null +++ b/lib/bundler/vendor/net-http-persistent/.document @@ -0,0 +1 @@ +# Vendored files do not need to be documented diff --git a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb index a4e1c5a750..c15b346330 100644 --- a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb +++ b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb @@ -1,14 +1,14 @@ -require 'net/http' -require_relative '../../../../uri/lib/uri' +require_relative '../../../../../vendored_net_http' +require_relative '../../../../../vendored_uri' require 'cgi' # for escaping require_relative '../../../../connection_pool/lib/connection_pool' autoload :OpenSSL, 'openssl' ## -# Persistent connections for Net::HTTP +# Persistent connections for Gem::Net::HTTP # -# Bundler::Persistent::Net::HTTP::Persistent maintains persistent connections across all the +# Gem::Net::HTTP::Persistent maintains persistent connections across all the # servers you wish to talk to. For each host:port you communicate with a # single persistent connection is created. # @@ -22,34 +22,34 @@ autoload :OpenSSL, 'openssl' # # require 'bundler/vendor/net-http-persistent/lib/net/http/persistent' # -# uri = Bundler::URI 'http://example.com/awesome/web/service' +# uri = Gem::URI 'http://example.com/awesome/web/service' # -# http = Bundler::Persistent::Net::HTTP::Persistent.new +# http = Gem::Net::HTTP::Persistent.new # # # perform a GET # response = http.request uri # # # or # -# get = Net::HTTP::Get.new uri.request_uri +# get = Gem::Net::HTTP::Get.new uri.request_uri # response = http.request get # # # create a POST # post_uri = uri + 'create' -# post = Net::HTTP::Post.new post_uri.path +# post = Gem::Net::HTTP::Post.new post_uri.path # post.set_form_data 'some' => 'cool data' # -# # perform the POST, the Bundler::URI is always required +# # perform the POST, the Gem::URI is always required # response http.request post_uri, post # # Note that for GET, HEAD and other requests that do not have a body you want -# to use Bundler::URI#request_uri not Bundler::URI#path. The request_uri contains the query +# to use Gem::URI#request_uri not Gem::URI#path. The request_uri contains the query # params which are sent in the body for other requests. # # == TLS/SSL # # TLS connections are automatically created depending upon the scheme of the -# Bundler::URI. TLS connections are automatically verified against the default +# Gem::URI. TLS connections are automatically verified against the default # certificate store for your computer. You can override this by changing # verify_mode or by specifying an alternate cert_store. # @@ -72,7 +72,7 @@ autoload :OpenSSL, 'openssl' # == Proxies # # A proxy can be set through #proxy= or at initialization time by providing a -# second argument to ::new. The proxy may be the Bundler::URI of the proxy server or +# second argument to ::new. The proxy may be the Gem::URI of the proxy server or # <code>:ENV</code> which will consult environment variables. # # See #proxy= and #proxy_from_env for details. @@ -92,7 +92,7 @@ autoload :OpenSSL, 'openssl' # # === Segregation # -# Each Bundler::Persistent::Net::HTTP::Persistent instance has its own pool of connections. There +# Each Gem::Net::HTTP::Persistent instance has its own pool of connections. There # is no sharing with other instances (as was true in earlier versions). # # === Idle Timeout @@ -131,7 +131,7 @@ autoload :OpenSSL, 'openssl' # # === Connection Termination # -# If you are done using the Bundler::Persistent::Net::HTTP::Persistent instance you may shut down +# If you are done using the Gem::Net::HTTP::Persistent instance you may shut down # all the connections in the current thread with #shutdown. This is not # recommended for normal use, it should only be used when it will be several # minutes before you make another HTTP request. @@ -141,7 +141,7 @@ autoload :OpenSSL, 'openssl' # Ruby will automatically garbage collect and shutdown your HTTP connections # when the thread terminates. -class Bundler::Persistent::Net::HTTP::Persistent +class Gem::Net::HTTP::Persistent ## # The beginning of Time @@ -172,12 +172,12 @@ class Bundler::Persistent::Net::HTTP::Persistent end ## - # The version of Bundler::Persistent::Net::HTTP::Persistent you are using + # The version of Gem::Net::HTTP::Persistent you are using - VERSION = '4.0.1' + VERSION = '4.0.2' ## - # Error class for errors raised by Bundler::Persistent::Net::HTTP::Persistent. Various + # Error class for errors raised by Gem::Net::HTTP::Persistent. Various # SystemCallErrors are re-raised with a human-readable message under this # class. @@ -197,10 +197,10 @@ class Bundler::Persistent::Net::HTTP::Persistent # NOTE: This may not work on ruby > 1.9. def self.detect_idle_timeout uri, max = 10 - uri = Bundler::URI uri unless Bundler::URI::Generic === uri + uri = Gem::URI uri unless Gem::URI::Generic === uri uri += '/' - req = Net::HTTP::Head.new uri.request_uri + req = Gem::Net::HTTP::Head.new uri.request_uri http = new 'net-http-persistent detect_idle_timeout' @@ -214,7 +214,7 @@ class Bundler::Persistent::Net::HTTP::Persistent $stderr.puts "HEAD #{uri} => #{response.code}" if $DEBUG - unless Net::HTTPOK === response then + unless Gem::Net::HTTPOK === response then raise Error, "bad response code #{response.code} detecting idle timeout" end @@ -238,7 +238,7 @@ class Bundler::Persistent::Net::HTTP::Persistent attr_reader :certificate ## - # For Net::HTTP parity + # For Gem::Net::HTTP parity alias cert certificate @@ -266,7 +266,7 @@ class Bundler::Persistent::Net::HTTP::Persistent attr_reader :ciphers ## - # Sends debug_output to this IO via Net::HTTP#set_debug_output. + # Sends debug_output to this IO via Gem::Net::HTTP#set_debug_output. # # Never use this method in production code, it causes a serious security # hole. @@ -279,7 +279,7 @@ class Bundler::Persistent::Net::HTTP::Persistent attr_reader :generation # :nodoc: ## - # Headers that are added to every request using Net::HTTP#add_field + # Headers that are added to every request using Gem::Net::HTTP#add_field attr_reader :headers @@ -304,7 +304,7 @@ class Bundler::Persistent::Net::HTTP::Persistent ## # Number of retries to perform if a request fails. # - # See also #max_retries=, Net::HTTP#max_retries=. + # See also #max_retries=, Gem::Net::HTTP#max_retries=. attr_reader :max_retries @@ -325,12 +325,12 @@ class Bundler::Persistent::Net::HTTP::Persistent attr_reader :name ## - # Seconds to wait until a connection is opened. See Net::HTTP#open_timeout + # Seconds to wait until a connection is opened. See Gem::Net::HTTP#open_timeout attr_accessor :open_timeout ## - # Headers that are added to every request using Net::HTTP#[]= + # Headers that are added to every request using Gem::Net::HTTP#[]= attr_reader :override_headers @@ -340,7 +340,7 @@ class Bundler::Persistent::Net::HTTP::Persistent attr_reader :private_key ## - # For Net::HTTP parity + # For Gem::Net::HTTP parity alias key private_key @@ -360,12 +360,12 @@ class Bundler::Persistent::Net::HTTP::Persistent attr_reader :pool # :nodoc: ## - # Seconds to wait until reading one block. See Net::HTTP#read_timeout + # Seconds to wait until reading one block. See Gem::Net::HTTP#read_timeout attr_accessor :read_timeout ## - # Seconds to wait until writing one block. See Net::HTTP#write_timeout + # Seconds to wait until writing one block. See Gem::Net::HTTP#write_timeout attr_accessor :write_timeout @@ -450,18 +450,18 @@ class Bundler::Persistent::Net::HTTP::Persistent attr_reader :verify_mode ## - # Creates a new Bundler::Persistent::Net::HTTP::Persistent. + # Creates a new Gem::Net::HTTP::Persistent. # # Set a +name+ for fun. Your library name should be good enough, but this # otherwise has no purpose. # - # +proxy+ may be set to a Bundler::URI::HTTP or :ENV to pick up proxy options from + # +proxy+ may be set to a Gem::URI::HTTP or :ENV to pick up proxy options from # the environment. See proxy_from_env for details. # - # In order to use a Bundler::URI for the proxy you may need to do some extra work - # beyond Bundler::URI parsing if the proxy requires a password: + # In order to use a Gem::URI for the proxy you may need to do some extra work + # beyond Gem::URI parsing if the proxy requires a password: # - # proxy = Bundler::URI 'http://proxy.example' + # proxy = Gem::URI 'http://proxy.example' # proxy.user = 'AzureDiamond' # proxy.password = 'hunter2' # @@ -492,8 +492,8 @@ class Bundler::Persistent::Net::HTTP::Persistent @socket_options << [Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1] if Socket.const_defined? :TCP_NODELAY - @pool = Bundler::Persistent::Net::HTTP::Persistent::Pool.new size: pool_size do |http_args| - Bundler::Persistent::Net::HTTP::Persistent::Connection.new Net::HTTP, http_args, @ssl_generation + @pool = Gem::Net::HTTP::Persistent::Pool.new size: pool_size do |http_args| + Gem::Net::HTTP::Persistent::Connection.new Gem::Net::HTTP, http_args, @ssl_generation end @certificate = nil @@ -510,7 +510,7 @@ class Bundler::Persistent::Net::HTTP::Persistent @verify_mode = nil @cert_store = nil - @generation = 0 # incremented when proxy Bundler::URI changes + @generation = 0 # incremented when proxy Gem::URI changes if HAVE_OPENSSL then @verify_mode = OpenSSL::SSL::VERIFY_PEER @@ -529,7 +529,7 @@ class Bundler::Persistent::Net::HTTP::Persistent reconnect_ssl end - # For Net::HTTP parity + # For Gem::Net::HTTP parity alias cert= certificate= ## @@ -648,7 +648,7 @@ class Bundler::Persistent::Net::HTTP::Persistent end ## - # Starts the Net::HTTP +connection+ + # Starts the Gem::Net::HTTP +connection+ def start http http.set_debug_output @debug_output if @debug_output @@ -666,7 +666,7 @@ class Bundler::Persistent::Net::HTTP::Persistent end ## - # Finishes the Net::HTTP +connection+ + # Finishes the Gem::Net::HTTP +connection+ def finish connection connection.finish @@ -716,16 +716,16 @@ class Bundler::Persistent::Net::HTTP::Persistent reconnect_ssl end - # For Net::HTTP parity + # For Gem::Net::HTTP parity alias key= private_key= ## - # Sets the proxy server. The +proxy+ may be the Bundler::URI of the proxy server, + # Sets the proxy server. The +proxy+ may be the Gem::URI of the proxy server, # the symbol +:ENV+ which will read the proxy from the environment or nil to # disable use of a proxy. See #proxy_from_env for details on setting the # proxy from the environment. # - # If the proxy Bundler::URI is set after requests have been made, the next request + # If the proxy Gem::URI is set after requests have been made, the next request # will shut-down and re-open all connections. # # The +no_proxy+ query parameter can be used to specify hosts which shouldn't @@ -736,9 +736,9 @@ class Bundler::Persistent::Net::HTTP::Persistent def proxy= proxy @proxy_uri = case proxy when :ENV then proxy_from_env - when Bundler::URI::HTTP then proxy + when Gem::URI::HTTP then proxy when nil then # ignore - else raise ArgumentError, 'proxy must be :ENV or a Bundler::URI::HTTP' + else raise ArgumentError, 'proxy must be :ENV or a Gem::URI::HTTP' end @no_proxy.clear @@ -763,13 +763,13 @@ class Bundler::Persistent::Net::HTTP::Persistent end ## - # Creates a Bundler::URI for an HTTP proxy server from ENV variables. + # Creates a Gem::URI for an HTTP proxy server from ENV variables. # # If +HTTP_PROXY+ is set a proxy will be returned. # - # If +HTTP_PROXY_USER+ or +HTTP_PROXY_PASS+ are set the Bundler::URI is given the + # If +HTTP_PROXY_USER+ or +HTTP_PROXY_PASS+ are set the Gem::URI is given the # indicated user and password unless HTTP_PROXY contains either of these in - # the Bundler::URI. + # the Gem::URI. # # The +NO_PROXY+ ENV variable can be used to specify hosts which shouldn't # be reached via proxy; if set it should be a comma separated list of @@ -785,7 +785,7 @@ class Bundler::Persistent::Net::HTTP::Persistent return nil if env_proxy.nil? or env_proxy.empty? - uri = Bundler::URI normalize_uri env_proxy + uri = Gem::URI normalize_uri env_proxy env_no_proxy = ENV['no_proxy'] || ENV['NO_PROXY'] @@ -835,7 +835,7 @@ class Bundler::Persistent::Net::HTTP::Persistent end ## - # Finishes then restarts the Net::HTTP +connection+ + # Finishes then restarts the Gem::Net::HTTP +connection+ def reset connection http = connection.http @@ -854,16 +854,16 @@ class Bundler::Persistent::Net::HTTP::Persistent end ## - # Makes a request on +uri+. If +req+ is nil a Net::HTTP::Get is performed + # Makes a request on +uri+. If +req+ is nil a Gem::Net::HTTP::Get is performed # against +uri+. # - # If a block is passed #request behaves like Net::HTTP#request (the body of + # If a block is passed #request behaves like Gem::Net::HTTP#request (the body of # the response will not have been read). # - # +req+ must be a Net::HTTPGenericRequest subclass (see Net::HTTP for a list). + # +req+ must be a Gem::Net::HTTPGenericRequest subclass (see Gem::Net::HTTP for a list). def request uri, req = nil, &block - uri = Bundler::URI uri + uri = Gem::URI uri req = request_setup req || uri response = nil @@ -896,14 +896,14 @@ class Bundler::Persistent::Net::HTTP::Persistent end ## - # Creates a GET request if +req_or_uri+ is a Bundler::URI and adds headers to the + # Creates a GET request if +req_or_uri+ is a Gem::URI and adds headers to the # request. # # Returns the request. def request_setup req_or_uri # :nodoc: req = if req_or_uri.respond_to? 'request_uri' then - Net::HTTP::Get.new req_or_uri.request_uri + Gem::Net::HTTP::Get.new req_or_uri.request_uri else req_or_uri end diff --git a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/connection.rb b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/connection.rb index a57a5d1352..8b9ab5cc78 100644 --- a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/connection.rb +++ b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/connection.rb @@ -1,8 +1,8 @@ ## -# A Net::HTTP connection wrapper that holds extra information for managing the +# A Gem::Net::HTTP connection wrapper that holds extra information for managing the # connection's lifetime. -class Bundler::Persistent::Net::HTTP::Persistent::Connection # :nodoc: +class Gem::Net::HTTP::Persistent::Connection # :nodoc: attr_accessor :http @@ -25,9 +25,10 @@ class Bundler::Persistent::Net::HTTP::Persistent::Connection # :nodoc: ensure reset end + alias_method :close, :finish def reset - @last_use = Bundler::Persistent::Net::HTTP::Persistent::EPOCH + @last_use = Gem::Net::HTTP::Persistent::EPOCH @requests = 0 end diff --git a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/pool.rb b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/pool.rb index 9dfa6ffdb1..04a1e754bf 100644 --- a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/pool.rb +++ b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/pool.rb @@ -1,4 +1,4 @@ -class Bundler::Persistent::Net::HTTP::Persistent::Pool < Bundler::ConnectionPool # :nodoc: +class Gem::Net::HTTP::Persistent::Pool < Bundler::ConnectionPool # :nodoc: attr_reader :available # :nodoc: attr_reader :key # :nodoc: @@ -6,25 +6,37 @@ class Bundler::Persistent::Net::HTTP::Persistent::Pool < Bundler::ConnectionPool def initialize(options = {}, &block) super - @available = Bundler::Persistent::Net::HTTP::Persistent::TimedStackMulti.new(@size, &block) + @available = Gem::Net::HTTP::Persistent::TimedStackMulti.new(@size, &block) @key = "current-#{@available.object_id}" end def checkin net_http_args - stack = Thread.current[@key][net_http_args] ||= [] + if net_http_args.is_a?(Hash) && net_http_args.size == 1 && net_http_args[:force] + # Bundler::ConnectionPool 2.4+ calls `checkin(force: true)` after fork. + # When this happens, we should remove all connections from Thread.current + if stacks = Thread.current[@key] + stacks.each do |http_args, connections| + connections.each do |conn| + @available.push conn, connection_args: http_args + end + connections.clear + end + end + else + stack = Thread.current[@key][net_http_args] ||= [] - raise Bundler::ConnectionPool::Error, 'no connections are checked out' if - stack.empty? + raise Bundler::ConnectionPool::Error, 'no connections are checked out' if + stack.empty? - conn = stack.pop + conn = stack.pop - if stack.empty? - @available.push conn, connection_args: net_http_args + if stack.empty? + @available.push conn, connection_args: net_http_args - Thread.current[@key].delete(net_http_args) - Thread.current[@key] = nil if Thread.current[@key].empty? + Thread.current[@key].delete(net_http_args) + Thread.current[@key] = nil if Thread.current[@key].empty? + end end - nil end diff --git a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi.rb b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi.rb index 2da881c554..214804fcd9 100644 --- a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi.rb +++ b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi.rb @@ -1,4 +1,4 @@ -class Bundler::Persistent::Net::HTTP::Persistent::TimedStackMulti < Bundler::ConnectionPool::TimedStack # :nodoc: +class Gem::Net::HTTP::Persistent::TimedStackMulti < Bundler::ConnectionPool::TimedStack # :nodoc: ## # Returns a new hash that has arrays for keys diff --git a/lib/bundler/vendor/pub_grub/.document b/lib/bundler/vendor/pub_grub/.document new file mode 100644 index 0000000000..0c43bbd6b3 --- /dev/null +++ b/lib/bundler/vendor/pub_grub/.document @@ -0,0 +1 @@ +# Vendored files do not need to be documented diff --git a/lib/bundler/vendor/pub_grub/lib/pub_grub/static_package_source.rb b/lib/bundler/vendor/pub_grub/lib/pub_grub/static_package_source.rb index 4bf61461b2..36ab06254d 100644 --- a/lib/bundler/vendor/pub_grub/lib/pub_grub/static_package_source.rb +++ b/lib/bundler/vendor/pub_grub/lib/pub_grub/static_package_source.rb @@ -1,4 +1,5 @@ require_relative 'package' +require_relative 'rubygems' require_relative 'version_constraint' require_relative 'incompatibility' require_relative 'basic_package_source' diff --git a/lib/bundler/vendor/pub_grub/lib/pub_grub/version_solver.rb b/lib/bundler/vendor/pub_grub/lib/pub_grub/version_solver.rb index 2cb8412cf3..4caf6b355b 100644 --- a/lib/bundler/vendor/pub_grub/lib/pub_grub/version_solver.rb +++ b/lib/bundler/vendor/pub_grub/lib/pub_grub/version_solver.rb @@ -162,7 +162,7 @@ module Bundler::PubGrub def resolve_conflict(incompatibility) logger.info { "conflict: #{incompatibility}" } - new_incompatibility = false + new_incompatibility = nil while !incompatibility.failure? most_recent_term = nil @@ -204,7 +204,7 @@ module Bundler::PubGrub solution.backtrack(previous_level) if new_incompatibility - add_incompatibility(incompatibility) + add_incompatibility(new_incompatibility) end return incompatibility @@ -219,9 +219,14 @@ module Bundler::PubGrub new_terms << difference.invert end - incompatibility = Incompatibility.new(new_terms, cause: Incompatibility::ConflictCause.new(incompatibility, most_recent_satisfier.cause)) + new_incompatibility = Incompatibility.new(new_terms, cause: Incompatibility::ConflictCause.new(incompatibility, most_recent_satisfier.cause)) - new_incompatibility = true + if incompatibility.to_s == new_incompatibility.to_s + logger.info { "!! failed to resolve conflicts, this shouldn't have happened" } + break + end + + incompatibility = new_incompatibility partially = difference ? " partially" : "" logger.info { "! #{most_recent_term} is#{partially} satisfied by #{most_recent_satisfier.term}" } diff --git a/lib/bundler/vendor/thor/.document b/lib/bundler/vendor/thor/.document new file mode 100644 index 0000000000..0c43bbd6b3 --- /dev/null +++ b/lib/bundler/vendor/thor/.document @@ -0,0 +1 @@ +# Vendored files do not need to be documented diff --git a/lib/bundler/vendor/thor/lib/thor.rb b/lib/bundler/vendor/thor/lib/thor.rb index 83ebe25593..627722164f 100644 --- a/lib/bundler/vendor/thor/lib/thor.rb +++ b/lib/bundler/vendor/thor/lib/thor.rb @@ -65,8 +65,15 @@ class Bundler::Thor # Defines the long description of the next command. # + # Long description is by default indented, line-wrapped and repeated whitespace merged. + # In order to print long description verbatim, with indentation and spacing exactly + # as found in the code, use the +wrap+ option + # + # long_desc 'your very long description', wrap: false + # # ==== Parameters # long description<String> + # options<Hash> # def long_desc(long_description, options = {}) if options[:for] @@ -74,6 +81,7 @@ class Bundler::Thor command.long_description = long_description if long_description else @long_desc = long_description + @long_desc_wrap = options[:wrap] != false end end @@ -133,7 +141,7 @@ class Bundler::Thor # # magic # end # - # method_option :foo => :bar, :for => :previous_command + # method_option :foo, :for => :previous_command # # def next_command # # magic @@ -153,6 +161,9 @@ class Bundler::Thor # :hide - If you want to hide this option from the help. # def method_option(name, options = {}) + unless [ Symbol, String ].any? { |klass| name.is_a?(klass) } + raise ArgumentError, "Expected a Symbol or String, got #{name.inspect}" + end scope = if options[:for] find_and_refresh_command(options[:for]).options else @@ -163,6 +174,81 @@ class Bundler::Thor end alias_method :option, :method_option + # Adds and declares option group for exclusive options in the + # block and arguments. You can declare options as the outside of the block. + # + # If :for is given as option, it allows you to change the options from + # a previous defined command. + # + # ==== Parameters + # Array[Bundler::Thor::Option.name] + # options<Hash>:: :for is applied for previous defined command. + # + # ==== Examples + # + # exclusive do + # option :one + # option :two + # end + # + # Or + # + # option :one + # option :two + # exclusive :one, :two + # + # If you give "--one" and "--two" at the same time ExclusiveArgumentsError + # will be raised. + # + def method_exclusive(*args, &block) + register_options_relation_for(:method_options, + :method_exclusive_option_names, *args, &block) + end + alias_method :exclusive, :method_exclusive + + # Adds and declares option group for required at least one of options in the + # block of arguments. You can declare options as the outside of the block. + # + # If :for is given as option, it allows you to change the options from + # a previous defined command. + # + # ==== Parameters + # Array[Bundler::Thor::Option.name] + # options<Hash>:: :for is applied for previous defined command. + # + # ==== Examples + # + # at_least_one do + # option :one + # option :two + # end + # + # Or + # + # option :one + # option :two + # at_least_one :one, :two + # + # If you do not give "--one" and "--two" AtLeastOneRequiredArgumentError + # will be raised. + # + # You can use at_least_one and exclusive at the same time. + # + # exclusive do + # at_least_one do + # option :one + # option :two + # end + # end + # + # Then it is required either only one of "--one" or "--two". + # + def method_at_least_one(*args, &block) + register_options_relation_for(:method_options, + :method_at_least_one_option_names, *args, &block) + end + alias_method :at_least_one, :method_at_least_one + # Prints help information for the given command. # # ==== Parameters @@ -178,9 +264,16 @@ class Bundler::Thor shell.say " #{banner(command).split("\n").join("\n ")}" shell.say class_options_help(shell, nil => command.options.values) + print_exclusive_options(shell, command) + print_at_least_one_required_options(shell, command) + if command.long_description shell.say "Description:" - shell.print_wrapped(command.long_description, :indent => 2) + if command.wrap_long_description + shell.print_wrapped(command.long_description, indent: 2) + else + shell.say command.long_description + end else shell.say command.description end @@ -197,7 +290,7 @@ class Bundler::Thor Bundler::Thor::Util.thor_classes_in(self).each do |klass| list += klass.printable_commands(false) end - list.sort! { |a, b| a[0] <=> b[0] } + sort_commands!(list) if defined?(@package_name) && @package_name shell.say "#{@package_name} commands:" @@ -205,9 +298,11 @@ class Bundler::Thor shell.say "Commands:" end - shell.print_table(list, :indent => 2, :truncate => true) + shell.print_table(list, indent: 2, truncate: true) shell.say class_options_help(shell) + print_exclusive_options(shell) + print_at_least_one_required_options(shell) end # Returns commands ready to be printed. @@ -238,7 +333,7 @@ class Bundler::Thor define_method(subcommand) do |*args| args, opts = Bundler::Thor::Arguments.split(args) - invoke_args = [args, opts, {:invoked_via_subcommand => true, :class_options => options}] + invoke_args = [args, opts, {invoked_via_subcommand: true, class_options: options}] invoke_args.unshift "help" if opts.delete("--help") || opts.delete("-h") invoke subcommand_class, *invoke_args end @@ -346,6 +441,24 @@ class Bundler::Thor protected + # Returns this class exclusive options array set. + # + # ==== Returns + # Array[Array[Bundler::Thor::Option.name]] + # + def method_exclusive_option_names #:nodoc: + @method_exclusive_option_names ||= [] + end + + # Returns this class at least one of required options array set. + # + # ==== Returns + # Array[Array[Bundler::Thor::Option.name]] + # + def method_at_least_one_option_names #:nodoc: + @method_at_least_one_option_names ||= [] + end + def stop_on_unknown_option #:nodoc: @stop_on_unknown_option ||= [] end @@ -355,6 +468,28 @@ class Bundler::Thor @disable_required_check ||= [:help] end + def print_exclusive_options(shell, command = nil) # :nodoc: + opts = [] + opts = command.method_exclusive_option_names unless command.nil? + opts += class_exclusive_option_names + unless opts.empty? + shell.say "Exclusive Options:" + shell.print_table(opts.map{ |ex| ex.map{ |e| "--#{e}"}}, indent: 2 ) + shell.say + end + end + + def print_at_least_one_required_options(shell, command = nil) # :nodoc: + opts = [] + opts = command.method_at_least_one_option_names unless command.nil? + opts += class_at_least_one_option_names + unless opts.empty? + shell.say "Required At Least One:" + shell.print_table(opts.map{ |ex| ex.map{ |e| "--#{e}"}}, indent: 2 ) + shell.say + end + end + # The method responsible for dispatching given the args. def dispatch(meth, given_args, given_opts, config) #:nodoc: meth ||= retrieve_command_name(given_args) @@ -415,12 +550,16 @@ class Bundler::Thor @usage ||= nil @desc ||= nil @long_desc ||= nil + @long_desc_wrap ||= nil @hide ||= nil if @usage && @desc base_class = @hide ? Bundler::Thor::HiddenCommand : Bundler::Thor::Command - commands[meth] = base_class.new(meth, @desc, @long_desc, @usage, method_options) - @usage, @desc, @long_desc, @method_options, @hide = nil + relations = {exclusive_option_names: method_exclusive_option_names, + at_least_one_option_names: method_at_least_one_option_names} + commands[meth] = base_class.new(meth, @desc, @long_desc, @long_desc_wrap, @usage, method_options, relations) + @usage, @desc, @long_desc, @long_desc_wrap, @method_options, @hide = nil + @method_exclusive_option_names, @method_at_least_one_option_names = nil true elsif all_commands[meth] || meth == "method_missing" true @@ -495,6 +634,14 @@ class Bundler::Thor " end alias_method :subtask_help, :subcommand_help + + # Sort the commands, lexicographically by default. + # + # Can be overridden in the subclass to change the display order of the + # commands. + def sort_commands!(list) + list.sort! { |a, b| a[0] <=> b[0] } + end end include Bundler::Thor::Base diff --git a/lib/bundler/vendor/thor/lib/thor/actions.rb b/lib/bundler/vendor/thor/lib/thor/actions.rb index a7afaf1d07..ca58182691 100644 --- a/lib/bundler/vendor/thor/lib/thor/actions.rb +++ b/lib/bundler/vendor/thor/lib/thor/actions.rb @@ -46,17 +46,17 @@ class Bundler::Thor # Add runtime options that help actions execution. # def add_runtime_options! - class_option :force, :type => :boolean, :aliases => "-f", :group => :runtime, - :desc => "Overwrite files that already exist" + class_option :force, type: :boolean, aliases: "-f", group: :runtime, + desc: "Overwrite files that already exist" - class_option :pretend, :type => :boolean, :aliases => "-p", :group => :runtime, - :desc => "Run but do not make any changes" + class_option :pretend, type: :boolean, aliases: "-p", group: :runtime, + desc: "Run but do not make any changes" - class_option :quiet, :type => :boolean, :aliases => "-q", :group => :runtime, - :desc => "Suppress status output" + class_option :quiet, type: :boolean, aliases: "-q", group: :runtime, + desc: "Suppress status output" - class_option :skip, :type => :boolean, :aliases => "-s", :group => :runtime, - :desc => "Skip files that already exist" + class_option :skip, type: :boolean, aliases: "-s", group: :runtime, + desc: "Skip files that already exist" end end @@ -113,9 +113,9 @@ class Bundler::Thor # def relative_to_original_destination_root(path, remove_dot = true) root = @destination_stack[0] - if path.start_with?(root) && [File::SEPARATOR, File::ALT_SEPARATOR, nil, ''].include?(path[root.size..root.size]) + if path.start_with?(root) && [File::SEPARATOR, File::ALT_SEPARATOR, nil, ""].include?(path[root.size..root.size]) path = path.dup - path[0...root.size] = '.' + path[0...root.size] = "." remove_dot ? (path[2..-1] || "") : path else path @@ -223,8 +223,7 @@ class Bundler::Thor contents = if is_uri require "open-uri" - # for ruby 2.1-2.4 - URI.send(:open, path, "Accept" => "application/x-thor-template", &:read) + URI.open(path, "Accept" => "application/x-thor-template", &:read) else File.open(path, &:read) end @@ -285,7 +284,7 @@ class Bundler::Thor # def run_ruby_script(command, config = {}) return unless behavior == :invoke - run command, config.merge(:with => Bundler::Thor::Util.ruby_command) + run command, config.merge(with: Bundler::Thor::Util.ruby_command) end # Run a thor command. A hash of options can be given and it's converted to @@ -316,7 +315,7 @@ class Bundler::Thor args.push Bundler::Thor::Options.to_switches(config) command = args.join(" ").strip - run command, :with => :thor, :verbose => verbose, :pretend => pretend, :capture => capture + run command, with: :thor, verbose: verbose, pretend: pretend, capture: capture end protected @@ -324,7 +323,7 @@ class Bundler::Thor # Allow current root to be shared between invocations. # def _shared_configuration #:nodoc: - super.merge!(:destination_root => destination_root) + super.merge!(destination_root: destination_root) end def _cleanup_options_and_set(options, key) #:nodoc: diff --git a/lib/bundler/vendor/thor/lib/thor/actions/create_file.rb b/lib/bundler/vendor/thor/lib/thor/actions/create_file.rb index 1b90e567ba..6724835b01 100644 --- a/lib/bundler/vendor/thor/lib/thor/actions/create_file.rb +++ b/lib/bundler/vendor/thor/lib/thor/actions/create_file.rb @@ -43,7 +43,8 @@ class Bundler::Thor # Boolean:: true if it is identical, false otherwise. # def identical? - exists? && File.binread(destination) == render + # binread uses ASCII-8BIT, so to avoid false negatives, the string must use the same + exists? && File.binread(destination) == String.new(render).force_encoding("ASCII-8BIT") end # Holds the content to be added to the file. diff --git a/lib/bundler/vendor/thor/lib/thor/actions/directory.rb b/lib/bundler/vendor/thor/lib/thor/actions/directory.rb index d37327a139..2f9687c0a5 100644 --- a/lib/bundler/vendor/thor/lib/thor/actions/directory.rb +++ b/lib/bundler/vendor/thor/lib/thor/actions/directory.rb @@ -58,7 +58,7 @@ class Bundler::Thor def initialize(base, source, destination = nil, config = {}, &block) @source = File.expand_path(Dir[Util.escape_globs(base.find_in_source_paths(source.to_s))].first) @block = block - super(base, destination, {:recursive => true}.merge(config)) + super(base, destination, {recursive: true}.merge(config)) end def invoke! diff --git a/lib/bundler/vendor/thor/lib/thor/actions/empty_directory.rb b/lib/bundler/vendor/thor/lib/thor/actions/empty_directory.rb index 284d92c19a..c0bca78525 100644 --- a/lib/bundler/vendor/thor/lib/thor/actions/empty_directory.rb +++ b/lib/bundler/vendor/thor/lib/thor/actions/empty_directory.rb @@ -33,7 +33,7 @@ class Bundler::Thor # def initialize(base, destination, config = {}) @base = base - @config = {:verbose => true}.merge(config) + @config = {verbose: true}.merge(config) self.destination = destination end diff --git a/lib/bundler/vendor/thor/lib/thor/actions/file_manipulation.rb b/lib/bundler/vendor/thor/lib/thor/actions/file_manipulation.rb index ffc29aa8e2..80a0255996 100644 --- a/lib/bundler/vendor/thor/lib/thor/actions/file_manipulation.rb +++ b/lib/bundler/vendor/thor/lib/thor/actions/file_manipulation.rb @@ -66,12 +66,15 @@ class Bundler::Thor # ==== Parameters # source<String>:: the address of the given content. # destination<String>:: the relative path to the destination root. - # config<Hash>:: give :verbose => false to not log the status. + # config<Hash>:: give :verbose => false to not log the status, and + # :http_headers => <Hash> to add headers to an http request. # # ==== Examples # # get "http://gist.github.com/103208", "doc/README" # + # get "http://gist.github.com/103208", "doc/README", :http_headers => {"Content-Type" => "application/json"} + # # get "http://gist.github.com/103208" do |content| # content.split("\n").first # end @@ -82,7 +85,7 @@ class Bundler::Thor render = if source =~ %r{^https?\://} require "open-uri" - URI.send(:open, source) { |input| input.binmode.read } + URI.send(:open, source, config.fetch(:http_headers, {})) { |input| input.binmode.read } else source = File.expand_path(find_in_source_paths(source.to_s)) File.open(source) { |input| input.binmode.read } @@ -120,12 +123,7 @@ class Bundler::Thor context = config.delete(:context) || instance_eval("binding") create_file destination, nil, config do - match = ERB.version.match(/(\d+\.\d+\.\d+)/) - capturable_erb = if match && match[1] >= "2.2.0" # Ruby 2.6+ - CapturableERB.new(::File.binread(source), :trim_mode => "-", :eoutvar => "@output_buffer") - else - CapturableERB.new(::File.binread(source), nil, "-", "@output_buffer") - end + capturable_erb = CapturableERB.new(::File.binread(source), trim_mode: "-", eoutvar: "@output_buffer") content = capturable_erb.tap do |erb| erb.filename = source end.result(context) diff --git a/lib/bundler/vendor/thor/lib/thor/actions/inject_into_file.rb b/lib/bundler/vendor/thor/lib/thor/actions/inject_into_file.rb index 6572f7a4f1..70526e615f 100644 --- a/lib/bundler/vendor/thor/lib/thor/actions/inject_into_file.rb +++ b/lib/bundler/vendor/thor/lib/thor/actions/inject_into_file.rb @@ -21,7 +21,7 @@ class Bundler::Thor # gems.split(" ").map{ |gem| " config.gem :#{gem}" }.join("\n") # end # - WARNINGS = { unchanged_no_flag: 'File unchanged! Either the supplied flag value not found or the content has already been inserted!' } + WARNINGS = {unchanged_no_flag: "File unchanged! Either the supplied flag value not found or the content has already been inserted!"} def insert_into_file(destination, *args, &block) data = block_given? ? block : args.shift @@ -37,7 +37,7 @@ class Bundler::Thor attr_reader :replacement, :flag, :behavior def initialize(base, destination, data, config) - super(base, destination, {:verbose => true}.merge(config)) + super(base, destination, {verbose: true}.merge(config)) @behavior, @flag = if @config.key?(:after) [:after, @config.delete(:after)] @@ -59,6 +59,8 @@ class Bundler::Thor if exists? if replace!(/#{flag}/, content, config[:force]) say_status(:invoke) + elsif replacement_present? + say_status(:unchanged, color: :blue) else say_status(:unchanged, warning: WARNINGS[:unchanged_no_flag], color: :red) end @@ -96,6 +98,8 @@ class Bundler::Thor end elsif warning warning + elsif behavior == :unchanged + :unchanged else :subtract end @@ -103,11 +107,18 @@ class Bundler::Thor super(status, (color || config[:verbose])) end + def content + @content ||= File.read(destination) + end + + def replacement_present? + content.include?(replacement) + end + # Adds the content to the file. # def replace!(regexp, string, force) - content = File.read(destination) - if force || !content.include?(replacement) + if force || !replacement_present? success = content.gsub!(regexp, string) File.open(destination, "wb") { |file| file.write(content) } unless pretend? diff --git a/lib/bundler/vendor/thor/lib/thor/base.rb b/lib/bundler/vendor/thor/lib/thor/base.rb index 211855680c..b156899c1e 100644 --- a/lib/bundler/vendor/thor/lib/thor/base.rb +++ b/lib/bundler/vendor/thor/lib/thor/base.rb @@ -24,9 +24,9 @@ class Bundler::Thor class << self def deprecation_warning(message) #:nodoc: - unless ENV['THOR_SILENCE_DEPRECATION'] + unless ENV["THOR_SILENCE_DEPRECATION"] warn "Deprecation warning: #{message}\n" + - 'You can silence deprecations warning by setting the environment variable THOR_SILENCE_DEPRECATION.' + "You can silence deprecations warning by setting the environment variable THOR_SILENCE_DEPRECATION." end end end @@ -60,6 +60,7 @@ class Bundler::Thor command_options = config.delete(:command_options) # hook for start parse_options = parse_options.merge(command_options) if command_options + if local_options.is_a?(Array) array_options = local_options hash_options = {} @@ -73,9 +74,24 @@ class Bundler::Thor # Let Bundler::Thor::Options parse the options first, so it can remove # declared options from the array. This will leave us with # a list of arguments that weren't declared. - stop_on_unknown = self.class.stop_on_unknown_option? config[:current_command] - disable_required_check = self.class.disable_required_check? config[:current_command] - opts = Bundler::Thor::Options.new(parse_options, hash_options, stop_on_unknown, disable_required_check) + current_command = config[:current_command] + stop_on_unknown = self.class.stop_on_unknown_option? current_command + + # Give a relation of options. + # After parsing, Bundler::Thor::Options check whether right relations are kept + relations = if current_command.nil? + {exclusive_option_names: [], at_least_one_option_names: []} + else + current_command.options_relation + end + + self.class.class_exclusive_option_names.map { |n| relations[:exclusive_option_names] << n } + self.class.class_at_least_one_option_names.map { |n| relations[:at_least_one_option_names] << n } + + disable_required_check = self.class.disable_required_check? current_command + + opts = Bundler::Thor::Options.new(parse_options, hash_options, stop_on_unknown, disable_required_check, relations) + self.options = opts.parse(array_options) self.options = config[:class_options].merge(options) if config[:class_options] @@ -310,9 +326,92 @@ class Bundler::Thor # :hide:: -- If you want to hide this option from the help. # def class_option(name, options = {}) + unless [ Symbol, String ].any? { |klass| name.is_a?(klass) } + raise ArgumentError, "Expected a Symbol or String, got #{name.inspect}" + end build_option(name, options, class_options) end + # Adds and declares option group for exclusive options in the + # block and arguments. You can declare options as the outside of the block. + # + # ==== Parameters + # Array[Bundler::Thor::Option.name] + # + # ==== Examples + # + # class_exclusive do + # class_option :one + # class_option :two + # end + # + # Or + # + # class_option :one + # class_option :two + # class_exclusive :one, :two + # + # If you give "--one" and "--two" at the same time ExclusiveArgumentsError + # will be raised. + # + def class_exclusive(*args, &block) + register_options_relation_for(:class_options, + :class_exclusive_option_names, *args, &block) + end + + # Adds and declares option group for required at least one of options in the + # block and arguments. You can declare options as the outside of the block. + # + # ==== Examples + # + # class_at_least_one do + # class_option :one + # class_option :two + # end + # + # Or + # + # class_option :one + # class_option :two + # class_at_least_one :one, :two + # + # If you do not give "--one" and "--two" AtLeastOneRequiredArgumentError + # will be raised. + # + # You can use class_at_least_one and class_exclusive at the same time. + # + # class_exclusive do + # class_at_least_one do + # class_option :one + # class_option :two + # end + # end + # + # Then it is required either only one of "--one" or "--two". + # + def class_at_least_one(*args, &block) + register_options_relation_for(:class_options, + :class_at_least_one_option_names, *args, &block) + end + + # Returns this class exclusive options array set, looking up in the ancestors chain. + # + # ==== Returns + # Array[Array[Bundler::Thor::Option.name]] + # + def class_exclusive_option_names + @class_exclusive_option_names ||= from_superclass(:class_exclusive_option_names, []) + end + + # Returns this class at least one of required options array set, looking up in the ancestors chain. + # + # ==== Returns + # Array[Array[Bundler::Thor::Option.name]] + # + def class_at_least_one_option_names + @class_at_least_one_option_names ||= from_superclass(:class_at_least_one_option_names, []) + end + # Removes a previous defined argument. If :undefine is given, undefine # accessors as well. # @@ -565,12 +664,12 @@ class Bundler::Thor item.push(option.description ? "# #{option.description}" : "") list << item - list << ["", "# Default: #{option.default}"] if option.show_default? - list << ["", "# Possible values: #{option.enum.join(', ')}"] if option.enum + list << ["", "# Default: #{option.print_default}"] if option.show_default? + list << ["", "# Possible values: #{option.enum_to_s}"] if option.enum end shell.say(group_name ? "#{group_name} options:" : "Options:") - shell.print_table(list, :indent => 2) + shell.print_table(list, indent: 2) shell.say "" end @@ -587,7 +686,7 @@ class Bundler::Thor # options<Hash>:: Described in both class_option and method_option. # scope<Hash>:: Options hash that is being built up def build_option(name, options, scope) #:nodoc: - scope[name] = Bundler::Thor::Option.new(name, {:check_default_type => check_default_type}.merge!(options)) + scope[name] = Bundler::Thor::Option.new(name, {check_default_type: check_default_type}.merge!(options)) end # Receives a hash of options, parse them and add to the scope. This is a @@ -693,6 +792,34 @@ class Bundler::Thor def dispatch(command, given_args, given_opts, config) #:nodoc: raise NotImplementedError end + + # Register a relation of options for target(method_option/class_option) + # by args and block. + def register_options_relation_for(target, relation, *args, &block) # :nodoc: + opt = args.pop if args.last.is_a? Hash + opt ||= {} + names = args.map{ |arg| arg.to_s } + names += built_option_names(target, opt, &block) if block_given? + command_scope_member(relation, opt) << names + end + + # Get target(method_options or class_options) options + # of before and after by block evaluation. + def built_option_names(target, opt = {}, &block) # :nodoc: + before = command_scope_member(target, opt).map{ |k,v| v.name } + instance_eval(&block) + after = command_scope_member(target, opt).map{ |k,v| v.name } + after - before + end + + # Get command scope member by name. + def command_scope_member(name, options = {}) # :nodoc: + if options[:for] + find_and_refresh_command(options[:for]).send(name) + else + send(name) + end + end end end end diff --git a/lib/bundler/vendor/thor/lib/thor/command.rb b/lib/bundler/vendor/thor/lib/thor/command.rb index 040c971c0c..68c8fffedb 100644 --- a/lib/bundler/vendor/thor/lib/thor/command.rb +++ b/lib/bundler/vendor/thor/lib/thor/command.rb @@ -1,14 +1,15 @@ class Bundler::Thor - class Command < Struct.new(:name, :description, :long_description, :usage, :options, :ancestor_name) + class Command < Struct.new(:name, :description, :long_description, :wrap_long_description, :usage, :options, :options_relation, :ancestor_name) FILE_REGEXP = /^#{Regexp.escape(File.dirname(__FILE__))}/ - def initialize(name, description, long_description, usage, options = nil) - super(name.to_s, description, long_description, usage, options || {}) + def initialize(name, description, long_description, wrap_long_description, usage, options = nil, options_relation = nil) + super(name.to_s, description, long_description, wrap_long_description, usage, options || {}, options_relation || {}) end def initialize_copy(other) #:nodoc: super(other) self.options = other.options.dup if other.options + self.options_relation = other.options_relation.dup if other.options_relation end def hidden? @@ -62,6 +63,14 @@ class Bundler::Thor end.join("\n") end + def method_exclusive_option_names #:nodoc: + self.options_relation[:exclusive_option_names] || [] + end + + def method_at_least_one_option_names #:nodoc: + self.options_relation[:at_least_one_option_names] || [] + end + protected # Add usage with required arguments @@ -127,7 +136,7 @@ class Bundler::Thor # A dynamic command that handles method missing scenarios. class DynamicCommand < Command def initialize(name, options = nil) - super(name.to_s, "A dynamically-generated command", name.to_s, name.to_s, options) + super(name.to_s, "A dynamically-generated command", name.to_s, nil, name.to_s, options) end def run(instance, args = []) diff --git a/lib/bundler/vendor/thor/lib/thor/core_ext/hash_with_indifferent_access.rb b/lib/bundler/vendor/thor/lib/thor/core_ext/hash_with_indifferent_access.rb index 3c4483e5dd..b16a98f782 100644 --- a/lib/bundler/vendor/thor/lib/thor/core_ext/hash_with_indifferent_access.rb +++ b/lib/bundler/vendor/thor/lib/thor/core_ext/hash_with_indifferent_access.rb @@ -38,6 +38,10 @@ class Bundler::Thor super(convert_key(key), *args) end + def slice(*keys) + super(*keys.map{ |key| convert_key(key) }) + end + def key?(key) super(convert_key(key)) end diff --git a/lib/bundler/vendor/thor/lib/thor/error.rb b/lib/bundler/vendor/thor/lib/thor/error.rb index dd64f750c9..928646e501 100644 --- a/lib/bundler/vendor/thor/lib/thor/error.rb +++ b/lib/bundler/vendor/thor/lib/thor/error.rb @@ -1,26 +1,15 @@ class Bundler::Thor Correctable = if defined?(DidYouMean::SpellChecker) && defined?(DidYouMean::Correctable) # rubocop:disable Naming/ConstantName - # In order to support versions of Ruby that don't have keyword - # arguments, we need our own spell checker class that doesn't take key - # words. Even though this code wouldn't be hit because of the check - # above, it's still necessary because the interpreter would otherwise be - # unable to parse the file. - class NoKwargSpellChecker < DidYouMean::SpellChecker # :nodoc: - def initialize(dictionary) - @dictionary = dictionary - end - end - - Module.new do - def to_s - super + DidYouMean.formatter.message_for(corrections) - end - - def corrections - @corrections ||= self.class.const_get(:SpellChecker).new(self).corrections - end - end - end + Module.new do + def to_s + super + DidYouMean.formatter.message_for(corrections) + end + + def corrections + @corrections ||= self.class.const_get(:SpellChecker).new(self).corrections + end + end + end # Bundler::Thor::Error is raised when it's caused by wrong usage of thor classes. Those # errors have their backtrace suppressed and are nicely shown to the user. @@ -45,7 +34,7 @@ class Bundler::Thor end def spell_checker - NoKwargSpellChecker.new(error.all_commands) + DidYouMean::SpellChecker.new(dictionary: error.all_commands) end end @@ -87,7 +76,7 @@ class Bundler::Thor end def spell_checker - @spell_checker ||= NoKwargSpellChecker.new(error.switches) + @spell_checker ||= DidYouMean::SpellChecker.new(dictionary: error.switches) end end @@ -108,4 +97,10 @@ class Bundler::Thor class MalformattedArgumentError < InvocationError end + + class ExclusiveArgumentError < InvocationError + end + + class AtLeastOneRequiredArgumentError < InvocationError + end end diff --git a/lib/bundler/vendor/thor/lib/thor/invocation.rb b/lib/bundler/vendor/thor/lib/thor/invocation.rb index 248a466f8e..5ce74710ba 100644 --- a/lib/bundler/vendor/thor/lib/thor/invocation.rb +++ b/lib/bundler/vendor/thor/lib/thor/invocation.rb @@ -143,7 +143,7 @@ class Bundler::Thor # Configuration values that are shared between invocations. def _shared_configuration #:nodoc: - {:invocations => @_invocations} + {invocations: @_invocations} end # This method simply retrieves the class and command to be invoked. diff --git a/lib/bundler/vendor/thor/lib/thor/nested_context.rb b/lib/bundler/vendor/thor/lib/thor/nested_context.rb index fd36b9d43f..7d60cb1c12 100644 --- a/lib/bundler/vendor/thor/lib/thor/nested_context.rb +++ b/lib/bundler/vendor/thor/lib/thor/nested_context.rb @@ -13,10 +13,10 @@ class Bundler::Thor end def entered? - @depth > 0 + @depth.positive? end - private + private def push @depth += 1 diff --git a/lib/bundler/vendor/thor/lib/thor/parser/argument.rb b/lib/bundler/vendor/thor/lib/thor/parser/argument.rb index dfe7398583..b9e94e4669 100644 --- a/lib/bundler/vendor/thor/lib/thor/parser/argument.rb +++ b/lib/bundler/vendor/thor/lib/thor/parser/argument.rb @@ -24,6 +24,17 @@ class Bundler::Thor validate! # Trigger specific validations end + def print_default + if @type == :array and @default.is_a?(Array) + @default.map { |x| + p = x.gsub('"','\\"') + "\"#{p}\"" + }.join(" ") + else + @default + end + end + def usage required? ? banner : "[#{banner}]" end @@ -41,11 +52,19 @@ class Bundler::Thor end end + def enum_to_s + if enum.respond_to? :join + enum.join(", ") + else + "#{enum.first}..#{enum.last}" + end + end + protected def validate! raise ArgumentError, "An argument cannot be required and have default value." if required? && !default.nil? - raise ArgumentError, "An argument cannot have an enum other than an array." if @enum && !@enum.is_a?(Array) + raise ArgumentError, "An argument cannot have an enum other than an enumerable." if @enum && !@enum.is_a?(Enumerable) end def valid_type?(type) diff --git a/lib/bundler/vendor/thor/lib/thor/parser/arguments.rb b/lib/bundler/vendor/thor/lib/thor/parser/arguments.rb index af395a0346..b6f9c9a37a 100644 --- a/lib/bundler/vendor/thor/lib/thor/parser/arguments.rb +++ b/lib/bundler/vendor/thor/lib/thor/parser/arguments.rb @@ -30,11 +30,7 @@ class Bundler::Thor arguments.each do |argument| if !argument.default.nil? - begin - @assigns[argument.human_name] = argument.default.dup - rescue TypeError # Compatibility shim for un-dup-able Fixnum in Ruby < 2.4 - @assigns[argument.human_name] = argument.default - end + @assigns[argument.human_name] = argument.default.dup elsif argument.required? @non_assigned_required << argument end @@ -121,8 +117,18 @@ class Bundler::Thor # def parse_array(name) return shift if peek.is_a?(Array) + array = [] - array << shift while current_is_value? + + while current_is_value? + value = shift + + if !value.empty? + validate_enum_value!(name, value, "Expected all values of '%s' to be one of %s; got %s") + end + + array << value + end array end @@ -138,11 +144,9 @@ class Bundler::Thor end value = $&.index(".") ? shift.to_f : shift.to_i - if @switches.is_a?(Hash) && switch = @switches[name] - if switch.enum && !switch.enum.include?(value) - raise MalformattedArgumentError, "Expected '#{name}' to be one of #{switch.enum.join(', ')}; got #{value}" - end - end + + validate_enum_value!(name, value, "Expected '%s' to be one of %s; got %s") + value end @@ -156,15 +160,27 @@ class Bundler::Thor nil else value = shift - if @switches.is_a?(Hash) && switch = @switches[name] - if switch.enum && !switch.enum.include?(value) - raise MalformattedArgumentError, "Expected '#{name}' to be one of #{switch.enum.join(', ')}; got #{value}" - end - end + + validate_enum_value!(name, value, "Expected '%s' to be one of %s; got %s") + value end end + # Raises an error if the switch is an enum and the values aren't included on it. + # + def validate_enum_value!(name, value, message) + return unless @switches.is_a?(Hash) + + switch = @switches[name] + + return unless switch + + if switch.enum && !switch.enum.include?(value) + raise MalformattedArgumentError, message % [name, switch.enum_to_s, value] + end + end + # Raises an error if @non_assigned_required array is not empty. # def check_requirement! diff --git a/lib/bundler/vendor/thor/lib/thor/parser/option.rb b/lib/bundler/vendor/thor/lib/thor/parser/option.rb index 393955f107..c6af4e1e87 100644 --- a/lib/bundler/vendor/thor/lib/thor/parser/option.rb +++ b/lib/bundler/vendor/thor/lib/thor/parser/option.rb @@ -11,7 +11,7 @@ class Bundler::Thor super @lazy_default = options[:lazy_default] @group = options[:group].to_s.capitalize if options[:group] - @aliases = Array(options[:aliases]) + @aliases = normalize_aliases(options[:aliases]) @hide = options[:hide] end @@ -69,7 +69,7 @@ class Bundler::Thor value.class.name.downcase.to_sym end - new(name.to_s, :required => required, :type => type, :default => default, :aliases => aliases) + new(name.to_s, required: required, type: type, default: default, aliases: aliases) end def switch_name @@ -90,7 +90,7 @@ class Bundler::Thor sample = "[#{sample}]".dup unless required? if boolean? - sample << ", [#{dasherize('no-' + human_name)}]" unless (name == "force") || name.start_with?("no-") + sample << ", [#{dasherize('no-' + human_name)}]" unless (name == "force") || name.match(/\Ano[\-_]/) end aliases_for_usage.ljust(padding) + sample @@ -104,6 +104,15 @@ class Bundler::Thor end end + def show_default? + case default + when TrueClass, FalseClass + true + else + super + end + end + VALID_TYPES.each do |type| class_eval <<-RUBY, __FILE__, __LINE__ + 1 def #{type}? @@ -142,8 +151,8 @@ class Bundler::Thor raise ArgumentError, err elsif @check_default_type == nil Bundler::Thor.deprecation_warning "#{err}.\n" + - 'This will be rejected in the future unless you explicitly pass the options `check_default_type: false`' + - ' or call `allow_incompatible_default_type!` in your code' + "This will be rejected in the future unless you explicitly pass the options `check_default_type: false`" + + " or call `allow_incompatible_default_type!` in your code" end end end @@ -159,5 +168,11 @@ class Bundler::Thor def dasherize(str) (str.length > 1 ? "--" : "-") + str.tr("_", "-") end + + private + + def normalize_aliases(aliases) + Array(aliases).map { |short| short.to_s.sub(/^(?!\-)/, "-") } + end end end diff --git a/lib/bundler/vendor/thor/lib/thor/parser/options.rb b/lib/bundler/vendor/thor/lib/thor/parser/options.rb index 499ce15339..978e76b132 100644 --- a/lib/bundler/vendor/thor/lib/thor/parser/options.rb +++ b/lib/bundler/vendor/thor/lib/thor/parser/options.rb @@ -29,8 +29,10 @@ class Bundler::Thor # # If +stop_on_unknown+ is true, #parse will stop as soon as it encounters # an unknown option or a regular argument. - def initialize(hash_options = {}, defaults = {}, stop_on_unknown = false, disable_required_check = false) + def initialize(hash_options = {}, defaults = {}, stop_on_unknown = false, disable_required_check = false, relations = {}) @stop_on_unknown = stop_on_unknown + @exclusives = (relations[:exclusive_option_names] || []).select{|array| !array.empty?} + @at_least_ones = (relations[:at_least_one_option_names] || []).select{|array| !array.empty?} @disable_required_check = disable_required_check options = hash_options.values super(options) @@ -50,8 +52,7 @@ class Bundler::Thor options.each do |option| @switches[option.switch_name] = option - option.aliases.each do |short| - name = short.to_s.sub(/^(?!\-)/, "-") + option.aliases.each do |name| @shorts[name] ||= option.switch_name end end @@ -101,7 +102,7 @@ class Bundler::Thor unshift($1.split("").map { |f| "-#{f}" }) next when EQ_RE - unshift($2, :is_value => true) + unshift($2, is_value: true) switch = $1 when SHORT_NUM unshift($2) @@ -132,12 +133,38 @@ class Bundler::Thor end check_requirement! unless @disable_required_check + check_exclusive! + check_at_least_one! assigns = Bundler::Thor::CoreExt::HashWithIndifferentAccess.new(@assigns) assigns.freeze assigns end + def check_exclusive! + opts = @assigns.keys + # When option A and B are exclusive, if A and B are given at the same time, + # the diffrence of argument array size will decrease. + found = @exclusives.find{ |ex| (ex - opts).size < ex.size - 1 } + if found + names = names_to_switch_names(found & opts).map{|n| "'#{n}'"} + class_name = self.class.name.split("::").last.downcase + fail ExclusiveArgumentError, "Found exclusive #{class_name} #{names.join(", ")}" + end + end + + def check_at_least_one! + opts = @assigns.keys + # When at least one is required of the options A and B, + # if the both options were not given, none? would be true. + found = @at_least_ones.find{ |one_reqs| one_reqs.none?{ |o| opts.include? o} } + if found + names = names_to_switch_names(found).map{|n| "'#{n}'"} + class_name = self.class.name.split("::").last.downcase + fail AtLeastOneRequiredArgumentError, "Not found at least one of required #{class_name} #{names.join(", ")}" + end + end + def check_unknown! to_check = @stopped_parsing_after_extra_index ? @extra[0...@stopped_parsing_after_extra_index] : @extra @@ -148,6 +175,17 @@ class Bundler::Thor protected + # Option names changes to swith name or human name + def names_to_switch_names(names = []) + @switches.map do |_, o| + if names.include? o.name + o.respond_to?(:switch_name) ? o.switch_name : o.human_name + else + nil + end + end.compact + end + def assign_result!(option, result) if option.repeatable && option.type == :hash (@assigns[option.human_name] ||= {}).merge!(result) diff --git a/lib/bundler/vendor/thor/lib/thor/runner.rb b/lib/bundler/vendor/thor/lib/thor/runner.rb index 8157c6c5b2..c7cc873131 100644 --- a/lib/bundler/vendor/thor/lib/thor/runner.rb +++ b/lib/bundler/vendor/thor/lib/thor/runner.rb @@ -23,7 +23,7 @@ class Bundler::Thor::Runner < Bundler::Thor #:nodoc: initialize_thorfiles(meth) klass, command = Bundler::Thor::Util.find_class_and_command_by_namespace(meth) self.class.handle_no_command_error(command, false) if klass.nil? - klass.start(["-h", command].compact, :shell => shell) + klass.start(["-h", command].compact, shell: shell) else super end @@ -38,11 +38,11 @@ class Bundler::Thor::Runner < Bundler::Thor #:nodoc: klass, command = Bundler::Thor::Util.find_class_and_command_by_namespace(meth) self.class.handle_no_command_error(command, false) if klass.nil? args.unshift(command) if command - klass.start(args, :shell => shell) + klass.start(args, shell: shell) end desc "install NAME", "Install an optionally named Bundler::Thor file into your system commands" - method_options :as => :string, :relative => :boolean, :force => :boolean + method_options as: :string, relative: :boolean, force: :boolean def install(name) # rubocop:disable Metrics/MethodLength initialize_thorfiles @@ -53,7 +53,7 @@ class Bundler::Thor::Runner < Bundler::Thor #:nodoc: package = :file require "open-uri" begin - contents = URI.send(:open, name, &:read) # Using `send` for Ruby 2.4- support + contents = URI.open(name, &:read) rescue OpenURI::HTTPError raise Error, "Error opening URI '#{name}'" end @@ -69,7 +69,7 @@ class Bundler::Thor::Runner < Bundler::Thor #:nodoc: base = name package = :file require "open-uri" - contents = URI.send(:open, name, &:read) # for ruby 2.1-2.4 + contents = URI.open(name, &:read) end rescue Errno::ENOENT raise Error, "Error opening file '#{name}'" @@ -101,9 +101,9 @@ class Bundler::Thor::Runner < Bundler::Thor #:nodoc: end thor_yaml[as] = { - :filename => Digest::SHA256.hexdigest(name + as), - :location => location, - :namespaces => Bundler::Thor::Util.namespaces_in_content(contents, base) + filename: Digest::SHA256.hexdigest(name + as), + location: location, + namespaces: Bundler::Thor::Util.namespaces_in_content(contents, base) } save_yaml(thor_yaml) @@ -164,14 +164,14 @@ class Bundler::Thor::Runner < Bundler::Thor #:nodoc: end desc "installed", "List the installed Bundler::Thor modules and commands" - method_options :internal => :boolean + method_options internal: :boolean def installed initialize_thorfiles(nil, true) display_klasses(true, options["internal"]) end desc "list [SEARCH]", "List the available thor commands (--substring means .*SEARCH)" - method_options :substring => :boolean, :group => :string, :all => :boolean, :debug => :boolean + method_options substring: :boolean, group: :string, all: :boolean, debug: :boolean def list(search = "") initialize_thorfiles @@ -313,7 +313,7 @@ private say shell.set_color(namespace, :blue, true) say "-" * namespace.size - print_table(list, :truncate => true) + print_table(list, truncate: true) say end alias_method :display_tasks, :display_commands diff --git a/lib/bundler/vendor/thor/lib/thor/shell.rb b/lib/bundler/vendor/thor/lib/thor/shell.rb index a4137d1bde..265f3ba046 100644 --- a/lib/bundler/vendor/thor/lib/thor/shell.rb +++ b/lib/bundler/vendor/thor/lib/thor/shell.rb @@ -75,7 +75,7 @@ class Bundler::Thor # Allow shell to be shared between invocations. # def _shared_configuration #:nodoc: - super.merge!(:shell => shell) + super.merge!(shell: shell) end end end diff --git a/lib/bundler/vendor/thor/lib/thor/shell/basic.rb b/lib/bundler/vendor/thor/lib/thor/shell/basic.rb index 7f414e9a08..dc3179e5f3 100644 --- a/lib/bundler/vendor/thor/lib/thor/shell/basic.rb +++ b/lib/bundler/vendor/thor/lib/thor/shell/basic.rb @@ -1,8 +1,10 @@ +require_relative "column_printer" +require_relative "table_printer" +require_relative "wrapped_printer" + class Bundler::Thor module Shell class Basic - DEFAULT_TERMINAL_WIDTH = 80 - attr_accessor :base attr_reader :padding @@ -145,14 +147,14 @@ class Bundler::Thor # "yes". # def yes?(statement, color = nil) - !!(ask(statement, color, :add_to_history => false) =~ is?(:yes)) + !!(ask(statement, color, add_to_history: false) =~ is?(:yes)) end # Make a question the to user and returns true if the user replies "n" or # "no". # def no?(statement, color = nil) - !!(ask(statement, color, :add_to_history => false) =~ is?(:no)) + !!(ask(statement, color, add_to_history: false) =~ is?(:no)) end # Prints values in columns @@ -161,16 +163,8 @@ class Bundler::Thor # Array[String, String, ...] # def print_in_columns(array) - return if array.empty? - colwidth = (array.map { |el| el.to_s.size }.max || 0) + 2 - array.each_with_index do |value, index| - # Don't output trailing spaces when printing the last column - if ((((index + 1) % (terminal_width / colwidth))).zero? && !index.zero?) || index + 1 == array.length - stdout.puts value - else - stdout.printf("%-#{colwidth}s", value) - end - end + printer = ColumnPrinter.new(stdout) + printer.print(array) end # Prints a table. @@ -181,58 +175,11 @@ class Bundler::Thor # ==== Options # indent<Integer>:: Indent the first column by indent value. # colwidth<Integer>:: Force the first column to colwidth spaces wide. + # borders<Boolean>:: Adds ascii borders. # def print_table(array, options = {}) # rubocop:disable Metrics/MethodLength - return if array.empty? - - formats = [] - indent = options[:indent].to_i - colwidth = options[:colwidth] - options[:truncate] = terminal_width if options[:truncate] == true - - formats << "%-#{colwidth + 2}s".dup if colwidth - start = colwidth ? 1 : 0 - - colcount = array.max { |a, b| a.size <=> b.size }.size - - maximas = [] - - start.upto(colcount - 1) do |index| - maxima = array.map { |row| row[index] ? row[index].to_s.size : 0 }.max - maximas << maxima - formats << if index == colcount - 1 - # Don't output 2 trailing spaces when printing the last column - "%-s".dup - else - "%-#{maxima + 2}s".dup - end - end - - formats[0] = formats[0].insert(0, " " * indent) - formats << "%s" - - array.each do |row| - sentence = "".dup - - row.each_with_index do |column, index| - maxima = maximas[index] - - f = if column.is_a?(Numeric) - if index == row.size - 1 - # Don't output 2 trailing spaces when printing the last column - "%#{maxima}s" - else - "%#{maxima}s " - end - else - formats[index] - end - sentence << f % column.to_s - end - - sentence = truncate(sentence, options[:truncate]) if options[:truncate] - stdout.puts sentence - end + printer = TablePrinter.new(stdout, options) + printer.print(array) end # Prints a long string, word-wrapping the text to the current width of the @@ -245,33 +192,8 @@ class Bundler::Thor # indent<Integer>:: Indent each line of the printed paragraph by indent value. # def print_wrapped(message, options = {}) - indent = options[:indent] || 0 - width = terminal_width - indent - paras = message.split("\n\n") - - paras.map! do |unwrapped| - words = unwrapped.split(" ") - counter = words.first.length - words.inject do |memo, word| - word = word.gsub(/\n\005/, "\n").gsub(/\005/, "\n") - counter = 0 if word.include? "\n" - if (counter + word.length + 1) < width - memo = "#{memo} #{word}" - counter += (word.length + 1) - else - memo = "#{memo}\n#{word}" - counter = word.length - end - memo - end - end.compact! - - paras.each do |para| - para.split("\n").each do |line| - stdout.puts line.insert(0, " " * indent) - end - stdout.puts unless para == paras.last - end + printer = WrappedPrinter.new(stdout, options) + printer.print(message) end # Deals with file collision and returns true if the file should be @@ -289,7 +211,7 @@ class Bundler::Thor loop do answer = ask( %[Overwrite #{destination}? (enter "h" for help) #{options}], - :add_to_history => false + add_to_history: false ) case answer @@ -316,24 +238,11 @@ class Bundler::Thor say "Please specify merge tool to `THOR_MERGE` env." else - say file_collision_help + say file_collision_help(block_given?) end end end - # This code was copied from Rake, available under MIT-LICENSE - # Copyright (c) 2003, 2004 Jim Weirich - def terminal_width - result = if ENV["THOR_COLUMNS"] - ENV["THOR_COLUMNS"].to_i - else - unix? ? dynamic_width : DEFAULT_TERMINAL_WIDTH - end - result < 10 ? DEFAULT_TERMINAL_WIDTH : result - rescue - DEFAULT_TERMINAL_WIDTH - end - # Called if something goes wrong during the execution. This is used by Bundler::Thor # internally and should not be used inside your scripts. If something went # wrong, you can always raise an exception. If you raise a Bundler::Thor::Error, it @@ -384,16 +293,21 @@ class Bundler::Thor end end - def file_collision_help #:nodoc: - <<-HELP + def file_collision_help(block_given) #:nodoc: + help = <<-HELP Y - yes, overwrite n - no, do not overwrite a - all, overwrite this and all others q - quit, abort - d - diff, show the differences between the old and the new h - help, show this help - m - merge, run merge tool HELP + if block_given + help << <<-HELP + d - diff, show the differences between the old and the new + m - merge, run merge tool + HELP + end + help end def show_diff(destination, content) #:nodoc: @@ -411,46 +325,8 @@ class Bundler::Thor mute? || (base && base.options[:quiet]) end - # Calculate the dynamic width of the terminal - def dynamic_width - @dynamic_width ||= (dynamic_width_stty.nonzero? || dynamic_width_tput) - end - - def dynamic_width_stty - `stty size 2>/dev/null`.split[1].to_i - end - - def dynamic_width_tput - `tput cols 2>/dev/null`.to_i - end - def unix? - RUBY_PLATFORM =~ /(aix|darwin|linux|(net|free|open)bsd|cygwin|solaris)/i - end - - def truncate(string, width) - as_unicode do - chars = string.chars.to_a - if chars.length <= width - chars.join - else - chars[0, width - 3].join + "..." - end - end - end - - if "".respond_to?(:encode) - def as_unicode - yield - end - else - def as_unicode - old = $KCODE - $KCODE = "U" - yield - ensure - $KCODE = old - end + Terminal.unix? end def ask_simply(statement, color, options) diff --git a/lib/bundler/vendor/thor/lib/thor/shell/color.rb b/lib/bundler/vendor/thor/lib/thor/shell/color.rb index ccb7c3c079..5d708fadca 100644 --- a/lib/bundler/vendor/thor/lib/thor/shell/color.rb +++ b/lib/bundler/vendor/thor/lib/thor/shell/color.rb @@ -105,52 +105,7 @@ class Bundler::Thor end def are_colors_disabled? - !ENV['NO_COLOR'].nil? && !ENV['NO_COLOR'].empty? - end - - # Overwrite show_diff to show diff with colors if Diff::LCS is - # available. - # - def show_diff(destination, content) #:nodoc: - if diff_lcs_loaded? && ENV["THOR_DIFF"].nil? && ENV["RAILS_DIFF"].nil? - actual = File.binread(destination).to_s.split("\n") - content = content.to_s.split("\n") - - Diff::LCS.sdiff(actual, content).each do |diff| - output_diff_line(diff) - end - else - super - end - end - - def output_diff_line(diff) #:nodoc: - case diff.action - when "-" - say "- #{diff.old_element.chomp}", :red, true - when "+" - say "+ #{diff.new_element.chomp}", :green, true - when "!" - say "- #{diff.old_element.chomp}", :red, true - say "+ #{diff.new_element.chomp}", :green, true - else - say " #{diff.old_element.chomp}", nil, true - end - end - - # Check if Diff::LCS is loaded. If it is, use it to create pretty output - # for diff. - # - def diff_lcs_loaded? #:nodoc: - return true if defined?(Diff::LCS) - return @diff_lcs_loaded unless @diff_lcs_loaded.nil? - - @diff_lcs_loaded = begin - require "diff/lcs" - true - rescue LoadError - false - end + !ENV["NO_COLOR"].nil? && !ENV["NO_COLOR"].empty? end end end diff --git a/lib/bundler/vendor/thor/lib/thor/shell/column_printer.rb b/lib/bundler/vendor/thor/lib/thor/shell/column_printer.rb new file mode 100644 index 0000000000..56a9e6181b --- /dev/null +++ b/lib/bundler/vendor/thor/lib/thor/shell/column_printer.rb @@ -0,0 +1,29 @@ +require_relative "terminal" + +class Bundler::Thor + module Shell + class ColumnPrinter + attr_reader :stdout, :options + + def initialize(stdout, options = {}) + @stdout = stdout + @options = options + @indent = options[:indent].to_i + end + + def print(array) + return if array.empty? + colwidth = (array.map { |el| el.to_s.size }.max || 0) + 2 + array.each_with_index do |value, index| + # Don't output trailing spaces when printing the last column + if ((((index + 1) % (Terminal.terminal_width / colwidth))).zero? && !index.zero?) || index + 1 == array.length + stdout.puts value + else + stdout.printf("%-#{colwidth}s", value) + end + end + end + end + end +end + diff --git a/lib/bundler/vendor/thor/lib/thor/shell/html.rb b/lib/bundler/vendor/thor/lib/thor/shell/html.rb index 77a6d13a23..0277b882b7 100644 --- a/lib/bundler/vendor/thor/lib/thor/shell/html.rb +++ b/lib/bundler/vendor/thor/lib/thor/shell/html.rb @@ -76,51 +76,6 @@ class Bundler::Thor def can_display_colors? true end - - # Overwrite show_diff to show diff with colors if Diff::LCS is - # available. - # - def show_diff(destination, content) #:nodoc: - if diff_lcs_loaded? && ENV["THOR_DIFF"].nil? && ENV["RAILS_DIFF"].nil? - actual = File.binread(destination).to_s.split("\n") - content = content.to_s.split("\n") - - Diff::LCS.sdiff(actual, content).each do |diff| - output_diff_line(diff) - end - else - super - end - end - - def output_diff_line(diff) #:nodoc: - case diff.action - when "-" - say "- #{diff.old_element.chomp}", :red, true - when "+" - say "+ #{diff.new_element.chomp}", :green, true - when "!" - say "- #{diff.old_element.chomp}", :red, true - say "+ #{diff.new_element.chomp}", :green, true - else - say " #{diff.old_element.chomp}", nil, true - end - end - - # Check if Diff::LCS is loaded. If it is, use it to create pretty output - # for diff. - # - def diff_lcs_loaded? #:nodoc: - return true if defined?(Diff::LCS) - return @diff_lcs_loaded unless @diff_lcs_loaded.nil? - - @diff_lcs_loaded = begin - require "diff/lcs" - true - rescue LoadError - false - end - end end end end diff --git a/lib/bundler/vendor/thor/lib/thor/shell/table_printer.rb b/lib/bundler/vendor/thor/lib/thor/shell/table_printer.rb new file mode 100644 index 0000000000..525f9ce5bb --- /dev/null +++ b/lib/bundler/vendor/thor/lib/thor/shell/table_printer.rb @@ -0,0 +1,134 @@ +require_relative "column_printer" +require_relative "terminal" + +class Bundler::Thor + module Shell + class TablePrinter < ColumnPrinter + BORDER_SEPARATOR = :separator + + def initialize(stdout, options = {}) + super + @formats = [] + @maximas = [] + @colwidth = options[:colwidth] + @truncate = options[:truncate] == true ? Terminal.terminal_width : options[:truncate] + @padding = 1 + end + + def print(array) + return if array.empty? + + prepare(array) + + print_border_separator if options[:borders] + + array.each do |row| + if options[:borders] && row == BORDER_SEPARATOR + print_border_separator + next + end + + sentence = "".dup + + row.each_with_index do |column, index| + sentence << format_cell(column, row.size, index) + end + + sentence = truncate(sentence) + sentence << "|" if options[:borders] + stdout.puts indentation + sentence + + end + print_border_separator if options[:borders] + end + + private + + def prepare(array) + array = array.reject{|row| row == BORDER_SEPARATOR } + + @formats << "%-#{@colwidth + 2}s".dup if @colwidth + start = @colwidth ? 1 : 0 + + colcount = array.max { |a, b| a.size <=> b.size }.size + + start.upto(colcount - 1) do |index| + maxima = array.map { |row| row[index] ? row[index].to_s.size : 0 }.max + + @maximas << maxima + @formats << if options[:borders] + "%-#{maxima}s".dup + elsif index == colcount - 1 + # Don't output 2 trailing spaces when printing the last column + "%-s".dup + else + "%-#{maxima + 2}s".dup + end + end + + @formats << "%s" + end + + def format_cell(column, row_size, index) + maxima = @maximas[index] + + f = if column.is_a?(Numeric) + if options[:borders] + # With borders we handle padding separately + "%#{maxima}s" + elsif index == row_size - 1 + # Don't output 2 trailing spaces when printing the last column + "%#{maxima}s" + else + "%#{maxima}s " + end + else + @formats[index] + end + + cell = "".dup + cell << "|" + " " * @padding if options[:borders] + cell << f % column.to_s + cell << " " * @padding if options[:borders] + cell + end + + def print_border_separator + separator = @maximas.map do |maxima| + "+" + "-" * (maxima + 2 * @padding) + end + stdout.puts indentation + separator.join + "+" + end + + def truncate(string) + return string unless @truncate + as_unicode do + chars = string.chars.to_a + if chars.length <= @truncate + chars.join + else + chars[0, @truncate - 3 - @indent].join + "..." + end + end + end + + def indentation + " " * @indent + end + + if "".respond_to?(:encode) + def as_unicode + yield + end + else + def as_unicode + old = $KCODE # rubocop:disable Style/GlobalVars + $KCODE = "U" # rubocop:disable Style/GlobalVars + yield + ensure + $KCODE = old # rubocop:disable Style/GlobalVars + end + end + end + end +end diff --git a/lib/bundler/vendor/thor/lib/thor/shell/terminal.rb b/lib/bundler/vendor/thor/lib/thor/shell/terminal.rb new file mode 100644 index 0000000000..2c60684308 --- /dev/null +++ b/lib/bundler/vendor/thor/lib/thor/shell/terminal.rb @@ -0,0 +1,42 @@ +class Bundler::Thor + module Shell + module Terminal + DEFAULT_TERMINAL_WIDTH = 80 + + class << self + # This code was copied from Rake, available under MIT-LICENSE + # Copyright (c) 2003, 2004 Jim Weirich + def terminal_width + result = if ENV["THOR_COLUMNS"] + ENV["THOR_COLUMNS"].to_i + else + unix? ? dynamic_width : DEFAULT_TERMINAL_WIDTH + end + result < 10 ? DEFAULT_TERMINAL_WIDTH : result + rescue + DEFAULT_TERMINAL_WIDTH + end + + def unix? + RUBY_PLATFORM =~ /(aix|darwin|linux|(net|free|open)bsd|cygwin|solaris)/i + end + + private + + # Calculate the dynamic width of the terminal + def dynamic_width + @dynamic_width ||= (dynamic_width_stty.nonzero? || dynamic_width_tput) + end + + def dynamic_width_stty + `stty size 2>/dev/null`.split[1].to_i + end + + def dynamic_width_tput + `tput cols 2>/dev/null`.to_i + end + + end + end + end +end diff --git a/lib/bundler/vendor/thor/lib/thor/shell/wrapped_printer.rb b/lib/bundler/vendor/thor/lib/thor/shell/wrapped_printer.rb new file mode 100644 index 0000000000..ba88e952db --- /dev/null +++ b/lib/bundler/vendor/thor/lib/thor/shell/wrapped_printer.rb @@ -0,0 +1,38 @@ +require_relative "column_printer" +require_relative "terminal" + +class Bundler::Thor + module Shell + class WrappedPrinter < ColumnPrinter + def print(message) + width = Terminal.terminal_width - @indent + paras = message.split("\n\n") + + paras.map! do |unwrapped| + words = unwrapped.split(" ") + counter = words.first.length + words.inject do |memo, word| + word = word.gsub(/\n\005/, "\n").gsub(/\005/, "\n") + counter = 0 if word.include? "\n" + if (counter + word.length + 1) < width + memo = "#{memo} #{word}" + counter += (word.length + 1) + else + memo = "#{memo}\n#{word}" + counter = word.length + end + memo + end + end.compact! + + paras.each do |para| + para.split("\n").each do |line| + stdout.puts line.insert(0, " " * @indent) + end + stdout.puts unless para == paras.last + end + end + end + end +end + diff --git a/lib/bundler/vendor/thor/lib/thor/util.rb b/lib/bundler/vendor/thor/lib/thor/util.rb index a17eabc22c..68916daf2e 100644 --- a/lib/bundler/vendor/thor/lib/thor/util.rb +++ b/lib/bundler/vendor/thor/lib/thor/util.rb @@ -130,9 +130,10 @@ class Bundler::Thor # def find_class_and_command_by_namespace(namespace, fallback = true) if namespace.include?(":") # look for a namespaced command - pieces = namespace.split(":") - command = pieces.pop - klass = Bundler::Thor::Util.find_by_namespace(pieces.join(":")) + *pieces, command = namespace.split(":") + namespace = pieces.join(":") + namespace = "default" if namespace.empty? + klass = Bundler::Thor::Base.subclasses.detect { |thor| thor.namespace == namespace && thor.commands.keys.include?(command) } end unless klass # look for a Bundler::Thor::Group with the right name klass = Bundler::Thor::Util.find_by_namespace(namespace) diff --git a/lib/bundler/vendor/thor/lib/thor/version.rb b/lib/bundler/vendor/thor/lib/thor/version.rb index 43e462c531..1fb00017ed 100644 --- a/lib/bundler/vendor/thor/lib/thor/version.rb +++ b/lib/bundler/vendor/thor/lib/thor/version.rb @@ -1,3 +1,3 @@ class Bundler::Thor - VERSION = "1.2.2" + VERSION = "1.3.0" end diff --git a/lib/bundler/vendor/tsort/.document b/lib/bundler/vendor/tsort/.document new file mode 100644 index 0000000000..0c43bbd6b3 --- /dev/null +++ b/lib/bundler/vendor/tsort/.document @@ -0,0 +1 @@ +# Vendored files do not need to be documented diff --git a/lib/bundler/vendor/tsort/lib/tsort.rb b/lib/bundler/vendor/tsort/lib/tsort.rb index 4a0e1a4e25..cf8731f760 100644 --- a/lib/bundler/vendor/tsort/lib/tsort.rb +++ b/lib/bundler/vendor/tsort/lib/tsort.rb @@ -122,6 +122,9 @@ # module Bundler::TSort + + VERSION = "0.2.0" + class Cyclic < StandardError end diff --git a/lib/bundler/vendor/uri/.document b/lib/bundler/vendor/uri/.document new file mode 100644 index 0000000000..0c43bbd6b3 --- /dev/null +++ b/lib/bundler/vendor/uri/.document @@ -0,0 +1 @@ +# Vendored files do not need to be documented diff --git a/lib/bundler/vendor/uri/lib/uri/common.rb b/lib/bundler/vendor/uri/lib/uri/common.rb index 914a4c7581..93f4f226ad 100644 --- a/lib/bundler/vendor/uri/lib/uri/common.rb +++ b/lib/bundler/vendor/uri/lib/uri/common.rb @@ -68,16 +68,32 @@ module Bundler::URI end private_constant :Schemes + # Registers the given +klass+ as the class to be instantiated + # when parsing a \Bundler::URI with the given +scheme+: # - # Register the given +klass+ to be instantiated when parsing URLs with the given +scheme+. - # Note that currently only schemes which after .upcase are valid constant names - # can be registered (no -/+/. allowed). + # Bundler::URI.register_scheme('MS_SEARCH', Bundler::URI::Generic) # => Bundler::URI::Generic + # Bundler::URI.scheme_list['MS_SEARCH'] # => Bundler::URI::Generic # + # Note that after calling String#upcase on +scheme+, it must be a valid + # constant name. def self.register_scheme(scheme, klass) Schemes.const_set(scheme.to_s.upcase, klass) end - # Returns a Hash of the defined schemes. + # Returns a hash of the defined schemes: + # + # Bundler::URI.scheme_list + # # => + # {"MAILTO"=>Bundler::URI::MailTo, + # "LDAPS"=>Bundler::URI::LDAPS, + # "WS"=>Bundler::URI::WS, + # "HTTP"=>Bundler::URI::HTTP, + # "HTTPS"=>Bundler::URI::HTTPS, + # "LDAP"=>Bundler::URI::LDAP, + # "FILE"=>Bundler::URI::File, + # "FTP"=>Bundler::URI::FTP} + # + # Related: Bundler::URI.register_scheme. def self.scheme_list Schemes.constants.map { |name| [name.to_s.upcase, Schemes.const_get(name)] @@ -88,9 +104,21 @@ module Bundler::URI private_constant :INITIAL_SCHEMES Ractor.make_shareable(INITIAL_SCHEMES) if defined?(Ractor) + # Returns a new object constructed from the given +scheme+, +arguments+, + # and +default+: + # + # - The new object is an instance of <tt>Bundler::URI.scheme_list[scheme.upcase]</tt>. + # - The object is initialized by calling the class initializer + # using +scheme+ and +arguments+. + # See Bundler::URI::Generic.new. + # + # Examples: # - # Construct a Bundler::URI instance, using the scheme to detect the appropriate class - # from +Bundler::URI.scheme_list+. + # values = ['john.doe', 'www.example.com', '123', nil, '/forum/questions/', nil, 'tag=networking&order=newest', 'top'] + # Bundler::URI.for('https', *values) + # # => #<Bundler::URI::HTTPS https://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top> + # Bundler::URI.for('foo', *values, default: Bundler::URI::HTTP) + # # => #<Bundler::URI::HTTP foo://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top> # def self.for(scheme, *arguments, default: Generic) const_name = scheme.to_s.upcase @@ -121,95 +149,49 @@ module Bundler::URI # class BadURIError < Error; end - # - # == Synopsis - # - # Bundler::URI::split(uri) - # - # == Args - # - # +uri+:: - # String with Bundler::URI. - # - # == Description - # - # Splits the string on following parts and returns array with result: - # - # * Scheme - # * Userinfo - # * Host - # * Port - # * Registry - # * Path - # * Opaque - # * Query - # * Fragment - # - # == Usage - # - # require 'bundler/vendor/uri/lib/uri' - # - # Bundler::URI.split("http://www.ruby-lang.org/") - # # => ["http", nil, "www.ruby-lang.org", nil, nil, "/", nil, nil, nil] + # Returns a 9-element array representing the parts of the \Bundler::URI + # formed from the string +uri+; + # each array element is a string or +nil+: + # + # names = %w[scheme userinfo host port registry path opaque query fragment] + # values = Bundler::URI.split('https://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top') + # names.zip(values) + # # => + # [["scheme", "https"], + # ["userinfo", "john.doe"], + # ["host", "www.example.com"], + # ["port", "123"], + # ["registry", nil], + # ["path", "/forum/questions/"], + # ["opaque", nil], + # ["query", "tag=networking&order=newest"], + # ["fragment", "top"]] # def self.split(uri) RFC3986_PARSER.split(uri) end + # Returns a new \Bundler::URI object constructed from the given string +uri+: # - # == Synopsis - # - # Bundler::URI::parse(uri_str) - # - # == Args - # - # +uri_str+:: - # String with Bundler::URI. - # - # == Description - # - # Creates one of the Bundler::URI's subclasses instance from the string. - # - # == Raises - # - # Bundler::URI::InvalidURIError:: - # Raised if Bundler::URI given is not a correct one. + # Bundler::URI.parse('https://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top') + # # => #<Bundler::URI::HTTPS https://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top> + # Bundler::URI.parse('http://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top') + # # => #<Bundler::URI::HTTP http://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top> # - # == Usage - # - # require 'bundler/vendor/uri/lib/uri' - # - # uri = Bundler::URI.parse("http://www.ruby-lang.org/") - # # => #<Bundler::URI::HTTP http://www.ruby-lang.org/> - # uri.scheme - # # => "http" - # uri.host - # # => "www.ruby-lang.org" - # - # It's recommended to first ::escape the provided +uri_str+ if there are any - # invalid Bundler::URI characters. + # It's recommended to first ::escape string +uri+ + # if it may contain invalid Bundler::URI characters. # def self.parse(uri) RFC3986_PARSER.parse(uri) end + # Merges the given Bundler::URI strings +str+ + # per {RFC 2396}[https://www.rfc-editor.org/rfc/rfc2396.html]. # - # == Synopsis - # - # Bundler::URI::join(str[, str, ...]) + # Each string in +str+ is converted to an + # {RFC3986 Bundler::URI}[https://www.rfc-editor.org/rfc/rfc3986.html] before being merged. # - # == Args - # - # +str+:: - # String(s) to work with, will be converted to RFC3986 URIs before merging. - # - # == Description - # - # Joins URIs. - # - # == Usage - # - # require 'bundler/vendor/uri/lib/uri' + # Examples: # # Bundler::URI.join("http://example.com/","main.rbx") # # => #<Bundler::URI::HTTP http://example.com/main.rbx> @@ -254,7 +236,7 @@ module Bundler::URI # Bundler::URI.extract("text here http://foo.example.org/bla and here mailto:test@example.com and here also.") # # => ["http://foo.example.com/bla", "mailto:test@example.com"] # - def self.extract(str, schemes = nil, &block) + def self.extract(str, schemes = nil, &block) # :nodoc: warn "Bundler::URI.extract is obsolete", uplevel: 1 if $VERBOSE DEFAULT_PARSER.extract(str, schemes, &block) end @@ -291,7 +273,7 @@ module Bundler::URI # p $& # end # - def self.regexp(schemes = nil) + def self.regexp(schemes = nil)# :nodoc: warn "Bundler::URI.regexp is obsolete", uplevel: 1 if $VERBOSE DEFAULT_PARSER.make_regexp(schemes) end @@ -314,40 +296,86 @@ module Bundler::URI TBLDECWWWCOMP_['+'] = ' ' TBLDECWWWCOMP_.freeze - # Encodes given +str+ to URL-encoded form data. + # Returns a URL-encoded string derived from the given string +str+. + # + # The returned string: + # + # - Preserves: + # + # - Characters <tt>'*'</tt>, <tt>'.'</tt>, <tt>'-'</tt>, and <tt>'_'</tt>. + # - Character in ranges <tt>'a'..'z'</tt>, <tt>'A'..'Z'</tt>, + # and <tt>'0'..'9'</tt>. + # + # Example: # - # This method doesn't convert *, -, ., 0-9, A-Z, _, a-z, but does convert SP - # (ASCII space) to + and converts others to %XX. + # Bundler::URI.encode_www_form_component('*.-_azAZ09') + # # => "*.-_azAZ09" # - # If +enc+ is given, convert +str+ to the encoding before percent encoding. + # - Converts: # - # This is an implementation of - # https://www.w3.org/TR/2013/CR-html5-20130806/forms.html#url-encoded-form-data. + # - Character <tt>' '</tt> to character <tt>'+'</tt>. + # - Any other character to "percent notation"; + # the percent notation for character <i>c</i> is <tt>'%%%X' % c.ord</tt>. # - # See Bundler::URI.decode_www_form_component, Bundler::URI.encode_www_form. + # Example: + # + # Bundler::URI.encode_www_form_component('Here are some punctuation characters: ,;?:') + # # => "Here+are+some+punctuation+characters%3A+%2C%3B%3F%3A" + # + # Encoding: + # + # - If +str+ has encoding Encoding::ASCII_8BIT, argument +enc+ is ignored. + # - Otherwise +str+ is converted first to Encoding::UTF_8 + # (with suitable character replacements), + # and then to encoding +enc+. + # + # In either case, the returned string has forced encoding Encoding::US_ASCII. + # + # Related: Bundler::URI.encode_uri_component (encodes <tt>' '</tt> as <tt>'%20'</tt>). def self.encode_www_form_component(str, enc=nil) _encode_uri_component(/[^*\-.0-9A-Z_a-z]/, TBLENCWWWCOMP_, str, enc) end - # Decodes given +str+ of URL-encoded form data. + # Returns a string decoded from the given \URL-encoded string +str+. + # + # The given string is first encoded as Encoding::ASCII-8BIT (using String#b), + # then decoded (as below), and finally force-encoded to the given encoding +enc+. + # + # The returned string: + # + # - Preserves: + # + # - Characters <tt>'*'</tt>, <tt>'.'</tt>, <tt>'-'</tt>, and <tt>'_'</tt>. + # - Character in ranges <tt>'a'..'z'</tt>, <tt>'A'..'Z'</tt>, + # and <tt>'0'..'9'</tt>. + # + # Example: + # + # Bundler::URI.decode_www_form_component('*.-_azAZ09') + # # => "*.-_azAZ09" + # + # - Converts: + # + # - Character <tt>'+'</tt> to character <tt>' '</tt>. + # - Each "percent notation" to an ASCII character. # - # This decodes + to SP. + # Example: # - # See Bundler::URI.encode_www_form_component, Bundler::URI.decode_www_form. + # Bundler::URI.decode_www_form_component('Here+are+some+punctuation+characters%3A+%2C%3B%3F%3A') + # # => "Here are some punctuation characters: ,;?:" + # + # Related: Bundler::URI.decode_uri_component (preserves <tt>'+'</tt>). def self.decode_www_form_component(str, enc=Encoding::UTF_8) _decode_uri_component(/\+|%\h\h/, str, enc) end - # Encodes +str+ using URL encoding - # - # This encodes SP to %20 instead of +. + # Like Bundler::URI.encode_www_form_component, except that <tt>' '</tt> (space) + # is encoded as <tt>'%20'</tt> (instead of <tt>'+'</tt>). def self.encode_uri_component(str, enc=nil) _encode_uri_component(/[^*\-.0-9A-Z_a-z]/, TBLENCURICOMP_, str, enc) end - # Decodes given +str+ of URL-encoded data. - # - # This does not decode + to SP. + # Like Bundler::URI.decode_www_form_component, except that <tt>'+'</tt> is preserved. def self.decode_uri_component(str, enc=Encoding::UTF_8) _decode_uri_component(/%\h\h/, str, enc) end @@ -372,33 +400,104 @@ module Bundler::URI end private_class_method :_decode_uri_component - # Generates URL-encoded form data from given +enum+. + # Returns a URL-encoded string derived from the given + # {Enumerable}[rdoc-ref:Enumerable@Enumerable+in+Ruby+Classes] + # +enum+. + # + # The result is suitable for use as form data + # for an \HTTP request whose <tt>Content-Type</tt> is + # <tt>'application/x-www-form-urlencoded'</tt>. + # + # The returned string consists of the elements of +enum+, + # each converted to one or more URL-encoded strings, + # and all joined with character <tt>'&'</tt>. + # + # Simple examples: + # + # Bundler::URI.encode_www_form([['foo', 0], ['bar', 1], ['baz', 2]]) + # # => "foo=0&bar=1&baz=2" + # Bundler::URI.encode_www_form({foo: 0, bar: 1, baz: 2}) + # # => "foo=0&bar=1&baz=2" + # + # The returned string is formed using method Bundler::URI.encode_www_form_component, + # which converts certain characters: + # + # Bundler::URI.encode_www_form('f#o': '/', 'b-r': '$', 'b z': '@') + # # => "f%23o=%2F&b-r=%24&b+z=%40" + # + # When +enum+ is Array-like, each element +ele+ is converted to a field: + # + # - If +ele+ is an array of two or more elements, + # the field is formed from its first two elements + # (and any additional elements are ignored): + # + # name = Bundler::URI.encode_www_form_component(ele[0], enc) + # value = Bundler::URI.encode_www_form_component(ele[1], enc) + # "#{name}=#{value}" # - # This generates application/x-www-form-urlencoded data defined in HTML5 - # from given an Enumerable object. + # Examples: # - # This internally uses Bundler::URI.encode_www_form_component(str). + # Bundler::URI.encode_www_form([%w[foo bar], %w[baz bat bah]]) + # # => "foo=bar&baz=bat" + # Bundler::URI.encode_www_form([['foo', 0], ['bar', :baz, 'bat']]) + # # => "foo=0&bar=baz" # - # This method doesn't convert the encoding of given items, so convert them - # before calling this method if you want to send data as other than original - # encoding or mixed encoding data. (Strings which are encoded in an HTML5 - # ASCII incompatible encoding are converted to UTF-8.) + # - If +ele+ is an array of one element, + # the field is formed from <tt>ele[0]</tt>: # - # This method doesn't handle files. When you send a file, use - # multipart/form-data. + # Bundler::URI.encode_www_form_component(ele[0]) # - # This refers https://url.spec.whatwg.org/#concept-urlencoded-serializer + # Example: # - # Bundler::URI.encode_www_form([["q", "ruby"], ["lang", "en"]]) - # #=> "q=ruby&lang=en" - # Bundler::URI.encode_www_form("q" => "ruby", "lang" => "en") - # #=> "q=ruby&lang=en" - # Bundler::URI.encode_www_form("q" => ["ruby", "perl"], "lang" => "en") - # #=> "q=ruby&q=perl&lang=en" - # Bundler::URI.encode_www_form([["q", "ruby"], ["q", "perl"], ["lang", "en"]]) - # #=> "q=ruby&q=perl&lang=en" + # Bundler::URI.encode_www_form([['foo'], [:bar], [0]]) + # # => "foo&bar&0" + # + # - Otherwise the field is formed from +ele+: + # + # Bundler::URI.encode_www_form_component(ele) + # + # Example: + # + # Bundler::URI.encode_www_form(['foo', :bar, 0]) + # # => "foo&bar&0" + # + # The elements of an Array-like +enum+ may be mixture: + # + # Bundler::URI.encode_www_form([['foo', 0], ['bar', 1, 2], ['baz'], :bat]) + # # => "foo=0&bar=1&baz&bat" + # + # When +enum+ is Hash-like, + # each +key+/+value+ pair is converted to one or more fields: + # + # - If +value+ is + # {Array-convertible}[rdoc-ref:implicit_conversion.rdoc@Array-Convertible+Objects], + # each element +ele+ in +value+ is paired with +key+ to form a field: + # + # name = Bundler::URI.encode_www_form_component(key, enc) + # value = Bundler::URI.encode_www_form_component(ele, enc) + # "#{name}=#{value}" + # + # Example: + # + # Bundler::URI.encode_www_form({foo: [:bar, 1], baz: [:bat, :bam, 2]}) + # # => "foo=bar&foo=1&baz=bat&baz=bam&baz=2" + # + # - Otherwise, +key+ and +value+ are paired to form a field: + # + # name = Bundler::URI.encode_www_form_component(key, enc) + # value = Bundler::URI.encode_www_form_component(value, enc) + # "#{name}=#{value}" + # + # Example: + # + # Bundler::URI.encode_www_form({foo: 0, bar: 1, baz: 2}) + # # => "foo=0&bar=1&baz=2" + # + # The elements of a Hash-like +enum+ may be mixture: + # + # Bundler::URI.encode_www_form({foo: [0, 1], bar: 2}) + # # => "foo=0&foo=1&bar=2" # - # See Bundler::URI.encode_www_form_component, Bundler::URI.decode_www_form. def self.encode_www_form(enum, enc=nil) enum.map do |k,v| if v.nil? @@ -419,22 +518,39 @@ module Bundler::URI end.join('&') end - # Decodes URL-encoded form data from given +str+. + # Returns name/value pairs derived from the given string +str+, + # which must be an ASCII string. + # + # The method may be used to decode the body of Net::HTTPResponse object +res+ + # for which <tt>res['Content-Type']</tt> is <tt>'application/x-www-form-urlencoded'</tt>. + # + # The returned data is an array of 2-element subarrays; + # each subarray is a name/value pair (both are strings). + # Each returned string has encoding +enc+, + # and has had invalid characters removed via + # {String#scrub}[rdoc-ref:String#scrub]. # - # This decodes application/x-www-form-urlencoded data - # and returns an array of key-value arrays. + # A simple example: # - # This refers http://url.spec.whatwg.org/#concept-urlencoded-parser, - # so this supports only &-separator, and doesn't support ;-separator. + # Bundler::URI.decode_www_form('foo=0&bar=1&baz') + # # => [["foo", "0"], ["bar", "1"], ["baz", ""]] # - # ary = Bundler::URI.decode_www_form("a=1&a=2&b=3") - # ary #=> [['a', '1'], ['a', '2'], ['b', '3']] - # ary.assoc('a').last #=> '1' - # ary.assoc('b').last #=> '3' - # ary.rassoc('a').last #=> '2' - # Hash[ary] #=> {"a"=>"2", "b"=>"3"} + # The returned strings have certain conversions, + # similar to those performed in Bundler::URI.decode_www_form_component: + # + # Bundler::URI.decode_www_form('f%23o=%2F&b-r=%24&b+z=%40') + # # => [["f#o", "/"], ["b-r", "$"], ["b z", "@"]] + # + # The given string may contain consecutive separators: + # + # Bundler::URI.decode_www_form('foo=0&&bar=1&&baz=2') + # # => [["foo", "0"], ["", ""], ["bar", "1"], ["", ""], ["baz", "2"]] + # + # A different separator may be specified: + # + # Bundler::URI.decode_www_form('foo=0--bar=1--baz', separator: '--') + # # => [["foo", "0"], ["bar", "1"], ["baz", ""]] # - # See Bundler::URI.decode_www_form_component, Bundler::URI.encode_www_form. def self.decode_www_form(str, enc=Encoding::UTF_8, separator: '&', use__charset_: false, isindex: false) raise ArgumentError, "the input of #{self.name}.#{__method__} must be ASCII only string" unless str.ascii_only? ary = [] @@ -713,7 +829,15 @@ end # module Bundler::URI module Bundler # - # Returns +uri+ converted to an Bundler::URI object. + # Returns a \Bundler::URI object derived from the given +uri+, + # which may be a \Bundler::URI string or an existing \Bundler::URI object: + # + # # Returns a new Bundler::URI. + # uri = Bundler::URI('http://github.com/ruby/ruby') + # # => #<Bundler::URI::HTTP http://github.com/ruby/ruby> + # # Returns the given Bundler::URI. + # Bundler::URI(uri) + # # => #<Bundler::URI::HTTP http://github.com/ruby/ruby> # def URI(uri) if uri.is_a?(Bundler::URI::Generic) diff --git a/lib/bundler/vendor/uri/lib/uri/generic.rb b/lib/bundler/vendor/uri/lib/uri/generic.rb index 9ae6915826..762c425ac1 100644 --- a/lib/bundler/vendor/uri/lib/uri/generic.rb +++ b/lib/bundler/vendor/uri/lib/uri/generic.rb @@ -1376,6 +1376,7 @@ module Bundler::URI end str end + alias to_str to_s # # Compares two URIs. diff --git a/lib/bundler/vendor/uri/lib/uri/rfc2396_parser.rb b/lib/bundler/vendor/uri/lib/uri/rfc2396_parser.rb index 2f8d55353b..09c22c9906 100644 --- a/lib/bundler/vendor/uri/lib/uri/rfc2396_parser.rb +++ b/lib/bundler/vendor/uri/lib/uri/rfc2396_parser.rb @@ -497,8 +497,8 @@ module Bundler::URI ret = {} # for Bundler::URI::split - ret[:ABS_URI] = Regexp.new('\A\s*' + pattern[:X_ABS_URI] + '\s*\z', Regexp::EXTENDED) - ret[:REL_URI] = Regexp.new('\A\s*' + pattern[:X_REL_URI] + '\s*\z', Regexp::EXTENDED) + ret[:ABS_URI] = Regexp.new('\A\s*+' + pattern[:X_ABS_URI] + '\s*\z', Regexp::EXTENDED) + ret[:REL_URI] = Regexp.new('\A\s*+' + pattern[:X_REL_URI] + '\s*\z', Regexp::EXTENDED) # for Bundler::URI::extract ret[:URI_REF] = Regexp.new(pattern[:URI_REF]) diff --git a/lib/bundler/vendor/uri/lib/uri/rfc3986_parser.rb b/lib/bundler/vendor/uri/lib/uri/rfc3986_parser.rb index d527072db4..4c9882f595 100644 --- a/lib/bundler/vendor/uri/lib/uri/rfc3986_parser.rb +++ b/lib/bundler/vendor/uri/lib/uri/rfc3986_parser.rb @@ -1,9 +1,73 @@ -# frozen_string_literal: false +# frozen_string_literal: true module Bundler::URI class RFC3986_Parser # :nodoc: # Bundler::URI defined in RFC3986 - RFC3986_URI = /\A(?<Bundler::URI>(?<scheme>[A-Za-z][+\-.0-9A-Za-z]*+):(?<hier-part>\/\/(?<authority>(?:(?<userinfo>(?:%\h\h|[!$&-.0-;=A-Z_a-z~])*+)@)?(?<host>(?<IP-literal>\[(?:(?<IPv6address>(?:\h{1,4}:){6}(?<ls32>\h{1,4}:\h{1,4}|(?<IPv4address>(?<dec-octet>[1-9]\d|1\d{2}|2[0-4]\d|25[0-5]|\d)\.\g<dec-octet>\.\g<dec-octet>\.\g<dec-octet>))|::(?:\h{1,4}:){5}\g<ls32>|\h{1,4}?::(?:\h{1,4}:){4}\g<ls32>|(?:(?:\h{1,4}:)?\h{1,4})?::(?:\h{1,4}:){3}\g<ls32>|(?:(?:\h{1,4}:){,2}\h{1,4})?::(?:\h{1,4}:){2}\g<ls32>|(?:(?:\h{1,4}:){,3}\h{1,4})?::\h{1,4}:\g<ls32>|(?:(?:\h{1,4}:){,4}\h{1,4})?::\g<ls32>|(?:(?:\h{1,4}:){,5}\h{1,4})?::\h{1,4}|(?:(?:\h{1,4}:){,6}\h{1,4})?::)|(?<IPvFuture>v\h++\.[!$&-.0-;=A-Z_a-z~]++))\])|\g<IPv4address>|(?<reg-name>(?:%\h\h|[!$&-.0-9;=A-Z_a-z~])*+))(?::(?<port>\d*+))?)(?<path-abempty>(?:\/(?<segment>(?:%\h\h|[!$&-.0-;=@-Z_a-z~])*+))*+)|(?<path-absolute>\/(?:(?<segment-nz>(?:%\h\h|[!$&-.0-;=@-Z_a-z~])++)(?:\/\g<segment>)*+)?)|(?<path-rootless>\g<segment-nz>(?:\/\g<segment>)*+)|(?<path-empty>))(?:\?(?<query>[^#]*+))?(?:\#(?<fragment>(?:%\h\h|[!$&-.0-;=@-Z_a-z~\/?])*+))?)\z/ - RFC3986_relative_ref = /\A(?<relative-ref>(?<relative-part>\/\/(?<authority>(?:(?<userinfo>(?:%\h\h|[!$&-.0-;=A-Z_a-z~])*+)@)?(?<host>(?<IP-literal>\[(?:(?<IPv6address>(?:\h{1,4}:){6}(?<ls32>\h{1,4}:\h{1,4}|(?<IPv4address>(?<dec-octet>[1-9]\d|1\d{2}|2[0-4]\d|25[0-5]|\d)\.\g<dec-octet>\.\g<dec-octet>\.\g<dec-octet>))|::(?:\h{1,4}:){5}\g<ls32>|\h{1,4}?::(?:\h{1,4}:){4}\g<ls32>|(?:(?:\h{1,4}:){,1}\h{1,4})?::(?:\h{1,4}:){3}\g<ls32>|(?:(?:\h{1,4}:){,2}\h{1,4})?::(?:\h{1,4}:){2}\g<ls32>|(?:(?:\h{1,4}:){,3}\h{1,4})?::\h{1,4}:\g<ls32>|(?:(?:\h{1,4}:){,4}\h{1,4})?::\g<ls32>|(?:(?:\h{1,4}:){,5}\h{1,4})?::\h{1,4}|(?:(?:\h{1,4}:){,6}\h{1,4})?::)|(?<IPvFuture>v\h++\.[!$&-.0-;=A-Z_a-z~]++))\])|\g<IPv4address>|(?<reg-name>(?:%\h\h|[!$&-.0-9;=A-Z_a-z~])++))?(?::(?<port>\d*+))?)(?<path-abempty>(?:\/(?<segment>(?:%\h\h|[!$&-.0-;=@-Z_a-z~])*+))*+)|(?<path-absolute>\/(?:(?<segment-nz>(?:%\h\h|[!$&-.0-;=@-Z_a-z~])++)(?:\/\g<segment>)*+)?)|(?<path-noscheme>(?<segment-nz-nc>(?:%\h\h|[!$&-.0-9;=@-Z_a-z~])++)(?:\/\g<segment>)*+)|(?<path-empty>))(?:\?(?<query>[^#]*+))?(?:\#(?<fragment>(?:%\h\h|[!$&-.0-;=@-Z_a-z~\/?])*+))?)\z/ + HOST = %r[ + (?<IP-literal>\[(?: + (?<IPv6address> + (?:\h{1,4}:){6} + (?<ls32>\h{1,4}:\h{1,4} + | (?<IPv4address>(?<dec-octet>[1-9]\d|1\d{2}|2[0-4]\d|25[0-5]|\d) + \.\g<dec-octet>\.\g<dec-octet>\.\g<dec-octet>) + ) + | ::(?:\h{1,4}:){5}\g<ls32> + | \h{1,4}?::(?:\h{1,4}:){4}\g<ls32> + | (?:(?:\h{1,4}:)?\h{1,4})?::(?:\h{1,4}:){3}\g<ls32> + | (?:(?:\h{1,4}:){,2}\h{1,4})?::(?:\h{1,4}:){2}\g<ls32> + | (?:(?:\h{1,4}:){,3}\h{1,4})?::\h{1,4}:\g<ls32> + | (?:(?:\h{1,4}:){,4}\h{1,4})?::\g<ls32> + | (?:(?:\h{1,4}:){,5}\h{1,4})?::\h{1,4} + | (?:(?:\h{1,4}:){,6}\h{1,4})?:: + ) + | (?<IPvFuture>v\h++\.[!$&-.0-9:;=A-Z_a-z~]++) + )\]) + | \g<IPv4address> + | (?<reg-name>(?:%\h\h|[!$&-.0-9;=A-Z_a-z~])*+) + ]x + + USERINFO = /(?:%\h\h|[!$&-.0-9:;=A-Z_a-z~])*+/ + + SCHEME = %r[[A-Za-z][+\-.0-9A-Za-z]*+].source + SEG = %r[(?:%\h\h|[!$&-.0-9:;=@A-Z_a-z~/])].source + SEG_NC = %r[(?:%\h\h|[!$&-.0-9;=@A-Z_a-z~])].source + FRAGMENT = %r[(?:%\h\h|[!$&-.0-9:;=@A-Z_a-z~/?])*+].source + + RFC3986_URI = %r[\A + (?<seg>#{SEG}){0} + (?<Bundler::URI> + (?<scheme>#{SCHEME}): + (?<hier-part>// + (?<authority> + (?:(?<userinfo>#{USERINFO.source})@)? + (?<host>#{HOST.source.delete(" \n")}) + (?::(?<port>\d*+))? + ) + (?<path-abempty>(?:/\g<seg>*+)?) + | (?<path-absolute>/((?!/)\g<seg>++)?) + | (?<path-rootless>(?!/)\g<seg>++) + | (?<path-empty>) + ) + (?:\?(?<query>[^\#]*+))? + (?:\#(?<fragment>#{FRAGMENT}))? + )\z]x + + RFC3986_relative_ref = %r[\A + (?<seg>#{SEG}){0} + (?<relative-ref> + (?<relative-part>// + (?<authority> + (?:(?<userinfo>#{USERINFO.source})@)? + (?<host>#{HOST.source.delete(" \n")}(?<!/))? + (?::(?<port>\d*+))? + ) + (?<path-abempty>(?:/\g<seg>*+)?) + | (?<path-absolute>/\g<seg>*+) + | (?<path-noscheme>#{SEG_NC}++(?:/\g<seg>*+)?) + | (?<path-empty>) + ) + (?:\?(?<query>[^#]*+))? + (?:\#(?<fragment>#{FRAGMENT}))? + )\z]x attr_reader :regexp def initialize @@ -19,9 +83,9 @@ module Bundler::URI uri.ascii_only? or raise InvalidURIError, "Bundler::URI must be ascii only #{uri.dump}" if m = RFC3986_URI.match(uri) - query = m["query".freeze] - scheme = m["scheme".freeze] - opaque = m["path-rootless".freeze] + query = m["query"] + scheme = m["scheme"] + opaque = m["path-rootless"] if opaque opaque << "?#{query}" if query [ scheme, @@ -32,35 +96,35 @@ module Bundler::URI nil, # path opaque, nil, # query - m["fragment".freeze] + m["fragment"] ] else # normal [ scheme, - m["userinfo".freeze], - m["host".freeze], - m["port".freeze], + m["userinfo"], + m["host"], + m["port"], nil, # registry - (m["path-abempty".freeze] || - m["path-absolute".freeze] || - m["path-empty".freeze]), + (m["path-abempty"] || + m["path-absolute"] || + m["path-empty"]), nil, # opaque query, - m["fragment".freeze] + m["fragment"] ] end elsif m = RFC3986_relative_ref.match(uri) [ nil, # scheme - m["userinfo".freeze], - m["host".freeze], - m["port".freeze], + m["userinfo"], + m["host"], + m["port"], nil, # registry, - (m["path-abempty".freeze] || - m["path-absolute".freeze] || - m["path-noscheme".freeze] || - m["path-empty".freeze]), + (m["path-abempty"] || + m["path-absolute"] || + m["path-noscheme"] || + m["path-empty"]), nil, # opaque - m["query".freeze], - m["fragment".freeze] + m["query"], + m["fragment"] ] else raise InvalidURIError, "bad Bundler::URI(is not Bundler::URI?): #{uri.inspect}" @@ -92,15 +156,15 @@ module Bundler::URI def default_regexp # :nodoc: { - SCHEME: /\A[A-Za-z][A-Za-z0-9+\-.]*\z/, - USERINFO: /\A(?:%\h\h|[!$&-.0-;=A-Z_a-z~])*\z/, - HOST: /\A(?:(?<IP-literal>\[(?:(?<IPv6address>(?:\h{1,4}:){6}(?<ls32>\h{1,4}:\h{1,4}|(?<IPv4address>(?<dec-octet>[1-9]\d|1\d{2}|2[0-4]\d|25[0-5]|\d)\.\g<dec-octet>\.\g<dec-octet>\.\g<dec-octet>))|::(?:\h{1,4}:){5}\g<ls32>|\h{,4}::(?:\h{1,4}:){4}\g<ls32>|(?:(?:\h{1,4}:)?\h{1,4})?::(?:\h{1,4}:){3}\g<ls32>|(?:(?:\h{1,4}:){,2}\h{1,4})?::(?:\h{1,4}:){2}\g<ls32>|(?:(?:\h{1,4}:){,3}\h{1,4})?::\h{1,4}:\g<ls32>|(?:(?:\h{1,4}:){,4}\h{1,4})?::\g<ls32>|(?:(?:\h{1,4}:){,5}\h{1,4})?::\h{1,4}|(?:(?:\h{1,4}:){,6}\h{1,4})?::)|(?<IPvFuture>v\h+\.[!$&-.0-;=A-Z_a-z~]+))\])|\g<IPv4address>|(?<reg-name>(?:%\h\h|[!$&-.0-9;=A-Z_a-z~])*))\z/, - ABS_PATH: /\A\/(?:%\h\h|[!$&-.0-;=@-Z_a-z~])*(?:\/(?:%\h\h|[!$&-.0-;=@-Z_a-z~])*)*\z/, - REL_PATH: /\A(?:%\h\h|[!$&-.0-;=@-Z_a-z~])+(?:\/(?:%\h\h|[!$&-.0-;=@-Z_a-z~])*)*\z/, - QUERY: /\A(?:%\h\h|[!$&-.0-;=@-Z_a-z~\/?])*\z/, - FRAGMENT: /\A(?:%\h\h|[!$&-.0-;=@-Z_a-z~\/?])*\z/, - OPAQUE: /\A(?:[^\/].*)?\z/, - PORT: /\A[\x09\x0a\x0c\x0d ]*\d*[\x09\x0a\x0c\x0d ]*\z/, + SCHEME: %r[\A#{SCHEME}\z]o, + USERINFO: %r[\A#{USERINFO}\z]o, + HOST: %r[\A#{HOST}\z]o, + ABS_PATH: %r[\A/#{SEG}*+\z]o, + REL_PATH: %r[\A(?!/)#{SEG}++\z]o, + QUERY: %r[\A(?:%\h\h|[!$&-.0-9:;=@A-Z_a-z~/?])*+\z], + FRAGMENT: %r[\A#{FRAGMENT}\z]o, + OPAQUE: %r[\A(?:[^/].*)?\z], + PORT: /\A[\x09\x0a\x0c\x0d ]*+\d*[\x09\x0a\x0c\x0d ]*\z/, } end diff --git a/lib/bundler/vendor/uri/lib/uri/version.rb b/lib/bundler/vendor/uri/lib/uri/version.rb index 500fdb3f99..1fa1c7c09a 100644 --- a/lib/bundler/vendor/uri/lib/uri/version.rb +++ b/lib/bundler/vendor/uri/lib/uri/version.rb @@ -1,6 +1,6 @@ module Bundler::URI # :stopdoc: - VERSION_CODE = '001201'.freeze + VERSION_CODE = '001300'.freeze VERSION = VERSION_CODE.scan(/../).collect{|n| n.to_i}.join('.').freeze # :startdoc: end diff --git a/lib/bundler/vendored_net_http.rb b/lib/bundler/vendored_net_http.rb new file mode 100644 index 0000000000..0dcabaa7d7 --- /dev/null +++ b/lib/bundler/vendored_net_http.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +begin + require "rubygems/vendored_net_http" +rescue LoadError + begin + require "rubygems/net/http" + rescue LoadError + require "net/http" + Gem::Net = Net + end +end diff --git a/lib/bundler/vendored_persistent.rb b/lib/bundler/vendored_persistent.rb index e29f27cdfd..ab985c267f 100644 --- a/lib/bundler/vendored_persistent.rb +++ b/lib/bundler/vendored_persistent.rb @@ -9,7 +9,3 @@ module Bundler end end require_relative "vendor/net-http-persistent/lib/net/http/persistent" - -module Bundler - PersistentHTTP = Persistent::Net::HTTP::Persistent -end diff --git a/lib/bundler/vendored_timeout.rb b/lib/bundler/vendored_timeout.rb new file mode 100644 index 0000000000..9b2507c0cc --- /dev/null +++ b/lib/bundler/vendored_timeout.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +begin + require "rubygems/vendored_timeout" +rescue LoadError + begin + require "rubygems/timeout" + rescue LoadError + require "timeout" + Gem::Timeout = Timeout + end +end diff --git a/lib/bundler/vendored_uri.rb b/lib/bundler/vendored_uri.rb index 905e8158e8..2efddc65f9 100644 --- a/lib/bundler/vendored_uri.rb +++ b/lib/bundler/vendored_uri.rb @@ -1,4 +1,21 @@ # frozen_string_literal: true module Bundler; end -require_relative "vendor/uri/lib/uri" + +# Use RubyGems vendored copy when available. Otherwise fallback to Bundler +# vendored copy. The vendored copy in Bundler can be removed once support for +# RubyGems 3.5 is dropped. + +begin + require "rubygems/vendor/uri/lib/uri" +rescue LoadError + require_relative "vendor/uri/lib/uri" + Gem::URI = Bundler::URI + + module Gem + def URI(uri) # rubocop:disable Naming/MethodName + Bundler::URI(uri) + end + module_function :URI + end +end diff --git a/lib/bundler/version.rb b/lib/bundler/version.rb index 7c03d8687f..f2f6236cda 100644 --- a/lib/bundler/version.rb +++ b/lib/bundler/version.rb @@ -1,7 +1,7 @@ # frozen_string_literal: false module Bundler - VERSION = "2.5.0.dev".freeze + VERSION = "2.6.0.dev".freeze def self.bundler_major_version @bundler_major_version ||= VERSION.split(".").first.to_i diff --git a/lib/bundler/vlad.rb b/lib/bundler/vlad.rb index 538e8c3e74..6179d0e4eb 100644 --- a/lib/bundler/vlad.rb +++ b/lib/bundler/vlad.rb @@ -13,5 +13,5 @@ require_relative "deployment" include Rake::DSL if defined? Rake::DSL namespace :vlad do - Bundler::Deployment.define_task(Rake::RemoteTask, :remote_task, :roles => :app) + Bundler::Deployment.define_task(Rake::RemoteTask, :remote_task, roles: :app) end diff --git a/lib/bundler/yaml_serializer.rb b/lib/bundler/yaml_serializer.rb index d5ecbd4aef..42e6aaf89d 100644 --- a/lib/bundler/yaml_serializer.rb +++ b/lib/bundler/yaml_serializer.rb @@ -17,7 +17,11 @@ module Bundler if v.is_a?(Hash) yaml << dump_hash(v).gsub(/^(?!$)/, " ") # indent all non-empty lines elsif v.is_a?(Array) # Expected to be array of strings - yaml << "\n- " << v.map {|s| s.to_s.gsub(/\s+/, " ").inspect }.join("\n- ") << "\n" + if v.empty? + yaml << " []\n" + else + yaml << "\n- " << v.map {|s| s.to_s.gsub(/\s+/, " ").inspect }.join("\n- ") << "\n" + end else yaml << " " << v.to_s.gsub(/\s+/, " ").inspect << "\n" end @@ -32,7 +36,7 @@ module Bundler (.*) # value \1 # matching closing quote $ - /xo.freeze + /xo HASH_REGEX = / ^ @@ -44,18 +48,20 @@ module Bundler (.*) # value \3 # matching closing quote $ - /xo.freeze + /xo def load(str) res = {} stack = [res] last_hash = nil last_empty_key = nil - str.split(/\r?\n/).each do |line| + str.split(/\r?\n/) do |line| if match = HASH_REGEX.match(line) indent, key, quote, val = match.captures - key = convert_to_backward_compatible_key(key) - depth = indent.scan(/ /).length + val = strip_comment(val) + + convert_to_backward_compatible_key!(key) + depth = indent.size / 2 if quote.empty? && val.empty? new_hash = {} stack[depth][key] = new_hash @@ -63,10 +69,13 @@ module Bundler last_empty_key = key last_hash = stack[depth] else + val = [] if val == "[]" # empty array stack[depth][key] = val end elsif match = ARRAY_REGEX.match(line) _, val = match.captures + val = strip_comment(val) + last_hash[last_empty_key] = [] unless last_hash[last_empty_key].is_a?(Array) last_hash[last_empty_key].push(val) @@ -75,15 +84,22 @@ module Bundler res end + def strip_comment(val) + if val.include?("#") && !val.start_with?("#") + val.split("#", 2).first.strip + else + val + end + end + # for settings' keys - def convert_to_backward_compatible_key(key) - key = "#{key}/" if key =~ /https?:/i && key !~ %r{/\Z} - key = key.gsub(".", "__") if key.include?(".") - key + def convert_to_backward_compatible_key!(key) + key << "/" if /https?:/i.match?(key) && !%r{/\Z}.match?(key) + key.gsub!(".", "__") end class << self - private :dump_hash, :convert_to_backward_compatible_key + private :dump_hash, :convert_to_backward_compatible_key! end end end diff --git a/lib/cgi.rb b/lib/cgi.rb index 4cd6b3bd8e..7af85e7fc8 100644 --- a/lib/cgi.rb +++ b/lib/cgi.rb @@ -288,7 +288,7 @@ # class CGI - VERSION = "0.3.6" + VERSION = "0.4.1" end require 'cgi/core' diff --git a/lib/cgi/util.rb b/lib/cgi/util.rb index 5f335898dc..4986e544e0 100644 --- a/lib/cgi/util.rb +++ b/lib/cgi/util.rb @@ -36,7 +36,7 @@ module CGI::Util # URL-encode a string following RFC 3986 # Space characters (+" "+) are encoded with (+"%20"+) - # url_encoded_string = CGI.escape("'Stop!' said Fred") + # url_encoded_string = CGI.escapeURIComponent("'Stop!' said Fred") # # => "%27Stop%21%27%20said%20Fred" def escapeURIComponent(string) encoding = string.encoding @@ -46,9 +46,10 @@ module CGI::Util end buffer.force_encoding(encoding) end + alias escape_uri_component escapeURIComponent # URL-decode a string following RFC 3986 with encoding(optional). - # string = CGI.unescape("%27Stop%21%27+said%20Fred") + # string = CGI.unescapeURIComponent("%27Stop%21%27+said%20Fred") # # => "'Stop!'+said Fred" def unescapeURIComponent(string, encoding = @@accept_charset) str = string.b @@ -59,6 +60,8 @@ module CGI::Util str.valid_encoding? ? str : str.force_encoding(string.encoding) end + alias unescape_uri_component unescapeURIComponent + # The set of special characters and their escaped values TABLE_FOR_ESCAPE_HTML__ = { "'" => ''', diff --git a/lib/csv.rb b/lib/csv.rb deleted file mode 100644 index d4dc569b83..0000000000 --- a/lib/csv.rb +++ /dev/null @@ -1,2881 +0,0 @@ -# encoding: US-ASCII -# frozen_string_literal: true -# = csv.rb -- CSV Reading and Writing -# -# Created by James Edward Gray II on 2005-10-31. -# -# See CSV for documentation. -# -# == Description -# -# Welcome to the new and improved CSV. -# -# This version of the CSV library began its life as FasterCSV. FasterCSV was -# intended as a replacement to Ruby's then standard CSV library. It was -# designed to address concerns users of that library had and it had three -# primary goals: -# -# 1. Be significantly faster than CSV while remaining a pure Ruby library. -# 2. Use a smaller and easier to maintain code base. (FasterCSV eventually -# grew larger, was also but considerably richer in features. The parsing -# core remains quite small.) -# 3. Improve on the CSV interface. -# -# Obviously, the last one is subjective. I did try to defer to the original -# interface whenever I didn't have a compelling reason to change it though, so -# hopefully this won't be too radically different. -# -# We must have met our goals because FasterCSV was renamed to CSV and replaced -# the original library as of Ruby 1.9. If you are migrating code from 1.8 or -# earlier, you may have to change your code to comply with the new interface. -# -# == What's the Different From the Old CSV? -# -# I'm sure I'll miss something, but I'll try to mention most of the major -# differences I am aware of, to help others quickly get up to speed: -# -# === \CSV Parsing -# -# * This parser is m17n aware. See CSV for full details. -# * This library has a stricter parser and will throw MalformedCSVErrors on -# problematic data. -# * This library has a less liberal idea of a line ending than CSV. What you -# set as the <tt>:row_sep</tt> is law. It can auto-detect your line endings -# though. -# * The old library returned empty lines as <tt>[nil]</tt>. This library calls -# them <tt>[]</tt>. -# * This library has a much faster parser. -# -# === Interface -# -# * CSV now uses keyword parameters to set options. -# * CSV no longer has generate_row() or parse_row(). -# * The old CSV's Reader and Writer classes have been dropped. -# * CSV::open() is now more like Ruby's open(). -# * CSV objects now support most standard IO methods. -# * CSV now has a new() method used to wrap objects like String and IO for -# reading and writing. -# * CSV::generate() is different from the old method. -# * CSV no longer supports partial reads. It works line-by-line. -# * CSV no longer allows the instance methods to override the separators for -# performance reasons. They must be set in the constructor. -# -# If you use this library and find yourself missing any functionality I have -# trimmed, please {let me know}[mailto:james@grayproductions.net]. -# -# == Documentation -# -# See CSV for documentation. -# -# == What is CSV, really? -# -# CSV maintains a pretty strict definition of CSV taken directly from -# {the RFC}[https://www.ietf.org/rfc/rfc4180.txt]. I relax the rules in only one -# place and that is to make using this library easier. CSV will parse all valid -# CSV. -# -# What you don't want to do is to feed CSV invalid data. Because of the way the -# CSV format works, it's common for a parser to need to read until the end of -# the file to be sure a field is invalid. This consumes a lot of time and memory. -# -# Luckily, when working with invalid CSV, Ruby's built-in methods will almost -# always be superior in every way. For example, parsing non-quoted fields is as -# easy as: -# -# data.split(",") -# -# == Questions and/or Comments -# -# Feel free to email {James Edward Gray II}[mailto:james@grayproductions.net] -# with any questions. - -require "forwardable" -require "date" -require "stringio" - -require_relative "csv/fields_converter" -require_relative "csv/input_record_separator" -require_relative "csv/parser" -require_relative "csv/row" -require_relative "csv/table" -require_relative "csv/writer" - -# == \CSV -# -# === In a Hurry? -# -# If you are familiar with \CSV data and have a particular task in mind, -# you may want to go directly to the: -# - {Recipes for CSV}[doc/csv/recipes/recipes_rdoc.html]. -# -# Otherwise, read on here, about the API: classes, methods, and constants. -# -# === \CSV Data -# -# \CSV (comma-separated values) data is a text representation of a table: -# - A _row_ _separator_ delimits table rows. -# A common row separator is the newline character <tt>"\n"</tt>. -# - A _column_ _separator_ delimits fields in a row. -# A common column separator is the comma character <tt>","</tt>. -# -# This \CSV \String, with row separator <tt>"\n"</tt> -# and column separator <tt>","</tt>, -# has three rows and two columns: -# "foo,0\nbar,1\nbaz,2\n" -# -# Despite the name \CSV, a \CSV representation can use different separators. -# -# For more about tables, see the Wikipedia article -# "{Table (information)}[https://en.wikipedia.org/wiki/Table_(information)]", -# especially its section -# "{Simple table}[https://en.wikipedia.org/wiki/Table_(information)#Simple_table]" -# -# == \Class \CSV -# -# Class \CSV provides methods for: -# - Parsing \CSV data from a \String object, a \File (via its file path), or an \IO object. -# - Generating \CSV data to a \String object. -# -# To make \CSV available: -# require 'csv' -# -# All examples here assume that this has been done. -# -# == Keeping It Simple -# -# A \CSV object has dozens of instance methods that offer fine-grained control -# of parsing and generating \CSV data. -# For many needs, though, simpler approaches will do. -# -# This section summarizes the singleton methods in \CSV -# that allow you to parse and generate without explicitly -# creating \CSV objects. -# For details, follow the links. -# -# === Simple Parsing -# -# Parsing methods commonly return either of: -# - An \Array of Arrays of Strings: -# - The outer \Array is the entire "table". -# - Each inner \Array is a row. -# - Each \String is a field. -# - A CSV::Table object. For details, see -# {\CSV with Headers}[#class-CSV-label-CSV+with+Headers]. -# -# ==== Parsing a \String -# -# The input to be parsed can be a string: -# string = "foo,0\nbar,1\nbaz,2\n" -# -# \Method CSV.parse returns the entire \CSV data: -# CSV.parse(string) # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] -# -# \Method CSV.parse_line returns only the first row: -# CSV.parse_line(string) # => ["foo", "0"] -# -# \CSV extends class \String with instance method String#parse_csv, -# which also returns only the first row: -# string.parse_csv # => ["foo", "0"] -# -# ==== Parsing Via a \File Path -# -# The input to be parsed can be in a file: -# string = "foo,0\nbar,1\nbaz,2\n" -# path = 't.csv' -# File.write(path, string) -# -# \Method CSV.read returns the entire \CSV data: -# CSV.read(path) # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] -# -# \Method CSV.foreach iterates, passing each row to the given block: -# CSV.foreach(path) do |row| -# p row -# end -# Output: -# ["foo", "0"] -# ["bar", "1"] -# ["baz", "2"] -# -# \Method CSV.table returns the entire \CSV data as a CSV::Table object: -# CSV.table(path) # => #<CSV::Table mode:col_or_row row_count:3> -# -# ==== Parsing from an Open \IO Stream -# -# The input to be parsed can be in an open \IO stream: -# -# \Method CSV.read returns the entire \CSV data: -# File.open(path) do |file| -# CSV.read(file) -# end # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] -# -# As does method CSV.parse: -# File.open(path) do |file| -# CSV.parse(file) -# end # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] -# -# \Method CSV.parse_line returns only the first row: -# File.open(path) do |file| -# CSV.parse_line(file) -# end # => ["foo", "0"] -# -# \Method CSV.foreach iterates, passing each row to the given block: -# File.open(path) do |file| -# CSV.foreach(file) do |row| -# p row -# end -# end -# Output: -# ["foo", "0"] -# ["bar", "1"] -# ["baz", "2"] -# -# \Method CSV.table returns the entire \CSV data as a CSV::Table object: -# File.open(path) do |file| -# CSV.table(file) -# end # => #<CSV::Table mode:col_or_row row_count:3> -# -# === Simple Generating -# -# \Method CSV.generate returns a \String; -# this example uses method CSV#<< to append the rows -# that are to be generated: -# output_string = CSV.generate do |csv| -# csv << ['foo', 0] -# csv << ['bar', 1] -# csv << ['baz', 2] -# end -# output_string # => "foo,0\nbar,1\nbaz,2\n" -# -# \Method CSV.generate_line returns a \String containing the single row -# constructed from an \Array: -# CSV.generate_line(['foo', '0']) # => "foo,0\n" -# -# \CSV extends class \Array with instance method <tt>Array#to_csv</tt>, -# which forms an \Array into a \String: -# ['foo', '0'].to_csv # => "foo,0\n" -# -# === "Filtering" \CSV -# -# \Method CSV.filter provides a Unix-style filter for \CSV data. -# The input data is processed to form the output data: -# in_string = "foo,0\nbar,1\nbaz,2\n" -# out_string = '' -# CSV.filter(in_string, out_string) do |row| -# row[0] = row[0].upcase -# row[1] *= 4 -# end -# out_string # => "FOO,0000\nBAR,1111\nBAZ,2222\n" -# -# == \CSV Objects -# -# There are three ways to create a \CSV object: -# - \Method CSV.new returns a new \CSV object. -# - \Method CSV.instance returns a new or cached \CSV object. -# - \Method \CSV() also returns a new or cached \CSV object. -# -# === Instance Methods -# -# \CSV has three groups of instance methods: -# - Its own internally defined instance methods. -# - Methods included by module Enumerable. -# - Methods delegated to class IO. See below. -# -# ==== Delegated Methods -# -# For convenience, a CSV object will delegate to many methods in class IO. -# (A few have wrapper "guard code" in \CSV.) You may call: -# * IO#binmode -# * #binmode? -# * IO#close -# * IO#close_read -# * IO#close_write -# * IO#closed? -# * #eof -# * #eof? -# * IO#external_encoding -# * IO#fcntl -# * IO#fileno -# * #flock -# * IO#flush -# * IO#fsync -# * IO#internal_encoding -# * #ioctl -# * IO#isatty -# * #path -# * IO#pid -# * IO#pos -# * IO#pos= -# * IO#reopen -# * #rewind -# * IO#seek -# * #stat -# * IO#string -# * IO#sync -# * IO#sync= -# * IO#tell -# * #to_i -# * #to_io -# * IO#truncate -# * IO#tty? -# -# === Options -# -# The default values for options are: -# DEFAULT_OPTIONS = { -# # For both parsing and generating. -# col_sep: ",", -# row_sep: :auto, -# quote_char: '"', -# # For parsing. -# field_size_limit: nil, -# converters: nil, -# unconverted_fields: nil, -# headers: false, -# return_headers: false, -# header_converters: nil, -# skip_blanks: false, -# skip_lines: nil, -# liberal_parsing: false, -# nil_value: nil, -# empty_value: "", -# strip: false, -# # For generating. -# write_headers: nil, -# quote_empty: true, -# force_quotes: false, -# write_converters: nil, -# write_nil_value: nil, -# write_empty_value: "", -# } -# -# ==== Options for Parsing -# -# Options for parsing, described in detail below, include: -# - +row_sep+: Specifies the row separator; used to delimit rows. -# - +col_sep+: Specifies the column separator; used to delimit fields. -# - +quote_char+: Specifies the quote character; used to quote fields. -# - +field_size_limit+: Specifies the maximum field size + 1 allowed. -# Deprecated since 3.2.3. Use +max_field_size+ instead. -# - +max_field_size+: Specifies the maximum field size allowed. -# - +converters+: Specifies the field converters to be used. -# - +unconverted_fields+: Specifies whether unconverted fields are to be available. -# - +headers+: Specifies whether data contains headers, -# or specifies the headers themselves. -# - +return_headers+: Specifies whether headers are to be returned. -# - +header_converters+: Specifies the header converters to be used. -# - +skip_blanks+: Specifies whether blanks lines are to be ignored. -# - +skip_lines+: Specifies how comments lines are to be recognized. -# - +strip+: Specifies whether leading and trailing whitespace are to be -# stripped from fields. This must be compatible with +col_sep+; if it is not, -# then an +ArgumentError+ exception will be raised. -# - +liberal_parsing+: Specifies whether \CSV should attempt to parse -# non-compliant data. -# - +nil_value+: Specifies the object that is to be substituted for each null (no-text) field. -# - +empty_value+: Specifies the object that is to be substituted for each empty field. -# -# :include: ../doc/csv/options/common/row_sep.rdoc -# -# :include: ../doc/csv/options/common/col_sep.rdoc -# -# :include: ../doc/csv/options/common/quote_char.rdoc -# -# :include: ../doc/csv/options/parsing/field_size_limit.rdoc -# -# :include: ../doc/csv/options/parsing/converters.rdoc -# -# :include: ../doc/csv/options/parsing/unconverted_fields.rdoc -# -# :include: ../doc/csv/options/parsing/headers.rdoc -# -# :include: ../doc/csv/options/parsing/return_headers.rdoc -# -# :include: ../doc/csv/options/parsing/header_converters.rdoc -# -# :include: ../doc/csv/options/parsing/skip_blanks.rdoc -# -# :include: ../doc/csv/options/parsing/skip_lines.rdoc -# -# :include: ../doc/csv/options/parsing/strip.rdoc -# -# :include: ../doc/csv/options/parsing/liberal_parsing.rdoc -# -# :include: ../doc/csv/options/parsing/nil_value.rdoc -# -# :include: ../doc/csv/options/parsing/empty_value.rdoc -# -# ==== Options for Generating -# -# Options for generating, described in detail below, include: -# - +row_sep+: Specifies the row separator; used to delimit rows. -# - +col_sep+: Specifies the column separator; used to delimit fields. -# - +quote_char+: Specifies the quote character; used to quote fields. -# - +write_headers+: Specifies whether headers are to be written. -# - +force_quotes+: Specifies whether each output field is to be quoted. -# - +quote_empty+: Specifies whether each empty output field is to be quoted. -# - +write_converters+: Specifies the field converters to be used in writing. -# - +write_nil_value+: Specifies the object that is to be substituted for each +nil+-valued field. -# - +write_empty_value+: Specifies the object that is to be substituted for each empty field. -# -# :include: ../doc/csv/options/common/row_sep.rdoc -# -# :include: ../doc/csv/options/common/col_sep.rdoc -# -# :include: ../doc/csv/options/common/quote_char.rdoc -# -# :include: ../doc/csv/options/generating/write_headers.rdoc -# -# :include: ../doc/csv/options/generating/force_quotes.rdoc -# -# :include: ../doc/csv/options/generating/quote_empty.rdoc -# -# :include: ../doc/csv/options/generating/write_converters.rdoc -# -# :include: ../doc/csv/options/generating/write_nil_value.rdoc -# -# :include: ../doc/csv/options/generating/write_empty_value.rdoc -# -# === \CSV with Headers -# -# CSV allows to specify column names of CSV file, whether they are in data, or -# provided separately. If headers are specified, reading methods return an instance -# of CSV::Table, consisting of CSV::Row. -# -# # Headers are part of data -# data = CSV.parse(<<~ROWS, headers: true) -# Name,Department,Salary -# Bob,Engineering,1000 -# Jane,Sales,2000 -# John,Management,5000 -# ROWS -# -# data.class #=> CSV::Table -# data.first #=> #<CSV::Row "Name":"Bob" "Department":"Engineering" "Salary":"1000"> -# data.first.to_h #=> {"Name"=>"Bob", "Department"=>"Engineering", "Salary"=>"1000"} -# -# # Headers provided by developer -# data = CSV.parse('Bob,Engineering,1000', headers: %i[name department salary]) -# data.first #=> #<CSV::Row name:"Bob" department:"Engineering" salary:"1000"> -# -# === \Converters -# -# By default, each value (field or header) parsed by \CSV is formed into a \String. -# You can use a _field_ _converter_ or _header_ _converter_ -# to intercept and modify the parsed values: -# - See {Field Converters}[#class-CSV-label-Field+Converters]. -# - See {Header Converters}[#class-CSV-label-Header+Converters]. -# -# Also by default, each value to be written during generation is written 'as-is'. -# You can use a _write_ _converter_ to modify values before writing. -# - See {Write Converters}[#class-CSV-label-Write+Converters]. -# -# ==== Specifying \Converters -# -# You can specify converters for parsing or generating in the +options+ -# argument to various \CSV methods: -# - Option +converters+ for converting parsed field values. -# - Option +header_converters+ for converting parsed header values. -# - Option +write_converters+ for converting values to be written (generated). -# -# There are three forms for specifying converters: -# - A converter proc: executable code to be used for conversion. -# - A converter name: the name of a stored converter. -# - A converter list: an array of converter procs, converter names, and converter lists. -# -# ===== Converter Procs -# -# This converter proc, +strip_converter+, accepts a value +field+ -# and returns <tt>field.strip</tt>: -# strip_converter = proc {|field| field.strip } -# In this call to <tt>CSV.parse</tt>, -# the keyword argument <tt>converters: string_converter</tt> -# specifies that: -# - \Proc +string_converter+ is to be called for each parsed field. -# - The converter's return value is to replace the +field+ value. -# Example: -# string = " foo , 0 \n bar , 1 \n baz , 2 \n" -# array = CSV.parse(string, converters: strip_converter) -# array # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] -# -# A converter proc can receive a second argument, +field_info+, -# that contains details about the field. -# This modified +strip_converter+ displays its arguments: -# strip_converter = proc do |field, field_info| -# p [field, field_info] -# field.strip -# end -# string = " foo , 0 \n bar , 1 \n baz , 2 \n" -# array = CSV.parse(string, converters: strip_converter) -# array # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] -# Output: -# [" foo ", #<struct CSV::FieldInfo index=0, line=1, header=nil>] -# [" 0 ", #<struct CSV::FieldInfo index=1, line=1, header=nil>] -# [" bar ", #<struct CSV::FieldInfo index=0, line=2, header=nil>] -# [" 1 ", #<struct CSV::FieldInfo index=1, line=2, header=nil>] -# [" baz ", #<struct CSV::FieldInfo index=0, line=3, header=nil>] -# [" 2 ", #<struct CSV::FieldInfo index=1, line=3, header=nil>] -# Each CSV::FieldInfo object shows: -# - The 0-based field index. -# - The 1-based line index. -# - The field header, if any. -# -# ===== Stored \Converters -# -# A converter may be given a name and stored in a structure where -# the parsing methods can find it by name. -# -# The storage structure for field converters is the \Hash CSV::Converters. -# It has several built-in converter procs: -# - <tt>:integer</tt>: converts each \String-embedded integer into a true \Integer. -# - <tt>:float</tt>: converts each \String-embedded float into a true \Float. -# - <tt>:date</tt>: converts each \String-embedded date into a true \Date. -# - <tt>:date_time</tt>: converts each \String-embedded date-time into a true \DateTime -# . -# This example creates a converter proc, then stores it: -# strip_converter = proc {|field| field.strip } -# CSV::Converters[:strip] = strip_converter -# Then the parsing method call can refer to the converter -# by its name, <tt>:strip</tt>: -# string = " foo , 0 \n bar , 1 \n baz , 2 \n" -# array = CSV.parse(string, converters: :strip) -# array # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] -# -# The storage structure for header converters is the \Hash CSV::HeaderConverters, -# which works in the same way. -# It also has built-in converter procs: -# - <tt>:downcase</tt>: Downcases each header. -# - <tt>:symbol</tt>: Converts each header to a \Symbol. -# -# There is no such storage structure for write headers. -# -# In order for the parsing methods to access stored converters in non-main-Ractors, the -# storage structure must be made shareable first. -# Therefore, <tt>Ractor.make_shareable(CSV::Converters)</tt> and -# <tt>Ractor.make_shareable(CSV::HeaderConverters)</tt> must be called before the creation -# of Ractors that use the converters stored in these structures. (Since making the storage -# structures shareable involves freezing them, any custom converters that are to be used -# must be added first.) -# -# ===== Converter Lists -# -# A _converter_ _list_ is an \Array that may include any assortment of: -# - Converter procs. -# - Names of stored converters. -# - Nested converter lists. -# -# Examples: -# numeric_converters = [:integer, :float] -# date_converters = [:date, :date_time] -# [numeric_converters, strip_converter] -# [strip_converter, date_converters, :float] -# -# Like a converter proc, a converter list may be named and stored in either -# \CSV::Converters or CSV::HeaderConverters: -# CSV::Converters[:custom] = [strip_converter, date_converters, :float] -# CSV::HeaderConverters[:custom] = [:downcase, :symbol] -# -# There are two built-in converter lists: -# CSV::Converters[:numeric] # => [:integer, :float] -# CSV::Converters[:all] # => [:date_time, :numeric] -# -# ==== Field \Converters -# -# With no conversion, all parsed fields in all rows become Strings: -# string = "foo,0\nbar,1\nbaz,2\n" -# ary = CSV.parse(string) -# ary # => # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] -# -# When you specify a field converter, each parsed field is passed to the converter; -# its return value becomes the stored value for the field. -# A converter might, for example, convert an integer embedded in a \String -# into a true \Integer. -# (In fact, that's what built-in field converter +:integer+ does.) -# -# There are three ways to use field \converters. -# -# - Using option {converters}[#class-CSV-label-Option+converters] with a parsing method: -# ary = CSV.parse(string, converters: :integer) -# ary # => [0, 1, 2] # => [["foo", 0], ["bar", 1], ["baz", 2]] -# - Using option {converters}[#class-CSV-label-Option+converters] with a new \CSV instance: -# csv = CSV.new(string, converters: :integer) -# # Field converters in effect: -# csv.converters # => [:integer] -# csv.read # => [["foo", 0], ["bar", 1], ["baz", 2]] -# - Using method #convert to add a field converter to a \CSV instance: -# csv = CSV.new(string) -# # Add a converter. -# csv.convert(:integer) -# csv.converters # => [:integer] -# csv.read # => [["foo", 0], ["bar", 1], ["baz", 2]] -# -# Installing a field converter does not affect already-read rows: -# csv = CSV.new(string) -# csv.shift # => ["foo", "0"] -# # Add a converter. -# csv.convert(:integer) -# csv.converters # => [:integer] -# csv.read # => [["bar", 1], ["baz", 2]] -# -# There are additional built-in \converters, and custom \converters are also supported. -# -# ===== Built-In Field \Converters -# -# The built-in field converters are in \Hash CSV::Converters: -# - Each key is a field converter name. -# - Each value is one of: -# - A \Proc field converter. -# - An \Array of field converter names. -# -# Display: -# CSV::Converters.each_pair do |name, value| -# if value.kind_of?(Proc) -# p [name, value.class] -# else -# p [name, value] -# end -# end -# Output: -# [:integer, Proc] -# [:float, Proc] -# [:numeric, [:integer, :float]] -# [:date, Proc] -# [:date_time, Proc] -# [:all, [:date_time, :numeric]] -# -# Each of these converters transcodes values to UTF-8 before attempting conversion. -# If a value cannot be transcoded to UTF-8 the conversion will -# fail and the value will remain unconverted. -# -# Converter +:integer+ converts each field that Integer() accepts: -# data = '0,1,2,x' -# # Without the converter -# csv = CSV.parse_line(data) -# csv # => ["0", "1", "2", "x"] -# # With the converter -# csv = CSV.parse_line(data, converters: :integer) -# csv # => [0, 1, 2, "x"] -# -# Converter +:float+ converts each field that Float() accepts: -# data = '1.0,3.14159,x' -# # Without the converter -# csv = CSV.parse_line(data) -# csv # => ["1.0", "3.14159", "x"] -# # With the converter -# csv = CSV.parse_line(data, converters: :float) -# csv # => [1.0, 3.14159, "x"] -# -# Converter +:numeric+ converts with both +:integer+ and +:float+.. -# -# Converter +:date+ converts each field that Date::parse accepts: -# data = '2001-02-03,x' -# # Without the converter -# csv = CSV.parse_line(data) -# csv # => ["2001-02-03", "x"] -# # With the converter -# csv = CSV.parse_line(data, converters: :date) -# csv # => [#<Date: 2001-02-03 ((2451944j,0s,0n),+0s,2299161j)>, "x"] -# -# Converter +:date_time+ converts each field that DateTime::parse accepts: -# data = '2020-05-07T14:59:00-05:00,x' -# # Without the converter -# csv = CSV.parse_line(data) -# csv # => ["2020-05-07T14:59:00-05:00", "x"] -# # With the converter -# csv = CSV.parse_line(data, converters: :date_time) -# csv # => [#<DateTime: 2020-05-07T14:59:00-05:00 ((2458977j,71940s,0n),-18000s,2299161j)>, "x"] -# -# Converter +:numeric+ converts with both +:date_time+ and +:numeric+.. -# -# As seen above, method #convert adds \converters to a \CSV instance, -# and method #converters returns an \Array of the \converters in effect: -# csv = CSV.new('0,1,2') -# csv.converters # => [] -# csv.convert(:integer) -# csv.converters # => [:integer] -# csv.convert(:date) -# csv.converters # => [:integer, :date] -# -# ===== Custom Field \Converters -# -# You can define a custom field converter: -# strip_converter = proc {|field| field.strip } -# string = " foo , 0 \n bar , 1 \n baz , 2 \n" -# array = CSV.parse(string, converters: strip_converter) -# array # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] -# You can register the converter in \Converters \Hash, -# which allows you to refer to it by name: -# CSV::Converters[:strip] = strip_converter -# string = " foo , 0 \n bar , 1 \n baz , 2 \n" -# array = CSV.parse(string, converters: :strip) -# array # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] -# -# ==== Header \Converters -# -# Header converters operate only on headers (and not on other rows). -# -# There are three ways to use header \converters; -# these examples use built-in header converter +:downcase+, -# which downcases each parsed header. -# -# - Option +header_converters+ with a singleton parsing method: -# string = "Name,Count\nFoo,0\n,Bar,1\nBaz,2" -# tbl = CSV.parse(string, headers: true, header_converters: :downcase) -# tbl.class # => CSV::Table -# tbl.headers # => ["name", "count"] -# -# - Option +header_converters+ with a new \CSV instance: -# csv = CSV.new(string, header_converters: :downcase) -# # Header converters in effect: -# csv.header_converters # => [:downcase] -# tbl = CSV.parse(string, headers: true) -# tbl.headers # => ["Name", "Count"] -# -# - Method #header_convert adds a header converter to a \CSV instance: -# csv = CSV.new(string) -# # Add a header converter. -# csv.header_convert(:downcase) -# csv.header_converters # => [:downcase] -# tbl = CSV.parse(string, headers: true) -# tbl.headers # => ["Name", "Count"] -# -# ===== Built-In Header \Converters -# -# The built-in header \converters are in \Hash CSV::HeaderConverters. -# The keys there are the names of the \converters: -# CSV::HeaderConverters.keys # => [:downcase, :symbol] -# -# Converter +:downcase+ converts each header by downcasing it: -# string = "Name,Count\nFoo,0\n,Bar,1\nBaz,2" -# tbl = CSV.parse(string, headers: true, header_converters: :downcase) -# tbl.class # => CSV::Table -# tbl.headers # => ["name", "count"] -# -# Converter +:symbol+ converts each header by making it into a \Symbol: -# string = "Name,Count\nFoo,0\n,Bar,1\nBaz,2" -# tbl = CSV.parse(string, headers: true, header_converters: :symbol) -# tbl.headers # => [:name, :count] -# Details: -# - Strips leading and trailing whitespace. -# - Downcases the header. -# - Replaces embedded spaces with underscores. -# - Removes non-word characters. -# - Makes the string into a \Symbol. -# -# ===== Custom Header \Converters -# -# You can define a custom header converter: -# upcase_converter = proc {|header| header.upcase } -# string = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" -# table = CSV.parse(string, headers: true, header_converters: upcase_converter) -# table # => #<CSV::Table mode:col_or_row row_count:4> -# table.headers # => ["NAME", "VALUE"] -# You can register the converter in \HeaderConverters \Hash, -# which allows you to refer to it by name: -# CSV::HeaderConverters[:upcase] = upcase_converter -# table = CSV.parse(string, headers: true, header_converters: :upcase) -# table # => #<CSV::Table mode:col_or_row row_count:4> -# table.headers # => ["NAME", "VALUE"] -# -# ===== Write \Converters -# -# When you specify a write converter for generating \CSV, -# each field to be written is passed to the converter; -# its return value becomes the new value for the field. -# A converter might, for example, strip whitespace from a field. -# -# Using no write converter (all fields unmodified): -# output_string = CSV.generate do |csv| -# csv << [' foo ', 0] -# csv << [' bar ', 1] -# csv << [' baz ', 2] -# end -# output_string # => " foo ,0\n bar ,1\n baz ,2\n" -# Using option +write_converters+ with two custom write converters: -# strip_converter = proc {|field| field.respond_to?(:strip) ? field.strip : field } -# upcase_converter = proc {|field| field.respond_to?(:upcase) ? field.upcase : field } -# write_converters = [strip_converter, upcase_converter] -# output_string = CSV.generate(write_converters: write_converters) do |csv| -# csv << [' foo ', 0] -# csv << [' bar ', 1] -# csv << [' baz ', 2] -# end -# output_string # => "FOO,0\nBAR,1\nBAZ,2\n" -# -# === Character Encodings (M17n or Multilingualization) -# -# This new CSV parser is m17n savvy. The parser works in the Encoding of the IO -# or String object being read from or written to. Your data is never transcoded -# (unless you ask Ruby to transcode it for you) and will literally be parsed in -# the Encoding it is in. Thus CSV will return Arrays or Rows of Strings in the -# Encoding of your data. This is accomplished by transcoding the parser itself -# into your Encoding. -# -# Some transcoding must take place, of course, to accomplish this multiencoding -# support. For example, <tt>:col_sep</tt>, <tt>:row_sep</tt>, and -# <tt>:quote_char</tt> must be transcoded to match your data. Hopefully this -# makes the entire process feel transparent, since CSV's defaults should just -# magically work for your data. However, you can set these values manually in -# the target Encoding to avoid the translation. -# -# It's also important to note that while all of CSV's core parser is now -# Encoding agnostic, some features are not. For example, the built-in -# converters will try to transcode data to UTF-8 before making conversions. -# Again, you can provide custom converters that are aware of your Encodings to -# avoid this translation. It's just too hard for me to support native -# conversions in all of Ruby's Encodings. -# -# Anyway, the practical side of this is simple: make sure IO and String objects -# passed into CSV have the proper Encoding set and everything should just work. -# CSV methods that allow you to open IO objects (CSV::foreach(), CSV::open(), -# CSV::read(), and CSV::readlines()) do allow you to specify the Encoding. -# -# One minor exception comes when generating CSV into a String with an Encoding -# that is not ASCII compatible. There's no existing data for CSV to use to -# prepare itself and thus you will probably need to manually specify the desired -# Encoding for most of those cases. It will try to guess using the fields in a -# row of output though, when using CSV::generate_line() or Array#to_csv(). -# -# I try to point out any other Encoding issues in the documentation of methods -# as they come up. -# -# This has been tested to the best of my ability with all non-"dummy" Encodings -# Ruby ships with. However, it is brave new code and may have some bugs. -# Please feel free to {report}[mailto:james@grayproductions.net] any issues you -# find with it. -# -class CSV - - # The error thrown when the parser encounters illegal CSV formatting. - class MalformedCSVError < RuntimeError - attr_reader :line_number - alias_method :lineno, :line_number - def initialize(message, line_number) - @line_number = line_number - super("#{message} in line #{line_number}.") - end - end - - # - # A FieldInfo Struct contains details about a field's position in the data - # source it was read from. CSV will pass this Struct to some blocks that make - # decisions based on field structure. See CSV.convert_fields() for an - # example. - # - # <b><tt>index</tt></b>:: The zero-based index of the field in its row. - # <b><tt>line</tt></b>:: The line of the data source this row is from. - # <b><tt>header</tt></b>:: The header for the column, when available. - # <b><tt>quoted?</tt></b>:: True or false, whether the original value is quoted or not. - # - FieldInfo = Struct.new(:index, :line, :header, :quoted?) - - # A Regexp used to find and convert some common Date formats. - DateMatcher = / \A(?: (\w+,?\s+)?\w+\s+\d{1,2},?\s+\d{2,4} | - \d{4}-\d{2}-\d{2} )\z /x - # A Regexp used to find and convert some common DateTime formats. - DateTimeMatcher = - / \A(?: (\w+,?\s+)?\w+\s+\d{1,2}\s+\d{1,2}:\d{1,2}:\d{1,2},?\s+\d{2,4} | - # ISO-8601 and RFC-3339 (space instead of T) recognized by DateTime.parse - \d{4}-\d{2}-\d{2} - (?:[T\s]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?(?:[+-]\d{2}(?::\d{2})|Z)?)?)? - )\z /x - - # The encoding used by all converters. - ConverterEncoding = Encoding.find("UTF-8") - - # A \Hash containing the names and \Procs for the built-in field converters. - # See {Built-In Field Converters}[#class-CSV-label-Built-In+Field+Converters]. - # - # This \Hash is intentionally left unfrozen, and may be extended with - # custom field converters. - # See {Custom Field Converters}[#class-CSV-label-Custom+Field+Converters]. - Converters = { - integer: lambda { |f| - Integer(f.encode(ConverterEncoding)) rescue f - }, - float: lambda { |f| - Float(f.encode(ConverterEncoding)) rescue f - }, - numeric: [:integer, :float], - date: lambda { |f| - begin - e = f.encode(ConverterEncoding) - e.match?(DateMatcher) ? Date.parse(e) : f - rescue # encoding conversion or date parse errors - f - end - }, - date_time: lambda { |f| - begin - e = f.encode(ConverterEncoding) - e.match?(DateTimeMatcher) ? DateTime.parse(e) : f - rescue # encoding conversion or date parse errors - f - end - }, - all: [:date_time, :numeric], - } - - # A \Hash containing the names and \Procs for the built-in header converters. - # See {Built-In Header Converters}[#class-CSV-label-Built-In+Header+Converters]. - # - # This \Hash is intentionally left unfrozen, and may be extended with - # custom field converters. - # See {Custom Header Converters}[#class-CSV-label-Custom+Header+Converters]. - HeaderConverters = { - downcase: lambda { |h| h.encode(ConverterEncoding).downcase }, - symbol: lambda { |h| - h.encode(ConverterEncoding).downcase.gsub(/[^\s\w]+/, "").strip. - gsub(/\s+/, "_").to_sym - }, - symbol_raw: lambda { |h| h.encode(ConverterEncoding).to_sym } - } - - # Default values for method options. - DEFAULT_OPTIONS = { - # For both parsing and generating. - col_sep: ",", - row_sep: :auto, - quote_char: '"', - # For parsing. - field_size_limit: nil, - max_field_size: nil, - converters: nil, - unconverted_fields: nil, - headers: false, - return_headers: false, - header_converters: nil, - skip_blanks: false, - skip_lines: nil, - liberal_parsing: false, - nil_value: nil, - empty_value: "", - strip: false, - # For generating. - write_headers: nil, - quote_empty: true, - force_quotes: false, - write_converters: nil, - write_nil_value: nil, - write_empty_value: "", - }.freeze - - class << self - # :call-seq: - # instance(string, **options) - # instance(io = $stdout, **options) - # instance(string, **options) {|csv| ... } - # instance(io = $stdout, **options) {|csv| ... } - # - # Creates or retrieves cached \CSV objects. - # For arguments and options, see CSV.new. - # - # This API is not Ractor-safe. - # - # --- - # - # With no block given, returns a \CSV object. - # - # The first call to +instance+ creates and caches a \CSV object: - # s0 = 's0' - # csv0 = CSV.instance(s0) - # csv0.class # => CSV - # - # Subsequent calls to +instance+ with that _same_ +string+ or +io+ - # retrieve that same cached object: - # csv1 = CSV.instance(s0) - # csv1.class # => CSV - # csv1.equal?(csv0) # => true # Same CSV object - # - # A subsequent call to +instance+ with a _different_ +string+ or +io+ - # creates and caches a _different_ \CSV object. - # s1 = 's1' - # csv2 = CSV.instance(s1) - # csv2.equal?(csv0) # => false # Different CSV object - # - # All the cached objects remains available: - # csv3 = CSV.instance(s0) - # csv3.equal?(csv0) # true # Same CSV object - # csv4 = CSV.instance(s1) - # csv4.equal?(csv2) # true # Same CSV object - # - # --- - # - # When a block is given, calls the block with the created or retrieved - # \CSV object; returns the block's return value: - # CSV.instance(s0) {|csv| :foo } # => :foo - def instance(data = $stdout, **options) - # create a _signature_ for this method call, data object and options - sig = [data.object_id] + - options.values_at(*DEFAULT_OPTIONS.keys) - - # fetch or create the instance for this signature - @@instances ||= Hash.new - instance = (@@instances[sig] ||= new(data, **options)) - - if block_given? - yield instance # run block, if given, returning result - else - instance # or return the instance - end - end - - # :call-seq: - # filter(in_string_or_io, **options) {|row| ... } -> array_of_arrays or csv_table - # filter(in_string_or_io, out_string_or_io, **options) {|row| ... } -> array_of_arrays or csv_table - # filter(**options) {|row| ... } -> array_of_arrays or csv_table - # - # - Parses \CSV from a source (\String, \IO stream, or ARGF). - # - Calls the given block with each parsed row: - # - Without headers, each row is an \Array. - # - With headers, each row is a CSV::Row. - # - Generates \CSV to an output (\String, \IO stream, or STDOUT). - # - Returns the parsed source: - # - Without headers, an \Array of \Arrays. - # - With headers, a CSV::Table. - # - # When +in_string_or_io+ is given, but not +out_string_or_io+, - # parses from the given +in_string_or_io+ - # and generates to STDOUT. - # - # \String input without headers: - # - # in_string = "foo,0\nbar,1\nbaz,2" - # CSV.filter(in_string) do |row| - # row[0].upcase! - # row[1] = - row[1].to_i - # end # => [["FOO", 0], ["BAR", -1], ["BAZ", -2]] - # - # Output (to STDOUT): - # - # FOO,0 - # BAR,-1 - # BAZ,-2 - # - # \String input with headers: - # - # in_string = "Name,Value\nfoo,0\nbar,1\nbaz,2" - # CSV.filter(in_string, headers: true) do |row| - # row[0].upcase! - # row[1] = - row[1].to_i - # end # => #<CSV::Table mode:col_or_row row_count:4> - # - # Output (to STDOUT): - # - # Name,Value - # FOO,0 - # BAR,-1 - # BAZ,-2 - # - # \IO stream input without headers: - # - # File.write('t.csv', "foo,0\nbar,1\nbaz,2") - # File.open('t.csv') do |in_io| - # CSV.filter(in_io) do |row| - # row[0].upcase! - # row[1] = - row[1].to_i - # end - # end # => [["FOO", 0], ["BAR", -1], ["BAZ", -2]] - # - # Output (to STDOUT): - # - # FOO,0 - # BAR,-1 - # BAZ,-2 - # - # \IO stream input with headers: - # - # File.write('t.csv', "Name,Value\nfoo,0\nbar,1\nbaz,2") - # File.open('t.csv') do |in_io| - # CSV.filter(in_io, headers: true) do |row| - # row[0].upcase! - # row[1] = - row[1].to_i - # end - # end # => #<CSV::Table mode:col_or_row row_count:4> - # - # Output (to STDOUT): - # - # Name,Value - # FOO,0 - # BAR,-1 - # BAZ,-2 - # - # When both +in_string_or_io+ and +out_string_or_io+ are given, - # parses from +in_string_or_io+ and generates to +out_string_or_io+. - # - # \String output without headers: - # - # in_string = "foo,0\nbar,1\nbaz,2" - # out_string = '' - # CSV.filter(in_string, out_string) do |row| - # row[0].upcase! - # row[1] = - row[1].to_i - # end # => [["FOO", 0], ["BAR", -1], ["BAZ", -2]] - # out_string # => "FOO,0\nBAR,-1\nBAZ,-2\n" - # - # \String output with headers: - # - # in_string = "Name,Value\nfoo,0\nbar,1\nbaz,2" - # out_string = '' - # CSV.filter(in_string, out_string, headers: true) do |row| - # row[0].upcase! - # row[1] = - row[1].to_i - # end # => #<CSV::Table mode:col_or_row row_count:4> - # out_string # => "Name,Value\nFOO,0\nBAR,-1\nBAZ,-2\n" - # - # \IO stream output without headers: - # - # in_string = "foo,0\nbar,1\nbaz,2" - # File.open('t.csv', 'w') do |out_io| - # CSV.filter(in_string, out_io) do |row| - # row[0].upcase! - # row[1] = - row[1].to_i - # end - # end # => [["FOO", 0], ["BAR", -1], ["BAZ", -2]] - # File.read('t.csv') # => "FOO,0\nBAR,-1\nBAZ,-2\n" - # - # \IO stream output with headers: - # - # in_string = "Name,Value\nfoo,0\nbar,1\nbaz,2" - # File.open('t.csv', 'w') do |out_io| - # CSV.filter(in_string, out_io, headers: true) do |row| - # row[0].upcase! - # row[1] = - row[1].to_i - # end - # end # => #<CSV::Table mode:col_or_row row_count:4> - # File.read('t.csv') # => "Name,Value\nFOO,0\nBAR,-1\nBAZ,-2\n" - # - # When neither +in_string_or_io+ nor +out_string_or_io+ given, - # parses from {ARGF}[rdoc-ref:ARGF] - # and generates to STDOUT. - # - # Without headers: - # - # # Put Ruby code into a file. - # ruby = <<-EOT - # require 'csv' - # CSV.filter do |row| - # row[0].upcase! - # row[1] = - row[1].to_i - # end - # EOT - # File.write('t.rb', ruby) - # # Put some CSV into a file. - # File.write('t.csv', "foo,0\nbar,1\nbaz,2") - # # Run the Ruby code with CSV filename as argument. - # system(Gem.ruby, "t.rb", "t.csv") - # - # Output (to STDOUT): - # - # FOO,0 - # BAR,-1 - # BAZ,-2 - # - # With headers: - # - # # Put Ruby code into a file. - # ruby = <<-EOT - # require 'csv' - # CSV.filter(headers: true) do |row| - # row[0].upcase! - # row[1] = - row[1].to_i - # end - # EOT - # File.write('t.rb', ruby) - # # Put some CSV into a file. - # File.write('t.csv', "Name,Value\nfoo,0\nbar,1\nbaz,2") - # # Run the Ruby code with CSV filename as argument. - # system(Gem.ruby, "t.rb", "t.csv") - # - # Output (to STDOUT): - # - # Name,Value - # FOO,0 - # BAR,-1 - # BAZ,-2 - # - # Arguments: - # - # * Argument +in_string_or_io+ must be a \String or an \IO stream. - # * Argument +out_string_or_io+ must be a \String or an \IO stream. - # * Arguments <tt>**options</tt> must be keyword options. - # See {Options for Parsing}[#class-CSV-label-Options+for+Parsing]. - def filter(input=nil, output=nil, **options) - # parse options for input, output, or both - in_options, out_options = Hash.new, {row_sep: InputRecordSeparator.value} - options.each do |key, value| - case key - when /\Ain(?:put)?_(.+)\Z/ - in_options[$1.to_sym] = value - when /\Aout(?:put)?_(.+)\Z/ - out_options[$1.to_sym] = value - else - in_options[key] = value - out_options[key] = value - end - end - - # build input and output wrappers - input = new(input || ARGF, **in_options) - output = new(output || $stdout, **out_options) - - # process headers - need_manual_header_output = - (in_options[:headers] and - out_options[:headers] == true and - out_options[:write_headers]) - if need_manual_header_output - first_row = input.shift - if first_row - if first_row.is_a?(Row) - headers = first_row.headers - yield headers - output << headers - end - yield first_row - output << first_row - end - end - - # read, yield, write - input.each do |row| - yield row - output << row - end - end - - # - # :call-seq: - # foreach(path_or_io, mode='r', **options) {|row| ... ) - # foreach(path_or_io, mode='r', **options) -> new_enumerator - # - # Calls the block with each row read from source +path_or_io+. - # - # \Path input without headers: - # - # string = "foo,0\nbar,1\nbaz,2\n" - # in_path = 't.csv' - # File.write(in_path, string) - # CSV.foreach(in_path) {|row| p row } - # - # Output: - # - # ["foo", "0"] - # ["bar", "1"] - # ["baz", "2"] - # - # \Path input with headers: - # - # string = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # in_path = 't.csv' - # File.write(in_path, string) - # CSV.foreach(in_path, headers: true) {|row| p row } - # - # Output: - # - # <CSV::Row "Name":"foo" "Value":"0"> - # <CSV::Row "Name":"bar" "Value":"1"> - # <CSV::Row "Name":"baz" "Value":"2"> - # - # \IO stream input without headers: - # - # string = "foo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # File.open('t.csv') do |in_io| - # CSV.foreach(in_io) {|row| p row } - # end - # - # Output: - # - # ["foo", "0"] - # ["bar", "1"] - # ["baz", "2"] - # - # \IO stream input with headers: - # - # string = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # File.open('t.csv') do |in_io| - # CSV.foreach(in_io, headers: true) {|row| p row } - # end - # - # Output: - # - # <CSV::Row "Name":"foo" "Value":"0"> - # <CSV::Row "Name":"bar" "Value":"1"> - # <CSV::Row "Name":"baz" "Value":"2"> - # - # With no block given, returns an \Enumerator: - # - # string = "foo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # CSV.foreach(path) # => #<Enumerator: CSV:foreach("t.csv", "r")> - # - # Arguments: - # * Argument +path_or_io+ must be a file path or an \IO stream. - # * Argument +mode+, if given, must be a \File mode - # See {Open Mode}[https://ruby-doc.org/core/IO.html#method-c-new-label-Open+Mode]. - # * Arguments <tt>**options</tt> must be keyword options. - # See {Options for Parsing}[#class-CSV-label-Options+for+Parsing]. - # * This method optionally accepts an additional <tt>:encoding</tt> option - # that you can use to specify the Encoding of the data read from +path+ or +io+. - # You must provide this unless your data is in the encoding - # given by <tt>Encoding::default_external</tt>. - # Parsing will use this to determine how to parse the data. - # You may provide a second Encoding to - # have the data transcoded as it is read. For example, - # encoding: 'UTF-32BE:UTF-8' - # would read +UTF-32BE+ data from the file - # but transcode it to +UTF-8+ before parsing. - def foreach(path, mode="r", **options, &block) - return to_enum(__method__, path, mode, **options) unless block_given? - open(path, mode, **options) do |csv| - csv.each(&block) - end - end - - # - # :call-seq: - # generate(csv_string, **options) {|csv| ... } - # generate(**options) {|csv| ... } - # - # * Argument +csv_string+, if given, must be a \String object; - # defaults to a new empty \String. - # * Arguments +options+, if given, should be generating options. - # See {Options for Generating}[#class-CSV-label-Options+for+Generating]. - # - # --- - # - # Creates a new \CSV object via <tt>CSV.new(csv_string, **options)</tt>; - # calls the block with the \CSV object, which the block may modify; - # returns the \String generated from the \CSV object. - # - # Note that a passed \String *is* modified by this method. - # Pass <tt>csv_string</tt>.dup if the \String must be preserved. - # - # This method has one additional option: <tt>:encoding</tt>, - # which sets the base Encoding for the output if no no +str+ is specified. - # CSV needs this hint if you plan to output non-ASCII compatible data. - # - # --- - # - # Add lines: - # input_string = "foo,0\nbar,1\nbaz,2\n" - # output_string = CSV.generate(input_string) do |csv| - # csv << ['bat', 3] - # csv << ['bam', 4] - # end - # output_string # => "foo,0\nbar,1\nbaz,2\nbat,3\nbam,4\n" - # input_string # => "foo,0\nbar,1\nbaz,2\nbat,3\nbam,4\n" - # output_string.equal?(input_string) # => true # Same string, modified - # - # Add lines into new string, preserving old string: - # input_string = "foo,0\nbar,1\nbaz,2\n" - # output_string = CSV.generate(input_string.dup) do |csv| - # csv << ['bat', 3] - # csv << ['bam', 4] - # end - # output_string # => "foo,0\nbar,1\nbaz,2\nbat,3\nbam,4\n" - # input_string # => "foo,0\nbar,1\nbaz,2\n" - # output_string.equal?(input_string) # => false # Different strings - # - # Create lines from nothing: - # output_string = CSV.generate do |csv| - # csv << ['foo', 0] - # csv << ['bar', 1] - # csv << ['baz', 2] - # end - # output_string # => "foo,0\nbar,1\nbaz,2\n" - # - # --- - # - # Raises an exception if +csv_string+ is not a \String object: - # # Raises TypeError (no implicit conversion of Integer into String) - # CSV.generate(0) - # - def generate(str=nil, **options) - encoding = options[:encoding] - # add a default empty String, if none was given - if str - str = StringIO.new(str) - str.seek(0, IO::SEEK_END) - str.set_encoding(encoding) if encoding - else - str = +"" - str.force_encoding(encoding) if encoding - end - csv = new(str, **options) # wrap - yield csv # yield for appending - csv.string # return final String - end - - # :call-seq: - # CSV.generate_line(ary) - # CSV.generate_line(ary, **options) - # - # Returns the \String created by generating \CSV from +ary+ - # using the specified +options+. - # - # Argument +ary+ must be an \Array. - # - # Special options: - # * Option <tt>:row_sep</tt> defaults to <tt>"\n"> on Ruby 3.0 or later - # and <tt>$INPUT_RECORD_SEPARATOR</tt> (<tt>$/</tt>) otherwise.: - # $INPUT_RECORD_SEPARATOR # => "\n" - # * This method accepts an additional option, <tt>:encoding</tt>, which sets the base - # Encoding for the output. This method will try to guess your Encoding from - # the first non-+nil+ field in +row+, if possible, but you may need to use - # this parameter as a backup plan. - # - # For other +options+, - # see {Options for Generating}[#class-CSV-label-Options+for+Generating]. - # - # --- - # - # Returns the \String generated from an \Array: - # CSV.generate_line(['foo', '0']) # => "foo,0\n" - # - # --- - # - # Raises an exception if +ary+ is not an \Array: - # # Raises NoMethodError (undefined method `find' for :foo:Symbol) - # CSV.generate_line(:foo) - # - def generate_line(row, **options) - options = {row_sep: InputRecordSeparator.value}.merge(options) - str = +"" - if options[:encoding] - str.force_encoding(options[:encoding]) - else - fallback_encoding = nil - output_encoding = nil - row.each do |field| - next unless field.is_a?(String) - fallback_encoding ||= field.encoding - next if field.ascii_only? - output_encoding = field.encoding - break - end - output_encoding ||= fallback_encoding - if output_encoding - str.force_encoding(output_encoding) - end - end - (new(str, **options) << row).string - end - - # :call-seq: - # CSV.generate_lines(rows) - # CSV.generate_lines(rows, **options) - # - # Returns the \String created by generating \CSV from - # using the specified +options+. - # - # Argument +rows+ must be an \Array of row. Row is \Array of \String or \CSV::Row. - # - # Special options: - # * Option <tt>:row_sep</tt> defaults to <tt>"\n"</tt> on Ruby 3.0 or later - # and <tt>$INPUT_RECORD_SEPARATOR</tt> (<tt>$/</tt>) otherwise.: - # $INPUT_RECORD_SEPARATOR # => "\n" - # * This method accepts an additional option, <tt>:encoding</tt>, which sets the base - # Encoding for the output. This method will try to guess your Encoding from - # the first non-+nil+ field in +row+, if possible, but you may need to use - # this parameter as a backup plan. - # - # For other +options+, - # see {Options for Generating}[#class-CSV-label-Options+for+Generating]. - # - # --- - # - # Returns the \String generated from an - # CSV.generate_lines([['foo', '0'], ['bar', '1'], ['baz', '2']]) # => "foo,0\nbar,1\nbaz,2\n" - # - # --- - # - # Raises an exception - # # Raises NoMethodError (undefined method `each' for :foo:Symbol) - # CSV.generate_lines(:foo) - # - def generate_lines(rows, **options) - self.generate(**options) do |csv| - rows.each do |row| - csv << row - end - end - end - - # - # :call-seq: - # open(file_path, mode = "rb", **options ) -> new_csv - # open(io, mode = "rb", **options ) -> new_csv - # open(file_path, mode = "rb", **options ) { |csv| ... } -> object - # open(io, mode = "rb", **options ) { |csv| ... } -> object - # - # possible options elements: - # keyword form: - # :invalid => nil # raise error on invalid byte sequence (default) - # :invalid => :replace # replace invalid byte sequence - # :undef => :replace # replace undefined conversion - # :replace => string # replacement string ("?" or "\uFFFD" if not specified) - # - # * Argument +path+, if given, must be the path to a file. - # :include: ../doc/csv/arguments/io.rdoc - # * Argument +mode+, if given, must be a \File mode - # See {Open Mode}[IO.html#method-c-new-label-Open+Mode]. - # * Arguments <tt>**options</tt> must be keyword options. - # See {Options for Generating}[#class-CSV-label-Options+for+Generating]. - # * This method optionally accepts an additional <tt>:encoding</tt> option - # that you can use to specify the Encoding of the data read from +path+ or +io+. - # You must provide this unless your data is in the encoding - # given by <tt>Encoding::default_external</tt>. - # Parsing will use this to determine how to parse the data. - # You may provide a second Encoding to - # have the data transcoded as it is read. For example, - # encoding: 'UTF-32BE:UTF-8' - # would read +UTF-32BE+ data from the file - # but transcode it to +UTF-8+ before parsing. - # - # --- - # - # These examples assume prior execution of: - # string = "foo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # - # --- - # - # With no block given, returns a new \CSV object. - # - # Create a \CSV object using a file path: - # csv = CSV.open(path) - # csv # => #<CSV io_type:File io_path:"t.csv" encoding:UTF-8 lineno:0 col_sep:"," row_sep:"\n" quote_char:"\""> - # - # Create a \CSV object using an open \File: - # csv = CSV.open(File.open(path)) - # csv # => #<CSV io_type:File io_path:"t.csv" encoding:UTF-8 lineno:0 col_sep:"," row_sep:"\n" quote_char:"\""> - # - # --- - # - # With a block given, calls the block with the created \CSV object; - # returns the block's return value: - # - # Using a file path: - # csv = CSV.open(path) {|csv| p csv} - # csv # => #<CSV io_type:File io_path:"t.csv" encoding:UTF-8 lineno:0 col_sep:"," row_sep:"\n" quote_char:"\""> - # Output: - # #<CSV io_type:File io_path:"t.csv" encoding:UTF-8 lineno:0 col_sep:"," row_sep:"\n" quote_char:"\""> - # - # Using an open \File: - # csv = CSV.open(File.open(path)) {|csv| p csv} - # csv # => #<CSV io_type:File io_path:"t.csv" encoding:UTF-8 lineno:0 col_sep:"," row_sep:"\n" quote_char:"\""> - # Output: - # #<CSV io_type:File io_path:"t.csv" encoding:UTF-8 lineno:0 col_sep:"," row_sep:"\n" quote_char:"\""> - # - # --- - # - # Raises an exception if the argument is not a \String object or \IO object: - # # Raises TypeError (no implicit conversion of Symbol into String) - # CSV.open(:foo) - def open(filename, mode="r", **options) - # wrap a File opened with the remaining +args+ with no newline - # decorator - file_opts = options.dup - unless file_opts.key?(:newline) - file_opts[:universal_newline] ||= false - end - options.delete(:invalid) - options.delete(:undef) - options.delete(:replace) - options.delete_if {|k, _| /newline\z/.match?(k)} - - begin - f = File.open(filename, mode, **file_opts) - rescue ArgumentError => e - raise unless /needs binmode/.match?(e.message) and mode == "r" - mode = "rb" - file_opts = {encoding: Encoding.default_external}.merge(file_opts) - retry - end - begin - csv = new(f, **options) - rescue Exception - f.close - raise - end - - # handle blocks like Ruby's open(), not like the CSV library - if block_given? - begin - yield csv - ensure - csv.close - end - else - csv - end - end - - # - # :call-seq: - # parse(string) -> array_of_arrays - # parse(io) -> array_of_arrays - # parse(string, headers: ..., **options) -> csv_table - # parse(io, headers: ..., **options) -> csv_table - # parse(string, **options) {|row| ... } - # parse(io, **options) {|row| ... } - # - # Parses +string+ or +io+ using the specified +options+. - # - # - Argument +string+ should be a \String object; - # it will be put into a new StringIO object positioned at the beginning. - # :include: ../doc/csv/arguments/io.rdoc - # - Argument +options+: see {Options for Parsing}[#class-CSV-label-Options+for+Parsing] - # - # ====== Without Option +headers+ - # - # Without {option +headers+}[#class-CSV-label-Option+headers] case. - # - # These examples assume prior execution of: - # string = "foo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # - # --- - # - # With no block given, returns an \Array of Arrays formed from the source. - # - # Parse a \String: - # a_of_a = CSV.parse(string) - # a_of_a # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] - # - # Parse an open \File: - # a_of_a = File.open(path) do |file| - # CSV.parse(file) - # end - # a_of_a # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] - # - # --- - # - # With a block given, calls the block with each parsed row: - # - # Parse a \String: - # CSV.parse(string) {|row| p row } - # - # Output: - # ["foo", "0"] - # ["bar", "1"] - # ["baz", "2"] - # - # Parse an open \File: - # File.open(path) do |file| - # CSV.parse(file) {|row| p row } - # end - # - # Output: - # ["foo", "0"] - # ["bar", "1"] - # ["baz", "2"] - # - # ====== With Option +headers+ - # - # With {option +headers+}[#class-CSV-label-Option+headers] case. - # - # These examples assume prior execution of: - # string = "Name,Count\nfoo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # - # --- - # - # With no block given, returns a CSV::Table object formed from the source. - # - # Parse a \String: - # csv_table = CSV.parse(string, headers: ['Name', 'Count']) - # csv_table # => #<CSV::Table mode:col_or_row row_count:5> - # - # Parse an open \File: - # csv_table = File.open(path) do |file| - # CSV.parse(file, headers: ['Name', 'Count']) - # end - # csv_table # => #<CSV::Table mode:col_or_row row_count:4> - # - # --- - # - # With a block given, calls the block with each parsed row, - # which has been formed into a CSV::Row object: - # - # Parse a \String: - # CSV.parse(string, headers: ['Name', 'Count']) {|row| p row } - # - # Output: - # # <CSV::Row "Name":"foo" "Count":"0"> - # # <CSV::Row "Name":"bar" "Count":"1"> - # # <CSV::Row "Name":"baz" "Count":"2"> - # - # Parse an open \File: - # File.open(path) do |file| - # CSV.parse(file, headers: ['Name', 'Count']) {|row| p row } - # end - # - # Output: - # # <CSV::Row "Name":"foo" "Count":"0"> - # # <CSV::Row "Name":"bar" "Count":"1"> - # # <CSV::Row "Name":"baz" "Count":"2"> - # - # --- - # - # Raises an exception if the argument is not a \String object or \IO object: - # # Raises NoMethodError (undefined method `close' for :foo:Symbol) - # CSV.parse(:foo) - def parse(str, **options, &block) - csv = new(str, **options) - - return csv.each(&block) if block_given? - - # slurp contents, if no block is given - begin - csv.read - ensure - csv.close - end - end - - # :call-seq: - # CSV.parse_line(string) -> new_array or nil - # CSV.parse_line(io) -> new_array or nil - # CSV.parse_line(string, **options) -> new_array or nil - # CSV.parse_line(io, **options) -> new_array or nil - # CSV.parse_line(string, headers: true, **options) -> csv_row or nil - # CSV.parse_line(io, headers: true, **options) -> csv_row or nil - # - # Returns the data created by parsing the first line of +string+ or +io+ - # using the specified +options+. - # - # - Argument +string+ should be a \String object; - # it will be put into a new StringIO object positioned at the beginning. - # :include: ../doc/csv/arguments/io.rdoc - # - Argument +options+: see {Options for Parsing}[#class-CSV-label-Options+for+Parsing] - # - # ====== Without Option +headers+ - # - # Without option +headers+, returns the first row as a new \Array. - # - # These examples assume prior execution of: - # string = "foo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # - # Parse the first line from a \String object: - # CSV.parse_line(string) # => ["foo", "0"] - # - # Parse the first line from a File object: - # File.open(path) do |file| - # CSV.parse_line(file) # => ["foo", "0"] - # end # => ["foo", "0"] - # - # Returns +nil+ if the argument is an empty \String: - # CSV.parse_line('') # => nil - # - # ====== With Option +headers+ - # - # With {option +headers+}[#class-CSV-label-Option+headers], - # returns the first row as a CSV::Row object. - # - # These examples assume prior execution of: - # string = "Name,Count\nfoo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # - # Parse the first line from a \String object: - # CSV.parse_line(string, headers: true) # => #<CSV::Row "Name":"foo" "Count":"0"> - # - # Parse the first line from a File object: - # File.open(path) do |file| - # CSV.parse_line(file, headers: true) - # end # => #<CSV::Row "Name":"foo" "Count":"0"> - # - # --- - # - # Raises an exception if the argument is +nil+: - # # Raises ArgumentError (Cannot parse nil as CSV): - # CSV.parse_line(nil) - # - def parse_line(line, **options) - new(line, **options).each.first - end - - # - # :call-seq: - # read(source, **options) -> array_of_arrays - # read(source, headers: true, **options) -> csv_table - # - # Opens the given +source+ with the given +options+ (see CSV.open), - # reads the source (see CSV#read), and returns the result, - # which will be either an \Array of Arrays or a CSV::Table. - # - # Without headers: - # string = "foo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # CSV.read(path) # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] - # - # With headers: - # string = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # CSV.read(path, headers: true) # => #<CSV::Table mode:col_or_row row_count:4> - def read(path, **options) - open(path, **options) { |csv| csv.read } - end - - # :call-seq: - # CSV.readlines(source, **options) - # - # Alias for CSV.read. - def readlines(path, **options) - read(path, **options) - end - - # :call-seq: - # CSV.table(source, **options) - # - # Calls CSV.read with +source+, +options+, and certain default options: - # - +headers+: +true+ - # - +converters+: +:numeric+ - # - +header_converters+: +:symbol+ - # - # Returns a CSV::Table object. - # - # Example: - # string = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # CSV.table(path) # => #<CSV::Table mode:col_or_row row_count:4> - def table(path, **options) - default_options = { - headers: true, - converters: :numeric, - header_converters: :symbol, - } - options = default_options.merge(options) - read(path, **options) - end - end - - # :call-seq: - # CSV.new(string) - # CSV.new(io) - # CSV.new(string, **options) - # CSV.new(io, **options) - # - # Returns the new \CSV object created using +string+ or +io+ - # and the specified +options+. - # - # - Argument +string+ should be a \String object; - # it will be put into a new StringIO object positioned at the beginning. - # :include: ../doc/csv/arguments/io.rdoc - # - Argument +options+: See: - # * {Options for Parsing}[#class-CSV-label-Options+for+Parsing] - # * {Options for Generating}[#class-CSV-label-Options+for+Generating] - # For performance reasons, the options cannot be overridden - # in a \CSV object, so those specified here will endure. - # - # In addition to the \CSV instance methods, several \IO methods are delegated. - # See {Delegated Methods}[#class-CSV-label-Delegated+Methods]. - # - # --- - # - # Create a \CSV object from a \String object: - # csv = CSV.new('foo,0') - # csv # => #<CSV io_type:StringIO encoding:UTF-8 lineno:0 col_sep:"," row_sep:"\n" quote_char:"\""> - # - # Create a \CSV object from a \File object: - # File.write('t.csv', 'foo,0') - # csv = CSV.new(File.open('t.csv')) - # csv # => #<CSV io_type:File io_path:"t.csv" encoding:UTF-8 lineno:0 col_sep:"," row_sep:"\n" quote_char:"\""> - # - # --- - # - # Raises an exception if the argument is +nil+: - # # Raises ArgumentError (Cannot parse nil as CSV): - # CSV.new(nil) - # - def initialize(data, - col_sep: ",", - row_sep: :auto, - quote_char: '"', - field_size_limit: nil, - max_field_size: nil, - converters: nil, - unconverted_fields: nil, - headers: false, - return_headers: false, - write_headers: nil, - header_converters: nil, - skip_blanks: false, - force_quotes: false, - skip_lines: nil, - liberal_parsing: false, - internal_encoding: nil, - external_encoding: nil, - encoding: nil, - nil_value: nil, - empty_value: "", - strip: false, - quote_empty: true, - write_converters: nil, - write_nil_value: nil, - write_empty_value: "") - raise ArgumentError.new("Cannot parse nil as CSV") if data.nil? - - if data.is_a?(String) - if encoding - if encoding.is_a?(String) - data_external_encoding, data_internal_encoding = encoding.split(":", 2) - if data_internal_encoding - data = data.encode(data_internal_encoding, data_external_encoding) - else - data = data.dup.force_encoding(data_external_encoding) - end - else - data = data.dup.force_encoding(encoding) - end - end - @io = StringIO.new(data) - else - @io = data - end - @encoding = determine_encoding(encoding, internal_encoding) - - @base_fields_converter_options = { - nil_value: nil_value, - empty_value: empty_value, - } - @write_fields_converter_options = { - nil_value: write_nil_value, - empty_value: write_empty_value, - } - @initial_converters = converters - @initial_header_converters = header_converters - @initial_write_converters = write_converters - - if max_field_size.nil? and field_size_limit - max_field_size = field_size_limit - 1 - end - @parser_options = { - column_separator: col_sep, - row_separator: row_sep, - quote_character: quote_char, - max_field_size: max_field_size, - unconverted_fields: unconverted_fields, - headers: headers, - return_headers: return_headers, - skip_blanks: skip_blanks, - skip_lines: skip_lines, - liberal_parsing: liberal_parsing, - encoding: @encoding, - nil_value: nil_value, - empty_value: empty_value, - strip: strip, - } - @parser = nil - @parser_enumerator = nil - @eof_error = nil - - @writer_options = { - encoding: @encoding, - force_encoding: (not encoding.nil?), - force_quotes: force_quotes, - headers: headers, - write_headers: write_headers, - column_separator: col_sep, - row_separator: row_sep, - quote_character: quote_char, - quote_empty: quote_empty, - } - - @writer = nil - writer if @writer_options[:write_headers] - end - - # :call-seq: - # csv.col_sep -> string - # - # Returns the encoded column separator; used for parsing and writing; - # see {Option +col_sep+}[#class-CSV-label-Option+col_sep]: - # CSV.new('').col_sep # => "," - def col_sep - parser.column_separator - end - - # :call-seq: - # csv.row_sep -> string - # - # Returns the encoded row separator; used for parsing and writing; - # see {Option +row_sep+}[#class-CSV-label-Option+row_sep]: - # CSV.new('').row_sep # => "\n" - def row_sep - parser.row_separator - end - - # :call-seq: - # csv.quote_char -> character - # - # Returns the encoded quote character; used for parsing and writing; - # see {Option +quote_char+}[#class-CSV-label-Option+quote_char]: - # CSV.new('').quote_char # => "\"" - def quote_char - parser.quote_character - end - - # :call-seq: - # csv.field_size_limit -> integer or nil - # - # Returns the limit for field size; used for parsing; - # see {Option +field_size_limit+}[#class-CSV-label-Option+field_size_limit]: - # CSV.new('').field_size_limit # => nil - # - # Deprecated since 3.2.3. Use +max_field_size+ instead. - def field_size_limit - parser.field_size_limit - end - - # :call-seq: - # csv.max_field_size -> integer or nil - # - # Returns the limit for field size; used for parsing; - # see {Option +max_field_size+}[#class-CSV-label-Option+max_field_size]: - # CSV.new('').max_field_size # => nil - # - # Since 3.2.3. - def max_field_size - parser.max_field_size - end - - # :call-seq: - # csv.skip_lines -> regexp or nil - # - # Returns the \Regexp used to identify comment lines; used for parsing; - # see {Option +skip_lines+}[#class-CSV-label-Option+skip_lines]: - # CSV.new('').skip_lines # => nil - def skip_lines - parser.skip_lines - end - - # :call-seq: - # csv.converters -> array - # - # Returns an \Array containing field converters; - # see {Field Converters}[#class-CSV-label-Field+Converters]: - # csv = CSV.new('') - # csv.converters # => [] - # csv.convert(:integer) - # csv.converters # => [:integer] - # csv.convert(proc {|x| x.to_s }) - # csv.converters - # - # Notes that you need to call - # +Ractor.make_shareable(CSV::Converters)+ on the main Ractor to use - # this method. - def converters - parser_fields_converter.map do |converter| - name = Converters.rassoc(converter) - name ? name.first : converter - end - end - - # :call-seq: - # csv.unconverted_fields? -> object - # - # Returns the value that determines whether unconverted fields are to be - # available; used for parsing; - # see {Option +unconverted_fields+}[#class-CSV-label-Option+unconverted_fields]: - # CSV.new('').unconverted_fields? # => nil - def unconverted_fields? - parser.unconverted_fields? - end - - # :call-seq: - # csv.headers -> object - # - # Returns the value that determines whether headers are used; used for parsing; - # see {Option +headers+}[#class-CSV-label-Option+headers]: - # CSV.new('').headers # => nil - def headers - if @writer - @writer.headers - else - parsed_headers = parser.headers - return parsed_headers if parsed_headers - raw_headers = @parser_options[:headers] - raw_headers = nil if raw_headers == false - raw_headers - end - end - - # :call-seq: - # csv.return_headers? -> true or false - # - # Returns the value that determines whether headers are to be returned; used for parsing; - # see {Option +return_headers+}[#class-CSV-label-Option+return_headers]: - # CSV.new('').return_headers? # => false - def return_headers? - parser.return_headers? - end - - # :call-seq: - # csv.write_headers? -> true or false - # - # Returns the value that determines whether headers are to be written; used for generating; - # see {Option +write_headers+}[#class-CSV-label-Option+write_headers]: - # CSV.new('').write_headers? # => nil - def write_headers? - @writer_options[:write_headers] - end - - # :call-seq: - # csv.header_converters -> array - # - # Returns an \Array containing header converters; used for parsing; - # see {Header Converters}[#class-CSV-label-Header+Converters]: - # CSV.new('').header_converters # => [] - # - # Notes that you need to call - # +Ractor.make_shareable(CSV::HeaderConverters)+ on the main Ractor - # to use this method. - def header_converters - header_fields_converter.map do |converter| - name = HeaderConverters.rassoc(converter) - name ? name.first : converter - end - end - - # :call-seq: - # csv.skip_blanks? -> true or false - # - # Returns the value that determines whether blank lines are to be ignored; used for parsing; - # see {Option +skip_blanks+}[#class-CSV-label-Option+skip_blanks]: - # CSV.new('').skip_blanks? # => false - def skip_blanks? - parser.skip_blanks? - end - - # :call-seq: - # csv.force_quotes? -> true or false - # - # Returns the value that determines whether all output fields are to be quoted; - # used for generating; - # see {Option +force_quotes+}[#class-CSV-label-Option+force_quotes]: - # CSV.new('').force_quotes? # => false - def force_quotes? - @writer_options[:force_quotes] - end - - # :call-seq: - # csv.liberal_parsing? -> true or false - # - # Returns the value that determines whether illegal input is to be handled; used for parsing; - # see {Option +liberal_parsing+}[#class-CSV-label-Option+liberal_parsing]: - # CSV.new('').liberal_parsing? # => false - def liberal_parsing? - parser.liberal_parsing? - end - - # :call-seq: - # csv.encoding -> encoding - # - # Returns the encoding used for parsing and generating; - # see {Character Encodings (M17n or Multilingualization)}[#class-CSV-label-Character+Encodings+-28M17n+or+Multilingualization-29]: - # CSV.new('').encoding # => #<Encoding:UTF-8> - attr_reader :encoding - - # :call-seq: - # csv.line_no -> integer - # - # Returns the count of the rows parsed or generated. - # - # Parsing: - # string = "foo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # CSV.open(path) do |csv| - # csv.each do |row| - # p [csv.lineno, row] - # end - # end - # Output: - # [1, ["foo", "0"]] - # [2, ["bar", "1"]] - # [3, ["baz", "2"]] - # - # Generating: - # CSV.generate do |csv| - # p csv.lineno; csv << ['foo', 0] - # p csv.lineno; csv << ['bar', 1] - # p csv.lineno; csv << ['baz', 2] - # end - # Output: - # 0 - # 1 - # 2 - def lineno - if @writer - @writer.lineno - else - parser.lineno - end - end - - # :call-seq: - # csv.line -> array - # - # Returns the line most recently read: - # string = "foo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # CSV.open(path) do |csv| - # csv.each do |row| - # p [csv.lineno, csv.line] - # end - # end - # Output: - # [1, "foo,0\n"] - # [2, "bar,1\n"] - # [3, "baz,2\n"] - def line - parser.line - end - - ### IO and StringIO Delegation ### - - extend Forwardable - def_delegators :@io, :binmode, :close, :close_read, :close_write, - :closed?, :external_encoding, :fcntl, - :fileno, :flush, :fsync, :internal_encoding, - :isatty, :pid, :pos, :pos=, :reopen, - :seek, :string, :sync, :sync=, :tell, - :truncate, :tty? - - def binmode? - if @io.respond_to?(:binmode?) - @io.binmode? - else - false - end - end - - def flock(*args) - raise NotImplementedError unless @io.respond_to?(:flock) - @io.flock(*args) - end - - def ioctl(*args) - raise NotImplementedError unless @io.respond_to?(:ioctl) - @io.ioctl(*args) - end - - def path - @io.path if @io.respond_to?(:path) - end - - def stat(*args) - raise NotImplementedError unless @io.respond_to?(:stat) - @io.stat(*args) - end - - def to_i - raise NotImplementedError unless @io.respond_to?(:to_i) - @io.to_i - end - - def to_io - @io.respond_to?(:to_io) ? @io.to_io : @io - end - - def eof? - return false if @eof_error - begin - parser_enumerator.peek - false - rescue MalformedCSVError => error - @eof_error = error - false - rescue StopIteration - true - end - end - alias_method :eof, :eof? - - # Rewinds the underlying IO object and resets CSV's lineno() counter. - def rewind - @parser = nil - @parser_enumerator = nil - @eof_error = nil - @writer.rewind if @writer - @io.rewind - end - - ### End Delegation ### - - # :call-seq: - # csv << row -> self - # - # Appends a row to +self+. - # - # - Argument +row+ must be an \Array object or a CSV::Row object. - # - The output stream must be open for writing. - # - # --- - # - # Append Arrays: - # CSV.generate do |csv| - # csv << ['foo', 0] - # csv << ['bar', 1] - # csv << ['baz', 2] - # end # => "foo,0\nbar,1\nbaz,2\n" - # - # Append CSV::Rows: - # headers = [] - # CSV.generate do |csv| - # csv << CSV::Row.new(headers, ['foo', 0]) - # csv << CSV::Row.new(headers, ['bar', 1]) - # csv << CSV::Row.new(headers, ['baz', 2]) - # end # => "foo,0\nbar,1\nbaz,2\n" - # - # Headers in CSV::Row objects are not appended: - # headers = ['Name', 'Count'] - # CSV.generate do |csv| - # csv << CSV::Row.new(headers, ['foo', 0]) - # csv << CSV::Row.new(headers, ['bar', 1]) - # csv << CSV::Row.new(headers, ['baz', 2]) - # end # => "foo,0\nbar,1\nbaz,2\n" - # - # --- - # - # Raises an exception if +row+ is not an \Array or \CSV::Row: - # CSV.generate do |csv| - # # Raises NoMethodError (undefined method `collect' for :foo:Symbol) - # csv << :foo - # end - # - # Raises an exception if the output stream is not opened for writing: - # path = 't.csv' - # File.write(path, '') - # File.open(path) do |file| - # CSV.open(file) do |csv| - # # Raises IOError (not opened for writing) - # csv << ['foo', 0] - # end - # end - def <<(row) - writer << row - self - end - alias_method :add_row, :<< - alias_method :puts, :<< - - # :call-seq: - # convert(converter_name) -> array_of_procs - # convert {|field, field_info| ... } -> array_of_procs - # - # - With no block, installs a field converter (a \Proc). - # - With a block, defines and installs a custom field converter. - # - Returns the \Array of installed field converters. - # - # - Argument +converter_name+, if given, should be the name - # of an existing field converter. - # - # See {Field Converters}[#class-CSV-label-Field+Converters]. - # --- - # - # With no block, installs a field converter: - # csv = CSV.new('') - # csv.convert(:integer) - # csv.convert(:float) - # csv.convert(:date) - # csv.converters # => [:integer, :float, :date] - # - # --- - # - # The block, if given, is called for each field: - # - Argument +field+ is the field value. - # - Argument +field_info+ is a CSV::FieldInfo object - # containing details about the field. - # - # The examples here assume the prior execution of: - # string = "foo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # - # Example giving a block: - # csv = CSV.open(path) - # csv.convert {|field, field_info| p [field, field_info]; field.upcase } - # csv.read # => [["FOO", "0"], ["BAR", "1"], ["BAZ", "2"]] - # - # Output: - # ["foo", #<struct CSV::FieldInfo index=0, line=1, header=nil>] - # ["0", #<struct CSV::FieldInfo index=1, line=1, header=nil>] - # ["bar", #<struct CSV::FieldInfo index=0, line=2, header=nil>] - # ["1", #<struct CSV::FieldInfo index=1, line=2, header=nil>] - # ["baz", #<struct CSV::FieldInfo index=0, line=3, header=nil>] - # ["2", #<struct CSV::FieldInfo index=1, line=3, header=nil>] - # - # The block need not return a \String object: - # csv = CSV.open(path) - # csv.convert {|field, field_info| field.to_sym } - # csv.read # => [[:foo, :"0"], [:bar, :"1"], [:baz, :"2"]] - # - # If +converter_name+ is given, the block is not called: - # csv = CSV.open(path) - # csv.convert(:integer) {|field, field_info| fail 'Cannot happen' } - # csv.read # => [["foo", 0], ["bar", 1], ["baz", 2]] - # - # --- - # - # Raises a parse-time exception if +converter_name+ is not the name of a built-in - # field converter: - # csv = CSV.open(path) - # csv.convert(:nosuch) => [nil] - # # Raises NoMethodError (undefined method `arity' for nil:NilClass) - # csv.read - def convert(name = nil, &converter) - parser_fields_converter.add_converter(name, &converter) - end - - # :call-seq: - # header_convert(converter_name) -> array_of_procs - # header_convert {|header, field_info| ... } -> array_of_procs - # - # - With no block, installs a header converter (a \Proc). - # - With a block, defines and installs a custom header converter. - # - Returns the \Array of installed header converters. - # - # - Argument +converter_name+, if given, should be the name - # of an existing header converter. - # - # See {Header Converters}[#class-CSV-label-Header+Converters]. - # --- - # - # With no block, installs a header converter: - # csv = CSV.new('') - # csv.header_convert(:symbol) - # csv.header_convert(:downcase) - # csv.header_converters # => [:symbol, :downcase] - # - # --- - # - # The block, if given, is called for each header: - # - Argument +header+ is the header value. - # - Argument +field_info+ is a CSV::FieldInfo object - # containing details about the header. - # - # The examples here assume the prior execution of: - # string = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # - # Example giving a block: - # csv = CSV.open(path, headers: true) - # csv.header_convert {|header, field_info| p [header, field_info]; header.upcase } - # table = csv.read - # table # => #<CSV::Table mode:col_or_row row_count:4> - # table.headers # => ["NAME", "VALUE"] - # - # Output: - # ["Name", #<struct CSV::FieldInfo index=0, line=1, header=nil>] - # ["Value", #<struct CSV::FieldInfo index=1, line=1, header=nil>] - - # The block need not return a \String object: - # csv = CSV.open(path, headers: true) - # csv.header_convert {|header, field_info| header.to_sym } - # table = csv.read - # table.headers # => [:Name, :Value] - # - # If +converter_name+ is given, the block is not called: - # csv = CSV.open(path, headers: true) - # csv.header_convert(:downcase) {|header, field_info| fail 'Cannot happen' } - # table = csv.read - # table.headers # => ["name", "value"] - # --- - # - # Raises a parse-time exception if +converter_name+ is not the name of a built-in - # field converter: - # csv = CSV.open(path, headers: true) - # csv.header_convert(:nosuch) - # # Raises NoMethodError (undefined method `arity' for nil:NilClass) - # csv.read - def header_convert(name = nil, &converter) - header_fields_converter.add_converter(name, &converter) - end - - include Enumerable - - # :call-seq: - # csv.each -> enumerator - # csv.each {|row| ...} - # - # Calls the block with each successive row. - # The data source must be opened for reading. - # - # Without headers: - # string = "foo,0\nbar,1\nbaz,2\n" - # csv = CSV.new(string) - # csv.each do |row| - # p row - # end - # Output: - # ["foo", "0"] - # ["bar", "1"] - # ["baz", "2"] - # - # With headers: - # string = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # csv = CSV.new(string, headers: true) - # csv.each do |row| - # p row - # end - # Output: - # <CSV::Row "Name":"foo" "Value":"0"> - # <CSV::Row "Name":"bar" "Value":"1"> - # <CSV::Row "Name":"baz" "Value":"2"> - # - # --- - # - # Raises an exception if the source is not opened for reading: - # string = "foo,0\nbar,1\nbaz,2\n" - # csv = CSV.new(string) - # csv.close - # # Raises IOError (not opened for reading) - # csv.each do |row| - # p row - # end - def each(&block) - return to_enum(__method__) unless block_given? - begin - while true - yield(parser_enumerator.next) - end - rescue StopIteration - end - end - - # :call-seq: - # csv.read -> array or csv_table - # - # Forms the remaining rows from +self+ into: - # - A CSV::Table object, if headers are in use. - # - An \Array of Arrays, otherwise. - # - # The data source must be opened for reading. - # - # Without headers: - # string = "foo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # csv = CSV.open(path) - # csv.read # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] - # - # With headers: - # string = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # path = 't.csv' - # File.write(path, string) - # csv = CSV.open(path, headers: true) - # csv.read # => #<CSV::Table mode:col_or_row row_count:4> - # - # --- - # - # Raises an exception if the source is not opened for reading: - # string = "foo,0\nbar,1\nbaz,2\n" - # csv = CSV.new(string) - # csv.close - # # Raises IOError (not opened for reading) - # csv.read - def read - rows = to_a - if parser.use_headers? - Table.new(rows, headers: parser.headers) - else - rows - end - end - alias_method :readlines, :read - - # :call-seq: - # csv.header_row? -> true or false - # - # Returns +true+ if the next row to be read is a header row\; - # +false+ otherwise. - # - # Without headers: - # string = "foo,0\nbar,1\nbaz,2\n" - # csv = CSV.new(string) - # csv.header_row? # => false - # - # With headers: - # string = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # csv = CSV.new(string, headers: true) - # csv.header_row? # => true - # csv.shift # => #<CSV::Row "Name":"foo" "Value":"0"> - # csv.header_row? # => false - # - # --- - # - # Raises an exception if the source is not opened for reading: - # string = "foo,0\nbar,1\nbaz,2\n" - # csv = CSV.new(string) - # csv.close - # # Raises IOError (not opened for reading) - # csv.header_row? - def header_row? - parser.header_row? - end - - # :call-seq: - # csv.shift -> array, csv_row, or nil - # - # Returns the next row of data as: - # - An \Array if no headers are used. - # - A CSV::Row object if headers are used. - # - # The data source must be opened for reading. - # - # Without headers: - # string = "foo,0\nbar,1\nbaz,2\n" - # csv = CSV.new(string) - # csv.shift # => ["foo", "0"] - # csv.shift # => ["bar", "1"] - # csv.shift # => ["baz", "2"] - # csv.shift # => nil - # - # With headers: - # string = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # csv = CSV.new(string, headers: true) - # csv.shift # => #<CSV::Row "Name":"foo" "Value":"0"> - # csv.shift # => #<CSV::Row "Name":"bar" "Value":"1"> - # csv.shift # => #<CSV::Row "Name":"baz" "Value":"2"> - # csv.shift # => nil - # - # --- - # - # Raises an exception if the source is not opened for reading: - # string = "foo,0\nbar,1\nbaz,2\n" - # csv = CSV.new(string) - # csv.close - # # Raises IOError (not opened for reading) - # csv.shift - def shift - if @eof_error - eof_error, @eof_error = @eof_error, nil - raise eof_error - end - begin - parser_enumerator.next - rescue StopIteration - nil - end - end - alias_method :gets, :shift - alias_method :readline, :shift - - # :call-seq: - # csv.inspect -> string - # - # Returns a \String showing certain properties of +self+: - # string = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # csv = CSV.new(string, headers: true) - # s = csv.inspect - # s # => "#<CSV io_type:StringIO encoding:UTF-8 lineno:0 col_sep:\",\" row_sep:\"\\n\" quote_char:\"\\\"\" headers:true>" - def inspect - str = ["#<", self.class.to_s, " io_type:"] - # show type of wrapped IO - if @io == $stdout then str << "$stdout" - elsif @io == $stdin then str << "$stdin" - elsif @io == $stderr then str << "$stderr" - else str << @io.class.to_s - end - # show IO.path(), if available - if @io.respond_to?(:path) and (p = @io.path) - str << " io_path:" << p.inspect - end - # show encoding - str << " encoding:" << @encoding.name - # show other attributes - ["lineno", "col_sep", "row_sep", "quote_char"].each do |attr_name| - if a = __send__(attr_name) - str << " " << attr_name << ":" << a.inspect - end - end - ["skip_blanks", "liberal_parsing"].each do |attr_name| - if a = __send__("#{attr_name}?") - str << " " << attr_name << ":" << a.inspect - end - end - _headers = headers - str << " headers:" << _headers.inspect if _headers - str << ">" - begin - str.join('') - rescue # any encoding error - str.map do |s| - e = Encoding::Converter.asciicompat_encoding(s.encoding) - e ? s.encode(e) : s.force_encoding("ASCII-8BIT") - end.join('') - end - end - - private - - def determine_encoding(encoding, internal_encoding) - # honor the IO encoding if we can, otherwise default to ASCII-8BIT - io_encoding = raw_encoding - return io_encoding if io_encoding - - return Encoding.find(internal_encoding) if internal_encoding - - if encoding - encoding, = encoding.split(":", 2) if encoding.is_a?(String) - return Encoding.find(encoding) - end - - Encoding.default_internal || Encoding.default_external - end - - def normalize_converters(converters) - converters ||= [] - unless converters.is_a?(Array) - converters = [converters] - end - converters.collect do |converter| - case converter - when Proc # custom code block - [nil, converter] - else # by name - [converter, nil] - end - end - end - - # - # Processes +fields+ with <tt>@converters</tt>, or <tt>@header_converters</tt> - # if +headers+ is passed as +true+, returning the converted field set. Any - # converter that changes the field into something other than a String halts - # the pipeline of conversion for that field. This is primarily an efficiency - # shortcut. - # - def convert_fields(fields, headers = false) - if headers - header_fields_converter.convert(fields, nil, 0) - else - parser_fields_converter.convert(fields, @headers, lineno) - end - end - - # - # Returns the encoding of the internal IO object. - # - def raw_encoding - if @io.respond_to? :internal_encoding - @io.internal_encoding || @io.external_encoding - elsif @io.respond_to? :encoding - @io.encoding - else - nil - end - end - - def parser_fields_converter - @parser_fields_converter ||= build_parser_fields_converter - end - - def build_parser_fields_converter - specific_options = { - builtin_converters_name: :Converters, - } - options = @base_fields_converter_options.merge(specific_options) - build_fields_converter(@initial_converters, options) - end - - def header_fields_converter - @header_fields_converter ||= build_header_fields_converter - end - - def build_header_fields_converter - specific_options = { - builtin_converters_name: :HeaderConverters, - accept_nil: true, - } - options = @base_fields_converter_options.merge(specific_options) - build_fields_converter(@initial_header_converters, options) - end - - def writer_fields_converter - @writer_fields_converter ||= build_writer_fields_converter - end - - def build_writer_fields_converter - build_fields_converter(@initial_write_converters, - @write_fields_converter_options) - end - - def build_fields_converter(initial_converters, options) - fields_converter = FieldsConverter.new(options) - normalize_converters(initial_converters).each do |name, converter| - fields_converter.add_converter(name, &converter) - end - fields_converter - end - - def parser - @parser ||= Parser.new(@io, parser_options) - end - - def parser_options - @parser_options.merge(header_fields_converter: header_fields_converter, - fields_converter: parser_fields_converter) - end - - def parser_enumerator - @parser_enumerator ||= parser.parse - end - - def writer - @writer ||= Writer.new(@io, writer_options) - end - - def writer_options - @writer_options.merge(header_fields_converter: header_fields_converter, - fields_converter: writer_fields_converter) - end -end - -# Passes +args+ to CSV::instance. -# -# CSV("CSV,data").read -# #=> [["CSV", "data"]] -# -# If a block is given, the instance is passed the block and the return value -# becomes the return value of the block. -# -# CSV("CSV,data") { |c| -# c.read.any? { |a| a.include?("data") } -# } #=> true -# -# CSV("CSV,data") { |c| -# c.read.any? { |a| a.include?("zombies") } -# } #=> false -# -# CSV options may also be given. -# -# io = StringIO.new -# CSV(io, col_sep: ";") { |csv| csv << ["a", "b", "c"] } -# -# This API is not Ractor-safe. -# -def CSV(*args, **options, &block) - CSV.instance(*args, **options, &block) -end - -require_relative "csv/version" -require_relative "csv/core_ext/array" -require_relative "csv/core_ext/string" diff --git a/lib/csv/core_ext/array.rb b/lib/csv/core_ext/array.rb deleted file mode 100644 index 8beb06b082..0000000000 --- a/lib/csv/core_ext/array.rb +++ /dev/null @@ -1,9 +0,0 @@ -class Array # :nodoc: - # Equivalent to CSV::generate_line(self, options) - # - # ["CSV", "data"].to_csv - # #=> "CSV,data\n" - def to_csv(**options) - CSV.generate_line(self, **options) - end -end diff --git a/lib/csv/core_ext/string.rb b/lib/csv/core_ext/string.rb deleted file mode 100644 index 9b1d31c2a4..0000000000 --- a/lib/csv/core_ext/string.rb +++ /dev/null @@ -1,9 +0,0 @@ -class String # :nodoc: - # Equivalent to CSV::parse_line(self, options) - # - # "CSV,data".parse_csv - # #=> ["CSV", "data"] - def parse_csv(**options) - CSV.parse_line(self, **options) - end -end diff --git a/lib/csv/csv.gemspec b/lib/csv/csv.gemspec deleted file mode 100644 index 11c5b0f2a6..0000000000 --- a/lib/csv/csv.gemspec +++ /dev/null @@ -1,64 +0,0 @@ -# frozen_string_literal: true - -begin - require_relative "lib/csv/version" -rescue LoadError - # for Ruby core repository - require_relative "version" -end - -Gem::Specification.new do |spec| - spec.name = "csv" - spec.version = CSV::VERSION - spec.authors = ["James Edward Gray II", "Kouhei Sutou"] - spec.email = [nil, "kou@cozmixng.org"] - - spec.summary = "CSV Reading and Writing" - spec.description = "The CSV library provides a complete interface to CSV files and data. It offers tools to enable you to read and write to and from Strings or IO objects, as needed." - spec.homepage = "https://github.com/ruby/csv" - spec.licenses = ["Ruby", "BSD-2-Clause"] - - lib_path = "lib" - spec.require_paths = [lib_path] - files = [] - lib_dir = File.join(__dir__, lib_path) - if File.exist?(lib_dir) - Dir.chdir(lib_dir) do - Dir.glob("**/*.rb").each do |file| - files << "lib/#{file}" - end - end - end - doc_dir = File.join(__dir__, "doc") - if File.exist?(doc_dir) - Dir.chdir(doc_dir) do - Dir.glob("**/*.rdoc").each do |rdoc_file| - files << "doc/#{rdoc_file}" - end - end - end - spec.files = files - spec.rdoc_options.concat(["--main", "README.md"]) - rdoc_files = [ - "LICENSE.txt", - "NEWS.md", - "README.md", - ] - recipes_dir = File.join(doc_dir, "csv", "recipes") - if File.exist?(recipes_dir) - Dir.chdir(recipes_dir) do - Dir.glob("**/*.rdoc").each do |recipe_file| - rdoc_files << "doc/csv/recipes/#{recipe_file}" - end - end - end - spec.extra_rdoc_files = rdoc_files - - spec.required_ruby_version = ">= 2.5.0" - - # spec.add_dependency "stringio", ">= 0.1.3" - spec.add_development_dependency "bundler" - spec.add_development_dependency "rake" - spec.add_development_dependency "benchmark_driver" - spec.add_development_dependency "test-unit", ">= 3.4.8" -end diff --git a/lib/csv/fields_converter.rb b/lib/csv/fields_converter.rb deleted file mode 100644 index d15977d379..0000000000 --- a/lib/csv/fields_converter.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -class CSV - # Note: Don't use this class directly. This is an internal class. - class FieldsConverter - include Enumerable - # - # A CSV::FieldsConverter is a data structure for storing the - # fields converter properties to be passed as a parameter - # when parsing a new file (e.g. CSV::Parser.new(@io, parser_options)) - # - - def initialize(options={}) - @converters = [] - @nil_value = options[:nil_value] - @empty_value = options[:empty_value] - @empty_value_is_empty_string = (@empty_value == "") - @accept_nil = options[:accept_nil] - @builtin_converters_name = options[:builtin_converters_name] - @need_static_convert = need_static_convert? - end - - def add_converter(name=nil, &converter) - if name.nil? # custom converter - @converters << converter - else # named converter - combo = builtin_converters[name] - case combo - when Array # combo converter - combo.each do |sub_name| - add_converter(sub_name) - end - else # individual named converter - @converters << combo - end - end - end - - def each(&block) - @converters.each(&block) - end - - def empty? - @converters.empty? - end - - def convert(fields, headers, lineno, quoted_fields) - return fields unless need_convert? - - fields.collect.with_index do |field, index| - if field.nil? - field = @nil_value - elsif field.is_a?(String) and field.empty? - field = @empty_value unless @empty_value_is_empty_string - end - @converters.each do |converter| - break if field.nil? and @accept_nil - if converter.arity == 1 # straight field converter - field = converter[field] - else # FieldInfo converter - if headers - header = headers[index] - else - header = nil - end - quoted = quoted_fields[index] - field = converter[field, FieldInfo.new(index, lineno, header, quoted)] - end - break unless field.is_a?(String) # short-circuit pipeline for speed - end - field # final state of each field, converted or original - end - end - - private - def need_static_convert? - not (@nil_value.nil? and @empty_value_is_empty_string) - end - - def need_convert? - @need_static_convert or - (not @converters.empty?) - end - - def builtin_converters - @builtin_converters ||= ::CSV.const_get(@builtin_converters_name) - end - end -end diff --git a/lib/csv/input_record_separator.rb b/lib/csv/input_record_separator.rb deleted file mode 100644 index 7a99343c0c..0000000000 --- a/lib/csv/input_record_separator.rb +++ /dev/null @@ -1,18 +0,0 @@ -require "English" -require "stringio" - -class CSV - module InputRecordSeparator - class << self - if RUBY_VERSION >= "3.0.0" - def value - "\n" - end - else - def value - $INPUT_RECORD_SEPARATOR - end - end - end - end -end diff --git a/lib/csv/parser.rb b/lib/csv/parser.rb deleted file mode 100644 index 1f8b150e19..0000000000 --- a/lib/csv/parser.rb +++ /dev/null @@ -1,1288 +0,0 @@ -# frozen_string_literal: true - -require "strscan" - -require_relative "input_record_separator" -require_relative "row" -require_relative "table" - -class CSV - # Note: Don't use this class directly. This is an internal class. - class Parser - # - # A CSV::Parser is m17n aware. The parser works in the Encoding of the IO - # or String object being read from or written to. Your data is never transcoded - # (unless you ask Ruby to transcode it for you) and will literally be parsed in - # the Encoding it is in. Thus CSV will return Arrays or Rows of Strings in the - # Encoding of your data. This is accomplished by transcoding the parser itself - # into your Encoding. - # - - # Raised when encoding is invalid. - class InvalidEncoding < StandardError - end - - # Raised when unexpected case is happen. - class UnexpectedError < StandardError - end - - # - # CSV::Scanner receives a CSV output, scans it and return the content. - # It also controls the life cycle of the object with its methods +keep_start+, - # +keep_end+, +keep_back+, +keep_drop+. - # - # Uses StringScanner (the official strscan gem). Strscan provides lexical - # scanning operations on a String. We inherit its object and take advantage - # on the methods. For more information, please visit: - # https://ruby-doc.org/stdlib-2.6.1/libdoc/strscan/rdoc/StringScanner.html - # - class Scanner < StringScanner - alias_method :scan_all, :scan - - def initialize(*args) - super - @keeps = [] - end - - def each_line(row_separator) - position = pos - rest.each_line(row_separator) do |line| - position += line.bytesize - self.pos = position - yield(line) - end - end - - def keep_start - @keeps.push(pos) - end - - def keep_end - start = @keeps.pop - string.byteslice(start, pos - start) - end - - def keep_back - self.pos = @keeps.pop - end - - def keep_drop - @keeps.pop - end - end - - # - # CSV::InputsScanner receives IO inputs, encoding and the chunk_size. - # It also controls the life cycle of the object with its methods +keep_start+, - # +keep_end+, +keep_back+, +keep_drop+. - # - # CSV::InputsScanner.scan() tries to match with pattern at the current position. - # If there's a match, the scanner advances the "scan pointer" and returns the matched string. - # Otherwise, the scanner returns nil. - # - # CSV::InputsScanner.rest() returns the "rest" of the string (i.e. everything after the scan pointer). - # If there is no more data (eos? = true), it returns "". - # - class InputsScanner - def initialize(inputs, encoding, row_separator, chunk_size: 8192) - @inputs = inputs.dup - @encoding = encoding - @row_separator = row_separator - @chunk_size = chunk_size - @last_scanner = @inputs.empty? - @keeps = [] - read_chunk - end - - def each_line(row_separator) - return enum_for(__method__, row_separator) unless block_given? - buffer = nil - input = @scanner.rest - position = @scanner.pos - offset = 0 - n_row_separator_chars = row_separator.size - # trace(__method__, :start, line, input) - while true - input.each_line(row_separator) do |line| - @scanner.pos += line.bytesize - if buffer - if n_row_separator_chars == 2 and - buffer.end_with?(row_separator[0]) and - line.start_with?(row_separator[1]) - buffer << line[0] - line = line[1..-1] - position += buffer.bytesize + offset - @scanner.pos = position - offset = 0 - yield(buffer) - buffer = nil - next if line.empty? - else - buffer << line - line = buffer - buffer = nil - end - end - if line.end_with?(row_separator) - position += line.bytesize + offset - @scanner.pos = position - offset = 0 - yield(line) - else - buffer = line - end - end - break unless read_chunk - input = @scanner.rest - position = @scanner.pos - offset = -buffer.bytesize if buffer - end - yield(buffer) if buffer - end - - def scan(pattern) - # trace(__method__, pattern, :start) - value = @scanner.scan(pattern) - # trace(__method__, pattern, :done, :last, value) if @last_scanner - return value if @last_scanner - - read_chunk if value and @scanner.eos? - # trace(__method__, pattern, :done, value) - value - end - - def scan_all(pattern) - # trace(__method__, pattern, :start) - value = @scanner.scan(pattern) - # trace(__method__, pattern, :done, :last, value) if @last_scanner - return value if @last_scanner - - return nil if value.nil? - while @scanner.eos? and read_chunk and (sub_value = @scanner.scan(pattern)) - # trace(__method__, pattern, :sub, sub_value) - value << sub_value - end - # trace(__method__, pattern, :done, value) - value - end - - def eos? - @scanner.eos? - end - - def keep_start - # trace(__method__, :start) - adjust_last_keep - @keeps.push([@scanner, @scanner.pos, nil]) - # trace(__method__, :done) - end - - def keep_end - # trace(__method__, :start) - scanner, start, buffer = @keeps.pop - if scanner == @scanner - keep = @scanner.string.byteslice(start, @scanner.pos - start) - else - keep = @scanner.string.byteslice(0, @scanner.pos) - end - if buffer - buffer << keep - keep = buffer - end - # trace(__method__, :done, keep) - keep - end - - def keep_back - # trace(__method__, :start) - scanner, start, buffer = @keeps.pop - if buffer - # trace(__method__, :rescan, start, buffer) - string = @scanner.string - if scanner == @scanner - keep = string.byteslice(start, string.bytesize - start) - else - keep = string - end - if keep and not keep.empty? - @inputs.unshift(StringIO.new(keep)) - @last_scanner = false - end - @scanner = StringScanner.new(buffer) - else - if @scanner != scanner - message = "scanners are different but no buffer: " - message += "#{@scanner.inspect}(#{@scanner.object_id}): " - message += "#{scanner.inspect}(#{scanner.object_id})" - raise UnexpectedError, message - end - # trace(__method__, :repos, start, buffer) - @scanner.pos = start - end - read_chunk if @scanner.eos? - end - - def keep_drop - _, _, buffer = @keeps.pop - # trace(__method__, :done, :empty) unless buffer - return unless buffer - - last_keep = @keeps.last - # trace(__method__, :done, :no_last_keep) unless last_keep - return unless last_keep - - if last_keep[2] - last_keep[2] << buffer - else - last_keep[2] = buffer - end - # trace(__method__, :done) - end - - def rest - @scanner.rest - end - - def check(pattern) - @scanner.check(pattern) - end - - private - def trace(*args) - pp([*args, @scanner, @scanner&.string, @scanner&.pos, @keeps]) - end - - def adjust_last_keep - # trace(__method__, :start) - - keep = @keeps.last - # trace(__method__, :done, :empty) if keep.nil? - return if keep.nil? - - scanner, start, buffer = keep - string = @scanner.string - if @scanner != scanner - start = 0 - end - if start == 0 and @scanner.eos? - keep_data = string - else - keep_data = string.byteslice(start, @scanner.pos - start) - end - if keep_data - if buffer - buffer << keep_data - else - keep[2] = keep_data.dup - end - end - - # trace(__method__, :done) - end - - def read_chunk - return false if @last_scanner - - adjust_last_keep - - input = @inputs.first - case input - when StringIO - string = input.read - raise InvalidEncoding unless string.valid_encoding? - # trace(__method__, :stringio, string) - @scanner = StringScanner.new(string) - @inputs.shift - @last_scanner = @inputs.empty? - true - else - chunk = input.gets(@row_separator, @chunk_size) - if chunk - raise InvalidEncoding unless chunk.valid_encoding? - # trace(__method__, :chunk, chunk) - @scanner = StringScanner.new(chunk) - if input.respond_to?(:eof?) and input.eof? - @inputs.shift - @last_scanner = @inputs.empty? - end - true - else - # trace(__method__, :no_chunk) - @scanner = StringScanner.new("".encode(@encoding)) - @inputs.shift - @last_scanner = @inputs.empty? - if @last_scanner - false - else - read_chunk - end - end - end - end - end - - def initialize(input, options) - @input = input - @options = options - @samples = [] - - prepare - end - - def column_separator - @column_separator - end - - def row_separator - @row_separator - end - - def quote_character - @quote_character - end - - def field_size_limit - @max_field_size&.succ - end - - def max_field_size - @max_field_size - end - - def skip_lines - @skip_lines - end - - def unconverted_fields? - @unconverted_fields - end - - def headers - @headers - end - - def header_row? - @use_headers and @headers.nil? - end - - def return_headers? - @return_headers - end - - def skip_blanks? - @skip_blanks - end - - def liberal_parsing? - @liberal_parsing - end - - def lineno - @lineno - end - - def line - last_line - end - - def parse(&block) - return to_enum(__method__) unless block_given? - - if @return_headers and @headers and @raw_headers - headers = Row.new(@headers, @raw_headers, true) - if @unconverted_fields - headers = add_unconverted_fields(headers, []) - end - yield headers - end - - begin - @scanner ||= build_scanner - if quote_character.nil? - parse_no_quote(&block) - elsif @need_robust_parsing - parse_quotable_robust(&block) - else - parse_quotable_loose(&block) - end - rescue InvalidEncoding - if @scanner - ignore_broken_line - lineno = @lineno - else - lineno = @lineno + 1 - end - message = "Invalid byte sequence in #{@encoding}" - raise MalformedCSVError.new(message, lineno) - rescue UnexpectedError => error - if @scanner - ignore_broken_line - lineno = @lineno - else - lineno = @lineno + 1 - end - message = "This should not be happen: #{error.message}: " - message += "Please report this to https://github.com/ruby/csv/issues" - raise MalformedCSVError.new(message, lineno) - end - end - - def use_headers? - @use_headers - end - - private - # A set of tasks to prepare the file in order to parse it - def prepare - prepare_variable - prepare_quote_character - prepare_backslash - prepare_skip_lines - prepare_strip - prepare_separators - validate_strip_and_col_sep_options - prepare_quoted - prepare_unquoted - prepare_line - prepare_header - prepare_parser - end - - def prepare_variable - @need_robust_parsing = false - @encoding = @options[:encoding] - liberal_parsing = @options[:liberal_parsing] - if liberal_parsing - @liberal_parsing = true - if liberal_parsing.is_a?(Hash) - @double_quote_outside_quote = - liberal_parsing[:double_quote_outside_quote] - @backslash_quote = liberal_parsing[:backslash_quote] - else - @double_quote_outside_quote = false - @backslash_quote = false - end - @need_robust_parsing = true - else - @liberal_parsing = false - @backslash_quote = false - end - @unconverted_fields = @options[:unconverted_fields] - @max_field_size = @options[:max_field_size] - @skip_blanks = @options[:skip_blanks] - @fields_converter = @options[:fields_converter] - @header_fields_converter = @options[:header_fields_converter] - end - - def prepare_quote_character - @quote_character = @options[:quote_character] - if @quote_character.nil? - @escaped_quote_character = nil - @escaped_quote = nil - else - @quote_character = @quote_character.to_s.encode(@encoding) - if @quote_character.length != 1 - message = ":quote_char has to be nil or a single character String" - raise ArgumentError, message - end - @escaped_quote_character = Regexp.escape(@quote_character) - @escaped_quote = Regexp.new(@escaped_quote_character) - end - end - - def prepare_backslash - return unless @backslash_quote - - @backslash_character = "\\".encode(@encoding) - - @escaped_backslash_character = Regexp.escape(@backslash_character) - @escaped_backslash = Regexp.new(@escaped_backslash_character) - if @quote_character.nil? - @backslash_quote_character = nil - else - @backslash_quote_character = - @backslash_character + @escaped_quote_character - end - end - - def prepare_skip_lines - skip_lines = @options[:skip_lines] - case skip_lines - when String - @skip_lines = skip_lines.encode(@encoding) - when Regexp, nil - @skip_lines = skip_lines - else - unless skip_lines.respond_to?(:match) - message = - ":skip_lines has to respond to \#match: #{skip_lines.inspect}" - raise ArgumentError, message - end - @skip_lines = skip_lines - end - end - - def prepare_strip - @strip = @options[:strip] - @escaped_strip = nil - @strip_value = nil - @rstrip_value = nil - if @strip.is_a?(String) - case @strip.length - when 0 - raise ArgumentError, ":strip must not be an empty String" - when 1 - # ok - else - raise ArgumentError, ":strip doesn't support 2 or more characters yet" - end - @strip = @strip.encode(@encoding) - @escaped_strip = Regexp.escape(@strip) - if @quote_character - @strip_value = Regexp.new(@escaped_strip + - "+".encode(@encoding)) - @rstrip_value = Regexp.new(@escaped_strip + - "+\\z".encode(@encoding)) - end - @need_robust_parsing = true - elsif @strip - strip_values = " \t\f\v" - @escaped_strip = strip_values.encode(@encoding) - if @quote_character - @strip_value = Regexp.new("[#{strip_values}]+".encode(@encoding)) - @rstrip_value = Regexp.new("[#{strip_values}]+\\z".encode(@encoding)) - end - @need_robust_parsing = true - end - end - - begin - StringScanner.new("x").scan("x") - rescue TypeError - STRING_SCANNER_SCAN_ACCEPT_STRING = false - else - STRING_SCANNER_SCAN_ACCEPT_STRING = true - end - - def prepare_separators - column_separator = @options[:column_separator] - @column_separator = column_separator.to_s.encode(@encoding) - if @column_separator.size < 1 - message = ":col_sep must be 1 or more characters: " - message += column_separator.inspect - raise ArgumentError, message - end - @row_separator = - resolve_row_separator(@options[:row_separator]).encode(@encoding) - - @escaped_column_separator = Regexp.escape(@column_separator) - @escaped_first_column_separator = Regexp.escape(@column_separator[0]) - if @column_separator.size > 1 - @column_end = Regexp.new(@escaped_column_separator) - @column_ends = @column_separator.each_char.collect do |char| - Regexp.new(Regexp.escape(char)) - end - @first_column_separators = Regexp.new(@escaped_first_column_separator + - "+".encode(@encoding)) - else - if STRING_SCANNER_SCAN_ACCEPT_STRING - @column_end = @column_separator - else - @column_end = Regexp.new(@escaped_column_separator) - end - @column_ends = nil - @first_column_separators = nil - end - - escaped_row_separator = Regexp.escape(@row_separator) - @row_end = Regexp.new(escaped_row_separator) - if @row_separator.size > 1 - @row_ends = @row_separator.each_char.collect do |char| - Regexp.new(Regexp.escape(char)) - end - else - @row_ends = nil - end - - @cr = "\r".encode(@encoding) - @lf = "\n".encode(@encoding) - @line_end = Regexp.new("\r\n|\n|\r".encode(@encoding)) - @not_line_end = Regexp.new("[^\r\n]+".encode(@encoding)) - end - - # This method verifies that there are no (obvious) ambiguities with the - # provided +col_sep+ and +strip+ parsing options. For example, if +col_sep+ - # and +strip+ were both equal to +\t+, then there would be no clear way to - # parse the input. - def validate_strip_and_col_sep_options - return unless @strip - - if @strip.is_a?(String) - if @column_separator.start_with?(@strip) || @column_separator.end_with?(@strip) - raise ArgumentError, - "The provided strip (#{@escaped_strip}) and " \ - "col_sep (#{@escaped_column_separator}) options are incompatible." - end - else - if Regexp.new("\\A[#{@escaped_strip}]|[#{@escaped_strip}]\\z").match?(@column_separator) - raise ArgumentError, - "The provided strip (true) and " \ - "col_sep (#{@escaped_column_separator}) options are incompatible." - end - end - end - - def prepare_quoted - if @quote_character - @quotes = Regexp.new(@escaped_quote_character + - "+".encode(@encoding)) - no_quoted_values = @escaped_quote_character.dup - if @backslash_quote - no_quoted_values << @escaped_backslash_character - end - @quoted_value = Regexp.new("[^".encode(@encoding) + - no_quoted_values + - "]+".encode(@encoding)) - end - if @escaped_strip - @split_column_separator = Regexp.new(@escaped_strip + - "*".encode(@encoding) + - @escaped_column_separator + - @escaped_strip + - "*".encode(@encoding)) - else - if @column_separator == " ".encode(@encoding) - @split_column_separator = Regexp.new(@escaped_column_separator) - else - @split_column_separator = @column_separator - end - end - end - - def prepare_unquoted - return if @quote_character.nil? - - no_unquoted_values = "\r\n".encode(@encoding) - no_unquoted_values << @escaped_first_column_separator - unless @liberal_parsing - no_unquoted_values << @escaped_quote_character - end - @unquoted_value = Regexp.new("[^".encode(@encoding) + - no_unquoted_values + - "]+".encode(@encoding)) - end - - def resolve_row_separator(separator) - if separator == :auto - cr = "\r".encode(@encoding) - lf = "\n".encode(@encoding) - if @input.is_a?(StringIO) - pos = @input.pos - separator = detect_row_separator(@input.read, cr, lf) - @input.seek(pos) - elsif @input.respond_to?(:gets) - if @input.is_a?(File) - chunk_size = 32 * 1024 - else - chunk_size = 1024 - end - begin - while separator == :auto - # - # if we run out of data, it's probably a single line - # (ensure will set default value) - # - break unless sample = @input.gets(nil, chunk_size) - - # extend sample if we're unsure of the line ending - if sample.end_with?(cr) - sample << (@input.gets(nil, 1) || "") - end - - @samples << sample - - separator = detect_row_separator(sample, cr, lf) - end - rescue IOError - # do nothing: ensure will set default - end - end - separator = InputRecordSeparator.value if separator == :auto - end - separator.to_s.encode(@encoding) - end - - def detect_row_separator(sample, cr, lf) - lf_index = sample.index(lf) - if lf_index - cr_index = sample[0, lf_index].index(cr) - else - cr_index = sample.index(cr) - end - if cr_index and lf_index - if cr_index + 1 == lf_index - cr + lf - elsif cr_index < lf_index - cr - else - lf - end - elsif cr_index - cr - elsif lf_index - lf - else - :auto - end - end - - def prepare_line - @lineno = 0 - @last_line = nil - @scanner = nil - end - - def last_line - if @scanner - @last_line ||= @scanner.keep_end - else - @last_line - end - end - - def prepare_header - @return_headers = @options[:return_headers] - - headers = @options[:headers] - case headers - when Array - @raw_headers = headers - quoted_fields = [false] * @raw_headers.size - @use_headers = true - when String - @raw_headers, quoted_fields = parse_headers(headers) - @use_headers = true - when nil, false - @raw_headers = nil - @use_headers = false - else - @raw_headers = nil - @use_headers = true - end - if @raw_headers - @headers = adjust_headers(@raw_headers, quoted_fields) - else - @headers = nil - end - end - - def parse_headers(row) - quoted_fields = [] - converter = lambda do |field, info| - quoted_fields << info.quoted? - field - end - headers = CSV.parse_line(row, - col_sep: @column_separator, - row_sep: @row_separator, - quote_char: @quote_character, - converters: [converter]) - [headers, quoted_fields] - end - - def adjust_headers(headers, quoted_fields) - adjusted_headers = @header_fields_converter.convert(headers, nil, @lineno, quoted_fields) - adjusted_headers.each {|h| h.freeze if h.is_a? String} - adjusted_headers - end - - def prepare_parser - @may_quoted = may_quoted? - end - - def may_quoted? - return false if @quote_character.nil? - - if @input.is_a?(StringIO) - pos = @input.pos - sample = @input.read - @input.seek(pos) - else - return false if @samples.empty? - sample = @samples.first - end - sample[0, 128].index(@quote_character) - end - - class UnoptimizedStringIO # :nodoc: - def initialize(string) - @io = StringIO.new(string, "rb:#{string.encoding}") - end - - def gets(*args) - @io.gets(*args) - end - - def each_line(*args, &block) - @io.each_line(*args, &block) - end - - def eof? - @io.eof? - end - end - - SCANNER_TEST = (ENV["CSV_PARSER_SCANNER_TEST"] == "yes") - if SCANNER_TEST - SCANNER_TEST_CHUNK_SIZE_NAME = "CSV_PARSER_SCANNER_TEST_CHUNK_SIZE" - SCANNER_TEST_CHUNK_SIZE_VALUE = ENV[SCANNER_TEST_CHUNK_SIZE_NAME] - def build_scanner - inputs = @samples.collect do |sample| - UnoptimizedStringIO.new(sample) - end - if @input.is_a?(StringIO) - inputs << UnoptimizedStringIO.new(@input.read) - else - inputs << @input - end - begin - chunk_size_value = ENV[SCANNER_TEST_CHUNK_SIZE_NAME] - rescue # Ractor::IsolationError - # Ractor on Ruby 3.0 can't read ENV value. - chunk_size_value = SCANNER_TEST_CHUNK_SIZE_VALUE - end - chunk_size = Integer((chunk_size_value || "1"), 10) - InputsScanner.new(inputs, - @encoding, - @row_separator, - chunk_size: chunk_size) - end - else - def build_scanner - string = nil - if @samples.empty? and @input.is_a?(StringIO) - string = @input.read - elsif @samples.size == 1 and - @input != ARGF and - @input.respond_to?(:eof?) and - @input.eof? - string = @samples[0] - end - if string - unless string.valid_encoding? - index = string.lines(@row_separator).index do |line| - !line.valid_encoding? - end - if index - message = "Invalid byte sequence in #{@encoding}" - raise MalformedCSVError.new(message, @lineno + index + 1) - end - end - Scanner.new(string) - else - inputs = @samples.collect do |sample| - StringIO.new(sample) - end - inputs << @input - InputsScanner.new(inputs, @encoding, @row_separator) - end - end - end - - def skip_needless_lines - return unless @skip_lines - - until @scanner.eos? - @scanner.keep_start - line = @scanner.scan_all(@not_line_end) || "".encode(@encoding) - line << @row_separator if parse_row_end - if skip_line?(line) - @lineno += 1 - @scanner.keep_drop - else - @scanner.keep_back - return - end - end - end - - def skip_line?(line) - line = line.delete_suffix(@row_separator) - case @skip_lines - when String - line.include?(@skip_lines) - when Regexp - @skip_lines.match?(line) - else - @skip_lines.match(line) - end - end - - def validate_field_size(field) - return unless @max_field_size - return if field.size <= @max_field_size - ignore_broken_line - message = "Field size exceeded: #{field.size} > #{@max_field_size}" - raise MalformedCSVError.new(message, @lineno) - end - - def parse_no_quote(&block) - @scanner.each_line(@row_separator) do |line| - next if @skip_lines and skip_line?(line) - original_line = line - line = line.delete_suffix(@row_separator) - - if line.empty? - next if @skip_blanks - row = [] - quoted_fields = [] - else - line = strip_value(line) - row = line.split(@split_column_separator, -1) - quoted_fields = [false] * row.size - if @max_field_size - row.each do |column| - validate_field_size(column) - end - end - n_columns = row.size - i = 0 - while i < n_columns - row[i] = nil if row[i].empty? - i += 1 - end - end - @last_line = original_line - emit_row(row, quoted_fields, &block) - end - end - - def parse_quotable_loose(&block) - @scanner.keep_start - @scanner.each_line(@row_separator) do |line| - if @skip_lines and skip_line?(line) - @scanner.keep_drop - @scanner.keep_start - next - end - original_line = line - line = line.delete_suffix(@row_separator) - - if line.empty? - if @skip_blanks - @scanner.keep_drop - @scanner.keep_start - next - end - row = [] - quoted_fields = [] - elsif line.include?(@cr) or line.include?(@lf) - @scanner.keep_back - @need_robust_parsing = true - return parse_quotable_robust(&block) - else - row = line.split(@split_column_separator, -1) - quoted_fields = [] - n_columns = row.size - i = 0 - while i < n_columns - column = row[i] - if column.empty? - quoted_fields << false - row[i] = nil - else - n_quotes = column.count(@quote_character) - if n_quotes.zero? - quoted_fields << false - # no quote - elsif n_quotes == 2 and - column.start_with?(@quote_character) and - column.end_with?(@quote_character) - quoted_fields << true - row[i] = column[1..-2] - else - @scanner.keep_back - @need_robust_parsing = true - return parse_quotable_robust(&block) - end - validate_field_size(row[i]) - end - i += 1 - end - end - @scanner.keep_drop - @scanner.keep_start - @last_line = original_line - emit_row(row, quoted_fields, &block) - end - @scanner.keep_drop - end - - def parse_quotable_robust(&block) - row = [] - quoted_fields = [] - skip_needless_lines - start_row - while true - @quoted_column_value = false - @unquoted_column_value = false - @scanner.scan_all(@strip_value) if @strip_value - value = parse_column_value - if value - @scanner.scan_all(@strip_value) if @strip_value - validate_field_size(value) - end - if parse_column_end - row << value - quoted_fields << @quoted_column_value - elsif parse_row_end - if row.empty? and value.nil? - emit_row([], [], &block) unless @skip_blanks - else - row << value - quoted_fields << @quoted_column_value - emit_row(row, quoted_fields, &block) - row = [] - quoted_fields = [] - end - skip_needless_lines - start_row - elsif @scanner.eos? - break if row.empty? and value.nil? - row << value - quoted_fields << @quoted_column_value - emit_row(row, quoted_fields, &block) - break - else - if @quoted_column_value - if liberal_parsing? and (new_line = @scanner.check(@line_end)) - message = - "Illegal end-of-line sequence outside of a quoted field " + - "<#{new_line.inspect}>" - else - message = "Any value after quoted field isn't allowed" - end - ignore_broken_line - raise MalformedCSVError.new(message, @lineno) - elsif @unquoted_column_value and - (new_line = @scanner.scan(@line_end)) - ignore_broken_line - message = "Unquoted fields do not allow new line " + - "<#{new_line.inspect}>" - raise MalformedCSVError.new(message, @lineno) - elsif @scanner.rest.start_with?(@quote_character) - ignore_broken_line - message = "Illegal quoting" - raise MalformedCSVError.new(message, @lineno) - elsif (new_line = @scanner.scan(@line_end)) - ignore_broken_line - message = "New line must be <#{@row_separator.inspect}> " + - "not <#{new_line.inspect}>" - raise MalformedCSVError.new(message, @lineno) - else - ignore_broken_line - raise MalformedCSVError.new("TODO: Meaningful message", - @lineno) - end - end - end - end - - def parse_column_value - if @liberal_parsing - quoted_value = parse_quoted_column_value - if quoted_value - @scanner.scan_all(@strip_value) if @strip_value - unquoted_value = parse_unquoted_column_value - if unquoted_value - if @double_quote_outside_quote - unquoted_value = unquoted_value.gsub(@quote_character * 2, - @quote_character) - if quoted_value.empty? # %Q{""...} case - return @quote_character + unquoted_value - end - end - @quote_character + quoted_value + @quote_character + unquoted_value - else - quoted_value - end - else - parse_unquoted_column_value - end - elsif @may_quoted - parse_quoted_column_value || - parse_unquoted_column_value - else - parse_unquoted_column_value || - parse_quoted_column_value - end - end - - def parse_unquoted_column_value - value = @scanner.scan_all(@unquoted_value) - return nil unless value - - @unquoted_column_value = true - if @first_column_separators - while true - @scanner.keep_start - is_column_end = @column_ends.all? do |column_end| - @scanner.scan(column_end) - end - @scanner.keep_back - break if is_column_end - sub_separator = @scanner.scan_all(@first_column_separators) - break if sub_separator.nil? - value << sub_separator - sub_value = @scanner.scan_all(@unquoted_value) - break if sub_value.nil? - value << sub_value - end - end - value.gsub!(@backslash_quote_character, @quote_character) if @backslash_quote - if @rstrip_value - value.gsub!(@rstrip_value, "") - end - value - end - - def parse_quoted_column_value - quotes = @scanner.scan_all(@quotes) - return nil unless quotes - - @quoted_column_value = true - n_quotes = quotes.size - if (n_quotes % 2).zero? - quotes[0, (n_quotes - 2) / 2] - else - value = quotes[0, n_quotes / 2] - while true - quoted_value = @scanner.scan_all(@quoted_value) - value << quoted_value if quoted_value - if @backslash_quote - if @scanner.scan(@escaped_backslash) - if @scanner.scan(@escaped_quote) - value << @quote_character - else - value << @backslash_character - end - next - end - end - - quotes = @scanner.scan_all(@quotes) - unless quotes - ignore_broken_line - message = "Unclosed quoted field" - raise MalformedCSVError.new(message, @lineno) - end - n_quotes = quotes.size - if n_quotes == 1 - break - else - value << quotes[0, n_quotes / 2] - break if (n_quotes % 2) == 1 - end - end - value - end - end - - def parse_column_end - return true if @scanner.scan(@column_end) - return false unless @column_ends - - @scanner.keep_start - if @column_ends.all? {|column_end| @scanner.scan(column_end)} - @scanner.keep_drop - true - else - @scanner.keep_back - false - end - end - - def parse_row_end - return true if @scanner.scan(@row_end) - return false unless @row_ends - @scanner.keep_start - if @row_ends.all? {|row_end| @scanner.scan(row_end)} - @scanner.keep_drop - true - else - @scanner.keep_back - false - end - end - - def strip_value(value) - return value unless @strip - return value if value.nil? - - case @strip - when String - while value.delete_prefix!(@strip) - # do nothing - end - while value.delete_suffix!(@strip) - # do nothing - end - else - value.strip! - end - value - end - - def ignore_broken_line - @scanner.scan_all(@not_line_end) - @scanner.scan_all(@line_end) - @lineno += 1 - end - - def start_row - if @last_line - @last_line = nil - else - @scanner.keep_drop - end - @scanner.keep_start - end - - def emit_row(row, quoted_fields, &block) - @lineno += 1 - - raw_row = row - if @use_headers - if @headers.nil? - @headers = adjust_headers(row, quoted_fields) - return unless @return_headers - row = Row.new(@headers, row, true) - else - row = Row.new(@headers, - @fields_converter.convert(raw_row, @headers, @lineno, quoted_fields)) - end - else - # convert fields, if needed... - row = @fields_converter.convert(raw_row, nil, @lineno, quoted_fields) - end - - # inject unconverted fields and accessor, if requested... - if @unconverted_fields and not row.respond_to?(:unconverted_fields) - add_unconverted_fields(row, raw_row) - end - - yield(row) - end - - # This method injects an instance variable <tt>unconverted_fields</tt> into - # +row+ and an accessor method for +row+ called unconverted_fields(). The - # variable is set to the contents of +fields+. - def add_unconverted_fields(row, fields) - class << row - attr_reader :unconverted_fields - end - row.instance_variable_set(:@unconverted_fields, fields) - row - end - end -end diff --git a/lib/csv/row.rb b/lib/csv/row.rb deleted file mode 100644 index 86323f7d0a..0000000000 --- a/lib/csv/row.rb +++ /dev/null @@ -1,757 +0,0 @@ -# frozen_string_literal: true - -require "forwardable" - -class CSV - # = \CSV::Row - # A \CSV::Row instance represents a \CSV table row. - # (see {class CSV}[../CSV.html]). - # - # The instance may have: - # - Fields: each is an object, not necessarily a \String. - # - Headers: each serves a key, and also need not be a \String. - # - # === Instance Methods - # - # \CSV::Row has three groups of instance methods: - # - Its own internally defined instance methods. - # - Methods included by module Enumerable. - # - Methods delegated to class Array.: - # * Array#empty? - # * Array#length - # * Array#size - # - # == Creating a \CSV::Row Instance - # - # Commonly, a new \CSV::Row instance is created by parsing \CSV source - # that has headers: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.each {|row| p row } - # Output: - # #<CSV::Row "Name":"foo" "Value":"0"> - # #<CSV::Row "Name":"bar" "Value":"1"> - # #<CSV::Row "Name":"baz" "Value":"2"> - # - # You can also create a row directly. See ::new. - # - # == Headers - # - # Like a \CSV::Table, a \CSV::Row has headers. - # - # A \CSV::Row that was created by parsing \CSV source - # inherits its headers from the table: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # row = table.first - # row.headers # => ["Name", "Value"] - # - # You can also create a new row with headers; - # like the keys in a \Hash, the headers need not be Strings: - # row = CSV::Row.new([:name, :value], ['foo', 0]) - # row.headers # => [:name, :value] - # - # The new row retains its headers even if added to a table - # that has headers: - # table << row # => #<CSV::Table mode:col_or_row row_count:5> - # row.headers # => [:name, :value] - # row[:name] # => "foo" - # row['Name'] # => nil - # - # - # - # == Accessing Fields - # - # You may access a field in a \CSV::Row with either its \Integer index - # (\Array-style) or its header (\Hash-style). - # - # Fetch a field using method #[]: - # row = CSV::Row.new(['Name', 'Value'], ['foo', 0]) - # row[1] # => 0 - # row['Value'] # => 0 - # - # Set a field using method #[]=: - # row = CSV::Row.new(['Name', 'Value'], ['foo', 0]) - # row # => #<CSV::Row "Name":"foo" "Value":0> - # row[0] = 'bar' - # row['Value'] = 1 - # row # => #<CSV::Row "Name":"bar" "Value":1> - # - class Row - # :call-seq: - # CSV::Row.new(headers, fields, header_row = false) -> csv_row - # - # Returns the new \CSV::Row instance constructed from - # arguments +headers+ and +fields+; both should be Arrays; - # note that the fields need not be Strings: - # row = CSV::Row.new(['Name', 'Value'], ['foo', 0]) - # row # => #<CSV::Row "Name":"foo" "Value":0> - # - # If the \Array lengths are different, the shorter is +nil+-filled: - # row = CSV::Row.new(['Name', 'Value', 'Date', 'Size'], ['foo', 0]) - # row # => #<CSV::Row "Name":"foo" "Value":0 "Date":nil "Size":nil> - # - # Each \CSV::Row object is either a <i>field row</i> or a <i>header row</i>; - # by default, a new row is a field row; for the row created above: - # row.field_row? # => true - # row.header_row? # => false - # - # If the optional argument +header_row+ is given as +true+, - # the created row is a header row: - # row = CSV::Row.new(['Name', 'Value'], ['foo', 0], header_row = true) - # row # => #<CSV::Row "Name":"foo" "Value":0> - # row.field_row? # => false - # row.header_row? # => true - def initialize(headers, fields, header_row = false) - @header_row = header_row - headers.each { |h| h.freeze if h.is_a? String } - - # handle extra headers or fields - @row = if headers.size >= fields.size - headers.zip(fields) - else - fields.zip(headers).each(&:reverse!) - end - end - - # Internal data format used to compare equality. - attr_reader :row - protected :row - - ### Array Delegation ### - - extend Forwardable - def_delegators :@row, :empty?, :length, :size - - # :call-seq: - # row.initialize_copy(other_row) -> self - # - # Calls superclass method. - def initialize_copy(other) - super_return_value = super - @row = @row.collect(&:dup) - super_return_value - end - - # :call-seq: - # row.header_row? -> true or false - # - # Returns +true+ if this is a header row, +false+ otherwise. - def header_row? - @header_row - end - - # :call-seq: - # row.field_row? -> true or false - # - # Returns +true+ if this is a field row, +false+ otherwise. - def field_row? - not header_row? - end - - # :call-seq: - # row.headers -> array_of_headers - # - # Returns the headers for this row: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # row = table.first - # row.headers # => ["Name", "Value"] - def headers - @row.map(&:first) - end - - # :call-seq: - # field(index) -> value - # field(header) -> value - # field(header, offset) -> value - # - # Returns the field value for the given +index+ or +header+. - # - # --- - # - # Fetch field value by \Integer index: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.field(0) # => "foo" - # row.field(1) # => "bar" - # - # Counts backward from the last column if +index+ is negative: - # row.field(-1) # => "0" - # row.field(-2) # => "foo" - # - # Returns +nil+ if +index+ is out of range: - # row.field(2) # => nil - # row.field(-3) # => nil - # - # --- - # - # Fetch field value by header (first found): - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.field('Name') # => "Foo" - # - # Fetch field value by header, ignoring +offset+ leading fields: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.field('Name', 2) # => "Baz" - # - # Returns +nil+ if the header does not exist. - def field(header_or_index, minimum_index = 0) - # locate the pair - finder = (header_or_index.is_a?(Integer) || header_or_index.is_a?(Range)) ? :[] : :assoc - pair = @row[minimum_index..-1].public_send(finder, header_or_index) - - # return the field if we have a pair - if pair.nil? - nil - else - header_or_index.is_a?(Range) ? pair.map(&:last) : pair.last - end - end - alias_method :[], :field - - # - # :call-seq: - # fetch(header) -> value - # fetch(header, default) -> value - # fetch(header) {|row| ... } -> value - # - # Returns the field value as specified by +header+. - # - # --- - # - # With the single argument +header+, returns the field value - # for that header (first found): - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.fetch('Name') # => "Foo" - # - # Raises exception +KeyError+ if the header does not exist. - # - # --- - # - # With arguments +header+ and +default+ given, - # returns the field value for the header (first found) - # if the header exists, otherwise returns +default+: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.fetch('Name', '') # => "Foo" - # row.fetch(:nosuch, '') # => "" - # - # --- - # - # With argument +header+ and a block given, - # returns the field value for the header (first found) - # if the header exists; otherwise calls the block - # and returns its return value: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.fetch('Name') {|header| fail 'Cannot happen' } # => "Foo" - # row.fetch(:nosuch) {|header| "Header '#{header} not found'" } # => "Header 'nosuch not found'" - def fetch(header, *varargs) - raise ArgumentError, "Too many arguments" if varargs.length > 1 - pair = @row.assoc(header) - if pair - pair.last - else - if block_given? - yield header - elsif varargs.empty? - raise KeyError, "key not found: #{header}" - else - varargs.first - end - end - end - - # :call-seq: - # row.has_key?(header) -> true or false - # - # Returns +true+ if there is a field with the given +header+, - # +false+ otherwise. - def has_key?(header) - !!@row.assoc(header) - end - alias_method :include?, :has_key? - alias_method :key?, :has_key? - alias_method :member?, :has_key? - alias_method :header?, :has_key? - - # - # :call-seq: - # row[index] = value -> value - # row[header, offset] = value -> value - # row[header] = value -> value - # - # Assigns the field value for the given +index+ or +header+; - # returns +value+. - # - # --- - # - # Assign field value by \Integer index: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row[0] = 'Bat' - # row[1] = 3 - # row # => #<CSV::Row "Name":"Bat" "Value":3> - # - # Counts backward from the last column if +index+ is negative: - # row[-1] = 4 - # row[-2] = 'Bam' - # row # => #<CSV::Row "Name":"Bam" "Value":4> - # - # Extends the row with <tt>nil:nil</tt> if positive +index+ is not in the row: - # row[4] = 5 - # row # => #<CSV::Row "Name":"bad" "Value":4 nil:nil nil:nil nil:5> - # - # Raises IndexError if negative +index+ is too small (too far from zero). - # - # --- - # - # Assign field value by header (first found): - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row['Name'] = 'Bat' - # row # => #<CSV::Row "Name":"Bat" "Name":"Bar" "Name":"Baz"> - # - # Assign field value by header, ignoring +offset+ leading fields: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row['Name', 2] = 4 - # row # => #<CSV::Row "Name":"Foo" "Name":"Bar" "Name":4> - # - # Append new field by (new) header: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row['New'] = 6 - # row# => #<CSV::Row "Name":"foo" "Value":"0" "New":6> - def []=(*args) - value = args.pop - - if args.first.is_a? Integer - if @row[args.first].nil? # extending past the end with index - @row[args.first] = [nil, value] - @row.map! { |pair| pair.nil? ? [nil, nil] : pair } - else # normal index assignment - @row[args.first][1] = value - end - else - index = index(*args) - if index.nil? # appending a field - self << [args.first, value] - else # normal header assignment - @row[index][1] = value - end - end - end - - # - # :call-seq: - # row << [header, value] -> self - # row << hash -> self - # row << value -> self - # - # Adds a field to +self+; returns +self+: - # - # If the argument is a 2-element \Array <tt>[header, value]</tt>, - # a field is added with the given +header+ and +value+: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row << ['NAME', 'Bat'] - # row # => #<CSV::Row "Name":"Foo" "Name":"Bar" "Name":"Baz" "NAME":"Bat"> - # - # If the argument is a \Hash, each <tt>key-value</tt> pair is added - # as a field with header +key+ and value +value+. - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row << {NAME: 'Bat', name: 'Bam'} - # row # => #<CSV::Row "Name":"Foo" "Name":"Bar" "Name":"Baz" NAME:"Bat" name:"Bam"> - # - # Otherwise, the given +value+ is added as a field with no header. - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row << 'Bag' - # row # => #<CSV::Row "Name":"Foo" "Name":"Bar" "Name":"Baz" nil:"Bag"> - def <<(arg) - if arg.is_a?(Array) and arg.size == 2 # appending a header and name - @row << arg - elsif arg.is_a?(Hash) # append header and name pairs - arg.each { |pair| @row << pair } - else # append field value - @row << [nil, arg] - end - - self # for chaining - end - - # :call-seq: - # row.push(*values) -> self - # - # Appends each of the given +values+ to +self+ as a field; returns +self+: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.push('Bat', 'Bam') - # row # => #<CSV::Row "Name":"Foo" "Name":"Bar" "Name":"Baz" nil:"Bat" nil:"Bam"> - def push(*args) - args.each { |arg| self << arg } - - self # for chaining - end - - # - # :call-seq: - # delete(index) -> [header, value] or nil - # delete(header) -> [header, value] or empty_array - # delete(header, offset) -> [header, value] or empty_array - # - # Removes a specified field from +self+; returns the 2-element \Array - # <tt>[header, value]</tt> if the field exists. - # - # If an \Integer argument +index+ is given, - # removes and returns the field at offset +index+, - # or returns +nil+ if the field does not exist: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.delete(1) # => ["Name", "Bar"] - # row.delete(50) # => nil - # - # Otherwise, if the single argument +header+ is given, - # removes and returns the first-found field with the given header, - # of returns a new empty \Array if the field does not exist: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.delete('Name') # => ["Name", "Foo"] - # row.delete('NAME') # => [] - # - # If argument +header+ and \Integer argument +offset+ are given, - # removes and returns the first-found field with the given header - # whose +index+ is at least as large as +offset+: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.delete('Name', 1) # => ["Name", "Bar"] - # row.delete('NAME', 1) # => [] - def delete(header_or_index, minimum_index = 0) - if header_or_index.is_a? Integer # by index - @row.delete_at(header_or_index) - elsif i = index(header_or_index, minimum_index) # by header - @row.delete_at(i) - else - [ ] - end - end - - # :call-seq: - # row.delete_if {|header, value| ... } -> self - # - # Removes fields from +self+ as selected by the block; returns +self+. - # - # Removes each field for which the block returns a truthy value: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.delete_if {|header, value| value.start_with?('B') } # => true - # row # => #<CSV::Row "Name":"Foo"> - # row.delete_if {|header, value| header.start_with?('B') } # => false - # - # If no block is given, returns a new Enumerator: - # row.delete_if # => #<Enumerator: #<CSV::Row "Name":"Foo">:delete_if> - def delete_if(&block) - return enum_for(__method__) { size } unless block_given? - - @row.delete_if(&block) - - self # for chaining - end - - # :call-seq: - # self.fields(*specifiers) -> array_of_fields - # - # Returns field values per the given +specifiers+, which may be any mixture of: - # - \Integer index. - # - \Range of \Integer indexes. - # - 2-element \Array containing a header and offset. - # - Header. - # - \Range of headers. - # - # For +specifier+ in one of the first four cases above, - # returns the result of <tt>self.field(specifier)</tt>; see #field. - # - # Although there may be any number of +specifiers+, - # the examples here will illustrate one at a time. - # - # When the specifier is an \Integer +index+, - # returns <tt>self.field(index)</tt>L - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.fields(1) # => ["Bar"] - # - # When the specifier is a \Range of \Integers +range+, - # returns <tt>self.field(range)</tt>: - # row.fields(1..2) # => ["Bar", "Baz"] - # - # When the specifier is a 2-element \Array +array+, - # returns <tt>self.field(array)</tt>L - # row.fields('Name', 1) # => ["Foo", "Bar"] - # - # When the specifier is a header +header+, - # returns <tt>self.field(header)</tt>L - # row.fields('Name') # => ["Foo"] - # - # When the specifier is a \Range of headers +range+, - # forms a new \Range +new_range+ from the indexes of - # <tt>range.start</tt> and <tt>range.end</tt>, - # and returns <tt>self.field(new_range)</tt>: - # source = "Name,NAME,name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.fields('Name'..'NAME') # => ["Foo", "Bar"] - # - # Returns all fields if no argument given: - # row.fields # => ["Foo", "Bar", "Baz"] - def fields(*headers_and_or_indices) - if headers_and_or_indices.empty? # return all fields--no arguments - @row.map(&:last) - else # or work like values_at() - all = [] - headers_and_or_indices.each do |h_or_i| - if h_or_i.is_a? Range - index_begin = h_or_i.begin.is_a?(Integer) ? h_or_i.begin : - index(h_or_i.begin) - index_end = h_or_i.end.is_a?(Integer) ? h_or_i.end : - index(h_or_i.end) - new_range = h_or_i.exclude_end? ? (index_begin...index_end) : - (index_begin..index_end) - all.concat(fields.values_at(new_range)) - else - all << field(*Array(h_or_i)) - end - end - return all - end - end - alias_method :values_at, :fields - - # :call-seq: - # index(header) -> index - # index(header, offset) -> index - # - # Returns the index for the given header, if it exists; - # otherwise returns +nil+. - # - # With the single argument +header+, returns the index - # of the first-found field with the given +header+: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.index('Name') # => 0 - # row.index('NAME') # => nil - # - # With arguments +header+ and +offset+, - # returns the index of the first-found field with given +header+, - # but ignoring the first +offset+ fields: - # row.index('Name', 1) # => 1 - # row.index('Name', 3) # => nil - def index(header, minimum_index = 0) - # find the pair - index = headers[minimum_index..-1].index(header) - # return the index at the right offset, if we found one - index.nil? ? nil : index + minimum_index - end - - # :call-seq: - # row.field?(value) -> true or false - # - # Returns +true+ if +value+ is a field in this row, +false+ otherwise: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.field?('Bar') # => true - # row.field?('BAR') # => false - def field?(data) - fields.include? data - end - - include Enumerable - - # :call-seq: - # row.each {|header, value| ... } -> self - # - # Calls the block with each header-value pair; returns +self+: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.each {|header, value| p [header, value] } - # Output: - # ["Name", "Foo"] - # ["Name", "Bar"] - # ["Name", "Baz"] - # - # If no block is given, returns a new Enumerator: - # row.each # => #<Enumerator: #<CSV::Row "Name":"Foo" "Name":"Bar" "Name":"Baz">:each> - def each(&block) - return enum_for(__method__) { size } unless block_given? - - @row.each(&block) - - self # for chaining - end - - alias_method :each_pair, :each - - # :call-seq: - # row == other -> true or false - # - # Returns +true+ if +other+ is a /CSV::Row that has the same - # fields (headers and values) in the same order as +self+; - # otherwise returns +false+: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # other_row = table[0] - # row == other_row # => true - # other_row = table[1] - # row == other_row # => false - def ==(other) - return @row == other.row if other.is_a? CSV::Row - @row == other - end - - # :call-seq: - # row.to_h -> hash - # - # Returns the new \Hash formed by adding each header-value pair in +self+ - # as a key-value pair in the \Hash. - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.to_h # => {"Name"=>"foo", "Value"=>"0"} - # - # Header order is preserved, but repeated headers are ignored: - # source = "Name,Name,Name\nFoo,Bar,Baz\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.to_h # => {"Name"=>"Foo"} - def to_h - hash = {} - each do |key, _value| - hash[key] = self[key] unless hash.key?(key) - end - hash - end - alias_method :to_hash, :to_h - - # :call-seq: - # row.deconstruct_keys(keys) -> hash - # - # Returns the new \Hash suitable for pattern matching containing only the - # keys specified as an argument. - def deconstruct_keys(keys) - if keys.nil? - to_h - else - keys.to_h { |key| [key, self[key]] } - end - end - - alias_method :to_ary, :to_a - - # :call-seq: - # row.deconstruct -> array - # - # Returns the new \Array suitable for pattern matching containing the values - # of the row. - def deconstruct - fields - end - - # :call-seq: - # row.to_csv -> csv_string - # - # Returns the row as a \CSV String. Headers are not included: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.to_csv # => "foo,0\n" - def to_csv(**options) - fields.to_csv(**options) - end - alias_method :to_s, :to_csv - - # :call-seq: - # row.dig(index_or_header, *identifiers) -> object - # - # Finds and returns the object in nested object that is specified - # by +index_or_header+ and +specifiers+. - # - # The nested objects may be instances of various classes. - # See {Dig Methods}[rdoc-ref:dig_methods.rdoc]. - # - # Examples: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.dig(1) # => "0" - # row.dig('Value') # => "0" - # row.dig(5) # => nil - def dig(index_or_header, *indexes) - value = field(index_or_header) - if value.nil? - nil - elsif indexes.empty? - value - else - unless value.respond_to?(:dig) - raise TypeError, "#{value.class} does not have \#dig method" - end - value.dig(*indexes) - end - end - - # :call-seq: - # row.inspect -> string - # - # Returns an ASCII-compatible \String showing: - # - Class \CSV::Row. - # - Header-value pairs. - # Example: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # row = table[0] - # row.inspect # => "#<CSV::Row \"Name\":\"foo\" \"Value\":\"0\">" - def inspect - str = ["#<", self.class.to_s] - each do |header, field| - str << " " << (header.is_a?(Symbol) ? header.to_s : header.inspect) << - ":" << field.inspect - end - str << ">" - begin - str.join('') - rescue # any encoding error - str.map do |s| - e = Encoding::Converter.asciicompat_encoding(s.encoding) - e ? s.encode(e) : s.force_encoding("ASCII-8BIT") - end.join('') - end - end - end -end diff --git a/lib/csv/table.rb b/lib/csv/table.rb deleted file mode 100644 index fb19f5453f..0000000000 --- a/lib/csv/table.rb +++ /dev/null @@ -1,1055 +0,0 @@ -# frozen_string_literal: true - -require "forwardable" - -class CSV - # = \CSV::Table - # A \CSV::Table instance represents \CSV data. - # (see {class CSV}[../CSV.html]). - # - # The instance may have: - # - Rows: each is a Table::Row object. - # - Headers: names for the columns. - # - # === Instance Methods - # - # \CSV::Table has three groups of instance methods: - # - Its own internally defined instance methods. - # - Methods included by module Enumerable. - # - Methods delegated to class Array.: - # * Array#empty? - # * Array#length - # * Array#size - # - # == Creating a \CSV::Table Instance - # - # Commonly, a new \CSV::Table instance is created by parsing \CSV source - # using headers: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.class # => CSV::Table - # - # You can also create an instance directly. See ::new. - # - # == Headers - # - # If a table has headers, the headers serve as labels for the columns of data. - # Each header serves as the label for its column. - # - # The headers for a \CSV::Table object are stored as an \Array of Strings. - # - # Commonly, headers are defined in the first row of \CSV source: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.headers # => ["Name", "Value"] - # - # If no headers are defined, the \Array is empty: - # table = CSV::Table.new([]) - # table.headers # => [] - # - # == Access Modes - # - # \CSV::Table provides three modes for accessing table data: - # - \Row mode. - # - Column mode. - # - Mixed mode (the default for a new table). - # - # The access mode for a\CSV::Table instance affects the behavior - # of some of its instance methods: - # - #[] - # - #[]= - # - #delete - # - #delete_if - # - #each - # - #values_at - # - # === \Row Mode - # - # Set a table to row mode with method #by_row!: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.by_row! # => #<CSV::Table mode:row row_count:4> - # - # Specify a single row by an \Integer index: - # # Get a row. - # table[1] # => #<CSV::Row "Name":"bar" "Value":"1"> - # # Set a row, then get it. - # table[1] = CSV::Row.new(['Name', 'Value'], ['bam', 3]) - # table[1] # => #<CSV::Row "Name":"bam" "Value":3> - # - # Specify a sequence of rows by a \Range: - # # Get rows. - # table[1..2] # => [#<CSV::Row "Name":"bam" "Value":3>, #<CSV::Row "Name":"baz" "Value":"2">] - # # Set rows, then get them. - # table[1..2] = [ - # CSV::Row.new(['Name', 'Value'], ['bat', 4]), - # CSV::Row.new(['Name', 'Value'], ['bad', 5]), - # ] - # table[1..2] # => [["Name", #<CSV::Row "Name":"bat" "Value":4>], ["Value", #<CSV::Row "Name":"bad" "Value":5>]] - # - # === Column Mode - # - # Set a table to column mode with method #by_col!: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.by_col! # => #<CSV::Table mode:col row_count:4> - # - # Specify a column by an \Integer index: - # # Get a column. - # table[0] - # # Set a column, then get it. - # table[0] = ['FOO', 'BAR', 'BAZ'] - # table[0] # => ["FOO", "BAR", "BAZ"] - # - # Specify a column by its \String header: - # # Get a column. - # table['Name'] # => ["FOO", "BAR", "BAZ"] - # # Set a column, then get it. - # table['Name'] = ['Foo', 'Bar', 'Baz'] - # table['Name'] # => ["Foo", "Bar", "Baz"] - # - # === Mixed Mode - # - # In mixed mode, you can refer to either rows or columns: - # - An \Integer index refers to a row. - # - A \Range index refers to multiple rows. - # - A \String index refers to a column. - # - # Set a table to mixed mode with method #by_col_or_row!: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.by_col_or_row! # => #<CSV::Table mode:col_or_row row_count:4> - # - # Specify a single row by an \Integer index: - # # Get a row. - # table[1] # => #<CSV::Row "Name":"bar" "Value":"1"> - # # Set a row, then get it. - # table[1] = CSV::Row.new(['Name', 'Value'], ['bam', 3]) - # table[1] # => #<CSV::Row "Name":"bam" "Value":3> - # - # Specify a sequence of rows by a \Range: - # # Get rows. - # table[1..2] # => [#<CSV::Row "Name":"bam" "Value":3>, #<CSV::Row "Name":"baz" "Value":"2">] - # # Set rows, then get them. - # table[1] = CSV::Row.new(['Name', 'Value'], ['bat', 4]) - # table[2] = CSV::Row.new(['Name', 'Value'], ['bad', 5]) - # table[1..2] # => [["Name", #<CSV::Row "Name":"bat" "Value":4>], ["Value", #<CSV::Row "Name":"bad" "Value":5>]] - # - # Specify a column by its \String header: - # # Get a column. - # table['Name'] # => ["foo", "bat", "bad"] - # # Set a column, then get it. - # table['Name'] = ['Foo', 'Bar', 'Baz'] - # table['Name'] # => ["Foo", "Bar", "Baz"] - class Table - # :call-seq: - # CSV::Table.new(array_of_rows, headers = nil) -> csv_table - # - # Returns a new \CSV::Table object. - # - # - Argument +array_of_rows+ must be an \Array of CSV::Row objects. - # - Argument +headers+, if given, may be an \Array of Strings. - # - # --- - # - # Create an empty \CSV::Table object: - # table = CSV::Table.new([]) - # table # => #<CSV::Table mode:col_or_row row_count:1> - # - # Create a non-empty \CSV::Table object: - # rows = [ - # CSV::Row.new([], []), - # CSV::Row.new([], []), - # CSV::Row.new([], []), - # ] - # table = CSV::Table.new(rows) - # table # => #<CSV::Table mode:col_or_row row_count:4> - # - # --- - # - # If argument +headers+ is an \Array of Strings, - # those Strings become the table's headers: - # table = CSV::Table.new([], headers: ['Name', 'Age']) - # table.headers # => ["Name", "Age"] - # - # If argument +headers+ is not given and the table has rows, - # the headers are taken from the first row: - # rows = [ - # CSV::Row.new(['Foo', 'Bar'], []), - # CSV::Row.new(['foo', 'bar'], []), - # CSV::Row.new(['FOO', 'BAR'], []), - # ] - # table = CSV::Table.new(rows) - # table.headers # => ["Foo", "Bar"] - # - # If argument +headers+ is not given and the table is empty (has no rows), - # the headers are also empty: - # table = CSV::Table.new([]) - # table.headers # => [] - # - # --- - # - # Raises an exception if argument +array_of_rows+ is not an \Array object: - # # Raises NoMethodError (undefined method `first' for :foo:Symbol): - # CSV::Table.new(:foo) - # - # Raises an exception if an element of +array_of_rows+ is not a \CSV::Table object: - # # Raises NoMethodError (undefined method `headers' for :foo:Symbol): - # CSV::Table.new([:foo]) - def initialize(array_of_rows, headers: nil) - @table = array_of_rows - @headers = headers - unless @headers - if @table.empty? - @headers = [] - else - @headers = @table.first.headers - end - end - - @mode = :col_or_row - end - - # The current access mode for indexing and iteration. - attr_reader :mode - - # Internal data format used to compare equality. - attr_reader :table - protected :table - - ### Array Delegation ### - - extend Forwardable - def_delegators :@table, :empty?, :length, :size - - # :call-seq: - # table.by_col -> table_dup - # - # Returns a duplicate of +self+, in column mode - # (see {Column Mode}[#class-CSV::Table-label-Column+Mode]): - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.mode # => :col_or_row - # dup_table = table.by_col - # dup_table.mode # => :col - # dup_table.equal?(table) # => false # It's a dup - # - # This may be used to chain method calls without changing the mode - # (but also will affect performance and memory usage): - # dup_table.by_col['Name'] - # - # Also note that changes to the duplicate table will not affect the original. - def by_col - self.class.new(@table.dup).by_col! - end - - # :call-seq: - # table.by_col! -> self - # - # Sets the mode for +self+ to column mode - # (see {Column Mode}[#class-CSV::Table-label-Column+Mode]); returns +self+: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.mode # => :col_or_row - # table1 = table.by_col! - # table.mode # => :col - # table1.equal?(table) # => true # Returned self - def by_col! - @mode = :col - - self - end - - # :call-seq: - # table.by_col_or_row -> table_dup - # - # Returns a duplicate of +self+, in mixed mode - # (see {Mixed Mode}[#class-CSV::Table-label-Mixed+Mode]): - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true).by_col! - # table.mode # => :col - # dup_table = table.by_col_or_row - # dup_table.mode # => :col_or_row - # dup_table.equal?(table) # => false # It's a dup - # - # This may be used to chain method calls without changing the mode - # (but also will affect performance and memory usage): - # dup_table.by_col_or_row['Name'] - # - # Also note that changes to the duplicate table will not affect the original. - def by_col_or_row - self.class.new(@table.dup).by_col_or_row! - end - - # :call-seq: - # table.by_col_or_row! -> self - # - # Sets the mode for +self+ to mixed mode - # (see {Mixed Mode}[#class-CSV::Table-label-Mixed+Mode]); returns +self+: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true).by_col! - # table.mode # => :col - # table1 = table.by_col_or_row! - # table.mode # => :col_or_row - # table1.equal?(table) # => true # Returned self - def by_col_or_row! - @mode = :col_or_row - - self - end - - # :call-seq: - # table.by_row -> table_dup - # - # Returns a duplicate of +self+, in row mode - # (see {Row Mode}[#class-CSV::Table-label-Row+Mode]): - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.mode # => :col_or_row - # dup_table = table.by_row - # dup_table.mode # => :row - # dup_table.equal?(table) # => false # It's a dup - # - # This may be used to chain method calls without changing the mode - # (but also will affect performance and memory usage): - # dup_table.by_row[1] - # - # Also note that changes to the duplicate table will not affect the original. - def by_row - self.class.new(@table.dup).by_row! - end - - # :call-seq: - # table.by_row! -> self - # - # Sets the mode for +self+ to row mode - # (see {Row Mode}[#class-CSV::Table-label-Row+Mode]); returns +self+: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.mode # => :col_or_row - # table1 = table.by_row! - # table.mode # => :row - # table1.equal?(table) # => true # Returned self - def by_row! - @mode = :row - - self - end - - # :call-seq: - # table.headers -> array_of_headers - # - # Returns a new \Array containing the \String headers for the table. - # - # If the table is not empty, returns the headers from the first row: - # rows = [ - # CSV::Row.new(['Foo', 'Bar'], []), - # CSV::Row.new(['FOO', 'BAR'], []), - # CSV::Row.new(['foo', 'bar'], []), - # ] - # table = CSV::Table.new(rows) - # table.headers # => ["Foo", "Bar"] - # table.delete(0) - # table.headers # => ["FOO", "BAR"] - # table.delete(0) - # table.headers # => ["foo", "bar"] - # - # If the table is empty, returns a copy of the headers in the table itself: - # table.delete(0) - # table.headers # => ["Foo", "Bar"] - def headers - if @table.empty? - @headers.dup - else - @table.first.headers - end - end - - # :call-seq: - # table[n] -> row or column_data - # table[range] -> array_of_rows or array_of_column_data - # table[header] -> array_of_column_data - # - # Returns data from the table; does not modify the table. - # - # --- - # - # Fetch a \Row by Its \Integer Index:: - # - Form: <tt>table[n]</tt>, +n+ an integer. - # - Access mode: <tt>:row</tt> or <tt>:col_or_row</tt>. - # - Return value: _nth_ row of the table, if that row exists; - # otherwise +nil+. - # - # Returns the _nth_ row of the table if that row exists: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.by_row! # => #<CSV::Table mode:row row_count:4> - # table[1] # => #<CSV::Row "Name":"bar" "Value":"1"> - # table.by_col_or_row! # => #<CSV::Table mode:col_or_row row_count:4> - # table[1] # => #<CSV::Row "Name":"bar" "Value":"1"> - # - # Counts backward from the last row if +n+ is negative: - # table[-1] # => #<CSV::Row "Name":"baz" "Value":"2"> - # - # Returns +nil+ if +n+ is too large or too small: - # table[4] # => nil - # table[-4] # => nil - # - # Raises an exception if the access mode is <tt>:row</tt> - # and +n+ is not an \Integer: - # table.by_row! # => #<CSV::Table mode:row row_count:4> - # # Raises TypeError (no implicit conversion of String into Integer): - # table['Name'] - # - # --- - # - # Fetch a Column by Its \Integer Index:: - # - Form: <tt>table[n]</tt>, +n+ an \Integer. - # - Access mode: <tt>:col</tt>. - # - Return value: _nth_ column of the table, if that column exists; - # otherwise an \Array of +nil+ fields of length <tt>self.size</tt>. - # - # Returns the _nth_ column of the table if that column exists: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.by_col! # => #<CSV::Table mode:col row_count:4> - # table[1] # => ["0", "1", "2"] - # - # Counts backward from the last column if +n+ is negative: - # table[-2] # => ["foo", "bar", "baz"] - # - # Returns an \Array of +nil+ fields if +n+ is too large or too small: - # table[4] # => [nil, nil, nil] - # table[-4] # => [nil, nil, nil] - # - # --- - # - # Fetch Rows by \Range:: - # - Form: <tt>table[range]</tt>, +range+ a \Range object. - # - Access mode: <tt>:row</tt> or <tt>:col_or_row</tt>. - # - Return value: rows from the table, beginning at row <tt>range.start</tt>, - # if those rows exists. - # - # Returns rows from the table, beginning at row <tt>range.first</tt>, - # if those rows exist: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.by_row! # => #<CSV::Table mode:row row_count:4> - # rows = table[1..2] # => #<CSV::Row "Name":"bar" "Value":"1"> - # rows # => [#<CSV::Row "Name":"bar" "Value":"1">, #<CSV::Row "Name":"baz" "Value":"2">] - # table.by_col_or_row! # => #<CSV::Table mode:col_or_row row_count:4> - # rows = table[1..2] # => #<CSV::Row "Name":"bar" "Value":"1"> - # rows # => [#<CSV::Row "Name":"bar" "Value":"1">, #<CSV::Row "Name":"baz" "Value":"2">] - # - # If there are too few rows, returns all from <tt>range.start</tt> to the end: - # rows = table[1..50] # => #<CSV::Row "Name":"bar" "Value":"1"> - # rows # => [#<CSV::Row "Name":"bar" "Value":"1">, #<CSV::Row "Name":"baz" "Value":"2">] - # - # Special case: if <tt>range.start == table.size</tt>, returns an empty \Array: - # table[table.size..50] # => [] - # - # If <tt>range.end</tt> is negative, calculates the ending index from the end: - # rows = table[0..-1] - # rows # => [#<CSV::Row "Name":"foo" "Value":"0">, #<CSV::Row "Name":"bar" "Value":"1">, #<CSV::Row "Name":"baz" "Value":"2">] - # - # If <tt>range.start</tt> is negative, calculates the starting index from the end: - # rows = table[-1..2] - # rows # => [#<CSV::Row "Name":"baz" "Value":"2">] - # - # If <tt>range.start</tt> is larger than <tt>table.size</tt>, returns +nil+: - # table[4..4] # => nil - # - # --- - # - # Fetch Columns by \Range:: - # - Form: <tt>table[range]</tt>, +range+ a \Range object. - # - Access mode: <tt>:col</tt>. - # - Return value: column data from the table, beginning at column <tt>range.start</tt>, - # if those columns exist. - # - # Returns column values from the table, if the column exists; - # the values are arranged by row: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.by_col! - # table[0..1] # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] - # - # Special case: if <tt>range.start == headers.size</tt>, - # returns an \Array (size: <tt>table.size</tt>) of empty \Arrays: - # table[table.headers.size..50] # => [[], [], []] - # - # If <tt>range.end</tt> is negative, calculates the ending index from the end: - # table[0..-1] # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] - # - # If <tt>range.start</tt> is negative, calculates the starting index from the end: - # table[-2..2] # => [["foo", "0"], ["bar", "1"], ["baz", "2"]] - # - # If <tt>range.start</tt> is larger than <tt>table.size</tt>, - # returns an \Array of +nil+ values: - # table[4..4] # => [nil, nil, nil] - # - # --- - # - # Fetch a Column by Its \String Header:: - # - Form: <tt>table[header]</tt>, +header+ a \String header. - # - Access mode: <tt>:col</tt> or <tt>:col_or_row</tt> - # - Return value: column data from the table, if that +header+ exists. - # - # Returns column values from the table, if the column exists: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.by_col! # => #<CSV::Table mode:col row_count:4> - # table['Name'] # => ["foo", "bar", "baz"] - # table.by_col_or_row! # => #<CSV::Table mode:col_or_row row_count:4> - # col = table['Name'] - # col # => ["foo", "bar", "baz"] - # - # Modifying the returned column values does not modify the table: - # col[0] = 'bat' - # col # => ["bat", "bar", "baz"] - # table['Name'] # => ["foo", "bar", "baz"] - # - # Returns an \Array of +nil+ values if there is no such column: - # table['Nosuch'] # => [nil, nil, nil] - def [](index_or_header) - if @mode == :row or # by index - (@mode == :col_or_row and (index_or_header.is_a?(Integer) or index_or_header.is_a?(Range))) - @table[index_or_header] - else # by header - @table.map { |row| row[index_or_header] } - end - end - - # :call-seq: - # table[n] = row -> row - # table[n] = field_or_array_of_fields -> field_or_array_of_fields - # table[header] = field_or_array_of_fields -> field_or_array_of_fields - # - # Puts data onto the table. - # - # --- - # - # Set a \Row by Its \Integer Index:: - # - Form: <tt>table[n] = row</tt>, +n+ an \Integer, - # +row+ a \CSV::Row instance or an \Array of fields. - # - Access mode: <tt>:row</tt> or <tt>:col_or_row</tt>. - # - Return value: +row+. - # - # If the row exists, it is replaced: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # new_row = CSV::Row.new(['Name', 'Value'], ['bat', 3]) - # table.by_row! # => #<CSV::Table mode:row row_count:4> - # return_value = table[0] = new_row - # return_value.equal?(new_row) # => true # Returned the row - # table[0].to_h # => {"Name"=>"bat", "Value"=>3} - # - # With access mode <tt>:col_or_row</tt>: - # table.by_col_or_row! # => #<CSV::Table mode:col_or_row row_count:4> - # table[0] = CSV::Row.new(['Name', 'Value'], ['bam', 4]) - # table[0].to_h # => {"Name"=>"bam", "Value"=>4} - # - # With an \Array instead of a \CSV::Row, inherits headers from the table: - # array = ['bad', 5] - # return_value = table[0] = array - # return_value.equal?(array) # => true # Returned the array - # table[0].to_h # => {"Name"=>"bad", "Value"=>5} - # - # If the row does not exist, extends the table by adding rows: - # assigns rows with +nil+ as needed: - # table.size # => 3 - # table[5] = ['bag', 6] - # table.size # => 6 - # table[3] # => nil - # table[4]# => nil - # table[5].to_h # => {"Name"=>"bag", "Value"=>6} - # - # Note that the +nil+ rows are actually +nil+, not a row of +nil+ fields. - # - # --- - # - # Set a Column by Its \Integer Index:: - # - Form: <tt>table[n] = array_of_fields</tt>, +n+ an \Integer, - # +array_of_fields+ an \Array of \String fields. - # - Access mode: <tt>:col</tt>. - # - Return value: +array_of_fields+. - # - # If the column exists, it is replaced: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # new_col = [3, 4, 5] - # table.by_col! # => #<CSV::Table mode:col row_count:4> - # return_value = table[1] = new_col - # return_value.equal?(new_col) # => true # Returned the column - # table[1] # => [3, 4, 5] - # # The rows, as revised: - # table.by_row! # => #<CSV::Table mode:row row_count:4> - # table[0].to_h # => {"Name"=>"foo", "Value"=>3} - # table[1].to_h # => {"Name"=>"bar", "Value"=>4} - # table[2].to_h # => {"Name"=>"baz", "Value"=>5} - # table.by_col! # => #<CSV::Table mode:col row_count:4> - # - # If there are too few values, fills with +nil+ values: - # table[1] = [0] - # table[1] # => [0, nil, nil] - # - # If there are too many values, ignores the extra values: - # table[1] = [0, 1, 2, 3, 4] - # table[1] # => [0, 1, 2] - # - # If a single value is given, replaces all fields in the column with that value: - # table[1] = 'bat' - # table[1] # => ["bat", "bat", "bat"] - # - # --- - # - # Set a Column by Its \String Header:: - # - Form: <tt>table[header] = field_or_array_of_fields</tt>, - # +header+ a \String header, +field_or_array_of_fields+ a field value - # or an \Array of \String fields. - # - Access mode: <tt>:col</tt> or <tt>:col_or_row</tt>. - # - Return value: +field_or_array_of_fields+. - # - # If the column exists, it is replaced: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # new_col = [3, 4, 5] - # table.by_col! # => #<CSV::Table mode:col row_count:4> - # return_value = table['Value'] = new_col - # return_value.equal?(new_col) # => true # Returned the column - # table['Value'] # => [3, 4, 5] - # # The rows, as revised: - # table.by_row! # => #<CSV::Table mode:row row_count:4> - # table[0].to_h # => {"Name"=>"foo", "Value"=>3} - # table[1].to_h # => {"Name"=>"bar", "Value"=>4} - # table[2].to_h # => {"Name"=>"baz", "Value"=>5} - # table.by_col! # => #<CSV::Table mode:col row_count:4> - # - # If there are too few values, fills with +nil+ values: - # table['Value'] = [0] - # table['Value'] # => [0, nil, nil] - # - # If there are too many values, ignores the extra values: - # table['Value'] = [0, 1, 2, 3, 4] - # table['Value'] # => [0, 1, 2] - # - # If the column does not exist, extends the table by adding columns: - # table['Note'] = ['x', 'y', 'z'] - # table['Note'] # => ["x", "y", "z"] - # # The rows, as revised: - # table.by_row! - # table[0].to_h # => {"Name"=>"foo", "Value"=>0, "Note"=>"x"} - # table[1].to_h # => {"Name"=>"bar", "Value"=>1, "Note"=>"y"} - # table[2].to_h # => {"Name"=>"baz", "Value"=>2, "Note"=>"z"} - # table.by_col! - # - # If a single value is given, replaces all fields in the column with that value: - # table['Value'] = 'bat' - # table['Value'] # => ["bat", "bat", "bat"] - def []=(index_or_header, value) - if @mode == :row or # by index - (@mode == :col_or_row and index_or_header.is_a? Integer) - if value.is_a? Array - @table[index_or_header] = Row.new(headers, value) - else - @table[index_or_header] = value - end - else # set column - unless index_or_header.is_a? Integer - index = @headers.index(index_or_header) || @headers.size - @headers[index] = index_or_header - end - if value.is_a? Array # multiple values - @table.each_with_index do |row, i| - if row.header_row? - row[index_or_header] = index_or_header - else - row[index_or_header] = value[i] - end - end - else # repeated value - @table.each do |row| - if row.header_row? - row[index_or_header] = index_or_header - else - row[index_or_header] = value - end - end - end - end - end - - # :call-seq: - # table.values_at(*indexes) -> array_of_rows - # table.values_at(*headers) -> array_of_columns_data - # - # If the access mode is <tt>:row</tt> or <tt>:col_or_row</tt>, - # and each argument is either an \Integer or a \Range, - # returns rows. - # Otherwise, returns columns data. - # - # In either case, the returned values are in the order - # specified by the arguments. Arguments may be repeated. - # - # --- - # - # Returns rows as an \Array of \CSV::Row objects. - # - # No argument: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.values_at # => [] - # - # One index: - # values = table.values_at(0) - # values # => [#<CSV::Row "Name":"foo" "Value":"0">] - # - # Two indexes: - # values = table.values_at(2, 0) - # values # => [#<CSV::Row "Name":"baz" "Value":"2">, #<CSV::Row "Name":"foo" "Value":"0">] - # - # One \Range: - # values = table.values_at(1..2) - # values # => [#<CSV::Row "Name":"bar" "Value":"1">, #<CSV::Row "Name":"baz" "Value":"2">] - # - # \Ranges and indexes: - # values = table.values_at(0..1, 1..2, 0, 2) - # pp values - # Output: - # [#<CSV::Row "Name":"foo" "Value":"0">, - # #<CSV::Row "Name":"bar" "Value":"1">, - # #<CSV::Row "Name":"bar" "Value":"1">, - # #<CSV::Row "Name":"baz" "Value":"2">, - # #<CSV::Row "Name":"foo" "Value":"0">, - # #<CSV::Row "Name":"baz" "Value":"2">] - # - # --- - # - # Returns columns data as row Arrays, - # each consisting of the specified columns data for that row: - # values = table.values_at('Name') - # values # => [["foo"], ["bar"], ["baz"]] - # values = table.values_at('Value', 'Name') - # values # => [["0", "foo"], ["1", "bar"], ["2", "baz"]] - def values_at(*indices_or_headers) - if @mode == :row or # by indices - ( @mode == :col_or_row and indices_or_headers.all? do |index| - index.is_a?(Integer) or - ( index.is_a?(Range) and - index.first.is_a?(Integer) and - index.last.is_a?(Integer) ) - end ) - @table.values_at(*indices_or_headers) - else # by headers - @table.map { |row| row.values_at(*indices_or_headers) } - end - end - - # :call-seq: - # table << row_or_array -> self - # - # If +row_or_array+ is a \CSV::Row object, - # it is appended to the table: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table << CSV::Row.new(table.headers, ['bat', 3]) - # table[3] # => #<CSV::Row "Name":"bat" "Value":3> - # - # If +row_or_array+ is an \Array, it is used to create a new - # \CSV::Row object which is then appended to the table: - # table << ['bam', 4] - # table[4] # => #<CSV::Row "Name":"bam" "Value":4> - def <<(row_or_array) - if row_or_array.is_a? Array # append Array - @table << Row.new(headers, row_or_array) - else # append Row - @table << row_or_array - end - - self # for chaining - end - - # - # :call-seq: - # table.push(*rows_or_arrays) -> self - # - # A shortcut for appending multiple rows. Equivalent to: - # rows.each {|row| self << row } - # - # Each argument may be either a \CSV::Row object or an \Array: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # rows = [ - # CSV::Row.new(table.headers, ['bat', 3]), - # ['bam', 4] - # ] - # table.push(*rows) - # table[3..4] # => [#<CSV::Row "Name":"bat" "Value":3>, #<CSV::Row "Name":"bam" "Value":4>] - def push(*rows) - rows.each { |row| self << row } - - self # for chaining - end - - # :call-seq: - # table.delete(*indexes) -> deleted_values - # table.delete(*headers) -> deleted_values - # - # If the access mode is <tt>:row</tt> or <tt>:col_or_row</tt>, - # and each argument is either an \Integer or a \Range, - # returns deleted rows. - # Otherwise, returns deleted columns data. - # - # In either case, the returned values are in the order - # specified by the arguments. Arguments may be repeated. - # - # --- - # - # Returns rows as an \Array of \CSV::Row objects. - # - # One index: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # deleted_values = table.delete(0) - # deleted_values # => [#<CSV::Row "Name":"foo" "Value":"0">] - # - # Two indexes: - # table = CSV.parse(source, headers: true) - # deleted_values = table.delete(2, 0) - # deleted_values # => [#<CSV::Row "Name":"baz" "Value":"2">, #<CSV::Row "Name":"foo" "Value":"0">] - # - # --- - # - # Returns columns data as column Arrays. - # - # One header: - # table = CSV.parse(source, headers: true) - # deleted_values = table.delete('Name') - # deleted_values # => ["foo", "bar", "baz"] - # - # Two headers: - # table = CSV.parse(source, headers: true) - # deleted_values = table.delete('Value', 'Name') - # deleted_values # => [["0", "1", "2"], ["foo", "bar", "baz"]] - def delete(*indexes_or_headers) - if indexes_or_headers.empty? - raise ArgumentError, "wrong number of arguments (given 0, expected 1+)" - end - deleted_values = indexes_or_headers.map do |index_or_header| - if @mode == :row or # by index - (@mode == :col_or_row and index_or_header.is_a? Integer) - @table.delete_at(index_or_header) - else # by header - if index_or_header.is_a? Integer - @headers.delete_at(index_or_header) - else - @headers.delete(index_or_header) - end - @table.map { |row| row.delete(index_or_header).last } - end - end - if indexes_or_headers.size == 1 - deleted_values[0] - else - deleted_values - end - end - - # :call-seq: - # table.delete_if {|row_or_column| ... } -> self - # - # Removes rows or columns for which the block returns a truthy value; - # returns +self+. - # - # Removes rows when the access mode is <tt>:row</tt> or <tt>:col_or_row</tt>; - # calls the block with each \CSV::Row object: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.by_row! # => #<CSV::Table mode:row row_count:4> - # table.size # => 3 - # table.delete_if {|row| row['Name'].start_with?('b') } - # table.size # => 1 - # - # Removes columns when the access mode is <tt>:col</tt>; - # calls the block with each column as a 2-element array - # containing the header and an \Array of column fields: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.by_col! # => #<CSV::Table mode:col row_count:4> - # table.headers.size # => 2 - # table.delete_if {|column_data| column_data[1].include?('2') } - # table.headers.size # => 1 - # - # Returns a new \Enumerator if no block is given: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.delete_if # => #<Enumerator: #<CSV::Table mode:col_or_row row_count:4>:delete_if> - def delete_if(&block) - return enum_for(__method__) { @mode == :row or @mode == :col_or_row ? size : headers.size } unless block_given? - - if @mode == :row or @mode == :col_or_row # by index - @table.delete_if(&block) - else # by header - headers.each do |header| - delete(header) if yield([header, self[header]]) - end - end - - self # for chaining - end - - include Enumerable - - # :call-seq: - # table.each {|row_or_column| ... ) -> self - # - # Calls the block with each row or column; returns +self+. - # - # When the access mode is <tt>:row</tt> or <tt>:col_or_row</tt>, - # calls the block with each \CSV::Row object: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.by_row! # => #<CSV::Table mode:row row_count:4> - # table.each {|row| p row } - # Output: - # #<CSV::Row "Name":"foo" "Value":"0"> - # #<CSV::Row "Name":"bar" "Value":"1"> - # #<CSV::Row "Name":"baz" "Value":"2"> - # - # When the access mode is <tt>:col</tt>, - # calls the block with each column as a 2-element array - # containing the header and an \Array of column fields: - # table.by_col! # => #<CSV::Table mode:col row_count:4> - # table.each {|column_data| p column_data } - # Output: - # ["Name", ["foo", "bar", "baz"]] - # ["Value", ["0", "1", "2"]] - # - # Returns a new \Enumerator if no block is given: - # table.each # => #<Enumerator: #<CSV::Table mode:col row_count:4>:each> - def each(&block) - return enum_for(__method__) { @mode == :col ? headers.size : size } unless block_given? - - if @mode == :col - headers.each.with_index do |header, i| - yield([header, @table.map {|row| row[header, i]}]) - end - else - @table.each(&block) - end - - self # for chaining - end - - # :call-seq: - # table == other_table -> true or false - # - # Returns +true+ if all each row of +self+ <tt>==</tt> - # the corresponding row of +other_table+, otherwise, +false+. - # - # The access mode does no affect the result. - # - # Equal tables: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # other_table = CSV.parse(source, headers: true) - # table == other_table # => true - # - # Different row count: - # other_table.delete(2) - # table == other_table # => false - # - # Different last row: - # other_table << ['bat', 3] - # table == other_table # => false - def ==(other) - return @table == other.table if other.is_a? CSV::Table - @table == other - end - - # :call-seq: - # table.to_a -> array_of_arrays - # - # Returns the table as an \Array of \Arrays; - # the headers are in the first row: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.to_a # => [["Name", "Value"], ["foo", "0"], ["bar", "1"], ["baz", "2"]] - def to_a - array = [headers] - @table.each do |row| - array.push(row.fields) unless row.header_row? - end - - array - end - - # :call-seq: - # table.to_csv(**options) -> csv_string - # - # Returns the table as \CSV string. - # See {Options for Generating}[../CSV.html#class-CSV-label-Options+for+Generating]. - # - # Defaults option +write_headers+ to +true+: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.to_csv # => "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # - # Omits the headers if option +write_headers+ is given as +false+ - # (see {Option +write_headers+}[../CSV.html#class-CSV-label-Option+write_headers]): - # table.to_csv(write_headers: false) # => "foo,0\nbar,1\nbaz,2\n" - # - # Limit rows if option +limit+ is given like +2+: - # table.to_csv(limit: 2) # => "Name,Value\nfoo,0\nbar,1\n" - def to_csv(write_headers: true, limit: nil, **options) - array = write_headers ? [headers.to_csv(**options)] : [] - limit ||= @table.size - limit = @table.size + 1 + limit if limit < 0 - limit = 0 if limit < 0 - @table.first(limit).each do |row| - array.push(row.fields.to_csv(**options)) unless row.header_row? - end - - array.join("") - end - alias_method :to_s, :to_csv - - # - # Extracts the nested value specified by the sequence of +index+ or +header+ objects by calling dig at each step, - # returning nil if any intermediate step is nil. - # - def dig(index_or_header, *index_or_headers) - value = self[index_or_header] - if value.nil? - nil - elsif index_or_headers.empty? - value - else - unless value.respond_to?(:dig) - raise TypeError, "#{value.class} does not have \#dig method" - end - value.dig(*index_or_headers) - end - end - - # :call-seq: - # table.inspect => string - # - # Returns a <tt>US-ASCII</tt>-encoded \String showing table: - # - Class: <tt>CSV::Table</tt>. - # - Access mode: <tt>:row</tt>, <tt>:col</tt>, or <tt>:col_or_row</tt>. - # - Size: Row count, including the header row. - # - # Example: - # source = "Name,Value\nfoo,0\nbar,1\nbaz,2\n" - # table = CSV.parse(source, headers: true) - # table.inspect # => "#<CSV::Table mode:col_or_row row_count:4>\nName,Value\nfoo,0\nbar,1\nbaz,2\n" - # - def inspect - inspected = +"#<#{self.class} mode:#{@mode} row_count:#{to_a.size}>" - summary = to_csv(limit: 5) - inspected << "\n" << summary if summary.encoding.ascii_compatible? - inspected - end - end -end diff --git a/lib/csv/version.rb b/lib/csv/version.rb deleted file mode 100644 index edafc6f128..0000000000 --- a/lib/csv/version.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -class CSV - # The version of the installed library. - VERSION = "3.2.7" -end diff --git a/lib/csv/writer.rb b/lib/csv/writer.rb deleted file mode 100644 index 030a295bc9..0000000000 --- a/lib/csv/writer.rb +++ /dev/null @@ -1,210 +0,0 @@ -# frozen_string_literal: true - -require_relative "input_record_separator" -require_relative "row" - -class CSV - # Note: Don't use this class directly. This is an internal class. - class Writer - # - # A CSV::Writer receives an output, prepares the header, format and output. - # It allows us to write new rows in the object and rewind it. - # - attr_reader :lineno - attr_reader :headers - - def initialize(output, options) - @output = output - @options = options - @lineno = 0 - @fields_converter = nil - prepare - if @options[:write_headers] and @headers - self << @headers - end - @fields_converter = @options[:fields_converter] - end - - # - # Adds a new row - # - def <<(row) - case row - when Row - row = row.fields - when Hash - row = @headers.collect {|header| row[header]} - end - - @headers ||= row if @use_headers - @lineno += 1 - - if @fields_converter - quoted_fields = [false] * row.size - row = @fields_converter.convert(row, nil, lineno, quoted_fields) - end - - i = -1 - converted_row = row.collect do |field| - i += 1 - quote(field, i) - end - line = converted_row.join(@column_separator) + @row_separator - if @output_encoding - line = line.encode(@output_encoding) - end - @output << line - - self - end - - # - # Winds back to the beginning - # - def rewind - @lineno = 0 - @headers = nil if @options[:headers].nil? - end - - private - def prepare - @encoding = @options[:encoding] - - prepare_header - prepare_format - prepare_output - end - - def prepare_header - headers = @options[:headers] - case headers - when Array - @headers = headers - @use_headers = true - when String - @headers = CSV.parse_line(headers, - col_sep: @options[:column_separator], - row_sep: @options[:row_separator], - quote_char: @options[:quote_character]) - @use_headers = true - when true - @headers = nil - @use_headers = true - else - @headers = nil - @use_headers = false - end - return unless @headers - - converter = @options[:header_fields_converter] - @headers = converter.convert(@headers, nil, 0, []) - @headers.each do |header| - header.freeze if header.is_a?(String) - end - end - - def prepare_force_quotes_fields(force_quotes) - @force_quotes_fields = {} - force_quotes.each do |name_or_index| - case name_or_index - when Integer - index = name_or_index - @force_quotes_fields[index] = true - when String, Symbol - name = name_or_index.to_s - if @headers.nil? - message = ":headers is required when you use field name " + - "in :force_quotes: " + - "#{name_or_index.inspect}: #{force_quotes.inspect}" - raise ArgumentError, message - end - index = @headers.index(name) - next if index.nil? - @force_quotes_fields[index] = true - else - message = ":force_quotes element must be " + - "field index or field name: " + - "#{name_or_index.inspect}: #{force_quotes.inspect}" - raise ArgumentError, message - end - end - end - - def prepare_format - @column_separator = @options[:column_separator].to_s.encode(@encoding) - row_separator = @options[:row_separator] - if row_separator == :auto - @row_separator = InputRecordSeparator.value.encode(@encoding) - else - @row_separator = row_separator.to_s.encode(@encoding) - end - @quote_character = @options[:quote_character] - force_quotes = @options[:force_quotes] - if force_quotes.is_a?(Array) - prepare_force_quotes_fields(force_quotes) - @force_quotes = false - elsif force_quotes - @force_quotes_fields = nil - @force_quotes = true - else - @force_quotes_fields = nil - @force_quotes = false - end - unless @force_quotes - @quotable_pattern = - Regexp.new("[\r\n".encode(@encoding) + - Regexp.escape(@column_separator) + - Regexp.escape(@quote_character.encode(@encoding)) + - "]".encode(@encoding)) - end - @quote_empty = @options.fetch(:quote_empty, true) - end - - def prepare_output - @output_encoding = nil - return unless @output.is_a?(StringIO) - - output_encoding = @output.internal_encoding || @output.external_encoding - if @encoding != output_encoding - if @options[:force_encoding] - @output_encoding = output_encoding - else - compatible_encoding = Encoding.compatible?(@encoding, output_encoding) - if compatible_encoding - @output.set_encoding(compatible_encoding) - @output.seek(0, IO::SEEK_END) - end - end - end - end - - def quote_field(field) - field = String(field) - encoded_quote_character = @quote_character.encode(field.encoding) - encoded_quote_character + - field.gsub(encoded_quote_character, - encoded_quote_character * 2) + - encoded_quote_character - end - - def quote(field, i) - if @force_quotes - quote_field(field) - elsif @force_quotes_fields and @force_quotes_fields[i] - quote_field(field) - else - if field.nil? # represent +nil+ fields as empty unquoted fields - "" - else - field = String(field) # Stringify fields - # represent empty fields as empty quoted fields - if (@quote_empty and field.empty?) or (field.valid_encoding? and @quotable_pattern.match?(field)) - quote_field(field) - else - field # unquoted field - end - end - end - end - end -end diff --git a/lib/delegate.rb b/lib/delegate.rb index 387a5f063d..1ea4fb985b 100644 --- a/lib/delegate.rb +++ b/lib/delegate.rb @@ -39,7 +39,7 @@ # Be advised, RDoc will not detect delegated methods. # class Delegator < BasicObject - VERSION = "0.3.0" + VERSION = "0.3.1" kernel = ::Kernel.dup kernel.class_eval do @@ -186,7 +186,7 @@ class Delegator < BasicObject # method calls are being delegated to. # def __getobj__ - __raise__ ::NotImplementedError, "need to define `__getobj__'" + __raise__ ::NotImplementedError, "need to define '__getobj__'" end # @@ -194,7 +194,7 @@ class Delegator < BasicObject # to _obj_. # def __setobj__(obj) - __raise__ ::NotImplementedError, "need to define `__setobj__'" + __raise__ ::NotImplementedError, "need to define '__setobj__'" end # diff --git a/lib/did_you_mean/did_you_mean.gemspec b/lib/did_you_mean/did_you_mean.gemspec index 8fe5723129..be4ac76b4b 100644 --- a/lib/did_you_mean/did_you_mean.gemspec +++ b/lib/did_you_mean/did_you_mean.gemspec @@ -22,6 +22,4 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.required_ruby_version = '>= 2.5.0' - - spec.add_development_dependency "rake" end diff --git a/lib/did_you_mean/jaro_winkler.rb b/lib/did_you_mean/jaro_winkler.rb index 56db130af4..9a3e57f6d7 100644 --- a/lib/did_you_mean/jaro_winkler.rb +++ b/lib/did_you_mean/jaro_winkler.rb @@ -8,8 +8,7 @@ module DidYouMean m = 0.0 t = 0.0 - range = (length2 / 2).floor - 1 - range = 0 if range < 0 + range = length2 > 3 ? length2 / 2 - 1 : 0 flags1 = 0 flags2 = 0 @@ -72,10 +71,8 @@ module DidYouMean codepoints2 = str2.codepoints prefix_bonus = 0 - i = 0 str1.each_codepoint do |char1| - char1 == codepoints2[i] && i < 4 ? prefix_bonus += 1 : break - i += 1 + char1 == codepoints2[prefix_bonus] && prefix_bonus < 4 ? prefix_bonus += 1 : break end jaro_distance + (prefix_bonus * WEIGHT * (1 - jaro_distance)) diff --git a/lib/did_you_mean/spell_checkers/key_error_checker.rb b/lib/did_you_mean/spell_checkers/key_error_checker.rb index be4bea7789..955bff1be6 100644 --- a/lib/did_you_mean/spell_checkers/key_error_checker.rb +++ b/lib/did_you_mean/spell_checkers/key_error_checker.rb @@ -14,7 +14,15 @@ module DidYouMean private def exact_matches - @exact_matches ||= @keys.select { |word| @key == word.to_s }.map(&:inspect) + @exact_matches ||= @keys.select { |word| @key == word.to_s }.map { |obj| format_object(obj) } + end + + def format_object(symbol_or_object) + if symbol_or_object.is_a?(Symbol) + ":#{symbol_or_object}" + else + symbol_or_object.to_s + end end end end diff --git a/lib/did_you_mean/spell_checkers/pattern_key_name_checker.rb b/lib/did_you_mean/spell_checkers/pattern_key_name_checker.rb index ed263c8f93..622d4dee25 100644 --- a/lib/did_you_mean/spell_checkers/pattern_key_name_checker.rb +++ b/lib/did_you_mean/spell_checkers/pattern_key_name_checker.rb @@ -14,7 +14,15 @@ module DidYouMean private def exact_matches - @exact_matches ||= @keys.select { |word| @key == word.to_s }.map(&:inspect) + @exact_matches ||= @keys.select { |word| @key == word.to_s }.map { |obj| format_object(obj) } + end + + def format_object(symbol_or_object) + if symbol_or_object.is_a?(Symbol) + ":#{symbol_or_object}" + else + symbol_or_object.to_s + end end end end diff --git a/lib/drb.rb b/lib/drb.rb deleted file mode 100644 index 2bb4716fa2..0000000000 --- a/lib/drb.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: false -require 'drb/drb' - diff --git a/lib/drb/acl.rb b/lib/drb/acl.rb deleted file mode 100644 index b004656f09..0000000000 --- a/lib/drb/acl.rb +++ /dev/null @@ -1,239 +0,0 @@ -# frozen_string_literal: false -# Copyright (c) 2000,2002,2003 Masatoshi SEKI -# -# acl.rb is copyrighted free software by Masatoshi SEKI. -# You can redistribute it and/or modify it under the same terms as Ruby. - -require 'ipaddr' - -## -# Simple Access Control Lists. -# -# Access control lists are composed of "allow" and "deny" halves to control -# access. Use "all" or "*" to match any address. To match a specific address -# use any address or address mask that IPAddr can understand. -# -# Example: -# -# list = %w[ -# deny all -# allow 192.168.1.1 -# allow ::ffff:192.168.1.2 -# allow 192.168.1.3 -# ] -# -# # From Socket#peeraddr, see also ACL#allow_socket? -# addr = ["AF_INET", 10, "lc630", "192.168.1.3"] -# -# acl = ACL.new -# p acl.allow_addr?(addr) # => true -# -# acl = ACL.new(list, ACL::DENY_ALLOW) -# p acl.allow_addr?(addr) # => true - -class ACL - - ## - # The current version of ACL - - VERSION=["2.0.0"] - - ## - # An entry in an ACL - - class ACLEntry - - ## - # Creates a new entry using +str+. - # - # +str+ may be "*" or "all" to match any address, an IP address string - # to match a specific address, an IP address mask per IPAddr, or one - # containing "*" to match part of an IPv4 address. - # - # IPAddr::InvalidPrefixError may be raised when an IP network - # address with an invalid netmask/prefix is given. - - def initialize(str) - if str == '*' or str == 'all' - @pat = [:all] - elsif str.include?('*') - @pat = [:name, dot_pat(str)] - else - begin - @pat = [:ip, IPAddr.new(str)] - rescue IPAddr::InvalidPrefixError - # In this case, `str` shouldn't be a host name pattern - # because it contains a slash. - raise - rescue ArgumentError - @pat = [:name, dot_pat(str)] - end - end - end - - private - - ## - # Creates a regular expression to match IPv4 addresses - - def dot_pat_str(str) - list = str.split('.').collect { |s| - (s == '*') ? '.+' : s - } - list.join("\\.") - end - - private - - ## - # Creates a Regexp to match an address. - - def dot_pat(str) - /\A#{dot_pat_str(str)}\z/ - end - - public - - ## - # Matches +addr+ against this entry. - - def match(addr) - case @pat[0] - when :all - true - when :ip - begin - ipaddr = IPAddr.new(addr[3]) - ipaddr = ipaddr.ipv4_mapped if @pat[1].ipv6? && ipaddr.ipv4? - rescue ArgumentError - return false - end - (@pat[1].include?(ipaddr)) ? true : false - when :name - (@pat[1] =~ addr[2]) ? true : false - else - false - end - end - end - - ## - # A list of ACLEntry objects. Used to implement the allow and deny halves - # of an ACL - - class ACLList - - ## - # Creates an empty ACLList - - def initialize - @list = [] - end - - public - - ## - # Matches +addr+ against each ACLEntry in this list. - - def match(addr) - @list.each do |e| - return true if e.match(addr) - end - false - end - - public - - ## - # Adds +str+ as an ACLEntry in this list - - def add(str) - @list.push(ACLEntry.new(str)) - end - - end - - ## - # Default to deny - - DENY_ALLOW = 0 - - ## - # Default to allow - - ALLOW_DENY = 1 - - ## - # Creates a new ACL from +list+ with an evaluation +order+ of DENY_ALLOW or - # ALLOW_DENY. - # - # An ACL +list+ is an Array of "allow" or "deny" and an address or address - # mask or "all" or "*" to match any address: - # - # %w[ - # deny all - # allow 192.0.2.2 - # allow 192.0.2.128/26 - # ] - - def initialize(list=nil, order = DENY_ALLOW) - @order = order - @deny = ACLList.new - @allow = ACLList.new - install_list(list) if list - end - - public - - ## - # Allow connections from Socket +soc+? - - def allow_socket?(soc) - allow_addr?(soc.peeraddr) - end - - public - - ## - # Allow connections from addrinfo +addr+? It must be formatted like - # Socket#peeraddr: - # - # ["AF_INET", 10, "lc630", "192.0.2.1"] - - def allow_addr?(addr) - case @order - when DENY_ALLOW - return true if @allow.match(addr) - return false if @deny.match(addr) - return true - when ALLOW_DENY - return false if @deny.match(addr) - return true if @allow.match(addr) - return false - else - false - end - end - - public - - ## - # Adds +list+ of ACL entries to this ACL. - - def install_list(list) - i = 0 - while i < list.size - permission, domain = list.slice(i,2) - case permission.downcase - when 'allow' - @allow.add(domain) - when 'deny' - @deny.add(domain) - else - raise "Invalid ACL entry #{list}" - end - i += 2 - end - end - -end diff --git a/lib/drb/drb.gemspec b/lib/drb/drb.gemspec deleted file mode 100644 index c9d7e40a51..0000000000 --- a/lib/drb/drb.gemspec +++ /dev/null @@ -1,43 +0,0 @@ -begin - require_relative "lib/drb/version" -rescue LoadError # Fallback to load version file in ruby core repository - require_relative "version" -end - -Gem::Specification.new do |spec| - spec.name = "drb" - spec.version = DRb::VERSION - spec.authors = ["Masatoshi SEKI"] - spec.email = ["seki@ruby-lang.org"] - - spec.summary = %q{Distributed object system for Ruby} - spec.description = %q{Distributed object system for Ruby} - spec.homepage = "https://github.com/ruby/drb" - spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0") - spec.licenses = ["Ruby", "BSD-2-Clause"] - - spec.metadata["homepage_uri"] = spec.homepage - spec.metadata["source_code_uri"] = spec.homepage - - spec.files = %w[ - LICENSE.txt - drb.gemspec - lib/drb.rb - lib/drb/acl.rb - lib/drb/drb.rb - lib/drb/eq.rb - lib/drb/extserv.rb - lib/drb/extservm.rb - lib/drb/gw.rb - lib/drb/invokemethod.rb - lib/drb/observer.rb - lib/drb/ssl.rb - lib/drb/timeridconv.rb - lib/drb/unix.rb - lib/drb/version.rb - lib/drb/weakidconv.rb - ] - spec.require_paths = ["lib"] - - spec.add_dependency "ruby2_keywords" -end diff --git a/lib/drb/drb.rb b/lib/drb/drb.rb deleted file mode 100644 index 23181bb834..0000000000 --- a/lib/drb/drb.rb +++ /dev/null @@ -1,1942 +0,0 @@ -# frozen_string_literal: false -# -# = drb/drb.rb -# -# Distributed Ruby: _dRuby_ version 2.0.4 -# -# Copyright (c) 1999-2003 Masatoshi SEKI. You can redistribute it and/or -# modify it under the same terms as Ruby. -# -# Author:: Masatoshi SEKI -# -# Documentation:: William Webber (william@williamwebber.com) -# -# == Overview -# -# dRuby is a distributed object system for Ruby. It allows an object in one -# Ruby process to invoke methods on an object in another Ruby process on the -# same or a different machine. -# -# The Ruby standard library contains the core classes of the dRuby package. -# However, the full package also includes access control lists and the -# Rinda tuple-space distributed task management system, as well as a -# large number of samples. The full dRuby package can be downloaded from -# the dRuby home page (see *References*). -# -# For an introduction and examples of usage see the documentation to the -# DRb module. -# -# == References -# -# [http://www2a.biglobe.ne.jp/~seki/ruby/druby.html] -# The dRuby home page, in Japanese. Contains the full dRuby package -# and links to other Japanese-language sources. -# -# [http://www2a.biglobe.ne.jp/~seki/ruby/druby.en.html] -# The English version of the dRuby home page. -# -# [http://pragprog.com/book/sidruby/the-druby-book] -# The dRuby Book: Distributed and Parallel Computing with Ruby -# by Masatoshi Seki and Makoto Inoue -# -# [http://www.ruby-doc.org/docs/ProgrammingRuby/html/ospace.html] -# The chapter from *Programming* *Ruby* by Dave Thomas and Andy Hunt -# which discusses dRuby. -# -# [http://www.clio.ne.jp/home/web-i31s/Flotuard/Ruby/PRC2K_seki/dRuby.en.html] -# Translation of presentation on Ruby by Masatoshi Seki. - -require 'socket' -require 'io/wait' -require 'monitor' -require_relative 'eq' - -# -# == Overview -# -# dRuby is a distributed object system for Ruby. It is written in -# pure Ruby and uses its own protocol. No add-in services are needed -# beyond those provided by the Ruby runtime, such as TCP sockets. It -# does not rely on or interoperate with other distributed object -# systems such as CORBA, RMI, or .NET. -# -# dRuby allows methods to be called in one Ruby process upon a Ruby -# object located in another Ruby process, even on another machine. -# References to objects can be passed between processes. Method -# arguments and return values are dumped and loaded in marshalled -# format. All of this is done transparently to both the caller of the -# remote method and the object that it is called upon. -# -# An object in a remote process is locally represented by a -# DRb::DRbObject instance. This acts as a sort of proxy for the -# remote object. Methods called upon this DRbObject instance are -# forwarded to its remote object. This is arranged dynamically at run -# time. There are no statically declared interfaces for remote -# objects, such as CORBA's IDL. -# -# dRuby calls made into a process are handled by a DRb::DRbServer -# instance within that process. This reconstitutes the method call, -# invokes it upon the specified local object, and returns the value to -# the remote caller. Any object can receive calls over dRuby. There -# is no need to implement a special interface, or mixin special -# functionality. Nor, in the general case, does an object need to -# explicitly register itself with a DRbServer in order to receive -# dRuby calls. -# -# One process wishing to make dRuby calls upon another process must -# somehow obtain an initial reference to an object in the remote -# process by some means other than as the return value of a remote -# method call, as there is initially no remote object reference it can -# invoke a method upon. This is done by attaching to the server by -# URI. Each DRbServer binds itself to a URI such as -# 'druby://example.com:8787'. A DRbServer can have an object attached -# to it that acts as the server's *front* *object*. A DRbObject can -# be explicitly created from the server's URI. This DRbObject's -# remote object will be the server's front object. This front object -# can then return references to other Ruby objects in the DRbServer's -# process. -# -# Method calls made over dRuby behave largely the same as normal Ruby -# method calls made within a process. Method calls with blocks are -# supported, as are raising exceptions. In addition to a method's -# standard errors, a dRuby call may also raise one of the -# dRuby-specific errors, all of which are subclasses of DRb::DRbError. -# -# Any type of object can be passed as an argument to a dRuby call or -# returned as its return value. By default, such objects are dumped -# or marshalled at the local end, then loaded or unmarshalled at the -# remote end. The remote end therefore receives a copy of the local -# object, not a distributed reference to it; methods invoked upon this -# copy are executed entirely in the remote process, not passed on to -# the local original. This has semantics similar to pass-by-value. -# -# However, if an object cannot be marshalled, a dRuby reference to it -# is passed or returned instead. This will turn up at the remote end -# as a DRbObject instance. All methods invoked upon this remote proxy -# are forwarded to the local object, as described in the discussion of -# DRbObjects. This has semantics similar to the normal Ruby -# pass-by-reference. -# -# The easiest way to signal that we want an otherwise marshallable -# object to be passed or returned as a DRbObject reference, rather -# than marshalled and sent as a copy, is to include the -# DRb::DRbUndumped mixin module. -# -# dRuby supports calling remote methods with blocks. As blocks (or -# rather the Proc objects that represent them) are not marshallable, -# the block executes in the local, not the remote, context. Each -# value yielded to the block is passed from the remote object to the -# local block, then the value returned by each block invocation is -# passed back to the remote execution context to be collected, before -# the collected values are finally returned to the local context as -# the return value of the method invocation. -# -# == Examples of usage -# -# For more dRuby samples, see the +samples+ directory in the full -# dRuby distribution. -# -# === dRuby in client/server mode -# -# This illustrates setting up a simple client-server drb -# system. Run the server and client code in different terminals, -# starting the server code first. -# -# ==== Server code -# -# require 'drb/drb' -# -# # The URI for the server to connect to -# URI="druby://localhost:8787" -# -# class TimeServer -# -# def get_current_time -# return Time.now -# end -# -# end -# -# # The object that handles requests on the server -# FRONT_OBJECT=TimeServer.new -# -# DRb.start_service(URI, FRONT_OBJECT) -# # Wait for the drb server thread to finish before exiting. -# DRb.thread.join -# -# ==== Client code -# -# require 'drb/drb' -# -# # The URI to connect to -# SERVER_URI="druby://localhost:8787" -# -# # Start a local DRbServer to handle callbacks. -# # -# # Not necessary for this small example, but will be required -# # as soon as we pass a non-marshallable object as an argument -# # to a dRuby call. -# # -# # Note: this must be called at least once per process to take any effect. -# # This is particularly important if your application forks. -# DRb.start_service -# -# timeserver = DRbObject.new_with_uri(SERVER_URI) -# puts timeserver.get_current_time -# -# === Remote objects under dRuby -# -# This example illustrates returning a reference to an object -# from a dRuby call. The Logger instances live in the server -# process. References to them are returned to the client process, -# where methods can be invoked upon them. These methods are -# executed in the server process. -# -# ==== Server code -# -# require 'drb/drb' -# -# URI="druby://localhost:8787" -# -# class Logger -# -# # Make dRuby send Logger instances as dRuby references, -# # not copies. -# include DRb::DRbUndumped -# -# def initialize(n, fname) -# @name = n -# @filename = fname -# end -# -# def log(message) -# File.open(@filename, "a") do |f| -# f.puts("#{Time.now}: #{@name}: #{message}") -# end -# end -# -# end -# -# # We have a central object for creating and retrieving loggers. -# # This retains a local reference to all loggers created. This -# # is so an existing logger can be looked up by name, but also -# # to prevent loggers from being garbage collected. A dRuby -# # reference to an object is not sufficient to prevent it being -# # garbage collected! -# class LoggerFactory -# -# def initialize(bdir) -# @basedir = bdir -# @loggers = {} -# end -# -# def get_logger(name) -# if !@loggers.has_key? name -# # make the filename safe, then declare it to be so -# fname = name.gsub(/[.\/\\\:]/, "_") -# @loggers[name] = Logger.new(name, @basedir + "/" + fname) -# end -# return @loggers[name] -# end -# -# end -# -# FRONT_OBJECT=LoggerFactory.new("/tmp/dlog") -# -# DRb.start_service(URI, FRONT_OBJECT) -# DRb.thread.join -# -# ==== Client code -# -# require 'drb/drb' -# -# SERVER_URI="druby://localhost:8787" -# -# DRb.start_service -# -# log_service=DRbObject.new_with_uri(SERVER_URI) -# -# ["loga", "logb", "logc"].each do |logname| -# -# logger=log_service.get_logger(logname) -# -# logger.log("Hello, world!") -# logger.log("Goodbye, world!") -# logger.log("=== EOT ===") -# -# end -# -# == Security -# -# As with all network services, security needs to be considered when -# using dRuby. By allowing external access to a Ruby object, you are -# not only allowing outside clients to call the methods you have -# defined for that object, but by default to execute arbitrary Ruby -# code on your server. Consider the following: -# -# # !!! UNSAFE CODE !!! -# ro = DRbObject::new_with_uri("druby://your.server.com:8989") -# class << ro -# undef :instance_eval # force call to be passed to remote object -# end -# ro.instance_eval("`rm -rf *`") -# -# The dangers posed by instance_eval and friends are such that a -# DRbServer should only be used when clients are trusted. -# -# A DRbServer can be configured with an access control list to -# selectively allow or deny access from specified IP addresses. The -# main druby distribution provides the ACL class for this purpose. In -# general, this mechanism should only be used alongside, rather than -# as a replacement for, a good firewall. -# -# == dRuby internals -# -# dRuby is implemented using three main components: a remote method -# call marshaller/unmarshaller; a transport protocol; and an -# ID-to-object mapper. The latter two can be directly, and the first -# indirectly, replaced, in order to provide different behaviour and -# capabilities. -# -# Marshalling and unmarshalling of remote method calls is performed by -# a DRb::DRbMessage instance. This uses the Marshal module to dump -# the method call before sending it over the transport layer, then -# reconstitute it at the other end. There is normally no need to -# replace this component, and no direct way is provided to do so. -# However, it is possible to implement an alternative marshalling -# scheme as part of an implementation of the transport layer. -# -# The transport layer is responsible for opening client and server -# network connections and forwarding dRuby request across them. -# Normally, it uses DRb::DRbMessage internally to manage marshalling -# and unmarshalling. The transport layer is managed by -# DRb::DRbProtocol. Multiple protocols can be installed in -# DRbProtocol at the one time; selection between them is determined by -# the scheme of a dRuby URI. The default transport protocol is -# selected by the scheme 'druby:', and implemented by -# DRb::DRbTCPSocket. This uses plain TCP/IP sockets for -# communication. An alternative protocol, using UNIX domain sockets, -# is implemented by DRb::DRbUNIXSocket in the file drb/unix.rb, and -# selected by the scheme 'drbunix:'. A sample implementation over -# HTTP can be found in the samples accompanying the main dRuby -# distribution. -# -# The ID-to-object mapping component maps dRuby object ids to the -# objects they refer to, and vice versa. The implementation to use -# can be specified as part of a DRb::DRbServer's configuration. The -# default implementation is provided by DRb::DRbIdConv. It uses an -# object's ObjectSpace id as its dRuby id. This means that the dRuby -# reference to that object only remains meaningful for the lifetime of -# the object's process and the lifetime of the object within that -# process. A modified implementation is provided by DRb::TimerIdConv -# in the file drb/timeridconv.rb. This implementation retains a local -# reference to all objects exported over dRuby for a configurable -# period of time (defaulting to ten minutes), to prevent them being -# garbage-collected within this time. Another sample implementation -# is provided in sample/name.rb in the main dRuby distribution. This -# allows objects to specify their own id or "name". A dRuby reference -# can be made persistent across processes by having each process -# register an object using the same dRuby name. -# -module DRb - - # Superclass of all errors raised in the DRb module. - class DRbError < RuntimeError; end - - # Error raised when an error occurs on the underlying communication - # protocol. - class DRbConnError < DRbError; end - - # Class responsible for converting between an object and its id. - # - # This, the default implementation, uses an object's local ObjectSpace - # __id__ as its id. This means that an object's identification over - # drb remains valid only while that object instance remains alive - # within the server runtime. - # - # For alternative mechanisms, see DRb::TimerIdConv in drb/timeridconv.rb - # and DRbNameIdConv in sample/name.rb in the full drb distribution. - class DRbIdConv - - # Convert an object reference id to an object. - # - # This implementation looks up the reference id in the local object - # space and returns the object it refers to. - def to_obj(ref) - ObjectSpace._id2ref(ref) - end - - # Convert an object into a reference id. - # - # This implementation returns the object's __id__ in the local - # object space. - def to_id(obj) - case obj - when Object - obj.nil? ? nil : obj.__id__ - when BasicObject - obj.__id__ - end - end - end - - # Mixin module making an object undumpable or unmarshallable. - # - # If an object which includes this module is returned by method - # called over drb, then the object remains in the server space - # and a reference to the object is returned, rather than the - # object being marshalled and moved into the client space. - module DRbUndumped - def _dump(dummy) # :nodoc: - raise TypeError, 'can\'t dump' - end - end - - # Error raised by the DRb module when an attempt is made to refer to - # the context's current drb server but the context does not have one. - # See #current_server. - class DRbServerNotFound < DRbError; end - - # Error raised by the DRbProtocol module when it cannot find any - # protocol implementation support the scheme specified in a URI. - class DRbBadURI < DRbError; end - - # Error raised by a dRuby protocol when it doesn't support the - # scheme specified in a URI. See DRb::DRbProtocol. - class DRbBadScheme < DRbError; end - - # An exception wrapping a DRb::DRbUnknown object - class DRbUnknownError < DRbError - - # Create a new DRbUnknownError for the DRb::DRbUnknown object +unknown+ - def initialize(unknown) - @unknown = unknown - super(unknown.name) - end - - # Get the wrapped DRb::DRbUnknown object. - attr_reader :unknown - - def self._load(s) # :nodoc: - Marshal::load(s) - end - - def _dump(lv) # :nodoc: - Marshal::dump(@unknown) - end - end - - # An exception wrapping an error object - class DRbRemoteError < DRbError - - # Creates a new remote error that wraps the Exception +error+ - def initialize(error) - @reason = error.class.to_s - super("#{error.message} (#{error.class})") - set_backtrace(error.backtrace) - end - - # the class of the error, as a string. - attr_reader :reason - end - - # Class wrapping a marshalled object whose type is unknown locally. - # - # If an object is returned by a method invoked over drb, but the - # class of the object is unknown in the client namespace, or - # the object is a constant unknown in the client namespace, then - # the still-marshalled object is returned wrapped in a DRbUnknown instance. - # - # If this object is passed as an argument to a method invoked over - # drb, then the wrapped object is passed instead. - # - # The class or constant name of the object can be read from the - # +name+ attribute. The marshalled object is held in the +buf+ - # attribute. - class DRbUnknown - - # Create a new DRbUnknown object. - # - # +buf+ is a string containing a marshalled object that could not - # be unmarshalled. +err+ is the error message that was raised - # when the unmarshalling failed. It is used to determine the - # name of the unmarshalled object. - def initialize(err, buf) - case err.to_s - when /uninitialized constant (\S+)/ - @name = $1 - when /undefined class\/module (\S+)/ - @name = $1 - else - @name = nil - end - @buf = buf - end - - # The name of the unknown thing. - # - # Class name for unknown objects; variable name for unknown - # constants. - attr_reader :name - - # Buffer contained the marshalled, unknown object. - attr_reader :buf - - def self._load(s) # :nodoc: - begin - Marshal::load(s) - rescue NameError, ArgumentError - DRbUnknown.new($!, s) - end - end - - def _dump(lv) # :nodoc: - @buf - end - - # Attempt to load the wrapped marshalled object again. - # - # If the class of the object is now known locally, the object - # will be unmarshalled and returned. Otherwise, a new - # but identical DRbUnknown object will be returned. - def reload - self.class._load(@buf) - end - - # Create a DRbUnknownError exception containing this object. - def exception - DRbUnknownError.new(self) - end - end - - # An Array wrapper that can be sent to another server via DRb. - # - # All entries in the array will be dumped or be references that point to - # the local server. - - class DRbArray - - # Creates a new DRbArray that either dumps or wraps all the items in the - # Array +ary+ so they can be loaded by a remote DRb server. - - def initialize(ary) - @ary = ary.collect { |obj| - if obj.kind_of? DRbUndumped - DRbObject.new(obj) - else - begin - Marshal.dump(obj) - obj - rescue - DRbObject.new(obj) - end - end - } - end - - def self._load(s) # :nodoc: - Marshal::load(s) - end - - def _dump(lv) # :nodoc: - Marshal.dump(@ary) - end - end - - # Handler for sending and receiving drb messages. - # - # This takes care of the low-level marshalling and unmarshalling - # of drb requests and responses sent over the wire between server - # and client. This relieves the implementor of a new drb - # protocol layer with having to deal with these details. - # - # The user does not have to directly deal with this object in - # normal use. - class DRbMessage - def initialize(config) # :nodoc: - @load_limit = config[:load_limit] - @argc_limit = config[:argc_limit] - end - - def dump(obj, error=false) # :nodoc: - case obj - when DRbUndumped - obj = make_proxy(obj, error) - when Object - # nothing - else - obj = make_proxy(obj, error) - end - begin - str = Marshal::dump(obj) - rescue - str = Marshal::dump(make_proxy(obj, error)) - end - [str.size].pack('N') + str - end - - def load(soc) # :nodoc: - begin - sz = soc.read(4) # sizeof (N) - rescue - raise(DRbConnError, $!.message, $!.backtrace) - end - raise(DRbConnError, 'connection closed') if sz.nil? - raise(DRbConnError, 'premature header') if sz.size < 4 - sz = sz.unpack('N')[0] - raise(DRbConnError, "too large packet #{sz}") if @load_limit < sz - begin - str = soc.read(sz) - rescue - raise(DRbConnError, $!.message, $!.backtrace) - end - raise(DRbConnError, 'connection closed') if str.nil? - raise(DRbConnError, 'premature marshal format(can\'t read)') if str.size < sz - DRb.mutex.synchronize do - begin - Marshal::load(str) - rescue NameError, ArgumentError - DRbUnknown.new($!, str) - end - end - end - - def send_request(stream, ref, msg_id, arg, b) # :nodoc: - ary = [] - ary.push(dump(ref.__drbref)) - ary.push(dump(msg_id.id2name)) - ary.push(dump(arg.length)) - arg.each do |e| - ary.push(dump(e)) - end - ary.push(dump(b)) - stream.write(ary.join('')) - rescue - raise(DRbConnError, $!.message, $!.backtrace) - end - - def recv_request(stream) # :nodoc: - ref = load(stream) - ro = DRb.to_obj(ref) - msg = load(stream) - argc = load(stream) - raise(DRbConnError, "too many arguments") if @argc_limit < argc - argv = Array.new(argc, nil) - argc.times do |n| - argv[n] = load(stream) - end - block = load(stream) - return ro, msg, argv, block - end - - def send_reply(stream, succ, result) # :nodoc: - stream.write(dump(succ) + dump(result, !succ)) - rescue - raise(DRbConnError, $!.message, $!.backtrace) - end - - def recv_reply(stream) # :nodoc: - succ = load(stream) - result = load(stream) - [succ, result] - end - - private - def make_proxy(obj, error=false) # :nodoc: - if error - DRbRemoteError.new(obj) - else - DRbObject.new(obj) - end - end - end - - # Module managing the underlying network protocol(s) used by drb. - # - # By default, drb uses the DRbTCPSocket protocol. Other protocols - # can be defined. A protocol must define the following class methods: - # - # [open(uri, config)] Open a client connection to the server at +uri+, - # using configuration +config+. Return a protocol - # instance for this connection. - # [open_server(uri, config)] Open a server listening at +uri+, - # using configuration +config+. Return a - # protocol instance for this listener. - # [uri_option(uri, config)] Take a URI, possibly containing an option - # component (e.g. a trailing '?param=val'), - # and return a [uri, option] tuple. - # - # All of these methods should raise a DRbBadScheme error if the URI - # does not identify the protocol they support (e.g. "druby:" for - # the standard Ruby protocol). This is how the DRbProtocol module, - # given a URI, determines which protocol implementation serves that - # protocol. - # - # The protocol instance returned by #open_server must have the - # following methods: - # - # [accept] Accept a new connection to the server. Returns a protocol - # instance capable of communicating with the client. - # [close] Close the server connection. - # [uri] Get the URI for this server. - # - # The protocol instance returned by #open must have the following methods: - # - # [send_request (ref, msg_id, arg, b)] - # Send a request to +ref+ with the given message id and arguments. - # This is most easily implemented by calling DRbMessage.send_request, - # providing a stream that sits on top of the current protocol. - # [recv_reply] - # Receive a reply from the server and return it as a [success-boolean, - # reply-value] pair. This is most easily implemented by calling - # DRb.recv_reply, providing a stream that sits on top of the - # current protocol. - # [alive?] - # Is this connection still alive? - # [close] - # Close this connection. - # - # The protocol instance returned by #open_server().accept() must have - # the following methods: - # - # [recv_request] - # Receive a request from the client and return a [object, message, - # args, block] tuple. This is most easily implemented by calling - # DRbMessage.recv_request, providing a stream that sits on top of - # the current protocol. - # [send_reply(succ, result)] - # Send a reply to the client. This is most easily implemented - # by calling DRbMessage.send_reply, providing a stream that sits - # on top of the current protocol. - # [close] - # Close this connection. - # - # A new protocol is registered with the DRbProtocol module using - # the add_protocol method. - # - # For examples of other protocols, see DRbUNIXSocket in drb/unix.rb, - # and HTTP0 in sample/http0.rb and sample/http0serv.rb in the full - # drb distribution. - module DRbProtocol - - # Add a new protocol to the DRbProtocol module. - def add_protocol(prot) - @protocol.push(prot) - end - module_function :add_protocol - - # Open a client connection to +uri+ with the configuration +config+. - # - # The DRbProtocol module asks each registered protocol in turn to - # try to open the URI. Each protocol signals that it does not handle that - # URI by raising a DRbBadScheme error. If no protocol recognises the - # URI, then a DRbBadURI error is raised. If a protocol accepts the - # URI, but an error occurs in opening it, a DRbConnError is raised. - def open(uri, config, first=true) - @protocol.each do |prot| - begin - return prot.open(uri, config) - rescue DRbBadScheme - rescue DRbConnError - raise($!) - rescue - raise(DRbConnError, "#{uri} - #{$!.inspect}") - end - end - if first && (config[:auto_load] != false) - auto_load(uri) - return open(uri, config, false) - end - raise DRbBadURI, 'can\'t parse uri:' + uri - end - module_function :open - - # Open a server listening for connections at +uri+ with - # configuration +config+. - # - # The DRbProtocol module asks each registered protocol in turn to - # try to open a server at the URI. Each protocol signals that it does - # not handle that URI by raising a DRbBadScheme error. If no protocol - # recognises the URI, then a DRbBadURI error is raised. If a protocol - # accepts the URI, but an error occurs in opening it, the underlying - # error is passed on to the caller. - def open_server(uri, config, first=true) - @protocol.each do |prot| - begin - return prot.open_server(uri, config) - rescue DRbBadScheme - end - end - if first && (config[:auto_load] != false) - auto_load(uri) - return open_server(uri, config, false) - end - raise DRbBadURI, 'can\'t parse uri:' + uri - end - module_function :open_server - - # Parse +uri+ into a [uri, option] pair. - # - # The DRbProtocol module asks each registered protocol in turn to - # try to parse the URI. Each protocol signals that it does not handle that - # URI by raising a DRbBadScheme error. If no protocol recognises the - # URI, then a DRbBadURI error is raised. - def uri_option(uri, config, first=true) - @protocol.each do |prot| - begin - uri, opt = prot.uri_option(uri, config) - # opt = nil if opt == '' - return uri, opt - rescue DRbBadScheme - end - end - if first && (config[:auto_load] != false) - auto_load(uri) - return uri_option(uri, config, false) - end - raise DRbBadURI, 'can\'t parse uri:' + uri - end - module_function :uri_option - - def auto_load(uri) # :nodoc: - if /\Adrb([a-z0-9]+):/ =~ uri - require("drb/#{$1}") rescue nil - end - end - module_function :auto_load - end - - # The default drb protocol which communicates over a TCP socket. - # - # The DRb TCP protocol URI looks like: - # <code>druby://<host>:<port>?<option></code>. The option is optional. - - class DRbTCPSocket - # :stopdoc: - private - def self.parse_uri(uri) - if /\Adruby:\/\/(.*?):(\d+)(\?(.*))?\z/ =~ uri - host = $1 - port = $2.to_i - option = $4 - [host, port, option] - else - raise(DRbBadScheme, uri) unless uri.start_with?('druby:') - raise(DRbBadURI, 'can\'t parse uri:' + uri) - end - end - - public - - # Open a client connection to +uri+ (DRb URI string) using configuration - # +config+. - # - # This can raise DRb::DRbBadScheme or DRb::DRbBadURI if +uri+ is not for a - # recognized protocol. See DRb::DRbServer.new for information on built-in - # URI protocols. - def self.open(uri, config) - host, port, = parse_uri(uri) - soc = TCPSocket.open(host, port) - self.new(uri, soc, config) - end - - # Returns the hostname of this server - def self.getservername - host = Socket::gethostname - begin - Socket::getaddrinfo(host, nil, - Socket::AF_UNSPEC, - Socket::SOCK_STREAM, - 0, - Socket::AI_PASSIVE)[0][3] - rescue - 'localhost' - end - end - - # For the families available for +host+, returns a TCPServer on +port+. - # If +port+ is 0 the first available port is used. IPv4 servers are - # preferred over IPv6 servers. - def self.open_server_inaddr_any(host, port) - infos = Socket::getaddrinfo(host, nil, - Socket::AF_UNSPEC, - Socket::SOCK_STREAM, - 0, - Socket::AI_PASSIVE) - families = Hash[*infos.collect { |af, *_| af }.uniq.zip([]).flatten] - return TCPServer.open('0.0.0.0', port) if families.has_key?('AF_INET') - return TCPServer.open('::', port) if families.has_key?('AF_INET6') - return TCPServer.open(port) - # :stopdoc: - end - - # Open a server listening for connections at +uri+ using - # configuration +config+. - def self.open_server(uri, config) - uri = 'druby://:0' unless uri - host, port, _ = parse_uri(uri) - config = {:tcp_original_host => host}.update(config) - if host.size == 0 - host = getservername - soc = open_server_inaddr_any(host, port) - else - soc = TCPServer.open(host, port) - end - port = soc.addr[1] if port == 0 - config[:tcp_port] = port - uri = "druby://#{host}:#{port}" - self.new(uri, soc, config) - end - - # Parse +uri+ into a [uri, option] pair. - def self.uri_option(uri, config) - host, port, option = parse_uri(uri) - return "druby://#{host}:#{port}", option - end - - # Create a new DRbTCPSocket instance. - # - # +uri+ is the URI we are connected to. - # +soc+ is the tcp socket we are bound to. +config+ is our - # configuration. - def initialize(uri, soc, config={}) - @uri = uri - @socket = soc - @config = config - @acl = config[:tcp_acl] - @msg = DRbMessage.new(config) - set_sockopt(@socket) - @shutdown_pipe_r, @shutdown_pipe_w = IO.pipe - end - - # Get the URI that we are connected to. - attr_reader :uri - - # Get the address of our TCP peer (the other end of the socket - # we are bound to. - def peeraddr - @socket.peeraddr - end - - # Get the socket. - def stream; @socket; end - - # On the client side, send a request to the server. - def send_request(ref, msg_id, arg, b) - @msg.send_request(stream, ref, msg_id, arg, b) - end - - # On the server side, receive a request from the client. - def recv_request - @msg.recv_request(stream) - end - - # On the server side, send a reply to the client. - def send_reply(succ, result) - @msg.send_reply(stream, succ, result) - end - - # On the client side, receive a reply from the server. - def recv_reply - @msg.recv_reply(stream) - end - - public - - # Close the connection. - # - # If this is an instance returned by #open_server, then this stops - # listening for new connections altogether. If this is an instance - # returned by #open or by #accept, then it closes this particular - # client-server session. - def close - shutdown - if @socket - @socket.close - @socket = nil - end - close_shutdown_pipe - end - - def close_shutdown_pipe - @shutdown_pipe_w.close - @shutdown_pipe_r.close - end - private :close_shutdown_pipe - - # On the server side, for an instance returned by #open_server, - # accept a client connection and return a new instance to handle - # the server's side of this client-server session. - def accept - while true - s = accept_or_shutdown - return nil unless s - break if (@acl ? @acl.allow_socket?(s) : true) - s.close - end - if @config[:tcp_original_host].to_s.size == 0 - uri = "druby://#{s.addr[3]}:#{@config[:tcp_port]}" - else - uri = @uri - end - self.class.new(uri, s, @config) - end - - def accept_or_shutdown - readables, = IO.select([@socket, @shutdown_pipe_r]) - if readables.include? @shutdown_pipe_r - return nil - end - @socket.accept - end - private :accept_or_shutdown - - # Graceful shutdown - def shutdown - @shutdown_pipe_w.close - end - - # Check to see if this connection is alive. - def alive? - return false unless @socket - if @socket.to_io.wait_readable(0) - close - return false - end - true - end - - def set_sockopt(soc) # :nodoc: - soc.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) - rescue IOError, Errno::ECONNRESET, Errno::EINVAL - # closed/shutdown socket, ignore error - end - end - - module DRbProtocol - @protocol = [DRbTCPSocket] # default - end - - class DRbURIOption # :nodoc: I don't understand the purpose of this class... - def initialize(option) - @option = option.to_s - end - attr_reader :option - def to_s; @option; end - - def ==(other) - return false unless DRbURIOption === other - @option == other.option - end - - def hash - @option.hash - end - - alias eql? == - end - - # Object wrapping a reference to a remote drb object. - # - # Method calls on this object are relayed to the remote - # object that this object is a stub for. - class DRbObject - - # Unmarshall a marshalled DRbObject. - # - # If the referenced object is located within the local server, then - # the object itself is returned. Otherwise, a new DRbObject is - # created to act as a stub for the remote referenced object. - def self._load(s) - uri, ref = Marshal.load(s) - - if DRb.here?(uri) - obj = DRb.to_obj(ref) - return obj - end - - self.new_with(uri, ref) - end - - # Creates a DRb::DRbObject given the reference information to the remote - # host +uri+ and object +ref+. - - def self.new_with(uri, ref) - it = self.allocate - it.instance_variable_set(:@uri, uri) - it.instance_variable_set(:@ref, ref) - it - end - - # Create a new DRbObject from a URI alone. - def self.new_with_uri(uri) - self.new(nil, uri) - end - - # Marshall this object. - # - # The URI and ref of the object are marshalled. - def _dump(lv) - Marshal.dump([@uri, @ref]) - end - - # Create a new remote object stub. - # - # +obj+ is the (local) object we want to create a stub for. Normally - # this is +nil+. +uri+ is the URI of the remote object that this - # will be a stub for. - def initialize(obj, uri=nil) - @uri = nil - @ref = nil - case obj - when Object - is_nil = obj.nil? - when BasicObject - is_nil = false - end - - if is_nil - return if uri.nil? - @uri, option = DRbProtocol.uri_option(uri, DRb.config) - @ref = DRbURIOption.new(option) unless option.nil? - else - @uri = uri ? uri : (DRb.uri rescue nil) - @ref = obj ? DRb.to_id(obj) : nil - end - end - - # Get the URI of the remote object. - def __drburi - @uri - end - - # Get the reference of the object, if local. - def __drbref - @ref - end - - undef :to_s - undef :to_a if respond_to?(:to_a) - - # Routes respond_to? to the referenced remote object. - def respond_to?(msg_id, priv=false) - case msg_id - when :_dump - true - when :marshal_dump - false - else - method_missing(:respond_to?, msg_id, priv) - end - end - - # Routes method calls to the referenced remote object. - ruby2_keywords def method_missing(msg_id, *a, &b) - if DRb.here?(@uri) - obj = DRb.to_obj(@ref) - DRb.current_server.check_insecure_method(obj, msg_id) - return obj.__send__(msg_id, *a, &b) - end - - succ, result = self.class.with_friend(@uri) do - DRbConn.open(@uri) do |conn| - conn.send_message(self, msg_id, a, b) - end - end - - if succ - return result - elsif DRbUnknown === result - raise result - else - bt = self.class.prepare_backtrace(@uri, result) - result.set_backtrace(bt + caller) - raise result - end - end - - # Given the +uri+ of another host executes the block provided. - def self.with_friend(uri) # :nodoc: - friend = DRb.fetch_server(uri) - return yield() unless friend - - save = Thread.current['DRb'] - Thread.current['DRb'] = { 'server' => friend } - return yield - ensure - Thread.current['DRb'] = save if friend - end - - # Returns a modified backtrace from +result+ with the +uri+ where each call - # in the backtrace came from. - def self.prepare_backtrace(uri, result) # :nodoc: - prefix = "(#{uri}) " - bt = [] - result.backtrace.each do |x| - break if /`__send__'$/ =~ x - if /\A\(druby:\/\// =~ x - bt.push(x) - else - bt.push(prefix + x) - end - end - bt - end - - def pretty_print(q) # :nodoc: - q.pp_object(self) - end - - def pretty_print_cycle(q) # :nodoc: - q.object_address_group(self) { - q.breakable - q.text '...' - } - end - end - - class ThreadObject - include MonitorMixin - - def initialize(&blk) - super() - @wait_ev = new_cond - @req_ev = new_cond - @res_ev = new_cond - @status = :wait - @req = nil - @res = nil - @thread = Thread.new(self, &blk) - end - - def alive? - @thread.alive? - end - - def kill - @thread.kill - @thread.join - end - - def method_missing(msg, *arg, &blk) - synchronize do - @wait_ev.wait_until { @status == :wait } - @req = [msg] + arg - @status = :req - @req_ev.broadcast - @res_ev.wait_until { @status == :res } - value = @res - @req = @res = nil - @status = :wait - @wait_ev.broadcast - return value - end - end - - def _execute() - synchronize do - @req_ev.wait_until { @status == :req } - @res = yield(@req) - @status = :res - @res_ev.signal - end - end - end - - # Class handling the connection between a DRbObject and the - # server the real object lives on. - # - # This class maintains a pool of connections, to reduce the - # overhead of starting and closing down connections for each - # method call. - # - # This class is used internally by DRbObject. The user does - # not normally need to deal with it directly. - class DRbConn - POOL_SIZE = 16 # :nodoc: - - def self.make_pool - ThreadObject.new do |queue| - pool = [] - while true - queue._execute do |message| - case(message[0]) - when :take then - remote_uri = message[1] - conn = nil - new_pool = [] - pool.each do |c| - if conn.nil? and c.uri == remote_uri - conn = c if c.alive? - else - new_pool.push c - end - end - pool = new_pool - conn - when :store then - conn = message[1] - pool.unshift(conn) - pool.pop.close while pool.size > POOL_SIZE - conn - else - nil - end - end - end - end - end - @pool_proxy = nil - - def self.stop_pool - @pool_proxy&.kill - @pool_proxy = nil - end - - def self.open(remote_uri) # :nodoc: - begin - @pool_proxy = make_pool unless @pool_proxy&.alive? - - conn = @pool_proxy.take(remote_uri) - conn = self.new(remote_uri) unless conn - succ, result = yield(conn) - return succ, result - - ensure - if conn - if succ - @pool_proxy.store(conn) - else - conn.close - end - end - end - end - - def initialize(remote_uri) # :nodoc: - @uri = remote_uri - @protocol = DRbProtocol.open(remote_uri, DRb.config) - end - attr_reader :uri # :nodoc: - - def send_message(ref, msg_id, arg, block) # :nodoc: - @protocol.send_request(ref, msg_id, arg, block) - @protocol.recv_reply - end - - def close # :nodoc: - @protocol.close - @protocol = nil - end - - def alive? # :nodoc: - return false unless @protocol - @protocol.alive? - end - end - - # Class representing a drb server instance. - # - # A DRbServer must be running in the local process before any incoming - # dRuby calls can be accepted, or any local objects can be passed as - # dRuby references to remote processes, even if those local objects are - # never actually called remotely. You do not need to start a DRbServer - # in the local process if you are only making outgoing dRuby calls - # passing marshalled parameters. - # - # Unless multiple servers are being used, the local DRbServer is normally - # started by calling DRb.start_service. - class DRbServer - @@acl = nil - @@idconv = DRbIdConv.new - @@secondary_server = nil - @@argc_limit = 256 - @@load_limit = 0xffffffff - @@verbose = false - - # Set the default value for the :argc_limit option. - # - # See #new(). The initial default value is 256. - def self.default_argc_limit(argc) - @@argc_limit = argc - end - - # Set the default value for the :load_limit option. - # - # See #new(). The initial default value is 25 MB. - def self.default_load_limit(sz) - @@load_limit = sz - end - - # Set the default access control list to +acl+. The default ACL is +nil+. - # - # See also DRb::ACL and #new() - def self.default_acl(acl) - @@acl = acl - end - - # Set the default value for the :id_conv option. - # - # See #new(). The initial default value is a DRbIdConv instance. - def self.default_id_conv(idconv) - @@idconv = idconv - end - - # Set the default value of the :verbose option. - # - # See #new(). The initial default value is false. - def self.verbose=(on) - @@verbose = on - end - - # Get the default value of the :verbose option. - def self.verbose - @@verbose - end - - def self.make_config(hash={}) # :nodoc: - default_config = { - :idconv => @@idconv, - :verbose => @@verbose, - :tcp_acl => @@acl, - :load_limit => @@load_limit, - :argc_limit => @@argc_limit, - } - default_config.update(hash) - end - - # Create a new DRbServer instance. - # - # +uri+ is the URI to bind to. This is normally of the form - # 'druby://<hostname>:<port>' where <hostname> is a hostname of - # the local machine. If nil, then the system's default hostname - # will be bound to, on a port selected by the system; these value - # can be retrieved from the +uri+ attribute. 'druby:' specifies - # the default dRuby transport protocol: another protocol, such - # as 'drbunix:', can be specified instead. - # - # +front+ is the front object for the server, that is, the object - # to which remote method calls on the server will be passed. If - # nil, then the server will not accept remote method calls. - # - # If +config_or_acl+ is a hash, it is the configuration to - # use for this server. The following options are recognised: - # - # :idconv :: an id-to-object conversion object. This defaults - # to an instance of the class DRb::DRbIdConv. - # :verbose :: if true, all unsuccessful remote calls on objects - # in the server will be logged to $stdout. false - # by default. - # :tcp_acl :: the access control list for this server. See - # the ACL class from the main dRuby distribution. - # :load_limit :: the maximum message size in bytes accepted by - # the server. Defaults to 25 MB (26214400). - # :argc_limit :: the maximum number of arguments to a remote - # method accepted by the server. Defaults to - # 256. - # The default values of these options can be modified on - # a class-wide basis by the class methods #default_argc_limit, - # #default_load_limit, #default_acl, #default_id_conv, - # and #verbose= - # - # If +config_or_acl+ is not a hash, but is not nil, it is - # assumed to be the access control list for this server. - # See the :tcp_acl option for more details. - # - # If no other server is currently set as the primary server, - # this will become the primary server. - # - # The server will immediately start running in its own thread. - def initialize(uri=nil, front=nil, config_or_acl=nil) - if Hash === config_or_acl - config = config_or_acl.dup - else - acl = config_or_acl || @@acl - config = { - :tcp_acl => acl - } - end - - @config = self.class.make_config(config) - - @protocol = DRbProtocol.open_server(uri, @config) - @uri = @protocol.uri - @exported_uri = [@uri] - - @front = front - @idconv = @config[:idconv] - - @grp = ThreadGroup.new - @thread = run - - DRb.regist_server(self) - end - - # The URI of this DRbServer. - attr_reader :uri - - # The main thread of this DRbServer. - # - # This is the thread that listens for and accepts connections - # from clients, not that handles each client's request-response - # session. - attr_reader :thread - - # The front object of the DRbServer. - # - # This object receives remote method calls made on the server's - # URI alone, with an object id. - attr_reader :front - - # The configuration of this DRbServer - attr_reader :config - - # Set whether to operate in verbose mode. - # - # In verbose mode, failed calls are logged to stdout. - def verbose=(v); @config[:verbose]=v; end - - # Get whether the server is in verbose mode. - # - # In verbose mode, failed calls are logged to stdout. - def verbose; @config[:verbose]; end - - # Is this server alive? - def alive? - @thread.alive? - end - - # Is +uri+ the URI for this server? - def here?(uri) - @exported_uri.include?(uri) - end - - # Stop this server. - def stop_service - DRb.remove_server(self) - if Thread.current['DRb'] && Thread.current['DRb']['server'] == self - Thread.current['DRb']['stop_service'] = true - else - shutdown - end - end - - # Convert a dRuby reference to the local object it refers to. - def to_obj(ref) - return front if ref.nil? - return front[ref.to_s] if DRbURIOption === ref - @idconv.to_obj(ref) - end - - # Convert a local object to a dRuby reference. - def to_id(obj) - return nil if obj.__id__ == front.__id__ - @idconv.to_id(obj) - end - - private - - def shutdown - current = Thread.current - if @protocol.respond_to? :shutdown - @protocol.shutdown - else - [@thread, *@grp.list].each { |thread| - thread.kill unless thread == current # xxx: Thread#kill - } - end - @thread.join unless @thread == current - end - - ## - # Starts the DRb main loop in a new thread. - - def run - Thread.start do - begin - while main_loop - end - ensure - @protocol.close if @protocol - end - end - end - - # List of insecure methods. - # - # These methods are not callable via dRuby. - INSECURE_METHOD = [ - :__send__ - ] - - # Has a method been included in the list of insecure methods? - def insecure_method?(msg_id) - INSECURE_METHOD.include?(msg_id) - end - - # Coerce an object to a string, providing our own representation if - # to_s is not defined for the object. - def any_to_s(obj) - "#{obj}:#{obj.class}" - rescue - Kernel.instance_method(:to_s).bind_call(obj) - end - - # Check that a method is callable via dRuby. - # - # +obj+ is the object we want to invoke the method on. +msg_id+ is the - # method name, as a Symbol. - # - # If the method is an insecure method (see #insecure_method?) a - # SecurityError is thrown. If the method is private or undefined, - # a NameError is thrown. - def check_insecure_method(obj, msg_id) - return true if Proc === obj && msg_id == :__drb_yield - raise(ArgumentError, "#{any_to_s(msg_id)} is not a symbol") unless Symbol == msg_id.class - raise(SecurityError, "insecure method `#{msg_id}'") if insecure_method?(msg_id) - - case obj - when Object - if obj.private_methods.include?(msg_id) - desc = any_to_s(obj) - raise NoMethodError, "private method `#{msg_id}' called for #{desc}" - elsif obj.protected_methods.include?(msg_id) - desc = any_to_s(obj) - raise NoMethodError, "protected method `#{msg_id}' called for #{desc}" - else - true - end - else - if Kernel.instance_method(:private_methods).bind(obj).call.include?(msg_id) - desc = any_to_s(obj) - raise NoMethodError, "private method `#{msg_id}' called for #{desc}" - elsif Kernel.instance_method(:protected_methods).bind(obj).call.include?(msg_id) - desc = any_to_s(obj) - raise NoMethodError, "protected method `#{msg_id}' called for #{desc}" - else - true - end - end - end - public :check_insecure_method - - class InvokeMethod # :nodoc: - def initialize(drb_server, client) - @drb_server = drb_server - @client = client - end - - def perform - begin - setup_message - ensure - @result = nil - @succ = false - end - - if @block - @result = perform_with_block - else - @result = perform_without_block - end - @succ = true - case @result - when Array - if @msg_id == :to_ary - @result = DRbArray.new(@result) - end - end - return @succ, @result - rescue NoMemoryError, SystemExit, SystemStackError, SecurityError - raise - rescue Exception - @result = $! - return @succ, @result - end - - private - def init_with_client - obj, msg, argv, block = @client.recv_request - @obj = obj - @msg_id = msg.intern - @argv = argv - @block = block - end - - def check_insecure_method - @drb_server.check_insecure_method(@obj, @msg_id) - end - - def setup_message - init_with_client - check_insecure_method - end - - def perform_without_block - if Proc === @obj && @msg_id == :__drb_yield - if @argv.size == 1 - ary = @argv - else - ary = [@argv] - end - ary.collect(&@obj)[0] - else - @obj.__send__(@msg_id, *@argv) - end - end - - end - - require_relative 'invokemethod' - class InvokeMethod - include InvokeMethod18Mixin - end - - def error_print(exception) - exception.backtrace.inject(true) do |first, x| - if first - $stderr.puts "#{x}: #{exception} (#{exception.class})" - else - $stderr.puts "\tfrom #{x}" - end - false - end - end - - # The main loop performed by a DRbServer's internal thread. - # - # Accepts a connection from a client, and starts up its own - # thread to handle it. This thread loops, receiving requests - # from the client, invoking them on a local object, and - # returning responses, until the client closes the connection - # or a local method call fails. - def main_loop - client0 = @protocol.accept - return nil if !client0 - Thread.start(client0) do |client| - @grp.add Thread.current - Thread.current['DRb'] = { 'client' => client , - 'server' => self } - DRb.mutex.synchronize do - client_uri = client.uri - @exported_uri << client_uri unless @exported_uri.include?(client_uri) - end - _last_invoke_method = nil - loop do - begin - succ = false - invoke_method = InvokeMethod.new(self, client) - succ, result = invoke_method.perform - error_print(result) if !succ && verbose - unless DRbConnError === result && result.message == 'connection closed' - client.send_reply(succ, result) - end - rescue Exception => e - error_print(e) if verbose - ensure - _last_invoke_method = invoke_method - client.close unless succ - if Thread.current['DRb']['stop_service'] - shutdown - break - end - break unless succ - end - end - end - end - end - - @primary_server = nil - - # Start a dRuby server locally. - # - # The new dRuby server will become the primary server, even - # if another server is currently the primary server. - # - # +uri+ is the URI for the server to bind to. If nil, - # the server will bind to random port on the default local host - # name and use the default dRuby protocol. - # - # +front+ is the server's front object. This may be nil. - # - # +config+ is the configuration for the new server. This may - # be nil. - # - # See DRbServer::new. - def start_service(uri=nil, front=nil, config=nil) - @primary_server = DRbServer.new(uri, front, config) - end - module_function :start_service - - # The primary local dRuby server. - # - # This is the server created by the #start_service call. - attr_accessor :primary_server - module_function :primary_server=, :primary_server - - # Get the 'current' server. - # - # In the context of execution taking place within the main - # thread of a dRuby server (typically, as a result of a remote - # call on the server or one of its objects), the current - # server is that server. Otherwise, the current server is - # the primary server. - # - # If the above rule fails to find a server, a DRbServerNotFound - # error is raised. - def current_server - drb = Thread.current['DRb'] - server = (drb && drb['server']) ? drb['server'] : @primary_server - raise DRbServerNotFound unless server - return server - end - module_function :current_server - - # Stop the local dRuby server. - # - # This operates on the primary server. If there is no primary - # server currently running, it is a noop. - def stop_service - @primary_server.stop_service if @primary_server - @primary_server = nil - end - module_function :stop_service - - # Get the URI defining the local dRuby space. - # - # This is the URI of the current server. See #current_server. - def uri - drb = Thread.current['DRb'] - client = (drb && drb['client']) - if client - uri = client.uri - return uri if uri - end - current_server.uri - end - module_function :uri - - # Is +uri+ the URI for the current local server? - def here?(uri) - current_server.here?(uri) rescue false - # (current_server.uri rescue nil) == uri - end - module_function :here? - - # Get the configuration of the current server. - # - # If there is no current server, this returns the default configuration. - # See #current_server and DRbServer::make_config. - def config - current_server.config - rescue - DRbServer.make_config - end - module_function :config - - # Get the front object of the current server. - # - # This raises a DRbServerNotFound error if there is no current server. - # See #current_server. - def front - current_server.front - end - module_function :front - - # Convert a reference into an object using the current server. - # - # This raises a DRbServerNotFound error if there is no current server. - # See #current_server. - def to_obj(ref) - current_server.to_obj(ref) - end - - # Get a reference id for an object using the current server. - # - # This raises a DRbServerNotFound error if there is no current server. - # See #current_server. - def to_id(obj) - current_server.to_id(obj) - end - module_function :to_id - module_function :to_obj - - # Get the thread of the primary server. - # - # This returns nil if there is no primary server. See #primary_server. - def thread - @primary_server ? @primary_server.thread : nil - end - module_function :thread - - # Set the default id conversion object. - # - # This is expected to be an instance such as DRb::DRbIdConv that responds to - # #to_id and #to_obj that can convert objects to and from DRb references. - # - # See DRbServer#default_id_conv. - def install_id_conv(idconv) - DRbServer.default_id_conv(idconv) - end - module_function :install_id_conv - - # Set the default ACL to +acl+. - # - # See DRb::DRbServer.default_acl. - def install_acl(acl) - DRbServer.default_acl(acl) - end - module_function :install_acl - - @mutex = Thread::Mutex.new - def mutex # :nodoc: - @mutex - end - module_function :mutex - - @server = {} - # Registers +server+ with DRb. - # - # This is called when a new DRb::DRbServer is created. - # - # If there is no primary server then +server+ becomes the primary server. - # - # Example: - # - # require 'drb' - # - # s = DRb::DRbServer.new # automatically calls regist_server - # DRb.fetch_server s.uri #=> #<DRb::DRbServer:0x...> - def regist_server(server) - @server[server.uri] = server - mutex.synchronize do - @primary_server = server unless @primary_server - end - end - module_function :regist_server - - # Removes +server+ from the list of registered servers. - def remove_server(server) - @server.delete(server.uri) - mutex.synchronize do - if @primary_server == server - @primary_server = nil - end - end - end - module_function :remove_server - - # Retrieves the server with the given +uri+. - # - # See also regist_server and remove_server. - def fetch_server(uri) - @server[uri] - end - module_function :fetch_server -end - -# :stopdoc: -DRbObject = DRb::DRbObject -DRbUndumped = DRb::DRbUndumped -DRbIdConv = DRb::DRbIdConv diff --git a/lib/drb/eq.rb b/lib/drb/eq.rb deleted file mode 100644 index 15ca5cae42..0000000000 --- a/lib/drb/eq.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: false -module DRb - class DRbObject # :nodoc: - def ==(other) - return false unless DRbObject === other - (@ref == other.__drbref) && (@uri == other.__drburi) - end - - def hash - [@uri, @ref].hash - end - - alias eql? == - end -end diff --git a/lib/drb/extserv.rb b/lib/drb/extserv.rb deleted file mode 100644 index 9523fe84e3..0000000000 --- a/lib/drb/extserv.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: false -=begin - external service - Copyright (c) 2000,2002 Masatoshi SEKI -=end - -require_relative 'drb' -require 'monitor' - -module DRb - class ExtServ - include MonitorMixin - include DRbUndumped - - def initialize(there, name, server=nil) - super() - @server = server || DRb::primary_server - @name = name - ro = DRbObject.new(nil, there) - synchronize do - @invoker = ro.register(name, DRbObject.new(self, @server.uri)) - end - end - attr_reader :server - - def front - DRbObject.new(nil, @server.uri) - end - - def stop_service - synchronize do - @invoker.unregister(@name) - server = @server - @server = nil - server.stop_service - true - end - end - - def alive? - @server ? @server.alive? : false - end - end -end diff --git a/lib/drb/extservm.rb b/lib/drb/extservm.rb deleted file mode 100644 index 9333a108e5..0000000000 --- a/lib/drb/extservm.rb +++ /dev/null @@ -1,94 +0,0 @@ -# frozen_string_literal: false -=begin - external service manager - Copyright (c) 2000 Masatoshi SEKI -=end - -require_relative 'drb' -require 'monitor' - -module DRb - class ExtServManager - include DRbUndumped - include MonitorMixin - - @@command = {} - - def self.command - @@command - end - - def self.command=(cmd) - @@command = cmd - end - - def initialize - super() - @cond = new_cond - @servers = {} - @waiting = [] - @queue = Thread::Queue.new - @thread = invoke_thread - @uri = nil - end - attr_accessor :uri - - def service(name) - synchronize do - while true - server = @servers[name] - return server if server && server.alive? # server may be `false' - invoke_service(name) - @cond.wait - end - end - end - - def register(name, ro) - synchronize do - @servers[name] = ro - @cond.signal - end - self - end - alias regist register - - def unregister(name) - synchronize do - @servers.delete(name) - end - end - alias unregist unregister - - private - def invoke_thread - Thread.new do - while name = @queue.pop - invoke_service_command(name, @@command[name]) - end - end - end - - def invoke_service(name) - @queue.push(name) - end - - def invoke_service_command(name, command) - raise "invalid command. name: #{name}" unless command - synchronize do - return if @servers.include?(name) - @servers[name] = false - end - uri = @uri || DRb.uri - if command.respond_to? :to_ary - command = command.to_ary + [uri, name] - pid = spawn(*command) - else - pid = spawn("#{command} #{uri} #{name}") - end - th = Process.detach(pid) - th[:drb_service] = name - th - end - end -end diff --git a/lib/drb/gw.rb b/lib/drb/gw.rb deleted file mode 100644 index 65a525476e..0000000000 --- a/lib/drb/gw.rb +++ /dev/null @@ -1,161 +0,0 @@ -# frozen_string_literal: false -require_relative 'drb' -require 'monitor' - -module DRb - - # Gateway id conversion forms a gateway between different DRb protocols or - # networks. - # - # The gateway needs to install this id conversion and create servers for - # each of the protocols or networks it will be a gateway between. It then - # needs to create a server that attaches to each of these networks. For - # example: - # - # require 'drb/drb' - # require 'drb/unix' - # require 'drb/gw' - # - # DRb.install_id_conv DRb::GWIdConv.new - # gw = DRb::GW.new - # s1 = DRb::DRbServer.new 'drbunix:/path/to/gateway', gw - # s2 = DRb::DRbServer.new 'druby://example:10000', gw - # - # s1.thread.join - # s2.thread.join - # - # Each client must register services with the gateway, for example: - # - # DRb.start_service 'drbunix:', nil # an anonymous server - # gw = DRbObject.new nil, 'drbunix:/path/to/gateway' - # gw[:unix] = some_service - # DRb.thread.join - - class GWIdConv < DRbIdConv - def to_obj(ref) # :nodoc: - if Array === ref && ref[0] == :DRbObject - return DRbObject.new_with(ref[1], ref[2]) - end - super(ref) - end - end - - # The GW provides a synchronized store for participants in the gateway to - # communicate. - - class GW - include MonitorMixin - - # Creates a new GW - - def initialize - super() - @hash = {} - end - - # Retrieves +key+ from the GW - - def [](key) - synchronize do - @hash[key] - end - end - - # Stores value +v+ at +key+ in the GW - - def []=(key, v) - synchronize do - @hash[key] = v - end - end - end - - class DRbObject # :nodoc: - def self._load(s) - uri, ref = Marshal.load(s) - if DRb.uri == uri - return ref ? DRb.to_obj(ref) : DRb.front - end - - self.new_with(DRb.uri, [:DRbObject, uri, ref]) - end - - def _dump(lv) - if DRb.uri == @uri - if Array === @ref && @ref[0] == :DRbObject - Marshal.dump([@ref[1], @ref[2]]) - else - Marshal.dump([@uri, @ref]) # ?? - end - else - Marshal.dump([DRb.uri, [:DRbObject, @uri, @ref]]) - end - end - end -end - -=begin -DRb.install_id_conv(DRb::GWIdConv.new) - -front = DRb::GW.new - -s1 = DRb::DRbServer.new('drbunix:/tmp/gw_b_a', front) -s2 = DRb::DRbServer.new('drbunix:/tmp/gw_b_c', front) - -s1.thread.join -s2.thread.join -=end - -=begin -# foo.rb - -require 'drb/drb' - -class Foo - include DRbUndumped - def initialize(name, peer=nil) - @name = name - @peer = peer - end - - def ping(obj) - puts "#{@name}: ping: #{obj.inspect}" - @peer.ping(self) if @peer - end -end -=end - -=begin -# gw_a.rb -require 'drb/unix' -require 'foo' - -obj = Foo.new('a') -DRb.start_service("drbunix:/tmp/gw_a", obj) - -robj = DRbObject.new_with_uri('drbunix:/tmp/gw_b_a') -robj[:a] = obj - -DRb.thread.join -=end - -=begin -# gw_c.rb -require 'drb/unix' -require 'foo' - -foo = Foo.new('c', nil) - -DRb.start_service("drbunix:/tmp/gw_c", nil) - -robj = DRbObject.new_with_uri("drbunix:/tmp/gw_b_c") - -puts "c->b" -a = robj[:a] -sleep 2 - -a.ping(foo) - -DRb.thread.join -=end - diff --git a/lib/drb/invokemethod.rb b/lib/drb/invokemethod.rb deleted file mode 100644 index 0fae6d52b6..0000000000 --- a/lib/drb/invokemethod.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: false -# for ruby-1.8.0 - -module DRb # :nodoc: all - class DRbServer - module InvokeMethod18Mixin - def block_yield(x) - if x.size == 1 && x[0].class == Array - x[0] = DRbArray.new(x[0]) - end - @block.call(*x) - end - - def perform_with_block - @obj.__send__(@msg_id, *@argv) do |*x| - jump_error = nil - begin - block_value = block_yield(x) - rescue LocalJumpError - jump_error = $! - end - if jump_error - case jump_error.reason - when :break - break(jump_error.exit_value) - else - raise jump_error - end - end - block_value - end - end - end - end -end diff --git a/lib/drb/observer.rb b/lib/drb/observer.rb deleted file mode 100644 index 0fb7301edf..0000000000 --- a/lib/drb/observer.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: false -require 'observer' - -module DRb - # The Observable module extended to DRb. See Observable for details. - module DRbObservable - include Observable - - # Notifies observers of a change in state. See also - # Observable#notify_observers - def notify_observers(*arg) - if defined? @observer_state and @observer_state - if defined? @observer_peers - @observer_peers.each do |observer, method| - begin - observer.__send__(method, *arg) - rescue - delete_observer(observer) - end - end - end - @observer_state = false - end - end - end -end diff --git a/lib/drb/ssl.rb b/lib/drb/ssl.rb deleted file mode 100644 index 54ab1ef395..0000000000 --- a/lib/drb/ssl.rb +++ /dev/null @@ -1,344 +0,0 @@ -# frozen_string_literal: false -require 'socket' -require 'openssl' -require_relative 'drb' -require 'singleton' - -module DRb - - # The protocol for DRb over an SSL socket - # - # The URI for a DRb socket over SSL is: - # <code>drbssl://<host>:<port>?<option></code>. The option is optional - class DRbSSLSocket < DRbTCPSocket - - # SSLConfig handles the needed SSL information for establishing a - # DRbSSLSocket connection, including generating the X509 / RSA pair. - # - # An instance of this config can be passed to DRbSSLSocket.new, - # DRbSSLSocket.open and DRbSSLSocket.open_server - # - # See DRb::DRbSSLSocket::SSLConfig.new for more details - class SSLConfig - - # Default values for a SSLConfig instance. - # - # See DRb::DRbSSLSocket::SSLConfig.new for more details - DEFAULT = { - :SSLCertificate => nil, - :SSLPrivateKey => nil, - :SSLClientCA => nil, - :SSLCACertificatePath => nil, - :SSLCACertificateFile => nil, - :SSLTmpDhCallback => nil, - :SSLVerifyMode => ::OpenSSL::SSL::VERIFY_NONE, - :SSLVerifyDepth => nil, - :SSLVerifyCallback => nil, # custom verification - :SSLCertificateStore => nil, - # Must specify if you use auto generated certificate. - :SSLCertName => nil, # e.g. [["CN","fqdn.example.com"]] - :SSLCertComment => "Generated by Ruby/OpenSSL" - } - - # Create a new DRb::DRbSSLSocket::SSLConfig instance - # - # The DRb::DRbSSLSocket will take either a +config+ Hash or an instance - # of SSLConfig, and will setup the certificate for its session for the - # configuration. If want it to generate a generic certificate, the bare - # minimum is to provide the :SSLCertName - # - # === Config options - # - # From +config+ Hash: - # - # :SSLCertificate :: - # An instance of OpenSSL::X509::Certificate. If this is not provided, - # then a generic X509 is generated, with a correspond :SSLPrivateKey - # - # :SSLPrivateKey :: - # A private key instance, like OpenSSL::PKey::RSA. This key must be - # the key that signed the :SSLCertificate - # - # :SSLClientCA :: - # An OpenSSL::X509::Certificate, or Array of certificates that will - # used as ClientCAs in the SSL Context - # - # :SSLCACertificatePath :: - # A path to the directory of CA certificates. The certificates must - # be in PEM format. - # - # :SSLCACertificateFile :: - # A path to a CA certificate file, in PEM format. - # - # :SSLTmpDhCallback :: - # A DH callback. See OpenSSL::SSL::SSLContext.tmp_dh_callback - # - # :SSLVerifyMode :: - # This is the SSL verification mode. See OpenSSL::SSL::VERIFY_* for - # available modes. The default is OpenSSL::SSL::VERIFY_NONE - # - # :SSLVerifyDepth :: - # Number of CA certificates to walk, when verifying a certificate - # chain. - # - # :SSLVerifyCallback :: - # A callback to be used for additional verification. See - # OpenSSL::SSL::SSLContext.verify_callback - # - # :SSLCertificateStore :: - # A OpenSSL::X509::Store used for verification of certificates - # - # :SSLCertName :: - # Issuer name for the certificate. This is required when generating - # the certificate (if :SSLCertificate and :SSLPrivateKey were not - # given). The value of this is to be an Array of pairs: - # - # [["C", "Raleigh"], ["ST","North Carolina"], - # ["CN","fqdn.example.com"]] - # - # See also OpenSSL::X509::Name - # - # :SSLCertComment :: - # A comment to be used for generating the certificate. The default is - # "Generated by Ruby/OpenSSL" - # - # - # === Example - # - # These values can be added after the fact, like a Hash. - # - # require 'drb/ssl' - # c = DRb::DRbSSLSocket::SSLConfig.new {} - # c[:SSLCertificate] = - # OpenSSL::X509::Certificate.new(File.read('mycert.crt')) - # c[:SSLPrivateKey] = OpenSSL::PKey::RSA.new(File.read('mycert.key')) - # c[:SSLVerifyMode] = OpenSSL::SSL::VERIFY_PEER - # c[:SSLCACertificatePath] = "/etc/ssl/certs/" - # c.setup_certificate - # - # or - # - # require 'drb/ssl' - # c = DRb::DRbSSLSocket::SSLConfig.new({ - # :SSLCertName => [["CN" => DRb::DRbSSLSocket.getservername]] - # }) - # c.setup_certificate - # - def initialize(config) - @config = config - @cert = config[:SSLCertificate] - @pkey = config[:SSLPrivateKey] - @ssl_ctx = nil - end - - # A convenience method to access the values like a Hash - def [](key); - @config[key] || DEFAULT[key] - end - - # Connect to IO +tcp+, with context of the current certificate - # configuration - def connect(tcp) - ssl = ::OpenSSL::SSL::SSLSocket.new(tcp, @ssl_ctx) - ssl.sync = true - ssl.connect - ssl - end - - # Accept connection to IO +tcp+, with context of the current certificate - # configuration - def accept(tcp) - ssl = OpenSSL::SSL::SSLSocket.new(tcp, @ssl_ctx) - ssl.sync = true - ssl.accept - ssl - end - - # Ensures that :SSLCertificate and :SSLPrivateKey have been provided - # or that a new certificate is generated with the other parameters - # provided. - def setup_certificate - if @cert && @pkey - return - end - - rsa = OpenSSL::PKey::RSA.new(2048){|p, n| - next unless self[:verbose] - case p - when 0; $stderr.putc "." # BN_generate_prime - when 1; $stderr.putc "+" # BN_generate_prime - when 2; $stderr.putc "*" # searching good prime, - # n = #of try, - # but also data from BN_generate_prime - when 3; $stderr.putc "\n" # found good prime, n==0 - p, n==1 - q, - # but also data from BN_generate_prime - else; $stderr.putc "*" # BN_generate_prime - end - } - - cert = OpenSSL::X509::Certificate.new - cert.version = 3 - cert.serial = 0 - name = OpenSSL::X509::Name.new(self[:SSLCertName]) - cert.subject = name - cert.issuer = name - cert.not_before = Time.now - cert.not_after = Time.now + (365*24*60*60) - cert.public_key = rsa.public_key - - ef = OpenSSL::X509::ExtensionFactory.new(nil,cert) - cert.extensions = [ - ef.create_extension("basicConstraints","CA:FALSE"), - ef.create_extension("subjectKeyIdentifier", "hash") ] - ef.issuer_certificate = cert - cert.add_extension(ef.create_extension("authorityKeyIdentifier", - "keyid:always,issuer:always")) - if comment = self[:SSLCertComment] - cert.add_extension(ef.create_extension("nsComment", comment)) - end - cert.sign(rsa, "SHA256") - - @cert = cert - @pkey = rsa - end - - # Establish the OpenSSL::SSL::SSLContext with the configuration - # parameters provided. - def setup_ssl_context - ctx = ::OpenSSL::SSL::SSLContext.new - ctx.cert = @cert - ctx.key = @pkey - ctx.client_ca = self[:SSLClientCA] - ctx.ca_path = self[:SSLCACertificatePath] - ctx.ca_file = self[:SSLCACertificateFile] - ctx.tmp_dh_callback = self[:SSLTmpDhCallback] - ctx.verify_mode = self[:SSLVerifyMode] - ctx.verify_depth = self[:SSLVerifyDepth] - ctx.verify_callback = self[:SSLVerifyCallback] - ctx.cert_store = self[:SSLCertificateStore] - @ssl_ctx = ctx - end - end - - # Parse the dRuby +uri+ for an SSL connection. - # - # Expects drbssl://... - # - # Raises DRbBadScheme or DRbBadURI if +uri+ is not matching or malformed - def self.parse_uri(uri) # :nodoc: - if /\Adrbssl:\/\/(.*?):(\d+)(\?(.*))?\z/ =~ uri - host = $1 - port = $2.to_i - option = $4 - [host, port, option] - else - raise(DRbBadScheme, uri) unless uri.start_with?('drbssl:') - raise(DRbBadURI, 'can\'t parse uri:' + uri) - end - end - - # Return an DRb::DRbSSLSocket instance as a client-side connection, - # with the SSL connected. This is called from DRb::start_service or while - # connecting to a remote object: - # - # DRb.start_service 'drbssl://localhost:0', front, config - # - # +uri+ is the URI we are connected to, - # <code>'drbssl://localhost:0'</code> above, +config+ is our - # configuration. Either a Hash or DRb::DRbSSLSocket::SSLConfig - def self.open(uri, config) - host, port, = parse_uri(uri) - soc = TCPSocket.open(host, port) - ssl_conf = SSLConfig::new(config) - ssl_conf.setup_ssl_context - ssl = ssl_conf.connect(soc) - self.new(uri, ssl, ssl_conf, true) - end - - # Returns a DRb::DRbSSLSocket instance as a server-side connection, with - # the SSL connected. This is called from DRb::start_service or while - # connecting to a remote object: - # - # DRb.start_service 'drbssl://localhost:0', front, config - # - # +uri+ is the URI we are connected to, - # <code>'drbssl://localhost:0'</code> above, +config+ is our - # configuration. Either a Hash or DRb::DRbSSLSocket::SSLConfig - def self.open_server(uri, config) - uri = 'drbssl://:0' unless uri - host, port, = parse_uri(uri) - if host.size == 0 - host = getservername - soc = open_server_inaddr_any(host, port) - else - soc = TCPServer.open(host, port) - end - port = soc.addr[1] if port == 0 - @uri = "drbssl://#{host}:#{port}" - - ssl_conf = SSLConfig.new(config) - ssl_conf.setup_certificate - ssl_conf.setup_ssl_context - self.new(@uri, soc, ssl_conf, false) - end - - # This is a convenience method to parse +uri+ and separate out any - # additional options appended in the +uri+. - # - # Returns an option-less uri and the option => [uri,option] - # - # The +config+ is completely unused, so passing nil is sufficient. - def self.uri_option(uri, config) # :nodoc: - host, port, option = parse_uri(uri) - return "drbssl://#{host}:#{port}", option - end - - # Create a DRb::DRbSSLSocket instance. - # - # +uri+ is the URI we are connected to. - # +soc+ is the tcp socket we are bound to. - # +config+ is our configuration. Either a Hash or SSLConfig - # +is_established+ is a boolean of whether +soc+ is currently established - # - # This is called automatically based on the DRb protocol. - def initialize(uri, soc, config, is_established) - @ssl = is_established ? soc : nil - super(uri, soc.to_io, config) - end - - # Returns the SSL stream - def stream; @ssl; end # :nodoc: - - # Closes the SSL stream before closing the dRuby connection. - def close # :nodoc: - if @ssl - @ssl.close - @ssl = nil - end - super - end - - def accept # :nodoc: - begin - while true - soc = accept_or_shutdown - return nil unless soc - break if (@acl ? @acl.allow_socket?(soc) : true) - soc.close - end - begin - ssl = @config.accept(soc) - rescue Exception - soc.close - raise - end - self.class.new(uri, ssl, @config, true) - rescue OpenSSL::SSL::SSLError - warn("#{$!.message} (#{$!.class})", uplevel: 0) if @config[:verbose] - retry - end - end - end - - DRbProtocol.add_protocol(DRbSSLSocket) -end diff --git a/lib/drb/timeridconv.rb b/lib/drb/timeridconv.rb deleted file mode 100644 index 3ead98a7f2..0000000000 --- a/lib/drb/timeridconv.rb +++ /dev/null @@ -1,97 +0,0 @@ -# frozen_string_literal: false -require_relative 'drb' -require 'monitor' - -module DRb - - # Timer id conversion keeps objects alive for a certain amount of time after - # their last access. The default time period is 600 seconds and can be - # changed upon initialization. - # - # To use TimerIdConv: - # - # DRb.install_id_conv TimerIdConv.new 60 # one minute - - class TimerIdConv < DRbIdConv - class TimerHolder2 # :nodoc: - include MonitorMixin - - class InvalidIndexError < RuntimeError; end - - def initialize(keeping=600) - super() - @sentinel = Object.new - @gc = {} - @renew = {} - @keeping = keeping - @expires = nil - end - - def add(obj) - synchronize do - rotate - key = obj.__id__ - @renew[key] = obj - invoke_keeper - return key - end - end - - def fetch(key) - synchronize do - rotate - obj = peek(key) - raise InvalidIndexError if obj == @sentinel - @renew[key] = obj # KeepIt - return obj - end - end - - private - def peek(key) - return @renew.fetch(key) { @gc.fetch(key, @sentinel) } - end - - def invoke_keeper - return if @expires - @expires = Time.now + @keeping - on_gc - end - - def on_gc - return unless Thread.main.alive? - return if @expires.nil? - Thread.new { rotate } if @expires < Time.now - ObjectSpace.define_finalizer(Object.new) {on_gc} - end - - def rotate - synchronize do - if @expires &.< Time.now - @gc = @renew # GCed - @renew = {} - @expires = @gc.empty? ? nil : Time.now + @keeping - end - end - end - end - - # Creates a new TimerIdConv which will hold objects for +keeping+ seconds. - def initialize(keeping=600) - @holder = TimerHolder2.new(keeping) - end - - def to_obj(ref) # :nodoc: - return super if ref.nil? - @holder.fetch(ref) - rescue TimerHolder2::InvalidIndexError - raise "invalid reference" - end - - def to_id(obj) # :nodoc: - return @holder.add(obj) - end - end -end - -# DRb.install_id_conv(TimerIdConv.new) diff --git a/lib/drb/unix.rb b/lib/drb/unix.rb deleted file mode 100644 index 1629ad3bcd..0000000000 --- a/lib/drb/unix.rb +++ /dev/null @@ -1,118 +0,0 @@ -# frozen_string_literal: false -require 'socket' -require_relative 'drb' -require 'tmpdir' - -raise(LoadError, "UNIXServer is required") unless defined?(UNIXServer) - -module DRb - - # Implements DRb over a UNIX socket - # - # DRb UNIX socket URIs look like <code>drbunix:<path>?<option></code>. The - # option is optional. - - class DRbUNIXSocket < DRbTCPSocket - # :stopdoc: - def self.parse_uri(uri) - if /\Adrbunix:(.*?)(\?(.*))?\z/ =~ uri - filename = $1 - option = $3 - [filename, option] - else - raise(DRbBadScheme, uri) unless uri.start_with?('drbunix:') - raise(DRbBadURI, 'can\'t parse uri:' + uri) - end - end - - def self.open(uri, config) - filename, = parse_uri(uri) - soc = UNIXSocket.open(filename) - self.new(uri, soc, config) - end - - def self.open_server(uri, config) - filename, = parse_uri(uri) - if filename.size == 0 - soc = temp_server - filename = soc.path - uri = 'drbunix:' + soc.path - else - soc = UNIXServer.open(filename) - end - owner = config[:UNIXFileOwner] - group = config[:UNIXFileGroup] - if owner || group - require 'etc' - owner = Etc.getpwnam( owner ).uid if owner - group = Etc.getgrnam( group ).gid if group - File.chown owner, group, filename - end - mode = config[:UNIXFileMode] - File.chmod(mode, filename) if mode - - self.new(uri, soc, config, true) - end - - def self.uri_option(uri, config) - filename, option = parse_uri(uri) - return "drbunix:#{filename}", option - end - - def initialize(uri, soc, config={}, server_mode = false) - super(uri, soc, config) - set_sockopt(@socket) - @server_mode = server_mode - @acl = nil - end - - # import from tempfile.rb - Max_try = 10 - private - def self.temp_server - tmpdir = Dir::tmpdir - n = 0 - while true - begin - tmpname = sprintf('%s/druby%d.%d', tmpdir, $$, n) - lock = tmpname + '.lock' - unless File.exist?(tmpname) or File.exist?(lock) - Dir.mkdir(lock) - break - end - rescue - raise "cannot generate tempfile `%s'" % tmpname if n >= Max_try - #sleep(1) - end - n += 1 - end - soc = UNIXServer.new(tmpname) - Dir.rmdir(lock) - soc - end - - public - def close - return unless @socket - shutdown # DRbProtocol#shutdown - path = @socket.path if @server_mode - @socket.close - File.unlink(path) if @server_mode - @socket = nil - close_shutdown_pipe - end - - def accept - s = accept_or_shutdown - return nil unless s - self.class.new(nil, s, @config) - end - - def set_sockopt(soc) - # no-op for now - end - end - - DRbProtocol.add_protocol(DRbUNIXSocket) - # :startdoc: -end diff --git a/lib/drb/version.rb b/lib/drb/version.rb deleted file mode 100644 index 10d33445b6..0000000000 --- a/lib/drb/version.rb +++ /dev/null @@ -1,3 +0,0 @@ -module DRb - VERSION = "2.1.1" -end diff --git a/lib/drb/weakidconv.rb b/lib/drb/weakidconv.rb deleted file mode 100644 index ecf0bf515f..0000000000 --- a/lib/drb/weakidconv.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: false -require_relative 'drb' -require 'monitor' - -module DRb - - # To use WeakIdConv: - # - # DRb.start_service(nil, nil, {:idconv => DRb::WeakIdConv.new}) - - class WeakIdConv < DRbIdConv - class WeakSet - include MonitorMixin - def initialize - super() - @immutable = {} - @map = ObjectSpace::WeakMap.new - end - - def add(obj) - synchronize do - begin - @map[obj] = self - rescue ArgumentError - @immutable[obj.__id__] = obj - end - return obj.__id__ - end - end - - def fetch(ref) - synchronize do - @immutable.fetch(ref) { - @map.each { |key, _| - return key if key.__id__ == ref - } - raise RangeError.new("invalid reference") - } - end - end - end - - def initialize() - super() - @weak_set = WeakSet.new - end - - def to_obj(ref) # :nodoc: - return super if ref.nil? - @weak_set.fetch(ref) - end - - def to_id(obj) # :nodoc: - return @weak_set.add(obj) - end - end -end - -# DRb.install_id_conv(WeakIdConv.new) diff --git a/lib/erb/compiler.rb b/lib/erb/compiler.rb index 547d2c4c44..7096c8dcea 100644 --- a/lib/erb/compiler.rb +++ b/lib/erb/compiler.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true #-- # ERB::Compiler # diff --git a/lib/erb/def_method.rb b/lib/erb/def_method.rb index 17f9c0f9fa..aee989a926 100644 --- a/lib/erb/def_method.rb +++ b/lib/erb/def_method.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true #-- # ERB::DefMethod # diff --git a/lib/erb/util.rb b/lib/erb/util.rb index 0c1e7482a8..1d2a36275d 100644 --- a/lib/erb/util.rb +++ b/lib/erb/util.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true #-- # ERB::Escape # diff --git a/lib/erb/version.rb b/lib/erb/version.rb index 38e1b76ff4..b5fe39b330 100644 --- a/lib/erb/version.rb +++ b/lib/erb/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true class ERB - VERSION = '4.0.2' + VERSION = '4.0.4' private_constant :VERSION end diff --git a/lib/error_highlight/base.rb b/lib/error_highlight/base.rb index 7d2ff0c889..b9c68b8eb8 100644 --- a/lib/error_highlight/base.rb +++ b/lib/error_highlight/base.rb @@ -54,7 +54,16 @@ module ErrorHighlight return nil unless Thread::Backtrace::Location === loc - node = RubyVM::AbstractSyntaxTree.of(loc, keep_script_lines: true) + node = + begin + RubyVM::AbstractSyntaxTree.of(loc, keep_script_lines: true) + rescue RuntimeError => error + # RubyVM::AbstractSyntaxTree.of raises an error with a message that + # includes "prism" when the ISEQ was compiled with the prism compiler. + # In this case, we'll set the node to `nil`. In the future, we will + # reparse with the prism parser and pass the parsed node to Spotter. + raise unless error.message.include?("prism") + end Spotter.new(node, **opts).spot diff --git a/lib/error_highlight/version.rb b/lib/error_highlight/version.rb index 5afe5f06d6..506d37fbc1 100644 --- a/lib/error_highlight/version.rb +++ b/lib/error_highlight/version.rb @@ -1,3 +1,3 @@ module ErrorHighlight - VERSION = "0.5.1" + VERSION = "0.6.0" end diff --git a/lib/fileutils.rb b/lib/fileutils.rb index 6e7c3174cb..e8cc355760 100644 --- a/lib/fileutils.rb +++ b/lib/fileutils.rb @@ -180,7 +180,8 @@ end # - {CVE-2004-0452}[https://cve.mitre.org/cgi-bin/cvename.cgi?name=CAN-2004-0452]. # module FileUtils - VERSION = "1.7.1" + # The version number. + VERSION = "1.7.2" def self.private_module_function(name) #:nodoc: module_function name @@ -1651,7 +1652,7 @@ module FileUtils when "a" mask | 07777 else - raise ArgumentError, "invalid `who' symbol in file mode: #{chr}" + raise ArgumentError, "invalid 'who' symbol in file mode: #{chr}" end end end @@ -1705,7 +1706,7 @@ module FileUtils copy_mask = user_mask(chr) (current_mode & copy_mask) / (copy_mask & 0111) * (user_mask & 0111) else - raise ArgumentError, "invalid `perm' symbol in file mode: #{chr}" + raise ArgumentError, "invalid 'perm' symbol in file mode: #{chr}" end end @@ -2028,21 +2029,22 @@ module FileUtils private - module StreamUtils_ + module StreamUtils_ # :nodoc: + private case (defined?(::RbConfig) ? ::RbConfig::CONFIG['host_os'] : ::RUBY_PLATFORM) when /mswin|mingw/ - def fu_windows?; true end + def fu_windows?; true end #:nodoc: else - def fu_windows?; false end + def fu_windows?; false end #:nodoc: end def fu_copy_stream0(src, dest, blksize = nil) #:nodoc: IO.copy_stream(src, dest) end - def fu_stream_blksize(*streams) + def fu_stream_blksize(*streams) #:nodoc: streams.each do |s| next unless s.respond_to?(:stat) size = fu_blksize(s.stat) @@ -2051,14 +2053,14 @@ module FileUtils fu_default_blksize() end - def fu_blksize(st) + def fu_blksize(st) #:nodoc: s = st.blksize return nil unless s return nil if s == 0 s end - def fu_default_blksize + def fu_default_blksize #:nodoc: 1024 end end @@ -2503,7 +2505,7 @@ module FileUtils end private_module_function :fu_output_message - def fu_split_path(path) + def fu_split_path(path) #:nodoc: path = File.path(path) list = [] until (parent, base = File.split(path); parent == path or parent == ".") @@ -2524,7 +2526,7 @@ module FileUtils end private_module_function :fu_relative_components_from - def fu_clean_components(*comp) + def fu_clean_components(*comp) #:nodoc: comp.shift while comp.first == "." return comp if comp.empty? clean = [comp.shift] @@ -2543,11 +2545,11 @@ module FileUtils private_module_function :fu_clean_components if fu_windows? - def fu_starting_path?(path) + def fu_starting_path?(path) #:nodoc: path&.start_with?(%r(\w:|/)) end else - def fu_starting_path?(path) + def fu_starting_path?(path) #:nodoc: path&.start_with?("/") end end diff --git a/lib/find.gemspec b/lib/find.gemspec index cb845e9409..aef24a5028 100644 --- a/lib/find.gemspec +++ b/lib/find.gemspec @@ -25,7 +25,5 @@ Gem::Specification.new do |spec| spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } end - spec.bindir = "exe" - spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ["lib"] end diff --git a/lib/find.rb b/lib/find.rb index 79e2e2854d..98a79cc76d 100644 --- a/lib/find.rb +++ b/lib/find.rb @@ -27,7 +27,7 @@ # module Find - VERSION = "0.1.1" + VERSION = "0.2.0" # # Calls the associated block with the name of every file and directory listed diff --git a/lib/getoptlong.gemspec b/lib/getoptlong.gemspec deleted file mode 100644 index c2f65dcaab..0000000000 --- a/lib/getoptlong.gemspec +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -name = File.basename(__FILE__, ".gemspec") -version = ["lib", Array.new(name.count("-")+1, ".").join("/")].find do |dir| - break File.foreach(File.join(__dir__, dir, "#{name.tr('-', '/')}.rb")) do |line| - /^\s*VERSION\s*=\s*"(.*)"/ =~ line and break $1 - end rescue nil -end - -Gem::Specification.new do |spec| - spec.name = name - spec.version = version - spec.authors = ["Yukihiro Matsumoto"] - spec.email = ["matz@ruby-lang.org"] - - spec.summary = %q{GetoptLong for Ruby} - spec.description = spec.summary - spec.homepage = "https://github.com/ruby/getoptlong" - spec.licenses = ["Ruby", "BSD-2-Clause"] - - spec.metadata["homepage_uri"] = spec.homepage - spec.metadata["source_code_uri"] = spec.homepage - - # Specify which files should be added to the gem when it is released. - # The `git ls-files -z` loads the files in the RubyGem that have been added into git. - spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do - `git ls-files -z 2>#{IO::NULL}`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } - end - spec.require_paths = ["lib"] -end diff --git a/lib/getoptlong.rb b/lib/getoptlong.rb deleted file mode 100644 index 5ae0e1497c..0000000000 --- a/lib/getoptlong.rb +++ /dev/null @@ -1,867 +0,0 @@ -# frozen_string_literal: true -# -# GetoptLong for Ruby -# -# Copyright (C) 1998, 1999, 2000 Motoyuki Kasahara. -# -# You may redistribute and/or modify this library under the same license -# terms as Ruby. - -# \Class \GetoptLong provides parsing both for options -# and for regular arguments. -# -# Using \GetoptLong, you can define options for your program. -# The program can then capture and respond to whatever options -# are included in the command that executes the program. -# -# A simple example: file <tt>simple.rb</tt>: -# -# :include: ../sample/getoptlong/simple.rb -# -# If you are somewhat familiar with options, -# you may want to skip to this -# {full example}[#class-GetoptLong-label-Full+Example]. -# -# == Options -# -# A \GetoptLong option has: -# -# - A string <em>option name</em>. -# - Zero or more string <em>aliases</em> for the name. -# - An <em>option type</em>. -# -# Options may be defined by calling singleton method GetoptLong.new, -# which returns a new \GetoptLong object. -# Options may then be processed by calling other methods -# such as GetoptLong#each. -# -# === Option Name and Aliases -# -# In the array that defines an option, -# the first element is the string option name. -# Often the name takes the 'long' form, beginning with two hyphens. -# -# The option name may have any number of aliases, -# which are defined by additional string elements. -# -# The name and each alias must be of one of two forms: -# -# - Two hyphens, followed by one or more letters. -# - One hyphen, followed by a single letter. -# -# File <tt>aliases.rb</tt>: -# -# :include: ../sample/getoptlong/aliases.rb -# -# An option may be cited by its name, -# or by any of its aliases; -# the parsed option always reports the name, not an alias: -# -# $ ruby aliases.rb -a -p --xxx --aaa -x -# -# Output: -# -# ["--xxx", ""] -# ["--xxx", ""] -# ["--xxx", ""] -# ["--xxx", ""] -# ["--xxx", ""] -# -# -# An option may also be cited by an abbreviation of its name or any alias, -# as long as that abbreviation is unique among the options. -# -# File <tt>abbrev.rb</tt>: -# -# :include: ../sample/getoptlong/abbrev.rb -# -# Command line: -# -# $ ruby abbrev.rb --xxx --xx --xyz --xy -# -# Output: -# -# ["--xxx", ""] -# ["--xxx", ""] -# ["--xyz", ""] -# ["--xyz", ""] -# -# This command line raises GetoptLong::AmbiguousOption: -# -# $ ruby abbrev.rb --x -# -# === Repetition -# -# An option may be cited more than once: -# -# $ ruby abbrev.rb --xxx --xyz --xxx --xyz -# -# Output: -# -# ["--xxx", ""] -# ["--xyz", ""] -# ["--xxx", ""] -# ["--xyz", ""] -# -# === Treating Remaining Options as Arguments -# -# A option-like token that appears -# anywhere after the token <tt>--</tt> is treated as an ordinary argument, -# and is not processed as an option: -# -# $ ruby abbrev.rb --xxx --xyz -- --xxx --xyz -# -# Output: -# -# ["--xxx", ""] -# ["--xyz", ""] -# -# === Option Types -# -# Each option definition includes an option type, -# which controls whether the option takes an argument. -# -# File <tt>types.rb</tt>: -# -# :include: ../sample/getoptlong/types.rb -# -# Note that an option type has to do with the <em>option argument</em> -# (whether it is required, optional, or forbidden), -# not with whether the option itself is required. -# -# ==== Option with Required Argument -# -# An option of type <tt>GetoptLong::REQUIRED_ARGUMENT</tt> -# must be followed by an argument, which is associated with that option: -# -# $ ruby types.rb --xxx foo -# -# Output: -# -# ["--xxx", "foo"] -# -# If the option is not last, its argument is whatever follows it -# (even if the argument looks like another option): -# -# $ ruby types.rb --xxx --yyy -# -# Output: -# -# ["--xxx", "--yyy"] -# -# If the option is last, an exception is raised: -# -# $ ruby types.rb -# # Raises GetoptLong::MissingArgument -# -# ==== Option with Optional Argument -# -# An option of type <tt>GetoptLong::OPTIONAL_ARGUMENT</tt> -# may be followed by an argument, which if given is associated with that option. -# -# If the option is last, it does not have an argument: -# -# $ ruby types.rb --yyy -# -# Output: -# -# ["--yyy", ""] -# -# If the option is followed by another option, it does not have an argument: -# -# $ ruby types.rb --yyy --zzz -# -# Output: -# -# ["--yyy", ""] -# ["--zzz", ""] -# -# Otherwise the option is followed by its argument, which is associated -# with that option: -# -# $ ruby types.rb --yyy foo -# -# Output: -# -# ["--yyy", "foo"] -# -# ==== Option with No Argument -# -# An option of type <tt>GetoptLong::NO_ARGUMENT</tt> takes no argument: -# -# ruby types.rb --zzz foo -# -# Output: -# -# ["--zzz", ""] -# -# === ARGV -# -# You can process options either with method #each and a block, -# or with method #get. -# -# During processing, each found option is removed, along with its argument -# if there is one. -# After processing, each remaining element was neither an option -# nor the argument for an option. -# -# File <tt>argv.rb</tt>: -# -# :include: ../sample/getoptlong/argv.rb -# -# Command line: -# -# $ ruby argv.rb --xxx Foo --yyy Bar Baz --zzz Bat Bam -# -# Output: -# -# Original ARGV: ["--xxx", "Foo", "--yyy", "Bar", "Baz", "--zzz", "Bat", "Bam"] -# ["--xxx", "Foo"] -# ["--yyy", "Bar"] -# ["--zzz", ""] -# Remaining ARGV: ["Baz", "Bat", "Bam"] -# -# === Ordering -# -# There are three settings that control the way the options -# are interpreted: -# -# - +PERMUTE+. -# - +REQUIRE_ORDER+. -# - +RETURN_IN_ORDER+. -# -# The initial setting for a new \GetoptLong object is +REQUIRE_ORDER+ -# if environment variable +POSIXLY_CORRECT+ is defined, +PERMUTE+ otherwise. -# -# ==== PERMUTE Ordering -# -# In the +PERMUTE+ ordering, options and other, non-option, -# arguments may appear in any order and any mixture. -# -# File <tt>permute.rb</tt>: -# -# :include: ../sample/getoptlong/permute.rb -# -# Command line: -# -# $ ruby permute.rb Foo --zzz Bar --xxx Baz --yyy Bat Bam --xxx Bag Bah -# -# Output: -# -# Original ARGV: ["Foo", "--zzz", "Bar", "--xxx", "Baz", "--yyy", "Bat", "Bam", "--xxx", "Bag", "Bah"] -# ["--zzz", ""] -# ["--xxx", "Baz"] -# ["--yyy", "Bat"] -# ["--xxx", "Bag"] -# Remaining ARGV: ["Foo", "Bar", "Bam", "Bah"] -# -# ==== REQUIRE_ORDER Ordering -# -# In the +REQUIRE_ORDER+ ordering, all options precede all non-options; -# that is, each word after the first non-option word -# is treated as a non-option word (even if it begins with a hyphen). -# -# File <tt>require_order.rb</tt>: -# -# :include: ../sample/getoptlong/require_order.rb -# -# Command line: -# -# $ ruby require_order.rb --xxx Foo Bar --xxx Baz --yyy Bat -zzz -# -# Output: -# -# Original ARGV: ["--xxx", "Foo", "Bar", "--xxx", "Baz", "--yyy", "Bat", "-zzz"] -# ["--xxx", "Foo"] -# Remaining ARGV: ["Bar", "--xxx", "Baz", "--yyy", "Bat", "-zzz"] -# -# ==== RETURN_IN_ORDER Ordering -# -# In the +RETURN_IN_ORDER+ ordering, every word is treated as an option. -# A word that begins with a hyphen (or two) is treated in the usual way; -# a word +word+ that does not so begin is treated as an option -# whose name is an empty string, and whose value is +word+. -# -# File <tt>return_in_order.rb</tt>: -# -# :include: ../sample/getoptlong/return_in_order.rb -# -# Command line: -# -# $ ruby return_in_order.rb Foo --xxx Bar Baz --zzz Bat Bam -# -# Output: -# -# Original ARGV: ["Foo", "--xxx", "Bar", "Baz", "--zzz", "Bat", "Bam"] -# ["", "Foo"] -# ["--xxx", "Bar"] -# ["", "Baz"] -# ["--zzz", ""] -# ["", "Bat"] -# ["", "Bam"] -# Remaining ARGV: [] -# -# === Full Example -# -# File <tt>fibonacci.rb</tt>: -# -# :include: ../sample/getoptlong/fibonacci.rb -# -# Command line: -# -# $ ruby fibonacci.rb -# -# Output: -# -# Option --number is required. -# Usage: -# -# -n n, --number n: -# Compute Fibonacci number for n. -# -v [boolean], --verbose [boolean]: -# Show intermediate results; default is 'false'. -# -h, --help: -# Show this help. -# -# Command line: -# -# $ ruby fibonacci.rb --number -# -# Raises GetoptLong::MissingArgument: -# -# fibonacci.rb: option `--number' requires an argument -# -# Command line: -# -# $ ruby fibonacci.rb --number 6 -# -# Output: -# -# 8 -# -# Command line: -# -# $ ruby fibonacci.rb --number 6 --verbose -# -# Output: -# 1 -# 2 -# 3 -# 5 -# 8 -# -# Command line: -# -# $ ruby fibonacci.rb --number 6 --verbose yes -# -# Output: -# -# --verbose argument must be true or false -# Usage: -# -# -n n, --number n: -# Compute Fibonacci number for n. -# -v [boolean], --verbose [boolean]: -# Show intermediate results; default is 'false'. -# -h, --help: -# Show this help. -# -class GetoptLong - # Version. - VERSION = "0.2.0" - - # - # Orderings. - # - ORDERINGS = [REQUIRE_ORDER = 0, PERMUTE = 1, RETURN_IN_ORDER = 2] - - # - # Argument flags. - # - ARGUMENT_FLAGS = [NO_ARGUMENT = 0, REQUIRED_ARGUMENT = 1, - OPTIONAL_ARGUMENT = 2] - - # - # Status codes. - # - STATUS_YET, STATUS_STARTED, STATUS_TERMINATED = 0, 1, 2 - - # - # Error types. - # - class Error < StandardError; end - class AmbiguousOption < Error; end - class NeedlessArgument < Error; end - class MissingArgument < Error; end - class InvalidOption < Error; end - - # - # Returns a new \GetoptLong object based on the given +arguments+. - # See {Options}[#class-GetoptLong-label-Options]. - # - # Example: - # - # :include: ../sample/getoptlong/simple.rb - # - # Raises an exception if: - # - # - Any of +arguments+ is not an array. - # - Any option name or alias is not a string. - # - Any option type is invalid. - # - def initialize(*arguments) - # - # Current ordering. - # - if ENV.include?('POSIXLY_CORRECT') - @ordering = REQUIRE_ORDER - else - @ordering = PERMUTE - end - - # - # Hash table of option names. - # Keys of the table are option names, and their values are canonical - # names of the options. - # - @canonical_names = Hash.new - - # - # Hash table of argument flags. - # Keys of the table are option names, and their values are argument - # flags of the options. - # - @argument_flags = Hash.new - - # - # Whether error messages are output to $stderr. - # - @quiet = false - - # - # Status code. - # - @status = STATUS_YET - - # - # Error code. - # - @error = nil - - # - # Error message. - # - @error_message = nil - - # - # Rest of catenated short options. - # - @rest_singles = '' - - # - # List of non-option-arguments. - # Append them to ARGV when option processing is terminated. - # - @non_option_arguments = Array.new - - if 0 < arguments.length - set_options(*arguments) - end - end - - # Sets the ordering; see {Ordering}[#class-GetoptLong-label-Ordering]; - # returns the new ordering. - # - # If the given +ordering+ is +PERMUTE+ and environment variable - # +POSIXLY_CORRECT+ is defined, sets the ordering to +REQUIRE_ORDER+; - # otherwise sets the ordering to +ordering+: - # - # options = GetoptLong.new - # options.ordering == GetoptLong::PERMUTE # => true - # options.ordering = GetoptLong::RETURN_IN_ORDER - # options.ordering == GetoptLong::RETURN_IN_ORDER # => true - # ENV['POSIXLY_CORRECT'] = 'true' - # options.ordering = GetoptLong::PERMUTE - # options.ordering == GetoptLong::REQUIRE_ORDER # => true - # - # Raises an exception if +ordering+ is invalid. - # - def ordering=(ordering) - # - # The method is failed if option processing has already started. - # - if @status != STATUS_YET - set_error(ArgumentError, "argument error") - raise RuntimeError, - "invoke ordering=, but option processing has already started" - end - - # - # Check ordering. - # - if !ORDERINGS.include?(ordering) - raise ArgumentError, "invalid ordering `#{ordering}'" - end - if ordering == PERMUTE && ENV.include?('POSIXLY_CORRECT') - @ordering = REQUIRE_ORDER - else - @ordering = ordering - end - end - - # - # Returns the ordering setting. - # - attr_reader :ordering - - # - # Replaces existing options with those given by +arguments+, - # which have the same form as the arguments to ::new; - # returns +self+. - # - # Raises an exception if option processing has begun. - # - def set_options(*arguments) - # - # The method is failed if option processing has already started. - # - if @status != STATUS_YET - raise RuntimeError, - "invoke set_options, but option processing has already started" - end - - # - # Clear tables of option names and argument flags. - # - @canonical_names.clear - @argument_flags.clear - - arguments.each do |arg| - if !arg.is_a?(Array) - raise ArgumentError, "the option list contains non-Array argument" - end - - # - # Find an argument flag and it set to `argument_flag'. - # - argument_flag = nil - arg.each do |i| - if ARGUMENT_FLAGS.include?(i) - if argument_flag != nil - raise ArgumentError, "too many argument-flags" - end - argument_flag = i - end - end - - raise ArgumentError, "no argument-flag" if argument_flag == nil - - canonical_name = nil - arg.each do |i| - # - # Check an option name. - # - next if i == argument_flag - begin - if !i.is_a?(String) || i !~ /\A-([^-]|-.+)\z/ - raise ArgumentError, "an invalid option `#{i}'" - end - if (@canonical_names.include?(i)) - raise ArgumentError, "option redefined `#{i}'" - end - rescue - @canonical_names.clear - @argument_flags.clear - raise - end - - # - # Register the option (`i') to the `@canonical_names' and - # `@canonical_names' Hashes. - # - if canonical_name == nil - canonical_name = i - end - @canonical_names[i] = canonical_name - @argument_flags[i] = argument_flag - end - raise ArgumentError, "no option name" if canonical_name == nil - end - return self - end - - # - # Sets quiet mode and returns the given argument: - # - # - When +false+ or +nil+, error messages are written to <tt>$stdout</tt>. - # - Otherwise, error messages are not written. - # - attr_writer :quiet - - # - # Returns the quiet mode setting. - # - attr_reader :quiet - alias quiet? quiet - - # - # Terminate option processing; - # returns +nil+ if processing has already terminated; - # otherwise returns +self+. - # - def terminate - return nil if @status == STATUS_TERMINATED - raise RuntimeError, "an error has occurred" if @error != nil - - @status = STATUS_TERMINATED - @non_option_arguments.reverse_each do |argument| - ARGV.unshift(argument) - end - - @canonical_names = nil - @argument_flags = nil - @rest_singles = nil - @non_option_arguments = nil - - return self - end - - # - # Returns +true+ if option processing has terminated, +false+ otherwise. - # - def terminated? - return @status == STATUS_TERMINATED - end - - # - # \Set an error (a protected method). - # - def set_error(type, message) - $stderr.print("#{$0}: #{message}\n") if !@quiet - - @error = type - @error_message = message - @canonical_names = nil - @argument_flags = nil - @rest_singles = nil - @non_option_arguments = nil - - raise type, message - end - protected :set_error - - # - # Returns whether option processing has failed. - # - attr_reader :error - alias error? error - - # Return the appropriate error message in POSIX-defined format. - # If no error has occurred, returns +nil+. - # - def error_message - return @error_message - end - - # - # Returns the next option as a 2-element array containing: - # - # - The option name (the name itself, not an alias). - # - The option value. - # - # Returns +nil+ if there are no more options. - # - def get - option_name, option_argument = nil, '' - - # - # Check status. - # - return nil if @error != nil - case @status - when STATUS_YET - @status = STATUS_STARTED - when STATUS_TERMINATED - return nil - end - - # - # Get next option argument. - # - if 0 < @rest_singles.length - argument = '-' + @rest_singles - elsif (ARGV.length == 0) - terminate - return nil - elsif @ordering == PERMUTE - while 0 < ARGV.length && ARGV[0] !~ /\A-./ - @non_option_arguments.push(ARGV.shift) - end - if ARGV.length == 0 - terminate - return nil - end - argument = ARGV.shift - elsif @ordering == REQUIRE_ORDER - if (ARGV[0] !~ /\A-./) - terminate - return nil - end - argument = ARGV.shift - else - argument = ARGV.shift - end - - # - # Check the special argument `--'. - # `--' indicates the end of the option list. - # - if argument == '--' && @rest_singles.length == 0 - terminate - return nil - end - - # - # Check for long and short options. - # - if argument =~ /\A(--[^=]+)/ && @rest_singles.length == 0 - # - # This is a long style option, which start with `--'. - # - pattern = $1 - if @canonical_names.include?(pattern) - option_name = pattern - else - # - # The option `option_name' is not registered in `@canonical_names'. - # It may be an abbreviated. - # - matches = [] - @canonical_names.each_key do |key| - if key.index(pattern) == 0 - option_name = key - matches << key - end - end - if 2 <= matches.length - set_error(AmbiguousOption, "option `#{argument}' is ambiguous between #{matches.join(', ')}") - elsif matches.length == 0 - set_error(InvalidOption, "unrecognized option `#{argument}'") - end - end - - # - # Check an argument to the option. - # - if @argument_flags[option_name] == REQUIRED_ARGUMENT - if argument =~ /=(.*)/m - option_argument = $1 - elsif 0 < ARGV.length - option_argument = ARGV.shift - else - set_error(MissingArgument, - "option `#{argument}' requires an argument") - end - elsif @argument_flags[option_name] == OPTIONAL_ARGUMENT - if argument =~ /=(.*)/m - option_argument = $1 - elsif 0 < ARGV.length && ARGV[0] !~ /\A-./ - option_argument = ARGV.shift - else - option_argument = '' - end - elsif argument =~ /=(.*)/m - set_error(NeedlessArgument, - "option `#{option_name}' doesn't allow an argument") - end - - elsif argument =~ /\A(-(.))(.*)/m - # - # This is a short style option, which start with `-' (not `--'). - # Short options may be catenated (e.g. `-l -g' is equivalent to - # `-lg'). - # - option_name, ch, @rest_singles = $1, $2, $3 - - if @canonical_names.include?(option_name) - # - # The option `option_name' is found in `@canonical_names'. - # Check its argument. - # - if @argument_flags[option_name] == REQUIRED_ARGUMENT - if 0 < @rest_singles.length - option_argument = @rest_singles - @rest_singles = '' - elsif 0 < ARGV.length - option_argument = ARGV.shift - else - # 1003.2 specifies the format of this message. - set_error(MissingArgument, "option requires an argument -- #{ch}") - end - elsif @argument_flags[option_name] == OPTIONAL_ARGUMENT - if 0 < @rest_singles.length - option_argument = @rest_singles - @rest_singles = '' - elsif 0 < ARGV.length && ARGV[0] !~ /\A-./ - option_argument = ARGV.shift - else - option_argument = '' - end - end - else - # - # This is an invalid option. - # 1003.2 specifies the format of this message. - # - if ENV.include?('POSIXLY_CORRECT') - set_error(InvalidOption, "invalid option -- #{ch}") - else - set_error(InvalidOption, "invalid option -- #{ch}") - end - end - else - # - # This is a non-option argument. - # Only RETURN_IN_ORDER fell into here. - # - return '', argument - end - - return @canonical_names[option_name], option_argument - end - alias get_option get - - # - # Calls the given block with each option; - # each option is a 2-element array containing: - # - # - The option name (the name itself, not an alias). - # - The option value. - # - # Example: - # - # :include: ../sample/getoptlong/each.rb - # - # Command line: - # - # ruby each.rb -xxx Foo -x Bar --yyy Baz -y Bat --zzz - # - # Output: - # - # Original ARGV: ["-xxx", "Foo", "-x", "Bar", "--yyy", "Baz", "-y", "Bat", "--zzz"] - # ["--xxx", "xx"] - # ["--xxx", "Bar"] - # ["--yyy", "Baz"] - # ["--yyy", "Bat"] - # ["--zzz", ""] - # Remaining ARGV: ["Foo"] - # - def each - loop do - option_name, option_argument = get_option - break if option_name == nil - yield option_name, option_argument - end - end - alias each_option each -end diff --git a/lib/ipaddr.rb b/lib/ipaddr.rb index 7a5cf94830..dbb213c90a 100644 --- a/lib/ipaddr.rb +++ b/lib/ipaddr.rb @@ -40,7 +40,7 @@ require 'socket' # p ipaddr3 #=> #<IPAddr: IPv4:192.168.2.0/255.255.255.0> class IPAddr - VERSION = "1.2.5" + VERSION = "1.2.6" # 32 bit mask for IPv4 IN4MASK = 0xffffffff @@ -52,7 +52,7 @@ class IPAddr # Regexp _internally_ used for parsing IPv4 address. RE_IPV4ADDRLIKE = %r{ \A - (\d+) \. (\d+) \. (\d+) \. (\d+) + \d+ \. \d+ \. \d+ \. \d+ \z }x @@ -110,8 +110,13 @@ class IPAddr # Convert a network byte ordered string form of an IP address into # human readable form. + # It expects the string to be encoded in Encoding::ASCII_8BIT (BINARY). def self.ntop(addr) - case addr.size + if addr.is_a?(String) && addr.encoding != Encoding::BINARY + raise InvalidAddressError, "invalid encoding (given #{addr.encoding}, expected BINARY)" + end + + case addr.bytesize when 4 addr.unpack('C4').join('.') when 16 @@ -176,9 +181,7 @@ class IPAddr def include?(other) other = coerce_other(other) return false unless other.family == family - range = to_range - other = other.to_range - range.begin <= other.begin && range.end >= other.end + begin_addr <= other.begin_addr && end_addr >= other.end_addr end alias === include? @@ -224,6 +227,12 @@ class IPAddr return str end + # Returns a string containing the IP address representation in + # cidr notation + def cidr + format("%s/%s", to_s, prefix) + end + # Returns a network byte ordered string form of the IP address. def hton case @family @@ -249,12 +258,17 @@ class IPAddr end # Returns true if the ipaddr is a loopback address. + # Loopback IPv4 addresses in the IPv4-mapped IPv6 + # address range are also considered as loopback addresses. def loopback? case @family when Socket::AF_INET - @addr & 0xff000000 == 0x7f000000 + @addr & 0xff000000 == 0x7f000000 # 127.0.0.1/8 when Socket::AF_INET6 - @addr == 1 + @addr == 1 || # ::1 + (@addr & 0xffff_0000_0000 == 0xffff_0000_0000 && ( + @addr & 0xff000000 == 0x7f000000 # ::ffff:127.0.0.1/8 + )) else raise AddressFamilyError, "unsupported address family" end @@ -263,7 +277,8 @@ class IPAddr # Returns true if the ipaddr is a private address. IPv4 addresses # in 10.0.0.0/8, 172.16.0.0/12 and 192.168.0.0/16 as defined in RFC # 1918 and IPv6 Unique Local Addresses in fc00::/7 as defined in RFC - # 4193 are considered private. + # 4193 are considered private. Private IPv4 addresses in the + # IPv4-mapped IPv6 address range are also considered private. def private? case @family when Socket::AF_INET @@ -271,22 +286,31 @@ class IPAddr @addr & 0xfff00000 == 0xac100000 || # 172.16.0.0/12 @addr & 0xffff0000 == 0xc0a80000 # 192.168.0.0/16 when Socket::AF_INET6 - @addr & 0xfe00_0000_0000_0000_0000_0000_0000_0000 == 0xfc00_0000_0000_0000_0000_0000_0000_0000 + @addr & 0xfe00_0000_0000_0000_0000_0000_0000_0000 == 0xfc00_0000_0000_0000_0000_0000_0000_0000 || + (@addr & 0xffff_0000_0000 == 0xffff_0000_0000 && ( + @addr & 0xff000000 == 0x0a000000 || # ::ffff:10.0.0.0/8 + @addr & 0xfff00000 == 0xac100000 || # ::ffff::172.16.0.0/12 + @addr & 0xffff0000 == 0xc0a80000 # ::ffff::192.168.0.0/16 + )) else raise AddressFamilyError, "unsupported address family" end end # Returns true if the ipaddr is a link-local address. IPv4 - # addresses in 169.254.0.0/16 reserved by RFC 3927 and Link-Local + # addresses in 169.254.0.0/16 reserved by RFC 3927 and link-local # IPv6 Unicast Addresses in fe80::/10 reserved by RFC 4291 are - # considered link-local. + # considered link-local. Link-local IPv4 addresses in the + # IPv4-mapped IPv6 address range are also considered link-local. def link_local? case @family when Socket::AF_INET @addr & 0xffff0000 == 0xa9fe0000 # 169.254.0.0/16 when Socket::AF_INET6 - @addr & 0xffc0_0000_0000_0000_0000_0000_0000_0000 == 0xfe80_0000_0000_0000_0000_0000_0000_0000 + @addr & 0xffc0_0000_0000_0000_0000_0000_0000_0000 == 0xfe80_0000_0000_0000_0000_0000_0000_0000 || # fe80::/10 + (@addr & 0xffff_0000_0000 == 0xffff_0000_0000 && ( + @addr & 0xffff0000 == 0xa9fe0000 # ::ffff:169.254.0.0/16 + )) else raise AddressFamilyError, "unsupported address family" end @@ -400,17 +424,6 @@ class IPAddr # Creates a Range object for the network address. def to_range - begin_addr = (@addr & @mask_addr) - - case @family - when Socket::AF_INET - end_addr = (@addr | (IN4MASK ^ @mask_addr)) - when Socket::AF_INET6 - end_addr = (@addr | (IN6MASK ^ @mask_addr)) - else - raise AddressFamilyError, "unsupported address family" - end - self.class.new(begin_addr, @family)..self.class.new(end_addr, @family) end @@ -439,7 +452,7 @@ class IPAddr when Integer mask!(prefix) else - raise InvalidPrefixError, "prefix must be an integer: #{@addr}" + raise InvalidPrefixError, "prefix must be an integer" end end @@ -464,6 +477,20 @@ class IPAddr _to_string(@mask_addr) end + # Returns the wildcard mask in string format e.g. 0.0.255.255 + def wildcard_mask + case @family + when Socket::AF_INET + mask = IN4MASK ^ @mask_addr + when Socket::AF_INET6 + mask = IN6MASK ^ @mask_addr + else + raise AddressFamilyError, "unsupported address family" + end + + _to_string(mask) + end + # Returns the IPv6 zone identifier, if present. # Raises InvalidAddressError if not an IPv6 address. def zone_id @@ -491,6 +518,21 @@ class IPAddr protected + def begin_addr + @addr & @mask_addr + end + + def end_addr + case @family + when Socket::AF_INET + @addr | (IN4MASK ^ @mask_addr) + when Socket::AF_INET6 + @addr | (IN6MASK ^ @mask_addr) + else + raise AddressFamilyError, "unsupported address family" + end + end + # Set +@addr+, the internal stored ip address, to given +addr+. The # parameter +addr+ is validated using the first +family+ member, # which is +Socket::AF_INET+ or +Socket::AF_INET6+. @@ -498,11 +540,11 @@ class IPAddr case family[0] ? family[0] : @family when Socket::AF_INET if addr < 0 || addr > IN4MASK - raise InvalidAddressError, "invalid address: #{@addr}" + raise InvalidAddressError, "invalid address: #{addr}" end when Socket::AF_INET6 if addr < 0 || addr > IN6MASK - raise InvalidAddressError, "invalid address: #{@addr}" + raise InvalidAddressError, "invalid address: #{addr}" end else raise AddressFamilyError, "unsupported address family" @@ -529,12 +571,12 @@ class IPAddr else m = IPAddr.new(mask) if m.family != @family - raise InvalidPrefixError, "address family is not same: #{@addr}" + raise InvalidPrefixError, "address family is not same" end @mask_addr = m.to_i n = @mask_addr ^ m.instance_variable_get(:@mask_addr) unless ((n + 1) & n).zero? - raise InvalidPrefixError, "invalid mask #{mask}: #{@addr}" + raise InvalidPrefixError, "invalid mask #{mask}" end @addr &= @mask_addr return self @@ -545,13 +587,13 @@ class IPAddr case @family when Socket::AF_INET if prefixlen < 0 || prefixlen > 32 - raise InvalidPrefixError, "invalid length: #{@addr}" + raise InvalidPrefixError, "invalid length" end masklen = 32 - prefixlen @mask_addr = ((IN4MASK >> masklen) << masklen) when Socket::AF_INET6 if prefixlen < 0 || prefixlen > 128 - raise InvalidPrefixError, "invalid length: #{@addr}" + raise InvalidPrefixError, "invalid length" end masklen = 128 - prefixlen @mask_addr = ((IN6MASK >> masklen) << masklen) @@ -647,12 +689,12 @@ class IPAddr when Array octets = addr else - m = RE_IPV4ADDRLIKE.match(addr) or return nil - octets = m.captures + RE_IPV4ADDRLIKE.match?(addr) or return nil + octets = addr.split('.') end octets.inject(0) { |i, s| (n = s.to_i) < 256 or raise InvalidAddressError, "invalid address: #{@addr}" - s.match(/\A0./) and raise InvalidAddressError, "zero-filled number in IPv4 address is ambiguous: #{@addr}" + (s != '0') && s.start_with?('0') and raise InvalidAddressError, "zero-filled number in IPv4 address is ambiguous: #{@addr}" i << 8 | n } end diff --git a/lib/irb.rb b/lib/irb.rb index 26432a0fac..b3435c257e 100644 --- a/lib/irb.rb +++ b/lib/irb.rb @@ -1,5 +1,6 @@ -# frozen_string_literal: false -# +# frozen_string_literal: true + +# :markup: markdown # irb.rb - irb main module # by Keiju ISHITSUKA(keiju@ruby-lang.org) # @@ -9,399 +10,886 @@ require "reline" require_relative "irb/init" require_relative "irb/context" -require_relative "irb/extend-command" +require_relative "irb/default_commands" require_relative "irb/ruby-lex" +require_relative "irb/statement" require_relative "irb/input-method" require_relative "irb/locale" require_relative "irb/color" require_relative "irb/version" require_relative "irb/easter-egg" +require_relative "irb/debug" +require_relative "irb/pager" -# IRB stands for "interactive Ruby" and is a tool to interactively execute Ruby -# expressions read from the standard input. +# ## IRB # -# The +irb+ command from your shell will start the interpreter. +# Module IRB ("Interactive Ruby") provides a shell-like interface that supports +# user interaction with the Ruby interpreter. # -# == Usage +# It operates as a *read-eval-print loop* +# ([REPL](https://en.wikipedia.org/wiki/Read%E2%80%93eval%E2%80%93print_loop)) +# that: # -# Use of irb is easy if you know Ruby. +# * ***Reads*** each character as you type. You can modify the IRB context to +# change the way input works. See [Input](rdoc-ref:IRB@Input). +# * ***Evaluates*** the code each time it has read a syntactically complete +# passage. +# * ***Prints*** after evaluating. You can modify the IRB context to change +# the way output works. See [Output](rdoc-ref:IRB@Output). # -# When executing irb, prompts are displayed as follows. Then, enter the Ruby -# expression. An input is executed when it is syntactically complete. +# +# Example: # # $ irb -# irb(main):001:0> 1+2 -# #=> 3 -# irb(main):002:0> class Foo -# irb(main):003:1> def foo -# irb(main):004:2> print 1 -# irb(main):005:2> end -# irb(main):006:1> end -# #=> nil -# -# The singleline editor module or multiline editor module can be used with irb. -# Use of multiline editor is default if it's installed. -# -# == Command line options -# -# :include: ./irb/lc/help-message -# -# == Commands -# -# The following commands are available on IRB. -# -# * cwws -# * Show the current workspace. -# * cb, cws, chws -# * Change the current workspace to an object. -# * bindings, workspaces -# * Show workspaces. -# * pushb, pushws -# * Push an object to the workspace stack. -# * popb, popws -# * Pop a workspace from the workspace stack. -# * load -# * Load a Ruby file. -# * require -# * Require a Ruby file. -# * source -# * Loads a given file in the current session. -# * irb -# * Start a child IRB. -# * jobs -# * List of current sessions. -# * fg -# * Switches to the session of the given number. -# * kill -# * Kills the session with the given number. -# * help -# * Enter the mode to look up RI documents. -# * irb_info -# * Show information about IRB. -# * ls -# * Show methods, constants, and variables. -# -g [query] or -G [query] allows you to filter out the output. -# * measure -# * measure enables the mode to measure processing time. measure :off disables it. -# * $, show_source -# * Show the source code of a given method or constant. -# * @, whereami -# * Show the source code around binding.irb again. -# * debug -# * Start the debugger of debug.gem. -# * break, delete, next, step, continue, finish, backtrace, info, catch -# * Start the debugger of debug.gem and run the command on it. -# -# == Configuration -# -# IRB reads a personal initialization file when it's invoked. -# IRB searches a file in the following order and loads the first one found. -# -# * <tt>$IRBRC</tt> (if <tt>$IRBRC</tt> is set) -# * <tt>$XDG_CONFIG_HOME/irb/irbrc</tt> (if <tt>$XDG_CONFIG_HOME</tt> is set) -# * <tt>~/.irbrc</tt> -# * +.config/irb/irbrc+ -# * +.irbrc+ -# * +irb.rc+ -# * +_irbrc+ -# * <code>$irbrc</code> -# -# The following are alternatives to the command line options. To use them type -# as follows in an +irb+ session: -# -# IRB.conf[:IRB_NAME]="irb" -# IRB.conf[:INSPECT_MODE]=nil -# IRB.conf[:IRB_RC] = nil -# IRB.conf[:BACK_TRACE_LIMIT]=16 -# IRB.conf[:USE_LOADER] = false -# IRB.conf[:USE_MULTILINE] = nil -# IRB.conf[:USE_SINGLELINE] = nil -# IRB.conf[:USE_COLORIZE] = true -# IRB.conf[:USE_TRACER] = false -# IRB.conf[:USE_AUTOCOMPLETE] = true -# IRB.conf[:IGNORE_SIGINT] = true -# IRB.conf[:IGNORE_EOF] = false -# IRB.conf[:PROMPT_MODE] = :DEFAULT -# IRB.conf[:PROMPT] = {...} -# -# === Auto indentation -# -# To disable auto-indent mode in irb, add the following to your +.irbrc+: +# irb(main):001> File.basename(Dir.pwd) +# => "irb" +# irb(main):002> Dir.entries('.').size +# => 25 +# irb(main):003* Dir.entries('.').select do |entry| +# irb(main):004* entry.start_with?('R') +# irb(main):005> end +# => ["README.md", "Rakefile"] +# +# The typed input may also include [\IRB-specific +# commands](rdoc-ref:IRB@IRB-Specific+Commands). +# +# As seen above, you can start IRB by using the shell command `irb`. +# +# You can stop an IRB session by typing command `exit`: +# +# irb(main):006> exit +# $ +# +# At that point, IRB calls any hooks found in array `IRB.conf[:AT_EXIT]`, then +# exits. +# +# ## Startup +# +# At startup, IRB: +# +# 1. Interprets (as Ruby code) the content of the [configuration +# file](rdoc-ref:IRB@Configuration+File) (if given). +# 2. Constructs the initial session context from [hash +# IRB.conf](rdoc-ref:IRB@Hash+IRB.conf) and from default values; the hash +# content may have been affected by [command-line +# options](rdoc-ref:IB@Command-Line+Options), and by direct assignments in +# the configuration file. +# 3. Assigns the context to variable `conf`. +# 4. Assigns command-line arguments to variable `ARGV`. +# 5. Prints the [prompt](rdoc-ref:IRB@Prompt+and+Return+Formats). +# 6. Puts the content of the [initialization +# script](rdoc-ref:IRB@Initialization+Script) onto the IRB shell, just as if +# it were user-typed commands. +# +# +# ### The Command Line +# +# On the command line, all options precede all arguments; the first item that is +# not recognized as an option is treated as an argument, as are all items that +# follow. +# +# #### Command-Line Options +# +# Many command-line options affect entries in hash `IRB.conf`, which in turn +# affect the initial configuration of the IRB session. +# +# Details of the options are described in the relevant subsections below. +# +# A cursory list of the IRB command-line options may be seen in the [help +# message](https://raw.githubusercontent.com/ruby/irb/master/lib/irb/lc/help-message), +# which is also displayed if you use command-line option `--help`. +# +# If you are interested in a specific option, consult the +# [index](rdoc-ref:doc/irb/indexes.md@Index+of+Command-Line+Options). +# +# #### Command-Line Arguments +# +# Command-line arguments are passed to IRB in array `ARGV`: +# +# $ irb --noscript Foo Bar Baz +# irb(main):001> ARGV +# => ["Foo", "Bar", "Baz"] +# irb(main):002> exit +# $ +# +# Command-line option `--` causes everything that follows to be treated as +# arguments, even those that look like options: +# +# $ irb --noscript -- --noscript -- Foo Bar Baz +# irb(main):001> ARGV +# => ["--noscript", "--", "Foo", "Bar", "Baz"] +# irb(main):002> exit +# $ +# +# ### Configuration File +# +# You can initialize IRB via a *configuration file*. +# +# If command-line option `-f` is given, no configuration file is looked for. +# +# Otherwise, IRB reads and interprets a configuration file if one is available. +# +# The configuration file can contain any Ruby code, and can usefully include +# user code that: +# +# * Can then be debugged in IRB. +# * Configures IRB itself. +# * Requires or loads files. +# +# +# The path to the configuration file is the first found among: +# +# * The value of variable `$IRBRC`, if defined. +# * The value of variable `$XDG_CONFIG_HOME/irb/irbrc`, if defined. +# * File `$HOME/.irbrc`, if it exists. +# * File `$HOME/.config/irb/irbrc`, if it exists. +# * File `.irbrc` in the current directory, if it exists. +# * File `irb.rc` in the current directory, if it exists. +# * File `_irbrc` in the current directory, if it exists. +# * File `$irbrc` in the current directory, if it exists. +# +# +# If the search fails, there is no configuration file. +# +# If the search succeeds, the configuration file is read as Ruby code, and so +# can contain any Ruby programming you like. +# +# Method `conf.rc?` returns `true` if a configuration file was read, `false` +# otherwise. Hash entry `IRB.conf[:RC]` also contains that value. +# +# ### Hash `IRB.conf` +# +# The initial entries in hash `IRB.conf` are determined by: +# +# * Default values. +# * Command-line options, which may override defaults. +# * Direct assignments in the configuration file. +# +# +# You can see the hash by typing `IRB.conf`. +# +# Details of the entries' meanings are described in the relevant subsections +# below. +# +# If you are interested in a specific entry, consult the +# [index](rdoc-ref:doc/irb/indexes.md@Index+of+IRB.conf+Entries). +# +# ### Notes on Initialization Precedence +# +# * Any conflict between an entry in hash `IRB.conf` and a command-line option +# is resolved in favor of the hash entry. +# * Hash `IRB.conf` affects the context only once, when the configuration file +# is interpreted; any subsequent changes to it do not affect the context and +# are therefore essentially meaningless. +# +# +# ### Initialization Script +# +# By default, the first command-line argument (after any options) is the path to +# a Ruby initialization script. +# +# IRB reads the initialization script and puts its content onto the IRB shell, +# just as if it were user-typed commands. +# +# Command-line option `--noscript` causes the first command-line argument to be +# treated as an ordinary argument (instead of an initialization script); +# `--script` is the default. +# +# ## Input +# +# This section describes the features that allow you to change the way IRB input +# works; see also [Input and Output](rdoc-ref:IRB@Input+and+Output). +# +# ### Input Command History +# +# By default, IRB stores a history of up to 1000 input commands in a file named +# `.irb_history`. The history file will be in the same directory as the +# [configuration file](rdoc-ref:IRB@Configuration+File) if one is found, or in +# `~/` otherwise. +# +# A new IRB session creates the history file if it does not exist, and appends +# to the file if it does exist. +# +# You can change the filepath by adding to your configuration file: +# `IRB.conf[:HISTORY_FILE] = *filepath*`, where *filepath* is a string filepath. +# +# During the session, method `conf.history_file` returns the filepath, and +# method `conf.history_file = *new_filepath*` copies the history to the file at +# *new_filepath*, which becomes the history file for the session. +# +# You can change the number of commands saved by adding to your configuration +# file: `IRB.conf[:SAVE_HISTORY] = *n*`, wheHISTORY_FILEre *n* is one of: +# +# * Positive integer: the number of commands to be saved, +# * Zero: all commands are to be saved. +# * `nil`: no commands are to be saved,. +# +# +# During the session, you can use methods `conf.save_history` or +# `conf.save_history=` to retrieve or change the count. +# +# ### Command Aliases +# +# By default, IRB defines several command aliases: +# +# irb(main):001> conf.command_aliases +# => {:"$"=>:show_source, :"@"=>:whereami} +# +# You can change the initial aliases in the configuration file with: +# +# IRB.conf[:COMMAND_ALIASES] = {foo: :show_source, bar: :whereami} +# +# You can replace the current aliases at any time with configuration method +# `conf.command_aliases=`; Because `conf.command_aliases` is a hash, you can +# modify it. +# +# ### End-of-File +# +# By default, `IRB.conf[:IGNORE_EOF]` is `false`, which means that typing the +# end-of-file character `Ctrl-D` causes the session to exit. +# +# You can reverse that behavior by adding `IRB.conf[:IGNORE_EOF] = true` to the +# configuration file. +# +# During the session, method `conf.ignore_eof?` returns the setting, and method +# `conf.ignore_eof = *boolean*` sets it. +# +# ### SIGINT +# +# By default, `IRB.conf[:IGNORE_SIGINT]` is `true`, which means that typing the +# interrupt character `Ctrl-C` causes the session to exit. +# +# You can reverse that behavior by adding `IRB.conf[:IGNORE_SIGING] = false` to +# the configuration file. +# +# During the session, method `conf.ignore_siging?` returns the setting, and +# method `conf.ignore_sigint = *boolean*` sets it. +# +# ### Automatic Completion +# +# By default, IRB enables [automatic +# completion](https://en.wikipedia.org/wiki/Autocomplete#In_command-line_interpr +# eters): +# +# You can disable it by either of these: +# +# * Adding `IRB.conf[:USE_AUTOCOMPLETE] = false` to the configuration file. +# * Giving command-line option `--noautocomplete` (`--autocomplete` is the +# default). +# +# +# Method `conf.use_autocomplete?` returns `true` if automatic completion is +# enabled, `false` otherwise. +# +# The setting may not be changed during the session. +# +# ### Automatic Indentation +# +# By default, IRB automatically indents lines of code to show structure (e.g., +# it indent the contents of a block). +# +# The current setting is returned by the configuration method +# `conf.auto_indent_mode`. +# +# The default initial setting is `true`: +# +# irb(main):001> conf.auto_indent_mode +# => true +# irb(main):002* Dir.entries('.').select do |entry| +# irb(main):003* entry.start_with?('R') +# irb(main):004> end +# => ["README.md", "Rakefile"] +# +# You can change the initial setting in the configuration file with: # # IRB.conf[:AUTO_INDENT] = false # -# === Autocompletion +# Note that the *current* setting *may not* be changed in the IRB session. +# +# ### Input Method +# +# The IRB input method determines how command input is to be read; by default, +# the input method for a session is IRB::RelineInputMethod. Unless the +# value of the TERM environment variable is 'dumb', in which case the +# most simplistic input method is used. +# +# You can set the input method by: +# +# * Adding to the configuration file: +# +# * `IRB.conf[:USE_SINGLELINE] = true` or `IRB.conf[:USE_MULTILINE]= +# false` sets the input method to IRB::ReadlineInputMethod. +# * `IRB.conf[:USE_SINGLELINE] = false` or `IRB.conf[:USE_MULTILINE] = +# true` sets the input method to IRB::RelineInputMethod. +# +# +# * Giving command-line options: +# +# * `--singleline` or `--nomultiline` sets the input method to +# IRB::ReadlineInputMethod. +# * `--nosingleline` or `--multiline` sets the input method to +# IRB::RelineInputMethod. +# * `--nosingleline` together with `--nomultiline` sets the +# input to IRB::StdioInputMethod. +# +# +# Method `conf.use_multiline?` and its synonym `conf.use_reline` return: +# +# * `true` if option `--multiline` was given. +# * `false` if option `--nomultiline` was given. +# * `nil` if neither was given. +# +# +# Method `conf.use_singleline?` and its synonym `conf.use_readline` return: +# +# * `true` if option `--singleline` was given. +# * `false` if option `--nosingleline` was given. +# * `nil` if neither was given. +# +# +# ## Output +# +# This section describes the features that allow you to change the way IRB +# output works; see also [Input and Output](rdoc-ref:IRB@Input+and+Output). +# +# ### Return-Value Printing (Echoing) +# +# By default, IRB prints (echoes) the values returned by all input commands. +# +# You can change the initial behavior and suppress all echoing by: +# +# * Adding to the configuration file: `IRB.conf[:ECHO] = false`. (The default +# value for this entry is `nil`, which means the same as `true`.) +# * Giving command-line option `--noecho`. (The default is `--echo`.) +# +# +# During the session, you can change the current setting with configuration +# method `conf.echo=` (set to `true` or `false`). +# +# As stated above, by default IRB prints the values returned by all input +# commands; but IRB offers special treatment for values returned by assignment +# statements, which may be: +# +# * Printed with truncation (to fit on a single line of output), which is the +# default; an ellipsis (`...` is suffixed, to indicate the truncation): +# +# irb(main):001> x = 'abc' * 100 +# +# +# > "abcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabcabc... +# +# * Printed in full (regardless of the length). +# * Suppressed (not printed at all) +# +# +# You can change the initial behavior by: +# +# * Adding to the configuration file: `IRB.conf[:ECHO_ON_ASSIGNMENT] = false`. +# (The default value for this entry is `niL`, which means the same as +# `:truncate`.) +# * Giving command-line option `--noecho-on-assignment` or +# `--echo-on-assignment`. (The default is `--truncate-echo-on-assignment`.) +# +# +# During the session, you can change the current setting with configuration +# method `conf.echo_on_assignment=` (set to `true`, `false`, or `:truncate`). +# +# By default, IRB formats returned values by calling method `inspect`. +# +# You can change the initial behavior by: # -# To disable autocompletion for irb, add the following to your +.irbrc+: +# * Adding to the configuration file: `IRB.conf[:INSPECT_MODE] = false`. (The +# default value for this entry is `true`.) +# * Giving command-line option `--noinspect`. (The default is `--inspect`.) # -# IRB.conf[:USE_AUTOCOMPLETE] = false # -# === History +# During the session, you can change the setting using method +# `conf.inspect_mode=`. # -# By default, irb will store the last 1000 commands you used in -# <code>IRB.conf[:HISTORY_FILE]</code> (<code>~/.irb_history</code> by default). +# ### Multiline Output # -# If you want to disable history, add the following to your +.irbrc+: +# By default, IRB prefixes a newline to a multiline response. # -# IRB.conf[:SAVE_HISTORY] = nil +# You can change the initial default value by adding to the configuration file: # -# See IRB::Context#save_history= for more information. +# IRB.conf[:NEWLINE_BEFORE_MULTILINE_OUTPUT] = false # -# The history of _results_ of commands evaluated is not stored by default, -# but can be turned on to be stored with this +.irbrc+ setting: +# During a session, you can retrieve or set the value using methods +# `conf.newline_before_multiline_output?` and +# `conf.newline_before_multiline_output=`. # -# IRB.conf[:EVAL_HISTORY] = <number> +# Examples: # -# See IRB::Context#eval_history= and History class. The history of command -# results is not permanently saved in any file. +# irb(main):001> conf.inspect_mode = false +# => false +# irb(main):002> "foo\nbar" +# => +# foo +# bar +# irb(main):003> conf.newline_before_multiline_output = false +# => false +# irb(main):004> "foo\nbar" +# => foo +# bar # -# == Customizing the IRB Prompt +# ### Evaluation History # -# In order to customize the prompt, you can change the following Hash: +# By default, IRB saves no history of evaluations (returned values), and the +# related methods `conf.eval_history`, `_`, and `__` are undefined. # -# IRB.conf[:PROMPT] +# You can turn on that history, and set the maximum number of evaluations to be +# stored: # -# This example can be used in your +.irbrc+ +# * In the configuration file: add `IRB.conf[:EVAL_HISTORY] = *n*`. (Examples +# below assume that we've added `IRB.conf[:EVAL_HISTORY] = 5`.) +# * In the session (at any time): `conf.eval_history = *n*`. # -# IRB.conf[:PROMPT][:MY_PROMPT] = { # name of prompt mode -# :AUTO_INDENT => false, # disables auto-indent mode -# :PROMPT_I => ">> ", # simple prompt -# :PROMPT_S => nil, # prompt for continuated strings -# :PROMPT_C => nil, # prompt for continuated statement -# :RETURN => " ==>%s\n" # format to return value -# } +# +# If `n` is zero, all evaluation history is stored. +# +# Doing either of the above: +# +# * Sets the maximum size of the evaluation history; defines method +# `conf.eval_history`, which returns the maximum size `n` of the evaluation +# history: +# +# irb(main):001> conf.eval_history = 5 +# => 5 +# irb(main):002> conf.eval_history +# => 5 +# +# * Defines variable `_`, which contains the most recent evaluation, or `nil` +# if none; same as method `conf.last_value`: +# +# irb(main):003> _ +# => 5 +# irb(main):004> :foo +# => :foo +# irb(main):005> :bar +# => :bar +# irb(main):006> _ +# => :bar +# irb(main):007> _ +# => :bar +# +# * Defines variable `__`: +# +# * `__` unadorned: contains all evaluation history: +# +# irb(main):008> :foo +# => :foo +# irb(main):009> :bar +# => :bar +# irb(main):010> :baz +# => :baz +# irb(main):011> :bat +# => :bat +# irb(main):012> :bam +# => :bam +# irb(main):013> __ +# => +# 9 :bar +# 10 :baz +# 11 :bat +# 12 :bam +# irb(main):014> __ +# => +# 10 :baz +# 11 :bat +# 12 :bam +# 13 ...self-history... +# +# Note that when the evaluation is multiline, it is displayed +# differently. +# +# * `__[`*m*`]`: +# +# * Positive *m*: contains the evaluation for the given line number, +# or `nil` if that line number is not in the evaluation history: +# +# irb(main):015> __[12] +# => :bam +# irb(main):016> __[1] +# => nil +# +# * Negative *m*: contains the `mth`-from-end evaluation, or `nil` if +# that evaluation is not in the evaluation history: +# +# irb(main):017> __[-3] +# => :bam +# irb(main):018> __[-13] +# => nil +# +# * Zero *m*: contains `nil`: +# +# irb(main):019> __[0] +# => nil +# +# +# +# +# ### Prompt and Return Formats +# +# By default, IRB uses the prompt and return value formats defined in its +# `:DEFAULT` prompt mode. +# +# #### The Default Prompt and Return Format +# +# The default prompt and return values look like this: +# +# irb(main):001> 1 + 1 +# => 2 +# irb(main):002> 2 + 2 +# => 4 +# +# The prompt includes: +# +# * The name of the running program (`irb`); see [IRB +# Name](rdoc-ref:IRB@IRB+Name). +# * The name of the current session (`main`); See [IRB +# Sessions](rdoc-ref:IRB@IRB+Sessions). +# * A 3-digit line number (1-based). +# +# +# The default prompt actually defines three formats: +# +# * One for most situations (as above): +# +# irb(main):003> Dir +# => Dir +# +# * One for when the typed command is a statement continuation (adds trailing +# asterisk): +# +# irb(main):004* Dir. +# +# * One for when the typed command is a string continuation (adds trailing +# single-quote): +# +# irb(main):005' Dir.entries('. +# +# +# You can see the prompt change as you type the characters in the following: +# +# irb(main):001* Dir.entries('.').select do |entry| +# irb(main):002* entry.start_with?('R') +# irb(main):003> end +# => ["README.md", "Rakefile"] +# +# #### Pre-Defined Prompts +# +# IRB has several pre-defined prompts, stored in hash `IRB.conf[:PROMPT]`: +# +# irb(main):001> IRB.conf[:PROMPT].keys +# => [:NULL, :DEFAULT, :CLASSIC, :SIMPLE, :INF_RUBY, :XMP] +# +# To see the full data for these, type `IRB.conf[:PROMPT]`. +# +# Most of these prompt definitions include specifiers that represent values like +# the IRB name, session name, and line number; see [Prompt +# Specifiers](rdoc-ref:IRB@Prompt+Specifiers). +# +# You can change the initial prompt and return format by: +# +# * Adding to the configuration file: `IRB.conf[:PROMPT] = *mode*` where +# *mode* is the symbol name of a prompt mode. +# * Giving a command-line option: +# +# * `--prompt *mode*`: sets the prompt mode to *mode*. where *mode* is the +# symbol name of a prompt mode. +# * `--simple-prompt` or `--sample-book-mode`: sets the prompt mode to +# `:SIMPLE`. +# * `--inf-ruby-mode`: sets the prompt mode to `:INF_RUBY` and suppresses +# both `--multiline` and `--singleline`. +# * `--noprompt`: suppresses prompting; does not affect echoing. +# +# +# +# You can retrieve or set the current prompt mode with methods +# +# `conf.prompt_mode` and `conf.prompt_mode=`. +# +# If you're interested in prompts and return formats other than the defaults, +# you might experiment by trying some of the others. +# +# #### Custom Prompts +# +# You can also define custom prompts and return formats, which may be done +# either in an IRB session or in the configuration file. +# +# A prompt in IRB actually defines three prompts, as seen above. For simple +# custom data, we'll make all three the same: +# +# irb(main):001* IRB.conf[:PROMPT][:MY_PROMPT] = { +# irb(main):002* PROMPT_I: ': ', +# irb(main):003* PROMPT_C: ': ', +# irb(main):004* PROMPT_S: ': ', +# irb(main):005* RETURN: '=> ' +# irb(main):006> } +# => {:PROMPT_I=>": ", :PROMPT_C=>": ", :PROMPT_S=>": ", :RETURN=>"=> "} +# +# If you define the custom prompt in the configuration file, you can also make +# it the current prompt by adding: # # IRB.conf[:PROMPT_MODE] = :MY_PROMPT # -# Or, invoke irb with the above prompt mode by: -# -# irb --prompt my-prompt -# -# Constants +PROMPT_I+, +PROMPT_S+ and +PROMPT_C+ specify the format. In the -# prompt specification, some special strings are available: -# -# %N # command name which is running -# %m # to_s of main object (self) -# %M # inspect of main object (self) -# %l # type of string(", ', /, ]), `]' is inner %w[...] -# %NNi # indent level. NN is digits and means as same as printf("%NNd"). -# # It can be omitted -# %NNn # line number. -# %% # % -# -# For instance, the default prompt mode is defined as follows: -# -# IRB.conf[:PROMPT_MODE][:DEFAULT] = { -# :PROMPT_I => "%N(%m):%03n:%i> ", -# :PROMPT_N => "%N(%m):%03n:%i> ", -# :PROMPT_S => "%N(%m):%03n:%i%l ", -# :PROMPT_C => "%N(%m):%03n:%i* ", -# :RETURN => "%s\n" # used to printf -# } -# -# irb comes with a number of available modes: -# -# # :NULL: -# # :PROMPT_I: -# # :PROMPT_N: -# # :PROMPT_S: -# # :PROMPT_C: -# # :RETURN: | -# # %s -# # :DEFAULT: -# # :PROMPT_I: ! '%N(%m):%03n:%i> ' -# # :PROMPT_N: ! '%N(%m):%03n:%i> ' -# # :PROMPT_S: ! '%N(%m):%03n:%i%l ' -# # :PROMPT_C: ! '%N(%m):%03n:%i* ' -# # :RETURN: | -# # => %s -# # :CLASSIC: -# # :PROMPT_I: ! '%N(%m):%03n:%i> ' -# # :PROMPT_N: ! '%N(%m):%03n:%i> ' -# # :PROMPT_S: ! '%N(%m):%03n:%i%l ' -# # :PROMPT_C: ! '%N(%m):%03n:%i* ' -# # :RETURN: | -# # %s -# # :SIMPLE: -# # :PROMPT_I: ! '>> ' -# # :PROMPT_N: ! '>> ' -# # :PROMPT_S: -# # :PROMPT_C: ! '?> ' -# # :RETURN: | -# # => %s -# # :INF_RUBY: -# # :PROMPT_I: ! '%N(%m):%03n:%i> ' -# # :PROMPT_N: -# # :PROMPT_S: -# # :PROMPT_C: -# # :RETURN: | -# # %s -# # :AUTO_INDENT: true -# # :XMP: -# # :PROMPT_I: -# # :PROMPT_N: -# # :PROMPT_S: -# # :PROMPT_C: -# # :RETURN: |2 -# # ==>%s -# -# == Restrictions -# -# Because irb evaluates input immediately after it is syntactically complete, -# the results may be slightly different than directly using Ruby. -# -# == IRB Sessions +# Regardless of where it's defined, you can make it the current prompt in a +# session: # -# IRB has a special feature, that allows you to manage many sessions at once. +# conf.prompt_mode = :MY_PROMPT # -# You can create new sessions with Irb.irb, and get a list of current sessions -# with the +jobs+ command in the prompt. +# You can view or modify the current prompt data with various configuration +# methods: +# +# * `conf.prompt_mode`, `conf.prompt_mode=`. +# * `conf.prompt_c`, `conf.c=`. +# * `conf.prompt_i`, `conf.i=`. +# * `conf.prompt_s`, `conf.s=`. +# * `conf.return_format`, `return_format=`. +# +# +# #### Prompt Specifiers +# +# A prompt's definition can include specifiers for which certain values are +# substituted: +# +# * `%N`: the name of the running program. +# * `%m`: the value of `self.to_s`. +# * `%M`: the value of `self.inspect`. +# * `%l`: an indication of the type of string; one of `"`, `'`, `/`, `]`. +# * `%NNi`: Indentation level. NN is a 2-digit number that specifies the number +# of digits of the indentation level (03 will result in 001). +# * `%NNn`: Line number. NN is a 2-digit number that specifies the number +# of digits of the line number (03 will result in 001). +# * `%%`: Literal `%`. +# +# +# ### Verbosity +# +# By default, IRB verbosity is disabled, which means that output is smaller +# rather than larger. +# +# You can enable verbosity by: +# +# * Adding to the configuration file: `IRB.conf[:VERBOSE] = true` (the default +# is `nil`). +# * Giving command-line options `--verbose` (the default is `--noverbose`). +# +# +# During a session, you can retrieve or set verbosity with methods +# `conf.verbose` and `conf.verbose=`. +# +# ### Help +# +# Command-line option `--version` causes IRB to print its help text and exit. +# +# ### Version +# +# Command-line option `--version` causes IRB to print its version text and exit. +# +# ## Input and Output +# +# ### Color Highlighting +# +# By default, IRB color highlighting is enabled, and is used for both: +# +# * Input: As you type, IRB reads the typed characters and highlights elements +# that it recognizes; it also highlights errors such as mismatched +# parentheses. +# * Output: IRB highlights syntactical elements. +# +# +# You can disable color highlighting by: +# +# * Adding to the configuration file: `IRB.conf[:USE_COLORIZE] = false` (the +# default value is `true`). +# * Giving command-line option `--nocolorize` +# +# +# ## Debugging +# +# Command-line option `-d` sets variables `$VERBOSE` and `$DEBUG` to `true`; +# these have no effect on IRB output. +# +# ### Warnings +# +# Command-line option `-w` suppresses warnings. +# +# Command-line option `-W[*level*]` sets warning level; +# +# * 0=silence +# * 1=medium +# * 2=verbose +# +# ## Other Features +# +# ### Load Modules +# +# You can specify the names of modules that are to be required at startup. +# +# Array `conf.load_modules` determines the modules (if any) that are to be +# required during session startup. The array is used only during session +# startup, so the initial value is the only one that counts. +# +# The default initial value is `[]` (load no modules): +# +# irb(main):001> conf.load_modules +# => [] +# +# You can set the default initial value via: +# +# * Command-line option `-r` +# +# $ irb -r csv -r json +# irb(main):001> conf.load_modules +# => ["csv", "json"] +# +# * Hash entry `IRB.conf[:LOAD_MODULES] = *array*`: +# +# IRB.conf[:LOAD_MODULES] = %w[csv, json] +# +# +# Note that the configuration file entry overrides the command-line options. +# +# ### RI Documentation Directories +# +# You can specify the paths to RI documentation directories that are to be +# loaded (in addition to the default directories) at startup; see details about +# RI by typing `ri --help`. +# +# Array `conf.extra_doc_dirs` determines the directories (if any) that are to be +# loaded during session startup. The array is used only during session startup, +# so the initial value is the only one that counts. +# +# The default initial value is `[]` (load no extra documentation): +# +# irb(main):001> conf.extra_doc_dirs +# => [] +# +# You can set the default initial value via: +# +# * Command-line option `--extra_doc_dir` +# +# $ irb --extra-doc-dir your_doc_dir --extra-doc-dir my_doc_dir +# irb(main):001> conf.extra_doc_dirs +# => ["your_doc_dir", "my_doc_dir"] +# +# * Hash entry `IRB.conf[:EXTRA_DOC_DIRS] = *array*`: +# +# IRB.conf[:EXTRA_DOC_DIRS] = %w[your_doc_dir my_doc_dir] +# +# +# Note that the configuration file entry overrides the command-line options. +# +# ### IRB Name +# +# You can specify a name for IRB. +# +# The default initial value is `'irb'`: +# +# irb(main):001> conf.irb_name +# => "irb" +# +# You can set the default initial value via hash entry `IRB.conf[:IRB_NAME] = +# *string*`: +# +# IRB.conf[:IRB_NAME] = 'foo' +# +# ### Application Name +# +# You can specify an application name for the IRB session. +# +# The default initial value is `'irb'`: +# +# irb(main):001> conf.ap_name +# => "irb" +# +# You can set the default initial value via hash entry `IRB.conf[:AP_NAME] = +# *string*`: # -# === Commands +# IRB.conf[:AP_NAME] = 'my_ap_name' # -# JobManager provides commands to handle the current sessions: +# ### Configuration Monitor # -# jobs # List of current sessions -# fg # Switches to the session of the given number -# kill # Kills the session with the given number +# You can monitor changes to the configuration by assigning a proc to +# `IRB.conf[:IRB_RC]` in the configuration file: # -# The +exit+ command, or ::irb_exit, will quit the current session and call any -# exit hooks with IRB.irb_at_exit. +# IRB.conf[:IRB_RC] = proc {|conf| puts conf.class } # -# A few commands for loading files within the session are also available: +# Each time the configuration is changed, that proc is called with argument +# `conf`: # -# +source+:: -# Loads a given file in the current session and displays the source lines, -# see IrbLoader#source_file -# +irb_load+:: -# Loads the given file similarly to Kernel#load, see IrbLoader#irb_load -# +irb_require+:: -# Loads the given file similarly to Kernel#require +# ### Encodings # -# === Configuration +# Command-line option `-E *ex*[:*in*]` sets initial external (ex) and internal +# (in) encodings. +# +# Command-line option `-U` sets both to UTF-8. +# +# ### Commands +# +# Please use the `help` command to see the list of available commands. +# +# ### IRB Sessions +# +# IRB has a special feature, that allows you to manage many sessions at once. +# +# You can create new sessions with Irb.irb, and get a list of current sessions +# with the `jobs` command in the prompt. +# +# #### Configuration # # The command line options, or IRB.conf, specify the default behavior of # Irb.irb. # -# On the other hand, each conf in IRB@Command+line+options is used to +# On the other hand, each conf in IRB@Command-Line+Options is used to # individually configure IRB.irb. # -# If a proc is set for <code>IRB.conf[:IRB_RC]</code>, its will be invoked after execution +# If a proc is set for `IRB.conf[:IRB_RC]`, its will be invoked after execution # of that proc with the context of the current session as its argument. Each # session can be configured using this mechanism. # -# === Session variables +# #### Session variables # # There are a few variables in every Irb session that can come in handy: # -# <code>_</code>:: -# The value command executed, as a local variable -# <code>__</code>:: -# The history of evaluated commands. Available only if -# <code>IRB.conf[:EVAL_HISTORY]</code> is not +nil+ (which is the default). -# See also IRB::Context#eval_history= and IRB::History. -# <code>__[line_no]</code>:: -# Returns the evaluation value at the given line number, +line_no+. -# If +line_no+ is a negative, the return value +line_no+ many lines before -# the most recent return value. -# -# === Example using IRB Sessions -# -# # invoke a new session -# irb(main):001:0> irb -# # list open sessions -# irb.1(main):001:0> jobs -# #0->irb on main (#<Thread:0x400fb7e4> : stop) -# #1->irb#1 on main (#<Thread:0x40125d64> : running) -# -# # change the active session -# irb.1(main):002:0> fg 0 -# # define class Foo in top-level session -# irb(main):002:0> class Foo;end -# # invoke a new session with the context of Foo -# irb(main):003:0> irb Foo -# # define Foo#foo -# irb.2(Foo):001:0> def foo -# irb.2(Foo):002:1> print 1 -# irb.2(Foo):003:1> end -# -# # change the active session -# irb.2(Foo):004:0> fg 0 -# # list open sessions -# irb(main):004:0> jobs -# #0->irb on main (#<Thread:0x400fb7e4> : running) -# #1->irb#1 on main (#<Thread:0x40125d64> : stop) -# #2->irb#2 on Foo (#<Thread:0x4011d54c> : stop) -# # check if Foo#foo is available -# irb(main):005:0> Foo.instance_methods #=> [:foo, ...] -# -# # change the active session -# irb(main):006:0> fg 2 -# # define Foo#bar in the context of Foo -# irb.2(Foo):005:0> def bar -# irb.2(Foo):006:1> print "bar" -# irb.2(Foo):007:1> end -# irb.2(Foo):010:0> Foo.instance_methods #=> [:bar, :foo, ...] -# -# # change the active session -# irb.2(Foo):011:0> fg 0 -# irb(main):007:0> f = Foo.new #=> #<Foo:0x4010af3c> -# # invoke a new session with the context of f (instance of Foo) -# irb(main):008:0> irb f -# # list open sessions -# irb.3(<Foo:0x4010af3c>):001:0> jobs -# #0->irb on main (#<Thread:0x400fb7e4> : stop) -# #1->irb#1 on main (#<Thread:0x40125d64> : stop) -# #2->irb#2 on Foo (#<Thread:0x4011d54c> : stop) -# #3->irb#3 on #<Foo:0x4010af3c> (#<Thread:0x4010a1e0> : running) -# # evaluate f.foo -# irb.3(<Foo:0x4010af3c>):002:0> foo #=> 1 => nil -# # evaluate f.bar -# irb.3(<Foo:0x4010af3c>):003:0> bar #=> bar => nil -# # kill jobs 1, 2, and 3 -# irb.3(<Foo:0x4010af3c>):004:0> kill 1, 2, 3 -# # list open sessions, should only include main session -# irb(main):009:0> jobs -# #0->irb on main (#<Thread:0x400fb7e4> : running) -# # quit irb -# irb(main):010:0> exit +# `_` +# : The value command executed, as a local variable +# `__` +# : The history of evaluated commands. Available only if +# `IRB.conf[:EVAL_HISTORY]` is not `nil` (which is the default). See also +# IRB::Context#eval_history= and IRB::History. +# `__[line_no]` +# : Returns the evaluation value at the given line number, `line_no`. If +# `line_no` is a negative, the return value `line_no` many lines before the +# most recent return value. +# +# +# ## Restrictions +# +# Ruby code typed into IRB behaves the same as Ruby code in a file, except that: +# +# * Because IRB evaluates input immediately after it is syntactically +# complete, some results may be slightly different. +# * Forking may not be well behaved. +# module IRB # An exception raised by IRB.irb_abort class Abort < Exception;end - @CONF = {} - - - # Displays current configuration. - # - # Modifying the configuration is achieved by sending a message to IRB.conf. - # - # See IRB@Configuration for more information. - def IRB.conf - @CONF - end - - # Returns the current version of IRB, including release version and last - # updated date. - def IRB.version - if v = @CONF[:VERSION] then return v end - - @CONF[:VERSION] = format("irb %s (%s)", @RELEASE_VERSION, @LAST_UPDATE_DATE) - end - # The current IRB::Context of the session, see IRB.conf # - # irb - # irb(main):001:0> IRB.CurrentContext.irb_name = "foo" - # foo(main):002:0> IRB.conf[:MAIN_CONTEXT].irb_name #=> "foo" - def IRB.CurrentContext + # irb + # irb(main):001:0> IRB.CurrentContext.irb_name = "foo" + # foo(main):002:0> IRB.conf[:MAIN_CONTEXT].irb_name #=> "foo" + def IRB.CurrentContext # :nodoc: IRB.conf[:MAIN_CONTEXT] end - # Initializes IRB and creates a new Irb.irb object at the +TOPLEVEL_BINDING+ + # Initializes IRB and creates a new Irb.irb object at the `TOPLEVEL_BINDING` def IRB.start(ap_path = nil) STDOUT.sync = true $0 = File::basename(ap_path, ".rb") if ap_path @@ -417,62 +905,48 @@ module IRB end # Quits irb - def IRB.irb_exit(irb, ret) - throw :IRB_EXIT, ret + def IRB.irb_exit(*) # :nodoc: + throw :IRB_EXIT, false end # Aborts then interrupts irb. # - # Will raise an Abort exception, or the given +exception+. - def IRB.irb_abort(irb, exception = Abort) + # Will raise an Abort exception, or the given `exception`. + def IRB.irb_abort(irb, exception = Abort) # :nodoc: irb.context.thread.raise exception, "abort then interrupt!" end class Irb - ASSIGNMENT_NODE_TYPES = [ - # Local, instance, global, class, constant, instance, and index assignment: - # "foo = bar", - # "@foo = bar", - # "$foo = bar", - # "@@foo = bar", - # "::Foo = bar", - # "a::Foo = bar", - # "Foo = bar" - # "foo.bar = 1" - # "foo[1] = bar" - :assign, - - # Operation assignment: - # "foo += bar" - # "foo -= bar" - # "foo ||= bar" - # "foo &&= bar" - :opassign, - - # Multiple assignment: - # "foo, bar = 1, 2 - :massign, - ] # Note: instance and index assignment expressions could also be written like: - # "foo.bar=(1)" and "foo.[]=(1, bar)", when expressed that way, the former - # be parsed as :assign and echo will be suppressed, but the latter is - # parsed as a :method_add_arg and the output won't be suppressed + # "foo.bar=(1)" and "foo.[]=(1, bar)", when expressed that way, the former be + # parsed as :assign and echo will be suppressed, but the latter is parsed as a + # :method_add_arg and the output won't be suppressed PROMPT_MAIN_TRUNCATE_LENGTH = 32 - PROMPT_MAIN_TRUNCATE_OMISSION = '...'.freeze - CONTROL_CHARACTERS_PATTERN = "\x00-\x1F".freeze + PROMPT_MAIN_TRUNCATE_OMISSION = '...' + CONTROL_CHARACTERS_PATTERN = "\x00-\x1F" + + # Returns the current context of this irb session + attr_reader :context + # The lexer used by this irb session + attr_accessor :scanner + + attr_reader :from_binding # Creates a new irb session - def initialize(workspace = nil, input_method = nil) + def initialize(workspace = nil, input_method = nil, from_binding: false) + @from_binding = from_binding @context = Context.new(self, workspace, input_method) - @context.main.extend ExtendCommandBundle + @context.workspace.load_helper_methods_to_main @signal_status = :IN_IRB - @scanner = RubyLex.new(@context) + @scanner = RubyLex.new + @line_no = 1 end - # A hook point for `debug` command's TracePoint after :IRB_EXIT as well as its clean-up + # A hook point for `debug` command's breakpoint after :IRB_EXIT as well as its + # clean-up def debug_break - # it means the debug command is executed + # it means the debug integration has been activated if defined?(DEBUGGER__) && DEBUGGER__.respond_to?(:capture_frames_without_irb) # after leaving this initial breakpoint, revert the capture_frames patch DEBUGGER__.singleton_class.send(:alias_method, :capture_frames, :capture_frames_without_irb) @@ -481,107 +955,101 @@ module IRB end end + def debug_readline(binding) + workspace = IRB::WorkSpace.new(binding) + context.replace_workspace(workspace) + context.workspace.load_helper_methods_to_main + @line_no += 1 + + # When users run: + # 1. Debugging commands, like `step 2` + # 2. Any input that's not irb-command, like `foo = 123` + # + # + # Irb#eval_input will simply return the input, and we need to pass it to the + # debugger. + input = nil + forced_exit = catch(:IRB_EXIT) do + if IRB.conf[:SAVE_HISTORY] && context.io.support_history_saving? + # Previous IRB session's history has been saved when `Irb#run` is exited We need + # to make sure the saved history is not saved again by resetting the counter + context.io.reset_history_counter + + begin + input = eval_input + ensure + context.io.save_history + end + else + input = eval_input + end + false + end + + Kernel.exit if forced_exit + + if input&.include?("\n") + @line_no += input.count("\n") - 1 + end + + input + end + def run(conf = IRB.conf) + in_nested_session = !!conf[:MAIN_CONTEXT] conf[:IRB_RC].call(context) if conf[:IRB_RC] + prev_context = conf[:MAIN_CONTEXT] conf[:MAIN_CONTEXT] = context + save_history = !in_nested_session && conf[:SAVE_HISTORY] && context.io.support_history_saving? + + if save_history + context.io.load_history + end + prev_trap = trap("SIGINT") do signal_handle end begin - catch(:IRB_EXIT) do + if defined?(RubyVM.keep_script_lines) + keep_script_lines_backup = RubyVM.keep_script_lines + RubyVM.keep_script_lines = true + end + + forced_exit = catch(:IRB_EXIT) do eval_input end ensure + # Do not restore to nil. It will cause IRB crash when used with threads. + IRB.conf[:MAIN_CONTEXT] = prev_context if prev_context + + RubyVM.keep_script_lines = keep_script_lines_backup if defined?(RubyVM.keep_script_lines) trap("SIGINT", prev_trap) conf[:AT_EXIT].each{|hook| hook.call} + + context.io.save_history if save_history + Kernel.exit if forced_exit end end - # Returns the current context of this irb session - attr_reader :context - # The lexer used by this irb session - attr_accessor :scanner - # Evaluates input for this session. def eval_input - exc = nil - - @scanner.set_prompt do - |ltype, indent, continue, line_no| - if ltype - f = @context.prompt_s - elsif continue - f = @context.prompt_c - elsif indent > 0 - f = @context.prompt_n - else - f = @context.prompt_i - end - f = "" unless f - if @context.prompting? - @context.io.prompt = p = prompt(f, ltype, indent, line_no) - else - @context.io.prompt = p = "" - end - if @context.auto_indent_mode and !@context.io.respond_to?(:auto_indent) - unless ltype - prompt_i = @context.prompt_i.nil? ? "" : @context.prompt_i - ind = prompt(prompt_i, ltype, indent, line_no)[/.*\z/].size + - indent * 2 - p.size - ind += 2 if continue - @context.io.prompt = p + " " * ind if ind > 0 - end - end - @context.io.prompt - end + configure_io - @scanner.set_input do - signal_status(:IN_INPUT) do - if l = @context.io.gets - print l if @context.verbose? - else - if @context.ignore_eof? and @context.io.readable_after_eof? - l = "\n" - if @context.verbose? - printf "Use \"exit\" to leave %s\n", @context.ap_name - end - else - print "\n" if @context.prompting? - end - end - l - end - end - - @scanner.configure_io(@context.io) - - @scanner.each_top_level_statement do |line, line_no| + each_top_level_statement do |statement, line_no| signal_status(:IN_EVAL) do begin - if IRB.conf[:MEASURE] && IRB.conf[:MEASURE_CALLBACKS].empty? - IRB.set_measure_callback - end - # Assignment expression check should be done before @context.evaluate to handle code like `a /2#/ if false; a = 1` - is_assignment = assignment_expression?(line) - if IRB.conf[:MEASURE] && !IRB.conf[:MEASURE_CALLBACKS].empty? - result = nil - last_proc = proc{ result = @context.evaluate(line, line_no, exception: exc) } - IRB.conf[:MEASURE_CALLBACKS].inject(last_proc) { |chain, item| - _name, callback, arg = item - proc { - callback.(@context, line, line_no, arg, exception: exc) do - chain.call - end - } - }.call - @context.set_last_value(result) - else - @context.evaluate(line, line_no, exception: exc) + # If the integration with debugger is activated, we return certain input if it + # should be dealt with by debugger + if @context.with_debugger && statement.should_be_handled_by_debugger? + return statement.code end - if @context.echo? - if is_assignment + + @context.evaluate(statement, line_no) + + if @context.echo? && !statement.suppresses_echo? + if statement.is_assignment? if @context.echo_on_assignment? output_value(@context.echo_on_assignment? == :truncate) end @@ -589,17 +1057,157 @@ module IRB output_value end end - rescue Interrupt => exc rescue SystemExit, SignalException raise - rescue Exception => exc + rescue Interrupt, Exception => exc + handle_exception(exc) + @context.workspace.local_variable_set(:_, exc) + end + end + end + end + + def read_input(prompt) + signal_status(:IN_INPUT) do + @context.io.prompt = prompt + if l = @context.io.gets + print l if @context.verbose? + else + if @context.ignore_eof? and @context.io.readable_after_eof? + l = "\n" + if @context.verbose? + printf "Use \"exit\" to leave %s\n", @context.ap_name + end + else + print "\n" if @context.prompting? + end + end + l + end + end + + def readmultiline + prompt = generate_prompt([], false, 0) + + # multiline + return read_input(prompt) if @context.io.respond_to?(:check_termination) + + # nomultiline + code = +'' + line_offset = 0 + loop do + line = read_input(prompt) + unless line + return code.empty? ? nil : code + end + + code << line + return code if command?(code) + + tokens, opens, terminated = @scanner.check_code_state(code, local_variables: @context.local_variables) + return code if terminated + + line_offset += 1 + continue = @scanner.should_continue?(tokens) + prompt = generate_prompt(opens, continue, line_offset) + end + end + + def each_top_level_statement + loop do + code = readmultiline + break unless code + yield build_statement(code), @line_no + @line_no += code.count("\n") + rescue RubyLex::TerminateLineInput + end + end + + def build_statement(code) + if code.match?(/\A\n*\z/) + return Statement::EmptyInput.new + end + + code.force_encoding(@context.io.encoding) + if (command, arg = parse_command(code)) + command_class = Command.load_command(command) + Statement::Command.new(code, command_class, arg) + else + is_assignment_expression = @scanner.assignment_expression?(code, local_variables: @context.local_variables) + Statement::Expression.new(code, is_assignment_expression) + end + end + + def parse_command(code) + command_name, arg = code.strip.split(/\s+/, 2) + return unless code.lines.size == 1 && command_name + + arg ||= '' + command = command_name.to_sym + # Command aliases are always command. example: $, @ + if (alias_name = @context.command_aliases[command]) + return [alias_name, arg] + end + + # Check visibility + public_method = !!Kernel.instance_method(:public_method).bind_call(@context.main, command) rescue false + private_method = !public_method && !!Kernel.instance_method(:method).bind_call(@context.main, command) rescue false + if Command.execute_as_command?(command, public_method: public_method, private_method: private_method) + [command, arg] + end + end + + def command?(code) + !!parse_command(code) + end + + def configure_io + if @context.io.respond_to?(:check_termination) + @context.io.check_termination do |code| + if Reline::IOGate.in_pasting? + rest = @scanner.check_termination_in_prev_line(code, local_variables: @context.local_variables) + if rest + Reline.delete_text + rest.bytes.reverse_each do |c| + Reline.ungetc(c) + end + true + else + false + end else - exc = nil - next + next true if command?(code) + + _tokens, _opens, terminated = @scanner.check_code_state(code, local_variables: @context.local_variables) + terminated end - handle_exception(exc) - @context.workspace.local_variable_set(:_, exc) - exc = nil + end + end + if @context.io.respond_to?(:dynamic_prompt) + @context.io.dynamic_prompt do |lines| + tokens = RubyLex.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, local_variables: @context.local_variables) + line_results = IRB::NestingParser.parse_by_line(tokens) + tokens_until_line = [] + line_results.map.with_index do |(line_tokens, _prev_opens, next_opens, _min_depth), line_num_offset| + line_tokens.each do |token, _s| + # Avoid appending duplicated token. Tokens that include "n" like multiline + # tstring_content can exist in multiple lines. + tokens_until_line << token if token != tokens_until_line.last + end + continue = @scanner.should_continue?(tokens_until_line) + generate_prompt(next_opens, continue, line_num_offset) + end + end + end + + if @context.io.respond_to?(:auto_indent) and @context.auto_indent_mode + @context.io.auto_indent do |lines, line_index, byte_pointer, is_newline| + next nil if lines == [nil] # Workaround for exit IRB with CTRL+d + next nil if !is_newline && lines[line_index]&.byteslice(0, byte_pointer)&.match?(/\A\s*\z/) + + code = lines[0..line_index].map { |l| "#{l}\n" }.join + tokens = RubyLex.ripper_lex_without_warning(code, local_variables: @context.local_variables) + @scanner.process_indent_level(tokens, lines, line_index, is_newline) end end end @@ -634,60 +1242,75 @@ module IRB end def handle_exception(exc) - if exc.backtrace && exc.backtrace[0] =~ /\/irb(2)?(\/.*|-.*|\.rb)?:/ && exc.class.to_s !~ /^IRB/ && + if exc.backtrace[0] =~ /\/irb(2)?(\/.*|-.*|\.rb)?:/ && exc.class.to_s !~ /^IRB/ && !(SyntaxError === exc) && !(EncodingError === exc) # The backtrace of invalid encoding hash (ex. {"\xAE": 1}) raises EncodingError without lineno. irb_bug = true else irb_bug = false - end + # To support backtrace filtering while utilizing Exception#full_message, we need to clone + # the exception to avoid modifying the original exception's backtrace. + exc = exc.clone + filtered_backtrace = exc.backtrace.map { |l| @context.workspace.filter_backtrace(l) }.compact + backtrace_filter = IRB.conf[:BACKTRACE_FILTER] - if exc.backtrace - order = nil - if RUBY_VERSION < '3.0.0' - if STDOUT.tty? - message = exc.full_message(order: :bottom) - order = :bottom + if backtrace_filter + if backtrace_filter.respond_to?(:call) + filtered_backtrace = backtrace_filter.call(filtered_backtrace) else - message = exc.full_message(order: :top) - order = :top + warn "IRB.conf[:BACKTRACE_FILTER] #{backtrace_filter} should respond to `call` method" end + end + + exc.set_backtrace(filtered_backtrace) + end + + highlight = Color.colorable? + + order = + if RUBY_VERSION < '3.0.0' + STDOUT.tty? ? :bottom : :top else # '3.0.0' <= RUBY_VERSION - message = exc.full_message(order: :top) - order = :top + :top end - message = convert_invalid_byte_sequence(message, exc.message.encoding) - message = encode_with_invalid_byte_sequence(message, IRB.conf[:LC_MESSAGES].encoding) unless message.encoding.to_s.casecmp?(IRB.conf[:LC_MESSAGES].encoding.to_s) - message = message.gsub(/((?:^\t.+$\n)+)/) { |m| - case order - when :top - lines = m.split("\n") - when :bottom - lines = m.split("\n").reverse - end - unless irb_bug - lines = lines.map { |l| @context.workspace.filter_backtrace(l) }.compact - if lines.size > @context.back_trace_limit - omit = lines.size - @context.back_trace_limit - lines = lines[0..(@context.back_trace_limit - 1)] - lines << "\t... %d levels..." % omit - end + + message = exc.full_message(order: order, highlight: highlight) + message = convert_invalid_byte_sequence(message, exc.message.encoding) + message = encode_with_invalid_byte_sequence(message, IRB.conf[:LC_MESSAGES].encoding) unless message.encoding.to_s.casecmp?(IRB.conf[:LC_MESSAGES].encoding.to_s) + message = message.gsub(/((?:^\t.+$\n)+)/) { |m| + case order + when :top + lines = m.split("\n") + when :bottom + lines = m.split("\n").reverse + end + unless irb_bug + if lines.size > @context.back_trace_limit + omit = lines.size - @context.back_trace_limit + lines = lines[0..(@context.back_trace_limit - 1)] + lines << "\t... %d levels..." % omit end - lines = lines.reverse if order == :bottom - lines.map{ |l| l + "\n" }.join - } - # The "<top (required)>" in "(irb)" may be the top level of IRB so imitate the main object. - message = message.gsub(/\(irb\):(?<num>\d+):in `<(?<frame>top \(required\))>'/) { "(irb):#{$~[:num]}:in `<main>'" } - puts message - end - print "Maybe IRB bug!\n" if irb_bug + end + lines = lines.reverse if order == :bottom + lines.map{ |l| l + "\n" }.join + } + # The "<top (required)>" in "(irb)" may be the top level of IRB so imitate the main object. + message = message.gsub(/\(irb\):(?<num>\d+):in (?<open_quote>[`'])<(?<frame>top \(required\))>'/) { "(irb):#{$~[:num]}:in #{$~[:open_quote]}<main>'" } + puts message + puts 'Maybe IRB bug!' if irb_bug + rescue Exception => handler_exc + begin + puts exc.inspect + puts "backtraces are hidden because #{handler_exc} was raised when processing them" + rescue Exception + puts 'Uninspectable exception occurred' + end end - # Evaluates the given block using the given +path+ as the Context#irb_path - # and +name+ as the Context#irb_name. + # Evaluates the given block using the given `path` as the Context#irb_path and + # `name` as the Context#irb_name. # - # Used by the irb command +source+, see IRB@IRB+Sessions for more - # information. + # Used by the irb command `source`, see IRB@IRB+Sessions for more information. def suspend_name(path = nil, name = nil) @context.irb_path, back_path = path, @context.irb_path if path @context.irb_name, back_name = name, @context.irb_name if name @@ -699,25 +1322,22 @@ module IRB end end - # Evaluates the given block using the given +workspace+ as the + # Evaluates the given block using the given `workspace` as the # Context#workspace. # - # Used by the irb command +irb_load+, see IRB@IRB+Sessions for more - # information. + # Used by the irb command `irb_load`, see IRB@IRB+Sessions for more information. def suspend_workspace(workspace) - @context.workspace, back_workspace = workspace, @context.workspace - begin - yield back_workspace - ensure - @context.workspace = back_workspace - end + current_workspace = @context.workspace + @context.replace_workspace(workspace) + yield + ensure + @context.replace_workspace current_workspace end - # Evaluates the given block using the given +input_method+ as the - # Context#io. + # Evaluates the given block using the given `input_method` as the Context#io. # - # Used by the irb commands +source+ and +irb_load+, see IRB@IRB+Sessions - # for more information. + # Used by the irb commands `source` and `irb_load`, see IRB@IRB+Sessions for + # more information. def suspend_input_method(input_method) back_io = @context.io @context.instance_eval{@io = input_method} @@ -728,16 +1348,6 @@ module IRB end end - # Evaluates the given block using the given +context+ as the Context. - def suspend_context(context) - @context, back_context = context, @context - begin - yield back_context - ensure - @context = back_context - end - end - # Handler for the signal SIGINT, see Kernel#trap for more information. def signal_handle unless @context.ignore_sigint? @@ -760,7 +1370,7 @@ module IRB end end - # Evaluates the given block using the given +status+. + # Evaluates the given block using the given `status`. def signal_status(status) return yield if @signal_status == :IN_LOAD @@ -773,54 +1383,6 @@ module IRB end end - def truncate_prompt_main(str) # :nodoc: - str = str.tr(CONTROL_CHARACTERS_PATTERN, ' ') - if str.size <= PROMPT_MAIN_TRUNCATE_LENGTH - str - else - str[0, PROMPT_MAIN_TRUNCATE_LENGTH - PROMPT_MAIN_TRUNCATE_OMISSION.size] + PROMPT_MAIN_TRUNCATE_OMISSION - end - end - - def prompt(prompt, ltype, indent, line_no) # :nodoc: - p = prompt.dup - p.gsub!(/%([0-9]+)?([a-zA-Z])/) do - case $2 - when "N" - @context.irb_name - when "m" - truncate_prompt_main(@context.main.to_s) - when "M" - truncate_prompt_main(@context.main.inspect) - when "l" - ltype - when "i" - if indent < 0 - if $1 - "-".rjust($1.to_i) - else - "-" - end - else - if $1 - format("%" + $1 + "d", indent) - else - indent.to_s - end - end - when "n" - if $1 - format("%" + $1 + "d", line_no) - else - line_no.to_s - end - when "%" - "%" - end - end - p - end - def output_value(omit = false) # :nodoc: str = @context.inspect_last_value multiline_p = str.include?("\n") @@ -850,15 +1412,16 @@ module IRB end end end + if multiline_p && @context.newline_before_multiline_output? - printf @context.return_format, "\n#{str}" - else - printf @context.return_format, str + str = "\n" + str end + + Pager.page_content(format(@context.return_format, str), retain_content: true) end - # Outputs the local variables to this current session, including - # #signal_status and #context, using IRB::Locale. + # Outputs the local variables to this current session, including #signal_status + # and #context, using IRB::Locale. def inspect ary = [] for iv in instance_variables @@ -874,55 +1437,94 @@ module IRB format("#<%s: %s>", self.class, ary.join(", ")) end - def assignment_expression?(line) - # Try to parse the line and check if the last of possibly multiple - # expressions is an assignment type. - - # If the expression is invalid, Ripper.sexp should return nil which will - # result in false being returned. Any valid expression should return an - # s-expression where the second element of the top level array is an - # array of parsed expressions. The first element of each expression is the - # expression's type. - verbose, $VERBOSE = $VERBOSE, nil - code = "#{RubyLex.generate_local_variables_assign_code(@context.local_variables) || 'nil;'}\n#{line}" - # Get the last node_type of the line. drop(1) is to ignore the local_variables_assign_code part. - node_type = Ripper.sexp(code)&.dig(1)&.drop(1)&.dig(-1, 0) - ASSIGNMENT_NODE_TYPES.include?(node_type) - ensure - $VERBOSE = verbose + private + + def generate_prompt(opens, continue, line_offset) + ltype = @scanner.ltype_from_open_tokens(opens) + indent = @scanner.calc_indent_level(opens) + continue = opens.any? || continue + line_no = @line_no + line_offset + + if ltype + f = @context.prompt_s + elsif continue + f = @context.prompt_c + else + f = @context.prompt_i + end + f = "" unless f + if @context.prompting? + p = format_prompt(f, ltype, indent, line_no) + else + p = "" + end + if @context.auto_indent_mode and !@context.io.respond_to?(:auto_indent) + unless ltype + prompt_i = @context.prompt_i.nil? ? "" : @context.prompt_i + ind = format_prompt(prompt_i, ltype, indent, line_no)[/.*\z/].size + + indent * 2 - p.size + p += " " * ind if ind > 0 + end + end + p end - end - def @CONF.inspect - IRB.version unless self[:VERSION] - - array = [] - for k, v in sort{|a1, a2| a1[0].id2name <=> a2[0].id2name} - case k - when :MAIN_CONTEXT, :__TMP__EHV__ - array.push format("CONF[:%s]=...myself...", k.id2name) - when :PROMPT - s = v.collect{ - |kk, vv| - ss = vv.collect{|kkk, vvv| ":#{kkk.id2name}=>#{vvv.inspect}"} - format(":%s=>{%s}", kk.id2name, ss.join(", ")) - } - array.push format("CONF[:%s]={%s}", k.id2name, s.join(", ")) + def truncate_prompt_main(str) # :nodoc: + str = str.tr(CONTROL_CHARACTERS_PATTERN, ' ') + if str.size <= PROMPT_MAIN_TRUNCATE_LENGTH + str else - array.push format("CONF[:%s]=%s", k.id2name, v.inspect) + str[0, PROMPT_MAIN_TRUNCATE_LENGTH - PROMPT_MAIN_TRUNCATE_OMISSION.size] + PROMPT_MAIN_TRUNCATE_OMISSION + end + end + + def format_prompt(format, ltype, indent, line_no) # :nodoc: + format.gsub(/%([0-9]+)?([a-zA-Z%])/) do + case $2 + when "N" + @context.irb_name + when "m" + main_str = @context.main.to_s rescue "!#{$!.class}" + truncate_prompt_main(main_str) + when "M" + main_str = @context.main.inspect rescue "!#{$!.class}" + truncate_prompt_main(main_str) + when "l" + ltype + when "i" + if indent < 0 + if $1 + "-".rjust($1.to_i) + else + "-" + end + else + if $1 + format("%" + $1 + "d", indent) + else + indent.to_s + end + end + when "n" + if $1 + format("%" + $1 + "d", line_no) + else + line_no.to_s + end + when "%" + "%" unless $1 + end end end - array.join("\n") end end class Binding - # Opens an IRB session where +binding.irb+ is called which allows for - # interactive debugging. You can call any methods or variables available in - # the current scope, and mutate state if you need to. - # + # Opens an IRB session where `binding.irb` is called which allows for + # interactive debugging. You can call any methods or variables available in the + # current scope, and mutate state if you need to. # - # Given a Ruby file called +potato.rb+ containing the following code: + # Given a Ruby file called `potato.rb` containing the following code: # # class Potato # def initialize @@ -934,8 +1536,8 @@ class Binding # # Potato.new # - # Running <code>ruby potato.rb</code> will open an IRB session where - # +binding.irb+ is called, and you will see the following: + # Running `ruby potato.rb` will open an IRB session where `binding.irb` is + # called, and you will see the following: # # $ ruby potato.rb # @@ -965,22 +1567,42 @@ class Binding # irb(#<Potato:0x00007feea1916670>):004:0> @cooked = true # => true # - # You can exit the IRB session with the +exit+ command. Note that exiting will - # resume execution where +binding.irb+ had paused it, as you can see from the + # You can exit the IRB session with the `exit` command. Note that exiting will + # resume execution where `binding.irb` had paused it, as you can see from the # output printed to standard output in this example: # # irb(#<Potato:0x00007feea1916670>):005:0> exit # Cooked potato: true # - # - # See IRB@IRB+Usage for more information. + # See IRB for more information. def irb(show_code: true) - IRB.setup(source_location[0], argv: []) + # Setup IRB with the current file's path and no command line arguments + IRB.setup(source_location[0], argv: []) unless IRB.initialized? + # Create a new workspace using the current binding workspace = IRB::WorkSpace.new(self) + # Print the code around the binding if show_code is true STDOUT.print(workspace.code_around_binding) if show_code - binding_irb = IRB::Irb.new(workspace) - binding_irb.context.irb_path = File.expand_path(source_location[0]) - binding_irb.run(IRB.conf) - binding_irb.debug_break + # Get the original IRB instance + debugger_irb = IRB.instance_variable_get(:@debugger_irb) + + irb_path = File.expand_path(source_location[0]) + + if debugger_irb + # If we're already in a debugger session, set the workspace and irb_path for the original IRB instance + debugger_irb.context.replace_workspace(workspace) + debugger_irb.context.irb_path = irb_path + # If we've started a debugger session and hit another binding.irb, we don't want + # to start an IRB session instead, we want to resume the irb:rdbg session. + IRB::Debug.setup(debugger_irb) + IRB::Debug.insert_debug_break + debugger_irb.debug_break + else + # If we're not in a debugger session, create a new IRB instance with the current + # workspace + binding_irb = IRB::Irb.new(workspace, from_binding: true) + binding_irb.context.irb_path = irb_path + binding_irb.run(IRB.conf) + binding_irb.debug_break + end end end diff --git a/lib/irb/cmd/backtrace.rb b/lib/irb/cmd/backtrace.rb deleted file mode 100644 index f632894618..0000000000 --- a/lib/irb/cmd/backtrace.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require_relative "debug" - -module IRB - # :stopdoc: - - module ExtendCommand - class Backtrace < DebugCommand - def self.transform_args(args) - args&.dump - end - - def execute(*args) - super(pre_cmds: ["backtrace", *args].join(" ")) - end - end - end - - # :startdoc: -end diff --git a/lib/irb/cmd/break.rb b/lib/irb/cmd/break.rb deleted file mode 100644 index df259a90ca..0000000000 --- a/lib/irb/cmd/break.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require_relative "debug" - -module IRB - # :stopdoc: - - module ExtendCommand - class Break < DebugCommand - def self.transform_args(args) - args&.dump - end - - def execute(args = nil) - super(pre_cmds: "break #{args}") - end - end - end - - # :startdoc: -end diff --git a/lib/irb/cmd/catch.rb b/lib/irb/cmd/catch.rb deleted file mode 100644 index 40b62c7533..0000000000 --- a/lib/irb/cmd/catch.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require_relative "debug" - -module IRB - # :stopdoc: - - module ExtendCommand - class Catch < DebugCommand - def self.transform_args(args) - args&.dump - end - - def execute(*args) - super(pre_cmds: ["catch", *args].join(" ")) - end - end - end - - # :startdoc: -end diff --git a/lib/irb/cmd/chws.rb b/lib/irb/cmd/chws.rb deleted file mode 100644 index 44712fa445..0000000000 --- a/lib/irb/cmd/chws.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: false -# -# change-ws.rb - -# by Keiju ISHITSUKA(keiju@ruby-lang.org) -# - -require_relative "nop" -require_relative "../ext/change-ws" - -module IRB - # :stopdoc: - - module ExtendCommand - - class CurrentWorkingWorkspace < Nop - category "IRB" - description "Show the current workspace." - - def execute(*obj) - irb_context.main - end - end - - class ChangeWorkspace < Nop - category "IRB" - description "Change the current workspace to an object." - - def execute(*obj) - irb_context.change_workspace(*obj) - irb_context.main - end - end - end - - # :startdoc: -end diff --git a/lib/irb/cmd/edit.rb b/lib/irb/cmd/edit.rb deleted file mode 100644 index 0103891cf4..0000000000 --- a/lib/irb/cmd/edit.rb +++ /dev/null @@ -1,61 +0,0 @@ -require 'shellwords' -require_relative "nop" - -module IRB - # :stopdoc: - - module ExtendCommand - class Edit < Nop - category "Misc" - description 'Open a file with the editor command defined with `ENV["EDITOR"]`.' - - class << self - def transform_args(args) - # Return a string literal as is for backward compatibility - if args.nil? || args.empty? || string_literal?(args) - args - else # Otherwise, consider the input as a String for convenience - args.strip.dump - end - end - end - - def execute(*args) - path = args.first - - if path.nil? && (irb_path = @irb_context.irb_path) - path = irb_path - end - - if !File.exist?(path) - require_relative "show_source" - - source = - begin - ShowSource.find_source(path, @irb_context) - rescue NameError - # if user enters a path that doesn't exist, it'll cause NameError when passed here because find_source would try to evaluate it as well - # in this case, we should just ignore the error - end - - if source && File.exist?(source.file) - path = source.file - else - puts "Can not find file: #{path}" - return - end - end - - if editor = ENV['EDITOR'] - puts "command: '#{editor}'" - puts " path: #{path}" - system(*Shellwords.split(editor), path) - else - puts "Can not find editor setting: ENV['EDITOR']" - end - end - end - end - - # :startdoc: -end diff --git a/lib/irb/cmd/fork.rb b/lib/irb/cmd/fork.rb deleted file mode 100644 index 1cd235997f..0000000000 --- a/lib/irb/cmd/fork.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: false -# -# fork.rb - -# by Keiju ISHITSUKA(keiju@ruby-lang.org) -# - -require_relative "nop" - -module IRB - # :stopdoc: - - module ExtendCommand - class Fork < Nop - def execute - pid = __send__ ExtendCommand.irb_original_method_name("fork") - unless pid - class << self - alias_method :exit, ExtendCommand.irb_original_method_name('exit') - end - if block_given? - begin - yield - ensure - exit - end - end - end - pid - end - end - end - - # :startdoc: -end diff --git a/lib/irb/cmd/help.rb b/lib/irb/cmd/help.rb deleted file mode 100644 index 64b885c383..0000000000 --- a/lib/irb/cmd/help.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require_relative "show_doc" - -module IRB - module ExtendCommand - class Help < ShowDoc - category "Context" - description "[DEPRECATED] Enter the mode to look up RI documents." - - DEPRECATION_MESSAGE = <<~MSG - [Deprecation] The `help` command will be repurposed to display command help in the future. - For RI document lookup, please use the `show_doc` command instead. - For command help, please use `show_cmds` for now. - MSG - - def execute(*names) - warn DEPRECATION_MESSAGE - super - end - end - end -end diff --git a/lib/irb/cmd/info.rb b/lib/irb/cmd/info.rb deleted file mode 100644 index 2c0a32b34f..0000000000 --- a/lib/irb/cmd/info.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require_relative "debug" - -module IRB - # :stopdoc: - - module ExtendCommand - class Info < DebugCommand - def self.transform_args(args) - args&.dump - end - - def execute(*args) - super(pre_cmds: ["info", *args].join(" ")) - end - end - end - - # :startdoc: -end diff --git a/lib/irb/cmd/nop.rb b/lib/irb/cmd/nop.rb index 7fb197c51f..9d2e3c4d47 100644 --- a/lib/irb/cmd/nop.rb +++ b/lib/irb/cmd/nop.rb @@ -1,53 +1,4 @@ -# frozen_string_literal: false -# -# nop.rb - -# by Keiju ISHITSUKA(keiju@ruby-lang.org) -# +# frozen_string_literal: true -module IRB - # :stopdoc: - - module ExtendCommand - class CommandArgumentError < StandardError; end - - class Nop - class << self - def category(category = nil) - @category = category if category - @category - end - - def description(description = nil) - @description = description if description - @description - end - - private - - def string_literal?(args) - sexp = Ripper.sexp(args) - sexp && sexp.size == 2 && sexp.last&.first&.first == :string_literal - end - end - - def self.execute(irb_context, *opts, **kwargs, &block) - command = new(irb_context) - command.execute(*opts, **kwargs, &block) - rescue CommandArgumentError => e - puts e.message - end - - def initialize(irb_context) - @irb_context = irb_context - end - - attr_reader :irb_context - - def execute(*opts) - #nop - end - end - end - - # :startdoc: -end +# This file is just a placeholder for backward-compatibility. +# Please require 'irb' and inherit your command from `IRB::Command::Base` instead. diff --git a/lib/irb/cmd/pushws.rb b/lib/irb/cmd/pushws.rb deleted file mode 100644 index d64d822182..0000000000 --- a/lib/irb/cmd/pushws.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: false -# -# change-ws.rb - -# by Keiju ISHITSUKA(keiju@ruby-lang.org) -# - -require_relative "nop" -require_relative "../ext/workspaces" - -module IRB - # :stopdoc: - - module ExtendCommand - class Workspaces < Nop - category "IRB" - description "Show workspaces." - - def execute(*obj) - irb_context.workspaces.collect{|ws| ws.main} - end - end - - class PushWorkspace < Workspaces - category "IRB" - description "Push an object to the workspace stack." - - def execute(*obj) - irb_context.push_workspace(*obj) - super - end - end - - class PopWorkspace < Workspaces - category "IRB" - description "Pop a workspace from the workspace stack." - - def execute(*obj) - irb_context.pop_workspace(*obj) - super - end - end - end - - # :startdoc: -end diff --git a/lib/irb/cmd/show_cmds.rb b/lib/irb/cmd/show_cmds.rb deleted file mode 100644 index acced27d48..0000000000 --- a/lib/irb/cmd/show_cmds.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -require "stringio" -require_relative "nop" - -module IRB - # :stopdoc: - - module ExtendCommand - class ShowCmds < Nop - category "IRB" - description "List all available commands and their description." - - def execute(*args) - commands_info = IRB::ExtendCommandBundle.all_commands_info - commands_grouped_by_categories = commands_info.group_by { |cmd| cmd[:category] } - longest_cmd_name_length = commands_info.map { |c| c[:display_name] }.max { |a, b| a.length <=> b.length }.length - - output = StringIO.new - - commands_grouped_by_categories.each do |category, cmds| - output.puts Color.colorize(category, [:BOLD]) - - cmds.each do |cmd| - output.puts " #{cmd[:display_name].to_s.ljust(longest_cmd_name_length)} #{cmd[:description]}" - end - - output.puts - end - - puts output.string - - nil - end - end - end - - # :startdoc: -end diff --git a/lib/irb/cmd/show_doc.rb b/lib/irb/cmd/show_doc.rb deleted file mode 100644 index 99dd9ab95a..0000000000 --- a/lib/irb/cmd/show_doc.rb +++ /dev/null @@ -1,48 +0,0 @@ -# frozen_string_literal: true - -require_relative "nop" - -module IRB - module ExtendCommand - class ShowDoc < Nop - class << self - def transform_args(args) - # Return a string literal as is for backward compatibility - if args.empty? || string_literal?(args) - args - else # Otherwise, consider the input as a String for convenience - args.strip.dump - end - end - end - - category "Context" - description "Enter the mode to look up RI documents." - - def execute(*names) - require 'rdoc/ri/driver' - - unless ShowDoc.const_defined?(:Ri) - opts = RDoc::RI::Driver.process_args([]) - ShowDoc.const_set(:Ri, RDoc::RI::Driver.new(opts)) - end - - if names.empty? - Ri.interactive - else - names.each do |name| - begin - Ri.display_name(name.to_s) - rescue RDoc::RI::Error - puts $!.message - end - end - end - - nil - rescue LoadError, SystemExit - warn "Can't display document because `rdoc` is not installed." - end - end - end -end diff --git a/lib/irb/cmd/show_source.rb b/lib/irb/cmd/show_source.rb deleted file mode 100644 index a74895b2dc..0000000000 --- a/lib/irb/cmd/show_source.rb +++ /dev/null @@ -1,113 +0,0 @@ -# frozen_string_literal: true - -require_relative "nop" -require_relative "../color" -require_relative "../ruby-lex" - -module IRB - # :stopdoc: - - module ExtendCommand - class ShowSource < Nop - category "Context" - description "Show the source code of a given method or constant." - - class << self - def transform_args(args) - # Return a string literal as is for backward compatibility - if args.empty? || string_literal?(args) - args - else # Otherwise, consider the input as a String for convenience - args.strip.dump - end - end - - def find_source(str, irb_context) - case str - when /\A[A-Z]\w*(::[A-Z]\w*)*\z/ # Const::Name - eval(str, irb_context.workspace.binding) # trigger autoload - base = irb_context.workspace.binding.receiver.yield_self { |r| r.is_a?(Module) ? r : Object } - file, line = base.const_source_location(str) - when /\A(?<owner>[A-Z]\w*(::[A-Z]\w*)*)#(?<method>[^ :.]+)\z/ # Class#method - owner = eval(Regexp.last_match[:owner], irb_context.workspace.binding) - method = Regexp.last_match[:method] - if owner.respond_to?(:instance_method) - methods = owner.instance_methods + owner.private_instance_methods - file, line = owner.instance_method(method).source_location if methods.include?(method.to_sym) - end - when /\A((?<receiver>.+)(\.|::))?(?<method>[^ :.]+)\z/ # method, receiver.method, receiver::method - receiver = eval(Regexp.last_match[:receiver] || 'self', irb_context.workspace.binding) - method = Regexp.last_match[:method] - file, line = receiver.method(method).source_location if receiver.respond_to?(method, true) - end - if file && line - Source.new(file: file, first_line: line, last_line: find_end(file, line, irb_context)) - end - end - - private - - def find_end(file, first_line, irb_context) - return first_line unless File.exist?(file) - lex = RubyLex.new(irb_context) - lines = File.read(file).lines[(first_line - 1)..-1] - tokens = RubyLex.ripper_lex_without_warning(lines.join) - prev_tokens = [] - - # chunk with line number - tokens.chunk { |tok| tok.pos[0] }.each do |lnum, chunk| - code = lines[0..lnum].join - prev_tokens.concat chunk - continue = lex.process_continue(prev_tokens) - code_block_open = lex.check_code_block(code, prev_tokens) - if !continue && !code_block_open - return first_line + lnum - end - end - first_line - end - end - - def execute(str = nil) - unless str.is_a?(String) - puts "Error: Expected a string but got #{str.inspect}" - return - end - - source = self.class.find_source(str, @irb_context) - if source && File.exist?(source.file) - show_source(source) - else - puts "Error: Couldn't locate a definition for #{str}" - end - nil - end - - private - - # @param [IRB::ExtendCommand::ShowSource::Source] source - def show_source(source) - puts - puts "#{bold("From")}: #{source.file}:#{source.first_line}" - puts - code = IRB::Color.colorize_code(File.read(source.file)) - puts code.lines[(source.first_line - 1)...source.last_line].join - puts - end - - def bold(str) - Color.colorize(str, [:BOLD]) - end - - Source = Struct.new( - :file, # @param [String] - file name - :first_line, # @param [String] - first line - :last_line, # @param [String] - last line - keyword_init: true, - ) - private_constant :Source - end - end - - # :startdoc: -end diff --git a/lib/irb/cmd/subirb.rb b/lib/irb/cmd/subirb.rb deleted file mode 100644 index 5f3e02c988..0000000000 --- a/lib/irb/cmd/subirb.rb +++ /dev/null @@ -1,66 +0,0 @@ -# frozen_string_literal: false -# -# multi.rb - -# by Keiju ISHITSUKA(keiju@ruby-lang.org) -# - -require_relative "nop" - -module IRB - # :stopdoc: - - module ExtendCommand - class MultiIRBCommand < Nop - def initialize(conf) - super - extend_irb_context - end - - private - - def extend_irb_context - # this extension patches IRB context like IRB.CurrentContext - require_relative "../ext/multi-irb" - end - end - - class IrbCommand < MultiIRBCommand - category "IRB" - description "Start a child IRB." - - def execute(*obj) - IRB.irb(nil, *obj) - end - end - - class Jobs < MultiIRBCommand - category "IRB" - description "List of current sessions." - - def execute - IRB.JobManager - end - end - - class Foreground < MultiIRBCommand - category "IRB" - description "Switches to the session of the given number." - - def execute(key = nil) - raise CommandArgumentError.new("Please specify the id of target IRB job (listed in the `jobs` command).") unless key - IRB.JobManager.switch(key) - end - end - - class Kill < MultiIRBCommand - category "IRB" - description "Kills the session with the given number." - - def execute(*keys) - IRB.JobManager.kill(*keys) - end - end - end - - # :startdoc: -end diff --git a/lib/irb/color.rb b/lib/irb/color.rb index fa99ed6fd6..fca942b28b 100644 --- a/lib/irb/color.rb +++ b/lib/irb/color.rb @@ -9,12 +9,14 @@ module IRB # :nodoc: BOLD = 1 UNDERLINE = 4 REVERSE = 7 + BLACK = 30 RED = 31 GREEN = 32 YELLOW = 33 BLUE = 34 MAGENTA = 35 CYAN = 36 + WHITE = 37 TOKEN_KEYWORDS = { on_kw: ['nil', 'self', 'true', 'false', '__FILE__', '__LINE__', '__ENCODING__'], @@ -77,12 +79,12 @@ module IRB # :nodoc: class << self def colorable? - supported = $stdout.tty? && (/mswin|mingw/ =~ RUBY_PLATFORM || (ENV.key?('TERM') && ENV['TERM'] != 'dumb')) + supported = $stdout.tty? && (/mswin|mingw/.match?(RUBY_PLATFORM) || (ENV.key?('TERM') && ENV['TERM'] != 'dumb')) # because ruby/debug also uses irb's color module selectively, # irb won't be activated in that case. if IRB.respond_to?(:conf) - supported && IRB.conf.fetch(:USE_COLORIZE, true) + supported && !!IRB.conf.fetch(:USE_COLORIZE, true) else supported end diff --git a/lib/irb/command.rb b/lib/irb/command.rb new file mode 100644 index 0000000000..68a4b52727 --- /dev/null +++ b/lib/irb/command.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +# +# irb/command.rb - irb command +# by Keiju ISHITSUKA(keiju@ruby-lang.org) +# + +require_relative "command/base" + +module IRB # :nodoc: + module Command + @commands = {} + + class << self + attr_reader :commands + + # Registers a command with the given name. + # Aliasing is intentionally not supported at the moment. + def register(name, command_class) + @commands[name.to_sym] = [command_class, []] + end + end + end +end diff --git a/lib/irb/command/backtrace.rb b/lib/irb/command/backtrace.rb new file mode 100644 index 0000000000..687bb075ac --- /dev/null +++ b/lib/irb/command/backtrace.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require_relative "debug" + +module IRB + # :stopdoc: + + module Command + class Backtrace < DebugCommand + def execute(arg) + execute_debug_command(pre_cmds: "backtrace #{arg}") + end + end + end + + # :startdoc: +end diff --git a/lib/irb/command/base.rb b/lib/irb/command/base.rb new file mode 100644 index 0000000000..1d406630a2 --- /dev/null +++ b/lib/irb/command/base.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true +# +# nop.rb - +# by Keiju ISHITSUKA(keiju@ruby-lang.org) +# + +module IRB + # :stopdoc: + + module Command + class CommandArgumentError < StandardError; end + + def self.extract_ruby_args(*args, **kwargs) + throw :EXTRACT_RUBY_ARGS, [args, kwargs] + end + + class Base + class << self + def category(category = nil) + @category = category if category + @category || "No category" + end + + def description(description = nil) + @description = description if description + @description || "No description provided." + end + + def help_message(help_message = nil) + @help_message = help_message if help_message + @help_message + end + + private + + def highlight(text) + Color.colorize(text, [:BOLD, :BLUE]) + end + end + + def self.execute(irb_context, arg) + new(irb_context).execute(arg) + rescue CommandArgumentError => e + puts e.message + end + + def initialize(irb_context) + @irb_context = irb_context + end + + attr_reader :irb_context + + def execute(arg) + #nop + end + end + + Nop = Base + end + + # :startdoc: +end diff --git a/lib/irb/command/break.rb b/lib/irb/command/break.rb new file mode 100644 index 0000000000..a8f81fe665 --- /dev/null +++ b/lib/irb/command/break.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require_relative "debug" + +module IRB + # :stopdoc: + + module Command + class Break < DebugCommand + def execute(arg) + execute_debug_command(pre_cmds: "break #{arg}") + end + end + end + + # :startdoc: +end diff --git a/lib/irb/command/catch.rb b/lib/irb/command/catch.rb new file mode 100644 index 0000000000..529dcbca5a --- /dev/null +++ b/lib/irb/command/catch.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require_relative "debug" + +module IRB + # :stopdoc: + + module Command + class Catch < DebugCommand + def execute(arg) + execute_debug_command(pre_cmds: "catch #{arg}") + end + end + end + + # :startdoc: +end diff --git a/lib/irb/command/chws.rb b/lib/irb/command/chws.rb new file mode 100644 index 0000000000..ef456d0961 --- /dev/null +++ b/lib/irb/command/chws.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true +# +# change-ws.rb - +# by Keiju ISHITSUKA(keiju@ruby-lang.org) +# +require_relative "../ext/change-ws" + +module IRB + # :stopdoc: + + module Command + + class CurrentWorkingWorkspace < Base + category "Workspace" + description "Show the current workspace." + + def execute(_arg) + puts "Current workspace: #{irb_context.main}" + end + end + + class ChangeWorkspace < Base + category "Workspace" + description "Change the current workspace to an object." + + def execute(arg) + if arg.empty? + irb_context.change_workspace + else + obj = eval(arg, irb_context.workspace.binding) + irb_context.change_workspace(obj) + end + + puts "Current workspace: #{irb_context.main}" + end + end + end + + # :startdoc: +end diff --git a/lib/irb/command/context.rb b/lib/irb/command/context.rb new file mode 100644 index 0000000000..b4fc807343 --- /dev/null +++ b/lib/irb/command/context.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module IRB + module Command + class Context < Base + category "IRB" + description "Displays current configuration." + + def execute(_arg) + # This command just displays the configuration. + # Modifying the configuration is achieved by sending a message to IRB.conf. + Pager.page_content(IRB.CurrentContext.inspect) + end + end + end +end diff --git a/lib/irb/cmd/continue.rb b/lib/irb/command/continue.rb index 9136177eef..0daa029b15 100644 --- a/lib/irb/cmd/continue.rb +++ b/lib/irb/command/continue.rb @@ -5,10 +5,10 @@ require_relative "debug" module IRB # :stopdoc: - module ExtendCommand + module Command class Continue < DebugCommand - def execute(*args) - super(do_cmds: ["continue", *args].join(" ")) + def execute(arg) + execute_debug_command(do_cmds: "continue #{arg}") end end end diff --git a/lib/irb/command/debug.rb b/lib/irb/command/debug.rb new file mode 100644 index 0000000000..8a091a49ed --- /dev/null +++ b/lib/irb/command/debug.rb @@ -0,0 +1,71 @@ +require_relative "../debug" + +module IRB + # :stopdoc: + + module Command + class Debug < Base + category "Debugging" + description "Start the debugger of debug.gem." + + def execute(_arg) + execute_debug_command + end + + def execute_debug_command(pre_cmds: nil, do_cmds: nil) + pre_cmds = pre_cmds&.rstrip + do_cmds = do_cmds&.rstrip + + if irb_context.with_debugger + # If IRB is already running with a debug session, throw the command and IRB.debug_readline will pass it to the debugger. + if cmd = pre_cmds || do_cmds + throw :IRB_EXIT, cmd + else + puts "IRB is already running with a debug session." + return + end + else + # If IRB is not running with a debug session yet, then: + # 1. Check if the debugging command is run from a `binding.irb` call. + # 2. If so, try setting up the debug gem. + # 3. Insert a debug breakpoint at `Irb#debug_break` with the intended command. + # 4. Exit the current Irb#run call via `throw :IRB_EXIT`. + # 5. `Irb#debug_break` will be called and trigger the breakpoint, which will run the intended command. + unless irb_context.from_binding? + puts "Debugging commands are only available when IRB is started with binding.irb" + return + end + + if IRB.respond_to?(:JobManager) + warn "Can't start the debugger when IRB is running in a multi-IRB session." + return + end + + unless IRB::Debug.setup(irb_context.irb) + puts <<~MSG + You need to install the debug gem before using this command. + If you use `bundle exec`, please add `gem "debug"` into your Gemfile. + MSG + return + end + + IRB::Debug.insert_debug_break(pre_cmds: pre_cmds, do_cmds: do_cmds) + + # exit current Irb#run call + throw :IRB_EXIT + end + end + end + + class DebugCommand < Debug + def self.category + "Debugging" + end + + def self.description + command_name = self.name.split("::").last.downcase + "Start the debugger of debug.gem and run its `#{command_name}` command." + end + end + end +end diff --git a/lib/irb/cmd/delete.rb b/lib/irb/command/delete.rb index aeb26d2572..2a57a4a3de 100644 --- a/lib/irb/cmd/delete.rb +++ b/lib/irb/command/delete.rb @@ -5,10 +5,10 @@ require_relative "debug" module IRB # :stopdoc: - module ExtendCommand + module Command class Delete < DebugCommand - def execute(*args) - super(pre_cmds: ["delete", *args].join(" ")) + def execute(arg) + execute_debug_command(pre_cmds: "delete #{arg}") end end end diff --git a/lib/irb/command/disable_irb.rb b/lib/irb/command/disable_irb.rb new file mode 100644 index 0000000000..0b00d0302b --- /dev/null +++ b/lib/irb/command/disable_irb.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module IRB + # :stopdoc: + + module Command + class DisableIrb < Base + category "IRB" + description "Disable binding.irb." + + def execute(*) + ::Binding.define_method(:irb) {} + IRB.irb_exit + end + end + end + + # :startdoc: +end diff --git a/lib/irb/command/edit.rb b/lib/irb/command/edit.rb new file mode 100644 index 0000000000..cb7e0c4873 --- /dev/null +++ b/lib/irb/command/edit.rb @@ -0,0 +1,63 @@ +require 'shellwords' + +require_relative "../color" +require_relative "../source_finder" + +module IRB + # :stopdoc: + + module Command + class Edit < Base + include RubyArgsExtractor + + category "Misc" + description 'Open a file or source location.' + help_message <<~HELP_MESSAGE + Usage: edit [FILE or constant or method signature] + + Open a file in the editor specified in #{highlight('ENV["VISUAL"]')} or #{highlight('ENV["EDITOR"]')} + + - If no arguments are provided, IRB will attempt to open the file the current context was defined in. + - If FILE is provided, IRB will open the file. + - If a constant or method signature is provided, IRB will attempt to locate the source file and open it. + + Examples: + + edit + edit foo.rb + edit Foo + edit Foo#bar + HELP_MESSAGE + + def execute(arg) + # Accept string literal for backward compatibility + path = unwrap_string_literal(arg) + + if path.nil? + path = @irb_context.irb_path + elsif !File.exist?(path) + source = SourceFinder.new(@irb_context).find_source(path) + + if source&.file_exist? && !source.binary_file? + path = source.file + end + end + + unless File.exist?(path) + puts "Can not find file: #{path}" + return + end + + if editor = (ENV['VISUAL'] || ENV['EDITOR']) + puts "command: '#{editor}'" + puts " path: #{path}" + system(*Shellwords.split(editor), path) + else + puts "Can not find editor setting: ENV['VISUAL'] or ENV['EDITOR']" + end + end + end + end + + # :startdoc: +end diff --git a/lib/irb/command/exit.rb b/lib/irb/command/exit.rb new file mode 100644 index 0000000000..b4436f0343 --- /dev/null +++ b/lib/irb/command/exit.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module IRB + # :stopdoc: + + module Command + class Exit < Base + category "IRB" + description "Exit the current irb session." + + def execute(_arg) + IRB.irb_exit + end + end + end + + # :startdoc: +end diff --git a/lib/irb/cmd/finish.rb b/lib/irb/command/finish.rb index 29f100feb5..3311a0e6e9 100644 --- a/lib/irb/cmd/finish.rb +++ b/lib/irb/command/finish.rb @@ -5,10 +5,10 @@ require_relative "debug" module IRB # :stopdoc: - module ExtendCommand + module Command class Finish < DebugCommand - def execute(*args) - super(do_cmds: ["finish", *args].join(" ")) + def execute(arg) + execute_debug_command(do_cmds: "finish #{arg}") end end end diff --git a/lib/irb/command/force_exit.rb b/lib/irb/command/force_exit.rb new file mode 100644 index 0000000000..14086aa849 --- /dev/null +++ b/lib/irb/command/force_exit.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module IRB + # :stopdoc: + + module Command + class ForceExit < Base + category "IRB" + description "Exit the current process." + + def execute(_arg) + throw :IRB_EXIT, true + end + end + end + + # :startdoc: +end diff --git a/lib/irb/command/help.rb b/lib/irb/command/help.rb new file mode 100644 index 0000000000..c2018f9b30 --- /dev/null +++ b/lib/irb/command/help.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module IRB + module Command + class Help < Base + category "Help" + description "List all available commands. Use `help <command>` to get information about a specific command." + + def execute(command_name) + content = + if command_name.empty? + help_message + else + if command_class = Command.load_command(command_name) + command_class.help_message || command_class.description + else + "Can't find command `#{command_name}`. Please check the command name and try again.\n\n" + end + end + Pager.page_content(content) + end + + private + + def help_message + commands_info = IRB::Command.all_commands_info + helper_methods_info = IRB::HelperMethod.all_helper_methods_info + commands_grouped_by_categories = commands_info.group_by { |cmd| cmd[:category] } + commands_grouped_by_categories["Helper methods"] = helper_methods_info + + if irb_context.with_debugger + # Remove the original "Debugging" category + commands_grouped_by_categories.delete("Debugging") + end + + longest_cmd_name_length = commands_info.map { |c| c[:display_name].length }.max + + output = StringIO.new + + help_cmds = commands_grouped_by_categories.delete("Help") + no_category_cmds = commands_grouped_by_categories.delete("No category") + aliases = irb_context.instance_variable_get(:@user_aliases).map do |alias_name, target| + { display_name: alias_name, description: "Alias for `#{target}`" } + end + + # Display help commands first + add_category_to_output("Help", help_cmds, output, longest_cmd_name_length) + + # Display the rest of the commands grouped by categories + commands_grouped_by_categories.each do |category, cmds| + add_category_to_output(category, cmds, output, longest_cmd_name_length) + end + + # Display commands without a category + if no_category_cmds + add_category_to_output("No category", no_category_cmds, output, longest_cmd_name_length) + end + + # Display aliases + add_category_to_output("Aliases", aliases, output, longest_cmd_name_length) + + # Append the debugger help at the end + if irb_context.with_debugger + # Add "Debugging (from debug.gem)" category as title + add_category_to_output("Debugging (from debug.gem)", [], output, longest_cmd_name_length) + output.puts DEBUGGER__.help + end + + output.string + end + + def add_category_to_output(category, cmds, output, longest_cmd_name_length) + output.puts Color.colorize(category, [:BOLD]) + + cmds.each do |cmd| + output.puts " #{cmd[:display_name].to_s.ljust(longest_cmd_name_length)} #{cmd[:description]}" + end + + output.puts + end + end + end +end diff --git a/lib/irb/command/history.rb b/lib/irb/command/history.rb new file mode 100644 index 0000000000..90f87f9102 --- /dev/null +++ b/lib/irb/command/history.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require "stringio" + +require_relative "../pager" + +module IRB + # :stopdoc: + + module Command + class History < Base + category "IRB" + description "Shows the input history. `-g [query]` or `-G [query]` allows you to filter the output." + + def execute(arg) + + if (match = arg&.match(/(-g|-G)\s+(?<grep>.+)\s*\n\z/)) + grep = Regexp.new(match[:grep]) + end + + formatted_inputs = irb_context.io.class::HISTORY.each_with_index.reverse_each.filter_map do |input, index| + next if grep && !input.match?(grep) + + header = "#{index}: " + + first_line, *other_lines = input.split("\n") + first_line = "#{header}#{first_line}" + + truncated_lines = other_lines.slice!(1..) # Show 1 additional line (2 total) + other_lines << "..." if truncated_lines&.any? + + other_lines.map! do |line| + " " * header.length + line + end + + [first_line, *other_lines].join("\n") + "\n" + end + + Pager.page_content(formatted_inputs.join) + end + end + end + + # :startdoc: +end diff --git a/lib/irb/command/info.rb b/lib/irb/command/info.rb new file mode 100644 index 0000000000..d08ce00a32 --- /dev/null +++ b/lib/irb/command/info.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require_relative "debug" + +module IRB + # :stopdoc: + + module Command + class Info < DebugCommand + def execute(arg) + execute_debug_command(pre_cmds: "info #{arg}") + end + end + end + + # :startdoc: +end diff --git a/lib/irb/command/internal_helpers.rb b/lib/irb/command/internal_helpers.rb new file mode 100644 index 0000000000..249b5cdede --- /dev/null +++ b/lib/irb/command/internal_helpers.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module IRB + module Command + # Internal use only, for default command's backward compatibility. + module RubyArgsExtractor # :nodoc: + def unwrap_string_literal(str) + return if str.empty? + + sexp = Ripper.sexp(str) + if sexp && sexp.size == 2 && sexp.last&.first&.first == :string_literal + @irb_context.workspace.binding.eval(str).to_s + else + str + end + end + + def ruby_args(arg) + # Use throw and catch to handle arg that includes `;` + # For example: "1, kw: (2; 3); 4" will be parsed to [[1], { kw: 3 }] + catch(:EXTRACT_RUBY_ARGS) do + @irb_context.workspace.binding.eval "IRB::Command.extract_ruby_args #{arg}" + end || [[], {}] + end + end + end +end diff --git a/lib/irb/cmd/irb_info.rb b/lib/irb/command/irb_info.rb index 75fdc38676..6d868de94c 100644 --- a/lib/irb/cmd/irb_info.rb +++ b/lib/irb/command/irb_info.rb @@ -1,20 +1,20 @@ -# frozen_string_literal: false - -require_relative "nop" +# frozen_string_literal: true module IRB # :stopdoc: - module ExtendCommand - class IrbInfo < Nop + module Command + class IrbInfo < Base category "IRB" description "Show information about IRB." - def execute + def execute(_arg) str = "Ruby version: #{RUBY_VERSION}\n" str += "IRB version: #{IRB.version}\n" str += "InputMethod: #{IRB.CurrentContext.io.inspect}\n" - str += ".irbrc path: #{IRB.rc_file}\n" if File.exist?(IRB.rc_file) + str += "Completion: #{IRB.CurrentContext.io.respond_to?(:completion_info) ? IRB.CurrentContext.io.completion_info : 'off'}\n" + rc_files = IRB.irbrc_files + str += ".irbrc paths: #{rc_files.join(", ")}\n" if rc_files.any? str += "RUBY_PLATFORM: #{RUBY_PLATFORM}\n" str += "LANG env: #{ENV["LANG"]}\n" if ENV["LANG"] && !ENV["LANG"].empty? str += "LC_ALL env: #{ENV["LC_ALL"]}\n" if ENV["LC_ALL"] && !ENV["LC_ALL"].empty? diff --git a/lib/irb/cmd/load.rb b/lib/irb/command/load.rb index a3e797a7e0..1cd3f279d1 100644 --- a/lib/irb/cmd/load.rb +++ b/lib/irb/command/load.rb @@ -1,17 +1,16 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # load.rb - # by Keiju ISHITSUKA(keiju@ruby-lang.org) # - -require_relative "nop" require_relative "../ext/loader" module IRB # :stopdoc: - module ExtendCommand - class LoaderCommand < Nop + module Command + class LoaderCommand < Base + include RubyArgsExtractor include IrbLoader def raise_cmd_argument_error @@ -23,7 +22,12 @@ module IRB category "IRB" description "Load a Ruby file." - def execute(file_name = nil, priv = nil) + def execute(arg) + args, kwargs = ruby_args(arg) + execute_internal(*args, **kwargs) + end + + def execute_internal(file_name = nil, priv = nil) raise_cmd_argument_error unless file_name irb_load(file_name, priv) end @@ -32,7 +36,13 @@ module IRB class Require < LoaderCommand category "IRB" description "Require a Ruby file." - def execute(file_name = nil) + + def execute(arg) + args, kwargs = ruby_args(arg) + execute_internal(*args, **kwargs) + end + + def execute_internal(file_name = nil) raise_cmd_argument_error unless file_name rex = Regexp.new("#{Regexp.quote(file_name)}(\.o|\.rb)?") @@ -65,7 +75,12 @@ module IRB category "IRB" description "Loads a given file in the current session." - def execute(file_name = nil) + def execute(arg) + args, kwargs = ruby_args(arg) + execute_internal(*args, **kwargs) + end + + def execute_internal(file_name = nil) raise_cmd_argument_error unless file_name source_file(file_name) diff --git a/lib/irb/cmd/ls.rb b/lib/irb/command/ls.rb index 063a88d33f..cbd9998bc4 100644 --- a/lib/irb/cmd/ls.rb +++ b/lib/irb/command/ls.rb @@ -1,39 +1,57 @@ # frozen_string_literal: true require "reline" -require_relative "nop" +require "stringio" + +require_relative "../pager" require_relative "../color" module IRB # :stopdoc: - module ExtendCommand - class Ls < Nop + module Command + class Ls < Base + include RubyArgsExtractor + category "Context" - description "Show methods, constants, and variables. `-g [query]` or `-G [query]` allows you to filter out the output." + description "Show methods, constants, and variables." + + help_message <<~HELP_MESSAGE + Usage: ls [obj] [-g [query]] - def self.transform_args(args) - if match = args&.match(/\A(?<args>.+\s|)(-g|-G)\s+(?<grep>[^\s]+)\s*\n\z/) - args = match[:args] - "#{args}#{',' unless args.chomp.empty?} grep: /#{match[:grep]}/" + -g [query] Filter the output with a query. + HELP_MESSAGE + + def execute(arg) + if match = arg.match(/\A(?<target>.+\s|)(-g|-G)\s+(?<grep>.+)$/) + if match[:target].empty? + use_main = true + else + obj = @irb_context.workspace.binding.eval(match[:target]) + end + grep = Regexp.new(match[:grep]) else - args + args, kwargs = ruby_args(arg) + use_main = args.empty? + obj = args.first + grep = kwargs[:grep] + end + + if use_main + obj = irb_context.workspace.main + locals = irb_context.workspace.binding.local_variables end - end - def execute(*arg, grep: nil) o = Output.new(grep: grep) - obj = arg.empty? ? irb_context.workspace.main : arg.first - locals = arg.empty? ? irb_context.workspace.binding.local_variables : [] klass = (obj.class == Class || obj.class == Module ? obj : obj.class) o.dump("constants", obj.constants) if obj.respond_to?(:constants) dump_methods(o, klass, obj) o.dump("instance variables", obj.instance_variables) o.dump("class variables", klass.class_variables) - o.dump("locals", locals) - nil + o.dump("locals", locals) if locals + o.print_result end def dump_methods(o, klass, obj) @@ -77,6 +95,11 @@ module IRB def initialize(grep: nil) @grep = grep @line_width = screen_width - MARGIN.length # right padding + @io = StringIO.new + end + + def print_result + Pager.page_content(@io.string) end def dump(name, strs) @@ -85,12 +108,12 @@ module IRB return if strs.empty? # Attempt a single line - print "#{Color.colorize(name, [:BOLD, :BLUE])}: " + @io.print "#{Color.colorize(name, [:BOLD, :BLUE])}: " if fits_on_line?(strs, cols: strs.size, offset: "#{name}: ".length) - puts strs.join(MARGIN) + @io.puts strs.join(MARGIN) return end - puts + @io.puts # Dump with the largest # of columns that fits on a line cols = strs.size @@ -99,7 +122,7 @@ module IRB end widths = col_widths(strs, cols: cols) strs.each_slice(cols) do |ss| - puts ss.map.with_index { |s, i| "#{MARGIN}%-#{widths[i]}s" % s }.join + @io.puts ss.map.with_index { |s, i| "#{MARGIN}%-#{widths[i]}s" % s }.join end end diff --git a/lib/irb/cmd/measure.rb b/lib/irb/command/measure.rb index 9122e2dac9..f96be20de8 100644 --- a/lib/irb/cmd/measure.rb +++ b/lib/irb/command/measure.rb @@ -1,10 +1,10 @@ -require_relative "nop" - module IRB # :stopdoc: - module ExtendCommand - class Measure < Nop + module Command + class Measure < Base + include RubyArgsExtractor + category "Misc" description "`measure` enables the mode to measure processing time. `measure :off` disables it." @@ -12,32 +12,33 @@ module IRB super(*args) end - def execute(type = nil, arg = nil, &block) + def execute(arg) + if arg&.match?(/^do$|^do[^\w]|^\{/) + warn 'Configure IRB.conf[:MEASURE_PROC] to add custom measure methods.' + return + end + args, kwargs = ruby_args(arg) + execute_internal(*args, **kwargs) + end + + def execute_internal(type = nil, arg = nil) # Please check IRB.init_config in lib/irb/init.rb that sets # IRB.conf[:MEASURE_PROC] to register default "measure" methods, # "measure :time" (abbreviated as "measure") and "measure :stackprof". + case type when :off - IRB.conf[:MEASURE] = nil IRB.unset_measure_callback(arg) when :list IRB.conf[:MEASURE_CALLBACKS].each do |type_name, _, arg_val| puts "- #{type_name}" + (arg_val ? "(#{arg_val.inspect})" : '') end when :on - IRB.conf[:MEASURE] = true - added = IRB.set_measure_callback(type, arg) + added = IRB.set_measure_callback(arg) puts "#{added[0]} is added." if added else - if block_given? - IRB.conf[:MEASURE] = true - added = IRB.set_measure_callback(&block) - puts "#{added[0]} is added." if added - else - IRB.conf[:MEASURE] = true - added = IRB.set_measure_callback(type, arg) - puts "#{added[0]} is added." if added - end + added = IRB.set_measure_callback(type, arg) + puts "#{added[0]} is added." if added end nil end diff --git a/lib/irb/cmd/next.rb b/lib/irb/command/next.rb index d29c82e7fc..3fc6b68d21 100644 --- a/lib/irb/cmd/next.rb +++ b/lib/irb/command/next.rb @@ -5,10 +5,10 @@ require_relative "debug" module IRB # :stopdoc: - module ExtendCommand + module Command class Next < DebugCommand - def execute(*args) - super(do_cmds: ["next", *args].join(" ")) + def execute(arg) + execute_debug_command(do_cmds: "next #{arg}") end end end diff --git a/lib/irb/command/pushws.rb b/lib/irb/command/pushws.rb new file mode 100644 index 0000000000..b51928c650 --- /dev/null +++ b/lib/irb/command/pushws.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true +# +# change-ws.rb - +# by Keiju ISHITSUKA(keiju@ruby-lang.org) +# + +require_relative "../ext/workspaces" + +module IRB + # :stopdoc: + + module Command + class Workspaces < Base + category "Workspace" + description "Show workspaces." + + def execute(_arg) + inspection_resuls = irb_context.instance_variable_get(:@workspace_stack).map do |ws| + truncated_inspect(ws.main) + end + + puts "[" + inspection_resuls.join(", ") + "]" + end + + private + + def truncated_inspect(obj) + obj_inspection = obj.inspect + + if obj_inspection.size > 20 + obj_inspection = obj_inspection[0, 19] + "...>" + end + + obj_inspection + end + end + + class PushWorkspace < Workspaces + category "Workspace" + description "Push an object to the workspace stack." + + def execute(arg) + if arg.empty? + irb_context.push_workspace + else + obj = eval(arg, irb_context.workspace.binding) + irb_context.push_workspace(obj) + end + super + end + end + + class PopWorkspace < Workspaces + category "Workspace" + description "Pop a workspace from the workspace stack." + + def execute(_arg) + irb_context.pop_workspace + super + end + end + end + + # :startdoc: +end diff --git a/lib/irb/command/show_doc.rb b/lib/irb/command/show_doc.rb new file mode 100644 index 0000000000..8a2188e4eb --- /dev/null +++ b/lib/irb/command/show_doc.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module IRB + module Command + class ShowDoc < Base + include RubyArgsExtractor + + category "Context" + description "Look up documentation with RI." + + help_message <<~HELP_MESSAGE + Usage: show_doc [name] + + When name is provided, IRB will look up the documentation for the given name. + When no name is provided, a RI session will be started. + + Examples: + + show_doc + show_doc Array + show_doc Array#each + + HELP_MESSAGE + + def execute(arg) + # Accept string literal for backward compatibility + name = unwrap_string_literal(arg) + require 'rdoc/ri/driver' + + unless ShowDoc.const_defined?(:Ri) + opts = RDoc::RI::Driver.process_args([]) + ShowDoc.const_set(:Ri, RDoc::RI::Driver.new(opts)) + end + + if name.nil? + Ri.interactive + else + begin + Ri.display_name(name) + rescue RDoc::RI::Error + puts $!.message + end + end + + nil + rescue LoadError, SystemExit + warn "Can't display document because `rdoc` is not installed." + end + end + end +end diff --git a/lib/irb/command/show_source.rb b/lib/irb/command/show_source.rb new file mode 100644 index 0000000000..f4c6f104a2 --- /dev/null +++ b/lib/irb/command/show_source.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require_relative "../source_finder" +require_relative "../pager" +require_relative "../color" + +module IRB + module Command + class ShowSource < Base + include RubyArgsExtractor + + category "Context" + description "Show the source code of a given method, class/module, or constant." + + help_message <<~HELP_MESSAGE + Usage: show_source [target] [-s] + + -s Show the super method. You can stack it like `-ss` to show the super of the super, etc. + + Examples: + + show_source Foo + show_source Foo#bar + show_source Foo#bar -s + show_source Foo.baz + show_source Foo::BAR + HELP_MESSAGE + + def execute(arg) + # Accept string literal for backward compatibility + str = unwrap_string_literal(arg) + unless str.is_a?(String) + puts "Error: Expected a string but got #{str.inspect}" + return + end + + str, esses = str.split(" -") + super_level = esses ? esses.count("s") : 0 + source = SourceFinder.new(@irb_context).find_source(str, super_level) + + if source + show_source(source) + elsif super_level > 0 + puts "Error: Couldn't locate a super definition for #{str}" + else + puts "Error: Couldn't locate a definition for #{str}" + end + nil + end + + private + + def show_source(source) + if source.binary_file? + content = "\n#{bold('Defined in binary file')}: #{source.file}\n\n" + else + code = source.colorized_content || 'Source not available' + content = <<~CONTENT + + #{bold("From")}: #{source.file}:#{source.line} + + #{code.chomp} + + CONTENT + end + Pager.page_content(content) + end + + def bold(str) + Color.colorize(str, [:BOLD]) + end + end + end +end diff --git a/lib/irb/cmd/step.rb b/lib/irb/command/step.rb index 2bc74a9d79..29e5e35ac0 100644 --- a/lib/irb/cmd/step.rb +++ b/lib/irb/command/step.rb @@ -5,10 +5,10 @@ require_relative "debug" module IRB # :stopdoc: - module ExtendCommand + module Command class Step < DebugCommand - def execute(*args) - super(do_cmds: ["step", *args].join(" ")) + def execute(arg) + execute_debug_command(do_cmds: "step #{arg}") end end end diff --git a/lib/irb/command/subirb.rb b/lib/irb/command/subirb.rb new file mode 100644 index 0000000000..85af28c1a5 --- /dev/null +++ b/lib/irb/command/subirb.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true +# +# multi.rb - +# by Keiju ISHITSUKA(keiju@ruby-lang.org) +# + +module IRB + # :stopdoc: + + module Command + class MultiIRBCommand < Base + include RubyArgsExtractor + + private + + def print_deprecated_warning + warn <<~MSG + Multi-irb commands are deprecated and will be removed in IRB 2.0.0. Please use workspace commands instead. + If you have any use case for multi-irb, please leave a comment at https://github.com/ruby/irb/issues/653 + MSG + end + + def extend_irb_context + # this extension patches IRB context like IRB.CurrentContext + require_relative "../ext/multi-irb" + end + + def print_debugger_warning + warn "Multi-IRB commands are not available when the debugger is enabled." + end + end + + class IrbCommand < MultiIRBCommand + category "Multi-irb (DEPRECATED)" + description "Start a child IRB." + + def execute(arg) + args, kwargs = ruby_args(arg) + execute_internal(*args, **kwargs) + end + + def execute_internal(*obj) + print_deprecated_warning + + if irb_context.with_debugger + print_debugger_warning + return + end + + extend_irb_context + IRB.irb(nil, *obj) + puts IRB.JobManager.inspect + end + end + + class Jobs < MultiIRBCommand + category "Multi-irb (DEPRECATED)" + description "List of current sessions." + + def execute(_arg) + print_deprecated_warning + + if irb_context.with_debugger + print_debugger_warning + return + end + + extend_irb_context + puts IRB.JobManager.inspect + end + end + + class Foreground < MultiIRBCommand + category "Multi-irb (DEPRECATED)" + description "Switches to the session of the given number." + + def execute(arg) + args, kwargs = ruby_args(arg) + execute_internal(*args, **kwargs) + end + + def execute_internal(key = nil) + print_deprecated_warning + + if irb_context.with_debugger + print_debugger_warning + return + end + + extend_irb_context + + raise CommandArgumentError.new("Please specify the id of target IRB job (listed in the `jobs` command).") unless key + IRB.JobManager.switch(key) + puts IRB.JobManager.inspect + end + end + + class Kill < MultiIRBCommand + category "Multi-irb (DEPRECATED)" + description "Kills the session with the given number." + + def execute(arg) + args, kwargs = ruby_args(arg) + execute_internal(*args, **kwargs) + end + + def execute_internal(*keys) + print_deprecated_warning + + if irb_context.with_debugger + print_debugger_warning + return + end + + extend_irb_context + IRB.JobManager.kill(*keys) + puts IRB.JobManager.inspect + end + end + end + + # :startdoc: +end diff --git a/lib/irb/cmd/whereami.rb b/lib/irb/command/whereami.rb index 8f56ba073d..c8439f1212 100644 --- a/lib/irb/cmd/whereami.rb +++ b/lib/irb/command/whereami.rb @@ -1,16 +1,14 @@ # frozen_string_literal: true -require_relative "nop" - module IRB # :stopdoc: - module ExtendCommand - class Whereami < Nop + module Command + class Whereami < Base category "Context" description "Show the source code around binding.irb again." - def execute(*) + def execute(_arg) code = irb_context.workspace.code_around_binding if code puts code diff --git a/lib/irb/completion.rb b/lib/irb/completion.rb index 64e387d49c..a3d89373c3 100644 --- a/lib/irb/completion.rb +++ b/lib/irb/completion.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/completion.rb - # by Keiju ISHITSUKA(keiju@ishitsuka.com) @@ -8,30 +8,7 @@ require_relative 'ruby-lex' module IRB - module InputCompletor # :nodoc: - using Module.new { - refine ::Binding do - def eval_methods - ::Kernel.instance_method(:methods).bind(eval("self")).call - end - - def eval_private_methods - ::Kernel.instance_method(:private_methods).bind(eval("self")).call - end - - def eval_instance_variables - ::Kernel.instance_method(:instance_variables).bind(eval("self")).call - end - - def eval_global_variables - ::Kernel.instance_method(:global_variables).bind(eval("self")).call - end - - def eval_class_constants - ::Module.instance_method(:constants).bind(eval("self.class")).call - end - end - } + class BaseCompletor # :nodoc: # Set of reserved words used by Ruby, you should not use these for # constants or variables @@ -56,7 +33,13 @@ module IRB yield ] - BASIC_WORD_BREAK_CHARACTERS = " \t\n`><=;|&{(" + def completion_candidates(preposing, target, postposing, bind:) + raise NotImplementedError + end + + def doc_namespace(preposing, matched, postposing, bind:) + raise NotImplementedError + end GEM_PATHS = if defined?(Gem::Specification) @@ -73,7 +56,7 @@ module IRB [] end.freeze - def self.retrieve_gem_and_system_load_path + def retrieve_gem_and_system_load_path candidates = (GEM_PATHS | $LOAD_PATH) candidates.map do |p| if p.respond_to?(:to_path) @@ -84,8 +67,8 @@ module IRB end.compact.sort end - def self.retrieve_files_to_require_from_load_path - @@files_from_load_path ||= + def retrieve_files_to_require_from_load_path + @files_from_load_path ||= ( shortest = [] rest = retrieve_gem_and_system_load_path.each_with_object([]) { |path, result| @@ -103,13 +86,74 @@ module IRB ) end - def self.retrieve_files_to_require_relative_from_current_dir - @@files_from_current_dir ||= Dir.glob("**/*.{rb,#{RbConfig::CONFIG['DLEXT']}}", base: '.').map { |path| + def command_completions(preposing, target) + if preposing.empty? && !target.empty? + IRB::Command.command_names.select { _1.start_with?(target) } + else + [] + end + end + + def retrieve_files_to_require_relative_from_current_dir + @files_from_current_dir ||= Dir.glob("**/*.{rb,#{RbConfig::CONFIG['DLEXT']}}", base: '.').map { |path| path.sub(/\.(rb|#{RbConfig::CONFIG['DLEXT']})\z/, '') } end + end + + class TypeCompletor < BaseCompletor # :nodoc: + def initialize(context) + @context = context + end + + def inspect + ReplTypeCompletor.info + end + + def completion_candidates(preposing, target, _postposing, bind:) + commands = command_completions(preposing, target) + result = ReplTypeCompletor.analyze(preposing + target, binding: bind, filename: @context.irb_path) + return commands unless result + + commands | result.completion_candidates.map { target + _1 } + end + + def doc_namespace(preposing, matched, _postposing, bind:) + result = ReplTypeCompletor.analyze(preposing + matched, binding: bind, filename: @context.irb_path) + result&.doc_namespace('') + end + end + + class RegexpCompletor < BaseCompletor # :nodoc: + using Module.new { + refine ::Binding do + def eval_methods + ::Kernel.instance_method(:methods).bind(eval("self")).call + end + + def eval_private_methods + ::Kernel.instance_method(:private_methods).bind(eval("self")).call + end + + def eval_instance_variables + ::Kernel.instance_method(:instance_variables).bind(eval("self")).call + end + + def eval_global_variables + ::Kernel.instance_method(:global_variables).bind(eval("self")).call + end + + def eval_class_constants + ::Module.instance_method(:constants).bind(eval("self.class")).call + end + end + } + + def inspect + 'RegexpCompletor' + end - CompletionRequireProc = lambda { |target, preposing = nil, postposing = nil| + def complete_require_path(target, preposing, postposing) if target =~ /\A(['"])([^'"]+)\Z/ quote = $1 actual_target = $2 @@ -124,39 +168,38 @@ module IRB break end end - result = [] - if tok && tok.event == :on_ident && tok.state == Ripper::EXPR_CMDARG - case tok.tok - when 'require' - result = retrieve_files_to_require_from_load_path.select { |path| - path.start_with?(actual_target) - }.map { |path| - quote + path - } - when 'require_relative' - result = retrieve_files_to_require_relative_from_current_dir.select { |path| - path.start_with?(actual_target) - }.map { |path| - quote + path - } - end + return unless tok&.event == :on_ident && tok.state == Ripper::EXPR_CMDARG + + case tok.tok + when 'require' + retrieve_files_to_require_from_load_path.select { |path| + path.start_with?(actual_target) + }.map { |path| + quote + path + } + when 'require_relative' + retrieve_files_to_require_relative_from_current_dir.select { |path| + path.start_with?(actual_target) + }.map { |path| + quote + path + } end - result - } + end - CompletionProc = lambda { |target, preposing = nil, postposing = nil| + def completion_candidates(preposing, target, postposing, bind:) if preposing && postposing - result = CompletionRequireProc.(target, preposing, postposing) - unless result - result = retrieve_completion_data(target).compact.map{ |i| i.encode(Encoding.default_external) } - end - result - else - retrieve_completion_data(target).compact.map{ |i| i.encode(Encoding.default_external) } + result = complete_require_path(target, preposing, postposing) + return result if result end - } + commands = command_completions(preposing || '', target) + commands | retrieve_completion_data(target, bind: bind, doc_namespace: false).compact.map{ |i| i.encode(Encoding.default_external) } + end + + def doc_namespace(_preposing, matched, _postposing, bind:) + retrieve_completion_data(matched, bind: bind, doc_namespace: true) + end - def self.retrieve_completion_data(input, bind: IRB.conf[:MAIN_CONTEXT].workspace.binding, doc_namespace: false) + def retrieve_completion_data(input, bind:, doc_namespace:) case input # this regexp only matches the closing character because of irb's Reline.completer_quote_characters setting # details are described in: https://github.com/ruby/irb/pull/523 @@ -199,16 +242,16 @@ module IRB end when /^([^\}]*\})\.([^.]*)$/ - # Proc or Hash + # Hash or Proc receiver = $1 message = $2 if doc_namespace - ["Proc.#{message}", "Hash.#{message}"] + ["Hash.#{message}", "Proc.#{message}"] else - proc_candidates = Proc.instance_methods.collect{|m| m.to_s} hash_candidates = Hash.instance_methods.collect{|m| m.to_s} - select_message(receiver, message, proc_candidates | hash_candidates) + proc_candidates = Proc.instance_methods.collect{|m| m.to_s} + select_message(receiver, message, hash_candidates | proc_candidates) end when /^(:[^:.]+)$/ @@ -218,7 +261,7 @@ module IRB else sym = $1 candidates = Symbol.all_symbols.collect do |s| - ":" + s.id2name.encode(Encoding.default_external) + s.inspect rescue EncodingError # ignore end @@ -356,7 +399,7 @@ module IRB if doc_namespace rec_class = rec.is_a?(Module) ? rec : rec.class - "#{rec_class.name}#{sep}#{candidates.find{ |i| i == message }}" + "#{rec_class.name}#{sep}#{candidates.find{ |i| i == message }}" rescue nil else select_message(receiver, message, candidates, sep) end @@ -374,13 +417,19 @@ module IRB else select_message(receiver, message, candidates.sort) end - + when /^\s*$/ + # empty input + if doc_namespace + nil + else + [] + end else if doc_namespace vars = (bind.local_variables | bind.eval_instance_variables).collect{|m| m.to_s} perfect_match_var = vars.find{|m| m.to_s == input} if perfect_match_var - eval("#{perfect_match_var}.class.name", bind) + eval("#{perfect_match_var}.class.name", bind) rescue nil else candidates = (bind.eval_methods | bind.eval_private_methods | bind.local_variables | bind.eval_instance_variables | bind.eval_class_constants).collect{|m| m.to_s} candidates |= ReservedWords @@ -394,44 +443,10 @@ module IRB end end - PerfectMatchedProc = ->(matched, bind: IRB.conf[:MAIN_CONTEXT].workspace.binding) { - begin - require 'rdoc' - rescue LoadError - return - end - - RDocRIDriver ||= RDoc::RI::Driver.new - - if matched =~ /\A(?:::)?RubyVM/ and not ENV['RUBY_YES_I_AM_NOT_A_NORMAL_USER'] - IRB.__send__(:easter_egg) - return - end - - namespace = retrieve_completion_data(matched, bind: bind, doc_namespace: true) - return unless namespace - - if namespace.is_a?(Array) - out = RDoc::Markup::Document.new - namespace.each do |m| - begin - RDocRIDriver.add_method(out, m) - rescue RDoc::RI::Driver::NotFoundError - end - end - RDocRIDriver.display(out) - else - begin - RDocRIDriver.display_names([namespace]) - rescue RDoc::RI::Driver::NotFoundError - end - end - } - # Set of available operators in Ruby Operators = %w[% & * ** + - / < << <= <=> == === =~ > >= >> [] []= ^ ! != !~] - def self.select_message(receiver, message, candidates, sep = ".") + def select_message(receiver, message, candidates, sep = ".") candidates.grep(/^#{Regexp.quote(message)}/).collect do |e| case e when /^[a-zA-Z_]/ @@ -443,4 +458,20 @@ module IRB end end end + + module InputCompletor + class << self + private def regexp_completor + @regexp_completor ||= RegexpCompletor.new + end + + def retrieve_completion_data(input, bind: IRB.conf[:MAIN_CONTEXT].workspace.binding, doc_namespace: false) + regexp_completor.retrieve_completion_data(input, bind: bind, doc_namespace: doc_namespace) + end + end + CompletionProc = ->(target, preposing = nil, postposing = nil) { + regexp_completor.completion_candidates(preposing, target, postposing, bind: IRB.conf[:MAIN_CONTEXT].workspace.binding) + } + end + deprecate_constant :InputCompletor end diff --git a/lib/irb/context.rb b/lib/irb/context.rb index f398c6f75d..aafce7aade 100644 --- a/lib/irb/context.rb +++ b/lib/irb/context.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/context.rb - irb context # by Keiju ISHITSUKA(keiju@ruby-lang.org) @@ -22,10 +22,11 @@ module IRB # +other+:: uses this as InputMethod def initialize(irb, workspace = nil, input_method = nil) @irb = irb + @workspace_stack = [] if workspace - @workspace = workspace + @workspace_stack << workspace else - @workspace = WorkSpace.new + @workspace_stack << WorkSpace.new end @thread = Thread.current @@ -61,7 +62,7 @@ module IRB @io = nil self.inspect_mode = IRB.conf[:INSPECT_MODE] - self.use_tracer = IRB.conf[:USE_TRACER] if IRB.conf[:USE_TRACER] + self.use_tracer = IRB.conf[:USE_TRACER] self.use_loader = IRB.conf[:USE_LOADER] if IRB.conf[:USE_LOADER] self.eval_history = IRB.conf[:EVAL_HISTORY] if IRB.conf[:EVAL_HISTORY] @@ -72,33 +73,34 @@ module IRB self.prompt_mode = IRB.conf[:PROMPT_MODE] - if IRB.conf[:SINGLE_IRB] or !defined?(IRB::JobManager) - @irb_name = IRB.conf[:IRB_NAME] - else - @irb_name = IRB.conf[:IRB_NAME]+"#"+IRB.JobManager.n_jobs.to_s + @irb_name = IRB.conf[:IRB_NAME] + + unless IRB.conf[:SINGLE_IRB] or !defined?(IRB::JobManager) + @irb_name = @irb_name + "#" + IRB.JobManager.n_jobs.to_s end - @irb_path = "(" + @irb_name + ")" + + self.irb_path = "(" + @irb_name + ")" case input_method when nil @io = nil case use_multiline? when nil - if STDIN.tty? && IRB.conf[:PROMPT_MODE] != :INF_RUBY && !use_singleline? + if term_interactive? && IRB.conf[:PROMPT_MODE] != :INF_RUBY && !use_singleline? # Both of multiline mode and singleline mode aren't specified. - @io = RelineInputMethod.new + @io = RelineInputMethod.new(build_completor) else @io = nil end when false @io = nil when true - @io = RelineInputMethod.new + @io = RelineInputMethod.new(build_completor) end unless @io case use_singleline? when nil - if (defined?(ReadlineInputMethod) && STDIN.tty? && + if (defined?(ReadlineInputMethod) && term_interactive? && IRB.conf[:PROMPT_MODE] != :INF_RUBY) @io = ReadlineInputMethod.new else @@ -121,16 +123,14 @@ module IRB when '-' @io = FileInputMethod.new($stdin) @irb_name = '-' - @irb_path = '-' + self.irb_path = '-' when String @io = FileInputMethod.new(input_method) @irb_name = File.basename(input_method) - @irb_path = input_method + self.irb_path = input_method else @io = input_method end - self.save_history = IRB.conf[:SAVE_HISTORY] if IRB.conf[:SAVE_HISTORY] - @extra_doc_dirs = IRB.conf[:EXTRA_DOC_DIRS] @echo = IRB.conf[:ECHO] @@ -148,18 +148,112 @@ module IRB @newline_before_multiline_output = true end - @command_aliases = IRB.conf[:COMMAND_ALIASES] + @user_aliases = IRB.conf[:COMMAND_ALIASES].dup + @command_aliases = @user_aliases.merge(KEYWORD_ALIASES) + end + + private def term_interactive? + return true if ENV['TEST_IRB_FORCE_INTERACTIVE'] + STDIN.tty? && ENV['TERM'] != 'dumb' + end + + # because all input will eventually be evaluated as Ruby code, + # command names that conflict with Ruby keywords need special workaround + # we can remove them once we implemented a better command system for IRB + KEYWORD_ALIASES = { + :break => :irb_break, + :catch => :irb_catch, + :next => :irb_next, + }.freeze + + private_constant :KEYWORD_ALIASES + + def use_tracer=(val) + require_relative "ext/tracer" if val + IRB.conf[:USE_TRACER] = val + end + + def eval_history=(val) + self.class.remove_method(__method__) + require_relative "ext/eval_history" + __send__(__method__, val) + end + + def use_loader=(val) + self.class.remove_method(__method__) + require_relative "ext/use-loader" + __send__(__method__, val) + end + + private def build_completor + completor_type = IRB.conf[:COMPLETOR] + case completor_type + when :regexp + return RegexpCompletor.new + when :type + completor = build_type_completor + return completor if completor + else + warn "Invalid value for IRB.conf[:COMPLETOR]: #{completor_type}" + end + # Fallback to RegexpCompletor + RegexpCompletor.new + end + + private def build_type_completor + if RUBY_ENGINE == 'truffleruby' + # Avoid SyntaxError. truffleruby does not support endless method definition yet. + warn 'TypeCompletor is not supported on TruffleRuby yet' + return + end + + begin + require 'repl_type_completor' + rescue LoadError => e + warn "TypeCompletor requires `gem repl_type_completor`: #{e.message}" + return + end + + ReplTypeCompletor.preload_rbs + TypeCompletor.new(self) + end + + def save_history=(val) + IRB.conf[:SAVE_HISTORY] = val + end + + def save_history + IRB.conf[:SAVE_HISTORY] + end + + # A copy of the default <code>IRB.conf[:HISTORY_FILE]</code> + def history_file + IRB.conf[:HISTORY_FILE] + end + + # Set <code>IRB.conf[:HISTORY_FILE]</code> to the given +hist+. + def history_file=(hist) + IRB.conf[:HISTORY_FILE] = hist + end + + # Workspace in the current context. + def workspace + @workspace_stack.last + end + + # Replace the current workspace with the given +workspace+. + def replace_workspace(workspace) + @workspace_stack.pop + @workspace_stack.push(workspace) end # The top-level workspace, see WorkSpace#main def main - @workspace.main + workspace.main end # The toplevel workspace, see #home_workspace attr_reader :workspace_home - # WorkSpace in the current context. - attr_accessor :workspace # The current thread in this context. attr_reader :thread # The current input method. @@ -180,9 +274,27 @@ module IRB # Can be either name from <code>IRB.conf[:IRB_NAME]</code>, or the number of # the current job set by JobManager, such as <code>irb#2</code> attr_accessor :irb_name - # Can be either the #irb_name surrounded by parenthesis, or the - # +input_method+ passed to Context.new - attr_accessor :irb_path + + # Can be one of the following: + # - the #irb_name surrounded by parenthesis + # - the +input_method+ passed to Context.new + # - the file path of the current IRB context in a binding.irb session + attr_reader :irb_path + + # Sets @irb_path to the given +path+ as well as @eval_path + # @eval_path is used for evaluating code in the context of IRB session + # It's the same as irb_path, but with the IRB name postfix + # This makes sure users can distinguish the methods defined in the IRB session + # from the methods defined in the current file's context, especially with binding.irb + def irb_path=(path) + @irb_path = path + + if File.exist?(path) + @eval_path = "#{path}(#{IRB.conf[:IRB_NAME]})" + else + @eval_path = path + end + end # Whether multiline editor mode is enabled or not. # @@ -203,18 +315,29 @@ module IRB attr_reader :prompt_mode # Standard IRB prompt. # - # See IRB@Customizing+the+IRB+Prompt for more information. + # See {Custom Prompts}[rdoc-ref:IRB@Custom+Prompts] for more information. attr_accessor :prompt_i # IRB prompt for continuated strings. # - # See IRB@Customizing+the+IRB+Prompt for more information. + # See {Custom Prompts}[rdoc-ref:IRB@Custom+Prompts] for more information. attr_accessor :prompt_s # IRB prompt for continuated statement. (e.g. immediately after an +if+) # - # See IRB@Customizing+the+IRB+Prompt for more information. + # See {Custom Prompts}[rdoc-ref:IRB@Custom+Prompts] for more information. attr_accessor :prompt_c - # See IRB@Customizing+the+IRB+Prompt for more information. - attr_accessor :prompt_n + + # TODO: Remove this when developing v2.0 + def prompt_n + warn "IRB::Context#prompt_n is deprecated and will be removed in the next major release." + "" + end + + # TODO: Remove this when developing v2.0 + def prompt_n=(_) + warn "IRB::Context#prompt_n= is deprecated and will be removed in the next major release." + "" + end + # Can be either the default <code>IRB.conf[:AUTO_INDENT]</code>, or the # mode set by #prompt_mode= # @@ -322,13 +445,13 @@ module IRB # The default value is 16. # # Can also be set using the +--back-trace-limit+ command line option. - # - # See IRB@Command+line+options for more command line options. attr_accessor :back_trace_limit # User-defined IRB command aliases attr_accessor :command_aliases + attr_accessor :with_debugger + # Alias for #use_multiline alias use_multiline? use_multiline # Alias for #use_singleline @@ -372,9 +495,7 @@ module IRB # StdioInputMethod or RelineInputMethod or ReadlineInputMethod, see #io # for more information. def prompting? - verbose? || (STDIN.tty? && @io.kind_of?(StdioInputMethod) || - @io.kind_of?(RelineInputMethod) || - (defined?(ReadlineInputMethod) && @io.kind_of?(ReadlineInputMethod))) + verbose? || @io.prompting? end # The return value of the last statement evaluated. @@ -384,19 +505,18 @@ module IRB # to #last_value. def set_last_value(value) @last_value = value - @workspace.local_variable_set :_, value + workspace.local_variable_set :_, value end # Sets the +mode+ of the prompt in this context. # - # See IRB@Customizing+the+IRB+Prompt for more information. + # See {Custom Prompts}[rdoc-ref:IRB@Custom+Prompts] for more information. def prompt_mode=(mode) @prompt_mode = mode pconf = IRB.conf[:PROMPT][mode] @prompt_i = pconf[:PROMPT_I] @prompt_s = pconf[:PROMPT_S] @prompt_c = pconf[:PROMPT_C] - @prompt_n = pconf[:PROMPT_N] @return_format = pconf[:RETURN] @return_format = "%s\n" if @return_format == nil if ai = pconf.include?(:AUTO_INDENT) @@ -428,8 +548,6 @@ module IRB # # Can also be set using the +--inspect+ and +--noinspect+ command line # options. - # - # See IRB@Command+line+options for more command line options. def inspect_mode=(opt) if i = Inspector::INSPECTORS[opt] @@ -473,42 +591,55 @@ module IRB @inspect_mode end - def evaluate(line, line_no, exception: nil) # :nodoc: + def evaluate(statement, line_no) # :nodoc: @line_no = line_no - if exception - line_no -= 1 - line = "begin ::Kernel.raise _; rescue _.class\n#{line}\n""end" - @workspace.local_variable_set(:_, exception) - end - # Transform a non-identifier alias (@, $) or keywords (next, break) - command, args = line.split(/\s/, 2) - if original = command_aliases[command.to_sym] - line = line.gsub(/\A#{Regexp.escape(command)}/, original.to_s) - command = original + case statement + when Statement::EmptyInput + return + when Statement::Expression + result = evaluate_expression(statement.code, line_no) + set_last_value(result) + when Statement::Command + statement.command_class.execute(self, statement.arg) + set_last_value(nil) end - # Hook command-specific transformation - command_class = ExtendCommandBundle.load_command(command) - if command_class&.respond_to?(:transform_args) - line = "#{command} #{command_class.transform_args(args)}" + nil + end + + def from_binding? + @irb.from_binding + end + + def evaluate_expression(code, line_no) # :nodoc: + result = nil + if IRB.conf[:MEASURE] && IRB.conf[:MEASURE_CALLBACKS].empty? + IRB.set_measure_callback end - set_last_value(@workspace.evaluate(line, irb_path, line_no)) + if IRB.conf[:MEASURE] && !IRB.conf[:MEASURE_CALLBACKS].empty? + last_proc = proc do + result = workspace.evaluate(code, @eval_path, line_no) + end + IRB.conf[:MEASURE_CALLBACKS].inject(last_proc) do |chain, item| + _name, callback, arg = item + proc do + callback.(self, code, line_no, arg) do + chain.call + end + end + end.call + else + result = workspace.evaluate(code, @eval_path, line_no) + end + result end def inspect_last_value # :nodoc: @inspect_method.inspect_value(@last_value) end - alias __exit__ exit - # Exits the current session, see IRB.irb_exit - def exit(ret = 0) - IRB.irb_exit(@irb, ret) - rescue UncaughtThrowError - super - end - NOPRINTING_IVARS = ["@last_value"] # :nodoc: NO_INSPECTING_IVARS = ["@irb", "@io"] # :nodoc: IDNAME_IVARS = ["@prompt_mode"] # :nodoc: @@ -539,17 +670,5 @@ module IRB def local_variables # :nodoc: workspace.binding.local_variables end - - # Return true if it's aliased from the argument and it's not an identifier. - def symbol_alias?(command) - return nil if command.match?(/\A\w+\z/) - command_aliases.key?(command.to_sym) - end - - # Return true if the command supports transforming args - def transform_args?(command) - command = command_aliases.fetch(command.to_sym, command) - ExtendCommandBundle.load_command(command)&.respond_to?(:transform_args) - end end end diff --git a/lib/irb/cmd/debug.rb b/lib/irb/debug.rb index 7d39b9fa27..1ec2335a8e 100644 --- a/lib/irb/cmd/debug.rb +++ b/lib/irb/debug.rb @@ -1,34 +1,13 @@ -require_relative "nop" +# frozen_string_literal: true module IRB - # :stopdoc: - - module ExtendCommand - class Debug < Nop - category "Debugging" - description "Start the debugger of debug.gem." - - BINDING_IRB_FRAME_REGEXPS = [ - '<internal:prelude>', - binding.method(:irb).source_location.first, - ].map { |file| /\A#{Regexp.escape(file)}:\d+:in `irb'\z/ } - IRB_DIR = File.expand_path('..', __dir__) - - def execute(pre_cmds: nil, do_cmds: nil) - unless binding_irb? - puts "`debug` command is only available when IRB is started with binding.irb" - return - end - - unless setup_debugger - puts <<~MSG - You need to install the debug gem before using this command. - If you use `bundle exec`, please add `gem "debug"` into your Gemfile. - MSG - return - end + module Debug + IRB_DIR = File.expand_path('..', __dir__) + class << self + def insert_debug_break(pre_cmds: nil, do_cmds: nil) options = { oneshot: true, hook_call: false } + if pre_cmds || do_cmds options[:command] = ['irb', pre_cmds, do_cmds] end @@ -41,37 +20,29 @@ module IRB # a TracePoint on #debug_break. file, lineno = IRB::Irb.instance_method(:debug_break).source_location DEBUGGER__::SESSION.add_line_breakpoint(file, lineno + 1, **options) - # exit current Irb#run call - throw :IRB_EXIT - end - - private - - def binding_irb? - caller.any? do |frame| - BINDING_IRB_FRAME_REGEXPS.any? do |regexp| - frame.match?(regexp) - end - end - end - - module SkipPathHelperForIRB - def skip_internal_path?(path) - # The latter can be removed once https://github.com/ruby/debug/issues/866 is resolved - super || path.match?(IRB_DIR) || path.match?('<internal:prelude>') - end end - def setup_debugger + def setup(irb) + # When debug session is not started at all unless defined?(DEBUGGER__::SESSION) begin require "debug/session" rescue LoadError # debug.gem is not written in Gemfile return false unless load_bundled_debug_gem end - DEBUGGER__.start(nonstop: true) + DEBUGGER__::CONFIG.set_config + configure_irb_for_debugger(irb) + + DEBUGGER__.initialize_session{ IRB::Debug::UI.new(irb) } + end + + # When debug session was previously started but not by IRB + if defined?(DEBUGGER__::SESSION) && !irb.context.with_debugger + configure_irb_for_debugger(irb) + DEBUGGER__::SESSION.reset_ui(IRB::Debug::UI.new(irb)) end + # Apply patches to debug gem so it skips IRB frames unless DEBUGGER__.respond_to?(:capture_frames_without_irb) DEBUGGER__.singleton_class.send(:alias_method, :capture_frames_without_irb, :capture_frames) @@ -86,9 +57,43 @@ module IRB DEBUGGER__::ThreadClient.prepend(SkipPathHelperForIRB) end + if !@output_modifier_defined && !DEBUGGER__::CONFIG[:no_hint] + irb_output_modifier_proc = Reline.output_modifier_proc + + Reline.output_modifier_proc = proc do |output, complete:| + unless output.strip.empty? + cmd = output.split(/\s/, 2).first + + if !complete && DEBUGGER__.commands.key?(cmd) + output = output.sub(/\n$/, " # debug command\n") + end + end + + irb_output_modifier_proc.call(output, complete: complete) + end + + @output_modifier_defined = true + end + true end + private + + def configure_irb_for_debugger(irb) + require 'irb/debug/ui' + IRB.instance_variable_set(:@debugger_irb, irb) + irb.context.with_debugger = true + irb.context.irb_name += ":rdbg" + end + + module SkipPathHelperForIRB + def skip_internal_path?(path) + # The latter can be removed once https://github.com/ruby/debug/issues/866 is resolved + super || path.match?(IRB_DIR) || path.match?('<internal:prelude>') + end + end + # This is used when debug.gem is not written in Gemfile. Even if it's not # installed by `bundle install`, debug.gem is installed by default because # it's a bundled gem. This method tries to activate and load that. @@ -121,16 +126,5 @@ module IRB end end end - - class DebugCommand < Debug - def self.category - "Debugging" - end - - def self.description - command_name = self.name.split("::").last.downcase - "Start the debugger of debug.gem and run its `#{command_name}` command." - end - end end end diff --git a/lib/irb/debug/ui.rb b/lib/irb/debug/ui.rb new file mode 100644 index 0000000000..307097b8c9 --- /dev/null +++ b/lib/irb/debug/ui.rb @@ -0,0 +1,103 @@ +require 'io/console/size' +require 'debug/console' + +module IRB + module Debug + class UI < DEBUGGER__::UI_Base + def initialize(irb) + @irb = irb + end + + def remote? + false + end + + def activate session, on_fork: false + end + + def deactivate + end + + def width + if (w = IO.console_size[1]) == 0 # for tests PTY + 80 + else + w + end + end + + def quit n + yield + exit n + end + + def ask prompt + setup_interrupt do + print prompt + ($stdin.gets || '').strip + end + end + + def puts str = nil + case str + when Array + str.each{|line| + $stdout.puts line.chomp + } + when String + str.each_line{|line| + $stdout.puts line.chomp + } + when nil + $stdout.puts + end + end + + def readline _ + setup_interrupt do + tc = DEBUGGER__::SESSION.instance_variable_get(:@tc) + cmd = @irb.debug_readline(tc.current_frame.binding || TOPLEVEL_BINDING) + + case cmd + when nil # when user types C-d + "continue" + else + cmd + end + end + end + + def setup_interrupt + DEBUGGER__::SESSION.intercept_trap_sigint false do + current_thread = Thread.current # should be session_server thread + + prev_handler = trap(:INT){ + current_thread.raise Interrupt + } + + yield + ensure + trap(:INT, prev_handler) + end + end + + def after_fork_parent + parent_pid = Process.pid + + at_exit{ + DEBUGGER__::SESSION.intercept_trap_sigint_end + trap(:SIGINT, :IGNORE) + + if Process.pid == parent_pid + # only check child process from its parent + begin + # wait for all child processes to keep terminal + Process.waitpid + rescue Errno::ESRCH, Errno::ECHILD + end + end + } + end + end + end +end diff --git a/lib/irb/default_commands.rb b/lib/irb/default_commands.rb new file mode 100644 index 0000000000..1bbc68efa7 --- /dev/null +++ b/lib/irb/default_commands.rb @@ -0,0 +1,260 @@ +# frozen_string_literal: true + +require_relative "command" +require_relative "command/internal_helpers" +require_relative "command/context" +require_relative "command/exit" +require_relative "command/force_exit" +require_relative "command/chws" +require_relative "command/pushws" +require_relative "command/subirb" +require_relative "command/load" +require_relative "command/debug" +require_relative "command/edit" +require_relative "command/break" +require_relative "command/catch" +require_relative "command/next" +require_relative "command/delete" +require_relative "command/step" +require_relative "command/continue" +require_relative "command/finish" +require_relative "command/backtrace" +require_relative "command/info" +require_relative "command/help" +require_relative "command/show_doc" +require_relative "command/irb_info" +require_relative "command/ls" +require_relative "command/measure" +require_relative "command/show_source" +require_relative "command/whereami" +require_relative "command/history" + +module IRB + module Command + NO_OVERRIDE = 0 + OVERRIDE_PRIVATE_ONLY = 0x01 + OVERRIDE_ALL = 0x02 + + class << self + # This API is for IRB's internal use only and may change at any time. + # Please do NOT use it. + def _register_with_aliases(name, command_class, *aliases) + @commands[name.to_sym] = [command_class, aliases] + end + + def all_commands_info + user_aliases = IRB.CurrentContext.command_aliases.each_with_object({}) do |(alias_name, target), result| + result[target] ||= [] + result[target] << alias_name + end + + commands.map do |command_name, (command_class, aliases)| + aliases = aliases.map { |a| a.first } + + if additional_aliases = user_aliases[command_name] + aliases += additional_aliases + end + + display_name = aliases.shift || command_name + { + display_name: display_name, + description: command_class.description, + category: command_class.category + } + end + end + + def command_override_policies + @@command_override_policies ||= commands.flat_map do |cmd_name, (cmd_class, aliases)| + [[cmd_name, OVERRIDE_ALL]] + aliases + end.to_h + end + + def execute_as_command?(name, public_method:, private_method:) + case command_override_policies[name] + when OVERRIDE_ALL + true + when OVERRIDE_PRIVATE_ONLY + !public_method + when NO_OVERRIDE + !public_method && !private_method + end + end + + def command_names + command_override_policies.keys.map(&:to_s) + end + + # Convert a command name to its implementation class if such command exists + def load_command(command) + command = command.to_sym + commands.each do |command_name, (command_class, aliases)| + if command_name == command || aliases.any? { |alias_name, _| alias_name == command } + return command_class + end + end + nil + end + end + + _register_with_aliases(:irb_context, Command::Context, + [:context, NO_OVERRIDE] + ) + + _register_with_aliases(:irb_exit, Command::Exit, + [:exit, OVERRIDE_PRIVATE_ONLY], + [:quit, OVERRIDE_PRIVATE_ONLY], + [:irb_quit, OVERRIDE_PRIVATE_ONLY] + ) + + _register_with_aliases(:irb_exit!, Command::ForceExit, + [:exit!, OVERRIDE_PRIVATE_ONLY] + ) + + _register_with_aliases(:irb_current_working_workspace, Command::CurrentWorkingWorkspace, + [:cwws, NO_OVERRIDE], + [:pwws, NO_OVERRIDE], + [:irb_print_working_workspace, OVERRIDE_ALL], + [:irb_cwws, OVERRIDE_ALL], + [:irb_pwws, OVERRIDE_ALL], + [:irb_current_working_binding, OVERRIDE_ALL], + [:irb_print_working_binding, OVERRIDE_ALL], + [:irb_cwb, OVERRIDE_ALL], + [:irb_pwb, OVERRIDE_ALL], + ) + + _register_with_aliases(:irb_change_workspace, Command::ChangeWorkspace, + [:chws, NO_OVERRIDE], + [:cws, NO_OVERRIDE], + [:irb_chws, OVERRIDE_ALL], + [:irb_cws, OVERRIDE_ALL], + [:irb_change_binding, OVERRIDE_ALL], + [:irb_cb, OVERRIDE_ALL], + [:cb, NO_OVERRIDE], + ) + + _register_with_aliases(:irb_workspaces, Command::Workspaces, + [:workspaces, NO_OVERRIDE], + [:irb_bindings, OVERRIDE_ALL], + [:bindings, NO_OVERRIDE], + ) + + _register_with_aliases(:irb_push_workspace, Command::PushWorkspace, + [:pushws, NO_OVERRIDE], + [:irb_pushws, OVERRIDE_ALL], + [:irb_push_binding, OVERRIDE_ALL], + [:irb_pushb, OVERRIDE_ALL], + [:pushb, NO_OVERRIDE], + ) + + _register_with_aliases(:irb_pop_workspace, Command::PopWorkspace, + [:popws, NO_OVERRIDE], + [:irb_popws, OVERRIDE_ALL], + [:irb_pop_binding, OVERRIDE_ALL], + [:irb_popb, OVERRIDE_ALL], + [:popb, NO_OVERRIDE], + ) + + _register_with_aliases(:irb_load, Command::Load) + _register_with_aliases(:irb_require, Command::Require) + _register_with_aliases(:irb_source, Command::Source, + [:source, NO_OVERRIDE] + ) + + _register_with_aliases(:irb, Command::IrbCommand) + _register_with_aliases(:irb_jobs, Command::Jobs, + [:jobs, NO_OVERRIDE] + ) + _register_with_aliases(:irb_fg, Command::Foreground, + [:fg, NO_OVERRIDE] + ) + _register_with_aliases(:irb_kill, Command::Kill, + [:kill, OVERRIDE_PRIVATE_ONLY] + ) + + _register_with_aliases(:irb_debug, Command::Debug, + [:debug, NO_OVERRIDE] + ) + _register_with_aliases(:irb_edit, Command::Edit, + [:edit, NO_OVERRIDE] + ) + + _register_with_aliases(:irb_break, Command::Break) + _register_with_aliases(:irb_catch, Command::Catch) + _register_with_aliases(:irb_next, Command::Next) + _register_with_aliases(:irb_delete, Command::Delete, + [:delete, NO_OVERRIDE] + ) + + _register_with_aliases(:irb_step, Command::Step, + [:step, NO_OVERRIDE] + ) + _register_with_aliases(:irb_continue, Command::Continue, + [:continue, NO_OVERRIDE] + ) + _register_with_aliases(:irb_finish, Command::Finish, + [:finish, NO_OVERRIDE] + ) + _register_with_aliases(:irb_backtrace, Command::Backtrace, + [:backtrace, NO_OVERRIDE], + [:bt, NO_OVERRIDE] + ) + + _register_with_aliases(:irb_debug_info, Command::Info, + [:info, NO_OVERRIDE] + ) + + _register_with_aliases(:irb_help, Command::Help, + [:help, NO_OVERRIDE], + [:show_cmds, NO_OVERRIDE] + ) + + _register_with_aliases(:irb_show_doc, Command::ShowDoc, + [:show_doc, NO_OVERRIDE] + ) + + _register_with_aliases(:irb_info, Command::IrbInfo) + + _register_with_aliases(:irb_ls, Command::Ls, + [:ls, NO_OVERRIDE] + ) + + _register_with_aliases(:irb_measure, Command::Measure, + [:measure, NO_OVERRIDE] + ) + + _register_with_aliases(:irb_show_source, Command::ShowSource, + [:show_source, NO_OVERRIDE] + ) + + _register_with_aliases(:irb_whereami, Command::Whereami, + [:whereami, NO_OVERRIDE] + ) + + _register_with_aliases(:irb_history, Command::History, + [:history, NO_OVERRIDE], + [:hist, NO_OVERRIDE] + ) + end + + ExtendCommand = Command + + # For backward compatibility, we need to keep this module: + # - As a container of helper methods + # - As a place to register commands with the deprecated def_extend_command method + module ExtendCommandBundle + # For backward compatibility + NO_OVERRIDE = Command::NO_OVERRIDE + OVERRIDE_PRIVATE_ONLY = Command::OVERRIDE_PRIVATE_ONLY + OVERRIDE_ALL = Command::OVERRIDE_ALL + + # Deprecated. Doesn't have any effect. + @EXTEND_COMMANDS = [] + + # Drepcated. Use Command.regiser instead. + def self.def_extend_command(cmd_name, cmd_class, _, *aliases) + Command._register_with_aliases(cmd_name, cmd_class, *aliases) + Command.class_variable_set(:@@command_override_policies, nil) + end + end +end diff --git a/lib/irb/easter-egg.rb b/lib/irb/easter-egg.rb index 3e79692de9..adc0834d55 100644 --- a/lib/irb/easter-egg.rb +++ b/lib/irb/easter-egg.rb @@ -98,18 +98,26 @@ module IRB end end + private def easter_egg_logo(type) + @easter_egg_logos ||= File.read(File.join(__dir__, 'ruby_logo.aa'), encoding: 'UTF-8:UTF-8') + .split(/TYPE: ([A-Z]+)\n/)[1..] + .each_slice(2) + .to_h + @easter_egg_logos[type.to_s.upcase] + end + private def easter_egg(type = nil) type ||= [:logo, :dancing].sample case type when :logo - File.open(File.join(__dir__, 'ruby_logo.aa')) do |f| - require "rdoc" - RDoc::RI::Driver.new.page do |io| - IO.copy_stream(f, io) - end + require "rdoc" + RDoc::RI::Driver.new.page do |io| + io.write easter_egg_logo(:large) end when :dancing - begin + STDOUT.cooked do + interrupted = false + prev_trap = trap("SIGINT") { interrupted = true } canvas = Canvas.new(Reline.get_screen_size) Reline::IOGate.set_winch_handler do canvas = Canvas.new(Reline.get_screen_size) @@ -125,10 +133,12 @@ module IRB buff[0, 20] = "\e[0mPress Ctrl+C to stop\e[31m\e[1m" print "\e[H" + buff sleep 0.05 + break if interrupted end rescue Interrupt ensure print "\e[0m\e[?1049l" + trap("SIGINT", prev_trap) end end end diff --git a/lib/irb/ext/change-ws.rb b/lib/irb/ext/change-ws.rb index c0f810a4c8..60e8afe31f 100644 --- a/lib/irb/ext/change-ws.rb +++ b/lib/irb/ext/change-ws.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/ext/cb.rb - # by Keiju ISHITSUKA(keiju@ruby-lang.org) @@ -12,7 +12,7 @@ module IRB # :nodoc: if defined? @home_workspace @home_workspace else - @home_workspace = @workspace + @home_workspace = workspace end end @@ -25,15 +25,13 @@ module IRB # :nodoc: # See IRB::WorkSpace.new for more information. def change_workspace(*_main) if _main.empty? - @workspace = home_workspace + replace_workspace(home_workspace) return main end - @workspace = WorkSpace.new(_main[0]) - - if !(class<<main;ancestors;end).include?(ExtendCommandBundle) - main.extend ExtendCommandBundle - end + workspace = WorkSpace.new(_main[0]) + replace_workspace(workspace) + workspace.load_helper_methods_to_main end end end diff --git a/lib/irb/ext/history.rb b/lib/irb/ext/eval_history.rb index 5243504efa..6c21ff00ee 100644 --- a/lib/irb/ext/history.rb +++ b/lib/irb/ext/eval_history.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # history.rb - # by Keiju ISHITSUKA(keiju@ruby-lang.org) @@ -18,7 +18,7 @@ module IRB # :nodoc: if defined?(@eval_history) && @eval_history @eval_history_values.push @line_no, @last_value - @workspace.evaluate "__ = IRB.CurrentContext.instance_eval{@eval_history_values}" + workspace.evaluate "__ = IRB.CurrentContext.instance_eval{@eval_history_values}" end @last_value @@ -40,16 +40,16 @@ module IRB # :nodoc: # # If +no+ is +nil+, execution result history isn't used (default). # - # History values are available via <code>__</code> variable, see - # IRB::History. + # EvalHistory values are available via <code>__</code> variable, see + # IRB::EvalHistory. def eval_history=(no) if no if defined?(@eval_history) && @eval_history @eval_history_values.size(no) else - @eval_history_values = History.new(no) + @eval_history_values = EvalHistory.new(no) IRB.conf[:__TMP__EHV__] = @eval_history_values - @workspace.evaluate("__ = IRB.conf[:__TMP__EHV__]") + workspace.evaluate("__ = IRB.conf[:__TMP__EHV__]") IRB.conf.delete(:__TMP_EHV__) end else @@ -89,7 +89,7 @@ module IRB # :nodoc: # __[1] # # => 10 # - class History + class EvalHistory def initialize(size = 16) # :nodoc: @size = size diff --git a/lib/irb/ext/loader.rb b/lib/irb/ext/loader.rb index d65695df3b..df5aaa8e5a 100644 --- a/lib/irb/ext/loader.rb +++ b/lib/irb/ext/loader.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # loader.rb - # by Keiju ISHITSUKA(keiju@ruby-lang.org) @@ -98,13 +98,13 @@ module IRB # :nodoc: def old # :nodoc: back_io = @io - back_path = @irb_path + back_path = irb_path back_name = @irb_name back_scanner = @irb.scanner begin @io = FileInputMethod.new(path) @irb_name = File.basename(path) - @irb_path = path + self.irb_path = path @irb.signal_status(:IN_LOAD) do if back_io.kind_of?(FileInputMethod) @irb.eval_input @@ -119,7 +119,7 @@ module IRB # :nodoc: ensure @io = back_io @irb_name = back_name - @irb_path = back_path + self.irb_path = back_path @irb.scanner = back_scanner end end diff --git a/lib/irb/ext/multi-irb.rb b/lib/irb/ext/multi-irb.rb index 1c20489137..9f234f0cdc 100644 --- a/lib/irb/ext/multi-irb.rb +++ b/lib/irb/ext/multi-irb.rb @@ -1,11 +1,11 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/multi-irb.rb - multiple irb module # by Keiju ISHITSUKA(keiju@ruby-lang.org) # module IRB - class JobManager + class JobManager # :nodoc: # Creates a new JobManager object def initialize @@ -166,12 +166,12 @@ module IRB @JobManager = JobManager.new # The current JobManager in the session - def IRB.JobManager + def IRB.JobManager # :nodoc: @JobManager end # The current Context in this session - def IRB.CurrentContext + def IRB.CurrentContext # :nodoc: IRB.JobManager.irb(Thread.current).context end @@ -179,7 +179,7 @@ module IRB # # The optional +file+ argument is given to Context.new, along with the # workspace created with the remaining arguments, see WorkSpace.new - def IRB.irb(file = nil, *main) + def IRB.irb(file = nil, *main) # :nodoc: workspace = WorkSpace.new(*main) parent_thread = Thread.current Thread.start do diff --git a/lib/irb/ext/tracer.rb b/lib/irb/ext/tracer.rb index 3eaeb70ef2..fd6daa88ae 100644 --- a/lib/irb/ext/tracer.rb +++ b/lib/irb/ext/tracer.rb @@ -1,78 +1,39 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/lib/tracer.rb - # by Keiju ISHITSUKA(keiju@ruby-lang.org) # - +# Loading the gem "tracer" will cause it to extend IRB commands with: +# https://github.com/ruby/tracer/blob/v0.2.2/lib/tracer/irb.rb begin require "tracer" rescue LoadError $stderr.puts "Tracer extension of IRB is enabled but tracer gem wasn't found." - module IRB - class Context - def use_tracer=(opt) - # do nothing - end - end - end return # This is about to disable loading below end module IRB + class CallTracer < ::CallTracer + IRB_DIR = File.expand_path('../..', __dir__) - # initialize tracing function - def IRB.initialize_tracer - Tracer.verbose = false - Tracer.add_filter { - |event, file, line, id, binding, *rests| - /^#{Regexp.quote(@CONF[:IRB_LIB_PATH])}/ !~ file and - File::basename(file) != "irb.rb" - } - end - - class Context - # Whether Tracer is used when evaluating statements in this context. - # - # See +lib/tracer.rb+ for more information. - attr_reader :use_tracer - alias use_tracer? use_tracer - - # Sets whether or not to use the Tracer library when evaluating statements - # in this context. - # - # See +lib/tracer.rb+ for more information. - def use_tracer=(opt) - if opt - Tracer.set_get_line_procs(@irb_path) { - |line_no, *rests| - @io.line(line_no) - } - elsif !opt && @use_tracer - Tracer.off - end - @use_tracer=opt + def skip?(tp) + super || tp.path.match?(IRB_DIR) || tp.path.match?('<internal:prelude>') end end - class WorkSpace alias __evaluate__ evaluate # Evaluate the context of this workspace and use the Tracer library to # output the exact lines of code are being executed in chronological order. # - # See +lib/tracer.rb+ for more information. - def evaluate(context, statements, file = nil, line = nil) - if context.use_tracer? && file != nil && line != nil - Tracer.on - begin + # See https://github.com/ruby/tracer for more information. + def evaluate(statements, file = __FILE__, line = __LINE__) + if IRB.conf[:USE_TRACER] == true + CallTracer.new(colorize: Color.colorable?).start do __evaluate__(statements, file, line) - ensure - Tracer.off end else - __evaluate__(statements, file || __FILE__, line || __LINE__) + __evaluate__(statements, file, line) end end end - - IRB.initialize_tracer end diff --git a/lib/irb/ext/use-loader.rb b/lib/irb/ext/use-loader.rb index d0b8c2d4f4..c8a3ea1fe8 100644 --- a/lib/irb/ext/use-loader.rb +++ b/lib/irb/ext/use-loader.rb @@ -1,10 +1,10 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # use-loader.rb - # by Keiju ISHITSUKA(keiju@ruby-lang.org) # -require_relative "../cmd/load" +require_relative "../command/load" require_relative "loader" class Object @@ -17,12 +17,12 @@ module IRB remove_method :irb_load if method_defined?(:irb_load) # Loads the given file similarly to Kernel#load, see IrbLoader#irb_load def irb_load(*opts, &b) - ExtendCommand::Load.execute(irb_context, *opts, &b) + Command::Load.execute(irb_context, *opts, &b) end remove_method :irb_require if method_defined?(:irb_require) # Loads the given file similarly to Kernel#require def irb_require(*opts, &b) - ExtendCommand::Require.execute(irb_context, *opts, &b) + Command::Require.execute(irb_context, *opts, &b) end end @@ -49,14 +49,12 @@ module IRB if IRB.conf[:USE_LOADER] != opt IRB.conf[:USE_LOADER] = opt if opt - if !$".include?("irb/cmd/load") - end - (class<<@workspace.main;self;end).instance_eval { + (class<<workspace.main;self;end).instance_eval { alias_method :load, :irb_load alias_method :require, :irb_require } else - (class<<@workspace.main;self;end).instance_eval { + (class<<workspace.main;self;end).instance_eval { alias_method :load, :__original__load__IRB_use_loader__ alias_method :require, :__original__require__IRB_use_loader__ } diff --git a/lib/irb/ext/workspaces.rb b/lib/irb/ext/workspaces.rb index 9defc3e17b..da09faa83e 100644 --- a/lib/irb/ext/workspaces.rb +++ b/lib/irb/ext/workspaces.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # push-ws.rb - # by Keiju ISHITSUKA(keiju@ruby-lang.org) @@ -6,21 +6,6 @@ module IRB # :nodoc: class Context - - # Size of the current WorkSpace stack - def irb_level - workspace_stack.size - end - - # WorkSpaces in the current stack - def workspaces - if defined? @workspaces - @workspaces - else - @workspaces = [] - end - end - # Creates a new workspace with the given object or binding, and appends it # onto the current #workspaces stack. # @@ -28,20 +13,15 @@ module IRB # :nodoc: # information. def push_workspace(*_main) if _main.empty? - if workspaces.empty? - print "No other workspace\n" - return nil + if @workspace_stack.size > 1 + # swap the top two workspaces + previous_workspace, current_workspace = @workspace_stack.pop(2) + @workspace_stack.push current_workspace, previous_workspace end - ws = workspaces.pop - workspaces.push @workspace - @workspace = ws - return workspaces - end - - workspaces.push @workspace - @workspace = WorkSpace.new(@workspace.binding, _main[0]) - if !(class<<main;ancestors;end).include?(ExtendCommandBundle) - main.extend ExtendCommandBundle + else + new_workspace = WorkSpace.new(workspace.binding, _main[0]) + @workspace_stack.push new_workspace + new_workspace.load_helper_methods_to_main end end @@ -50,11 +30,7 @@ module IRB # :nodoc: # # Also, see #push_workspace. def pop_workspace - if workspaces.empty? - print "workspace stack empty\n" - return - end - @workspace = workspaces.pop + @workspace_stack.pop if @workspace_stack.size > 1 end end end diff --git a/lib/irb/extend-command.rb b/lib/irb/extend-command.rb deleted file mode 100644 index e377148fb4..0000000000 --- a/lib/irb/extend-command.rb +++ /dev/null @@ -1,356 +0,0 @@ -# frozen_string_literal: false -# -# irb/extend-command.rb - irb extend command -# by Keiju ISHITSUKA(keiju@ruby-lang.org) -# - -module IRB # :nodoc: - # Installs the default irb extensions command bundle. - module ExtendCommandBundle - EXCB = ExtendCommandBundle # :nodoc: - - # See #install_alias_method. - NO_OVERRIDE = 0 - # See #install_alias_method. - OVERRIDE_PRIVATE_ONLY = 0x01 - # See #install_alias_method. - OVERRIDE_ALL = 0x02 - - # Quits the current irb context - # - # +ret+ is the optional signal or message to send to Context#exit - # - # Same as <code>IRB.CurrentContext.exit</code>. - def irb_exit(ret = 0) - irb_context.exit(ret) - end - - # Displays current configuration. - # - # Modifying the configuration is achieved by sending a message to IRB.conf. - def irb_context - IRB.CurrentContext - end - - @ALIASES = [ - [:context, :irb_context, NO_OVERRIDE], - [:conf, :irb_context, NO_OVERRIDE], - [:irb_quit, :irb_exit, OVERRIDE_PRIVATE_ONLY], - [:exit, :irb_exit, OVERRIDE_PRIVATE_ONLY], - [:quit, :irb_exit, OVERRIDE_PRIVATE_ONLY], - ] - - - @EXTEND_COMMANDS = [ - [ - :irb_current_working_workspace, :CurrentWorkingWorkspace, "cmd/chws", - [:cwws, NO_OVERRIDE], - [:pwws, NO_OVERRIDE], - [:irb_print_working_workspace, OVERRIDE_ALL], - [:irb_cwws, OVERRIDE_ALL], - [:irb_pwws, OVERRIDE_ALL], - [:irb_current_working_binding, OVERRIDE_ALL], - [:irb_print_working_binding, OVERRIDE_ALL], - [:irb_cwb, OVERRIDE_ALL], - [:irb_pwb, OVERRIDE_ALL], - ], - [ - :irb_change_workspace, :ChangeWorkspace, "cmd/chws", - [:chws, NO_OVERRIDE], - [:cws, NO_OVERRIDE], - [:irb_chws, OVERRIDE_ALL], - [:irb_cws, OVERRIDE_ALL], - [:irb_change_binding, OVERRIDE_ALL], - [:irb_cb, OVERRIDE_ALL], - [:cb, NO_OVERRIDE], - ], - - [ - :irb_workspaces, :Workspaces, "cmd/pushws", - [:workspaces, NO_OVERRIDE], - [:irb_bindings, OVERRIDE_ALL], - [:bindings, NO_OVERRIDE], - ], - [ - :irb_push_workspace, :PushWorkspace, "cmd/pushws", - [:pushws, NO_OVERRIDE], - [:irb_pushws, OVERRIDE_ALL], - [:irb_push_binding, OVERRIDE_ALL], - [:irb_pushb, OVERRIDE_ALL], - [:pushb, NO_OVERRIDE], - ], - [ - :irb_pop_workspace, :PopWorkspace, "cmd/pushws", - [:popws, NO_OVERRIDE], - [:irb_popws, OVERRIDE_ALL], - [:irb_pop_binding, OVERRIDE_ALL], - [:irb_popb, OVERRIDE_ALL], - [:popb, NO_OVERRIDE], - ], - - [ - :irb_load, :Load, "cmd/load"], - [ - :irb_require, :Require, "cmd/load"], - [ - :irb_source, :Source, "cmd/load", - [:source, NO_OVERRIDE], - ], - - [ - :irb, :IrbCommand, "cmd/subirb"], - [ - :irb_jobs, :Jobs, "cmd/subirb", - [:jobs, NO_OVERRIDE], - ], - [ - :irb_fg, :Foreground, "cmd/subirb", - [:fg, NO_OVERRIDE], - ], - [ - :irb_kill, :Kill, "cmd/subirb", - [:kill, OVERRIDE_PRIVATE_ONLY], - ], - - [ - :irb_debug, :Debug, "cmd/debug", - [:debug, NO_OVERRIDE], - ], - [ - :irb_edit, :Edit, "cmd/edit", - [:edit, NO_OVERRIDE], - ], - [ - :irb_break, :Break, "cmd/break", - ], - [ - :irb_catch, :Catch, "cmd/catch", - ], - [ - :irb_next, :Next, "cmd/next" - ], - [ - :irb_delete, :Delete, "cmd/delete", - [:delete, NO_OVERRIDE], - ], - [ - :irb_step, :Step, "cmd/step", - [:step, NO_OVERRIDE], - ], - [ - :irb_continue, :Continue, "cmd/continue", - [:continue, NO_OVERRIDE], - ], - [ - :irb_finish, :Finish, "cmd/finish", - [:finish, NO_OVERRIDE], - ], - [ - :irb_backtrace, :Backtrace, "cmd/backtrace", - [:backtrace, NO_OVERRIDE], - [:bt, NO_OVERRIDE], - ], - [ - :irb_debug_info, :Info, "cmd/info", - [:info, NO_OVERRIDE], - ], - - [ - :irb_help, :Help, "cmd/help", - [:help, NO_OVERRIDE], - ], - - [ - :irb_show_doc, :ShowDoc, "cmd/show_doc", - [:show_doc, NO_OVERRIDE], - ], - - [ - :irb_info, :IrbInfo, "cmd/irb_info" - ], - - [ - :irb_ls, :Ls, "cmd/ls", - [:ls, NO_OVERRIDE], - ], - - [ - :irb_measure, :Measure, "cmd/measure", - [:measure, NO_OVERRIDE], - ], - - [ - :irb_show_source, :ShowSource, "cmd/show_source", - [:show_source, NO_OVERRIDE], - ], - - [ - :irb_whereami, :Whereami, "cmd/whereami", - [:whereami, NO_OVERRIDE], - ], - [ - :irb_show_cmds, :ShowCmds, "cmd/show_cmds", - [:show_cmds, NO_OVERRIDE], - ] - ] - - - @@commands = [] - - def self.all_commands_info - return @@commands unless @@commands.empty? - user_aliases = IRB.CurrentContext.command_aliases.each_with_object({}) do |(alias_name, target), result| - result[target] ||= [] - result[target] << alias_name - end - - @EXTEND_COMMANDS.each do |cmd_name, cmd_class, load_file, *aliases| - if !defined?(ExtendCommand) || !ExtendCommand.const_defined?(cmd_class, false) - require_relative load_file - end - - klass = ExtendCommand.const_get(cmd_class, false) - aliases = aliases.map { |a| a.first } - - if additional_aliases = user_aliases[cmd_name] - aliases += additional_aliases - end - - display_name = aliases.shift || cmd_name - @@commands << { display_name: display_name, description: klass.description, category: klass.category } - end - - @@commands - end - - # Convert a command name to its implementation class if such command exists - def self.load_command(command) - command = command.to_sym - @EXTEND_COMMANDS.each do |cmd_name, cmd_class, load_file, *aliases| - next if cmd_name != command && aliases.all? { |alias_name, _| alias_name != command } - - if !defined?(ExtendCommand) || !ExtendCommand.const_defined?(cmd_class, false) - require_relative load_file - end - return ExtendCommand.const_get(cmd_class, false) - end - nil - end - - # Installs the default irb commands. - def self.install_extend_commands - for args in @EXTEND_COMMANDS - def_extend_command(*args) - end - end - - # Evaluate the given +cmd_name+ on the given +cmd_class+ Class. - # - # Will also define any given +aliases+ for the method. - # - # The optional +load_file+ parameter will be required within the method - # definition. - def self.def_extend_command(cmd_name, cmd_class, load_file, *aliases) - case cmd_class - when Symbol - cmd_class = cmd_class.id2name - when String - when Class - cmd_class = cmd_class.name - end - - line = __LINE__; eval %[ - def #{cmd_name}(*opts, **kwargs, &b) - Kernel.require_relative "#{load_file}" - ::IRB::ExtendCommand::#{cmd_class}.execute(irb_context, *opts, **kwargs, &b) - end - ], nil, __FILE__, line - - for ali, flag in aliases - @ALIASES.push [ali, cmd_name, flag] - end - end - - # Installs alias methods for the default irb commands, see - # ::install_extend_commands. - def install_alias_method(to, from, override = NO_OVERRIDE) - to = to.id2name unless to.kind_of?(String) - from = from.id2name unless from.kind_of?(String) - - if override == OVERRIDE_ALL or - (override == OVERRIDE_PRIVATE_ONLY) && !respond_to?(to) or - (override == NO_OVERRIDE) && !respond_to?(to, true) - target = self - (class << self; self; end).instance_eval{ - if target.respond_to?(to, true) && - !target.respond_to?(EXCB.irb_original_method_name(to), true) - alias_method(EXCB.irb_original_method_name(to), to) - end - alias_method to, from - } - else - Kernel.print "irb: warn: can't alias #{to} from #{from}.\n" - end - end - - def self.irb_original_method_name(method_name) # :nodoc: - "irb_" + method_name + "_org" - end - - # Installs alias methods for the default irb commands on the given object - # using #install_alias_method. - def self.extend_object(obj) - unless (class << obj; ancestors; end).include?(EXCB) - super - for ali, com, flg in @ALIASES - obj.install_alias_method(ali, com, flg) - end - end - end - - install_extend_commands - end - - # Extends methods for the Context module - module ContextExtender - CE = ContextExtender # :nodoc: - - @EXTEND_COMMANDS = [ - [:eval_history=, "ext/history.rb"], - [:use_tracer=, "ext/tracer.rb"], - [:use_loader=, "ext/use-loader.rb"], - [:save_history=, "ext/save-history.rb"], - ] - - # Installs the default context extensions as irb commands: - # - # Context#eval_history=:: +irb/ext/history.rb+ - # Context#use_tracer=:: +irb/ext/tracer.rb+ - # Context#use_loader=:: +irb/ext/use-loader.rb+ - # Context#save_history=:: +irb/ext/save-history.rb+ - def self.install_extend_commands - for args in @EXTEND_COMMANDS - def_extend_command(*args) - end - end - - # Evaluate the given +command+ from the given +load_file+ on the Context - # module. - # - # Will also define any given +aliases+ for the method. - def self.def_extend_command(cmd_name, load_file, *aliases) - line = __LINE__; Context.module_eval %[ - def #{cmd_name}(*opts, &b) - Context.module_eval {remove_method(:#{cmd_name})} - require_relative "#{load_file}" - __send__ :#{cmd_name}, *opts, &b - end - for ali in aliases - alias_method ali, cmd_name - end - ], __FILE__, line - end - - CE.install_extend_commands - end -end diff --git a/lib/irb/frame.rb b/lib/irb/frame.rb index 14768bd8f6..4b697c8719 100644 --- a/lib/irb/frame.rb +++ b/lib/irb/frame.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # frame.rb - # by Keiju ISHITSUKA(Nihon Rational Software Co.,Ltd) diff --git a/lib/irb/help.rb b/lib/irb/help.rb index ca12810de1..a24bc10a15 100644 --- a/lib/irb/help.rb +++ b/lib/irb/help.rb @@ -1,12 +1,12 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/help.rb - print usage module # by Keiju ISHITSUKA(keiju@ishitsuka.com) # module IRB - # Outputs the irb help message, see IRB@Command+line+options. - def IRB.print_usage + # Outputs the irb help message, see IRB@Command-Line+Options. + def IRB.print_usage # :nodoc: lc = IRB.conf[:LC_MESSAGES] path = lc.find("irb/help-message") space_line = false diff --git a/lib/irb/helper_method.rb b/lib/irb/helper_method.rb new file mode 100644 index 0000000000..f1f6fff915 --- /dev/null +++ b/lib/irb/helper_method.rb @@ -0,0 +1,29 @@ +require_relative "helper_method/base" + +module IRB + module HelperMethod + @helper_methods = {} + + class << self + attr_reader :helper_methods + + def register(name, helper_class) + @helper_methods[name] = helper_class + + if defined?(HelpersContainer) + HelpersContainer.install_helper_methods + end + end + + def all_helper_methods_info + @helper_methods.map do |name, helper_class| + { display_name: name, description: helper_class.description } + end + end + end + + # Default helper_methods + require_relative "helper_method/conf" + register(:conf, HelperMethod::Conf) + end +end diff --git a/lib/irb/helper_method/base.rb b/lib/irb/helper_method/base.rb new file mode 100644 index 0000000000..a68001ed28 --- /dev/null +++ b/lib/irb/helper_method/base.rb @@ -0,0 +1,16 @@ +require "singleton" + +module IRB + module HelperMethod + class Base + include Singleton + + class << self + def description(description = nil) + @description = description if description + @description + end + end + end + end +end diff --git a/lib/irb/helper_method/conf.rb b/lib/irb/helper_method/conf.rb new file mode 100644 index 0000000000..718ed279c0 --- /dev/null +++ b/lib/irb/helper_method/conf.rb @@ -0,0 +1,11 @@ +module IRB + module HelperMethod + class Conf < Base + description "Returns the current IRB context." + + def execute + IRB.CurrentContext + end + end + end +end diff --git a/lib/irb/ext/save-history.rb b/lib/irb/history.rb index b1987ea646..685354b2d8 100644 --- a/lib/irb/ext/save-history.rb +++ b/lib/irb/history.rb @@ -1,70 +1,23 @@ -# frozen_string_literal: false -# -# save-history.rb - -# by Keiju ISHITSUKA(keiju@ruby-lang.org) -# +require "pathname" module IRB module HistorySavingAbility # :nodoc: - end - - class Context - def init_save_history# :nodoc: - unless (class<<@io;self;end).include?(HistorySavingAbility) - @io.extend(HistorySavingAbility) - end - end - - # A copy of the default <code>IRB.conf[:SAVE_HISTORY]</code> - def save_history - IRB.conf[:SAVE_HISTORY] - end - - remove_method(:save_history=) if method_defined?(:save_history=) - # Sets <code>IRB.conf[:SAVE_HISTORY]</code> to the given +val+ and calls - # #init_save_history with this context. - # - # Will store the number of +val+ entries of history in the #history_file - # - # Add the following to your +.irbrc+ to change the number of history - # entries stored to 1000: - # - # IRB.conf[:SAVE_HISTORY] = 1000 - def save_history=(val) - IRB.conf[:SAVE_HISTORY] = val - if val - main_context = IRB.conf[:MAIN_CONTEXT] - main_context = self unless main_context - main_context.init_save_history - end - end - - # A copy of the default <code>IRB.conf[:HISTORY_FILE]</code> - def history_file - IRB.conf[:HISTORY_FILE] + def support_history_saving? + true end - # Set <code>IRB.conf[:HISTORY_FILE]</code> to the given +hist+. - def history_file=(hist) - IRB.conf[:HISTORY_FILE] = hist - end - end - - module HistorySavingAbility # :nodoc: - def HistorySavingAbility.extended(obj) - IRB.conf[:AT_EXIT].push proc{obj.save_history} - obj.load_history - obj + def reset_history_counter + @loaded_history_lines = self.class::HISTORY.size end def load_history - return unless self.class.const_defined?(:HISTORY) history = self.class::HISTORY + if history_file = IRB.conf[:HISTORY_FILE] history_file = File.expand_path(history_file) end history_file = IRB.rc_file("_history") unless history_file - if File.exist?(history_file) + if history_file && File.exist?(history_file) File.open(history_file, "r:#{IRB.conf[:LC_MESSAGES].encoding}") do |f| f.each { |l| l = l.chomp @@ -82,14 +35,17 @@ module IRB end def save_history - return unless self.class.const_defined?(:HISTORY) - history = self.class::HISTORY + history = self.class::HISTORY.to_a + if num = IRB.conf[:SAVE_HISTORY] and (num = num.to_i) != 0 if history_file = IRB.conf[:HISTORY_FILE] history_file = File.expand_path(history_file) end history_file = IRB.rc_file("_history") unless history_file + # When HOME and XDG_CONFIG_HOME are not available, history_file might be nil + return unless history_file + # Change the permission of a file that already exists[BUG #7694] begin if File.stat(history_file).mode & 066 != 0 @@ -108,13 +64,19 @@ module IRB append_history = true end + pathname = Pathname.new(history_file) + unless Dir.exist?(pathname.dirname) + warn "Warning: The directory to save IRB's history file does not exist. Please double check `IRB.conf[:HISTORY_FILE]`'s value." + return + end + File.open(history_file, (append_history ? 'a' : 'w'), 0o600, encoding: IRB.conf[:LC_MESSAGES]&.encoding) do |f| - hist = history.map{ |l| l.split("\n").join("\\\n") } + hist = history.map{ |l| l.scrub.split("\n").join("\\\n") } unless append_history begin hist = hist.last(num) if hist.size > num and num > 0 rescue RangeError # bignum too big to convert into `long' - # Do nothing because the bignum should be treated as inifinity + # Do nothing because the bignum should be treated as infinity end end f.puts(hist) diff --git a/lib/irb/init.rb b/lib/irb/init.rb index 75a7f8d74c..7dc08912ef 100644 --- a/lib/irb/init.rb +++ b/lib/irb/init.rb @@ -1,10 +1,50 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/init.rb - irb initialize module # by Keiju ISHITSUKA(keiju@ruby-lang.org) # module IRB # :nodoc: + @CONF = {} + @INITIALIZED = false + # Displays current configuration. + # + # Modifying the configuration is achieved by sending a message to IRB.conf. + # + # See IRB@Configuration for more information. + def IRB.conf + @CONF + end + + def @CONF.inspect + array = [] + for k, v in sort{|a1, a2| a1[0].id2name <=> a2[0].id2name} + case k + when :MAIN_CONTEXT, :__TMP__EHV__ + array.push format("CONF[:%s]=...myself...", k.id2name) + when :PROMPT + s = v.collect{ + |kk, vv| + ss = vv.collect{|kkk, vvv| ":#{kkk.id2name}=>#{vvv.inspect}"} + format(":%s=>{%s}", kk.id2name, ss.join(", ")) + } + array.push format("CONF[:%s]={%s}", k.id2name, s.join(", ")) + else + array.push format("CONF[:%s]=%s", k.id2name, v.inspect) + end + end + array.join("\n") + end + + # Returns the current version of IRB, including release version and last + # updated date. + def IRB.version + format("irb %s (%s)", @RELEASE_VERSION, @LAST_UPDATE_DATE) + end + + def IRB.initialized? + !!@INITIALIZED + end # initialize config def IRB.setup(ap_path, argv: ::ARGV) @@ -12,22 +52,22 @@ module IRB # :nodoc: IRB.init_error IRB.parse_opts(argv: argv) IRB.run_config + IRB.validate_config IRB.load_modules unless @CONF[:PROMPT][@CONF[:PROMPT_MODE]] fail UndefinedPromptMode, @CONF[:PROMPT_MODE] end + @INITIALIZED = true end # @CONF default setting def IRB.init_config(ap_path) - # class instance variables - @TRACER_INITIALIZED = false - # default configurations unless ap_path and @CONF[:AP_NAME] ap_path = File.join(File.dirname(File.dirname(__FILE__)), "irb.rb") end + @CONF[:VERSION] = version @CONF[:AP_NAME] = File::basename(ap_path, ".rb") @CONF[:IRB_NAME] = "irb" @@ -40,11 +80,13 @@ module IRB # :nodoc: @CONF[:USE_SINGLELINE] = false unless defined?(ReadlineInputMethod) @CONF[:USE_COLORIZE] = (nc = ENV['NO_COLOR']).nil? || nc.empty? @CONF[:USE_AUTOCOMPLETE] = ENV.fetch("IRB_USE_AUTOCOMPLETE", "true") != "false" + @CONF[:COMPLETOR] = ENV.fetch("IRB_COMPLETOR", "regexp").to_sym @CONF[:INSPECT_MODE] = true @CONF[:USE_TRACER] = false @CONF[:USE_LOADER] = false @CONF[:IGNORE_SIGINT] = true @CONF[:IGNORE_EOF] = false + @CONF[:USE_PAGER] = true @CONF[:EXTRA_DOC_DIRS] = [] @CONF[:ECHO] = nil @CONF[:ECHO_ON_ASSIGNMENT] = nil @@ -58,35 +100,30 @@ module IRB # :nodoc: @CONF[:PROMPT] = { :NULL => { :PROMPT_I => nil, - :PROMPT_N => nil, :PROMPT_S => nil, :PROMPT_C => nil, :RETURN => "%s\n" }, :DEFAULT => { - :PROMPT_I => "%N(%m):%03n:%i> ", - :PROMPT_N => "%N(%m):%03n:%i> ", - :PROMPT_S => "%N(%m):%03n:%i%l ", - :PROMPT_C => "%N(%m):%03n:%i* ", + :PROMPT_I => "%N(%m):%03n> ", + :PROMPT_S => "%N(%m):%03n%l ", + :PROMPT_C => "%N(%m):%03n* ", :RETURN => "=> %s\n" }, :CLASSIC => { :PROMPT_I => "%N(%m):%03n:%i> ", - :PROMPT_N => "%N(%m):%03n:%i> ", :PROMPT_S => "%N(%m):%03n:%i%l ", :PROMPT_C => "%N(%m):%03n:%i* ", :RETURN => "%s\n" }, :SIMPLE => { :PROMPT_I => ">> ", - :PROMPT_N => ">> ", :PROMPT_S => "%l> ", :PROMPT_C => "?> ", :RETURN => "=> %s\n" }, :INF_RUBY => { - :PROMPT_I => "%N(%m):%03n:%i> ", - :PROMPT_N => nil, + :PROMPT_I => "%N(%m):%03n> ", :PROMPT_S => nil, :PROMPT_C => nil, :RETURN => "%s\n", @@ -94,7 +131,6 @@ module IRB # :nodoc: }, :XMP => { :PROMPT_I => nil, - :PROMPT_N => nil, :PROMPT_S => nil, :PROMPT_C => nil, :RETURN => " ==>%s\n" @@ -157,10 +193,6 @@ module IRB # :nodoc: # Symbol aliases :'$' => :show_source, :'@' => :whereami, - # Keyword aliases - :break => :irb_break, - :catch => :irb_catch, - :next => :irb_next, } end @@ -187,6 +219,7 @@ module IRB # :nodoc: added = [:TIME, IRB.conf[:MEASURE_PROC][:TIME], arg] end if added + IRB.conf[:MEASURE] = true found = IRB.conf[:MEASURE_CALLBACKS].find{ |m| m[0] == added[0] && m[2] == added[2] } if found # already added @@ -207,6 +240,7 @@ module IRB # :nodoc: type_sym = type.upcase.to_sym IRB.conf[:MEASURE_CALLBACKS].reject!{ |t, | t == type_sym } end + IRB.conf[:MEASURE] = nil if IRB.conf[:MEASURE_CALLBACKS].empty? end def IRB.init_error @@ -254,6 +288,8 @@ module IRB # :nodoc: end when "--noinspect" @CONF[:INSPECT_MODE] = false + when "--no-pager" + @CONF[:USE_PAGER] = false when "--singleline", "--readline", "--legacy" @CONF[:USE_SINGLELINE] = true when "--nosingleline", "--noreadline" @@ -299,6 +335,10 @@ module IRB # :nodoc: @CONF[:USE_AUTOCOMPLETE] = true when "--noautocomplete" @CONF[:USE_AUTOCOMPLETE] = false + when "--regexp-completor" + @CONF[:COMPLETOR] = :regexp + when "--type-completor" + @CONF[:COMPLETOR] = :type when /^--prompt-mode(?:=(.+))?/, /^--prompt(?:=(.+))?/ opt = $1 || argv.shift prompt_mode = opt.upcase.tr("-", "_").intern @@ -353,61 +393,73 @@ module IRB # :nodoc: $LOAD_PATH.unshift(*load_path) end - # running config + # Run the config file def IRB.run_config if @CONF[:RC] - begin - load rc_file - rescue LoadError, Errno::ENOENT - rescue # StandardError, ScriptError - print "load error: #{rc_file}\n" - print $!.class, ": ", $!, "\n" - for err in $@[0, $@.size - 2] - print "\t", err, "\n" - end + irbrc_files.each do |rc| + load rc + rescue StandardError, ScriptError => e + warn "Error loading RC file '#{rc}':\n#{e.full_message(highlight: false)}" end end end IRBRC_EXT = "rc" - def IRB.rc_file(ext = IRBRC_EXT) - if !@CONF[:RC_NAME_GENERATOR] - rc_file_generators do |rcgen| - @CONF[:RC_NAME_GENERATOR] ||= rcgen - if File.exist?(rcgen.call(IRBRC_EXT)) - @CONF[:RC_NAME_GENERATOR] = rcgen - break - end - end + + def IRB.rc_file(ext) + prepare_irbrc_name_generators + + # When irbrc exist in default location + if (rcgen = @existing_rc_name_generators.first) + return rcgen.call(ext) end - case rc_file = @CONF[:RC_NAME_GENERATOR].call(ext) - when String - return rc_file - else - fail IllegalRCNameGenerator + + # When irbrc does not exist in default location + rc_file_generators do |rcgen| + return rcgen.call(ext) end + + # When HOME and XDG_CONFIG_HOME are not available + nil + end + + def IRB.irbrc_files + prepare_irbrc_name_generators + @irbrc_files end - # enumerate possible rc-file base name generators - def IRB.rc_file_generators - if irbrc = ENV["IRBRC"] - yield proc{|rc| rc == "rc" ? irbrc : irbrc+rc} + def IRB.validate_config + conf[:IRB_NAME] = conf[:IRB_NAME].to_s + + irb_rc = conf[:IRB_RC] + unless irb_rc.nil? || irb_rc.respond_to?(:call) + raise_validation_error "IRB.conf[:IRB_RC] should be a callable object. Got #{irb_rc.inspect}." + end + + back_trace_limit = conf[:BACK_TRACE_LIMIT] + unless back_trace_limit.is_a?(Integer) + raise_validation_error "IRB.conf[:BACK_TRACE_LIMIT] should be an integer. Got #{back_trace_limit.inspect}." end - if xdg_config_home = ENV["XDG_CONFIG_HOME"] - irb_home = File.join(xdg_config_home, "irb") - if File.directory?(irb_home) - yield proc{|rc| irb_home + "/irb#{rc}"} + + prompt = conf[:PROMPT] + unless prompt.is_a?(Hash) + msg = "IRB.conf[:PROMPT] should be a Hash. Got #{prompt.inspect}." + + if prompt.is_a?(Symbol) + msg += " Did you mean to set `IRB.conf[:PROMPT_MODE]`?" end + + raise_validation_error msg end - if home = ENV["HOME"] - yield proc{|rc| home+"/.irb#{rc}"} - yield proc{|rc| home+"/.config/irb/irb#{rc}"} + + eval_history = conf[:EVAL_HISTORY] + unless eval_history.nil? || eval_history.is_a?(Integer) + raise_validation_error "IRB.conf[:EVAL_HISTORY] should be an integer. Got #{eval_history.inspect}." end - current_dir = Dir.pwd - yield proc{|rc| current_dir+"/.irb#{rc}"} - yield proc{|rc| current_dir+"/irb#{rc.sub(/\A_?/, '.')}"} - yield proc{|rc| current_dir+"/_irb#{rc}"} - yield proc{|rc| current_dir+"/$irb#{rc}"} + end + + def IRB.raise_validation_error(msg) + raise TypeError, msg, @irbrc_files end # loading modules @@ -423,6 +475,50 @@ module IRB # :nodoc: class << IRB private + + def prepare_irbrc_name_generators + return if @existing_rc_name_generators + + @existing_rc_name_generators = [] + @irbrc_files = [] + rc_file_generators do |rcgen| + irbrc = rcgen.call(IRBRC_EXT) + if File.exist?(irbrc) + @irbrc_files << irbrc + @existing_rc_name_generators << rcgen + end + end + generate_current_dir_irbrc_files.each do |irbrc| + @irbrc_files << irbrc if File.exist?(irbrc) + end + @irbrc_files.uniq! + end + + # enumerate possible rc-file base name generators + def rc_file_generators + if irbrc = ENV["IRBRC"] + yield proc{|rc| rc == "rc" ? irbrc : irbrc+rc} + end + if xdg_config_home = ENV["XDG_CONFIG_HOME"] + irb_home = File.join(xdg_config_home, "irb") + if File.directory?(irb_home) + yield proc{|rc| irb_home + "/irb#{rc}"} + end + end + if home = ENV["HOME"] + yield proc{|rc| home+"/.irb#{rc}"} + if xdg_config_home.nil? || xdg_config_home.empty? + yield proc{|rc| home+"/.config/irb/irb#{rc}"} + end + end + end + + # possible irbrc files in current directory + def generate_current_dir_irbrc_files + current_dir = Dir.pwd + %w[.irbrc irbrc _irbrc $irbrc].map { |file| "#{current_dir}/#{file}" } + end + def set_encoding(extern, intern = nil, override: true) verbose, $VERBOSE = $VERBOSE, nil Encoding.default_external = extern unless extern.nil? || extern.empty? diff --git a/lib/irb/input-method.rb b/lib/irb/input-method.rb index 4e049b22db..684527edc4 100644 --- a/lib/irb/input-method.rb +++ b/lib/irb/input-method.rb @@ -1,24 +1,17 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/input-method.rb - input methods used irb # by Keiju ISHITSUKA(keiju@ruby-lang.org) # -require_relative 'src_encoding' require_relative 'completion' +require_relative "history" require 'io/console' require 'reline' module IRB - STDIN_FILE_NAME = "(line)" # :nodoc: class InputMethod - - # Creates a new input method object - def initialize(file = STDIN_FILE_NAME) - @file_name = file - end - # The file name of this input method, usually given during initialization. - attr_reader :file_name + BASIC_WORD_BREAK_CHARACTERS = " \t\n`><=;|&{(" # The irb prompt associated with this input method attr_accessor :prompt @@ -27,7 +20,7 @@ module IRB # # See IO#gets for more information. def gets - fail NotImplementedError, "gets" + fail NotImplementedError end public :gets @@ -47,6 +40,14 @@ module IRB false end + def support_history_saving? + false + end + + def prompting? + false + end + # For debug message def inspect 'Abstract InputMethod' @@ -56,7 +57,6 @@ module IRB class StdioInputMethod < InputMethod # Creates a new input method object def initialize - super @line_no = 0 @line = [] @stdin = IO.open(STDIN.to_i, :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-") @@ -67,6 +67,7 @@ module IRB # # See IO#gets for more information. def gets + puts if @stdout.tty? # workaround for debug compatibility test print @prompt line = @stdin.gets @line[@line_no += 1] = line @@ -95,6 +96,10 @@ module IRB true end + def prompting? + STDIN.tty? + end + # Returns the current line number for #io. # # #line counts the number of times #gets is called. @@ -130,12 +135,9 @@ module IRB # Creates a new input method object def initialize(file) - super @io = file.is_a?(IO) ? file : File.open(file) @external_encoding = @io.external_encoding end - # The file name of this input method, usually given during initialization. - attr_reader :file_name # Whether the end of this input method has been reached, returns +true+ if # there is no more data to read. @@ -168,114 +170,99 @@ module IRB end end - begin - class ReadlineInputMethod < InputMethod - def self.initialize_readline - require "readline" - rescue LoadError - else - include ::Readline - end + class ReadlineInputMethod < StdioInputMethod + def self.initialize_readline + require "readline" + rescue LoadError + else + include ::Readline + end - # Creates a new input method object using Readline - def initialize - self.class.initialize_readline - if Readline.respond_to?(:encoding_system_needs) - IRB.__send__(:set_encoding, Readline.encoding_system_needs.name, override: false) - end - super + include HistorySavingAbility - @line_no = 0 - @line = [] - @eof = false + # Creates a new input method object using Readline + def initialize + self.class.initialize_readline + if Readline.respond_to?(:encoding_system_needs) + IRB.__send__(:set_encoding, Readline.encoding_system_needs.name, override: false) + end - @stdin = IO.open(STDIN.to_i, :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-") - @stdout = IO.open(STDOUT.to_i, 'w', :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-") + super - if Readline.respond_to?("basic_word_break_characters=") - Readline.basic_word_break_characters = IRB::InputCompletor::BASIC_WORD_BREAK_CHARACTERS - end - Readline.completion_append_character = nil - Readline.completion_proc = IRB::InputCompletor::CompletionProc - end + @eof = false + @completor = RegexpCompletor.new - # Reads the next line from this input method. - # - # See IO#gets for more information. - def gets - Readline.input = @stdin - Readline.output = @stdout - if l = readline(@prompt, false) - HISTORY.push(l) if !l.empty? - @line[@line_no += 1] = l + "\n" - else - @eof = true - l - end + if Readline.respond_to?("basic_word_break_characters=") + Readline.basic_word_break_characters = BASIC_WORD_BREAK_CHARACTERS end + Readline.completion_append_character = nil + Readline.completion_proc = ->(target) { + bind = IRB.conf[:MAIN_CONTEXT].workspace.binding + @completor.completion_candidates('', target, '', bind: bind) + } + end - # Whether the end of this input method has been reached, returns +true+ - # if there is no more data to read. - # - # See IO#eof? for more information. - def eof? - @eof - end + def completion_info + 'RegexpCompletor' + end - # Whether this input method is still readable when there is no more data to - # read. - # - # See IO#eof for more information. - def readable_after_eof? - true + # Reads the next line from this input method. + # + # See IO#gets for more information. + def gets + Readline.input = @stdin + Readline.output = @stdout + if l = readline(@prompt, false) + HISTORY.push(l) if !l.empty? + @line[@line_no += 1] = l + "\n" + else + @eof = true + l end + end - # Returns the current line number for #io. - # - # #line counts the number of times #gets is called. - # - # See IO#lineno for more information. - def line(line_no) - @line[line_no] - end + # Whether the end of this input method has been reached, returns +true+ + # if there is no more data to read. + # + # See IO#eof? for more information. + def eof? + @eof + end - # The external encoding for standard input. - def encoding - @stdin.external_encoding - end + def prompting? + true + end - # For debug message - def inspect - readline_impl = (defined?(Reline) && Readline == Reline) ? 'Reline' : 'ext/readline' - str = "ReadlineInputMethod with #{readline_impl} #{Readline::VERSION}" - inputrc_path = File.expand_path(ENV['INPUTRC'] || '~/.inputrc') - str += " and #{inputrc_path}" if File.exist?(inputrc_path) - str - end + # For debug message + def inspect + readline_impl = (defined?(Reline) && Readline == Reline) ? 'Reline' : 'ext/readline' + str = "ReadlineInputMethod with #{readline_impl} #{Readline::VERSION}" + inputrc_path = File.expand_path(ENV['INPUTRC'] || '~/.inputrc') + str += " and #{inputrc_path}" if File.exist?(inputrc_path) + str end end - class RelineInputMethod < InputMethod - include Reline - + class RelineInputMethod < StdioInputMethod + HISTORY = Reline::HISTORY + include HistorySavingAbility # Creates a new input method object using Reline - def initialize + def initialize(completor) IRB.__send__(:set_encoding, Reline.encoding_system_needs.name, override: false) - super - @line_no = 0 - @line = [] - @eof = false + super() - @stdin = ::IO.open(STDIN.to_i, :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-") - @stdout = ::IO.open(STDOUT.to_i, 'w', :external_encoding => IRB.conf[:LC_MESSAGES].encoding, :internal_encoding => "-") + @eof = false + @completor = completor - if Reline.respond_to?("basic_word_break_characters=") - Reline.basic_word_break_characters = IRB::InputCompletor::BASIC_WORD_BREAK_CHARACTERS - end + Reline.basic_word_break_characters = BASIC_WORD_BREAK_CHARACTERS Reline.completion_append_character = nil Reline.completer_quote_characters = '' - Reline.completion_proc = IRB::InputCompletor::CompletionProc + Reline.completion_proc = ->(target, preposing, postposing) { + bind = IRB.conf[:MAIN_CONTEXT].workspace.binding + @completion_params = [preposing, target, postposing, bind] + @completor.completion_candidates(preposing, target, postposing, bind: bind) + } Reline.output_modifier_proc = if IRB.conf[:USE_COLORIZE] proc do |output, complete: | @@ -288,18 +275,23 @@ module IRB Reline::Unicode.escape_for_print(output) end end - Reline.dig_perfect_match_proc = IRB::InputCompletor::PerfectMatchedProc + Reline.dig_perfect_match_proc = ->(matched) { display_document(matched) } Reline.autocompletion = IRB.conf[:USE_AUTOCOMPLETE] if IRB.conf[:USE_AUTOCOMPLETE] begin require 'rdoc' - Reline.add_dialog_proc(:show_doc, SHOW_DOC_DIALOG, Reline::DEFAULT_DIALOG_CONTEXT) + Reline.add_dialog_proc(:show_doc, show_doc_dialog_proc, Reline::DEFAULT_DIALOG_CONTEXT) rescue LoadError end end end + def completion_info + autocomplete_message = Reline.autocompletion ? 'Autocomplete' : 'Tab Complete' + "#{autocomplete_message}, #{@completor.inspect}" + end + def check_termination(&block) @check_termination_proc = block end @@ -312,100 +304,164 @@ module IRB @auto_indent_proc = block end - SHOW_DOC_DIALOG = ->() { - dialog.trap_key = nil - alt_d = [ - [Reline::Key.new(nil, 0xE4, true)], # Normal Alt+d. - [27, 100], # Normal Alt+d when convert-meta isn't used. - [195, 164], # The "ä" that appears when Alt+d is pressed on xterm. - [226, 136, 130] # The "∂" that appears when Alt+d in pressed on iTerm2. - ] + def retrieve_doc_namespace(matched) + preposing, _target, postposing, bind = @completion_params + @completor.doc_namespace(preposing, matched, postposing, bind: bind) + end - if just_cursor_moving and completion_journey_data.nil? - return nil - end - cursor_pos_to_render, result, pointer, autocomplete_dialog = context.pop(4) - return nil if result.nil? or pointer.nil? or pointer < 0 - name = result[pointer] - name = IRB::InputCompletor.retrieve_completion_data(name, doc_namespace: true) + def rdoc_ri_driver + return @rdoc_ri_driver if defined?(@rdoc_ri_driver) - options = {} - options[:extra_doc_dirs] = IRB.conf[:EXTRA_DOC_DIRS] unless IRB.conf[:EXTRA_DOC_DIRS].empty? - driver = RDoc::RI::Driver.new(options) + begin + require 'rdoc' + rescue LoadError + @rdoc_ri_driver = nil + else + options = {} + options[:extra_doc_dirs] = IRB.conf[:EXTRA_DOC_DIRS] unless IRB.conf[:EXTRA_DOC_DIRS].empty? + @rdoc_ri_driver = RDoc::RI::Driver.new(options) + end + end - if key.match?(dialog.name) - begin - driver.display_names([name]) - rescue RDoc::RI::Driver::NotFoundError + def show_doc_dialog_proc + input_method = self # self is changed in the lambda below. + ->() { + dialog.trap_key = nil + alt_d = [ + [Reline::Key.new(nil, 0xE4, true)], # Normal Alt+d. + [27, 100], # Normal Alt+d when convert-meta isn't used. + [195, 164], # The "ä" that appears when Alt+d is pressed on xterm. + [226, 136, 130] # The "∂" that appears when Alt+d in pressed on iTerm2. + ] + + if just_cursor_moving and completion_journey_data.nil? + return nil end - end + cursor_pos_to_render, result, pointer, autocomplete_dialog = context.pop(4) + return nil if result.nil? or pointer.nil? or pointer < 0 - begin - name = driver.expand_name(name) - rescue RDoc::RI::Driver::NotFoundError - return nil - rescue - return nil # unknown error - end - doc = nil - used_for_class = false - if not name =~ /#|\./ - found, klasses, includes, extends = driver.classes_and_includes_and_extends_for(name) - if not found.empty? - doc = driver.class_document(name, found, klasses, includes, extends) - used_for_class = true + name = input_method.retrieve_doc_namespace(result[pointer]) + # Use first one because document dialog does not support multiple namespaces. + name = name.first if name.is_a?(Array) + + show_easter_egg = name&.match?(/\ARubyVM/) && !ENV['RUBY_YES_I_AM_NOT_A_NORMAL_USER'] + + driver = input_method.rdoc_ri_driver + + if key.match?(dialog.name) + if show_easter_egg + IRB.__send__(:easter_egg) + else + begin + driver.display_names([name]) + rescue RDoc::RI::Driver::NotFoundError + end + end end - end - unless used_for_class - doc = RDoc::Markup::Document.new + begin - driver.add_method(doc, name) + name = driver.expand_name(name) rescue RDoc::RI::Driver::NotFoundError - doc = nil + return nil rescue return nil # unknown error end - end - return nil if doc.nil? - width = 40 - - right_x = cursor_pos_to_render.x + autocomplete_dialog.width - if right_x + width > screen_width - right_width = screen_width - (right_x + 1) - left_x = autocomplete_dialog.column - width - left_x = 0 if left_x < 0 - left_width = width > autocomplete_dialog.column ? autocomplete_dialog.column : width - if right_width.positive? and left_width.positive? - if right_width >= left_width + doc = nil + used_for_class = false + if not name =~ /#|\./ + found, klasses, includes, extends = driver.classes_and_includes_and_extends_for(name) + if not found.empty? + doc = driver.class_document(name, found, klasses, includes, extends) + used_for_class = true + end + end + unless used_for_class + doc = RDoc::Markup::Document.new + begin + driver.add_method(doc, name) + rescue RDoc::RI::Driver::NotFoundError + doc = nil + rescue + return nil # unknown error + end + end + return nil if doc.nil? + width = 40 + + right_x = cursor_pos_to_render.x + autocomplete_dialog.width + if right_x + width > screen_width + right_width = screen_width - (right_x + 1) + left_x = autocomplete_dialog.column - width + left_x = 0 if left_x < 0 + left_width = width > autocomplete_dialog.column ? autocomplete_dialog.column : width + if right_width.positive? and left_width.positive? + if right_width >= left_width + width = right_width + x = right_x + else + width = left_width + x = left_x + end + elsif right_width.positive? and left_width <= 0 width = right_width x = right_x - else + elsif right_width <= 0 and left_width.positive? width = left_width x = left_x + else # Both are negative width. + return nil end - elsif right_width.positive? and left_width <= 0 - width = right_width + else x = right_x - elsif right_width <= 0 and left_width.positive? - width = left_width - x = left_x - else # Both are negative width. - return nil end + formatter = RDoc::Markup::ToAnsi.new + formatter.width = width + dialog.trap_key = alt_d + mod_key = RUBY_PLATFORM.match?(/darwin/) ? "Option" : "Alt" + if show_easter_egg + type = STDOUT.external_encoding == Encoding::UTF_8 ? :unicode : :ascii + contents = IRB.send(:easter_egg_logo, type).split("\n") + message = "Press #{mod_key}+d to see more" + contents[0][0, message.size] = message + else + message = "Press #{mod_key}+d to read the full document" + contents = [message] + doc.accept(formatter).split("\n") + end + contents = contents.take(preferred_dialog_height) + + y = cursor_pos_to_render.y + Reline::DialogRenderInfo.new(pos: Reline::CursorPos.new(x, y), contents: contents, width: width, bg_color: '49') + } + end + + def display_document(matched) + driver = rdoc_ri_driver + return unless driver + + if matched =~ /\A(?:::)?RubyVM/ and not ENV['RUBY_YES_I_AM_NOT_A_NORMAL_USER'] + IRB.__send__(:easter_egg) + return + end + + namespace = retrieve_doc_namespace(matched) + return unless namespace + + if namespace.is_a?(Array) + out = RDoc::Markup::Document.new + namespace.each do |m| + begin + driver.add_method(out, m) + rescue RDoc::RI::Driver::NotFoundError + end + end + driver.display(out) else - x = right_x + begin + driver.display_names([namespace]) + rescue RDoc::RI::Driver::NotFoundError + end end - formatter = RDoc::Markup::ToAnsi.new - formatter.width = width - dialog.trap_key = alt_d - mod_key = RUBY_PLATFORM.match?(/darwin/) ? "Option" : "Alt" - message = "Press #{mod_key}+d to read the full document" - contents = [message] + doc.accept(formatter).split("\n") - contents = contents.take(preferred_dialog_height) if respond_to?(:preferred_dialog_height) - - y = cursor_pos_to_render.y - DialogRenderInfo.new(pos: Reline::CursorPos.new(x, y), contents: contents, width: width, bg_color: '49') - } + end # Reads the next line from this input method. # @@ -415,8 +471,8 @@ module IRB Reline.output = @stdout Reline.prompt_proc = @prompt_proc Reline.auto_indent_proc = @auto_indent_proc if @auto_indent_proc - if l = readmultiline(@prompt, false, &@check_termination_proc) - HISTORY.push(l) if !l.empty? + if l = Reline.readmultiline(@prompt, false, &@check_termination_proc) + Reline::HISTORY.push(l) if !l.empty? @line[@line_no += 1] = l + "\n" else @eof = true @@ -432,37 +488,15 @@ module IRB @eof end - # Whether this input method is still readable when there is no more data to - # read. - # - # See IO#eof for more information. - def readable_after_eof? + def prompting? true end - # Returns the current line number for #io. - # - # #line counts the number of times #gets is called. - # - # See IO#lineno for more information. - def line(line_no) - @line[line_no] - end - - # The external encoding for standard input. - def encoding - @stdin.external_encoding - end - # For debug message def inspect config = Reline::Config.new str = "RelineInputMethod with Reline #{Reline::VERSION}" - if config.respond_to?(:inputrc_path) - inputrc_path = File.expand_path(config.inputrc_path) - else - inputrc_path = File.expand_path(ENV['INPUTRC'] || '~/.inputrc') - end + inputrc_path = File.expand_path(config.inputrc_path) str += " and #{inputrc_path}" if File.exist?(inputrc_path) str end diff --git a/lib/irb/inspector.rb b/lib/irb/inspector.rb index ee3b19efdc..667087ccba 100644 --- a/lib/irb/inspector.rb +++ b/lib/irb/inspector.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/inspector.rb - inspect methods # by Keiju ISHITSUKA(keiju@ruby-lang.org) @@ -46,7 +46,7 @@ module IRB # :nodoc: # Determines the inspector to use where +inspector+ is one of the keys passed # during inspector definition. def self.keys_with_inspector(inspector) - INSPECTORS.select{|k,v| v == inspector}.collect{|k, v| k} + INSPECTORS.select{|k, v| v == inspector}.collect{|k, v| k} end # Example @@ -113,7 +113,7 @@ module IRB # :nodoc: Color.colorize_code(v.inspect, colorable: Color.colorable? && Color.inspect_colorable?(v)) } Inspector.def_inspector([true, :pp, :pretty_inspect], proc{require_relative "color_printer"}){|v| - IRB::ColorPrinter.pp(v, '').chomp + IRB::ColorPrinter.pp(v, +'').chomp } Inspector.def_inspector([:yaml, :YAML], proc{require "yaml"}){|v| begin diff --git a/lib/irb/irb.gemspec b/lib/irb/irb.gemspec index e62429ca7b..b29002f593 100644 --- a/lib/irb/irb.gemspec +++ b/lib/irb/irb.gemspec @@ -41,5 +41,6 @@ Gem::Specification.new do |spec| spec.required_ruby_version = Gem::Requirement.new(">= 2.7") - spec.add_dependency "reline", ">= 0.3.0" + spec.add_dependency "reline", ">= 0.4.2" + spec.add_dependency "rdoc", ">= 4.0.0" end diff --git a/lib/irb/lc/error.rb b/lib/irb/lc/error.rb index a5ec150865..ee0f047822 100644 --- a/lib/irb/lc/error.rb +++ b/lib/irb/lc/error.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/lc/error.rb - # by Keiju ISHITSUKA(keiju@ruby-lang.org) @@ -12,11 +12,6 @@ module IRB super("Unrecognized switch: #{val}") end end - class NotImplementedError < StandardError - def initialize(val) - super("Need to define `#{val}'") - end - end class CantReturnToNormalMode < StandardError def initialize super("Can't return to normal mode.") @@ -52,11 +47,6 @@ module IRB super("Undefined prompt mode(#{val}).") end end - class IllegalRCGenerator < StandardError - def initialize - super("Define illegal RC_NAME_GENERATOR.") - end - end # :startdoc: end diff --git a/lib/irb/lc/help-message b/lib/irb/lc/help-message index 7e66100be9..37347306e8 100644 --- a/lib/irb/lc/help-message +++ b/lib/irb/lc/help-message @@ -22,6 +22,7 @@ Usage: irb.rb [options] [programfile] [arguments] Show truncated result on assignment (default). --inspect Use 'inspect' for output. --noinspect Don't use 'inspect' for output. + --no-pager Don't use pager. --multiline Use multiline editor module (default). --nomultiline Don't use multiline editor module. --singleline Use single line editor module. @@ -30,6 +31,9 @@ Usage: irb.rb [options] [programfile] [arguments] --nocolorize Don't use color-highlighting. --autocomplete Use auto-completion (default). --noautocomplete Don't use auto-completion. + --regexp-completor + Use regexp based completion (default). + --type-completor Use type based completion. --prompt prompt-mode, --prompt-mode prompt-mode Set prompt mode. Pre-defined prompt modes are: 'default', 'classic', 'simple', 'inf-ruby', 'xmp', 'null'. diff --git a/lib/irb/lc/ja/error.rb b/lib/irb/lc/ja/error.rb index 50d72c4a10..9e2e5b8870 100644 --- a/lib/irb/lc/ja/error.rb +++ b/lib/irb/lc/ja/error.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/lc/ja/error.rb - # by Keiju ISHITSUKA(keiju@ruby-lang.org) @@ -12,11 +12,6 @@ module IRB super("スイッチ(#{val})が分りません") end end - class NotImplementedError < StandardError - def initialize(val) - super("`#{val}'の定義が必要です") - end - end class CantReturnToNormalMode < StandardError def initialize super("Normalモードに戻れません.") @@ -52,11 +47,6 @@ module IRB super("プロンプトモード(#{val})は定義されていません.") end end - class IllegalRCGenerator < StandardError - def initialize - super("RC_NAME_GENERATORが正しく定義されていません.") - end - end # :startdoc: end diff --git a/lib/irb/lc/ja/help-message b/lib/irb/lc/ja/help-message index 1c15d331ea..99f4449b3b 100644 --- a/lib/irb/lc/ja/help-message +++ b/lib/irb/lc/ja/help-message @@ -9,10 +9,18 @@ Usage: irb.rb [options] [programfile] [arguments] -W[level=2] ruby -W と同じ. --context-mode n 新しいワークスペースを作成した時に関連する Binding オブジェクトの作成方法を 0 から 3 のいずれかに設定する. + --extra-doc-dir 指定したディレクトリのドキュメントを追加で読み込む. --echo 実行結果を表示する(デフォルト). --noecho 実行結果を表示しない. + --echo-on-assignment + 代入結果を表示する. + --noecho-on-assignment + 代入結果を表示しない. + --truncate-echo-on-assignment + truncateされた代入結果を表示する(デフォルト). --inspect 結果出力にinspectを用いる. --noinspect 結果出力にinspectを用いない. + --no-pager ページャを使用しない. --multiline マルチラインエディタを利用する. --nomultiline マルチラインエディタを利用しない. --singleline シングルラインエディタを利用する. @@ -21,6 +29,9 @@ Usage: irb.rb [options] [programfile] [arguments] --nocolorize 色付けを利用しない. --autocomplete オートコンプリートを利用する. --noautocomplete オートコンプリートを利用しない. + --regexp-completor + 補完に正規表現を利用する. + --type-completor 補完に型情報を利用する. --prompt prompt-mode/--prompt-mode prompt-mode プロンプトモードを切替えます. 現在定義されているプ ロンプトモードは, default, simple, xmp, inf-rubyが @@ -31,6 +42,8 @@ Usage: irb.rb [options] [programfile] [arguments] --sample-book-mode/--simple-prompt 非常にシンプルなプロンプトを用いるモードです. --noprompt プロンプト表示を行なわない. + --script スクリプトモード(最初の引数をスクリプトファイルとして扱う、デフォルト) + --noscript 引数をargvとして扱う. --single-irb irb 中で self を実行して得られるオブジェクトをサ ブ irb と共有する. --tracer コマンド実行時にトレースを行なう. diff --git a/lib/irb/locale.rb b/lib/irb/locale.rb index f94aa0f40b..2abcc7354b 100644 --- a/lib/irb/locale.rb +++ b/lib/irb/locale.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/locale.rb - internationalization module # by Keiju ISHITSUKA(keiju@ruby-lang.org) @@ -94,7 +94,7 @@ module IRB # :nodoc: end end - def find(file , paths = $:) + def find(file, paths = $:) dir = File.dirname(file) dir = "" if dir == "." base = File.basename(file) diff --git a/lib/irb/nesting_parser.rb b/lib/irb/nesting_parser.rb new file mode 100644 index 0000000000..5aa940cc28 --- /dev/null +++ b/lib/irb/nesting_parser.rb @@ -0,0 +1,237 @@ +# frozen_string_literal: true +module IRB + module NestingParser + IGNORE_TOKENS = %i[on_sp on_ignored_nl on_comment on_embdoc_beg on_embdoc on_embdoc_end] + + # Scan each token and call the given block with array of token and other information for parsing + def self.scan_opens(tokens) + opens = [] + pending_heredocs = [] + first_token_on_line = true + tokens.each do |t| + skip = false + last_tok, state, args = opens.last + case state + when :in_alias_undef + skip = t.event == :on_kw + when :in_unquoted_symbol + unless IGNORE_TOKENS.include?(t.event) + opens.pop + skip = true + end + when :in_lambda_head + opens.pop if t.event == :on_tlambeg || (t.event == :on_kw && t.tok == 'do') + when :in_method_head + unless IGNORE_TOKENS.include?(t.event) + next_args = [] + body = nil + if args.include?(:receiver) + case t.event + when :on_lparen, :on_ivar, :on_gvar, :on_cvar + # def (receiver). | def @ivar. | def $gvar. | def @@cvar. + next_args << :dot + when :on_kw + case t.tok + when 'self', 'true', 'false', 'nil' + # def self(arg) | def self. + next_args.push(:arg, :dot) + else + # def if(arg) + skip = true + next_args << :arg + end + when :on_op, :on_backtick + # def +(arg) + skip = true + next_args << :arg + when :on_ident, :on_const + # def a(arg) | def a. + next_args.push(:arg, :dot) + end + end + if args.include?(:dot) + # def receiver.name + next_args << :name if t.event == :on_period || (t.event == :on_op && t.tok == '::') + end + if args.include?(:name) + if %i[on_ident on_const on_op on_kw on_backtick].include?(t.event) + # def name(arg) | def receiver.name(arg) + next_args << :arg + skip = true + end + end + if args.include?(:arg) + case t.event + when :on_nl, :on_semicolon + # def receiver.f; + body = :normal + when :on_lparen + # def receiver.f() + next_args << :eq + else + if t.event == :on_op && t.tok == '=' + # def receiver.f = + body = :oneliner + else + # def receiver.f arg + next_args << :arg_without_paren + end + end + end + if args.include?(:eq) + if t.event == :on_op && t.tok == '=' + body = :oneliner + else + body = :normal + end + end + if args.include?(:arg_without_paren) + if %i[on_semicolon on_nl].include?(t.event) + # def f a; + body = :normal + else + # def f a, b + next_args << :arg_without_paren + end + end + if body == :oneliner + opens.pop + elsif body + opens[-1] = [last_tok, nil] + else + opens[-1] = [last_tok, :in_method_head, next_args] + end + end + when :in_for_while_until_condition + if t.event == :on_semicolon || t.event == :on_nl || (t.event == :on_kw && t.tok == 'do') + skip = true if t.event == :on_kw && t.tok == 'do' + opens[-1] = [last_tok, nil] + end + end + + unless skip + case t.event + when :on_kw + case t.tok + when 'begin', 'class', 'module', 'do', 'case' + opens << [t, nil] + when 'end' + opens.pop + when 'def' + opens << [t, :in_method_head, [:receiver, :name]] + when 'if', 'unless' + unless t.state.allbits?(Ripper::EXPR_LABEL) + opens << [t, nil] + end + when 'while', 'until' + unless t.state.allbits?(Ripper::EXPR_LABEL) + opens << [t, :in_for_while_until_condition] + end + when 'ensure', 'rescue' + unless t.state.allbits?(Ripper::EXPR_LABEL) + opens.pop + opens << [t, nil] + end + when 'alias' + opens << [t, :in_alias_undef, 2] + when 'undef' + opens << [t, :in_alias_undef, 1] + when 'elsif', 'else', 'when' + opens.pop + opens << [t, nil] + when 'for' + opens << [t, :in_for_while_until_condition] + when 'in' + if last_tok&.event == :on_kw && %w[case in].include?(last_tok.tok) && first_token_on_line + opens.pop + opens << [t, nil] + end + end + when :on_tlambda + opens << [t, :in_lambda_head] + when :on_lparen, :on_lbracket, :on_lbrace, :on_tlambeg, :on_embexpr_beg, :on_embdoc_beg + opens << [t, nil] + when :on_rparen, :on_rbracket, :on_rbrace, :on_embexpr_end, :on_embdoc_end + opens.pop + when :on_heredoc_beg + pending_heredocs << t + when :on_heredoc_end + opens.pop + when :on_backtick + opens << [t, nil] if t.state.allbits?(Ripper::EXPR_BEG) + when :on_tstring_beg, :on_words_beg, :on_qwords_beg, :on_symbols_beg, :on_qsymbols_beg, :on_regexp_beg + opens << [t, nil] + when :on_tstring_end, :on_regexp_end, :on_label_end + opens.pop + when :on_symbeg + if t.tok == ':' + opens << [t, :in_unquoted_symbol] + else + opens << [t, nil] + end + end + end + if t.event == :on_nl || t.event == :on_semicolon + first_token_on_line = true + elsif t.event != :on_sp + first_token_on_line = false + end + if pending_heredocs.any? && t.tok.include?("\n") + pending_heredocs.reverse_each { |t| opens << [t, nil] } + pending_heredocs = [] + end + if opens.last && opens.last[1] == :in_alias_undef && !IGNORE_TOKENS.include?(t.event) && t.event != :on_heredoc_end + tok, state, arg = opens.pop + opens << [tok, state, arg - 1] if arg >= 1 + end + yield t, opens if block_given? + end + opens.map(&:first) + pending_heredocs.reverse + end + + def self.open_tokens(tokens) + # scan_opens without block will return a list of open tokens at last token position + scan_opens(tokens) + end + + # Calculates token information [line_tokens, prev_opens, next_opens, min_depth] for each line. + # Example code + # ["hello + # world"+( + # First line + # line_tokens: [[lbracket, '['], [tstring_beg, '"'], [tstring_content("hello\nworld"), "hello\n"]] + # prev_opens: [] + # next_tokens: [lbracket, tstring_beg] + # min_depth: 0 (minimum at beginning of line) + # Second line + # line_tokens: [[tstring_content("hello\nworld"), "world"], [tstring_end, '"'], [op, '+'], [lparen, '(']] + # prev_opens: [lbracket, tstring_beg] + # next_tokens: [lbracket, lparen] + # min_depth: 1 (minimum just after tstring_end) + def self.parse_by_line(tokens) + line_tokens = [] + prev_opens = [] + min_depth = 0 + output = [] + last_opens = scan_opens(tokens) do |t, opens| + depth = t == opens.last&.first ? opens.size - 1 : opens.size + min_depth = depth if depth < min_depth + if t.tok.include?("\n") + t.tok.each_line do |line| + line_tokens << [t, line] + next if line[-1] != "\n" + next_opens = opens.map(&:first) + output << [line_tokens, prev_opens, next_opens, min_depth] + prev_opens = next_opens + min_depth = prev_opens.size + line_tokens = [] + end + else + line_tokens << [t, t.tok] + end + end + output << [line_tokens, prev_opens, last_opens, min_depth] if line_tokens.any? + output + end + end +end diff --git a/lib/irb/notifier.rb b/lib/irb/notifier.rb index 612de3df16..dc1b9ef14b 100644 --- a/lib/irb/notifier.rb +++ b/lib/irb/notifier.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # notifier.rb - output methods used by irb # by Keiju ISHITSUKA(keiju@ruby-lang.org) diff --git a/lib/irb/output-method.rb b/lib/irb/output-method.rb index f5ea57111d..69942f47a2 100644 --- a/lib/irb/output-method.rb +++ b/lib/irb/output-method.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # output-method.rb - output methods used by irb # by Keiju ISHITSUKA(keiju@ruby-lang.org) @@ -9,16 +9,10 @@ module IRB # IRB::Notifier. You can define your own output method to use with Irb.new, # or Context.new class OutputMethod - class NotImplementedError < StandardError - def initialize(val) - super("Need to define `#{val}'") - end - end - # Open this method to implement your own output method, raises a # NotImplementedError if you don't define #print in your own class. def print(*opts) - raise NotImplementedError, "print" + raise NotImplementedError end # Prints the given +opts+, with a newline delimiter. diff --git a/lib/irb/pager.rb b/lib/irb/pager.rb new file mode 100644 index 0000000000..3391b32c66 --- /dev/null +++ b/lib/irb/pager.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module IRB + # The implementation of this class is borrowed from RDoc's lib/rdoc/ri/driver.rb. + # Please do NOT use this class directly outside of IRB. + class Pager + PAGE_COMMANDS = [ENV['RI_PAGER'], ENV['PAGER'], 'less', 'more'].compact.uniq + + class << self + def page_content(content, **options) + if content_exceeds_screen_height?(content) + page(**options) do |io| + io.puts content + end + else + $stdout.puts content + end + end + + def page(retain_content: false) + if should_page? && pager = setup_pager(retain_content: retain_content) + begin + pid = pager.pid + yield pager + ensure + pager.close + end + else + yield $stdout + end + # When user presses Ctrl-C, IRB would raise `IRB::Abort` + # But since Pager is implemented by running paging commands like `less` in another process with `IO.popen`, + # the `IRB::Abort` exception only interrupts IRB's execution but doesn't affect the pager + # So to properly terminate the pager with Ctrl-C, we need to catch `IRB::Abort` and kill the pager process + rescue IRB::Abort + Process.kill("TERM", pid) if pid + nil + rescue Errno::EPIPE + end + + private + + def should_page? + IRB.conf[:USE_PAGER] && STDIN.tty? && (ENV.key?("TERM") && ENV["TERM"] != "dumb") + end + + def content_exceeds_screen_height?(content) + screen_height, screen_width = begin + Reline.get_screen_size + rescue Errno::EINVAL + [24, 80] + end + + pageable_height = screen_height - 3 # leave some space for previous and the current prompt + + # If the content has more lines than the pageable height + content.lines.count > pageable_height || + # Or if the content is a few long lines + pageable_height * screen_width < Reline::Unicode.calculate_width(content, true) + end + + def setup_pager(retain_content:) + require 'shellwords' + + PAGE_COMMANDS.each do |pager_cmd| + cmd = Shellwords.split(pager_cmd) + next if cmd.empty? + + if cmd.first == 'less' + cmd << '-R' unless cmd.include?('-R') + cmd << '-X' if retain_content && !cmd.include?('-X') + end + + begin + io = IO.popen(cmd, 'w') + rescue + next + end + + if $? && $?.pid == io.pid && $?.exited? # pager didn't work + next + end + + return io + end + + nil + end + end + end +end diff --git a/lib/irb/ruby-lex.rb b/lib/irb/ruby-lex.rb index e29d52e47c..f6ac7f0f5f 100644 --- a/lib/irb/ruby-lex.rb +++ b/lib/irb/ruby-lex.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/ruby-lex.rb - ruby lexcal analyzer # by Keiju ISHITSUKA(keiju@ruby-lang.org) @@ -6,824 +6,469 @@ require "ripper" require "jruby" if RUBY_ENGINE == "jruby" - -# :stopdoc: -class RubyLex - - class TerminateLineInput < StandardError - def initialize - super("Terminate Line Input") - end - end - - def initialize(context) - @context = context - @line_no = 1 - @prompt = nil - end - - def self.compile_with_errors_suppressed(code, line_no: 1) - begin - result = yield code, line_no - rescue ArgumentError - # Ruby can issue an error for the code if there is an - # incomplete magic comment for encoding in it. Force an - # expression with a new line before the code in this - # case to prevent magic comment handling. To make sure - # line numbers in the lexed code remain the same, - # decrease the line number by one. - code = ";\n#{code}" - line_no -= 1 - result = yield code, line_no +require_relative "nesting_parser" + +module IRB + # :stopdoc: + class RubyLex + ASSIGNMENT_NODE_TYPES = [ + # Local, instance, global, class, constant, instance, and index assignment: + # "foo = bar", + # "@foo = bar", + # "$foo = bar", + # "@@foo = bar", + # "::Foo = bar", + # "a::Foo = bar", + # "Foo = bar" + # "foo.bar = 1" + # "foo[1] = bar" + :assign, + + # Operation assignment: + # "foo += bar" + # "foo -= bar" + # "foo ||= bar" + # "foo &&= bar" + :opassign, + + # Multiple assignment: + # "foo, bar = 1, 2 + :massign, + ] + + class TerminateLineInput < StandardError + def initialize + super("Terminate Line Input") + end end - result - end - - def single_line_command?(code) - command = code.split(/\s/, 2).first - @context.symbol_alias?(command) || @context.transform_args?(command) - end - # io functions - def set_input(&block) - @input = block - end - - def configure_io(io) - @io = io - if @io.respond_to?(:check_termination) - @io.check_termination do |code| - if Reline::IOGate.in_pasting? - lex = RubyLex.new(@context) - rest = lex.check_termination_in_prev_line(code) - if rest - Reline.delete_text - rest.bytes.reverse_each do |c| - Reline.ungetc(c) - end - true - else - false - end - else - # Accept any single-line input for symbol aliases or commands that transform args - next true if single_line_command?(code) - - ltype, indent, continue, code_block_open = check_code_state(code) - if ltype or indent > 0 or continue or code_block_open - false - else - true - end - end + def self.compile_with_errors_suppressed(code, line_no: 1) + begin + result = yield code, line_no + rescue ArgumentError + # Ruby can issue an error for the code if there is an + # incomplete magic comment for encoding in it. Force an + # expression with a new line before the code in this + # case to prevent magic comment handling. To make sure + # line numbers in the lexed code remain the same, + # decrease the line number by one. + code = ";\n#{code}" + line_no -= 1 + result = yield code, line_no end + result end - if @io.respond_to?(:dynamic_prompt) - @io.dynamic_prompt do |lines| - lines << '' if lines.empty? - result = [] - tokens = self.class.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, context: @context) - code = String.new - partial_tokens = [] - unprocessed_tokens = [] - line_num_offset = 0 - tokens.each do |t| - partial_tokens << t - unprocessed_tokens << t - if t.tok.include?("\n") - t_str = t.tok - t_str.each_line("\n") do |s| - code << s - next unless s.include?("\n") - ltype, indent, continue, code_block_open = check_state(code, partial_tokens) - result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset) - line_num_offset += 1 - end - unprocessed_tokens = [] - else - code << t.tok - end - end - unless unprocessed_tokens.empty? - ltype, indent, continue, code_block_open = check_state(code, unprocessed_tokens) - result << @prompt.call(ltype, indent, continue || code_block_open, @line_no + line_num_offset) - end - result - end + ERROR_TOKENS = [ + :on_parse_error, + :compile_error, + :on_assign_error, + :on_alias_error, + :on_class_name_error, + :on_param_error + ] + + def self.generate_local_variables_assign_code(local_variables) + "#{local_variables.join('=')}=nil;" unless local_variables.empty? end - if @io.respond_to?(:auto_indent) and @context.auto_indent_mode - @io.auto_indent do |lines, line_index, byte_pointer, is_newline| - if is_newline - @tokens = self.class.ripper_lex_without_warning(lines[0..line_index].join("\n"), context: @context) - prev_spaces = find_prev_spaces(line_index) - depth_difference = check_newline_depth_difference - depth_difference = 0 if depth_difference < 0 - prev_spaces + depth_difference * 2 - else - code = line_index.zero? ? '' : lines[0..(line_index - 1)].map{ |l| l + "\n" }.join - last_line = lines[line_index]&.byteslice(0, byte_pointer) - code += last_line if last_line - @tokens = self.class.ripper_lex_without_warning(code, context: @context) - check_corresponding_token_depth(lines, line_index) + # Some part of the code is not included in Ripper's token. + # Example: DATA part, token after heredoc_beg when heredoc has unclosed embexpr. + # With interpolated tokens, tokens.map(&:tok).join will be equal to code. + def self.interpolate_ripper_ignored_tokens(code, tokens) + line_positions = [0] + code.lines.each do |line| + line_positions << line_positions.last + line.bytesize + end + prev_byte_pos = 0 + interpolated = [] + prev_line = 1 + tokens.each do |t| + line, col = t.pos + byte_pos = line_positions[line - 1] + col + if prev_byte_pos < byte_pos + tok = code.byteslice(prev_byte_pos...byte_pos) + pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]] + interpolated << Ripper::Lexer::Elem.new(pos, :on_ignored_by_ripper, tok, 0) + prev_line += tok.count("\n") end + interpolated << t + prev_byte_pos = byte_pos + t.tok.bytesize + prev_line += t.tok.count("\n") end - end - end - - def set_prompt(&block) - @prompt = block - end - - ERROR_TOKENS = [ - :on_parse_error, - :compile_error, - :on_assign_error, - :on_alias_error, - :on_class_name_error, - :on_param_error - ] - - def self.generate_local_variables_assign_code(local_variables) - "#{local_variables.join('=')}=nil;" unless local_variables.empty? - end - - def self.ripper_lex_without_warning(code, context: nil) - verbose, $VERBOSE = $VERBOSE, nil - lvars_code = generate_local_variables_assign_code(context&.local_variables || []) - if lvars_code - code = "#{lvars_code}\n#{code}" - line_no = 0 - else - line_no = 1 + if prev_byte_pos < code.bytesize + tok = code.byteslice(prev_byte_pos..) + pos = [prev_line, prev_byte_pos - line_positions[prev_line - 1]] + interpolated << Ripper::Lexer::Elem.new(pos, :on_ignored_by_ripper, tok, 0) + end + interpolated end - compile_with_errors_suppressed(code, line_no: line_no) do |inner_code, line_no| - lexer = Ripper::Lexer.new(inner_code, '-', line_no) - lexer.scan.each_with_object([]) do |t, tokens| - next if t.pos.first == 0 - prev_tk = tokens.last - position_overlapped = prev_tk && t.pos[0] == prev_tk.pos[0] && t.pos[1] < prev_tk.pos[1] + prev_tk.tok.bytesize - if position_overlapped - tokens[-1] = t if ERROR_TOKENS.include?(prev_tk.event) && !ERROR_TOKENS.include?(t.event) - else - tokens << t - end + def self.ripper_lex_without_warning(code, local_variables: []) + verbose, $VERBOSE = $VERBOSE, nil + lvars_code = generate_local_variables_assign_code(local_variables) + original_code = code + if lvars_code + code = "#{lvars_code}\n#{code}" + line_no = 0 + else + line_no = 1 end - end - ensure - $VERBOSE = verbose - end - def find_prev_spaces(line_index) - return 0 if @tokens.size == 0 - md = @tokens[0].tok.match(/(\A +)/) - prev_spaces = md.nil? ? 0 : md[1].count(' ') - line_count = 0 - @tokens.each_with_index do |t, i| - if t.tok.include?("\n") - line_count += t.tok.count("\n") - if line_count >= line_index - return prev_spaces - end - next if t.event == :on_tstring_content || t.event == :on_words_sep - if (@tokens.size - 1) > i - md = @tokens[i + 1].tok.match(/(\A +)/) - prev_spaces = md.nil? ? 0 : md[1].count(' ') + compile_with_errors_suppressed(code, line_no: line_no) do |inner_code, line_no| + lexer = Ripper::Lexer.new(inner_code, '-', line_no) + tokens = [] + lexer.scan.each do |t| + next if t.pos.first == 0 + prev_tk = tokens.last + position_overlapped = prev_tk && t.pos[0] == prev_tk.pos[0] && t.pos[1] < prev_tk.pos[1] + prev_tk.tok.bytesize + if position_overlapped + tokens[-1] = t if ERROR_TOKENS.include?(prev_tk.event) && !ERROR_TOKENS.include?(t.event) + else + tokens << t + end end + interpolate_ripper_ignored_tokens(original_code, tokens) end + ensure + $VERBOSE = verbose end - prev_spaces - end - - def check_state(code, tokens) - ltype = process_literal_type(tokens) - indent = process_nesting_level(tokens) - continue = process_continue(tokens) - lvars_code = self.class.generate_local_variables_assign_code(@context.local_variables) - code = "#{lvars_code}\n#{code}" if lvars_code - code_block_open = check_code_block(code, tokens) - [ltype, indent, continue, code_block_open] - end - - def check_code_state(code) - check_target_code = code.gsub(/\s*\z/, '').concat("\n") - tokens = self.class.ripper_lex_without_warning(check_target_code, context: @context) - check_state(check_target_code, tokens) - end - - def save_prompt_to_context_io(ltype, indent, continue, line_num_offset) - # Implicitly saves prompt string to `@context.io.prompt`. This will be used in the next `@input.call`. - @prompt.call(ltype, indent, continue, @line_no + line_num_offset) - end - - def readmultiline - save_prompt_to_context_io(nil, 0, false, 0) - - # multiline - return @input.call if @io.respond_to?(:check_termination) - - # nomultiline - code = '' - line_offset = 0 - loop do - line = @input.call - unless line - return code.empty? ? nil : code - end - - code << line - # Accept any single-line input for symbol aliases or commands that transform args - return code if single_line_command?(code) - ltype, indent, continue, code_block_open = check_code_state(code) - return code unless ltype or indent > 0 or continue or code_block_open - - line_offset += 1 - save_prompt_to_context_io(ltype, indent, continue, line_offset) + def check_code_state(code, local_variables:) + tokens = self.class.ripper_lex_without_warning(code, local_variables: local_variables) + opens = NestingParser.open_tokens(tokens) + [tokens, opens, code_terminated?(code, tokens, opens, local_variables: local_variables)] end - end - - def each_top_level_statement - loop do - code = readmultiline - break unless code - if code != "\n" - code.force_encoding(@io.encoding) - yield code, @line_no + def code_terminated?(code, tokens, opens, local_variables:) + case check_code_syntax(code, local_variables: local_variables) + when :unrecoverable_error + true + when :recoverable_error + false + when :other_error + opens.empty? && !should_continue?(tokens) + when :valid + !should_continue?(tokens) end - @line_no += code.count("\n") - rescue TerminateLineInput end - end - def process_continue(tokens) - # last token is always newline - if tokens.size >= 2 and tokens[-2].event == :on_regexp_end - # end of regexp literal - return false - elsif tokens.size >= 2 and tokens[-2].event == :on_semicolon - return false - elsif tokens.size >= 2 and tokens[-2].event == :on_kw and ['begin', 'else', 'ensure'].include?(tokens[-2].tok) - return false - elsif !tokens.empty? and tokens.last.tok == "\\\n" - return true - elsif tokens.size >= 1 and tokens[-1].event == :on_heredoc_end # "EOH\n" - return false - elsif tokens.size >= 2 and tokens[-2].state.anybits?(Ripper::EXPR_BEG | Ripper::EXPR_FNAME) and tokens[-2].tok !~ /\A\.\.\.?\z/ - # end of literal except for regexp - # endless range at end of line is not a continue - return true - end - false - end - - def check_code_block(code, tokens) - return true if tokens.empty? - if tokens.last.event == :on_heredoc_beg - return true - end + def assignment_expression?(code, local_variables:) + # Try to parse the code and check if the last of possibly multiple + # expressions is an assignment type. - begin # check if parser error are available + # If the expression is invalid, Ripper.sexp should return nil which will + # result in false being returned. Any valid expression should return an + # s-expression where the second element of the top level array is an + # array of parsed expressions. The first element of each expression is the + # expression's type. verbose, $VERBOSE = $VERBOSE, nil - case RUBY_ENGINE - when 'ruby' - self.class.compile_with_errors_suppressed(code) do |inner_code, line_no| - RubyVM::InstructionSequence.compile(inner_code, nil, nil, line_no) - end - when 'jruby' - JRuby.compile_ir(code) - else - catch(:valid) do - eval("BEGIN { throw :valid, true }\n#{code}") - false - end - end - rescue EncodingError - # This is for a hash with invalid encoding symbol, {"\xAE": 1} - rescue SyntaxError => e - case e.message - when /unterminated (?:string|regexp) meets end of file/ - # "unterminated regexp meets end of file" - # - # example: - # / - # - # "unterminated string meets end of file" - # - # example: - # ' - return true - when /syntax error, unexpected end-of-input/ - # "syntax error, unexpected end-of-input, expecting keyword_end" - # - # example: - # if true - # hoge - # if false - # fuga - # end - return true - when /syntax error, unexpected keyword_end/ - # "syntax error, unexpected keyword_end" - # - # example: - # if ( - # end - # - # example: - # end - return false - when /syntax error, unexpected '\.'/ - # "syntax error, unexpected '.'" - # - # example: - # . - return false - when /unexpected tREGEXP_BEG/ - # "syntax error, unexpected tREGEXP_BEG, expecting keyword_do or '{' or '('" - # - # example: - # method / f / - return false - end + code = "#{RubyLex.generate_local_variables_assign_code(local_variables) || 'nil;'}\n#{code}" + # Get the last node_type of the line. drop(1) is to ignore the local_variables_assign_code part. + node_type = Ripper.sexp(code)&.dig(1)&.drop(1)&.dig(-1, 0) + ASSIGNMENT_NODE_TYPES.include?(node_type) ensure $VERBOSE = verbose end - last_lex_state = tokens.last.state - - if last_lex_state.allbits?(Ripper::EXPR_BEG) - return false - elsif last_lex_state.allbits?(Ripper::EXPR_DOT) - return true - elsif last_lex_state.allbits?(Ripper::EXPR_CLASS) - return true - elsif last_lex_state.allbits?(Ripper::EXPR_FNAME) - return true - elsif last_lex_state.allbits?(Ripper::EXPR_VALUE) - return true - elsif last_lex_state.allbits?(Ripper::EXPR_ARG) - return false + def should_continue?(tokens) + # Look at the last token and check if IRB need to continue reading next line. + # Example code that should continue: `a\` `a +` `a.` + # Trailing spaces, newline, comments are skipped + return true if tokens.last&.event == :on_sp && tokens.last.tok == "\\\n" + + tokens.reverse_each do |token| + case token.event + when :on_sp, :on_nl, :on_ignored_nl, :on_comment, :on_embdoc_beg, :on_embdoc, :on_embdoc_end + # Skip + when :on_regexp_end, :on_heredoc_end, :on_semicolon + # State is EXPR_BEG but should not continue + return false + else + # Endless range should not continue + return false if token.event == :on_op && token.tok.match?(/\A\.\.\.?\z/) + + # EXPR_DOT and most of the EXPR_BEG should continue + return token.state.anybits?(Ripper::EXPR_BEG | Ripper::EXPR_DOT) + end + end + false end - false - end + def check_code_syntax(code, local_variables:) + lvars_code = RubyLex.generate_local_variables_assign_code(local_variables) + code = "#{lvars_code}\n#{code}" - def process_nesting_level(tokens) - indent = 0 - in_oneliner_def = nil - tokens.each_with_index { |t, index| - # detecting one-liner method definition - if in_oneliner_def.nil? - if t.state.allbits?(Ripper::EXPR_ENDFN) - in_oneliner_def = :ENDFN - end - else - if t.state.allbits?(Ripper::EXPR_ENDFN) - # continuing - elsif t.state.allbits?(Ripper::EXPR_BEG) - if t.tok == '=' - in_oneliner_def = :BODY + begin # check if parser error are available + verbose, $VERBOSE = $VERBOSE, nil + case RUBY_ENGINE + when 'ruby' + self.class.compile_with_errors_suppressed(code) do |inner_code, line_no| + RubyVM::InstructionSequence.compile(inner_code, nil, nil, line_no) end + when 'jruby' + JRuby.compile_ir(code) else - if in_oneliner_def == :BODY - # one-liner method definition - indent -= 1 + catch(:valid) do + eval("BEGIN { throw :valid, true }\n#{code}") + false end - in_oneliner_def = nil end - end - - case t.event - when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg - indent += 1 - when :on_rbracket, :on_rbrace, :on_rparen - indent -= 1 - when :on_kw - next if index > 0 and tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME) - case t.tok - when 'do' - syntax_of_do = take_corresponding_syntax_to_kw_do(tokens, index) - indent += 1 if syntax_of_do == :method_calling - when 'def', 'case', 'for', 'begin', 'class', 'module' - indent += 1 - when 'if', 'unless', 'while', 'until' - # postfix if/unless/while/until must be Ripper::EXPR_LABEL - indent += 1 unless t.state.allbits?(Ripper::EXPR_LABEL) - when 'end' - indent -= 1 + rescue EncodingError + # This is for a hash with invalid encoding symbol, {"\xAE": 1} + :unrecoverable_error + rescue SyntaxError => e + case e.message + when /unexpected keyword_end/ + # "syntax error, unexpected keyword_end" + # + # example: + # if ( + # end + # + # example: + # end + return :unrecoverable_error + when /unexpected '\.'/ + # "syntax error, unexpected '.'" + # + # example: + # . + return :unrecoverable_error + when /unexpected tREGEXP_BEG/ + # "syntax error, unexpected tREGEXP_BEG, expecting keyword_do or '{' or '('" + # + # example: + # method / f / + return :unrecoverable_error + when /unterminated (?:string|regexp) meets end of file/ + # "unterminated regexp meets end of file" + # + # example: + # / + # + # "unterminated string meets end of file" + # + # example: + # ' + return :recoverable_error + when /unexpected end-of-input/ + # "syntax error, unexpected end-of-input, expecting keyword_end" + # + # example: + # if true + # hoge + # if false + # fuga + # end + return :recoverable_error + else + return :other_error end + ensure + $VERBOSE = verbose end - # percent literals are not indented - } - indent - end + :valid + end - def is_method_calling?(tokens, index) - tk = tokens[index] - if tk.state.anybits?(Ripper::EXPR_CMDARG) and tk.event == :on_ident - # The target method call to pass the block with "do". - return true - elsif tk.state.anybits?(Ripper::EXPR_ARG) and tk.event == :on_ident - non_sp_index = tokens[0..(index - 1)].rindex{ |t| t.event != :on_sp } - if non_sp_index - prev_tk = tokens[non_sp_index] - if prev_tk.state.anybits?(Ripper::EXPR_DOT) and prev_tk.event == :on_period - # The target method call with receiver to pass the block with "do". - return true + def calc_indent_level(opens) + indent_level = 0 + opens.each_with_index do |t, index| + case t.event + when :on_heredoc_beg + if opens[index + 1]&.event != :on_heredoc_beg + if t.tok.match?(/^<<[~-]/) + indent_level += 1 + else + indent_level = 0 + end + end + when :on_tstring_beg, :on_regexp_beg, :on_symbeg, :on_backtick + # No indent: "", //, :"", `` + # Indent: %(), %r(), %i(), %x() + indent_level += 1 if t.tok.start_with? '%' + when :on_embdoc_beg + indent_level = 0 + else + indent_level += 1 unless t.tok == 'alias' || t.tok == 'undef' end end + indent_level end - false - end - def take_corresponding_syntax_to_kw_do(tokens, index) - syntax_of_do = nil - # Finding a syntax corresponding to "do". - index.downto(0) do |i| - tk = tokens[i] - # In "continue", the token isn't the corresponding syntax to "do". - non_sp_index = tokens[0..(i - 1)].rindex{ |t| t.event != :on_sp } - first_in_fomula = false - if non_sp_index.nil? - first_in_fomula = true - elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index].event) - first_in_fomula = true - end - if is_method_calling?(tokens, i) - syntax_of_do = :method_calling - break if first_in_fomula - elsif tk.event == :on_kw && %w{while until for}.include?(tk.tok) - # A loop syntax in front of "do" found. - # - # while cond do # also "until" or "for" - # end - # - # This "do" doesn't increment indent because the loop syntax already - # incremented. - syntax_of_do = :loop_syntax - break if first_in_fomula - end + FREE_INDENT_TOKENS = %i[on_tstring_beg on_backtick on_regexp_beg on_symbeg] + + def free_indent_token?(token) + FREE_INDENT_TOKENS.include?(token&.event) end - syntax_of_do - end - def is_the_in_correspond_to_a_for(tokens, index) - syntax_of_in = nil - # Finding a syntax corresponding to "do". - index.downto(0) do |i| - tk = tokens[i] - # In "continue", the token isn't the corresponding syntax to "do". - non_sp_index = tokens[0..(i - 1)].rindex{ |t| t.event != :on_sp } - first_in_fomula = false - if non_sp_index.nil? - first_in_fomula = true - elsif [:on_ignored_nl, :on_nl, :on_comment].include?(tokens[non_sp_index].event) - first_in_fomula = true - end - if tk.event == :on_kw && tk.tok == 'for' - # A loop syntax in front of "do" found. - # - # while cond do # also "until" or "for" - # end - # - # This "do" doesn't increment indent because the loop syntax already - # incremented. - syntax_of_in = :for + # Calculates the difference of pasted code's indent and indent calculated from tokens + def indent_difference(lines, line_results, line_index) + loop do + _tokens, prev_opens, _next_opens, min_depth = line_results[line_index] + open_token = prev_opens.last + if !open_token || (open_token.event != :on_heredoc_beg && !free_indent_token?(open_token)) + # If the leading whitespace is an indent, return the difference + indent_level = calc_indent_level(prev_opens.take(min_depth)) + calculated_indent = 2 * indent_level + actual_indent = lines[line_index][/^ */].size + return actual_indent - calculated_indent + elsif open_token.event == :on_heredoc_beg && open_token.tok.match?(/^<<[^-~]/) + return 0 + end + # If the leading whitespace is not an indent but part of a multiline token + # Calculate base_indent of the multiline token's beginning line + line_index = open_token.pos[0] - 1 end - break if first_in_fomula end - syntax_of_in - end - def check_newline_depth_difference - depth_difference = 0 - open_brace_on_line = 0 - in_oneliner_def = nil - @tokens.each_with_index do |t, index| - # detecting one-liner method definition - if in_oneliner_def.nil? - if t.state.allbits?(Ripper::EXPR_ENDFN) - in_oneliner_def = :ENDFN - end + def process_indent_level(tokens, lines, line_index, is_newline) + line_results = NestingParser.parse_by_line(tokens) + result = line_results[line_index] + if result + _tokens, prev_opens, next_opens, min_depth = result else - if t.state.allbits?(Ripper::EXPR_ENDFN) - # continuing - elsif t.state.allbits?(Ripper::EXPR_BEG) - if t.tok == '=' - in_oneliner_def = :BODY - end - else - if in_oneliner_def == :BODY - # one-liner method definition - depth_difference -= 1 - end - in_oneliner_def = nil - end + # When last line is empty + prev_opens = next_opens = line_results.last[2] + min_depth = next_opens.size end - case t.event - when :on_ignored_nl, :on_nl, :on_comment - if index != (@tokens.size - 1) and in_oneliner_def != :BODY - depth_difference = 0 - open_brace_on_line = 0 - end - next - when :on_sp - next - end + # To correctly indent line like `end.map do`, we use shortest open tokens on each line for indent calculation. + # Shortest open tokens can be calculated by `opens.take(min_depth)` + indent = 2 * calc_indent_level(prev_opens.take(min_depth)) - case t.event - when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg - depth_difference += 1 - open_brace_on_line += 1 - when :on_rbracket, :on_rbrace, :on_rparen - depth_difference -= 1 if open_brace_on_line > 0 - when :on_kw - next if index > 0 and @tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME) - case t.tok - when 'do' - syntax_of_do = take_corresponding_syntax_to_kw_do(@tokens, index) - depth_difference += 1 if syntax_of_do == :method_calling - when 'def', 'case', 'for', 'begin', 'class', 'module' - depth_difference += 1 - when 'if', 'unless', 'while', 'until', 'rescue' - # postfix if/unless/while/until/rescue must be Ripper::EXPR_LABEL - unless t.state.allbits?(Ripper::EXPR_LABEL) - depth_difference += 1 - end - when 'else', 'elsif', 'ensure', 'when' - depth_difference += 1 - when 'in' - unless is_the_in_correspond_to_a_for(@tokens, index) - depth_difference += 1 - end - when 'end' - depth_difference -= 1 - end - end - end - depth_difference - end + preserve_indent = lines[line_index - (is_newline ? 1 : 0)][/^ */].size - def check_corresponding_token_depth(lines, line_index) - corresponding_token_depth = nil - is_first_spaces_of_line = true - is_first_printable_of_line = true - spaces_of_nest = [] - spaces_at_line_head = 0 - open_brace_on_line = 0 - in_oneliner_def = nil - - if heredoc_scope? - return lines[line_index][/^ */].length - end + prev_open_token = prev_opens.last + next_open_token = next_opens.last - @tokens.each_with_index do |t, index| - # detecting one-liner method definition - if in_oneliner_def.nil? - if t.state.allbits?(Ripper::EXPR_ENDFN) - in_oneliner_def = :ENDFN - end + # Calculates base indent for pasted code on the line where prev_open_token is located + # irb(main):001:1* if a # base_indent is 2, indent calculated from tokens is 0 + # irb(main):002:1* if b # base_indent is 6, indent calculated from tokens is 2 + # irb(main):003:0> c # base_indent is 6, indent calculated from tokens is 4 + if prev_open_token + base_indent = [0, indent_difference(lines, line_results, prev_open_token.pos[0] - 1)].max else - if t.state.allbits?(Ripper::EXPR_ENDFN) - # continuing - elsif t.state.allbits?(Ripper::EXPR_BEG) - if t.tok == '=' - in_oneliner_def = :BODY - end - else - if in_oneliner_def == :BODY - # one-liner method definition - if is_first_printable_of_line - corresponding_token_depth = spaces_of_nest.pop - else - spaces_of_nest.pop - corresponding_token_depth = nil - end - end - in_oneliner_def = nil - end + base_indent = 0 end - case t.event - when :on_ignored_nl, :on_nl, :on_comment, :on_heredoc_end, :on_embdoc_end - if in_oneliner_def != :BODY - corresponding_token_depth = nil - spaces_at_line_head = 0 - is_first_spaces_of_line = true - is_first_printable_of_line = true - open_brace_on_line = 0 + if free_indent_token?(prev_open_token) + if is_newline && prev_open_token.pos[0] == line_index + # First newline inside free-indent token + base_indent + indent + else + # Accept any number of indent inside free-indent token + preserve_indent end - next - when :on_sp - spaces_at_line_head = t.tok.count(' ') if is_first_spaces_of_line - is_first_spaces_of_line = false - next - end - - case t.event - when :on_lbracket, :on_lbrace, :on_lparen, :on_tlambeg - spaces_of_nest.push(spaces_at_line_head + open_brace_on_line * 2) - open_brace_on_line += 1 - when :on_rbracket, :on_rbrace, :on_rparen - if is_first_printable_of_line - corresponding_token_depth = spaces_of_nest.pop + elsif prev_open_token&.event == :on_embdoc_beg || next_open_token&.event == :on_embdoc_beg + if prev_open_token&.event == next_open_token&.event + # Accept any number of indent inside embdoc content + preserve_indent else - spaces_of_nest.pop - corresponding_token_depth = nil + # =begin or =end + 0 end - open_brace_on_line -= 1 - when :on_kw - next if index > 0 and @tokens[index - 1].state.allbits?(Ripper::EXPR_FNAME) - case t.tok - when 'do' - syntax_of_do = take_corresponding_syntax_to_kw_do(@tokens, index) - if syntax_of_do == :method_calling - spaces_of_nest.push(spaces_at_line_head) - end - when 'def', 'case', 'for', 'begin', 'class', 'module' - spaces_of_nest.push(spaces_at_line_head) - when 'rescue' - unless t.state.allbits?(Ripper::EXPR_LABEL) - corresponding_token_depth = spaces_of_nest.last - end - when 'if', 'unless', 'while', 'until' - # postfix if/unless/while/until must be Ripper::EXPR_LABEL - unless t.state.allbits?(Ripper::EXPR_LABEL) - spaces_of_nest.push(spaces_at_line_head) - end - when 'else', 'elsif', 'ensure', 'when' - corresponding_token_depth = spaces_of_nest.last - when 'in' - if in_keyword_case_scope? - corresponding_token_depth = spaces_of_nest.last - end - when 'end' - if is_first_printable_of_line - corresponding_token_depth = spaces_of_nest.pop + elsif prev_open_token&.event == :on_heredoc_beg + tok = prev_open_token.tok + if prev_opens.size <= next_opens.size + if is_newline && lines[line_index].empty? && line_results[line_index - 1][1].last != next_open_token + # First line in heredoc + tok.match?(/^<<[-~]/) ? base_indent + indent : indent + elsif tok.match?(/^<<~/) + # Accept extra indent spaces inside `<<~` heredoc + [base_indent + indent, preserve_indent].max else - spaces_of_nest.pop - corresponding_token_depth = nil + # Accept any number of indent inside other heredoc + preserve_indent end + else + # Heredoc close + prev_line_indent_level = calc_indent_level(prev_opens) + tok.match?(/^<<[~-]/) ? base_indent + 2 * (prev_line_indent_level - 1) : 0 end + else + base_indent + indent end - is_first_spaces_of_line = false - is_first_printable_of_line = false end - corresponding_token_depth - end - - def check_string_literal(tokens) - i = 0 - start_token = [] - end_type = [] - pending_heredocs = [] - while i < tokens.size - t = tokens[i] - case t.event - when *end_type.last - start_token.pop - end_type.pop - when :on_tstring_beg - start_token << t - end_type << [:on_tstring_end, :on_label_end] - when :on_regexp_beg - start_token << t - end_type << :on_regexp_end - when :on_symbeg - acceptable_single_tokens = %i{on_ident on_const on_op on_cvar on_ivar on_gvar on_kw on_int on_backtick} - if (i + 1) < tokens.size - if acceptable_single_tokens.all?{ |st| tokens[i + 1].event != st } - start_token << t - end_type << :on_tstring_end - else - i += 1 - end - end - when :on_backtick - if t.state.allbits?(Ripper::EXPR_BEG) - start_token << t - end_type << :on_tstring_end - end - when :on_qwords_beg, :on_words_beg, :on_qsymbols_beg, :on_symbols_beg - start_token << t - end_type << :on_tstring_end - when :on_heredoc_beg - pending_heredocs << t - end - if pending_heredocs.any? && t.tok.include?("\n") - pending_heredocs.reverse_each do |t| - start_token << t - end_type << :on_heredoc_end - end - pending_heredocs = [] - end - i += 1 - end - pending_heredocs.first || start_token.last - end + LTYPE_TOKENS = %i[ + on_heredoc_beg on_tstring_beg + on_regexp_beg on_symbeg on_backtick + on_symbols_beg on_qsymbols_beg + on_words_beg on_qwords_beg + ] - def process_literal_type(tokens) - start_token = check_string_literal(tokens) - return nil if start_token == "" - - case start_token&.event - when :on_tstring_beg - case start_token&.tok - when ?" then ?" - when /^%.$/ then ?" - when /^%Q.$/ then ?" - when ?' then ?' - when /^%q.$/ then ?' + def ltype_from_open_tokens(opens) + start_token = opens.reverse_each.find do |tok| + LTYPE_TOKENS.include?(tok.event) end - when :on_regexp_beg then ?/ - when :on_symbeg then ?: - when :on_backtick then ?` - when :on_qwords_beg then ?] - when :on_words_beg then ?] - when :on_qsymbols_beg then ?] - when :on_symbols_beg then ?] - when :on_heredoc_beg - start_token&.tok =~ /<<[-~]?(['"`])\w+\1/ - $1 || ?" - else - nil - end - end + return nil unless start_token - def check_termination_in_prev_line(code) - tokens = self.class.ripper_lex_without_warning(code, context: @context) - past_first_newline = false - index = tokens.rindex do |t| - # traverse first token before last line - if past_first_newline - if t.tok.include?("\n") - true + case start_token&.event + when :on_tstring_beg + case start_token&.tok + when ?" then ?" + when /^%.$/ then ?" + when /^%Q.$/ then ?" + when ?' then ?' + when /^%q.$/ then ?' end - elsif t.tok.include?("\n") - past_first_newline = true - false + when :on_regexp_beg then ?/ + when :on_symbeg then ?: + when :on_backtick then ?` + when :on_qwords_beg then ?] + when :on_words_beg then ?] + when :on_qsymbols_beg then ?] + when :on_symbols_beg then ?] + when :on_heredoc_beg + start_token&.tok =~ /<<[-~]?(['"`])\w+\1/ + $1 || ?" else - false + nil end end - if index - first_token = nil - last_line_tokens = tokens[(index + 1)..(tokens.size - 1)] - last_line_tokens.each do |t| - unless [:on_sp, :on_ignored_sp, :on_comment].include?(t.event) - first_token = t - break - end - end - - if first_token.nil? - return false - elsif first_token && first_token.state == Ripper::EXPR_DOT - return false - else - tokens_without_last_line = tokens[0..index] - ltype = process_literal_type(tokens_without_last_line) - indent = process_nesting_level(tokens_without_last_line) - continue = process_continue(tokens_without_last_line) - code_block_open = check_code_block(tokens_without_last_line.map(&:tok).join(''), tokens_without_last_line) - if ltype or indent > 0 or continue or code_block_open - return false + def check_termination_in_prev_line(code, local_variables:) + tokens = self.class.ripper_lex_without_warning(code, local_variables: local_variables) + past_first_newline = false + index = tokens.rindex do |t| + # traverse first token before last line + if past_first_newline + if t.tok.include?("\n") + true + end + elsif t.tok.include?("\n") + past_first_newline = true + false else - return last_line_tokens.map(&:tok).join('') + false end end - end - false - end - - private - def heredoc_scope? - heredoc_tokens = @tokens.select { |t| [:on_heredoc_beg, :on_heredoc_end].include?(t.event) } - heredoc_tokens[-1]&.event == :on_heredoc_beg - end + if index + first_token = nil + last_line_tokens = tokens[(index + 1)..(tokens.size - 1)] + last_line_tokens.each do |t| + unless [:on_sp, :on_ignored_sp, :on_comment].include?(t.event) + first_token = t + break + end + end - def in_keyword_case_scope? - kw_tokens = @tokens.select { |t| t.event == :on_kw && ['case', 'for', 'end'].include?(t.tok) } - counter = 0 - kw_tokens.reverse.each do |t| - if t.tok == 'case' - return true if counter.zero? - counter += 1 - elsif t.tok == 'for' - counter += 1 - elsif t.tok == 'end' - counter -= 1 + if first_token && first_token.state != Ripper::EXPR_DOT + tokens_without_last_line = tokens[0..index] + code_without_last_line = tokens_without_last_line.map(&:tok).join + opens_without_last_line = NestingParser.open_tokens(tokens_without_last_line) + if code_terminated?(code_without_last_line, tokens_without_last_line, opens_without_last_line, local_variables: local_variables) + return last_line_tokens.map(&:tok).join + end + end end + false end - false end + # :startdoc: end -# :startdoc: + +RubyLex = IRB::RubyLex +Object.deprecate_constant(:RubyLex) diff --git a/lib/irb/ruby_logo.aa b/lib/irb/ruby_logo.aa index a34a3e2f28..61fe22c94a 100644 --- a/lib/irb/ruby_logo.aa +++ b/lib/irb/ruby_logo.aa @@ -1,3 +1,4 @@ +TYPE: LARGE -+smJYYN?mm- HB"BBYT TQg NggT @@ -35,3 +36,45 @@ m7 NW H N HSVO1z=?11- NgTH bB kH WBHWWHBHWmQgg&gggggNNN NNggggggNN +TYPE: ASCII + ,,,;;;;''''';;;'';, + ,,;'' ';;,;;; ', + ,,'' ;;'';'''';;;;;; + ,;' ;; ',, ; + ,;' ,;' ';, ; + ;' ,;; ',,,; + ,' ,;;,,,,,,,,,,,;;;; + ;' ;;';;;; ,;; + ;' ,;' ;; '',, ,;;; + ;; ,;' ; '';, ,; ;' +;; ,;;' ;; ;; ;; +;;, ,,;;' ; ;'; ;; +;';;,,,,;;;;;;;,,, ;; ,' ; ;; +; ;;''' ,;'; ''';,,, ; ,;' ;;;; +;;;;, ; '; ''';;;' ';;; +;'; ;, ;' '; ,;' ', ;;; +;;; ; ,; '; ,,' ',, ;; +;;; '; ;' ';,,'' ';,;; + '; ';,; ,,;''''''''';;;;;;,,;;; + ';,,;;,,;;;;;;;;;;'''''''''''''' +TYPE: UNICODE + ⣀⣤⣴⣾⣿⣿⣿⡛⠛⠛⠛⠛⣻⣿⠿⠛⠛⠶⣤⡀ + ⣀⣴⠾⠛⠉⠁ ⠙⣿⣶⣤⣶⣟⣉ ⠈⠻⣦ + ⣀⣴⠟⠋ ⢸⣿⠟⠻⣯⡙⠛⠛⠛⠶⠶⠶⢶⣽⣇ + ⣠⡾⠋⠁ ⣾⡿ ⠈⠛⢦⣄ ⣿ + ⣠⡾⠋ ⣰⣿⠃ ⠙⠷⣤⡀ ⣿ + ⢀⡾⠋ ⣰⣿⡏ ⠈⠻⣦⣄⢠⣿ + ⣰⠟⠁ ⣴⣿⣿⣁⣀⣠⣤⣤⣤⣤⣤⣤⣤⣴⠶⠿⣿⡏ + ⣼⠏ ⢀⣾⣿⠟⣿⠿⣯⣍⠁ ⣰⣿⡇ + ⢀⣼⠋ ⢀⣴⣿⠟⠁ ⢸⡇ ⠙⠻⢦⣄⡀ ⢠⡿⣿⡇ +⢀⣾⡏ ⢀⣴⣿⠟⠁ ⣿ ⠉⠻⢶⣄⡀⣰⡟ ⣿⠃ +⣾⣿⠁ ⣠⣶⡿⠋⠁ ⢹⡇ ⠈⣿⡏ ⢸⣿ +⣿⣿⡆ ⢀⣠⣴⣿⡿⠋ ⠈⣿ ⢀⡾⠋⣿ ⢸⣿ +⣿⠸⣿⣶⣤⣤⣤⣤⣶⣾⠿⠿⣿⣿⠶⣤⣤⣀⡀ ⢹⡇ ⣴⠟⠁ ⣿⡀⢸⣿ +⣿⢀⣿⣟⠛⠋⠉⠁ ⢰⡟⠹⣧ ⠈⠉⠛⠻⠶⢦⣤⣀⡀ ⠈⣿ ⣠⡾⠃ ⢸⡇⢸⡇ +⣿⣾⣿⢿⡄ ⣿⠁ ⠘⣧ ⠉⠙⠛⠷⣿⣿⡋ ⠸⣇⣸⡇ +⣿⠃⣿⠈⢿⡄ ⣸⠇ ⠘⣧ ⢀⣤⠾⠋⠈⠻⣦⡀ ⣿⣿⡇ +⣿⢸⡏ ⠈⣷⡀ ⢠⡿ ⠘⣧⡀ ⣠⡴⠟⠁ ⠈⠻⣦⣀ ⢿⣿⠁ +⢻⣾⡇ ⠘⣷ ⣼⠃ ⠘⣷⣠⣴⠟⠋ ⠙⢷⣄⢸⣿ + ⠻⣧⡀ ⠘⣧⣰⡏ ⢀⣠⣤⠶⠛⠉⠛⠛⠛⠛⠛⠛⠻⢶⣶⣶⣶⣶⣶⣤⣤⣽⣿⣿ + ⠈⠛⠷⢦⣤⣽⣿⣥⣤⣶⣶⡿⠿⠿⠶⠶⠶⠶⠾⠛⠛⠛⠛⠛⠛⠛⠋⠉⠉⠉⠉⠉⠉⠁ diff --git a/lib/irb/source_finder.rb b/lib/irb/source_finder.rb new file mode 100644 index 0000000000..5d7d729d19 --- /dev/null +++ b/lib/irb/source_finder.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +require_relative "ruby-lex" + +module IRB + class SourceFinder + class EvaluationError < StandardError; end + + class Source + attr_reader :file, :line + def initialize(file, line, ast_source = nil) + @file = file + @line = line + @ast_source = ast_source + end + + def file_exist? + File.exist?(@file) + end + + def binary_file? + # If the line is zero, it means that the target's source is probably in a binary file. + @line.zero? + end + + def file_content + @file_content ||= File.read(@file) + end + + def colorized_content + if !binary_file? && file_exist? + end_line = find_end + # To correctly colorize, we need to colorize full content and extract the relevant lines. + colored = IRB::Color.colorize_code(file_content) + colored.lines[@line - 1...end_line].join + elsif @ast_source + IRB::Color.colorize_code(@ast_source) + end + end + + private + + def find_end + lex = RubyLex.new + code = file_content + lines = code.lines[(@line - 1)..-1] + tokens = RubyLex.ripper_lex_without_warning(lines.join) + prev_tokens = [] + + # chunk with line number + tokens.chunk { |tok| tok.pos[0] }.each do |lnum, chunk| + code = lines[0..lnum].join + prev_tokens.concat chunk + continue = lex.should_continue?(prev_tokens) + syntax = lex.check_code_syntax(code, local_variables: []) + if !continue && syntax == :valid + return @line + lnum + end + end + @line + end + end + + private_constant :Source + + def initialize(irb_context) + @irb_context = irb_context + end + + def find_source(signature, super_level = 0) + case signature + when /\A(::)?[A-Z]\w*(::[A-Z]\w*)*\z/ # ConstName, ::ConstName, ConstPath::ConstName + eval_receiver_or_owner(signature) # trigger autoload + *parts, name = signature.split('::', -1) + base = + if parts.empty? # ConstName + find_const_owner(name) + elsif parts == [''] # ::ConstName + Object + else # ConstPath::ConstName + eval_receiver_or_owner(parts.join('::')) + end + file, line = base.const_source_location(name) + when /\A(?<owner>[A-Z]\w*(::[A-Z]\w*)*)#(?<method>[^ :.]+)\z/ # Class#method + owner = eval_receiver_or_owner(Regexp.last_match[:owner]) + method = Regexp.last_match[:method] + return unless owner.respond_to?(:instance_method) + method = method_target(owner, super_level, method, "owner") + file, line = method&.source_location + when /\A((?<receiver>.+)(\.|::))?(?<method>[^ :.]+)\z/ # method, receiver.method, receiver::method + receiver = eval_receiver_or_owner(Regexp.last_match[:receiver] || 'self') + method = Regexp.last_match[:method] + return unless receiver.respond_to?(method, true) + method = method_target(receiver, super_level, method, "receiver") + file, line = method&.source_location + end + return unless file && line + + if File.exist?(file) + Source.new(file, line) + elsif method + # Method defined with eval, probably in IRB session + source = RubyVM::AbstractSyntaxTree.of(method)&.source rescue nil + Source.new(file, line, source) + end + rescue EvaluationError + nil + end + + private + + def method_target(owner_receiver, super_level, method, type) + case type + when "owner" + target_method = owner_receiver.instance_method(method) + when "receiver" + target_method = owner_receiver.method(method) + end + super_level.times do |s| + target_method = target_method.super_method if target_method + end + target_method + rescue NameError + nil + end + + def eval_receiver_or_owner(code) + context_binding = @irb_context.workspace.binding + eval(code, context_binding) + rescue NameError + raise EvaluationError + end + + def find_const_owner(name) + module_nesting = @irb_context.workspace.binding.eval('::Module.nesting') + module_nesting.find { |mod| mod.const_defined?(name, false) } || module_nesting.find { |mod| mod.const_defined?(name) } || Object + end + end +end diff --git a/lib/irb/src_encoding.rb b/lib/irb/src_encoding.rb deleted file mode 100644 index 99aea2b43e..0000000000 --- a/lib/irb/src_encoding.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: false -# DO NOT WRITE ANY MAGIC COMMENT HERE. -module IRB - def self.default_src_encoding - return __ENCODING__ - end -end diff --git a/lib/irb/statement.rb b/lib/irb/statement.rb new file mode 100644 index 0000000000..a3391c12a3 --- /dev/null +++ b/lib/irb/statement.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module IRB + class Statement + attr_reader :code + + def is_assignment? + raise NotImplementedError + end + + def suppresses_echo? + raise NotImplementedError + end + + def should_be_handled_by_debugger? + raise NotImplementedError + end + + class EmptyInput < Statement + def is_assignment? + false + end + + def suppresses_echo? + true + end + + # Debugger takes empty input to repeat the last command + def should_be_handled_by_debugger? + true + end + + def code + "" + end + end + + class Expression < Statement + def initialize(code, is_assignment) + @code = code + @is_assignment = is_assignment + end + + def suppresses_echo? + @code.match?(/;\s*\z/) + end + + def should_be_handled_by_debugger? + true + end + + def is_assignment? + @is_assignment + end + end + + class Command < Statement + attr_reader :command_class, :arg + + def initialize(original_code, command_class, arg) + @code = original_code + @command_class = command_class + @arg = arg + end + + def is_assignment? + false + end + + def suppresses_echo? + false + end + + def should_be_handled_by_debugger? + require_relative 'command/debug' + IRB::Command::DebugCommand > @command_class + end + end + end +end diff --git a/lib/irb/version.rb b/lib/irb/version.rb index a1e15815a2..c41917329c 100644 --- a/lib/irb/version.rb +++ b/lib/irb/version.rb @@ -1,11 +1,11 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/version.rb - irb version definition file # by Keiju ISHITSUKA(keiju@ishitsuka.com) # module IRB # :nodoc: - VERSION = "1.7.0" + VERSION = "1.13.1" @RELEASE_VERSION = VERSION - @LAST_UPDATE_DATE = "2023-06-03" + @LAST_UPDATE_DATE = "2024-05-05" end diff --git a/lib/irb/workspace.rb b/lib/irb/workspace.rb index d6fa67053d..d24d1cc38d 100644 --- a/lib/irb/workspace.rb +++ b/lib/irb/workspace.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/workspace-binding.rb - # by Keiju ISHITSUKA(keiju@ruby-lang.org) @@ -6,6 +6,8 @@ require "delegate" +require_relative "helper_method" + IRB::TOPLEVEL_BINDING = binding module IRB # :nodoc: class WorkSpace @@ -90,11 +92,11 @@ EOF IRB.conf[:__MAIN__] = @main @main.singleton_class.class_eval do private - define_method(:exit) do |*a, &b| - # Do nothing, will be overridden - end define_method(:binding, Kernel.instance_method(:binding)) define_method(:local_variables, Kernel.instance_method(:local_variables)) + # Define empty method to avoid delegator warning, will be overridden. + define_method(:exit) {|*a, &b| } + define_method(:exit!) {|*a, &b| } end @binding = eval("IRB.conf[:__MAIN__].instance_eval('binding', __FILE__, __LINE__)", @binding, *@binding.source_location) end @@ -108,6 +110,12 @@ EOF # <code>IRB.conf[:__MAIN__]</code> attr_reader :main + def load_helper_methods_to_main + ancestors = class<<main;ancestors;end + main.extend ExtendCommandBundle if !ancestors.include?(ExtendCommandBundle) + main.extend HelpersContainer if !ancestors.include?(HelpersContainer) + end + # Evaluate the given +statements+ within the context of this workspace. def evaluate(statements, file = __FILE__, line = __LINE__) eval(statements, @binding, file, line) @@ -166,4 +174,16 @@ EOF "\nFrom: #{file} @ line #{pos + 1} :\n\n#{body}#{Color.clear}\n" end end + + module HelpersContainer + def self.install_helper_methods + HelperMethod.helper_methods.each do |name, helper_method_class| + define_method name do |*args, **opts, &block| + helper_method_class.instance.execute(*args, **opts, &block) + end unless method_defined?(name) + end + end + + install_helper_methods + end end diff --git a/lib/irb/ws-for-case-2.rb b/lib/irb/ws-for-case-2.rb index a0f617e4ed..03f42d73d9 100644 --- a/lib/irb/ws-for-case-2.rb +++ b/lib/irb/ws-for-case-2.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # irb/ws-for-case-2.rb - # by Keiju ISHITSUKA(keiju@ruby-lang.org) diff --git a/lib/irb/xmp.rb b/lib/irb/xmp.rb index 94c700b484..b1bc53283e 100644 --- a/lib/irb/xmp.rb +++ b/lib/irb/xmp.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # xmp.rb - irb version of gotoken xmp # by Keiju ISHITSUKA(Nippon Rational Inc.) @@ -44,8 +44,8 @@ class XMP # The top-level binding or, optional +bind+ parameter will be used when # creating the workspace. See WorkSpace.new for more information. # - # This uses the +:XMP+ prompt mode, see IRB@Customizing+the+IRB+Prompt for - # full detail. + # This uses the +:XMP+ prompt mode. + # See {Custom Prompts}[rdoc-ref:IRB@Custom+Prompts] for more information. def initialize(bind = nil) IRB.init_config(nil) diff --git a/lib/logger.rb b/lib/logger.rb index acc65ac8bb..4099955ef2 100644 --- a/lib/logger.rb +++ b/lib/logger.rb @@ -265,8 +265,7 @@ require_relative 'logger/errors' # logger.error! # => 3 # logger.fatal! # => 4 # -# You can retrieve the log level with method -# {level}[Logger.html#attribute-i-level]: +# You can retrieve the log level with method #level. # # logger.level = Logger::ERROR # logger.level # => 3 diff --git a/lib/logger/logger.gemspec b/lib/logger/logger.gemspec index d12db625d9..5e8232e4ab 100644 --- a/lib/logger/logger.gemspec +++ b/lib/logger/logger.gemspec @@ -18,9 +18,5 @@ Gem::Specification.new do |spec| spec.files = Dir.glob("lib/**/*.rb") + ["logger.gemspec"] spec.require_paths = ["lib"] - spec.required_ruby_version = ">= 2.3.0" - - spec.add_development_dependency "bundler", ">= 0" - spec.add_development_dependency "rake", ">= 12.3.3" - spec.add_development_dependency "test-unit" + spec.required_ruby_version = ">= 2.5.0" end diff --git a/lib/logger/version.rb b/lib/logger/version.rb index f85c72eed3..202b6e4fba 100644 --- a/lib/logger/version.rb +++ b/lib/logger/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true class Logger - VERSION = "1.5.3" + VERSION = "1.6.0" end diff --git a/lib/mkmf.rb b/lib/mkmf.rb index 20389ed705..73459ffeb9 100644 --- a/lib/mkmf.rb +++ b/lib/mkmf.rb @@ -44,6 +44,23 @@ end # correctly compile and link the C extension to Ruby and a third-party # library. module MakeMakefile + + target_rbconfig = nil + ARGV.delete_if do |arg| + opt = arg.delete_prefix("--target-rbconfig=") + unless opt == arg + target_rbconfig = opt + end + end + if target_rbconfig + # Load the RbConfig for the target platform into this module. + # Cross-compiling needs the same version of Ruby. + Kernel.load target_rbconfig, self + else + # The RbConfig for the target platform where the built extension runs. + RbConfig = ::RbConfig + end + #### defer until this module become global-state free. # def self.extended(obj) # obj.init_mkmf @@ -59,6 +76,9 @@ module MakeMakefile # The makefile configuration using the defaults from when Ruby was built. CONFIG = RbConfig::MAKEFILE_CONFIG + + ## + # The saved original value of +LIB+ environment variable ORIG_LIBPATH = ENV['LIB'] ## @@ -245,12 +265,16 @@ MESSAGE CSRCFLAG = CONFIG['CSRCFLAG'] CPPOUTFILE = config_string('CPPOUTFILE') {|str| str.sub(/\bconftest\b/, CONFTEST)} + # :startdoc: + + # Removes _files_. def rm_f(*files) opt = (Hash === files.last ? [files.pop] : []) FileUtils.rm_f(Dir[*files.flatten], *opt) end module_function :rm_f + # Removes _files_ recursively. def rm_rf(*files) opt = (Hash === files.last ? [files.pop] : []) FileUtils.rm_rf(Dir[*files.flatten], *opt) @@ -265,6 +289,8 @@ MESSAGE t if times.all? {|n| n <= t} end + # :stopdoc: + def split_libs(*strs) sep = $mswin ? /\s+/ : /\s+(?=-|\z)/ strs.flat_map {|s| s.lstrip.split(sep)} @@ -390,6 +416,11 @@ MESSAGE env, *commands = commands if Hash === commands.first envs.merge!(env) if env end + + # disable ASAN leak reporting - conftest programs almost always don't bother + # to free their memory. + envs['ASAN_OPTIONS'] = "detect_leaks=0" unless ENV.key?('ASAN_OPTIONS') + return envs, expand[commands] end @@ -397,11 +428,19 @@ MESSAGE envs.map {|e, v| "#{e}=#{v.quote}"} end - def xsystem command, opts = nil + # :startdoc: + + # call-seq: + # xsystem(command, werror: false) -> true or false + # + # Executes _command_ with expanding variables, and returns the exit + # status like as Kernel#system. If _werror_ is true and the error + # output is not empty, returns +false+. The output will logged. + def xsystem(command, werror: false) env, command = expand_command(command) Logging::open do puts [env_quote(env), command.quote].join(' ') - if opts and opts[:werror] + if werror result = nil Logging.postpone do |log| output = IO.popen(env, command, &:read) @@ -415,6 +454,7 @@ MESSAGE end end + # Executes _command_ similarly to xsystem, but yields opened pipe. def xpopen command, *mode, &block env, commands = expand_command(command) command = [env_quote(env), command].join(' ') @@ -429,6 +469,7 @@ MESSAGE end end + # Logs _src_ def log_src(src, heading="checked program was") src = src.split(/^/) fmt = "%#{src.size.to_s.size}d: %s" @@ -443,10 +484,15 @@ EOM EOM end + # Returns the language-dependent source file name for configuration + # checks. def conftest_source CONFTEST_C end + # Creats temporary source file from +COMMON_HEADERS+ and _src_. + # Yields the created source string and uses the returned string as + # the source code, if the block is given. def create_tmpsrc(src) src = "#{COMMON_HEADERS}\n#{src}" src = yield(src) if block_given? @@ -467,6 +513,8 @@ EOM src end + # :stopdoc: + def have_devel? unless defined? $have_devel $have_devel = true @@ -475,7 +523,7 @@ EOM $have_devel end - def try_do(src, command, *opts, &b) + def try_do(src, command, **opts, &b) unless have_devel? raise <<MSG The compiler failed to generate an executable file. @@ -484,7 +532,7 @@ MSG end begin src = create_tmpsrc(src, &b) - xsystem(command, *opts) + xsystem(command, **opts) ensure log_src(src) end @@ -545,18 +593,17 @@ MSG }.join end + def werror_flag(opt = nil) + config_string("WERRORFLAG") {|flag| opt = opt && !opt.empty? ? "#{opt} #{flag}" : flag} + opt + end + def with_werror(opt, opts = nil) - if opts - if opts[:werror] and config_string("WERRORFLAG") {|flag| opt = opt ? "#{opt} #{flag}" : flag} - (opts = opts.dup).delete(:werror) - end - yield(opt, opts) - else - yield(opt) - end + opt = werror_flag(opt) if opts and (opts = opts.dup).delete(:werror) + yield(opt, opts) end - def try_link0(src, opt="", *opts, &b) # :nodoc: + def try_link0(src, opt = "", **opts, &b) # :nodoc: exe = CONFTEST+$EXEEXT cmd = link_command("", opt) if $universal @@ -564,13 +611,13 @@ MSG Dir.mktmpdir("mkmf_", oldtmpdir = ENV["TMPDIR"]) do |tmpdir| begin ENV["TMPDIR"] = tmpdir - try_do(src, cmd, *opts, &b) + try_do(src, cmd, **opts, &b) ensure ENV["TMPDIR"] = oldtmpdir end end else - try_do(src, cmd, *opts, &b) + try_do(src, cmd, **opts, &b) end and File.executable?(exe) or return nil exe ensure @@ -579,31 +626,32 @@ MSG # Returns whether or not the +src+ can be compiled as a C source and linked # with its depending libraries successfully. +opt+ is passed to the linker - # as options. Note that +$CFLAGS+ and +$LDFLAGS+ are also passed to the - # linker. + # as options. Note that <tt>$CFLAGS</tt> and <tt>$LDFLAGS</tt> are also + # passed to the linker. # # If a block given, it is called with the source before compilation. You can # modify the source in the block. # # [+src+] a String which contains a C source # [+opt+] a String which contains linker options - def try_link(src, opt="", *opts, &b) - exe = try_link0(src, opt, *opts, &b) or return false + def try_link(src, opt = "", **opts, &b) + exe = try_link0(src, opt, **opts, &b) or return false MakeMakefile.rm_f exe true end # Returns whether or not the +src+ can be compiled as a C source. +opt+ is - # passed to the C compiler as options. Note that +$CFLAGS+ is also passed to - # the compiler. + # passed to the C compiler as options. Note that <tt>$CFLAGS</tt> is also + # passed to the compiler. # # If a block given, it is called with the source before compilation. You can # modify the source in the block. # # [+src+] a String which contains a C source # [+opt+] a String which contains compiler options - def try_compile(src, opt="", *opts, &b) - with_werror(opt, *opts) {|_opt, *| try_do(src, cc_command(_opt), *opts, &b)} and + def try_compile(src, opt = "", werror: nil, **opts, &b) + opt = werror_flag(opt) if werror + try_do(src, cc_command(opt), werror: werror, **opts, &b) and File.file?("#{CONFTEST}.#{$OBJEXT}") ensure MakeMakefile.rm_f "#{CONFTEST}*" @@ -611,15 +659,15 @@ MSG # Returns whether or not the +src+ can be preprocessed with the C # preprocessor. +opt+ is passed to the preprocessor as options. Note that - # +$CFLAGS+ is also passed to the preprocessor. + # <tt>$CFLAGS</tt> is also passed to the preprocessor. # # If a block given, it is called with the source before preprocessing. You # can modify the source in the block. # # [+src+] a String which contains a C source # [+opt+] a String which contains preprocessor options - def try_cpp(src, opt="", *opts, &b) - try_do(src, cpp_command(CPPOUTFILE, opt), *opts, &b) and + def try_cpp(src, opt = "", **opts, &b) + try_do(src, cpp_command(CPPOUTFILE, opt), **opts, &b) and File.file?("#{CONFTEST}.i") ensure MakeMakefile.rm_f "#{CONFTEST}*" @@ -636,6 +684,14 @@ MSG end end + # :startdoc: + + # Sets <tt>$CPPFLAGS</tt> to _flags_ and yields. If the block returns a + # falsy value, <tt>$CPPFLAGS</tt> is reset to its previous value, remains + # set to _flags_ otherwise. + # + # [+flags+] a C preprocessor flag as a +String+ + # def with_cppflags(flags) cppflags = $CPPFLAGS $CPPFLAGS = flags.dup @@ -644,20 +700,29 @@ MSG $CPPFLAGS = cppflags unless ret end - def try_cppflags(flags, opts = {}) - try_header(MAIN_DOES_NOTHING, flags, {:werror => true}.update(opts)) + # :nodoc: + def try_cppflags(flags, werror: true, **opts) + try_header(MAIN_DOES_NOTHING, flags, werror: werror, **opts) end - def append_cppflags(flags, *opts) + # Check whether each given C preprocessor flag is acceptable and append it + # to <tt>$CPPFLAGS</tt> if so. + # + # [+flags+] a C preprocessor flag as a +String+ or an +Array+ of them + # + def append_cppflags(flags, **opts) Array(flags).each do |flag| if checking_for("whether #{flag} is accepted as CPPFLAGS") { - try_cppflags(flag, *opts) + try_cppflags(flag, **opts) } $CPPFLAGS << " " << flag end end end + # Sets <tt>$CFLAGS</tt> to _flags_ and yields. If the block returns a falsy + # value, <tt>$CFLAGS</tt> is reset to its previous value, remains set to + # _flags_ otherwise. def with_cflags(flags) cflags = $CFLAGS $CFLAGS = flags.dup @@ -666,10 +731,14 @@ MSG $CFLAGS = cflags unless ret end - def try_cflags(flags, opts = {}) - try_compile(MAIN_DOES_NOTHING, flags, {:werror => true}.update(opts)) + # :nodoc: + def try_cflags(flags, werror: true, **opts) + try_compile(MAIN_DOES_NOTHING, flags, werror: werror, **opts) end + # Sets <tt>$LDFLAGS</tt> to _flags_ and yields. If the block returns a + # falsy value, <tt>$LDFLAGS</tt> is reset to its previous value, remains set + # to _flags_ otherwise. def with_ldflags(flags) ldflags = $LDFLAGS $LDFLAGS = flags.dup @@ -678,21 +747,30 @@ MSG $LDFLAGS = ldflags unless ret end - def try_ldflags(flags, opts = {}) - opts = {:werror => true}.update(opts) if $mswin - try_link(MAIN_DOES_NOTHING, flags, opts) + # :nodoc: + def try_ldflags(flags, werror: $mswin, **opts) + try_link(MAIN_DOES_NOTHING, flags, werror: werror, **opts) end - def append_ldflags(flags, *opts) + # :startdoc: + + # Check whether each given linker flag is acceptable and append it to + # <tt>$LDFLAGS</tt> if so. + # + # [+flags+] a linker flag as a +String+ or an +Array+ of them + # + def append_ldflags(flags, **opts) Array(flags).each do |flag| if checking_for("whether #{flag} is accepted as LDFLAGS") { - try_ldflags(flag, *opts) + try_ldflags(flag, **opts) } $LDFLAGS << " " << flag end end end + # :stopdoc: + def try_static_assert(expr, headers = nil, opt = "", &b) headers = cpp_include(headers) try_compile(<<SRC, opt, &b) @@ -828,6 +906,8 @@ int t(void) { const volatile void *volatile p; p = &(&#{var})[0]; return !p; } SRC end + # :startdoc: + # Returns whether or not the +src+ can be preprocessed with the C # preprocessor and matches with +pat+. # @@ -866,6 +946,8 @@ SRC log_src(src) end + # :stopdoc: + # This is used internally by the have_macro? method. def macro_defined?(macro, src, opt = "", &b) src = src.sub(/[^\n]\z/, "\\&\n") @@ -885,8 +967,8 @@ SRC # * the linked file can be invoked as an executable # * and the executable exits successfully # - # +opt+ is passed to the linker as options. Note that +$CFLAGS+ and - # +$LDFLAGS+ are also passed to the linker. + # +opt+ is passed to the linker as options. Note that <tt>$CFLAGS</tt> and + # <tt>$LDFLAGS</tt> are also passed to the linker. # # If a block given, it is called with the source before compilation. You can # modify the source in the block. @@ -958,6 +1040,10 @@ SRC format(LIBARG, lib) + " " + libs end + # Prints messages to $stdout, if verbose mode. + # + # Internal use only. + # def message(*s) unless Logging.quiet and not $VERBOSE printf(*s) @@ -989,6 +1075,10 @@ SRC r end + # Build a message for checking. + # + # Internal use only. + # def checking_message(target, place = nil, opt = nil) [["in", place], ["with", opt]].inject("#{target}") do |msg, (pre, noun)| if noun @@ -1013,10 +1103,10 @@ SRC # # [+flags+] a C compiler flag as a +String+ or an +Array+ of them # - def append_cflags(flags, *opts) + def append_cflags(flags, **opts) Array(flags).each do |flag| if checking_for("whether #{flag} is accepted as CFLAGS") { - try_cflags(flag, *opts) + try_cflags(flag, **opts) } $CFLAGS << " " << flag end @@ -1250,6 +1340,7 @@ SRC end end + # :nodoc: # Returns whether or not the static type +type+ is defined. # # See also +have_type+ @@ -1307,6 +1398,7 @@ SRC end end + # :nodoc: # Returns whether or not the constant +const+ is defined. # # See also +have_const+ @@ -1466,7 +1558,7 @@ SRC u = "unsigned " if signed > 0 prelude << "extern rbcv_typedef_ foo();" compat = UNIVERSAL_INTS.find {|t| - try_compile([prelude, "extern #{u}#{t} foo();"].join("\n"), opts, :werror=>true, &b) + try_compile([prelude, "extern #{u}#{t} foo();"].join("\n"), opts, werror: true, &b) } end if compat @@ -1490,7 +1582,7 @@ SRC # Used internally by the what_type? method to determine if +type+ is a scalar # pointer. def scalar_ptr_type?(type, member = nil, headers = nil, &b) - try_compile(<<"SRC", &b) # pointer + try_compile(<<"SRC", &b) #{cpp_include(headers)} /*top*/ volatile #{type} conftestval; @@ -1503,7 +1595,7 @@ SRC # Used internally by the what_type? method to determine if +type+ is a scalar # pointer. def scalar_type?(type, member = nil, headers = nil, &b) - try_compile(<<"SRC", &b) # pointer + try_compile(<<"SRC", &b) #{cpp_include(headers)} /*top*/ volatile #{type} conftestval; @@ -1525,6 +1617,10 @@ SRC end end + # :startdoc: + + # Returns a string represents the type of _type_, or _member_ of + # _type_ if _member_ is not +nil+. def what_type?(type, member = nil, headers = nil, &b) m = "#{type}" var = val = "*rbcv_var_" @@ -1584,6 +1680,8 @@ SRC end end + # :nodoc: + # # This method is used internally by the find_executable method. # # Internal use only. @@ -1622,8 +1720,6 @@ SRC nil end - # :startdoc: - # Searches for the executable +bin+ on +path+. The default path is your # +PATH+ environment variable. If that isn't defined, it will resort to # searching /usr/local/bin, /usr/ucb, /usr/bin and /bin. @@ -1792,7 +1888,8 @@ SRC # application. # def dir_config(target, idefault=nil, ldefault=nil) - if conf = $config_dirs[target] + key = [target, idefault, ldefault].compact.join("\0") + if conf = $config_dirs[key] return conf end @@ -1802,9 +1899,13 @@ SRC end idir = with_config(target + "-include", idefault) - $arg_config.last[1] ||= "${#{target}-dir}/include" + if conf = $arg_config.assoc("--with-#{target}-include") + conf[1] ||= "${#{target}-dir}/include" + end ldir = with_config(target + "-lib", ldefault) - $arg_config.last[1] ||= "${#{target}-dir}/#{_libdir_basename}" + if conf = $arg_config.assoc("--with-#{target}-lib") + conf[1] ||= "${#{target}-dir}/#{_libdir_basename}" + end idirs = idir ? Array === idir ? idir.dup : idir.split(File::PATH_SEPARATOR) : [] if defaults @@ -1826,7 +1927,7 @@ SRC end $LIBPATH = ldirs | $LIBPATH - $config_dirs[target] = [idir, ldir] + $config_dirs[key] = [idir, ldir] end # Returns compile/link information about an installed library in a tuple of <code>[cflags, @@ -2064,7 +2165,9 @@ ARCH_FLAG = #{$ARCH_FLAG} DLDFLAGS = $(ldflags) $(dldflags) $(ARCH_FLAG) LDSHARED = #{CONFIG['LDSHARED']} LDSHAREDXX = #{config_string('LDSHAREDXX') || '$(LDSHARED)'} +POSTLINK = #{config_string('POSTLINK', RbConfig::CONFIG)} AR = #{CONFIG['AR']} +LD = #{CONFIG['LD']} EXEEXT = #{CONFIG['EXEEXT']} } @@ -2372,7 +2475,7 @@ TARGET_ENTRY = #{EXPORT_PREFIX || ''}Init_$(TARGET_NAME) DLLIB = #{dllib} EXTSTATIC = #{$static || ""} STATIC_LIB = #{staticlib unless $static.nil?} -#{!$extout && defined?($installed_list) ? "INSTALLED_LIST = #{$installed_list}\n" : ""} +#{!$extout && defined?($installed_list) ? %[INSTALLED_LIST = #{$installed_list}\n] : ""} TIMESTAMP_DIR = #{$extout && $extmk ? '$(extout)/.timestamp' : '.'} " #" # TODO: fixme @@ -2399,7 +2502,7 @@ TARGET_SO_DIR_TIMESTAMP = #{timestamp_file(sodir, target_prefix)} mfile.puts(conf) mfile.print " all: #{$extout ? "install" : target ? "$(DLLIB)" : "Makefile"} -static: #{$extmk && !$static ? "all" : "$(STATIC_LIB)#{$extout ? " install-rb" : ""}"} +static: #{$extmk && !$static ? "all" : %[$(STATIC_LIB)#{$extout ? " install-rb" : ""}]} .PHONY: all install static install-so install-rb .PHONY: clean clean-so clean-static clean-rb " #" @@ -2430,6 +2533,7 @@ static: #{$extmk && !$static ? "all" : "$(STATIC_LIB)#{$extout ? " install-rb" : mfile.puts dest mfile.print "clean-so::\n" mfile.print "\t-$(Q)$(RM) #{fseprepl[dest]} #{fseprepl[stamp]}\n" + mfile.print "\t-$(Q)$(RM_RF) #{fseprepl['$(CLEANLIBS)']}\n" mfile.print "\t-$(Q)$(RMDIRS) #{fseprepl[dir]}#{$ignore_error}\n" else mfile.print "#{f} #{stamp}\n" @@ -2699,7 +2803,7 @@ MESSAGE when $mswin $nmake = ?m if /nmake/i =~ make end - $ignore_error = $nmake ? '' : ' 2> /dev/null || true' + $ignore_error = " 2> #{File::NULL} || #{$mswin ? 'exit /b0' : 'true'}" RbConfig::CONFIG["srcdir"] = CONFIG["srcdir"] = $srcdir = arg_config("--srcdir", File.dirname($0)) @@ -2722,6 +2826,9 @@ MESSAGE split = Shellwords.method(:shellwords).to_proc + ## + # The prefix added to exported symbols automatically + EXPORT_PREFIX = config_string('EXPORT_PREFIX') {|s| s.strip} hdr = ['#include "ruby.h"' "\n"] @@ -2751,6 +2858,10 @@ MESSAGE # make compile rules COMPILE_RULES = config_string('COMPILE_RULES', &split) || %w[.%s.%s:] + + ## + # Substitution in rules for NMake + RULE_SUBST = config_string('RULE_SUBST') ## @@ -2796,6 +2907,10 @@ MESSAGE # Argument which will add a library path to the linker LIBPATHFLAG = config_string('LIBPATHFLAG') || ' -L%s' + + ## + # Argument which will add a runtime library path to the linker + RPATHFLAG = config_string('RPATHFLAG') || '' ## @@ -2807,6 +2922,10 @@ MESSAGE # A C main function which does no work MAIN_DOES_NOTHING = config_string('MAIN_DOES_NOTHING') || "int main(int argc, char **argv)\n{\n return !!argv[argc];\n}" + + ## + # The type names for convertible_int + UNIVERSAL_INTS = config_string('UNIVERSAL_INTS') {|s| Shellwords.shellwords(s)} || %w[int short long long\ long] @@ -2837,18 +2956,32 @@ realclean: distclean @lang = Hash.new(self) + ## + # Retrieves the module for _name_ language. def self.[](name) @lang.fetch(name) end + ## + # Defines the module for _name_ language. def self.[]=(name, mod) @lang[name] = mod end - self["C++"] = Module.new do + ## + # The language that this module is for + LANGUAGE = -"C" + + self[self::LANGUAGE] = self + + cxx = Module.new do + # Module for C++ + include MakeMakefile extend self + # :stopdoc: + CONFTEST_CXX = "#{CONFTEST}.#{config_string('CXX_EXT') || CXX_EXT[0]}" TRY_LINK_CXX = config_string('TRY_LINK_CXX') || @@ -2878,7 +3011,12 @@ realclean: distclean conf = link_config(ldflags, *opts) RbConfig::expand(TRY_LINK_CXX.dup, conf) end + + # :startdoc: end + + cxx::LANGUAGE = -"C++" + self[cxx::LANGUAGE] = cxx end # MakeMakefile::Global = # diff --git a/lib/mutex_m.gemspec b/lib/mutex_m.gemspec deleted file mode 100644 index ebbda2606c..0000000000 --- a/lib/mutex_m.gemspec +++ /dev/null @@ -1,28 +0,0 @@ -begin - require_relative "lib/mutex_m" -rescue LoadError - # for Ruby core repository - require_relative "mutex_m" -end - -Gem::Specification.new do |spec| - spec.name = "mutex_m" - spec.version = Mutex_m::VERSION - spec.authors = ["Keiju ISHITSUKA"] - spec.email = ["keiju@ruby-lang.org"] - - spec.summary = %q{Mixin to extend objects to be handled like a Mutex.} - spec.description = %q{Mixin to extend objects to be handled like a Mutex.} - spec.homepage = "https://github.com/ruby/mutex_m" - spec.licenses = ["Ruby", "BSD-2-Clause"] - - spec.files = ["Gemfile", "LICENSE.txt", "README.md", "Rakefile", "lib/mutex_m.rb", "mutex_m.gemspec"] - spec.bindir = "exe" - spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } - spec.require_paths = ["lib"] - spec.required_ruby_version = '>= 2.5' - - spec.add_development_dependency "bundler" - spec.add_development_dependency "rake" - spec.add_development_dependency "test-unit" -end diff --git a/lib/mutex_m.rb b/lib/mutex_m.rb deleted file mode 100644 index 4c888d6a17..0000000000 --- a/lib/mutex_m.rb +++ /dev/null @@ -1,116 +0,0 @@ -# frozen_string_literal: false -# -# mutex_m.rb - -# $Release Version: 3.0$ -# $Revision: 1.7 $ -# Original from mutex.rb -# by Keiju ISHITSUKA(keiju@ishitsuka.com) -# modified by matz -# patched by akira yamada -# -# -- - -# = mutex_m.rb -# -# When 'mutex_m' is required, any object that extends or includes Mutex_m will -# be treated like a Mutex. -# -# Start by requiring the standard library Mutex_m: -# -# require "mutex_m.rb" -# -# From here you can extend an object with Mutex instance methods: -# -# obj = Object.new -# obj.extend Mutex_m -# -# Or mixin Mutex_m into your module to your class inherit Mutex instance -# methods --- remember to call super() in your class initialize method. -# -# class Foo -# include Mutex_m -# def initialize -# # ... -# super() -# end -# # ... -# end -# obj = Foo.new -# # this obj can be handled like Mutex -# -module Mutex_m - - VERSION = "0.1.2" - Ractor.make_shareable(VERSION) if defined?(Ractor) - - def Mutex_m.define_aliases(cl) # :nodoc: - cl.alias_method(:locked?, :mu_locked?) - cl.alias_method(:lock, :mu_lock) - cl.alias_method(:unlock, :mu_unlock) - cl.alias_method(:try_lock, :mu_try_lock) - cl.alias_method(:synchronize, :mu_synchronize) - end - - def Mutex_m.append_features(cl) # :nodoc: - super - define_aliases(cl) unless cl.instance_of?(Module) - end - - def Mutex_m.extend_object(obj) # :nodoc: - super - obj.mu_extended - end - - def mu_extended # :nodoc: - unless (defined? locked? and - defined? lock and - defined? unlock and - defined? try_lock and - defined? synchronize) - Mutex_m.define_aliases(singleton_class) - end - mu_initialize - end - - # See Thread::Mutex#synchronize - def mu_synchronize(&block) - @_mutex.synchronize(&block) - end - - # See Thread::Mutex#locked? - def mu_locked? - @_mutex.locked? - end - - # See Thread::Mutex#try_lock - def mu_try_lock - @_mutex.try_lock - end - - # See Thread::Mutex#lock - def mu_lock - @_mutex.lock - end - - # See Thread::Mutex#unlock - def mu_unlock - @_mutex.unlock - end - - # See Thread::Mutex#sleep - def sleep(timeout = nil) - @_mutex.sleep(timeout) - end - - private - - def mu_initialize # :nodoc: - @_mutex = Thread::Mutex.new - end - - def initialize(*args) # :nodoc: - mu_initialize - super - end - ruby2_keywords(:initialize) if respond_to?(:ruby2_keywords, true) -end diff --git a/lib/net/http.rb b/lib/net/http.rb index 89277ca49d..6b78c264af 100644 --- a/lib/net/http.rb +++ b/lib/net/http.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # = net/http.rb # @@ -456,6 +456,10 @@ module Net #:nodoc: # # == What's Here # + # First, what's elsewhere. Class Net::HTTP: + # + # - Inherits from {class Object}[rdoc-ref:Object@What-27s+Here]. + # # This is a categorized summary of methods and attributes. # # === \Net::HTTP Objects @@ -722,8 +726,7 @@ module Net #:nodoc: class HTTP < Protocol # :stopdoc: - VERSION = "0.3.2" - Revision = %q$Revision$.split[1] + VERSION = "0.4.1" HTTPVersion = '1.1' begin require 'zlib' @@ -1075,7 +1078,7 @@ module Net #:nodoc: elsif p_addr == :ENV then http.proxy_from_env = true else - if p_addr && p_no_proxy && !URI::Generic.use_proxy?(p_addr, p_addr, p_port, p_no_proxy) + if p_addr && p_no_proxy && !URI::Generic.use_proxy?(address, address, port, p_no_proxy) p_addr = nil p_port = nil end @@ -1615,8 +1618,8 @@ module Net #:nodoc: write_timeout: @write_timeout, continue_timeout: @continue_timeout, debug_output: @debug_output) - buf = "CONNECT #{conn_address}:#{@port} HTTP/#{HTTPVersion}\r\n" - buf << "Host: #{@address}:#{@port}\r\n" + buf = +"CONNECT #{conn_address}:#{@port} HTTP/#{HTTPVersion}\r\n" \ + "Host: #{@address}:#{@port}\r\n" if proxy_user credential = ["#{proxy_user}:#{proxy_pass}"].pack('m0') buf << "Proxy-Authorization: Basic #{credential}\r\n" @@ -1798,7 +1801,7 @@ module Net #:nodoc: def proxy_uri # :nodoc: return if @proxy_uri == false @proxy_uri ||= URI::HTTP.new( - "http".freeze, nil, address, port, nil, nil, nil, nil, nil + "http", nil, address, port, nil, nil, nil, nil, nil ).find_proxy || false @proxy_uri || nil end @@ -2351,7 +2354,10 @@ module Net #:nodoc: res } res.reading_body(@socket, req.response_body_permitted?) { - yield res if block_given? + if block_given? + count = max_retries # Don't restart in the middle of a download + yield res + end } rescue Net::OpenTimeout raise diff --git a/lib/net/http/backward.rb b/lib/net/http/backward.rb index 691e41e4f1..b44577edbd 100644 --- a/lib/net/http/backward.rb +++ b/lib/net/http/backward.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # for backward compatibility # :enddoc: diff --git a/lib/net/http/exceptions.rb b/lib/net/http/exceptions.rb index 9c425cae16..ceec8f7b0a 100644 --- a/lib/net/http/exceptions.rb +++ b/lib/net/http/exceptions.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true module Net # Net::HTTP exception class. # You cannot use Net::HTTPExceptions directly; instead, you must use diff --git a/lib/net/http/generic_request.rb b/lib/net/http/generic_request.rb index 8877cd04ae..44e329a0c8 100644 --- a/lib/net/http/generic_request.rb +++ b/lib/net/http/generic_request.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # \HTTPGenericRequest is the parent of the Net::HTTPRequest class. # @@ -23,7 +23,7 @@ class Net::HTTPGenericRequest raise ArgumentError, "no host component for URI" unless (hostname && hostname.length > 0) @uri = uri_or_path.dup host = @uri.hostname.dup - host << ":".freeze << @uri.port.to_s if @uri.port != @uri.default_port + host << ":" << @uri.port.to_s if @uri.port != @uri.default_port @path = uri_or_path.request_uri raise ArgumentError, "no HTTP request path given" unless @path else @@ -212,15 +212,15 @@ class Net::HTTPGenericRequest return unless @uri if ssl - scheme = 'https'.freeze + scheme = 'https' klass = URI::HTTPS else - scheme = 'http'.freeze + scheme = 'http' klass = URI::HTTP end if host = self['host'] - host.sub!(/:.*/m, ''.freeze) + host.sub!(/:.*/m, '') elsif host = @uri.host else host = addr @@ -316,7 +316,7 @@ class Net::HTTPGenericRequest boundary ||= SecureRandom.urlsafe_base64(40) chunked_p = chunked? - buf = '' + buf = +'' params.each do |key, value, h={}| key = quote_string(key, charset) filename = @@ -401,7 +401,7 @@ class Net::HTTPGenericRequest if /[\r\n]/ =~ reqline raise ArgumentError, "A Request-Line must not contain CR or LF" end - buf = "" + buf = +'' buf << reqline << "\r\n" each_capitalized do |k,v| buf << "#{k}: #{v}\r\n" diff --git a/lib/net/http/header.rb b/lib/net/http/header.rb index b704860c90..6660c8075a 100644 --- a/lib/net/http/header.rb +++ b/lib/net/http/header.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # # The \HTTPHeader module provides access to \HTTP headers. # @@ -699,10 +699,14 @@ module Net::HTTPHeader # res.content_type # => "application/json" # def content_type - return nil unless main_type() - if sub_type() - then "#{main_type()}/#{sub_type()}" - else main_type() + main = main_type() + return nil unless main + + sub = sub_type() + if sub + "#{main}/#{sub}" + else + main end end diff --git a/lib/net/http/proxy_delta.rb b/lib/net/http/proxy_delta.rb index a2f770ebdb..e7d30def64 100644 --- a/lib/net/http/proxy_delta.rb +++ b/lib/net/http/proxy_delta.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true module Net::HTTP::ProxyDelta #:nodoc: internal use only private diff --git a/lib/net/http/request.rb b/lib/net/http/request.rb index e900b8a17a..4a138572e9 100644 --- a/lib/net/http/request.rb +++ b/lib/net/http/request.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # This class is the base class for \Net::HTTP request classes. # The class should not be used directly; diff --git a/lib/net/http/requests.rb b/lib/net/http/requests.rb index 96cedcabcc..5724164205 100644 --- a/lib/net/http/requests.rb +++ b/lib/net/http/requests.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # HTTP/1.1 methods --- RFC2616 diff --git a/lib/net/http/response.rb b/lib/net/http/response.rb index 1d83935c10..40de963868 100644 --- a/lib/net/http/response.rb +++ b/lib/net/http/response.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true # This class is the base class for \Net::HTTP response classes. # @@ -273,7 +273,7 @@ class Net::HTTPResponse def error! #:nodoc: message = @code - message += ' ' + @message.dump if @message + message = "#{message} #{@message.dump}" if @message raise error_type().new(message, self) end @@ -366,6 +366,7 @@ class Net::HTTPResponse @body = nil end @read = true + return if @body.nil? case enc = @body_encoding when Encoding, false, nil @@ -639,7 +640,7 @@ class Net::HTTPResponse end def stream_check - raise IOError, 'attempt to read body out of block' if @socket.closed? + raise IOError, 'attempt to read body out of block' if @socket.nil? || @socket.closed? end def procdest(dest, block) @@ -648,7 +649,7 @@ class Net::HTTPResponse if block Net::ReadAdapter.new(block) else - dest || '' + dest || +'' end end diff --git a/lib/net/https.rb b/lib/net/https.rb index d46721c82a..0f23e1fb13 100644 --- a/lib/net/https.rb +++ b/lib/net/https.rb @@ -1,4 +1,4 @@ -# frozen_string_literal: false +# frozen_string_literal: true =begin = net/https -- SSL/TLS enhancement for Net::HTTP. diff --git a/lib/net/net-protocol.gemspec b/lib/net/net-protocol.gemspec index 29ad2504eb..f9fd83f12b 100644 --- a/lib/net/net-protocol.gemspec +++ b/lib/net/net-protocol.gemspec @@ -21,6 +21,7 @@ Gem::Specification.new do |spec| spec.metadata["homepage_uri"] = spec.homepage spec.metadata["source_code_uri"] = spec.homepage + spec.metadata["changelog_uri"] = spec.homepage + "/releases" # Specify which files should be added to the gem when it is released. # The `git ls-files -z` loads the files in the RubyGem that have been added into git. diff --git a/lib/net/protocol.rb b/lib/net/protocol.rb index ea0752a971..197ea09089 100644 --- a/lib/net/protocol.rb +++ b/lib/net/protocol.rb @@ -26,7 +26,7 @@ require 'io/wait' module Net # :nodoc: class Protocol #:nodoc: internal use only - VERSION = "0.2.1" + VERSION = "0.2.2" private def Protocol.protocol_param(name, val) diff --git a/lib/observer.gemspec b/lib/observer.gemspec deleted file mode 100644 index 93e61b8c84..0000000000 --- a/lib/observer.gemspec +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -name = File.basename(__FILE__, ".gemspec") -version = ["lib", Array.new(name.count("-")+1, ".").join("/")].find do |dir| - break File.foreach(File.join(__dir__, dir, "#{name.tr('-', '/')}.rb")) do |line| - /^\s*VERSION\s*=\s*"(.*)"/ =~ line and break $1 - end rescue nil -end - -Gem::Specification.new do |spec| - spec.name = name - spec.version = version - spec.authors = ["Yukihiro Matsumoto"] - spec.email = ["matz@ruby-lang.org"] - - spec.summary = %q{Implementation of the Observer object-oriented design pattern.} - spec.description = spec.summary - spec.homepage = "https://github.com/ruby/observer" - spec.licenses = ["Ruby", "BSD-2-Clause"] - - spec.metadata["homepage_uri"] = spec.homepage - spec.metadata["source_code_uri"] = spec.homepage - - # Specify which files should be added to the gem when it is released. - # The `git ls-files -z` loads the files in the RubyGem that have been added into git. - spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do - `git ls-files -z 2>#{IO::NULL}`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } - end - spec.bindir = "exe" - spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } - spec.require_paths = ["lib"] -end diff --git a/lib/observer.rb b/lib/observer.rb deleted file mode 100644 index ef70e39dd8..0000000000 --- a/lib/observer.rb +++ /dev/null @@ -1,229 +0,0 @@ -# frozen_string_literal: true -# -# Implementation of the _Observer_ object-oriented design pattern. The -# following documentation is copied, with modifications, from "Programming -# Ruby", by Hunt and Thomas; http://www.ruby-doc.org/docs/ProgrammingRuby/html/lib_patterns.html. -# -# See Observable for more info. - -# The Observer pattern (also known as publish/subscribe) provides a simple -# mechanism for one object to inform a set of interested third-party objects -# when its state changes. -# -# == Mechanism -# -# The notifying class mixes in the +Observable+ -# module, which provides the methods for managing the associated observer -# objects. -# -# The observable object must: -# * assert that it has +#changed+ -# * call +#notify_observers+ -# -# An observer subscribes to updates using Observable#add_observer, which also -# specifies the method called via #notify_observers. The default method for -# #notify_observers is #update. -# -# === Example -# -# The following example demonstrates this nicely. A +Ticker+, when run, -# continually receives the stock +Price+ for its <tt>@symbol</tt>. A +Warner+ -# is a general observer of the price, and two warners are demonstrated, a -# +WarnLow+ and a +WarnHigh+, which print a warning if the price is below or -# above their set limits, respectively. -# -# The +update+ callback allows the warners to run without being explicitly -# called. The system is set up with the +Ticker+ and several observers, and the -# observers do their duty without the top-level code having to interfere. -# -# Note that the contract between publisher and subscriber (observable and -# observer) is not declared or enforced. The +Ticker+ publishes a time and a -# price, and the warners receive that. But if you don't ensure that your -# contracts are correct, nothing else can warn you. -# -# require "observer" -# -# class Ticker ### Periodically fetch a stock price. -# include Observable -# -# def initialize(symbol) -# @symbol = symbol -# end -# -# def run -# last_price = nil -# loop do -# price = Price.fetch(@symbol) -# print "Current price: #{price}\n" -# if price != last_price -# changed # notify observers -# last_price = price -# notify_observers(Time.now, price) -# end -# sleep 1 -# end -# end -# end -# -# class Price ### A mock class to fetch a stock price (60 - 140). -# def self.fetch(symbol) -# 60 + rand(80) -# end -# end -# -# class Warner ### An abstract observer of Ticker objects. -# def initialize(ticker, limit) -# @limit = limit -# ticker.add_observer(self) -# end -# end -# -# class WarnLow < Warner -# def update(time, price) # callback for observer -# if price < @limit -# print "--- #{time.to_s}: Price below #@limit: #{price}\n" -# end -# end -# end -# -# class WarnHigh < Warner -# def update(time, price) # callback for observer -# if price > @limit -# print "+++ #{time.to_s}: Price above #@limit: #{price}\n" -# end -# end -# end -# -# ticker = Ticker.new("MSFT") -# WarnLow.new(ticker, 80) -# WarnHigh.new(ticker, 120) -# ticker.run -# -# Produces: -# -# Current price: 83 -# Current price: 75 -# --- Sun Jun 09 00:10:25 CDT 2002: Price below 80: 75 -# Current price: 90 -# Current price: 134 -# +++ Sun Jun 09 00:10:25 CDT 2002: Price above 120: 134 -# Current price: 134 -# Current price: 112 -# Current price: 79 -# --- Sun Jun 09 00:10:25 CDT 2002: Price below 80: 79 -# -# === Usage with procs -# -# The +#notify_observers+ method can also be used with +proc+s by using -# the +:call+ as +func+ parameter. -# -# The following example illustrates the use of a lambda: -# -# require 'observer' -# -# class Ticker -# include Observable -# -# def run -# # logic to retrieve the price (here 77.0) -# changed -# notify_observers(77.0) -# end -# end -# -# ticker = Ticker.new -# warner = ->(price) { puts "New price received: #{price}" } -# ticker.add_observer(warner, :call) -# ticker.run -module Observable - VERSION = "0.1.1" - - # - # Add +observer+ as an observer on this object. So that it will receive - # notifications. - # - # +observer+:: the object that will be notified of changes. - # +func+:: Symbol naming the method that will be called when this Observable - # has changes. - # - # This method must return true for +observer.respond_to?+ and will - # receive <tt>*arg</tt> when #notify_observers is called, where - # <tt>*arg</tt> is the value passed to #notify_observers by this - # Observable - def add_observer(observer, func=:update) - @observer_peers = {} unless defined? @observer_peers - unless observer.respond_to? func - raise NoMethodError, "observer does not respond to `#{func}'" - end - @observer_peers[observer] = func - end - - # - # Remove +observer+ as an observer on this object so that it will no longer - # receive notifications. - # - # +observer+:: An observer of this Observable - def delete_observer(observer) - @observer_peers.delete observer if defined? @observer_peers - end - - # - # Remove all observers associated with this object. - # - def delete_observers - @observer_peers.clear if defined? @observer_peers - end - - # - # Return the number of observers associated with this object. - # - def count_observers - if defined? @observer_peers - @observer_peers.size - else - 0 - end - end - - # - # Set the changed state of this object. Notifications will be sent only if - # the changed +state+ is +true+. - # - # +state+:: Boolean indicating the changed state of this Observable. - # - def changed(state=true) - @observer_state = state - end - - # - # Returns true if this object's state has been changed since the last - # #notify_observers call. - # - def changed? - if defined? @observer_state and @observer_state - true - else - false - end - end - - # - # Notify observers of a change in state *if* this object's changed state is - # +true+. - # - # This will invoke the method named in #add_observer, passing <tt>*arg</tt>. - # The changed state is then set to +false+. - # - # <tt>*arg</tt>:: Any arguments to pass to the observers. - def notify_observers(*arg) - if defined? @observer_state and @observer_state - if defined? @observer_peers - @observer_peers.each do |k, v| - k.__send__(v, *arg) - end - end - @observer_state = false - end - end - -end diff --git a/lib/open-uri.rb b/lib/open-uri.rb index 0052bb49fb..ba2379325f 100644 --- a/lib/open-uri.rb +++ b/lib/open-uri.rb @@ -31,6 +31,7 @@ module URI super end end + singleton_class.send(:ruby2_keywords, :open) if respond_to?(:ruby2_keywords, true) end # OpenURI is an easy-to-use wrapper for Net::HTTP, Net::HTTPS and Net::FTP. @@ -90,7 +91,7 @@ end module OpenURI - VERSION = "0.3.0" + VERSION = "0.4.1" Options = { :proxy => true, @@ -107,6 +108,7 @@ module OpenURI :ftp_active_mode => false, :redirect => true, :encoding => nil, + :max_redirects => 64, } def OpenURI.check_options(options) # :nodoc: @@ -210,6 +212,7 @@ module OpenURI end uri_set = {} + max_redirects = options[:max_redirects] buf = nil while true redirect = catch(:open_uri_redirect) { @@ -237,6 +240,7 @@ module OpenURI uri = redirect raise "HTTP redirection loop: #{uri}" if uri_set.include? uri.to_s uri_set[uri.to_s] = true + raise TooManyRedirects.new("Too many redirects", buf.io) if max_redirects && uri_set.size > max_redirects else break end @@ -391,6 +395,9 @@ module OpenURI attr_reader :uri end + class TooManyRedirects < HTTPError + end + class Buffer # :nodoc: all def initialize @io = StringIO.new diff --git a/lib/open3.rb b/lib/open3.rb index 9652b27194..74d00b86d9 100644 --- a/lib/open3.rb +++ b/lib/open3.rb @@ -31,57 +31,189 @@ require 'open3/version' +# \Module \Open3 supports creating child processes +# with access to their $stdin, $stdout, and $stderr streams. +# +# == What's Here +# +# Each of these methods executes a given command in a new process or subshell, +# or multiple commands in new processes and/or subshells: +# +# - Each of these methods executes a single command in a process or subshell, +# accepts a string for input to $stdin, +# and returns string output from $stdout, $stderr, or both: +# +# - Open3.capture2: Executes the command; +# returns the string from $stdout. +# - Open3.capture2e: Executes the command; +# returns the string from merged $stdout and $stderr. +# - Open3.capture3: Executes the command; +# returns strings from $stdout and $stderr. +# +# - Each of these methods executes a single command in a process or subshell, +# and returns pipes for $stdin, $stdout, and/or $stderr: +# +# - Open3.popen2: Executes the command; +# returns pipes for $stdin and $stdout. +# - Open3.popen2e: Executes the command; +# returns pipes for $stdin and merged $stdout and $stderr. +# - Open3.popen3: Executes the command; +# returns pipes for $stdin, $stdout, and $stderr. +# +# - Each of these methods executes one or more commands in processes and/or subshells, +# returns pipes for the first $stdin, the last $stdout, or both: +# +# - Open3.pipeline_r: Returns a pipe for the last $stdout. +# - Open3.pipeline_rw: Returns pipes for the first $stdin and the last $stdout. +# - Open3.pipeline_w: Returns a pipe for the first $stdin. +# - Open3.pipeline_start: Does not wait for processes to complete. +# - Open3.pipeline: Waits for processes to complete. +# +# Each of the methods above accepts: +# +# - An optional hash of environment variable names and values; +# see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. +# - A required string argument that is a +command_line+ or +exe_path+; +# see {Argument command_line or exe_path}[rdoc-ref:Process@Argument+command_line+or+exe_path]. +# - An optional hash of execution options; +# see {Execution Options}[rdoc-ref:Process@Execution+Options]. +# module Open3 - # Open stdin, stdout, and stderr streams and start external executable. - # In addition, a thread to wait for the started process is created. - # The thread has a pid method and a thread variable :pid which is the pid of - # the started process. + # :call-seq: + # Open3.popen3([env, ] command_line, options = {}) -> [stdin, stdout, stderr, wait_thread] + # Open3.popen3([env, ] exe_path, *args, options = {}) -> [stdin, stdout, stderr, wait_thread] + # Open3.popen3([env, ] command_line, options = {}) {|stdin, stdout, stderr, wait_thread| ... } -> object + # Open3.popen3([env, ] exe_path, *args, options = {}) {|stdin, stdout, stderr, wait_thread| ... } -> object + # + # Basically a wrapper for + # {Process.spawn}[rdoc-ref:Process.spawn] + # that: + # + # - Creates a child process, by calling Process.spawn with the given arguments. + # - Creates streams +stdin+, +stdout+, and +stderr+, + # which are the standard input, standard output, and standard error streams + # in the child process. + # - Creates thread +wait_thread+ that waits for the child process to exit; + # the thread has method +pid+, which returns the process ID + # of the child process. + # + # With no block given, returns the array + # <tt>[stdin, stdout, stderr, wait_thread]</tt>. + # The caller should close each of the three returned streams. + # + # stdin, stdout, stderr, wait_thread = Open3.popen3('echo') + # # => [#<IO:fd 8>, #<IO:fd 10>, #<IO:fd 12>, #<Process::Waiter:0x00007f58d5428f58 run>] + # stdin.close + # stdout.close + # stderr.close + # wait_thread.pid # => 2210481 + # wait_thread.value # => #<Process::Status: pid 2210481 exit 0> + # + # With a block given, calls the block with the four variables + # (three streams and the wait thread) + # and returns the block's return value. + # The caller need not close the streams: + # + # Open3.popen3('echo') do |stdin, stdout, stderr, wait_thread| + # p stdin + # p stdout + # p stderr + # p wait_thread + # p wait_thread.pid + # p wait_thread.value + # end # - # Block form: + # Output: # - # Open3.popen3([env,] cmd... [, opts]) {|stdin, stdout, stderr, wait_thr| - # pid = wait_thr.pid # pid of the started process. - # ... - # exit_status = wait_thr.value # Process::Status object returned. - # } + # #<IO:fd 6> + # #<IO:fd 7> + # #<IO:fd 9> + # #<Process::Waiter:0x00007f58d53606e8 sleep> + # 2211047 + # #<Process::Status: pid 2211047 exit 0> # - # Non-block form: + # Like Process.spawn, this method has potential security vulnerabilities + # if called with untrusted input; + # see {Command Injection}[rdoc-ref:command_injection.rdoc@Command+Injection]. # - # stdin, stdout, stderr, wait_thr = Open3.popen3([env,] cmd... [, opts]) - # pid = wait_thr[:pid] # pid of the started process - # ... - # stdin.close # stdin, stdout and stderr should be closed explicitly in this form. - # stdout.close - # stderr.close - # exit_status = wait_thr.value # Process::Status object returned. + # Unlike Process.spawn, this method waits for the child process to exit + # before returning, so the caller need not do so. + # + # If the first argument is a hash, it becomes leading argument +env+ + # in the call to Process.spawn; + # see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. + # + # If the last argument is a hash, it becomes trailing argument +options+ + # in the call to Process.spawn; + # see {Execution Options}[rdoc-ref:Process@Execution+Options]. + # + # The single required argument is one of the following: + # + # - +command_line+ if it is a string, + # and if it begins with a shell reserved word or special built-in, + # or if it contains one or more metacharacters. + # - +exe_path+ otherwise. + # + # <b>Argument +command_line+</b> # - # The parameters env, cmd, and opts are passed to Process.spawn. - # A commandline string and a list of argument strings can be accepted as follows: + # \String argument +command_line+ is a command line to be passed to a shell; + # it must begin with a shell reserved word, begin with a special built-in, + # or contain meta characters: # - # Open3.popen3("echo abc") {|i, o, e, t| ... } - # Open3.popen3("echo", "abc") {|i, o, e, t| ... } - # Open3.popen3(["echo", "argv0"], "abc") {|i, o, e, t| ... } + # Open3.popen3('if true; then echo "Foo"; fi') {|*args| p args } # Shell reserved word. + # Open3.popen3('echo') {|*args| p args } # Built-in. + # Open3.popen3('date > date.tmp') {|*args| p args } # Contains meta character. # - # If the last parameter, opts, is a Hash, it is recognized as an option for Process.spawn. + # Output (similar for each call above): # - # Open3.popen3("pwd", :chdir=>"/") {|i,o,e,t| - # p o.read.chomp #=> "/" - # } + # [#<IO:(closed)>, #<IO:(closed)>, #<IO:(closed)>, #<Process::Waiter:0x00007f58d52f28c8 dead>] # - # wait_thr.value waits for the termination of the process. - # The block form also waits for the process when it returns. + # The command line may also contain arguments and options for the command: + # + # Open3.popen3('echo "Foo"') { |i, o, e, t| o.gets } + # "Foo\n" + # + # <b>Argument +exe_path+</b> + # + # Argument +exe_path+ is one of the following: + # + # - The string path to an executable to be called. + # - A 2-element array containing the path to an executable + # and the string to be used as the name of the executing process. + # + # Example: # - # Closing stdin, stdout and stderr does not wait for the process to complete. + # Open3.popen3('/usr/bin/date') { |i, o, e, t| o.gets } + # # => "Wed Sep 27 02:56:44 PM CDT 2023\n" # - # You should be careful to avoid deadlocks. - # Since pipes are fixed length buffers, - # Open3.popen3("prog") {|i, o, e, t| o.read } deadlocks if - # the program generates too much output on stderr. - # You should read stdout and stderr simultaneously (using threads or IO.select). - # However, if you don't need stderr output, you can use Open3.popen2. - # If merged stdout and stderr output is not a problem, you can use Open3.popen2e. - # If you really need stdout and stderr output as separate strings, you can consider Open3.capture3. + # Ruby invokes the executable directly, with no shell and no shell expansion: + # + # Open3.popen3('doesnt_exist') { |i, o, e, t| o.gets } # Raises Errno::ENOENT + # + # If one or more +args+ is given, each is an argument or option + # to be passed to the executable: + # + # Open3.popen3('echo', 'C #') { |i, o, e, t| o.gets } + # # => "C #\n" + # Open3.popen3('echo', 'hello', 'world') { |i, o, e, t| o.gets } + # # => "hello world\n" + # + # Take care to avoid deadlocks. + # Output streams +stdout+ and +stderr+ have fixed-size buffers, + # so reading extensively from one but not the other can cause a deadlock + # when the unread buffer fills. + # To avoid that, +stdout+ and +stderr+ should be read simultaneously + # (using threads or IO.select). + # + # Related: + # + # - Open3.popen2: Makes the standard input and standard output streams + # of the child process available as separate streams, + # with no access to the standard error stream. + # - Open3.popen2e: Makes the standard input and the merge + # of the standard output and standard error streams + # of the child process available as separate streams. # def popen3(*cmd, &block) if Hash === cmd.last @@ -104,45 +236,131 @@ module Open3 end module_function :popen3 - # Open3.popen2 is similar to Open3.popen3 except that it doesn't create a pipe for - # the standard error stream. + # :call-seq: + # Open3.popen2([env, ] command_line, options = {}) -> [stdin, stdout, wait_thread] + # Open3.popen2([env, ] exe_path, *args, options = {}) -> [stdin, stdout, wait_thread] + # Open3.popen2([env, ] command_line, options = {}) {|stdin, stdout, wait_thread| ... } -> object + # Open3.popen2([env, ] exe_path, *args, options = {}) {|stdin, stdout, wait_thread| ... } -> object + # + # Basically a wrapper for + # {Process.spawn}[rdoc-ref:Process.spawn] + # that: + # + # - Creates a child process, by calling Process.spawn with the given arguments. + # - Creates streams +stdin+ and +stdout+, + # which are the standard input and standard output streams + # in the child process. + # - Creates thread +wait_thread+ that waits for the child process to exit; + # the thread has method +pid+, which returns the process ID + # of the child process. + # + # With no block given, returns the array + # <tt>[stdin, stdout, wait_thread]</tt>. + # The caller should close each of the two returned streams. + # + # stdin, stdout, wait_thread = Open3.popen2('echo') + # # => [#<IO:fd 6>, #<IO:fd 7>, #<Process::Waiter:0x00007f58d52dbe98 run>] + # stdin.close + # stdout.close + # wait_thread.pid # => 2263572 + # wait_thread.value # => #<Process::Status: pid 2263572 exit 0> + # + # With a block given, calls the block with the three variables + # (two streams and the wait thread) + # and returns the block's return value. + # The caller need not close the streams: + # + # Open3.popen2('echo') do |stdin, stdout, wait_thread| + # p stdin + # p stdout + # p wait_thread + # p wait_thread.pid + # p wait_thread.value + # end # - # Block form: + # Output: # - # Open3.popen2([env,] cmd... [, opts]) {|stdin, stdout, wait_thr| - # pid = wait_thr.pid # pid of the started process. - # ... - # exit_status = wait_thr.value # Process::Status object returned. - # } + # #<IO:fd 6> + # #<IO:fd 7> + # #<Process::Waiter:0x00007f58d59a34b0 sleep> + # 2263636 + # #<Process::Status: pid 2263636 exit 0> # - # Non-block form: + # Like Process.spawn, this method has potential security vulnerabilities + # if called with untrusted input; + # see {Command Injection}[rdoc-ref:command_injection.rdoc@Command+Injection]. # - # stdin, stdout, wait_thr = Open3.popen2([env,] cmd... [, opts]) - # ... - # stdin.close # stdin and stdout should be closed explicitly in this form. - # stdout.close + # Unlike Process.spawn, this method waits for the child process to exit + # before returning, so the caller need not do so. + # + # If the first argument is a hash, it becomes leading argument +env+ + # in the call to Process.spawn; + # see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. + # + # If the last argument is a hash, it becomes trailing argument +options+ + # in the call to Process.spawn; + # see {Execution Options}[rdoc-ref:Process@Execution+Options]. + # + # The single required argument is one of the following: + # + # - +command_line+ if it is a string, + # and if it begins with a shell reserved word or special built-in, + # or if it contains one or more metacharacters. + # - +exe_path+ otherwise. # - # See Process.spawn for the optional hash arguments _env_ and _opts_. + # <b>Argument +command_line+</b> + # + # \String argument +command_line+ is a command line to be passed to a shell; + # it must begin with a shell reserved word, begin with a special built-in, + # or contain meta characters: + # + # Open3.popen2('if true; then echo "Foo"; fi') {|*args| p args } # Shell reserved word. + # Open3.popen2('echo') {|*args| p args } # Built-in. + # Open3.popen2('date > date.tmp') {|*args| p args } # Contains meta character. + # + # Output (similar for each call above): + # + # # => [#<IO:(closed)>, #<IO:(closed)>, #<Process::Waiter:0x00007f7577dfe410 dead>] + # + # The command line may also contain arguments and options for the command: + # + # Open3.popen2('echo "Foo"') { |i, o, t| o.gets } + # "Foo\n" + # + # <b>Argument +exe_path+</b> + # + # Argument +exe_path+ is one of the following: + # + # - The string path to an executable to be called. + # - A 2-element array containing the path to an executable + # and the string to be used as the name of the executing process. # # Example: # - # Open3.popen2("wc -c") {|i,o,t| - # i.print "answer to life the universe and everything" - # i.close - # p o.gets #=> "42\n" - # } + # Open3.popen2('/usr/bin/date') { |i, o, t| o.gets } + # # => "Thu Sep 28 09:41:06 AM CDT 2023\n" + # + # Ruby invokes the executable directly, with no shell and no shell expansion: + # + # Open3.popen2('doesnt_exist') { |i, o, t| o.gets } # Raises Errno::ENOENT + # + # If one or more +args+ is given, each is an argument or option + # to be passed to the executable: + # + # Open3.popen2('echo', 'C #') { |i, o, t| o.gets } + # # => "C #\n" + # Open3.popen2('echo', 'hello', 'world') { |i, o, t| o.gets } + # # => "hello world\n" # - # Open3.popen2("bc -q") {|i,o,t| - # i.puts "obase=13" - # i.puts "6 * 9" - # p o.gets #=> "42\n" - # } # - # Open3.popen2("dc") {|i,o,t| - # i.print "42P" - # i.close - # p o.read #=> "*" - # } + # Related: + # + # - Open3.popen2e: Makes the standard input and the merge + # of the standard output and standard error streams + # of the child process available as separate streams. + # - Open3.popen3: Makes the standard input, standard output, + # and standard error streams + # of the child process available as separate streams. # def popen2(*cmd, &block) if Hash === cmd.last @@ -162,36 +380,130 @@ module Open3 end module_function :popen2 - # Open3.popen2e is similar to Open3.popen3 except that it merges - # the standard output stream and the standard error stream. + # :call-seq: + # Open3.popen2e([env, ] command_line, options = {}) -> [stdin, stdout_and_stderr, wait_thread] + # Open3.popen2e([env, ] exe_path, *args, options = {}) -> [stdin, stdout_and_stderr, wait_thread] + # Open3.popen2e([env, ] command_line, options = {}) {|stdin, stdout_and_stderr, wait_thread| ... } -> object + # Open3.popen2e([env, ] exe_path, *args, options = {}) {|stdin, stdout_and_stderr, wait_thread| ... } -> object + # + # Basically a wrapper for + # {Process.spawn}[rdoc-ref:Process.spawn] + # that: + # + # - Creates a child process, by calling Process.spawn with the given arguments. + # - Creates streams +stdin+, +stdout_and_stderr+, + # which are the standard input and the merge of the standard output + # and standard error streams in the child process. + # - Creates thread +wait_thread+ that waits for the child process to exit; + # the thread has method +pid+, which returns the process ID + # of the child process. + # + # With no block given, returns the array + # <tt>[stdin, stdout_and_stderr, wait_thread]</tt>. + # The caller should close each of the two returned streams. + # + # stdin, stdout_and_stderr, wait_thread = Open3.popen2e('echo') + # # => [#<IO:fd 6>, #<IO:fd 7>, #<Process::Waiter:0x00007f7577da4398 run>] + # stdin.close + # stdout_and_stderr.close + # wait_thread.pid # => 2274600 + # wait_thread.value # => #<Process::Status: pid 2274600 exit 0> + # + # With a block given, calls the block with the three variables + # (two streams and the wait thread) + # and returns the block's return value. + # The caller need not close the streams: + # + # Open3.popen2e('echo') do |stdin, stdout_and_stderr, wait_thread| + # p stdin + # p stdout_and_stderr + # p wait_thread + # p wait_thread.pid + # p wait_thread.value + # end # - # Block form: + # Output: # - # Open3.popen2e([env,] cmd... [, opts]) {|stdin, stdout_and_stderr, wait_thr| - # pid = wait_thr.pid # pid of the started process. - # ... - # exit_status = wait_thr.value # Process::Status object returned. - # } + # #<IO:fd 6> + # #<IO:fd 7> + # #<Process::Waiter:0x00007f75777578c8 sleep> + # 2274763 + # #<Process::Status: pid 2274763 exit 0> # - # Non-block form: + # Like Process.spawn, this method has potential security vulnerabilities + # if called with untrusted input; + # see {Command Injection}[rdoc-ref:command_injection.rdoc@Command+Injection]. # - # stdin, stdout_and_stderr, wait_thr = Open3.popen2e([env,] cmd... [, opts]) - # ... - # stdin.close # stdin and stdout_and_stderr should be closed explicitly in this form. - # stdout_and_stderr.close + # Unlike Process.spawn, this method waits for the child process to exit + # before returning, so the caller need not do so. + # + # If the first argument is a hash, it becomes leading argument +env+ + # in the call to Process.spawn; + # see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. + # + # If the last argument is a hash, it becomes trailing argument +options+ + # in the call to Process.spawn; + # see {Execution Options}[rdoc-ref:Process@Execution+Options]. + # + # The single required argument is one of the following: + # + # - +command_line+ if it is a string, + # and if it begins with a shell reserved word or special built-in, + # or if it contains one or more metacharacters. + # - +exe_path+ otherwise. + # + # <b>Argument +command_line+</b> + # + # \String argument +command_line+ is a command line to be passed to a shell; + # it must begin with a shell reserved word, begin with a special built-in, + # or contain meta characters: + # + # Open3.popen2e('if true; then echo "Foo"; fi') {|*args| p args } # Shell reserved word. + # Open3.popen2e('echo') {|*args| p args } # Built-in. + # Open3.popen2e('date > date.tmp') {|*args| p args } # Contains meta character. + # + # Output (similar for each call above): + # + # # => [#<IO:(closed)>, #<IO:(closed)>, #<Process::Waiter:0x00007f7577d8a1f0 dead>] # - # See Process.spawn for the optional hash arguments _env_ and _opts_. + # The command line may also contain arguments and options for the command: + # + # Open3.popen2e('echo "Foo"') { |i, o_and_e, t| o_and_e.gets } + # "Foo\n" + # + # <b>Argument +exe_path+</b> + # + # Argument +exe_path+ is one of the following: + # + # - The string path to an executable to be called. + # - A 2-element array containing the path to an executable + # and the string to be used as the name of the executing process. # # Example: - # # check gcc warnings - # source = "foo.c" - # Open3.popen2e("gcc", "-Wall", source) {|i,oe,t| - # oe.each {|line| - # if /warning/ =~ line - # ... - # end - # } - # } + # + # Open3.popen2e('/usr/bin/date') { |i, o_and_e, t| o_and_e.gets } + # # => "Thu Sep 28 01:58:45 PM CDT 2023\n" + # + # Ruby invokes the executable directly, with no shell and no shell expansion: + # + # Open3.popen2e('doesnt_exist') { |i, o_and_e, t| o_and_e.gets } # Raises Errno::ENOENT + # + # If one or more +args+ is given, each is an argument or option + # to be passed to the executable: + # + # Open3.popen2e('echo', 'C #') { |i, o_and_e, t| o_and_e.gets } + # # => "C #\n" + # Open3.popen2e('echo', 'hello', 'world') { |i, o_and_e, t| o_and_e.gets } + # # => "hello world\n" + # + # Related: + # + # - Open3.popen2: Makes the standard input and standard output streams + # of the child process available as separate streams, + # with no access to the standard error stream. + # - Open3.popen3: Makes the standard input, standard output, + # and standard error streams + # of the child process available as separate streams. # def popen2e(*cmd, &block) if Hash === cmd.last @@ -238,44 +550,100 @@ module Open3 private :popen_run end - # Open3.capture3 captures the standard output and the standard error of a command. + # :call-seq: + # Open3.capture3([env, ] command_line, options = {}) -> [stdout_s, stderr_s, status] + # Open3.capture3([env, ] exe_path, *args, options = {}) -> [stdout_s, stderr_s, status] # - # stdout_str, stderr_str, status = Open3.capture3([env,] cmd... [, opts]) + # Basically a wrapper for Open3.popen3 that: # - # The arguments env, cmd and opts are passed to Open3.popen3 except - # <code>opts[:stdin_data]</code> and <code>opts[:binmode]</code>. See Process.spawn. + # - Creates a child process, by calling Open3.popen3 with the given arguments + # (except for certain entries in hash +options+; see below). + # - Returns as strings +stdout_s+ and +stderr_s+ the standard output + # and standard error of the child process. + # - Returns as +status+ a <tt>Process::Status</tt> object + # that represents the exit status of the child process. # - # If <code>opts[:stdin_data]</code> is specified, it is sent to the command's standard input. + # Returns the array <tt>[stdout_s, stderr_s, status]</tt>: # - # If <code>opts[:binmode]</code> is true, internal pipes are set to binary mode. + # stdout_s, stderr_s, status = Open3.capture3('echo "Foo"') + # # => ["Foo\n", "", #<Process::Status: pid 2281954 exit 0>] # - # Examples: + # Like Process.spawn, this method has potential security vulnerabilities + # if called with untrusted input; + # see {Command Injection}[rdoc-ref:command_injection.rdoc@Command+Injection]. # - # # dot is a command of graphviz. - # graph = <<'End' - # digraph g { - # a -> b - # } - # End - # drawn_graph, dot_log = Open3.capture3("dot -v", :stdin_data=>graph) + # Unlike Process.spawn, this method waits for the child process to exit + # before returning, so the caller need not do so. # - # o, e, s = Open3.capture3("echo abc; sort >&2", :stdin_data=>"foo\nbar\nbaz\n") - # p o #=> "abc\n" - # p e #=> "bar\nbaz\nfoo\n" - # p s #=> #<Process::Status: pid 32682 exit 0> + # If the first argument is a hash, it becomes leading argument +env+ + # in the call to Open3.popen3; + # see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. # - # # generate a thumbnail image using the convert command of ImageMagick. - # # However, if the image is really stored in a file, - # # system("convert", "-thumbnail", "80", "png:#{filename}", "png:-") is better - # # because of reduced memory consumption. - # # But if the image is stored in a DB or generated by the gnuplot Open3.capture2 example, - # # Open3.capture3 should be considered. - # # - # image = File.read("/usr/share/openclipart/png/animals/mammals/sheep-md-v0.1.png", :binmode=>true) - # thumbnail, err, s = Open3.capture3("convert -thumbnail 80 png:- png:-", :stdin_data=>image, :binmode=>true) - # if s.success? - # STDOUT.binmode; print thumbnail - # end + # If the last argument is a hash, it becomes trailing argument +options+ + # in the call to Open3.popen3; + # see {Execution Options}[rdoc-ref:Process@Execution+Options]. + # + # The hash +options+ is given; + # two options have local effect in method Open3.capture3: + # + # - If entry <tt>options[:stdin_data]</tt> exists, the entry is removed + # and its string value is sent to the command's standard input: + # + # Open3.capture3('tee', stdin_data: 'Foo') + # # => ["Foo", "", #<Process::Status: pid 2319575 exit 0>] + # + # - If entry <tt>options[:binmode]</tt> exists, the entry is removed and + # the internal streams are set to binary mode. + # + # The single required argument is one of the following: + # + # - +command_line+ if it is a string, + # and if it begins with a shell reserved word or special built-in, + # or if it contains one or more metacharacters. + # - +exe_path+ otherwise. + # + # <b>Argument +command_line+</b> + # + # \String argument +command_line+ is a command line to be passed to a shell; + # it must begin with a shell reserved word, begin with a special built-in, + # or contain meta characters: + # + # Open3.capture3('if true; then echo "Foo"; fi') # Shell reserved word. + # # => ["Foo\n", "", #<Process::Status: pid 2282025 exit 0>] + # Open3.capture3('echo') # Built-in. + # # => ["\n", "", #<Process::Status: pid 2282092 exit 0>] + # Open3.capture3('date > date.tmp') # Contains meta character. + # # => ["", "", #<Process::Status: pid 2282110 exit 0>] + # + # The command line may also contain arguments and options for the command: + # + # Open3.capture3('echo "Foo"') + # # => ["Foo\n", "", #<Process::Status: pid 2282092 exit 0>] + # + # <b>Argument +exe_path+</b> + # + # Argument +exe_path+ is one of the following: + # + # - The string path to an executable to be called. + # - A 2-element array containing the path to an executable + # and the string to be used as the name of the executing process. + # + # Example: + # + # Open3.capture3('/usr/bin/date') + # # => ["Thu Sep 28 05:03:51 PM CDT 2023\n", "", #<Process::Status: pid 2282300 exit 0>] + # + # Ruby invokes the executable directly, with no shell and no shell expansion: + # + # Open3.capture3('doesnt_exist') # Raises Errno::ENOENT + # + # If one or more +args+ is given, each is an argument or option + # to be passed to the executable: + # + # Open3.capture3('echo', 'C #') + # # => ["C #\n", "", #<Process::Status: pid 2282368 exit 0>] + # Open3.capture3('echo', 'hello', 'world') + # # => ["hello world\n", "", #<Process::Status: pid 2282372 exit 0>] # def capture3(*cmd) if Hash === cmd.last @@ -309,34 +677,100 @@ module Open3 end module_function :capture3 - # Open3.capture2 captures the standard output of a command. + # :call-seq: + # Open3.capture2([env, ] command_line, options = {}) -> [stdout_s, status] + # Open3.capture2([env, ] exe_path, *args, options = {}) -> [stdout_s, status] + # + # Basically a wrapper for Open3.popen3 that: + # + # - Creates a child process, by calling Open3.popen3 with the given arguments + # (except for certain entries in hash +options+; see below). + # - Returns as string +stdout_s+ the standard output of the child process. + # - Returns as +status+ a <tt>Process::Status</tt> object + # that represents the exit status of the child process. + # + # Returns the array <tt>[stdout_s, status]</tt>: + # + # stdout_s, status = Open3.capture2('echo "Foo"') + # # => ["Foo\n", #<Process::Status: pid 2326047 exit 0>] + # + # Like Process.spawn, this method has potential security vulnerabilities + # if called with untrusted input; + # see {Command Injection}[rdoc-ref:command_injection.rdoc@Command+Injection]. + # + # Unlike Process.spawn, this method waits for the child process to exit + # before returning, so the caller need not do so. # - # stdout_str, status = Open3.capture2([env,] cmd... [, opts]) + # If the first argument is a hash, it becomes leading argument +env+ + # in the call to Open3.popen3; + # see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. # - # The arguments env, cmd and opts are passed to Open3.popen3 except - # <code>opts[:stdin_data]</code> and <code>opts[:binmode]</code>. See Process.spawn. + # If the last argument is a hash, it becomes trailing argument +options+ + # in the call to Open3.popen3; + # see {Execution Options}[rdoc-ref:Process@Execution+Options]. # - # If <code>opts[:stdin_data]</code> is specified, it is sent to the command's standard input. + # The hash +options+ is given; + # two options have local effect in method Open3.capture2: # - # If <code>opts[:binmode]</code> is true, internal pipes are set to binary mode. + # - If entry <tt>options[:stdin_data]</tt> exists, the entry is removed + # and its string value is sent to the command's standard input: + # + # Open3.capture2('tee', stdin_data: 'Foo') + # + # # => ["Foo", #<Process::Status: pid 2326087 exit 0>] + # + # - If entry <tt>options[:binmode]</tt> exists, the entry is removed and + # the internal streams are set to binary mode. + # + # The single required argument is one of the following: + # + # - +command_line+ if it is a string, + # and if it begins with a shell reserved word or special built-in, + # or if it contains one or more metacharacters. + # - +exe_path+ otherwise. + # + # <b>Argument +command_line+</b> + # + # \String argument +command_line+ is a command line to be passed to a shell; + # it must begin with a shell reserved word, begin with a special built-in, + # or contain meta characters: + # + # Open3.capture2('if true; then echo "Foo"; fi') # Shell reserved word. + # # => ["Foo\n", #<Process::Status: pid 2326131 exit 0>] + # Open3.capture2('echo') # Built-in. + # # => ["\n", #<Process::Status: pid 2326139 exit 0>] + # Open3.capture2('date > date.tmp') # Contains meta character. + # # => ["", #<Process::Status: pid 2326174 exit 0>] + # + # The command line may also contain arguments and options for the command: + # + # Open3.capture2('echo "Foo"') + # # => ["Foo\n", #<Process::Status: pid 2326183 exit 0>] + # + # <b>Argument +exe_path+</b> + # + # Argument +exe_path+ is one of the following: + # + # - The string path to an executable to be called. + # - A 2-element array containing the path to an executable + # and the string to be used as the name of the executing process. # # Example: # - # # factor is a command for integer factorization. - # o, s = Open3.capture2("factor", :stdin_data=>"42") - # p o #=> "42: 2 3 7\n" - # - # # generate x**2 graph in png using gnuplot. - # gnuplot_commands = <<"End" - # set terminal png - # plot x**2, "-" with lines - # 1 14 - # 2 1 - # 3 8 - # 4 5 - # e - # End - # image, s = Open3.capture2("gnuplot", :stdin_data=>gnuplot_commands, :binmode=>true) + # Open3.capture2('/usr/bin/date') + # # => ["Fri Sep 29 01:00:39 PM CDT 2023\n", #<Process::Status: pid 2326222 exit 0>] + # + # Ruby invokes the executable directly, with no shell and no shell expansion: + # + # Open3.capture2('doesnt_exist') # Raises Errno::ENOENT + # + # If one or more +args+ is given, each is an argument or option + # to be passed to the executable: + # + # Open3.capture2('echo', 'C #') + # # => ["C #\n", #<Process::Status: pid 2326267 exit 0>] + # Open3.capture2('echo', 'hello', 'world') + # # => ["hello world\n", #<Process::Status: pid 2326299 exit 0>] # def capture2(*cmd) if Hash === cmd.last @@ -370,21 +804,100 @@ module Open3 end module_function :capture2 - # Open3.capture2e captures the standard output and the standard error of a command. + # :call-seq: + # Open3.capture2e([env, ] command_line, options = {}) -> [stdout_and_stderr_s, status] + # Open3.capture2e([env, ] exe_path, *args, options = {}) -> [stdout_and_stderr_s, status] + # + # Basically a wrapper for Open3.popen3 that: + # + # - Creates a child process, by calling Open3.popen3 with the given arguments + # (except for certain entries in hash +options+; see below). + # - Returns as string +stdout_and_stderr_s+ the merged standard output + # and standard error of the child process. + # - Returns as +status+ a <tt>Process::Status</tt> object + # that represents the exit status of the child process. + # + # Returns the array <tt>[stdout_and_stderr_s, status]</tt>: + # + # stdout_and_stderr_s, status = Open3.capture2e('echo "Foo"') + # # => ["Foo\n", #<Process::Status: pid 2371692 exit 0>] + # + # Like Process.spawn, this method has potential security vulnerabilities + # if called with untrusted input; + # see {Command Injection}[rdoc-ref:command_injection.rdoc@Command+Injection]. + # + # Unlike Process.spawn, this method waits for the child process to exit + # before returning, so the caller need not do so. + # + # If the first argument is a hash, it becomes leading argument +env+ + # in the call to Open3.popen3; + # see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. + # + # If the last argument is a hash, it becomes trailing argument +options+ + # in the call to Open3.popen3; + # see {Execution Options}[rdoc-ref:Process@Execution+Options]. + # + # The hash +options+ is given; + # two options have local effect in method Open3.capture2e: + # + # - If entry <tt>options[:stdin_data]</tt> exists, the entry is removed + # and its string value is sent to the command's standard input: # - # stdout_and_stderr_str, status = Open3.capture2e([env,] cmd... [, opts]) + # Open3.capture2e('tee', stdin_data: 'Foo') + # # => ["Foo", #<Process::Status: pid 2371732 exit 0>] # - # The arguments env, cmd and opts are passed to Open3.popen3 except - # <code>opts[:stdin_data]</code> and <code>opts[:binmode]</code>. See Process.spawn. + # - If entry <tt>options[:binmode]</tt> exists, the entry is removed and + # the internal streams are set to binary mode. # - # If <code>opts[:stdin_data]</code> is specified, it is sent to the command's standard input. + # The single required argument is one of the following: # - # If <code>opts[:binmode]</code> is true, internal pipes are set to binary mode. + # - +command_line+ if it is a string, + # and if it begins with a shell reserved word or special built-in, + # or if it contains one or more metacharacters. + # - +exe_path+ otherwise. + # + # <b>Argument +command_line+</b> + # + # \String argument +command_line+ is a command line to be passed to a shell; + # it must begin with a shell reserved word, begin with a special built-in, + # or contain meta characters: + # + # Open3.capture2e('if true; then echo "Foo"; fi') # Shell reserved word. + # # => ["Foo\n", #<Process::Status: pid 2371740 exit 0>] + # Open3.capture2e('echo') # Built-in. + # # => ["\n", #<Process::Status: pid 2371774 exit 0>] + # Open3.capture2e('date > date.tmp') # Contains meta character. + # # => ["", #<Process::Status: pid 2371812 exit 0>] + # + # The command line may also contain arguments and options for the command: + # + # Open3.capture2e('echo "Foo"') + # # => ["Foo\n", #<Process::Status: pid 2326183 exit 0>] + # + # <b>Argument +exe_path+</b> + # + # Argument +exe_path+ is one of the following: + # + # - The string path to an executable to be called. + # - A 2-element array containing the path to an executable + # and the string to be used as the name of the executing process. # # Example: # - # # capture make log - # make_log, s = Open3.capture2e("make") + # Open3.capture2e('/usr/bin/date') + # # => ["Sat Sep 30 09:01:46 AM CDT 2023\n", #<Process::Status: pid 2371820 exit 0>] + # + # Ruby invokes the executable directly, with no shell and no shell expansion: + # + # Open3.capture2e('doesnt_exist') # Raises Errno::ENOENT + # + # If one or more +args+ is given, each is an argument or option + # to be passed to the executable: + # + # Open3.capture2e('echo', 'C #') + # # => ["C #\n", #<Process::Status: pid 2371856 exit 0>] + # Open3.capture2e('echo', 'hello', 'world') + # # => ["hello world\n", #<Process::Status: pid 2371894 exit 0>] # def capture2e(*cmd) if Hash === cmd.last @@ -418,48 +931,86 @@ module Open3 end module_function :capture2e - # Open3.pipeline_rw starts a list of commands as a pipeline with pipes - # which connect to stdin of the first command and stdout of the last command. - # - # Open3.pipeline_rw(cmd1, cmd2, ... [, opts]) {|first_stdin, last_stdout, wait_threads| - # ... - # } + # :call-seq: + # Open3.pipeline_rw([env, ] *cmds, options = {}) -> [first_stdin, last_stdout, wait_threads] # - # first_stdin, last_stdout, wait_threads = Open3.pipeline_rw(cmd1, cmd2, ... [, opts]) - # ... - # first_stdin.close - # last_stdout.close + # Basically a wrapper for + # {Process.spawn}[rdoc-ref:Process.spawn] + # that: # - # Each cmd is a string or an array. - # If it is an array, the elements are passed to Process.spawn. + # - Creates a child process for each of the given +cmds+ + # by calling Process.spawn. + # - Pipes the +stdout+ from each child to the +stdin+ of the next child, + # or, for the first child, from the caller's +stdin+, + # or, for the last child, to the caller's +stdout+. # - # cmd: - # commandline command line string which is passed to a shell - # [env, commandline, opts] command line string which is passed to a shell - # [env, cmdname, arg1, ..., opts] command name and one or more arguments (no shell) - # [env, [cmdname, argv0], arg1, ..., opts] command name and arguments including argv[0] (no shell) + # The method does not wait for child processes to exit, + # so the caller must do so. # - # Note that env and opts are optional, as for Process.spawn. + # With no block given, returns a 3-element array containing: # - # The options to pass to Process.spawn are constructed by merging - # +opts+, the last hash element of the array, and - # specifications for the pipes between each of the commands. + # - The +stdin+ stream of the first child process. + # - The +stdout+ stream of the last child process. + # - An array of the wait threads for all of the child processes. # # Example: # - # Open3.pipeline_rw("tr -dc A-Za-z", "wc -c") {|i, o, ts| - # i.puts "All persons more than a mile high to leave the court." - # i.close - # p o.gets #=> "42\n" - # } - # - # Open3.pipeline_rw("sort", "cat -n") {|stdin, stdout, wait_thrs| - # stdin.puts "foo" - # stdin.puts "bar" - # stdin.puts "baz" - # stdin.close # send EOF to sort. - # p stdout.read #=> " 1\tbar\n 2\tbaz\n 3\tfoo\n" - # } + # first_stdin, last_stdout, wait_threads = Open3.pipeline_rw('sort', 'cat -n') + # # => [#<IO:fd 20>, #<IO:fd 21>, [#<Process::Waiter:0x000055e8de29ab40 sleep>, #<Process::Waiter:0x000055e8de29a690 sleep>]] + # first_stdin.puts("foo\nbar\nbaz") + # first_stdin.close # Send EOF to sort. + # puts last_stdout.read + # wait_threads.each do |wait_thread| + # wait_thread.join + # end + # + # Output: + # + # 1 bar + # 2 baz + # 3 foo + # + # With a block given, calls the block with the +stdin+ stream of the first child, + # the +stdout+ stream of the last child, + # and an array of the wait processes: + # + # Open3.pipeline_rw('sort', 'cat -n') do |first_stdin, last_stdout, wait_threads| + # first_stdin.puts "foo\nbar\nbaz" + # first_stdin.close # send EOF to sort. + # puts last_stdout.read + # wait_threads.each do |wait_thread| + # wait_thread.join + # end + # end + # + # Output: + # + # 1 bar + # 2 baz + # 3 foo + # + # Like Process.spawn, this method has potential security vulnerabilities + # if called with untrusted input; + # see {Command Injection}[rdoc-ref:command_injection.rdoc@Command+Injection]. + # + # If the first argument is a hash, it becomes leading argument +env+ + # in each call to Process.spawn; + # see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. + # + # If the last argument is a hash, it becomes trailing argument +options+ + # in each call to Process.spawn; + # see {Execution Options}[rdoc-ref:Process@Execution+Options]. + # + # Each remaining argument in +cmds+ is one of: + # + # - A +command_line+: a string that begins with a shell reserved word + # or special built-in, or contains one or more metacharacters. + # - An +exe_path+: the string path to an executable to be called. + # - An array containing a +command_line+ or an +exe_path+, + # along with zero or more string arguments for the command. + # + # See {Argument command_line or exe_path}[rdoc-ref:Process@Argument+command_line+or+exe_path]. + # def pipeline_rw(*cmds, &block) if Hash === cmds.last opts = cmds.pop.dup @@ -478,43 +1029,77 @@ module Open3 end module_function :pipeline_rw - # Open3.pipeline_r starts a list of commands as a pipeline with a pipe - # which connects to stdout of the last command. + # :call-seq: + # Open3.pipeline_r([env, ] *cmds, options = {}) -> [last_stdout, wait_threads] # - # Open3.pipeline_r(cmd1, cmd2, ... [, opts]) {|last_stdout, wait_threads| - # ... - # } + # Basically a wrapper for + # {Process.spawn}[rdoc-ref:Process.spawn] + # that: # - # last_stdout, wait_threads = Open3.pipeline_r(cmd1, cmd2, ... [, opts]) - # ... - # last_stdout.close + # - Creates a child process for each of the given +cmds+ + # by calling Process.spawn. + # - Pipes the +stdout+ from each child to the +stdin+ of the next child, + # or, for the last child, to the caller's +stdout+. # - # Each cmd is a string or an array. - # If it is an array, the elements are passed to Process.spawn. + # The method does not wait for child processes to exit, + # so the caller must do so. # - # cmd: - # commandline command line string which is passed to a shell - # [env, commandline, opts] command line string which is passed to a shell - # [env, cmdname, arg1, ..., opts] command name and one or more arguments (no shell) - # [env, [cmdname, argv0], arg1, ..., opts] command name and arguments including argv[0] (no shell) + # With no block given, returns a 2-element array containing: # - # Note that env and opts are optional, as for Process.spawn. + # - The +stdout+ stream of the last child process. + # - An array of the wait threads for all of the child processes. # # Example: # - # Open3.pipeline_r("zcat /var/log/apache2/access.log.*.gz", - # [{"LANG"=>"C"}, "grep", "GET /favicon.ico"], - # "logresolve") {|o, ts| - # o.each_line {|line| - # ... - # } - # } + # last_stdout, wait_threads = Open3.pipeline_r('ls', 'grep R') + # # => [#<IO:fd 5>, [#<Process::Waiter:0x000055e8de2f9898 dead>, #<Process::Waiter:0x000055e8de2f94b0 sleep>]] + # puts last_stdout.read + # wait_threads.each do |wait_thread| + # wait_thread.join + # end + # + # Output: + # + # Rakefile + # README.md + # + # With a block given, calls the block with the +stdout+ stream + # of the last child process, + # and an array of the wait processes: + # + # Open3.pipeline_r('ls', 'grep R') do |last_stdout, wait_threads| + # puts last_stdout.read + # wait_threads.each do |wait_thread| + # wait_thread.join + # end + # end + # + # Output: + # + # Rakefile + # README.md + # + # Like Process.spawn, this method has potential security vulnerabilities + # if called with untrusted input; + # see {Command Injection}[rdoc-ref:command_injection.rdoc@Command+Injection]. + # + # If the first argument is a hash, it becomes leading argument +env+ + # in each call to Process.spawn; + # see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. + # + # If the last argument is a hash, it becomes trailing argument +options+ + # in each call to Process.spawn; + # see {Execution Options}[rdoc-ref:Process@Execution+Options]. # - # Open3.pipeline_r("yes", "head -10") {|o, ts| - # p o.read #=> "y\ny\ny\ny\ny\ny\ny\ny\ny\ny\n" - # p ts[0].value #=> #<Process::Status: pid 24910 SIGPIPE (signal 13)> - # p ts[1].value #=> #<Process::Status: pid 24913 exit 0> - # } + # Each remaining argument in +cmds+ is one of: + # + # - A +command_line+: a string that begins with a shell reserved word + # or special built-in, or contains one or more metacharacters. + # - An +exe_path+: the string path to an executable to be called. + # - An array containing a +command_line+ or an +exe_path+, + # along with zero or more string arguments for the command. + # + # See {Argument command_line or exe_path}[rdoc-ref:Process@Argument+command_line+or+exe_path]. # def pipeline_r(*cmds, &block) if Hash === cmds.last @@ -530,33 +1115,82 @@ module Open3 end module_function :pipeline_r - # Open3.pipeline_w starts a list of commands as a pipeline with a pipe - # which connects to stdin of the first command. + + # :call-seq: + # Open3.pipeline_w([env, ] *cmds, options = {}) -> [first_stdin, wait_threads] # - # Open3.pipeline_w(cmd1, cmd2, ... [, opts]) {|first_stdin, wait_threads| - # ... - # } + # Basically a wrapper for + # {Process.spawn}[rdoc-ref:Process.spawn] + # that: # - # first_stdin, wait_threads = Open3.pipeline_w(cmd1, cmd2, ... [, opts]) - # ... - # first_stdin.close + # - Creates a child process for each of the given +cmds+ + # by calling Process.spawn. + # - Pipes the +stdout+ from each child to the +stdin+ of the next child, + # or, for the first child, pipes the caller's +stdout+ to the child's +stdin+. # - # Each cmd is a string or an array. - # If it is an array, the elements are passed to Process.spawn. + # The method does not wait for child processes to exit, + # so the caller must do so. # - # cmd: - # commandline command line string which is passed to a shell - # [env, commandline, opts] command line string which is passed to a shell - # [env, cmdname, arg1, ..., opts] command name and one or more arguments (no shell) - # [env, [cmdname, argv0], arg1, ..., opts] command name and arguments including argv[0] (no shell) + # With no block given, returns a 2-element array containing: # - # Note that env and opts are optional, as for Process.spawn. + # - The +stdin+ stream of the first child process. + # - An array of the wait threads for all of the child processes. # # Example: # - # Open3.pipeline_w("bzip2 -c", :out=>"/tmp/hello.bz2") {|i, ts| - # i.puts "hello" - # } + # first_stdin, wait_threads = Open3.pipeline_w('sort', 'cat -n') + # # => [#<IO:fd 7>, [#<Process::Waiter:0x000055e8de928278 run>, #<Process::Waiter:0x000055e8de923e80 run>]] + # first_stdin.puts("foo\nbar\nbaz") + # first_stdin.close # Send EOF to sort. + # wait_threads.each do |wait_thread| + # wait_thread.join + # end + # + # Output: + # + # 1 bar + # 2 baz + # 3 foo + # + # With a block given, calls the block with the +stdin+ stream + # of the first child process, + # and an array of the wait processes: + # + # Open3.pipeline_w('sort', 'cat -n') do |first_stdin, wait_threads| + # first_stdin.puts("foo\nbar\nbaz") + # first_stdin.close # Send EOF to sort. + # wait_threads.each do |wait_thread| + # wait_thread.join + # end + # end + # + # Output: + # + # 1 bar + # 2 baz + # 3 foo + # + # Like Process.spawn, this method has potential security vulnerabilities + # if called with untrusted input; + # see {Command Injection}[rdoc-ref:command_injection.rdoc@Command+Injection]. + # + # If the first argument is a hash, it becomes leading argument +env+ + # in each call to Process.spawn; + # see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. + # + # If the last argument is a hash, it becomes trailing argument +options+ + # in each call to Process.spawn; + # see {Execution Options}[rdoc-ref:Process@Execution+Options]. + # + # Each remaining argument in +cmds+ is one of: + # + # - A +command_line+: a string that begins with a shell reserved word + # or special built-in, or contains one or more metacharacters. + # - An +exe_path+: the string path to an executable to be called. + # - An array containing a +command_line+ or an +exe_path+, + # along with zero or more string arguments for the command. + # + # See {Argument command_line or exe_path}[rdoc-ref:Process@Argument+command_line+or+exe_path]. # def pipeline_w(*cmds, &block) if Hash === cmds.last @@ -573,49 +1207,67 @@ module Open3 end module_function :pipeline_w - # Open3.pipeline_start starts a list of commands as a pipeline. - # No pipes are created for stdin of the first command and - # stdout of the last command. + # :call-seq: + # Open3.pipeline_start([env, ] *cmds, options = {}) -> [wait_threads] # - # Open3.pipeline_start(cmd1, cmd2, ... [, opts]) {|wait_threads| - # ... - # } + # Basically a wrapper for + # {Process.spawn}[rdoc-ref:Process.spawn] + # that: # - # wait_threads = Open3.pipeline_start(cmd1, cmd2, ... [, opts]) - # ... + # - Creates a child process for each of the given +cmds+ + # by calling Process.spawn. + # - Does not wait for child processes to exit. # - # Each cmd is a string or an array. - # If it is an array, the elements are passed to Process.spawn. + # With no block given, returns an array of the wait threads + # for all of the child processes. # - # cmd: - # commandline command line string which is passed to a shell - # [env, commandline, opts] command line string which is passed to a shell - # [env, cmdname, arg1, ..., opts] command name and one or more arguments (no shell) - # [env, [cmdname, argv0], arg1, ..., opts] command name and arguments including argv[0] (no shell) + # Example: # - # Note that env and opts are optional, as for Process.spawn. + # wait_threads = Open3.pipeline_start('ls', 'grep R') + # # => [#<Process::Waiter:0x000055e8de9d2bb0 run>, #<Process::Waiter:0x000055e8de9d2890 run>] + # wait_threads.each do |wait_thread| + # wait_thread.join + # end # - # Example: + # Output: + # + # Rakefile + # README.md + # + # With a block given, calls the block with an array of the wait processes: + # + # Open3.pipeline_start('ls', 'grep R') do |wait_threads| + # wait_threads.each do |wait_thread| + # wait_thread.join + # end + # end + # + # Output: + # + # Rakefile + # README.md # - # # Run xeyes in 10 seconds. - # Open3.pipeline_start("xeyes") {|ts| - # sleep 10 - # t = ts[0] - # Process.kill("TERM", t.pid) - # p t.value #=> #<Process::Status: pid 911 SIGTERM (signal 15)> - # } - # - # # Convert pdf to ps and send it to a printer. - # # Collect error message of pdftops and lpr. - # pdf_file = "paper.pdf" - # printer = "printer-name" - # err_r, err_w = IO.pipe - # Open3.pipeline_start(["pdftops", pdf_file, "-"], - # ["lpr", "-P#{printer}"], - # :err=>err_w) {|ts| - # err_w.close - # p err_r.read # error messages of pdftops and lpr. - # } + # Like Process.spawn, this method has potential security vulnerabilities + # if called with untrusted input; + # see {Command Injection}[rdoc-ref:command_injection.rdoc@Command+Injection]. + # + # If the first argument is a hash, it becomes leading argument +env+ + # in each call to Process.spawn; + # see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. + # + # If the last argument is a hash, it becomes trailing argument +options+ + # in each call to Process.spawn; + # see {Execution Options}[rdoc-ref:Process@Execution+Options]. + # + # Each remaining argument in +cmds+ is one of: + # + # - A +command_line+: a string that begins with a shell reserved word + # or special built-in, or contains one or more metacharacters. + # - An +exe_path+: the string path to an executable to be called. + # - An array containing a +command_line+ or an +exe_path+, + # along with zero or more string arguments for the command. + # + # See {Argument command_line or exe_path}[rdoc-ref:Process@Argument+command_line+or+exe_path]. # def pipeline_start(*cmds, &block) if Hash === cmds.last @@ -633,57 +1285,51 @@ module Open3 end module_function :pipeline_start - # Open3.pipeline starts a list of commands as a pipeline. - # It waits for the completion of the commands. - # No pipes are created for stdin of the first command and - # stdout of the last command. + # :call-seq: + # Open3.pipeline([env, ] *cmds, options = {}) -> array_of_statuses # - # status_list = Open3.pipeline(cmd1, cmd2, ... [, opts]) + # Basically a wrapper for + # {Process.spawn}[rdoc-ref:Process.spawn] + # that: # - # Each cmd is a string or an array. - # If it is an array, the elements are passed to Process.spawn. + # - Creates a child process for each of the given +cmds+ + # by calling Process.spawn. + # - Pipes the +stdout+ from each child to the +stdin+ of the next child, + # or, for the last child, to the caller's +stdout+. + # - Waits for the child processes to exit. + # - Returns an array of Process::Status objects (one for each child). # - # cmd: - # commandline command line string which is passed to a shell - # [env, commandline, opts] command line string which is passed to a shell - # [env, cmdname, arg1, ..., opts] command name and one or more arguments (no shell) - # [env, [cmdname, argv0], arg1, ..., opts] command name and arguments including argv[0] (no shell) + # Example: # - # Note that env and opts are optional, as Process.spawn. + # wait_threads = Open3.pipeline('ls', 'grep R') + # # => [#<Process::Status: pid 2139200 exit 0>, #<Process::Status: pid 2139202 exit 0>] # - # Example: + # Output: + # + # Rakefile + # README.md + # + # Like Process.spawn, this method has potential security vulnerabilities + # if called with untrusted input; + # see {Command Injection}[rdoc-ref:command_injection.rdoc@Command+Injection]. + # + # If the first argument is a hash, it becomes leading argument +env+ + # in each call to Process.spawn; + # see {Execution Environment}[rdoc-ref:Process@Execution+Environment]. + # + # If the last argument is a hash, it becomes trailing argument +options+ + # in each call to Process.spawn' + # see {Execution Options}[rdoc-ref:Process@Execution+Options]. + # + # Each remaining argument in +cmds+ is one of: + # + # - A +command_line+: a string that begins with a shell reserved word + # or special built-in, or contains one or more metacharacters. + # - An +exe_path+: the string path to an executable to be called. + # - An array containing a +command_line+ or an +exe_path+, + # along with zero or more string arguments for the command. # - # fname = "/usr/share/man/man1/ruby.1.gz" - # p Open3.pipeline(["zcat", fname], "nroff -man", "less") - # #=> [#<Process::Status: pid 11817 exit 0>, - # # #<Process::Status: pid 11820 exit 0>, - # # #<Process::Status: pid 11828 exit 0>] - # - # fname = "/usr/share/man/man1/ls.1.gz" - # Open3.pipeline(["zcat", fname], "nroff -man", "colcrt") - # - # # convert PDF to PS and send to a printer by lpr - # pdf_file = "paper.pdf" - # printer = "printer-name" - # Open3.pipeline(["pdftops", pdf_file, "-"], - # ["lpr", "-P#{printer}"]) - # - # # count lines - # Open3.pipeline("sort", "uniq -c", :in=>"names.txt", :out=>"count") - # - # # cyclic pipeline - # r,w = IO.pipe - # w.print "ibase=14\n10\n" - # Open3.pipeline("bc", "tee /dev/tty", :in=>r, :out=>w) - # #=> 14 - # # 18 - # # 22 - # # 30 - # # 42 - # # 58 - # # 78 - # # 106 - # # 202 + # See {Argument command_line or exe_path}[rdoc-ref:Process@Argument+command_line+or+exe_path]. # def pipeline(*cmds) if Hash === cmds.last diff --git a/lib/open3/version.rb b/lib/open3/version.rb index b6b6ee2c9c..bfcec44ccc 100644 --- a/lib/open3/version.rb +++ b/lib/open3/version.rb @@ -1,3 +1,3 @@ module Open3 - VERSION = "0.1.2" + VERSION = "0.2.1" end diff --git a/lib/optparse.rb b/lib/optparse.rb index 28b64af613..069c3e436e 100644 --- a/lib/optparse.rb +++ b/lib/optparse.rb @@ -8,7 +8,6 @@ # See OptionParser for documentation. # - #-- # == Developer Documentation (not for RDoc output) # @@ -48,7 +47,7 @@ # # == OptionParser # -# === New to \OptionParser? +# === New to +OptionParser+? # # See the {Tutorial}[optparse/tutorial.rdoc]. # @@ -425,7 +424,8 @@ # If you have any questions, file a ticket at http://bugs.ruby-lang.org. # class OptionParser - OptionParser::Version = "0.4.0.pre.1" + # The version string + OptionParser::Version = "0.5.0" # :stopdoc: NoArgument = [NO_ARGUMENT = :NONE, nil].freeze @@ -438,6 +438,8 @@ class OptionParser # and resolved against a list of acceptable values. # module Completion + # :nodoc: + def self.regexp(key, icase) Regexp.new('\A' + Regexp.quote(key).gsub(/\w+\b/, '\&\w*'), icase) end @@ -459,7 +461,7 @@ class OptionParser candidates end - def candidate(key, icase = false, pat = nil) + def candidate(key, icase = false, pat = nil, &_) Completion.candidate(key, icase, pat, &method(:each)) end @@ -510,6 +512,8 @@ class OptionParser # RequiredArgument, etc. # class Switch + # :nodoc: + attr_reader :pattern, :conv, :short, :long, :arg, :desc, :block # @@ -697,6 +701,11 @@ class OptionParser q.object_group(self) {pretty_print_contents(q)} end + def omitted_argument(val) # :nodoc: + val.pop if val.size == 3 and val.last.nil? + val + end + # # Switch that takes no arguments. # @@ -710,10 +719,10 @@ class OptionParser conv_arg(arg) end - def self.incompatible_argument_styles(*) + def self.incompatible_argument_styles(*) # :nodoc: end - def self.pattern + def self.pattern # :nodoc: Object end @@ -730,7 +739,7 @@ class OptionParser # # Raises an exception if argument is not present. # - def parse(arg, argv) + def parse(arg, argv, &_) unless arg raise MissingArgument if argv.empty? arg = argv.shift @@ -755,7 +764,7 @@ class OptionParser if arg conv_arg(*parse_arg(arg, &error)) else - conv_arg(arg) + omitted_argument conv_arg(arg) end end @@ -774,13 +783,14 @@ class OptionParser # def parse(arg, argv, &error) if !(val = arg) and (argv.empty? or /\A-./ =~ (val = argv[0])) - return nil, block, nil + return nil, block end opt = (val = parse_arg(val, &error))[1] val = conv_arg(*val) if opt and !arg argv.shift else + omitted_argument val val[0] = nil end val @@ -798,6 +808,8 @@ class OptionParser # matching pattern and converter pair. Also provides summary feature. # class List + # :nodoc: + # Map from acceptable argument types to pattern and converter pairs. attr_reader :atype @@ -837,7 +849,7 @@ class OptionParser def accept(t, pat = /.*/m, &block) if pat pat.respond_to?(:match) or - raise TypeError, "has no `match'", ParseError.filter_backtrace(caller(2)) + raise TypeError, "has no 'match'", ParseError.filter_backtrace(caller(2)) else pat = t if t.respond_to?(:match) end @@ -1033,11 +1045,31 @@ XXX to << "#compdef #{name}\n" to << COMPSYS_HEADER visit(:compsys, {}, {}) {|o, d| - to << %Q[ "#{o}[#{d.gsub(/[\"\[\]]/, '\\\\\&')}]" \\\n] + to << %Q[ "#{o}[#{d.gsub(/[\\\"\[\]]/, '\\\\\&')}]" \\\n] } to << " '*:file:_files' && return 0\n" end + def help_exit + if STDOUT.tty? && (pager = ENV.values_at(*%w[RUBY_PAGER PAGER]).find {|e| e && !e.empty?}) + less = ENV["LESS"] + args = [{"LESS" => "#{!less || less.empty? ? '-' : less}Fe"}, pager, "w"] + print = proc do |f| + f.puts help + rescue Errno::EPIPE + # pager terminated + end + if Process.respond_to?(:fork) and false + IO.popen("-") {|f| f ? Process.exec(*args, in: f) : print.call(STDOUT)} + # unreachable + end + IO.popen(*args, &print) + else + puts help + end + exit + end + # # Default options for ARGV, which never appear in option summary. # @@ -1049,8 +1081,7 @@ XXX # Officious['help'] = proc do |parser| Switch::NoArgument.new do |arg| - puts parser.help - exit + parser.help_exit end end @@ -1129,6 +1160,10 @@ XXX default.to_i + 1 end end + + # + # See self.inc + # def inc(*args) self.class.inc(*args) end @@ -1167,11 +1202,19 @@ XXX def terminate(arg = nil) self.class.terminate(arg) end + # + # See #terminate. + # def self.terminate(arg = nil) throw :terminate, arg end @stack = [DefaultList] + # + # Returns the global top option list. + # + # Do not use directly. + # def self.top() DefaultList end # @@ -1192,9 +1235,9 @@ XXX # # Directs to reject specified class argument. # - # +t+:: Argument class specifier, any object including Class. + # +type+:: Argument class specifier, any object including Class. # - # reject(t) + # reject(type) # def reject(*args, &blk) top.reject(*args, &blk) end # @@ -1284,10 +1327,24 @@ XXX end end + # + # Shows warning message with the program name + # + # +mesg+:: Message, defaulted to +$!+. + # + # See Kernel#warn. + # def warn(mesg = $!) super("#{program_name}: #{mesg}") end + # + # Shows message with the program name then aborts. + # + # +mesg+:: Message, defaulted to +$!+. + # + # See Kernel#abort. + # def abort(mesg = $!) super("#{program_name}: #{mesg}") end @@ -1309,6 +1366,9 @@ XXX # # Pushes a new List. # + # If a block is given, yields +self+ and returns the result of the + # block, otherwise returns +self+. + # def new @stack.push(List.new) if block_given? @@ -1532,6 +1592,12 @@ XXX nolong end + # ---- + # Option definition phase methods + # + # These methods are used to define options, or to construct an + # OptionParser instance in other words. + # :call-seq: # define(*params, &block) # @@ -1607,6 +1673,13 @@ XXX top.append(string, nil, nil) end + # ---- + # Arguments parse phase methods + # + # These methods parse +argv+, convert, and store the results by + # calling handlers. As these methods do not modify +self+, +self+ + # can be frozen. + # # Parses command line arguments +argv+ in order. When a block is given, # each non-option argument is yielded. When optional +into+ keyword @@ -1616,21 +1689,21 @@ XXX # # Returns the rest of +argv+ left unparsed. # - def order(*argv, into: nil, &nonopt) + def order(*argv, **keywords, &nonopt) argv = argv[0].dup if argv.size == 1 and Array === argv[0] - order!(argv, into: into, &nonopt) + order!(argv, **keywords, &nonopt) end # # Same as #order, but removes switches destructively. # Non-option arguments remain in +argv+. # - def order!(argv = default_argv, into: nil, &nonopt) + def order!(argv = default_argv, into: nil, **keywords, &nonopt) setter = ->(name, val) {into[name.to_sym] = val} if into - parse_in_order(argv, setter, &nonopt) + parse_in_order(argv, setter, **keywords, &nonopt) end - def parse_in_order(argv = default_argv, setter = nil, &nonopt) # :nodoc: + def parse_in_order(argv = default_argv, setter = nil, exact: require_exact, **, &nonopt) # :nodoc: opt, arg, val, rest = nil nonopt ||= proc {|a| throw :terminate, a} argv.unshift(arg) if arg = catch(:terminate) { @@ -1641,19 +1714,24 @@ XXX opt, rest = $1, $2 opt.tr!('_', '-') begin - sw, = complete(:long, opt, true) - if require_exact && !sw.long.include?(arg) - throw :terminate, arg unless raise_unknown - raise InvalidOption, arg + if exact + sw, = search(:long, opt) + else + sw, = complete(:long, opt, true) end rescue ParseError throw :terminate, arg unless raise_unknown raise $!.set_option(arg, true) + else + unless sw + throw :terminate, arg unless raise_unknown + raise InvalidOption, arg + end end begin - opt, cb, val = sw.parse(rest, argv) {|*exc| raise(*exc)} - val = cb.call(val) if cb - setter.call(sw.switch_name, val) if setter + opt, cb, *val = sw.parse(rest, argv) {|*exc| raise(*exc)} + val = callback!(cb, 1, *val) if cb + callback!(setter, 2, sw.switch_name, *val) if setter rescue ParseError raise $!.set_option(arg, rest) end @@ -1671,7 +1749,7 @@ XXX val = arg.delete_prefix('-') has_arg = true rescue InvalidOption - raise if require_exact + raise if exact # if no short options match, try completion with long # options. sw, = complete(:long, opt) @@ -1683,7 +1761,7 @@ XXX raise $!.set_option(arg, true) end begin - opt, cb, val = sw.parse(val, argv) {|*exc| raise(*exc) if eq} + opt, cb, *val = sw.parse(val, argv) {|*exc| raise(*exc) if eq} rescue ParseError raise $!.set_option(arg, arg.length > 2) else @@ -1691,8 +1769,8 @@ XXX end begin argv.unshift(opt) if opt and (!rest or (opt = opt.sub(/\A-*/, '-')) != '-') - val = cb.call(val) if cb - setter.call(sw.switch_name, val) if setter + val = callback!(cb, 1, *val) if cb + callback!(setter, 2, sw.switch_name, *val) if setter rescue ParseError raise $!.set_option(arg, arg.length > 2) end @@ -1718,6 +1796,17 @@ XXX end private :parse_in_order + # Calls callback with _val_. + def callback!(cb, max_arity, *args) # :nodoc: + if (size = args.size) < max_arity and cb.to_proc.lambda? + (arity = cb.arity) < 0 and arity = (1-arity) + arity = max_arity if arity > max_arity + args[arity - 1] = nil if arity > size + end + cb.call(*args) + end + private :callback! + # # Parses command line arguments +argv+ in permutation mode and returns # list of non-option arguments. When optional +into+ keyword @@ -1725,18 +1814,18 @@ XXX # <code>[]=</code> method (so it can be Hash, or OpenStruct, or other # similar object). # - def permute(*argv, into: nil) + def permute(*argv, **keywords) argv = argv[0].dup if argv.size == 1 and Array === argv[0] - permute!(argv, into: into) + permute!(argv, **keywords) end # # Same as #permute, but removes switches destructively. # Non-option arguments remain in +argv+. # - def permute!(argv = default_argv, into: nil) + def permute!(argv = default_argv, **keywords) nonopts = [] - order!(argv, into: into, &nonopts.method(:<<)) + order!(argv, **keywords, &nonopts.method(:<<)) argv[0, 0] = nonopts argv end @@ -1748,20 +1837,20 @@ XXX # values are stored there via <code>[]=</code> method (so it can be Hash, # or OpenStruct, or other similar object). # - def parse(*argv, into: nil) + def parse(*argv, **keywords) argv = argv[0].dup if argv.size == 1 and Array === argv[0] - parse!(argv, into: into) + parse!(argv, **keywords) end # # Same as #parse, but removes switches destructively. # Non-option arguments remain in +argv+. # - def parse!(argv = default_argv, into: nil) + def parse!(argv = default_argv, **keywords) if ENV.include?('POSIXLY_CORRECT') - order!(argv, into: into) + order!(argv, **keywords) else - permute!(argv, into: into) + permute!(argv, **keywords) end end @@ -1784,7 +1873,7 @@ XXX # # params[:bar] = "x" # --bar x # # params[:zot] = "z" # --zot Z # - def getopts(*args, symbolize_names: false) + def getopts(*args, symbolize_names: false, **keywords) argv = Array === args.first ? args.shift : default_argv single_options, *long_options = *args @@ -1812,7 +1901,7 @@ XXX end end - parse_in_order(argv, result.method(:[]=)) + parse_in_order(argv, result.method(:[]=), **keywords) symbolize_names ? result.transform_keys(&:to_sym) : result end @@ -1881,6 +1970,9 @@ XXX DidYouMean.formatter.message_for(all_candidates & checker.correct(opt)) end + # + # Return candidates for +word+. + # def candidate(word) list = [] case word @@ -1922,10 +2014,10 @@ XXX # The optional +into+ keyword argument works exactly like that accepted in # method #parse. # - def load(filename = nil, into: nil) + def load(filename = nil, **keywords) unless filename basename = File.basename($0, '.*') - return true if load(File.expand_path(basename, '~/.options'), into: into) rescue nil + return true if load(File.expand_path(basename, '~/.options'), **keywords) rescue nil basename << ".options" return [ # XDG @@ -1937,11 +2029,11 @@ XXX '~/config/settings', ].any? {|dir| next if !dir or dir.empty? - load(File.expand_path(basename, dir), into: into) rescue nil + load(File.expand_path(basename, dir), **keywords) rescue nil } end begin - parse(*File.readlines(filename, chomp: true), into: into) + parse(*File.readlines(filename, chomp: true), **keywords) true rescue Errno::ENOENT, Errno::ENOTDIR false @@ -1954,10 +2046,10 @@ XXX # # +env+ defaults to the basename of the program. # - def environment(env = File.basename($0, '.*')) + def environment(env = File.basename($0, '.*'), **keywords) env = ENV[env] || ENV[env.upcase] or return require 'shellwords' - parse(*Shellwords.shellwords(env)) + parse(*Shellwords.shellwords(env), **keywords) end # @@ -2123,6 +2215,7 @@ XXX # Reason which caused the error. Reason = 'parse error' + # :nodoc: def initialize(*args, additional: nil) @additional = additional @arg0, = args @@ -2273,19 +2366,19 @@ XXX # Parses +self+ destructively in order and returns +self+ containing the # rest arguments left unparsed. # - def order!(&blk) options.order!(self, &blk) end + def order!(**keywords, &blk) options.order!(self, **keywords, &blk) end # # Parses +self+ destructively in permutation mode and returns +self+ # containing the rest arguments left unparsed. # - def permute!() options.permute!(self) end + def permute!(**keywords) options.permute!(self, **keywords) end # # Parses +self+ destructively and returns +self+ containing the # rest arguments left unparsed. # - def parse!() options.parse!(self) end + def parse!(**keywords) options.parse!(self, **keywords) end # # Substitution of getopts is possible as follows. Also see @@ -2298,8 +2391,8 @@ XXX # rescue OptionParser::ParseError # end # - def getopts(*args, symbolize_names: false) - options.getopts(self, *args, symbolize_names: symbolize_names) + def getopts(*args, symbolize_names: false, **keywords) + options.getopts(self, *args, symbolize_names: symbolize_names, **keywords) end # @@ -2309,7 +2402,8 @@ XXX super obj.instance_eval {@optparse = nil} end - def initialize(*args) + + def initialize(*args) # :nodoc: super @optparse = nil end diff --git a/lib/optparse/ac.rb b/lib/optparse/ac.rb index 0953725e46..23fc740d10 100644 --- a/lib/optparse/ac.rb +++ b/lib/optparse/ac.rb @@ -1,7 +1,11 @@ # frozen_string_literal: false require_relative '../optparse' +# +# autoconf-like options. +# class OptionParser::AC < OptionParser + # :stopdoc: private def _check_ac_args(name, block) @@ -14,6 +18,7 @@ class OptionParser::AC < OptionParser end ARG_CONV = proc {|val| val.nil? ? true : val} + private_constant :ARG_CONV def _ac_arg_enable(prefix, name, help_string, block) _check_ac_args(name, block) @@ -29,16 +34,27 @@ class OptionParser::AC < OptionParser enable end + # :startdoc: + public + # Define <tt>--enable</tt> / <tt>--disable</tt> style option + # + # Appears as <tt>--enable-<i>name</i></tt> in help message. def ac_arg_enable(name, help_string, &block) _ac_arg_enable("enable", name, help_string, block) end + # Define <tt>--enable</tt> / <tt>--disable</tt> style option + # + # Appears as <tt>--disable-<i>name</i></tt> in help message. def ac_arg_disable(name, help_string, &block) _ac_arg_enable("disable", name, help_string, block) end + # Define <tt>--with</tt> / <tt>--without</tt> style option + # + # Appears as <tt>--with-<i>name</i></tt> in help message. def ac_arg_with(name, help_string, &block) _check_ac_args(name, block) diff --git a/lib/optparse/kwargs.rb b/lib/optparse/kwargs.rb index 992aca467e..59a2f61544 100644 --- a/lib/optparse/kwargs.rb +++ b/lib/optparse/kwargs.rb @@ -7,12 +7,17 @@ class OptionParser # # :include: ../../doc/optparse/creates_option.rdoc # - def define_by_keywords(options, meth, **opts) - meth.parameters.each do |type, name| + # Defines options which set in to _options_ for keyword parameters + # of _method_. + # + # Parameters for each keywords are given as elements of _params_. + # + def define_by_keywords(options, method, **params) + method.parameters.each do |type, name| case type when :key, :keyreq op, cl = *(type == :key ? %w"[ ]" : ["", ""]) - define("--#{name}=#{op}#{name.upcase}#{cl}", *opts[name]) do |o| + define("--#{name}=#{op}#{name.upcase}#{cl}", *params[name]) do |o| options[name] = o end end diff --git a/lib/optparse/optparse.gemspec b/lib/optparse/optparse.gemspec index a4287ddeee..1aa54aa781 100644 --- a/lib/optparse/optparse.gemspec +++ b/lib/optparse/optparse.gemspec @@ -22,7 +22,8 @@ Gem::Specification.new do |spec| spec.metadata["homepage_uri"] = spec.homepage spec.metadata["source_code_uri"] = spec.homepage - spec.files = Dir["{doc,lib,misc}/**/*"] + %w[README.md ChangeLog COPYING] + spec.files = Dir["{doc,lib,misc}/**/{*,.document}"] + + %w[README.md ChangeLog COPYING .document .rdoc_options] spec.rdoc_options = ["--main=README.md", "--op=rdoc", "--page-dir=doc"] spec.bindir = "exe" spec.executables = [] diff --git a/lib/optparse/version.rb b/lib/optparse/version.rb index b869d8fe51..b5ed695146 100644 --- a/lib/optparse/version.rb +++ b/lib/optparse/version.rb @@ -2,6 +2,11 @@ # OptionParser internal utility class << OptionParser + # + # Shows version string in packages if Version is defined. + # + # +pkgs+:: package list + # def show_version(*pkgs) progname = ARGV.options.program_name result = false @@ -47,6 +52,8 @@ class << OptionParser result end + # :stopdoc: + def each_const(path, base = ::Object) path.split(/::|\//).inject(base) do |klass, name| raise NameError, path unless Module === klass @@ -68,4 +75,6 @@ class << OptionParser end end end + + # :startdoc: end diff --git a/lib/ostruct.gemspec b/lib/ostruct.gemspec index 21cce18226..08a7aefb05 100644 --- a/lib/ostruct.gemspec +++ b/lib/ostruct.gemspec @@ -21,7 +21,4 @@ Gem::Specification.new do |spec| spec.files = [".gitignore", "Gemfile", "LICENSE.txt", "README.md", "Rakefile", "bin/console", "bin/setup", "lib/ostruct.rb", "ostruct.gemspec"] spec.require_paths = ["lib"] - - spec.add_development_dependency "bundler" - spec.add_development_dependency "rake" end diff --git a/lib/ostruct.rb b/lib/ostruct.rb index a08561d6c9..c762baa5a5 100644 --- a/lib/ostruct.rb +++ b/lib/ostruct.rb @@ -107,7 +107,15 @@ # For all these reasons, consider not using OpenStruct at all. # class OpenStruct - VERSION = "0.5.5" + VERSION = "0.6.0" + + HAS_PERFORMANCE_WARNINGS = begin + Warning[:performance] + true + rescue NoMethodError, ArgumentError + false + end + private_constant :HAS_PERFORMANCE_WARNINGS # # Creates a new OpenStruct object. By default, the resulting OpenStruct @@ -124,6 +132,10 @@ class OpenStruct # data # => #<OpenStruct country="Australia", capital="Canberra"> # def initialize(hash=nil) + if HAS_PERFORMANCE_WARNINGS && Warning[:performance] + warn "OpenStruct use is discouraged for performance reasons", uplevel: 1, category: :performance + end + if hash update_to_values!(hash) else @@ -364,7 +376,7 @@ class OpenStruct end @table.delete(sym) do return yield if block - raise! NameError.new("no field `#{sym}' in #{self}", sym) + raise! NameError.new("no field '#{sym}' in #{self}", sym) end end @@ -46,6 +46,7 @@ require 'prettyprint' # # To define a customized pretty printing function for your classes, # redefine method <code>#pretty_print(pp)</code> in the class. +# Note that <code>require 'pp'</code> is needed before redefining <code>#pretty_print(pp)</code>. # # <code>#pretty_print</code> takes the +pp+ argument, which is an instance of the PP class. # The method uses #text, #breakable, #nest, #group and #pp to print the @@ -62,7 +63,7 @@ require 'prettyprint' class PP < PrettyPrint - VERSION = "0.4.0" + VERSION = "0.5.0" # Returns the usable width for +out+. # As the width of +out+: @@ -92,7 +93,7 @@ class PP < PrettyPrint # # PP.pp returns +out+. def PP.pp(obj, out=$>, width=width_for(out)) - q = PP.new(out, width) + q = new(out, width) q.guard_inspect_key {q.pp obj} q.flush #$pp = q @@ -285,16 +286,21 @@ class PP < PrettyPrint group(1, '{', '}') { seplist(obj, nil, :each_pair) {|k, v| group { - pp k - text '=>' - group(1) { - breakable '' - pp v - } + pp_hash_pair k, v } } } end + + # A pretty print for a pair of Hash + def pp_hash_pair(k, v) + pp k + text '=>' + group(1) { + breakable '' + pp v + } + end end include PPMethods @@ -421,8 +427,10 @@ end class Data # :nodoc: def pretty_print(q) # :nodoc: - q.group(1, sprintf("#<data %s", PP.mcall(self, Kernel, :class).name), '>') { - q.seplist(PP.mcall(self, Data, :members), lambda { q.text "," }) {|member| + class_name = PP.mcall(self, Kernel, :class).name + class_name = " #{class_name}" if class_name + q.group(1, "#<data#{class_name}", '>') { + q.seplist(PP.mcall(self, Kernel, :class).members, lambda { q.text "," }) {|member| q.breakable q.text member.to_s q.text '=' @@ -437,11 +445,11 @@ class Data # :nodoc: def pretty_print_cycle(q) # :nodoc: q.text sprintf("#<data %s:...>", PP.mcall(self, Kernel, :class).name) end -end if "3.2" <= RUBY_VERSION +end if defined?(Data.define) class Range # :nodoc: def pretty_print(q) # :nodoc: - q.pp self.begin + q.pp self.begin if self.begin q.breakable '' q.text(self.exclude_end? ? '...' : '..') q.breakable '' @@ -632,10 +640,6 @@ end module Kernel # Returns a pretty printed object as a string. # - # In order to use this method you must first require the PP module: - # - # require 'pp' - # # See the PP module for more information. def pretty_inspect PP.pp(self, ''.dup) diff --git a/lib/prettyprint.rb b/lib/prettyprint.rb index 540b12fb70..6f50192f5d 100644 --- a/lib/prettyprint.rb +++ b/lib/prettyprint.rb @@ -23,17 +23,17 @@ # # == References # Christian Lindig, Strictly Pretty, March 2000, -# http://www.st.cs.uni-sb.de/~lindig/papers/#pretty +# https://lindig.github.io/papers/strictly-pretty-2000.pdf # # Philip Wadler, A prettier printer, March 1998, -# http://homepages.inf.ed.ac.uk/wadler/topics/language-design.html#prettier +# https://homepages.inf.ed.ac.uk/wadler/topics/language-design.html#prettier # # == Author # Tanaka Akira <akr@fsij.org> # class PrettyPrint - VERSION = "0.1.1" + VERSION = "0.2.0" # This is a convenience method which is same as follows: # diff --git a/lib/prism.rb b/lib/prism.rb new file mode 100644 index 0000000000..66a64e7fd0 --- /dev/null +++ b/lib/prism.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +# The Prism Ruby parser. +# +# "Parsing Ruby is suddenly manageable!" +# - You, hopefully +# +module Prism + # There are many files in prism that are templated to handle every node type, + # which means the files can end up being quite large. We autoload them to make + # our require speed faster since consuming libraries are unlikely to use all + # of these features. + + autoload :BasicVisitor, "prism/visitor" + autoload :Compiler, "prism/compiler" + autoload :DesugarCompiler, "prism/desugar_compiler" + autoload :Dispatcher, "prism/dispatcher" + autoload :DotVisitor, "prism/dot_visitor" + autoload :DSL, "prism/dsl" + autoload :InspectVisitor, "prism/inspect_visitor" + autoload :LexCompat, "prism/lex_compat" + autoload :LexRipper, "prism/lex_compat" + autoload :MutationCompiler, "prism/mutation_compiler" + autoload :Pack, "prism/pack" + autoload :Pattern, "prism/pattern" + autoload :Reflection, "prism/reflection" + autoload :Serialize, "prism/serialize" + autoload :Translation, "prism/translation" + autoload :Visitor, "prism/visitor" + + # Some of these constants are not meant to be exposed, so marking them as + # private here. + + private_constant :LexCompat + private_constant :LexRipper + + # :call-seq: + # Prism::lex_compat(source, **options) -> LexCompat::Result + # + # Returns a parse result whose value is an array of tokens that closely + # resembles the return value of Ripper::lex. The main difference is that the + # `:on_sp` token is not emitted. + # + # For supported options, see Prism::parse. + def self.lex_compat(source, **options) + LexCompat.new(source, **options).result # steep:ignore + end + + # :call-seq: + # Prism::lex_ripper(source) -> Array + # + # This lexes with the Ripper lex. It drops any space events but otherwise + # returns the same tokens. Raises SyntaxError if the syntax in source is + # invalid. + def self.lex_ripper(source) + LexRipper.new(source).result # steep:ignore + end + + # :call-seq: + # Prism::load(source, serialized) -> ParseResult + # + # Load the serialized AST using the source as a reference into a tree. + def self.load(source, serialized) + Serialize.load(source, serialized) + end +end + +require_relative "prism/polyfill/byteindex" +require_relative "prism/node" +require_relative "prism/node_ext" +require_relative "prism/parse_result" + +# This is a Ruby implementation of the prism parser. If we're running on CRuby +# and we haven't explicitly set the PRISM_FFI_BACKEND environment variable, then +# it's going to require the built library. Otherwise, it's going to require a +# module that uses FFI to call into the library. +if RUBY_ENGINE == "ruby" and !ENV["PRISM_FFI_BACKEND"] + require "prism/prism" + + # The C extension is the default backend on CRuby. + Prism::BACKEND = :CEXT +else + require_relative "prism/ffi" + + # The FFI backend is used on other Ruby implementations. + Prism::BACKEND = :FFI +end diff --git a/lib/prism/desugar_compiler.rb b/lib/prism/desugar_compiler.rb new file mode 100644 index 0000000000..de02445149 --- /dev/null +++ b/lib/prism/desugar_compiler.rb @@ -0,0 +1,354 @@ +# frozen_string_literal: true + +module Prism + class DesugarAndWriteNode # :nodoc: + attr_reader :node, :source, :read_class, :write_class, :arguments + + def initialize(node, source, read_class, write_class, *arguments) + @node = node + @source = source + @read_class = read_class + @write_class = write_class + @arguments = arguments + end + + # Desugar `x &&= y` to `x && x = y` + def compile + AndNode.new( + source, + read_class.new(source, *arguments, node.name_loc), + write_class.new(source, *arguments, node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + end + + class DesugarOrWriteDefinedNode # :nodoc: + attr_reader :node, :source, :read_class, :write_class, :arguments + + def initialize(node, source, read_class, write_class, *arguments) + @node = node + @source = source + @read_class = read_class + @write_class = write_class + @arguments = arguments + end + + # Desugar `x ||= y` to `defined?(x) ? x : x = y` + def compile + IfNode.new( + source, + node.operator_loc, + DefinedNode.new(source, nil, read_class.new(source, *arguments, node.name_loc), nil, node.operator_loc, node.name_loc), + node.operator_loc, + StatementsNode.new(source, [read_class.new(source, *arguments, node.name_loc)], node.location), + ElseNode.new( + source, + node.operator_loc, + StatementsNode.new( + source, + [write_class.new(source, *arguments, node.name_loc, node.value, node.operator_loc, node.location)], + node.location + ), + node.operator_loc, + node.location + ), + node.operator_loc, + node.location + ) + end + end + + class DesugarOperatorWriteNode # :nodoc: + attr_reader :node, :source, :read_class, :write_class, :arguments + + def initialize(node, source, read_class, write_class, *arguments) + @node = node + @source = source + @read_class = read_class + @write_class = write_class + @arguments = arguments + end + + # Desugar `x += y` to `x = x + y` + def compile + binary_operator_loc = node.binary_operator_loc.chop + + write_class.new( + source, + *arguments, + node.name_loc, + CallNode.new( + source, + 0, + read_class.new(source, *arguments, node.name_loc), + nil, + binary_operator_loc.slice.to_sym, + binary_operator_loc, + nil, + ArgumentsNode.new(source, 0, [node.value], node.value.location), + nil, + nil, + node.location + ), + node.binary_operator_loc.copy(start_offset: node.binary_operator_loc.end_offset - 1, length: 1), + node.location + ) + end + end + + class DesugarOrWriteNode # :nodoc: + attr_reader :node, :source, :read_class, :write_class, :arguments + + def initialize(node, source, read_class, write_class, *arguments) + @node = node + @source = source + @read_class = read_class + @write_class = write_class + @arguments = arguments + end + + # Desugar `x ||= y` to `x || x = y` + def compile + OrNode.new( + source, + read_class.new(source, *arguments, node.name_loc), + write_class.new(source, *arguments, node.name_loc, node.value, node.operator_loc, node.location), + node.operator_loc, + node.location + ) + end + end + + private_constant :DesugarAndWriteNode, :DesugarOrWriteNode, :DesugarOrWriteDefinedNode, :DesugarOperatorWriteNode + + class ClassVariableAndWriteNode + def desugar # :nodoc: + DesugarAndWriteNode.new(self, source, ClassVariableReadNode, ClassVariableWriteNode, name).compile + end + end + + class ClassVariableOrWriteNode + def desugar # :nodoc: + DesugarOrWriteDefinedNode.new(self, source, ClassVariableReadNode, ClassVariableWriteNode, name).compile + end + end + + class ClassVariableOperatorWriteNode + def desugar # :nodoc: + DesugarOperatorWriteNode.new(self, source, ClassVariableReadNode, ClassVariableWriteNode, name).compile + end + end + + class ConstantAndWriteNode + def desugar # :nodoc: + DesugarAndWriteNode.new(self, source, ConstantReadNode, ConstantWriteNode, name).compile + end + end + + class ConstantOrWriteNode + def desugar # :nodoc: + DesugarOrWriteDefinedNode.new(self, source, ConstantReadNode, ConstantWriteNode, name).compile + end + end + + class ConstantOperatorWriteNode + def desugar # :nodoc: + DesugarOperatorWriteNode.new(self, source, ConstantReadNode, ConstantWriteNode, name).compile + end + end + + class GlobalVariableAndWriteNode + def desugar # :nodoc: + DesugarAndWriteNode.new(self, source, GlobalVariableReadNode, GlobalVariableWriteNode, name).compile + end + end + + class GlobalVariableOrWriteNode + def desugar # :nodoc: + DesugarOrWriteDefinedNode.new(self, source, GlobalVariableReadNode, GlobalVariableWriteNode, name).compile + end + end + + class GlobalVariableOperatorWriteNode + def desugar # :nodoc: + DesugarOperatorWriteNode.new(self, source, GlobalVariableReadNode, GlobalVariableWriteNode, name).compile + end + end + + class InstanceVariableAndWriteNode + def desugar # :nodoc: + DesugarAndWriteNode.new(self, source, InstanceVariableReadNode, InstanceVariableWriteNode, name).compile + end + end + + class InstanceVariableOrWriteNode + def desugar # :nodoc: + DesugarOrWriteNode.new(self, source, InstanceVariableReadNode, InstanceVariableWriteNode, name).compile + end + end + + class InstanceVariableOperatorWriteNode + def desugar # :nodoc: + DesugarOperatorWriteNode.new(self, source, InstanceVariableReadNode, InstanceVariableWriteNode, name).compile + end + end + + class LocalVariableAndWriteNode + def desugar # :nodoc: + DesugarAndWriteNode.new(self, source, LocalVariableReadNode, LocalVariableWriteNode, name, depth).compile + end + end + + class LocalVariableOrWriteNode + def desugar # :nodoc: + DesugarOrWriteNode.new(self, source, LocalVariableReadNode, LocalVariableWriteNode, name, depth).compile + end + end + + class LocalVariableOperatorWriteNode + def desugar # :nodoc: + DesugarOperatorWriteNode.new(self, source, LocalVariableReadNode, LocalVariableWriteNode, name, depth).compile + end + end + + # DesugarCompiler is a compiler that desugars Ruby code into a more primitive + # form. This is useful for consumers that want to deal with fewer node types. + class DesugarCompiler < MutationCompiler + # @@foo &&= bar + # + # becomes + # + # @@foo && @@foo = bar + def visit_class_variable_and_write_node(node) + node.desugar + end + + # @@foo ||= bar + # + # becomes + # + # defined?(@@foo) ? @@foo : @@foo = bar + def visit_class_variable_or_write_node(node) + node.desugar + end + + # @@foo += bar + # + # becomes + # + # @@foo = @@foo + bar + def visit_class_variable_operator_write_node(node) + node.desugar + end + + # Foo &&= bar + # + # becomes + # + # Foo && Foo = bar + def visit_constant_and_write_node(node) + node.desugar + end + + # Foo ||= bar + # + # becomes + # + # defined?(Foo) ? Foo : Foo = bar + def visit_constant_or_write_node(node) + node.desugar + end + + # Foo += bar + # + # becomes + # + # Foo = Foo + bar + def visit_constant_operator_write_node(node) + node.desugar + end + + # $foo &&= bar + # + # becomes + # + # $foo && $foo = bar + def visit_global_variable_and_write_node(node) + node.desugar + end + + # $foo ||= bar + # + # becomes + # + # defined?($foo) ? $foo : $foo = bar + def visit_global_variable_or_write_node(node) + node.desugar + end + + # $foo += bar + # + # becomes + # + # $foo = $foo + bar + def visit_global_variable_operator_write_node(node) + node.desugar + end + + # @foo &&= bar + # + # becomes + # + # @foo && @foo = bar + def visit_instance_variable_and_write_node(node) + node.desugar + end + + # @foo ||= bar + # + # becomes + # + # @foo || @foo = bar + def visit_instance_variable_or_write_node(node) + node.desugar + end + + # @foo += bar + # + # becomes + # + # @foo = @foo + bar + def visit_instance_variable_operator_write_node(node) + node.desugar + end + + # foo &&= bar + # + # becomes + # + # foo && foo = bar + def visit_local_variable_and_write_node(node) + node.desugar + end + + # foo ||= bar + # + # becomes + # + # foo || foo = bar + def visit_local_variable_or_write_node(node) + node.desugar + end + + # foo += bar + # + # becomes + # + # foo = foo + bar + def visit_local_variable_operator_write_node(node) + node.desugar + end + end +end diff --git a/lib/prism/ffi.rb b/lib/prism/ffi.rb new file mode 100644 index 0000000000..df7e183310 --- /dev/null +++ b/lib/prism/ffi.rb @@ -0,0 +1,437 @@ +# frozen_string_literal: true +# typed: ignore + +# This file is responsible for mirroring the API provided by the C extension by +# using FFI to call into the shared library. + +require "rbconfig" +require "ffi" + +module Prism + module LibRubyParser # :nodoc: + extend FFI::Library + + # Define the library that we will be pulling functions from. Note that this + # must align with the build shared library from make/rake. + ffi_lib File.expand_path("../../build/libprism.#{RbConfig::CONFIG["SOEXT"]}", __dir__) + + # Convert a native C type declaration into a symbol that FFI understands. + # For example: + # + # const char * -> :pointer + # bool -> :bool + # size_t -> :size_t + # void -> :void + # + def self.resolve_type(type, callbacks) + type = type.strip + + if !type.end_with?("*") + type.delete_prefix("const ").to_sym + else + type = type.delete_suffix("*").rstrip + callbacks.include?(type.to_sym) ? type.to_sym : :pointer + end + end + + # Read through the given header file and find the declaration of each of the + # given functions. For each one, define a function with the same name and + # signature as the C function. + def self.load_exported_functions_from(header, *functions, callbacks) + File.foreach(File.expand_path("../../include/#{header}", __dir__)) do |line| + # We only want to attempt to load exported functions. + next unless line.start_with?("PRISM_EXPORTED_FUNCTION ") + + # We only want to load the functions that we are interested in. + next unless functions.any? { |function| line.include?(function) } + + # Parse the function declaration. + unless /^PRISM_EXPORTED_FUNCTION (?<return_type>.+) (?<name>\w+)\((?<arg_types>.+)\);$/ =~ line + raise "Could not parse #{line}" + end + + # Delete the function from the list of functions we are looking for to + # mark it as having been found. + functions.delete(name) + + # Split up the argument types into an array, ensure we handle the case + # where there are no arguments (by explicit void). + arg_types = arg_types.split(",").map(&:strip) + arg_types = [] if arg_types == %w[void] + + # Resolve the type of the argument by dropping the name of the argument + # first if it is present. + arg_types.map! { |type| resolve_type(type.sub(/\w+$/, ""), callbacks) } + + # Attach the function using the FFI library. + attach_function name, arg_types, resolve_type(return_type, []) + end + + # If we didn't find all of the functions, raise an error. + raise "Could not find functions #{functions.inspect}" unless functions.empty? + end + + callback :pm_parse_stream_fgets_t, [:pointer, :int, :pointer], :pointer + + load_exported_functions_from( + "prism.h", + "pm_version", + "pm_serialize_parse", + "pm_serialize_parse_stream", + "pm_serialize_parse_comments", + "pm_serialize_lex", + "pm_serialize_parse_lex", + "pm_parse_success_p", + [:pm_parse_stream_fgets_t] + ) + + load_exported_functions_from( + "prism/util/pm_buffer.h", + "pm_buffer_sizeof", + "pm_buffer_init", + "pm_buffer_value", + "pm_buffer_length", + "pm_buffer_free", + [] + ) + + load_exported_functions_from( + "prism/util/pm_string.h", + "pm_string_mapped_init", + "pm_string_free", + "pm_string_source", + "pm_string_length", + "pm_string_sizeof", + [] + ) + + # This object represents a pm_buffer_t. We only use it as an opaque pointer, + # so it doesn't need to know the fields of pm_buffer_t. + class PrismBuffer # :nodoc: + SIZEOF = LibRubyParser.pm_buffer_sizeof + + attr_reader :pointer + + def initialize(pointer) + @pointer = pointer + end + + def value + LibRubyParser.pm_buffer_value(pointer) + end + + def length + LibRubyParser.pm_buffer_length(pointer) + end + + def read + value.read_string(length) + end + + # Initialize a new buffer and yield it to the block. The buffer will be + # automatically freed when the block returns. + def self.with + FFI::MemoryPointer.new(SIZEOF) do |pointer| + raise unless LibRubyParser.pm_buffer_init(pointer) + return yield new(pointer) + ensure + LibRubyParser.pm_buffer_free(pointer) + end + end + end + + # This object represents a pm_string_t. We only use it as an opaque pointer, + # so it doesn't have to be an FFI::Struct. + class PrismString # :nodoc: + SIZEOF = LibRubyParser.pm_string_sizeof + + attr_reader :pointer, :length + + def initialize(pointer, length, from_string) + @pointer = pointer + @length = length + @from_string = from_string + end + + def read + raise "should use the original String instead" if @from_string + @pointer.read_string(@length) + end + + # Yields a pm_string_t pointer to the given block. + def self.with_string(string) + raise TypeError unless string.is_a?(String) + + length = string.bytesize + # + 1 to never get an address of 0, which pm_parser_init() asserts + FFI::MemoryPointer.new(:char, length + 1, false) do |pointer| + pointer.write_string(string) + # since we have the extra byte we might as well \0-terminate + pointer.put_char(length, 0) + return yield new(pointer, length, true) + end + end + + # Yields a pm_string_t pointer to the given block. + def self.with_file(filepath) + raise TypeError unless filepath.is_a?(String) + + FFI::MemoryPointer.new(SIZEOF) do |pm_string| + if LibRubyParser.pm_string_mapped_init(pm_string, filepath) + pointer = LibRubyParser.pm_string_source(pm_string) + length = LibRubyParser.pm_string_length(pm_string) + return yield new(pointer, length, false) + else + raise SystemCallError.new(filepath, FFI.errno) + end + ensure + LibRubyParser.pm_string_free(pm_string) + end + end + end + end + + # Mark the LibRubyParser module as private as it should only be called through + # the prism module. + private_constant :LibRubyParser + + # The version constant is set by reading the result of calling pm_version. + VERSION = LibRubyParser.pm_version.read_string + + class << self + # Mirror the Prism.dump API by using the serialization API. + def dump(code, **options) + LibRubyParser::PrismString.with_string(code) { |string| dump_common(string, options) } + end + + # Mirror the Prism.dump_file API by using the serialization API. + def dump_file(filepath, **options) + options[:filepath] = filepath + LibRubyParser::PrismString.with_file(filepath) { |string| dump_common(string, options) } + end + + # Mirror the Prism.lex API by using the serialization API. + def lex(code, **options) + LibRubyParser::PrismString.with_string(code) { |string| lex_common(string, code, options) } + end + + # Mirror the Prism.lex_file API by using the serialization API. + def lex_file(filepath, **options) + options[:filepath] = filepath + LibRubyParser::PrismString.with_file(filepath) { |string| lex_common(string, string.read, options) } + end + + # Mirror the Prism.parse API by using the serialization API. + def parse(code, **options) + LibRubyParser::PrismString.with_string(code) { |string| parse_common(string, code, options) } + end + + # Mirror the Prism.parse_file API by using the serialization API. This uses + # native strings instead of Ruby strings because it allows us to use mmap + # when it is available. + def parse_file(filepath, **options) + options[:filepath] = filepath + LibRubyParser::PrismString.with_file(filepath) { |string| parse_common(string, string.read, options) } + end + + # Mirror the Prism.parse_stream API by using the serialization API. + def parse_stream(stream, **options) + LibRubyParser::PrismBuffer.with do |buffer| + source = +"" + callback = -> (string, size, _) { + raise "Expected size to be >= 0, got: #{size}" if size <= 0 + + if !(line = stream.gets(size - 1)).nil? + source << line + string.write_string("#{line}\x00", line.bytesize + 1) + end + } + + # In the pm_serialize_parse_stream function it accepts a pointer to the + # IO object as a void* and then passes it through to the callback as the + # third argument, but it never touches it itself. As such, since we have + # access to the IO object already through the closure of the lambda, we + # can pass a null pointer here and not worry. + LibRubyParser.pm_serialize_parse_stream(buffer.pointer, nil, callback, dump_options(options)) + Prism.load(source, buffer.read) + end + end + + # Mirror the Prism.parse_comments API by using the serialization API. + def parse_comments(code, **options) + LibRubyParser::PrismString.with_string(code) { |string| parse_comments_common(string, code, options) } + end + + # Mirror the Prism.parse_file_comments API by using the serialization + # API. This uses native strings instead of Ruby strings because it allows us + # to use mmap when it is available. + def parse_file_comments(filepath, **options) + options[:filepath] = filepath + LibRubyParser::PrismString.with_file(filepath) { |string| parse_comments_common(string, string.read, options) } + end + + # Mirror the Prism.parse_lex API by using the serialization API. + def parse_lex(code, **options) + LibRubyParser::PrismString.with_string(code) { |string| parse_lex_common(string, code, options) } + end + + # Mirror the Prism.parse_lex_file API by using the serialization API. + def parse_lex_file(filepath, **options) + options[:filepath] = filepath + LibRubyParser::PrismString.with_file(filepath) { |string| parse_lex_common(string, string.read, options) } + end + + # Mirror the Prism.parse_success? API by using the serialization API. + def parse_success?(code, **options) + LibRubyParser::PrismString.with_string(code) { |string| parse_file_success_common(string, options) } + end + + # Mirror the Prism.parse_failure? API by using the serialization API. + def parse_failure?(code, **options) + !parse_success?(code, **options) + end + + # Mirror the Prism.parse_file_success? API by using the serialization API. + def parse_file_success?(filepath, **options) + options[:filepath] = filepath + LibRubyParser::PrismString.with_file(filepath) { |string| parse_file_success_common(string, options) } + end + + # Mirror the Prism.parse_file_failure? API by using the serialization API. + def parse_file_failure?(filepath, **options) + !parse_file_success?(filepath, **options) + end + + private + + def dump_common(string, options) # :nodoc: + LibRubyParser::PrismBuffer.with do |buffer| + LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options)) + buffer.read + end + end + + def lex_common(string, code, options) # :nodoc: + serialized = LibRubyParser::PrismBuffer.with do |buffer| + LibRubyParser.pm_serialize_lex(buffer.pointer, string.pointer, string.length, dump_options(options)) + buffer.read + end + + Serialize.load_tokens(Source.for(code), serialized) + end + + def parse_common(string, code, options) # :nodoc: + serialized = dump_common(string, options) + Prism.load(code, serialized) + end + + def parse_comments_common(string, code, options) # :nodoc: + LibRubyParser::PrismBuffer.with do |buffer| + LibRubyParser.pm_serialize_parse_comments(buffer.pointer, string.pointer, string.length, dump_options(options)) + + source = Source.for(code) + loader = Serialize::Loader.new(source, buffer.read) + + loader.load_header + loader.load_encoding + loader.load_start_line + loader.load_comments + end + end + + def parse_lex_common(string, code, options) # :nodoc: + LibRubyParser::PrismBuffer.with do |buffer| + LibRubyParser.pm_serialize_parse_lex(buffer.pointer, string.pointer, string.length, dump_options(options)) + + source = Source.for(code) + loader = Serialize::Loader.new(source, buffer.read) + + tokens = loader.load_tokens + node, comments, magic_comments, data_loc, errors, warnings = loader.load_nodes + tokens.each { |token,| token.value.force_encoding(loader.encoding) } + + ParseLexResult.new([node, tokens], comments, magic_comments, data_loc, errors, warnings, source) + end + end + + def parse_file_success_common(string, options) # :nodoc: + LibRubyParser.pm_parse_success_p(string.pointer, string.length, dump_options(options)) + end + + # Return the value that should be dumped for the command_line option. + def dump_options_command_line(options) + command_line = options.fetch(:command_line, "") + raise ArgumentError, "command_line must be a string" unless command_line.is_a?(String) + + command_line.each_char.inject(0) do |value, char| + case char + when "a" then value | 0b000001 + when "e" then value | 0b000010 + when "l" then value | 0b000100 + when "n" then value | 0b001000 + when "p" then value | 0b010000 + when "x" then value | 0b100000 + else raise ArgumentError, "invalid command_line option: #{char}" + end + end + end + + # Convert the given options into a serialized options string. + def dump_options(options) + template = +"" + values = [] + + template << "L" + if (filepath = options[:filepath]) + values.push(filepath.bytesize, filepath.b) + template << "A*" + else + values << 0 + end + + template << "l" + values << options.fetch(:line, 1) + + template << "L" + if (encoding = options[:encoding]) + name = encoding.is_a?(Encoding) ? encoding.name : encoding + values.push(name.bytesize, name.b) + template << "A*" + else + values << 0 + end + + template << "C" + values << (options.fetch(:frozen_string_literal, false) ? 1 : 0) + + template << "C" + values << dump_options_command_line(options) + + template << "C" + values << { nil => 0, "3.3.0" => 1, "3.3.1" => 1, "3.4.0" => 0, "latest" => 0 }.fetch(options[:version]) + + template << "L" + if (scopes = options[:scopes]) + values << scopes.length + + scopes.each do |scope| + template << "L" + values << scope.length + + scope.each do |local| + name = local.name + template << "L" + values << name.bytesize + + template << "A*" + values << name.b + end + end + else + values << 0 + end + + values.pack(template) + end + end +end diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb new file mode 100644 index 0000000000..4f8e443a3b --- /dev/null +++ b/lib/prism/lex_compat.rb @@ -0,0 +1,927 @@ +# frozen_string_literal: true + +require "delegate" +require "ripper" + +module Prism + # This class is responsible for lexing the source using prism and then + # converting those tokens to be compatible with Ripper. In the vast majority + # of cases, this is a one-to-one mapping of the token type. Everything else + # generally lines up. However, there are a few cases that require special + # handling. + class LexCompat # :nodoc: + # A result class specialized for holding tokens produced by the lexer. + class Result < Prism::Result + # The list of tokens that were produced by the lexer. + attr_reader :value + + # Create a new lex compat result object with the given values. + def initialize(value, comments, magic_comments, data_loc, errors, warnings, source) + @value = value + super(comments, magic_comments, data_loc, errors, warnings, source) + end + + # Implement the hash pattern matching interface for Result. + def deconstruct_keys(keys) + super.merge!(value: value) + end + end + + # This is a mapping of prism token types to Ripper token types. This is a + # many-to-one mapping because we split up our token types, whereas Ripper + # tends to group them. + RIPPER = { + AMPERSAND: :on_op, + AMPERSAND_AMPERSAND: :on_op, + AMPERSAND_AMPERSAND_EQUAL: :on_op, + AMPERSAND_DOT: :on_op, + AMPERSAND_EQUAL: :on_op, + BACK_REFERENCE: :on_backref, + BACKTICK: :on_backtick, + BANG: :on_op, + BANG_EQUAL: :on_op, + BANG_TILDE: :on_op, + BRACE_LEFT: :on_lbrace, + BRACE_RIGHT: :on_rbrace, + BRACKET_LEFT: :on_lbracket, + BRACKET_LEFT_ARRAY: :on_lbracket, + BRACKET_LEFT_RIGHT: :on_op, + BRACKET_LEFT_RIGHT_EQUAL: :on_op, + BRACKET_RIGHT: :on_rbracket, + CARET: :on_op, + CARET_EQUAL: :on_op, + CHARACTER_LITERAL: :on_CHAR, + CLASS_VARIABLE: :on_cvar, + COLON: :on_op, + COLON_COLON: :on_op, + COMMA: :on_comma, + COMMENT: :on_comment, + CONSTANT: :on_const, + DOT: :on_period, + DOT_DOT: :on_op, + DOT_DOT_DOT: :on_op, + EMBDOC_BEGIN: :on_embdoc_beg, + EMBDOC_END: :on_embdoc_end, + EMBDOC_LINE: :on_embdoc, + EMBEXPR_BEGIN: :on_embexpr_beg, + EMBEXPR_END: :on_embexpr_end, + EMBVAR: :on_embvar, + EOF: :on_eof, + EQUAL: :on_op, + EQUAL_EQUAL: :on_op, + EQUAL_EQUAL_EQUAL: :on_op, + EQUAL_GREATER: :on_op, + EQUAL_TILDE: :on_op, + FLOAT: :on_float, + FLOAT_IMAGINARY: :on_imaginary, + FLOAT_RATIONAL: :on_rational, + FLOAT_RATIONAL_IMAGINARY: :on_imaginary, + GREATER: :on_op, + GREATER_EQUAL: :on_op, + GREATER_GREATER: :on_op, + GREATER_GREATER_EQUAL: :on_op, + GLOBAL_VARIABLE: :on_gvar, + HEREDOC_END: :on_heredoc_end, + HEREDOC_START: :on_heredoc_beg, + IDENTIFIER: :on_ident, + IGNORED_NEWLINE: :on_ignored_nl, + INTEGER: :on_int, + INTEGER_IMAGINARY: :on_imaginary, + INTEGER_RATIONAL: :on_rational, + INTEGER_RATIONAL_IMAGINARY: :on_imaginary, + INSTANCE_VARIABLE: :on_ivar, + INVALID: :INVALID, + KEYWORD___ENCODING__: :on_kw, + KEYWORD___LINE__: :on_kw, + KEYWORD___FILE__: :on_kw, + KEYWORD_ALIAS: :on_kw, + KEYWORD_AND: :on_kw, + KEYWORD_BEGIN: :on_kw, + KEYWORD_BEGIN_UPCASE: :on_kw, + KEYWORD_BREAK: :on_kw, + KEYWORD_CASE: :on_kw, + KEYWORD_CLASS: :on_kw, + KEYWORD_DEF: :on_kw, + KEYWORD_DEFINED: :on_kw, + KEYWORD_DO: :on_kw, + KEYWORD_DO_LOOP: :on_kw, + KEYWORD_ELSE: :on_kw, + KEYWORD_ELSIF: :on_kw, + KEYWORD_END: :on_kw, + KEYWORD_END_UPCASE: :on_kw, + KEYWORD_ENSURE: :on_kw, + KEYWORD_FALSE: :on_kw, + KEYWORD_FOR: :on_kw, + KEYWORD_IF: :on_kw, + KEYWORD_IF_MODIFIER: :on_kw, + KEYWORD_IN: :on_kw, + KEYWORD_MODULE: :on_kw, + KEYWORD_NEXT: :on_kw, + KEYWORD_NIL: :on_kw, + KEYWORD_NOT: :on_kw, + KEYWORD_OR: :on_kw, + KEYWORD_REDO: :on_kw, + KEYWORD_RESCUE: :on_kw, + KEYWORD_RESCUE_MODIFIER: :on_kw, + KEYWORD_RETRY: :on_kw, + KEYWORD_RETURN: :on_kw, + KEYWORD_SELF: :on_kw, + KEYWORD_SUPER: :on_kw, + KEYWORD_THEN: :on_kw, + KEYWORD_TRUE: :on_kw, + KEYWORD_UNDEF: :on_kw, + KEYWORD_UNLESS: :on_kw, + KEYWORD_UNLESS_MODIFIER: :on_kw, + KEYWORD_UNTIL: :on_kw, + KEYWORD_UNTIL_MODIFIER: :on_kw, + KEYWORD_WHEN: :on_kw, + KEYWORD_WHILE: :on_kw, + KEYWORD_WHILE_MODIFIER: :on_kw, + KEYWORD_YIELD: :on_kw, + LABEL: :on_label, + LABEL_END: :on_label_end, + LAMBDA_BEGIN: :on_tlambeg, + LESS: :on_op, + LESS_EQUAL: :on_op, + LESS_EQUAL_GREATER: :on_op, + LESS_LESS: :on_op, + LESS_LESS_EQUAL: :on_op, + METHOD_NAME: :on_ident, + MINUS: :on_op, + MINUS_EQUAL: :on_op, + MINUS_GREATER: :on_tlambda, + NEWLINE: :on_nl, + NUMBERED_REFERENCE: :on_backref, + PARENTHESIS_LEFT: :on_lparen, + PARENTHESIS_LEFT_PARENTHESES: :on_lparen, + PARENTHESIS_RIGHT: :on_rparen, + PERCENT: :on_op, + PERCENT_EQUAL: :on_op, + PERCENT_LOWER_I: :on_qsymbols_beg, + PERCENT_LOWER_W: :on_qwords_beg, + PERCENT_LOWER_X: :on_backtick, + PERCENT_UPPER_I: :on_symbols_beg, + PERCENT_UPPER_W: :on_words_beg, + PIPE: :on_op, + PIPE_EQUAL: :on_op, + PIPE_PIPE: :on_op, + PIPE_PIPE_EQUAL: :on_op, + PLUS: :on_op, + PLUS_EQUAL: :on_op, + QUESTION_MARK: :on_op, + RATIONAL_FLOAT: :on_rational, + RATIONAL_INTEGER: :on_rational, + REGEXP_BEGIN: :on_regexp_beg, + REGEXP_END: :on_regexp_end, + SEMICOLON: :on_semicolon, + SLASH: :on_op, + SLASH_EQUAL: :on_op, + STAR: :on_op, + STAR_EQUAL: :on_op, + STAR_STAR: :on_op, + STAR_STAR_EQUAL: :on_op, + STRING_BEGIN: :on_tstring_beg, + STRING_CONTENT: :on_tstring_content, + STRING_END: :on_tstring_end, + SYMBOL_BEGIN: :on_symbeg, + TILDE: :on_op, + UAMPERSAND: :on_op, + UCOLON_COLON: :on_op, + UDOT_DOT: :on_op, + UDOT_DOT_DOT: :on_op, + UMINUS: :on_op, + UMINUS_NUM: :on_op, + UPLUS: :on_op, + USTAR: :on_op, + USTAR_STAR: :on_op, + WORDS_SEP: :on_words_sep, + "__END__": :on___end__ + }.freeze + + # When we produce tokens, we produce the same arrays that Ripper does. + # However, we add a couple of convenience methods onto them to make them a + # little easier to work with. We delegate all other methods to the array. + class Token < SimpleDelegator + # @dynamic initialize, each, [] + + # The location of the token in the source. + def location + self[0] + end + + # The type of the token. + def event + self[1] + end + + # The slice of the source that this token represents. + def value + self[2] + end + + # The state of the lexer when this token was produced. + def state + self[3] + end + end + + # Ripper doesn't include the rest of the token in the event, so we need to + # trim it down to just the content on the first line when comparing. + class EndContentToken < Token + def ==(other) # :nodoc: + [self[0], self[1], self[2][0..self[2].index("\n")], self[3]] == other + end + end + + # Tokens where state should be ignored + # used for :on_comment, :on_heredoc_end, :on_embexpr_end + class IgnoreStateToken < Token + def ==(other) # :nodoc: + self[0...-1] == other[0...-1] + end + end + + # Ident tokens for the most part are exactly the same, except sometimes we + # know an ident is a local when ripper doesn't (when they are introduced + # through named captures in regular expressions). In that case we don't + # compare the state. + class IdentToken < Token + def ==(other) # :nodoc: + (self[0...-1] == other[0...-1]) && ( + (other[3] == Ripper::EXPR_LABEL | Ripper::EXPR_END) || + (other[3] & Ripper::EXPR_ARG_ANY != 0) + ) + end + end + + # Ignored newlines can occasionally have a LABEL state attached to them, so + # we compare the state differently here. + class IgnoredNewlineToken < Token + def ==(other) # :nodoc: + return false unless self[0...-1] == other[0...-1] + + if self[3] == Ripper::EXPR_ARG | Ripper::EXPR_LABELED + other[3] & Ripper::EXPR_ARG | Ripper::EXPR_LABELED != 0 + else + self[3] == other[3] + end + end + end + + # If we have an identifier that follows a method name like: + # + # def foo bar + # + # then Ripper will mark bar as END|LABEL if there is a local in a parent + # scope named bar because it hasn't pushed the local table yet. We do this + # more accurately, so we need to allow comparing against both END and + # END|LABEL. + class ParamToken < Token + def ==(other) # :nodoc: + (self[0...-1] == other[0...-1]) && ( + (other[3] == Ripper::EXPR_END) || + (other[3] == Ripper::EXPR_END | Ripper::EXPR_LABEL) + ) + end + end + + # A heredoc in this case is a list of tokens that belong to the body of the + # heredoc that should be appended onto the list of tokens when the heredoc + # closes. + module Heredoc # :nodoc: + # Heredocs that are no dash or tilde heredocs are just a list of tokens. + # We need to keep them around so that we can insert them in the correct + # order back into the token stream and set the state of the last token to + # the state that the heredoc was opened in. + class PlainHeredoc # :nodoc: + attr_reader :tokens + + def initialize + @tokens = [] + end + + def <<(token) + tokens << token + end + + def to_a + tokens + end + end + + # Dash heredocs are a little more complicated. They are a list of tokens + # that need to be split on "\\\n" to mimic Ripper's behavior. We also need + # to keep track of the state that the heredoc was opened in. + class DashHeredoc # :nodoc: + attr_reader :split, :tokens + + def initialize(split) + @split = split + @tokens = [] + end + + def <<(token) + tokens << token + end + + def to_a + embexpr_balance = 0 + + tokens.each_with_object([]) do |token, results| #$ Array[Token] + case token.event + when :on_embexpr_beg + embexpr_balance += 1 + results << token + when :on_embexpr_end + embexpr_balance -= 1 + results << token + when :on_tstring_content + if embexpr_balance == 0 + lineno = token[0][0] + column = token[0][1] + + if split + # Split on "\\\n" to mimic Ripper's behavior. Use a lookbehind + # to keep the delimiter in the result. + token.value.split(/(?<=[^\\]\\\n)|(?<=[^\\]\\\r\n)/).each_with_index do |value, index| + column = 0 if index > 0 + results << Token.new([[lineno, column], :on_tstring_content, value, token.state]) + lineno += value.count("\n") + end + else + results << token + end + else + results << token + end + else + results << token + end + end + end + end + + # Heredocs that are dedenting heredocs are a little more complicated. + # Ripper outputs on_ignored_sp tokens for the whitespace that is being + # removed from the output. prism only modifies the node itself and keeps + # the token the same. This simplifies prism, but makes comparing against + # Ripper much harder because there is a length mismatch. + # + # Fortunately, we already have to pull out the heredoc tokens in order to + # insert them into the stream in the correct order. As such, we can do + # some extra manipulation on the tokens to make them match Ripper's + # output by mirroring the dedent logic that Ripper uses. + class DedentingHeredoc # :nodoc: + TAB_WIDTH = 8 + + attr_reader :tokens, :dedent_next, :dedent, :embexpr_balance + + def initialize + @tokens = [] + @dedent_next = true + @dedent = nil + @embexpr_balance = 0 + @ended_on_newline = false + end + + # As tokens are coming in, we track the minimum amount of common leading + # whitespace on plain string content tokens. This allows us to later + # remove that amount of whitespace from the beginning of each line. + def <<(token) + case token.event + when :on_embexpr_beg, :on_heredoc_beg + @embexpr_balance += 1 + @dedent = 0 if @dedent_next && @ended_on_newline + when :on_embexpr_end, :on_heredoc_end + @embexpr_balance -= 1 + when :on_tstring_content + if embexpr_balance == 0 + line = token.value + + if dedent_next && !(line.strip.empty? && line.end_with?("\n")) + leading = line[/\A(\s*)\n?/, 1] + next_dedent = 0 + + leading.each_char do |char| + if char == "\t" + next_dedent = next_dedent - (next_dedent % TAB_WIDTH) + TAB_WIDTH + else + next_dedent += 1 + end + end + + @dedent = [dedent, next_dedent].compact.min + @dedent_next = true + @ended_on_newline = line.end_with?("\n") + tokens << token + return + end + end + end + + @dedent_next = token.event == :on_tstring_content && embexpr_balance == 0 + @ended_on_newline = false + tokens << token + end + + def to_a + # If every line in the heredoc is blank, we still need to split up the + # string content token into multiple tokens. + if dedent.nil? + results = [] #: Array[Token] + embexpr_balance = 0 + + tokens.each do |token| + case token.event + when :on_embexpr_beg, :on_heredoc_beg + embexpr_balance += 1 + results << token + when :on_embexpr_end, :on_heredoc_end + embexpr_balance -= 1 + results << token + when :on_tstring_content + if embexpr_balance == 0 + lineno = token[0][0] + column = token[0][1] + + token.value.split(/(?<=\n)/).each_with_index do |value, index| + column = 0 if index > 0 + results << Token.new([[lineno, column], :on_tstring_content, value, token.state]) + lineno += 1 + end + else + results << token + end + else + results << token + end + end + + return results + end + + # If the minimum common whitespace is 0, then we need to concatenate + # string nodes together that are immediately adjacent. + if dedent == 0 + results = [] #: Array[Token] + embexpr_balance = 0 + + index = 0 + max_index = tokens.length + + while index < max_index + token = tokens[index] + results << token + index += 1 + + case token.event + when :on_embexpr_beg, :on_heredoc_beg + embexpr_balance += 1 + when :on_embexpr_end, :on_heredoc_end + embexpr_balance -= 1 + when :on_tstring_content + if embexpr_balance == 0 + while index < max_index && tokens[index].event == :on_tstring_content + token.value << tokens[index].value + index += 1 + end + end + end + end + + return results + end + + # Otherwise, we're going to run through each token in the list and + # insert on_ignored_sp tokens for the amount of dedent that we need to + # perform. We also need to remove the dedent from the beginning of + # each line of plain string content tokens. + results = [] #: Array[Token] + dedent_next = true + embexpr_balance = 0 + + tokens.each do |token| + # Notice that the structure of this conditional largely matches the + # whitespace calculation we performed above. This is because + # checking if the subsequent token needs to be dedented is common to + # both the dedent calculation and the ignored_sp insertion. + case token.event + when :on_embexpr_beg + embexpr_balance += 1 + results << token + when :on_embexpr_end + embexpr_balance -= 1 + results << token + when :on_tstring_content + if embexpr_balance == 0 + # Here we're going to split the string on newlines, but maintain + # the newlines in the resulting array. We'll do that with a look + # behind assertion. + splits = token.value.split(/(?<=\n)/) + index = 0 + + while index < splits.length + line = splits[index] + lineno = token[0][0] + index + column = token[0][1] + + # Blank lines do not count toward common leading whitespace + # calculation and do not need to be dedented. + if dedent_next || index > 0 + column = 0 + end + + # If the dedent is 0 and we're not supposed to dedent the next + # line or this line doesn't start with whitespace, then we + # should concatenate the rest of the string to match ripper. + if dedent == 0 && (!dedent_next || !line.start_with?(/\s/)) + line = splits[index..].join + index = splits.length + end + + # If we are supposed to dedent this line or if this is not the + # first line of the string and this line isn't entirely blank, + # then we need to insert an on_ignored_sp token and remove the + # dedent from the beginning of the line. + if (dedent > 0) && (dedent_next || index > 0) + deleting = 0 + deleted_chars = [] #: Array[String] + + # Gather up all of the characters that we're going to + # delete, stopping when you hit a character that would put + # you over the dedent amount. + line.each_char.with_index do |char, i| + case char + when "\r" + if line[i + 1] == "\n" + break + end + when "\n" + break + when "\t" + deleting = deleting - (deleting % TAB_WIDTH) + TAB_WIDTH + else + deleting += 1 + end + + break if deleting > dedent + deleted_chars << char + end + + # If we have something to delete, then delete it from the + # string and insert an on_ignored_sp token. + if deleted_chars.any? + ignored = deleted_chars.join + line.delete_prefix!(ignored) + + results << Token.new([[lineno, 0], :on_ignored_sp, ignored, token[3]]) + column = ignored.length + end + end + + results << Token.new([[lineno, column], token[1], line, token[3]]) unless line.empty? + index += 1 + end + else + results << token + end + else + results << token + end + + dedent_next = + ((token.event == :on_tstring_content) || (token.event == :on_heredoc_end)) && + embexpr_balance == 0 + end + + results + end + end + + # Here we will split between the two types of heredocs and return the + # object that will store their tokens. + def self.build(opening) + case opening.value[2] + when "~" + DedentingHeredoc.new + when "-" + DashHeredoc.new(opening.value[3] != "'") + else + PlainHeredoc.new + end + end + end + + private_constant :Heredoc + + attr_reader :source, :options + + def initialize(source, **options) + @source = source + @options = options + end + + def result + tokens = [] #: Array[LexCompat::Token] + + state = :default + heredoc_stack = [[]] #: Array[Array[Heredoc::PlainHeredoc | Heredoc::DashHeredoc | Heredoc::DedentingHeredoc]] + + result = Prism.lex(source, **options) + result_value = result.value + previous_state = nil #: Ripper::Lexer::State? + last_heredoc_end = nil #: Integer? + + # In previous versions of Ruby, Ripper wouldn't flush the bom before the + # first token, so we had to have a hack in place to account for that. This + # checks for that behavior. + bom_flushed = Ripper.lex("\xEF\xBB\xBF# test")[0][0][1] == 0 + bom = source.byteslice(0..2) == "\xEF\xBB\xBF" + + result_value.each_with_index do |(token, lex_state), index| + lineno = token.location.start_line + column = token.location.start_column + + # If there's a UTF-8 byte-order mark as the start of the file, then for + # certain tokens ripper sets the first token back by 3 bytes. It also + # keeps the byte order mark in the first token's value. This is weird, + # and I don't want to mirror that in our parser. So instead, we'll match + # up the columns and values here. + if bom && lineno == 1 + column -= 3 + + if index == 0 && column == 0 && !bom_flushed + flushed = + case token.type + when :BACK_REFERENCE, :INSTANCE_VARIABLE, :CLASS_VARIABLE, + :GLOBAL_VARIABLE, :NUMBERED_REFERENCE, :PERCENT_LOWER_I, + :PERCENT_LOWER_X, :PERCENT_LOWER_W, :PERCENT_UPPER_I, + :PERCENT_UPPER_W, :STRING_BEGIN + true + when :REGEXP_BEGIN, :SYMBOL_BEGIN + token.value.start_with?("%") + else + false + end + + unless flushed + column -= 3 + value = token.value + value.prepend(String.new("\xEF\xBB\xBF", encoding: value.encoding)) + end + end + end + + event = RIPPER.fetch(token.type) + value = token.value + lex_state = Ripper::Lexer::State.new(lex_state) + + token = + case event + when :on___end__ + EndContentToken.new([[lineno, column], event, value, lex_state]) + when :on_comment + IgnoreStateToken.new([[lineno, column], event, value, lex_state]) + when :on_heredoc_end + # Heredoc end tokens can be emitted in an odd order, so we don't + # want to bother comparing the state on them. + last_heredoc_end = token.location.end_offset + IgnoreStateToken.new([[lineno, column], event, value, lex_state]) + when :on_ident + if lex_state == Ripper::EXPR_END + # If we have an identifier that follows a method name like: + # + # def foo bar + # + # then Ripper will mark bar as END|LABEL if there is a local in a + # parent scope named bar because it hasn't pushed the local table + # yet. We do this more accurately, so we need to allow comparing + # against both END and END|LABEL. + ParamToken.new([[lineno, column], event, value, lex_state]) + elsif lex_state == Ripper::EXPR_END | Ripper::EXPR_LABEL + # In the event that we're comparing identifiers, we're going to + # allow a little divergence. Ripper doesn't account for local + # variables introduced through named captures in regexes, and we + # do, which accounts for this difference. + IdentToken.new([[lineno, column], event, value, lex_state]) + else + Token.new([[lineno, column], event, value, lex_state]) + end + when :on_embexpr_end + IgnoreStateToken.new([[lineno, column], event, value, lex_state]) + when :on_ignored_nl + # Ignored newlines can occasionally have a LABEL state attached to + # them which doesn't actually impact anything. We don't mirror that + # state so we ignored it. + IgnoredNewlineToken.new([[lineno, column], event, value, lex_state]) + when :on_regexp_end + # On regex end, Ripper scans and then sets end state, so the ripper + # lexed output is begin, when it should be end. prism sets lex state + # correctly to end state, but we want to be able to compare against + # Ripper's lexed state. So here, if it's a regexp end token, we + # output the state as the previous state, solely for the sake of + # comparison. + previous_token = result_value[index - 1][0] + lex_state = + if RIPPER.fetch(previous_token.type) == :on_embexpr_end + # If the previous token is embexpr_end, then we have to do even + # more processing. The end of an embedded expression sets the + # state to the state that it had at the beginning of the + # embedded expression. So we have to go and find that state and + # set it here. + counter = 1 + current_index = index - 1 + + until counter == 0 + current_index -= 1 + current_event = RIPPER.fetch(result_value[current_index][0].type) + counter += { on_embexpr_beg: -1, on_embexpr_end: 1 }[current_event] || 0 + end + + Ripper::Lexer::State.new(result_value[current_index][1]) + else + previous_state + end + + Token.new([[lineno, column], event, value, lex_state]) + when :on_eof + previous_token = result_value[index - 1][0] + + # If we're at the end of the file and the previous token was a + # comment and there is still whitespace after the comment, then + # Ripper will append a on_nl token (even though there isn't + # necessarily a newline). We mirror that here. + if previous_token.type == :COMMENT + # If the comment is at the start of a heredoc: <<HEREDOC # comment + # then the comment's end_offset is up near the heredoc_beg. + # This is not the correct offset to use for figuring out if + # there is trailing whitespace after the last token. + # Use the greater offset of the two to determine the start of + # the trailing whitespace. + start_offset = [previous_token.location.end_offset, last_heredoc_end].compact.max + end_offset = token.location.start_offset + + if start_offset < end_offset + if bom + start_offset += 3 + end_offset += 3 + end + + tokens << Token.new([[lineno, 0], :on_nl, source.byteslice(start_offset...end_offset), lex_state]) + end + end + + Token.new([[lineno, column], event, value, lex_state]) + else + Token.new([[lineno, column], event, value, lex_state]) + end + + previous_state = lex_state + + # The order in which tokens appear in our lexer is different from the + # order that they appear in Ripper. When we hit the declaration of a + # heredoc in prism, we skip forward and lex the rest of the content of + # the heredoc before going back and lexing at the end of the heredoc + # identifier. + # + # To match up to ripper, we keep a small state variable around here to + # track whether we're in the middle of a heredoc or not. In this way we + # can shuffle around the token to match Ripper's output. + case state + when :default + # The default state is when there are no heredocs at all. In this + # state we can append the token to the list of tokens and move on. + tokens << token + + # If we get the declaration of a heredoc, then we open a new heredoc + # and move into the heredoc_opened state. + if event == :on_heredoc_beg + state = :heredoc_opened + heredoc_stack.last << Heredoc.build(token) + end + when :heredoc_opened + # The heredoc_opened state is when we've seen the declaration of a + # heredoc and are now lexing the body of the heredoc. In this state we + # push tokens onto the most recently created heredoc. + heredoc_stack.last.last << token + + case event + when :on_heredoc_beg + # If we receive a heredoc declaration while lexing the body of a + # heredoc, this means we have nested heredocs. In this case we'll + # push a new heredoc onto the stack and stay in the heredoc_opened + # state since we're now lexing the body of the new heredoc. + heredoc_stack << [Heredoc.build(token)] + when :on_heredoc_end + # If we receive the end of a heredoc, then we're done lexing the + # body of the heredoc. In this case we now have a completed heredoc + # but need to wait for the next newline to push it into the token + # stream. + state = :heredoc_closed + end + when :heredoc_closed + if %i[on_nl on_ignored_nl on_comment].include?(event) || (event == :on_tstring_content && value.end_with?("\n")) + if heredoc_stack.size > 1 + flushing = heredoc_stack.pop + heredoc_stack.last.last << token + + flushing.each do |heredoc| + heredoc.to_a.each do |flushed_token| + heredoc_stack.last.last << flushed_token + end + end + + state = :heredoc_opened + next + end + elsif event == :on_heredoc_beg + tokens << token + state = :heredoc_opened + heredoc_stack.last << Heredoc.build(token) + next + elsif heredoc_stack.size > 1 + heredoc_stack[-2].last << token + next + end + + heredoc_stack.last.each do |heredoc| + tokens.concat(heredoc.to_a) + end + + heredoc_stack.last.clear + state = :default + + tokens << token + end + end + + # Drop the EOF token from the list + tokens = tokens[0...-1] + + # We sort by location to compare against Ripper's output + tokens.sort_by!(&:location) + + Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, Source.for(source)) + end + end + + private_constant :LexCompat + + # This is a class that wraps the Ripper lexer to produce almost exactly the + # same tokens. + class LexRipper # :nodoc: + attr_reader :source + + def initialize(source) + @source = source + end + + def result + previous = [] #: [[Integer, Integer], Symbol, String, untyped] | [] + results = [] #: Array[[[Integer, Integer], Symbol, String, untyped]] + + lex(source).each do |token| + case token[1] + when :on_sp + # skip + when :on_tstring_content + if previous[1] == :on_tstring_content && (token[2].start_with?("\#$") || token[2].start_with?("\#@")) + previous[2] << token[2] + else + results << token + previous = token + end + when :on_words_sep + if previous[1] == :on_words_sep + previous[2] << token[2] + else + results << token + previous = token + end + else + results << token + previous = token + end + end + + results + end + + private + + if Ripper.method(:lex).parameters.assoc(:keyrest) + def lex(source) + Ripper.lex(source, raise_errors: true) + end + else + def lex(source) + ripper = Ripper::Lexer.new(source) + ripper.lex.tap do |result| + raise SyntaxError, ripper.errors.map(&:message).join(' ;') if ripper.errors.any? + end + end + end + end + + private_constant :LexRipper +end diff --git a/lib/prism/node_ext.rb b/lib/prism/node_ext.rb new file mode 100644 index 0000000000..cdc4b2bf08 --- /dev/null +++ b/lib/prism/node_ext.rb @@ -0,0 +1,443 @@ +# frozen_string_literal: true + +# Here we are reopening the prism module to provide methods on nodes that aren't +# templated and are meant as convenience methods. +module Prism + class Node + def deprecated(*replacements) # :nodoc: + suggest = replacements.map { |replacement| "#{self.class}##{replacement}" } + warn(<<~MSG, category: :deprecated) + [deprecation]: #{self.class}##{caller_locations(1, 1)[0].label} is deprecated \ + and will be removed in the next major version. Use #{suggest.join("/")} instead. + #{(caller(1, 3) || []).join("\n")} + MSG + end + end + + module RegularExpressionOptions # :nodoc: + # Returns a numeric value that represents the flags that were used to create + # the regular expression. + def options + o = flags & (RegularExpressionFlags::IGNORE_CASE | RegularExpressionFlags::EXTENDED | RegularExpressionFlags::MULTI_LINE) + o |= Regexp::FIXEDENCODING if flags.anybits?(RegularExpressionFlags::EUC_JP | RegularExpressionFlags::WINDOWS_31J | RegularExpressionFlags::UTF_8) + o |= Regexp::NOENCODING if flags.anybits?(RegularExpressionFlags::ASCII_8BIT) + o + end + end + + class InterpolatedMatchLastLineNode < Node + include RegularExpressionOptions + end + + class InterpolatedRegularExpressionNode < Node + include RegularExpressionOptions + end + + class MatchLastLineNode < Node + include RegularExpressionOptions + end + + class RegularExpressionNode < Node + include RegularExpressionOptions + end + + private_constant :RegularExpressionOptions + + module HeredocQuery # :nodoc: + # Returns true if this node was represented as a heredoc in the source code. + def heredoc? + opening&.start_with?("<<") + end + end + + class InterpolatedStringNode < Node + include HeredocQuery + end + + class InterpolatedXStringNode < Node + include HeredocQuery + end + + class StringNode < Node + include HeredocQuery + + # Occasionally it's helpful to treat a string as if it were interpolated so + # that there's a consistent interface for working with strings. + def to_interpolated + InterpolatedStringNode.new( + source, + frozen? ? InterpolatedStringNodeFlags::FROZEN : 0, + opening_loc, + [copy(opening_loc: nil, closing_loc: nil, location: content_loc)], + closing_loc, + location + ) + end + end + + class XStringNode < Node + include HeredocQuery + + # Occasionally it's helpful to treat a string as if it were interpolated so + # that there's a consistent interface for working with strings. + def to_interpolated + InterpolatedXStringNode.new( + source, + opening_loc, + [StringNode.new(source, 0, nil, content_loc, nil, unescaped, content_loc)], + closing_loc, + location + ) + end + end + + private_constant :HeredocQuery + + class ImaginaryNode < Node + # Returns the value of the node as a Ruby Complex. + def value + Complex(0, numeric.value) + end + end + + class RationalNode < Node + # Returns the value of the node as a Ruby Rational. + def value + Rational(numerator, denominator) + end + + # Returns the value of the node as an IntegerNode or a FloatNode. This + # method is deprecated in favor of #value or #numerator/#denominator. + def numeric + deprecated("value", "numerator", "denominator") + + if denominator == 1 + IntegerNode.new(source, flags, numerator, location.chop) + else + FloatNode.new(source, numerator.to_f / denominator, location.chop) + end + end + end + + class ConstantReadNode < Node + # Returns the list of parts for the full name of this constant. + # For example: [:Foo] + def full_name_parts + [name] + end + + # Returns the full name of this constant. For example: "Foo" + def full_name + name.to_s + end + end + + class ConstantWriteNode < Node + # Returns the list of parts for the full name of this constant. + # For example: [:Foo] + def full_name_parts + [name] + end + + # Returns the full name of this constant. For example: "Foo" + def full_name + name.to_s + end + end + + class ConstantPathNode < Node + # An error class raised when dynamic parts are found while computing a + # constant path's full name. For example: + # Foo::Bar::Baz -> does not raise because all parts of the constant path are + # simple constants + # var::Bar::Baz -> raises because the first part of the constant path is a + # local variable + class DynamicPartsInConstantPathError < StandardError; end + + # An error class raised when missing nodes are found while computing a + # constant path's full name. For example: + # Foo:: -> raises because the constant path is missing the last part + class MissingNodesInConstantPathError < StandardError; end + + # Returns the list of parts for the full name of this constant path. + # For example: [:Foo, :Bar] + def full_name_parts + parts = [] #: Array[Symbol] + current = self #: node? + + while current.is_a?(ConstantPathNode) + name = current.name + if name.nil? + raise MissingNodesInConstantPathError, "Constant path contains missing nodes. Cannot compute full name" + end + + parts.unshift(name) + current = current.parent + end + + if !current.is_a?(ConstantReadNode) && !current.nil? + raise DynamicPartsInConstantPathError, "Constant path contains dynamic parts. Cannot compute full name" + end + + parts.unshift(current&.name || :"") + end + + # Returns the full name of this constant path. For example: "Foo::Bar" + def full_name + full_name_parts.join("::") + end + + # Previously, we had a child node on this class that contained either a + # constant read or a missing node. To not cause a breaking change, we + # continue to supply that API. + def child + deprecated("name", "name_loc") + name ? ConstantReadNode.new(source, name, name_loc) : MissingNode.new(source, location) + end + end + + class ConstantPathTargetNode < Node + # Returns the list of parts for the full name of this constant path. + # For example: [:Foo, :Bar] + def full_name_parts + parts = + case parent + when ConstantPathNode, ConstantReadNode + parent.full_name_parts + when nil + [:""] + else + # e.g. self::Foo, (var)::Bar = baz + raise ConstantPathNode::DynamicPartsInConstantPathError, "Constant target path contains dynamic parts. Cannot compute full name" + end + + if name.nil? + raise ConstantPathNode::MissingNodesInConstantPathError, "Constant target path contains missing nodes. Cannot compute full name" + end + + parts.push(name) + end + + # Returns the full name of this constant path. For example: "Foo::Bar" + def full_name + full_name_parts.join("::") + end + + # Previously, we had a child node on this class that contained either a + # constant read or a missing node. To not cause a breaking change, we + # continue to supply that API. + def child + deprecated("name", "name_loc") + name ? ConstantReadNode.new(source, name, name_loc) : MissingNode.new(source, location) + end + end + + class ConstantTargetNode < Node + # Returns the list of parts for the full name of this constant. + # For example: [:Foo] + def full_name_parts + [name] + end + + # Returns the full name of this constant. For example: "Foo" + def full_name + name.to_s + end + end + + class ParametersNode < Node + # Mirrors the Method#parameters method. + def signature + names = [] #: Array[[Symbol, Symbol] | [Symbol]] + + requireds.each do |param| + names << (param.is_a?(MultiTargetNode) ? [:req] : [:req, param.name]) + end + + optionals.each { |param| names << [:opt, param.name] } + + if rest && rest.is_a?(RestParameterNode) + names << [:rest, rest.name || :*] + end + + posts.each do |param| + if param.is_a?(MultiTargetNode) + names << [:req] + elsif param.is_a?(NoKeywordsParameterNode) + # Invalid syntax, e.g. "def f(**nil, ...)" moves the NoKeywordsParameterNode to posts + raise "Invalid syntax" + else + names << [:req, param.name] + end + end + + # Regardless of the order in which the keywords were defined, the required + # keywords always come first followed by the optional keywords. + keyopt = [] #: Array[OptionalKeywordParameterNode] + keywords.each do |param| + if param.is_a?(OptionalKeywordParameterNode) + keyopt << param + else + names << [:keyreq, param.name] + end + end + + keyopt.each { |param| names << [:key, param.name] } + + case keyword_rest + when ForwardingParameterNode + names.concat([[:rest, :*], [:keyrest, :**], [:block, :&]]) + when KeywordRestParameterNode + names << [:keyrest, keyword_rest.name || :**] + when NoKeywordsParameterNode + names << [:nokey] + end + + names << [:block, block.name || :&] if block + names + end + end + + class CallNode < Node + # When a call node has the attribute_write flag set, it means that the call + # is using the attribute write syntax. This is either a method call to []= + # or a method call to a method that ends with =. Either way, the = sign is + # present in the source. + # + # Prism returns the message_loc _without_ the = sign attached, because there + # can be any amount of space between the message and the = sign. However, + # sometimes you want the location of the full message including the inner + # space and the = sign. This method provides that. + def full_message_loc + attribute_write? ? message_loc&.adjoin("=") : message_loc + end + end + + class CallOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class ClassVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class ConstantOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class ConstantPathOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class GlobalVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class IndexOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class InstanceVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class LocalVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end +end diff --git a/lib/prism/pack.rb b/lib/prism/pack.rb new file mode 100644 index 0000000000..c0de8ab8b7 --- /dev/null +++ b/lib/prism/pack.rb @@ -0,0 +1,228 @@ +# frozen_string_literal: true +# typed: ignore + +module Prism + # A parser for the pack template language. + module Pack + %i[ + SPACE + COMMENT + INTEGER + UTF8 + BER + FLOAT + STRING_SPACE_PADDED + STRING_NULL_PADDED + STRING_NULL_TERMINATED + STRING_MSB + STRING_LSB + STRING_HEX_HIGH + STRING_HEX_LOW + STRING_UU + STRING_MIME + STRING_BASE64 + STRING_FIXED + STRING_POINTER + MOVE + BACK + NULL + + UNSIGNED + SIGNED + SIGNED_NA + + AGNOSTIC_ENDIAN + LITTLE_ENDIAN + BIG_ENDIAN + NATIVE_ENDIAN + ENDIAN_NA + + SIZE_SHORT + SIZE_INT + SIZE_LONG + SIZE_LONG_LONG + SIZE_8 + SIZE_16 + SIZE_32 + SIZE_64 + SIZE_P + SIZE_NA + + LENGTH_FIXED + LENGTH_MAX + LENGTH_RELATIVE + LENGTH_NA + ].each do |const| + const_set(const, const) + end + + # A directive in the pack template language. + class Directive + # A symbol representing the version of Ruby. + attr_reader :version + + # A symbol representing whether or not we are packing or unpacking. + attr_reader :variant + + # A byteslice of the source string that this directive represents. + attr_reader :source + + # The type of the directive. + attr_reader :type + + # The type of signedness of the directive. + attr_reader :signed + + # The type of endianness of the directive. + attr_reader :endian + + # The size of the directive. + attr_reader :size + + # The length type of this directive (used for integers). + attr_reader :length_type + + # The length of this directive (used for integers). + attr_reader :length + + # Initialize a new directive with the given values. + def initialize(version, variant, source, type, signed, endian, size, length_type, length) + @version = version + @variant = variant + @source = source + @type = type + @signed = signed + @endian = endian + @size = size + @length_type = length_type + @length = length + end + + # The descriptions of the various types of endianness. + ENDIAN_DESCRIPTIONS = { + AGNOSTIC_ENDIAN: "agnostic", + LITTLE_ENDIAN: "little-endian (VAX)", + BIG_ENDIAN: "big-endian (network)", + NATIVE_ENDIAN: "native-endian", + ENDIAN_NA: "n/a" + } + + # The descriptions of the various types of signedness. + SIGNED_DESCRIPTIONS = { + UNSIGNED: "unsigned", + SIGNED: "signed", + SIGNED_NA: "n/a" + } + + # The descriptions of the various types of sizes. + SIZE_DESCRIPTIONS = { + SIZE_SHORT: "short", + SIZE_INT: "int-width", + SIZE_LONG: "long", + SIZE_LONG_LONG: "long long", + SIZE_8: "8-bit", + SIZE_16: "16-bit", + SIZE_32: "32-bit", + SIZE_64: "64-bit", + SIZE_P: "pointer-width" + } + + # Provide a human-readable description of the directive. + def describe + case type + when SPACE + "whitespace" + when COMMENT + "comment" + when INTEGER + if size == SIZE_8 + base = "#{SIGNED_DESCRIPTIONS[signed]} #{SIZE_DESCRIPTIONS[size]} integer" + else + base = "#{SIGNED_DESCRIPTIONS[signed]} #{SIZE_DESCRIPTIONS[size]} #{ENDIAN_DESCRIPTIONS[endian]} integer" + end + case length_type + when LENGTH_FIXED + if length > 1 + base + ", x#{length}" + else + base + end + when LENGTH_MAX + base + ", as many as possible" + else + raise + end + when UTF8 + "UTF-8 character" + when BER + "BER-compressed integer" + when FLOAT + "#{SIZE_DESCRIPTIONS[size]} #{ENDIAN_DESCRIPTIONS[endian]} float" + when STRING_SPACE_PADDED + "arbitrary binary string (space padded)" + when STRING_NULL_PADDED + "arbitrary binary string (null padded, count is width)" + when STRING_NULL_TERMINATED + "arbitrary binary string (null padded, count is width), except that null is added with *" + when STRING_MSB + "bit string (MSB first)" + when STRING_LSB + "bit string (LSB first)" + when STRING_HEX_HIGH + "hex string (high nibble first)" + when STRING_HEX_LOW + "hex string (low nibble first)" + when STRING_UU + "UU-encoded string" + when STRING_MIME + "quoted printable, MIME encoding" + when STRING_BASE64 + "base64 encoded string" + when STRING_FIXED + "pointer to a structure (fixed-length string)" + when STRING_POINTER + "pointer to a null-terminated string" + when MOVE + "move to absolute position" + when BACK + "back up a byte" + when NULL + "null byte" + else + raise + end + end + end + + # The result of parsing a pack template. + class Format + # A list of the directives in the template. + attr_reader :directives + + # The encoding of the template. + attr_reader :encoding + + # Create a new Format with the given directives and encoding. + def initialize(directives, encoding) + @directives = directives + @encoding = encoding + end + + # Provide a human-readable description of the format. + def describe + source_width = directives.map { |d| d.source.inspect.length }.max + directive_lines = directives.map do |directive| + if directive.type == SPACE + source = directive.source.inspect + else + source = directive.source + end + # @type var source_width: Integer + " #{source.ljust(source_width)} #{directive.describe}" + end + + (["Directives:"] + directive_lines + ["Encoding:", " #{encoding}"]).join("\n") + end + end + end +end diff --git a/lib/prism/parse_result.rb b/lib/prism/parse_result.rb new file mode 100644 index 0000000000..798fde09e5 --- /dev/null +++ b/lib/prism/parse_result.rb @@ -0,0 +1,698 @@ +# frozen_string_literal: true + +module Prism + # This represents a source of Ruby code that has been parsed. It is used in + # conjunction with locations to allow them to resolve line numbers and source + # ranges. + class Source + # Create a new source object with the given source code. This method should + # be used instead of `new` and it will return either a `Source` or a + # specialized and more performant `ASCIISource` if no multibyte characters + # are present in the source code. + def self.for(source, start_line = 1, offsets = []) + source.ascii_only? ? ASCIISource.new(source, start_line, offsets): new(source, start_line, offsets) + end + + # The source code that this source object represents. + attr_reader :source + + # The line number where this source starts. + attr_reader :start_line + + # The list of newline byte offsets in the source code. + attr_reader :offsets + + # Create a new source object with the given source code. + def initialize(source, start_line = 1, offsets = []) + @source = source + @start_line = start_line # set after parsing is done + @offsets = offsets # set after parsing is done + end + + # Returns the encoding of the source code, which is set by parameters to the + # parser or by the encoding magic comment. + def encoding + source.encoding + end + + # Returns the lines of the source code as an array of strings. + def lines + source.lines + end + + # Perform a byteslice on the source code using the given byte offset and + # byte length. + def slice(byte_offset, length) + source.byteslice(byte_offset, length) or raise + end + + # Binary search through the offsets to find the line number for the given + # byte offset. + def line(byte_offset) + start_line + find_line(byte_offset) + end + + # Return the byte offset of the start of the line corresponding to the given + # byte offset. + def line_start(byte_offset) + offsets[find_line(byte_offset)] + end + + # Returns the byte offset of the end of the line corresponding to the given + # byte offset. + def line_end(byte_offset) + offsets[find_line(byte_offset) + 1] || source.bytesize + end + + # Return the column number for the given byte offset. + def column(byte_offset) + byte_offset - line_start(byte_offset) + end + + # Return the character offset for the given byte offset. + def character_offset(byte_offset) + (source.byteslice(0, byte_offset) or raise).length + end + + # Return the column number in characters for the given byte offset. + def character_column(byte_offset) + character_offset(byte_offset) - character_offset(line_start(byte_offset)) + end + + # Returns the offset from the start of the file for the given byte offset + # counting in code units for the given encoding. + # + # This method is tested with UTF-8, UTF-16, and UTF-32. If there is the + # concept of code units that differs from the number of characters in other + # encodings, it is not captured here. + def code_units_offset(byte_offset, encoding) + byteslice = (source.byteslice(0, byte_offset) or raise).encode(encoding) + (encoding == Encoding::UTF_16LE || encoding == Encoding::UTF_16BE) ? (byteslice.bytesize / 2) : byteslice.length + end + + # Returns the column number in code units for the given encoding for the + # given byte offset. + def code_units_column(byte_offset, encoding) + code_units_offset(byte_offset, encoding) - code_units_offset(line_start(byte_offset), encoding) + end + + private + + # Binary search through the offsets to find the line number for the given + # byte offset. + def find_line(byte_offset) + left = 0 + right = offsets.length - 1 + + while left <= right + mid = left + (right - left) / 2 + return mid if (offset = offsets[mid]) == byte_offset + + if offset < byte_offset + left = mid + 1 + else + right = mid - 1 + end + end + + left - 1 + end + end + + # Specialized version of Prism::Source for source code that includes ASCII + # characters only. This class is used to apply performance optimizations that + # cannot be applied to sources that include multibyte characters. Sources that + # include multibyte characters are represented by the Prism::Source class. + class ASCIISource < Source + # Return the character offset for the given byte offset. + def character_offset(byte_offset) + byte_offset + end + + # Return the column number in characters for the given byte offset. + def character_column(byte_offset) + byte_offset - line_start(byte_offset) + end + + # Returns the offset from the start of the file for the given byte offset + # counting in code units for the given encoding. + # + # This method is tested with UTF-8, UTF-16, and UTF-32. If there is the + # concept of code units that differs from the number of characters in other + # encodings, it is not captured here. + def code_units_offset(byte_offset, encoding) + byte_offset + end + + # Specialized version of `code_units_column` that does not depend on + # `code_units_offset`, which is a more expensive operation. This is + # essentialy the same as `Prism::Source#column`. + def code_units_column(byte_offset, encoding) + byte_offset - line_start(byte_offset) + end + end + + # This represents a location in the source. + class Location + # A Source object that is used to determine more information from the given + # offset and length. + attr_reader :source + protected :source + + # The byte offset from the beginning of the source where this location + # starts. + attr_reader :start_offset + + # The length of this location in bytes. + attr_reader :length + + # Create a new location object with the given source, start byte offset, and + # byte length. + def initialize(source, start_offset, length) + @source = source + @start_offset = start_offset + @length = length + + # These are used to store comments that are associated with this location. + # They are initialized to `nil` to save on memory when there are no + # comments to be attached and/or the comment-related APIs are not used. + @leading_comments = nil + @trailing_comments = nil + end + + # These are the comments that are associated with this location that exist + # before the start of this location. + def leading_comments + @leading_comments ||= [] + end + + # Attach a comment to the leading comments of this location. + def leading_comment(comment) + leading_comments << comment + end + + # These are the comments that are associated with this location that exist + # after the end of this location. + def trailing_comments + @trailing_comments ||= [] + end + + # Attach a comment to the trailing comments of this location. + def trailing_comment(comment) + trailing_comments << comment + end + + # Returns all comments that are associated with this location (both leading + # and trailing comments). + def comments + [*@leading_comments, *@trailing_comments] + end + + # Create a new location object with the given options. + def copy(source: self.source, start_offset: self.start_offset, length: self.length) + Location.new(source, start_offset, length) + end + + # Returns a new location that is the result of chopping off the last byte. + def chop + copy(length: length == 0 ? length : length - 1) + end + + # Returns a string representation of this location. + def inspect + "#<Prism::Location @start_offset=#{@start_offset} @length=#{@length} start_line=#{start_line}>" + end + + # Returns all of the lines of the source code associated with this location. + def source_lines + source.lines + end + + # The source code that this location represents. + def slice + source.slice(start_offset, length) + end + + # The source code that this location represents starting from the beginning + # of the line that this location starts on to the end of the line that this + # location ends on. + def slice_lines + line_start = source.line_start(start_offset) + line_end = source.line_end(end_offset) + source.slice(line_start, line_end - line_start) + end + + # The character offset from the beginning of the source where this location + # starts. + def start_character_offset + source.character_offset(start_offset) + end + + # The offset from the start of the file in code units of the given encoding. + def start_code_units_offset(encoding = Encoding::UTF_16LE) + source.code_units_offset(start_offset, encoding) + end + + # The byte offset from the beginning of the source where this location ends. + def end_offset + start_offset + length + end + + # The character offset from the beginning of the source where this location + # ends. + def end_character_offset + source.character_offset(end_offset) + end + + # The offset from the start of the file in code units of the given encoding. + def end_code_units_offset(encoding = Encoding::UTF_16LE) + source.code_units_offset(end_offset, encoding) + end + + # The line number where this location starts. + def start_line + source.line(start_offset) + end + + # The content of the line where this location starts before this location. + def start_line_slice + offset = source.line_start(start_offset) + source.slice(offset, start_offset - offset) + end + + # The line number where this location ends. + def end_line + source.line(end_offset) + end + + # The column number in bytes where this location starts from the start of + # the line. + def start_column + source.column(start_offset) + end + + # The column number in characters where this location ends from the start of + # the line. + def start_character_column + source.character_column(start_offset) + end + + # The column number in code units of the given encoding where this location + # starts from the start of the line. + def start_code_units_column(encoding = Encoding::UTF_16LE) + source.code_units_column(start_offset, encoding) + end + + # The column number in bytes where this location ends from the start of the + # line. + def end_column + source.column(end_offset) + end + + # The column number in characters where this location ends from the start of + # the line. + def end_character_column + source.character_column(end_offset) + end + + # The column number in code units of the given encoding where this location + # ends from the start of the line. + def end_code_units_column(encoding = Encoding::UTF_16LE) + source.code_units_column(end_offset, encoding) + end + + # Implement the hash pattern matching interface for Location. + def deconstruct_keys(keys) + { start_offset: start_offset, end_offset: end_offset } + end + + # Implement the pretty print interface for Location. + def pretty_print(q) + q.text("(#{start_line},#{start_column})-(#{end_line},#{end_column})") + end + + # Returns true if the given other location is equal to this location. + def ==(other) + Location === other && + other.start_offset == start_offset && + other.end_offset == end_offset + end + + # Returns a new location that stretches from this location to the given + # other location. Raises an error if this location is not before the other + # location or if they don't share the same source. + def join(other) + raise "Incompatible sources" if source != other.source + raise "Incompatible locations" if start_offset > other.start_offset + + Location.new(source, start_offset, other.end_offset - start_offset) + end + + # Join this location with the first occurrence of the string in the source + # that occurs after this location on the same line, and return the new + # location. This will raise an error if the string does not exist. + def adjoin(string) + line_suffix = source.slice(end_offset, source.line_end(end_offset) - end_offset) + + line_suffix_index = line_suffix.byteindex(string) + raise "Could not find #{string}" if line_suffix_index.nil? + + Location.new(source, start_offset, length + line_suffix_index + string.bytesize) + end + end + + # This represents a comment that was encountered during parsing. It is the + # base class for all comment types. + class Comment + # The location of this comment in the source. + attr_reader :location + + # Create a new comment object with the given location. + def initialize(location) + @location = location + end + + # Implement the hash pattern matching interface for Comment. + def deconstruct_keys(keys) + { location: location } + end + + # Returns the content of the comment by slicing it from the source code. + def slice + location.slice + end + end + + # InlineComment objects are the most common. They correspond to comments in + # the source file like this one that start with #. + class InlineComment < Comment + # Returns true if this comment happens on the same line as other code and + # false if the comment is by itself. + def trailing? + !location.start_line_slice.strip.empty? + end + + # Returns a string representation of this comment. + def inspect + "#<Prism::InlineComment @location=#{location.inspect}>" + end + end + + # EmbDocComment objects correspond to comments that are surrounded by =begin + # and =end. + class EmbDocComment < Comment + # This can only be true for inline comments. + def trailing? + false + end + + # Returns a string representation of this comment. + def inspect + "#<Prism::EmbDocComment @location=#{location.inspect}>" + end + end + + # This represents a magic comment that was encountered during parsing. + class MagicComment + # A Location object representing the location of the key in the source. + attr_reader :key_loc + + # A Location object representing the location of the value in the source. + attr_reader :value_loc + + # Create a new magic comment object with the given key and value locations. + def initialize(key_loc, value_loc) + @key_loc = key_loc + @value_loc = value_loc + end + + # Returns the key of the magic comment by slicing it from the source code. + def key + key_loc.slice + end + + # Returns the value of the magic comment by slicing it from the source code. + def value + value_loc.slice + end + + # Implement the hash pattern matching interface for MagicComment. + def deconstruct_keys(keys) + { key_loc: key_loc, value_loc: value_loc } + end + + # Returns a string representation of this magic comment. + def inspect + "#<Prism::MagicComment @key=#{key.inspect} @value=#{value.inspect}>" + end + end + + # This represents an error that was encountered during parsing. + class ParseError + # The type of error. This is an _internal_ symbol that is used for + # communicating with translation layers. It is not meant to be public API. + attr_reader :type + + # The message associated with this error. + attr_reader :message + + # A Location object representing the location of this error in the source. + attr_reader :location + + # The level of this error. + attr_reader :level + + # Create a new error object with the given message and location. + def initialize(type, message, location, level) + @type = type + @message = message + @location = location + @level = level + end + + # Implement the hash pattern matching interface for ParseError. + def deconstruct_keys(keys) + { type: type, message: message, location: location, level: level } + end + + # Returns a string representation of this error. + def inspect + "#<Prism::ParseError @type=#{@type.inspect} @message=#{@message.inspect} @location=#{@location.inspect} @level=#{@level.inspect}>" + end + end + + # This represents a warning that was encountered during parsing. + class ParseWarning + # The type of warning. This is an _internal_ symbol that is used for + # communicating with translation layers. It is not meant to be public API. + attr_reader :type + + # The message associated with this warning. + attr_reader :message + + # A Location object representing the location of this warning in the source. + attr_reader :location + + # The level of this warning. + attr_reader :level + + # Create a new warning object with the given message and location. + def initialize(type, message, location, level) + @type = type + @message = message + @location = location + @level = level + end + + # Implement the hash pattern matching interface for ParseWarning. + def deconstruct_keys(keys) + { type: type, message: message, location: location, level: level } + end + + # Returns a string representation of this warning. + def inspect + "#<Prism::ParseWarning @type=#{@type.inspect} @message=#{@message.inspect} @location=#{@location.inspect} @level=#{@level.inspect}>" + end + end + + # This represents the result of a call to ::parse or ::parse_file. It contains + # the requested structure, any comments that were encounters, and any errors + # that were encountered. + class Result + # The list of comments that were encountered during parsing. + attr_reader :comments + + # The list of magic comments that were encountered during parsing. + attr_reader :magic_comments + + # An optional location that represents the location of the __END__ marker + # and the rest of the content of the file. This content is loaded into the + # DATA constant when the file being parsed is the main file being executed. + attr_reader :data_loc + + # The list of errors that were generated during parsing. + attr_reader :errors + + # The list of warnings that were generated during parsing. + attr_reader :warnings + + # A Source instance that represents the source code that was parsed. + attr_reader :source + + # Create a new result object with the given values. + def initialize(comments, magic_comments, data_loc, errors, warnings, source) + @comments = comments + @magic_comments = magic_comments + @data_loc = data_loc + @errors = errors + @warnings = warnings + @source = source + end + + # Implement the hash pattern matching interface for Result. + def deconstruct_keys(keys) + { comments: comments, magic_comments: magic_comments, data_loc: data_loc, errors: errors, warnings: warnings } + end + + # Returns the encoding of the source code that was parsed. + def encoding + source.encoding + end + + # Returns true if there were no errors during parsing and false if there + # were. + def success? + errors.empty? + end + + # Returns true if there were errors during parsing and false if there were + # not. + def failure? + !success? + end + end + + # This is a result specific to the `parse` and `parse_file` methods. + class ParseResult < Result + autoload :Comments, "prism/parse_result/comments" + autoload :Newlines, "prism/parse_result/newlines" + + private_constant :Comments + private_constant :Newlines + + # The syntax tree that was parsed from the source code. + attr_reader :value + + # Create a new parse result object with the given values. + def initialize(value, comments, magic_comments, data_loc, errors, warnings, source) + @value = value + super(comments, magic_comments, data_loc, errors, warnings, source) + end + + # Implement the hash pattern matching interface for ParseResult. + def deconstruct_keys(keys) + super.merge!(value: value) + end + + # Attach the list of comments to their respective locations in the tree. + def attach_comments! + Comments.new(self).attach! # steep:ignore + end + + # Walk the tree and mark nodes that are on a new line, loosely emulating + # the behavior of CRuby's `:line` tracepoint event. + def mark_newlines! + value.accept(Newlines.new(source.offsets.size)) # steep:ignore + end + end + + # This is a result specific to the `lex` and `lex_file` methods. + class LexResult < Result + # The list of tokens that were parsed from the source code. + attr_reader :value + + # Create a new lex result object with the given values. + def initialize(value, comments, magic_comments, data_loc, errors, warnings, source) + @value = value + super(comments, magic_comments, data_loc, errors, warnings, source) + end + + # Implement the hash pattern matching interface for LexResult. + def deconstruct_keys(keys) + super.merge!(value: value) + end + end + + # This is a result specific to the `parse_lex` and `parse_lex_file` methods. + class ParseLexResult < Result + # A tuple of the syntax tree and the list of tokens that were parsed from + # the source code. + attr_reader :value + + # Create a new parse lex result object with the given values. + def initialize(value, comments, magic_comments, data_loc, errors, warnings, source) + @value = value + super(comments, magic_comments, data_loc, errors, warnings, source) + end + + # Implement the hash pattern matching interface for ParseLexResult. + def deconstruct_keys(keys) + super.merge!(value: value) + end + end + + # This represents a token from the Ruby source. + class Token + # The Source object that represents the source this token came from. + attr_reader :source + private :source + + # The type of token that this token is. + attr_reader :type + + # A byteslice of the source that this token represents. + attr_reader :value + + # Create a new token object with the given type, value, and location. + def initialize(source, type, value, location) + @source = source + @type = type + @value = value + @location = location + end + + # Implement the hash pattern matching interface for Token. + def deconstruct_keys(keys) + { type: type, value: value, location: location } + end + + # A Location object representing the location of this token in the source. + def location + location = @location + return location if location.is_a?(Location) + @location = Location.new(source, location >> 32, location & 0xFFFFFFFF) + end + + # Implement the pretty print interface for Token. + def pretty_print(q) + q.group do + q.text(type.to_s) + self.location.pretty_print(q) + q.text("(") + q.nest(2) do + q.breakable("") + q.pp(value) + end + q.breakable("") + q.text(")") + end + end + + # Returns true if the given other token is equal to this token. + def ==(other) + Token === other && + other.type == type && + other.value == value + end + end +end diff --git a/lib/prism/parse_result/comments.rb b/lib/prism/parse_result/comments.rb new file mode 100644 index 0000000000..22c4148b2c --- /dev/null +++ b/lib/prism/parse_result/comments.rb @@ -0,0 +1,187 @@ +# frozen_string_literal: true + +module Prism + class ParseResult < Result + # When we've parsed the source, we have both the syntax tree and the list of + # comments that we found in the source. This class is responsible for + # walking the tree and finding the nearest location to attach each comment. + # + # It does this by first finding the nearest locations to each comment. + # Locations can either come from nodes directly or from location fields on + # nodes. For example, a `ClassNode` has an overall location encompassing the + # entire class, but it also has a location for the `class` keyword. + # + # Once the nearest locations are found, it determines which one to attach + # to. If it's a trailing comment (a comment on the same line as other source + # code), it will favor attaching to the nearest location that occurs before + # the comment. Otherwise it will favor attaching to the nearest location + # that is after the comment. + class Comments + # A target for attaching comments that is based on a specific node's + # location. + class NodeTarget # :nodoc: + attr_reader :node + + def initialize(node) + @node = node + end + + def start_offset + node.start_offset + end + + def end_offset + node.end_offset + end + + def encloses?(comment) + start_offset <= comment.location.start_offset && + comment.location.end_offset <= end_offset + end + + def leading_comment(comment) + node.location.leading_comment(comment) + end + + def trailing_comment(comment) + node.location.trailing_comment(comment) + end + end + + # A target for attaching comments that is based on a location field on a + # node. For example, the `end` token of a ClassNode. + class LocationTarget # :nodoc: + attr_reader :location + + def initialize(location) + @location = location + end + + def start_offset + location.start_offset + end + + def end_offset + location.end_offset + end + + def encloses?(comment) + false + end + + def leading_comment(comment) + location.leading_comment(comment) + end + + def trailing_comment(comment) + location.trailing_comment(comment) + end + end + + # The parse result that we are attaching comments to. + attr_reader :parse_result + + # Create a new Comments object that will attach comments to the given + # parse result. + def initialize(parse_result) + @parse_result = parse_result + end + + # Attach the comments to their respective locations in the tree by + # mutating the parse result. + def attach! + parse_result.comments.each do |comment| + preceding, enclosing, following = nearest_targets(parse_result.value, comment) + + if comment.trailing? + if preceding + preceding.trailing_comment(comment) + else + (following || enclosing || NodeTarget.new(parse_result.value)).leading_comment(comment) + end + else + # If a comment exists on its own line, prefer a leading comment. + if following + following.leading_comment(comment) + elsif preceding + preceding.trailing_comment(comment) + else + (enclosing || NodeTarget.new(parse_result.value)).leading_comment(comment) + end + end + end + end + + private + + # Responsible for finding the nearest targets to the given comment within + # the context of the given encapsulating node. + def nearest_targets(node, comment) + comment_start = comment.location.start_offset + comment_end = comment.location.end_offset + + targets = [] #: Array[_Target] + node.comment_targets.map do |value| + case value + when StatementsNode + targets.concat(value.body.map { |node| NodeTarget.new(node) }) + when Node + targets << NodeTarget.new(value) + when Location + targets << LocationTarget.new(value) + end + end + + targets.sort_by!(&:start_offset) + preceding = nil #: _Target? + following = nil #: _Target? + + left = 0 + right = targets.length + + # This is a custom binary search that finds the nearest nodes to the + # given comment. When it finds a node that completely encapsulates the + # comment, it recurses downward into the tree. + while left < right + middle = (left + right) / 2 + target = targets[middle] + + target_start = target.start_offset + target_end = target.end_offset + + if target.encloses?(comment) + # @type var target: NodeTarget + # The comment is completely contained by this target. Abandon the + # binary search at this level. + return nearest_targets(target.node, comment) + end + + if target_end <= comment_start + # This target falls completely before the comment. Because we will + # never consider this target or any targets before it again, this + # target must be the closest preceding target we have encountered so + # far. + preceding = target + left = middle + 1 + next + end + + if comment_end <= target_start + # This target falls completely after the comment. Because we will + # never consider this target or any targets after it again, this + # target must be the closest following target we have encountered so + # far. + following = target + right = middle + next + end + + # This should only happen if there is a bug in this parser. + raise "Comment location overlaps with a target location" + end + + [preceding, NodeTarget.new(node), following] + end + end + end +end diff --git a/lib/prism/parse_result/newlines.rb b/lib/prism/parse_result/newlines.rb new file mode 100644 index 0000000000..cc1343dfda --- /dev/null +++ b/lib/prism/parse_result/newlines.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +module Prism + class ParseResult < Result + # The :line tracepoint event gets fired whenever the Ruby VM encounters an + # expression on a new line. The types of expressions that can trigger this + # event are: + # + # * if statements + # * unless statements + # * nodes that are children of statements lists + # + # In order to keep track of the newlines, we have a list of offsets that + # come back from the parser. We assign these offsets to the first nodes that + # we find in the tree that are on those lines. + # + # Note that the logic in this file should be kept in sync with the Java + # MarkNewlinesVisitor, since that visitor is responsible for marking the + # newlines for JRuby/TruffleRuby. + # + # This file is autoloaded only when `mark_newlines!` is called, so the + # re-opening of the various nodes in this file will only be performed in + # that case. We do that to avoid storing the extra `@newline` instance + # variable on every node if we don't need it. + class Newlines < Visitor + # Create a new Newlines visitor with the given newline offsets. + def initialize(lines) + @lines = Array.new(1 + lines, false) + end + + # Permit block/lambda nodes to mark newlines within themselves. + def visit_block_node(node) + old_lines = @lines + @lines = Array.new(old_lines.size, false) + + begin + super(node) + ensure + @lines = old_lines + end + end + + alias_method :visit_lambda_node, :visit_block_node + + # Mark if/unless nodes as newlines. + def visit_if_node(node) + node.newline!(@lines) + super(node) + end + + alias_method :visit_unless_node, :visit_if_node + + # Permit statements lists to mark newlines within themselves. + def visit_statements_node(node) + node.body.each do |child| + child.newline!(@lines) + end + super(node) + end + end + end + + class Node + def newline? # :nodoc: + @newline ? true : false + end + + def newline!(lines) # :nodoc: + line = location.start_line + unless lines[line] + lines[line] = true + @newline = true + end + end + end + + class BeginNode < Node + def newline!(lines) # :nodoc: + # Never mark BeginNode with a newline flag, mark children instead. + end + end + + class ParenthesesNode < Node + def newline!(lines) # :nodoc: + # Never mark ParenthesesNode with a newline flag, mark children instead. + end + end + + class IfNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class UnlessNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class UntilNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class WhileNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class RescueModifierNode < Node + def newline!(lines) # :nodoc: + expression.newline!(lines) + end + end + + class InterpolatedMatchLastLineNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end + + class InterpolatedRegularExpressionNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end + + class InterpolatedStringNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end + + class InterpolatedSymbolNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end + + class InterpolatedXStringNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end +end diff --git a/lib/prism/pattern.rb b/lib/prism/pattern.rb new file mode 100644 index 0000000000..03fec26789 --- /dev/null +++ b/lib/prism/pattern.rb @@ -0,0 +1,268 @@ +# frozen_string_literal: true + +module Prism + # A pattern is an object that wraps a Ruby pattern matching expression. The + # expression would normally be passed to an `in` clause within a `case` + # expression or a rightward assignment expression. For example, in the + # following snippet: + # + # case node + # in ConstantPathNode[ConstantReadNode[name: :Prism], ConstantReadNode[name: :Pattern]] + # end + # + # the pattern is the <tt>ConstantPathNode[...]</tt> expression. + # + # The pattern gets compiled into an object that responds to #call by running + # the #compile method. This method itself will run back through Prism to + # parse the expression into a tree, then walk the tree to generate the + # necessary callable objects. For example, if you wanted to compile the + # expression above into a callable, you would: + # + # callable = Prism::Pattern.new("ConstantPathNode[ConstantReadNode[name: :Prism], ConstantReadNode[name: :Pattern]]").compile + # callable.call(node) + # + # The callable object returned by #compile is guaranteed to respond to #call + # with a single argument, which is the node to match against. It also is + # guaranteed to respond to #===, which means it itself can be used in a `case` + # expression, as in: + # + # case node + # when callable + # end + # + # If the query given to the initializer cannot be compiled into a valid + # matcher (either because of a syntax error or because it is using syntax we + # do not yet support) then a Prism::Pattern::CompilationError will be + # raised. + class Pattern + # Raised when the query given to a pattern is either invalid Ruby syntax or + # is using syntax that we don't yet support. + class CompilationError < StandardError + # Create a new CompilationError with the given representation of the node + # that caused the error. + def initialize(repr) + super(<<~ERROR) + prism was unable to compile the pattern you provided into a usable + expression. It failed on to understand the node represented by: + + #{repr} + + Note that not all syntax supported by Ruby's pattern matching syntax + is also supported by prism's patterns. If you're using some syntax + that you believe should be supported, please open an issue on + GitHub at https://github.com/ruby/prism/issues/new. + ERROR + end + end + + # The query that this pattern was initialized with. + attr_reader :query + + # Create a new pattern with the given query. The query should be a string + # containing a Ruby pattern matching expression. + def initialize(query) + @query = query + @compiled = nil + end + + # Compile the query into a callable object that can be used to match against + # nodes. + def compile + result = Prism.parse("case nil\nin #{query}\nend") + + case_match_node = result.value.statements.body.last + raise CompilationError, case_match_node.inspect unless case_match_node.is_a?(CaseMatchNode) + + in_node = case_match_node.conditions.last + raise CompilationError, in_node.inspect unless in_node.is_a?(InNode) + + compile_node(in_node.pattern) + end + + # Scan the given node and all of its children for nodes that match the + # pattern. If a block is given, it will be called with each node that + # matches the pattern. If no block is given, an enumerator will be returned + # that will yield each node that matches the pattern. + def scan(root) + return to_enum(:scan, root) unless block_given? + + @compiled ||= compile + queue = [root] + + while (node = queue.shift) + yield node if @compiled.call(node) # steep:ignore + queue.concat(node.compact_child_nodes) + end + end + + private + + # Shortcut for combining two procs into one that returns true if both return + # true. + def combine_and(left, right) + ->(other) { left.call(other) && right.call(other) } + end + + # Shortcut for combining two procs into one that returns true if either + # returns true. + def combine_or(left, right) + ->(other) { left.call(other) || right.call(other) } + end + + # Raise an error because the given node is not supported. + def compile_error(node) + raise CompilationError, node.inspect + end + + # in [foo, bar, baz] + def compile_array_pattern_node(node) + compile_error(node) if !node.rest.nil? || node.posts.any? + + constant = node.constant + compiled_constant = compile_node(constant) if constant + + preprocessed = node.requireds.map { |required| compile_node(required) } + + compiled_requireds = ->(other) do + deconstructed = other.deconstruct + + deconstructed.length == preprocessed.length && + preprocessed + .zip(deconstructed) + .all? { |(matcher, value)| matcher.call(value) } + end + + if compiled_constant + combine_and(compiled_constant, compiled_requireds) + else + compiled_requireds + end + end + + # in foo | bar + def compile_alternation_pattern_node(node) + combine_or(compile_node(node.left), compile_node(node.right)) + end + + # in Prism::ConstantReadNode + def compile_constant_path_node(node) + parent = node.parent + + if parent.is_a?(ConstantReadNode) && parent.slice == "Prism" + name = node.name + raise CompilationError, node.inspect if name.nil? + + compile_constant_name(node, name) + else + compile_error(node) + end + end + + # in ConstantReadNode + # in String + def compile_constant_read_node(node) + compile_constant_name(node, node.name) + end + + # Compile a name associated with a constant. + def compile_constant_name(node, name) + if Prism.const_defined?(name, false) + clazz = Prism.const_get(name) + + ->(other) { clazz === other } + elsif Object.const_defined?(name, false) + clazz = Object.const_get(name) + + ->(other) { clazz === other } + else + compile_error(node) + end + end + + # in InstanceVariableReadNode[name: Symbol] + # in { name: Symbol } + def compile_hash_pattern_node(node) + compile_error(node) if node.rest + compiled_constant = compile_node(node.constant) if node.constant + + preprocessed = + node.elements.to_h do |element| + key = element.key + if key.is_a?(SymbolNode) + [key.unescaped.to_sym, compile_node(element.value)] + else + raise CompilationError, element.inspect + end + end + + compiled_keywords = ->(other) do + deconstructed = other.deconstruct_keys(preprocessed.keys) + + preprocessed.all? do |keyword, matcher| + deconstructed.key?(keyword) && matcher.call(deconstructed[keyword]) + end + end + + if compiled_constant + combine_and(compiled_constant, compiled_keywords) + else + compiled_keywords + end + end + + # in nil + def compile_nil_node(node) + ->(attribute) { attribute.nil? } + end + + # in /foo/ + def compile_regular_expression_node(node) + regexp = Regexp.new(node.unescaped, node.closing[1..]) + + ->(attribute) { regexp === attribute } + end + + # in "" + # in "foo" + def compile_string_node(node) + string = node.unescaped + + ->(attribute) { string === attribute } + end + + # in :+ + # in :foo + def compile_symbol_node(node) + symbol = node.unescaped.to_sym + + ->(attribute) { symbol === attribute } + end + + # Compile any kind of node. Dispatch out to the individual compilation + # methods based on the type of node. + def compile_node(node) + case node + when AlternationPatternNode + compile_alternation_pattern_node(node) + when ArrayPatternNode + compile_array_pattern_node(node) + when ConstantPathNode + compile_constant_path_node(node) + when ConstantReadNode + compile_constant_read_node(node) + when HashPatternNode + compile_hash_pattern_node(node) + when NilNode + compile_nil_node(node) + when RegularExpressionNode + compile_regular_expression_node(node) + when StringNode + compile_string_node(node) + when SymbolNode + compile_symbol_node(node) + else + compile_error(node) + end + end + end +end diff --git a/lib/prism/polyfill/byteindex.rb b/lib/prism/polyfill/byteindex.rb new file mode 100644 index 0000000000..98c6089f14 --- /dev/null +++ b/lib/prism/polyfill/byteindex.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# Polyfill for String#byteindex, which didn't exist until Ruby 3.2. +if !("".respond_to?(:byteindex)) + String.include( + Module.new { + def byteindex(needle, offset = 0) + charindex = index(needle, offset) + slice(0...charindex).bytesize if charindex + end + } + ) +end diff --git a/lib/prism/polyfill/unpack1.rb b/lib/prism/polyfill/unpack1.rb new file mode 100644 index 0000000000..3fa9b5a0c5 --- /dev/null +++ b/lib/prism/polyfill/unpack1.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# Polyfill for String#unpack1 with the offset parameter. Not all Ruby engines +# have Method#parameters implemented, so we check the arity instead if +# necessary. +if (unpack1 = String.instance_method(:unpack1)).respond_to?(:parameters) ? unpack1.parameters.none? { |_, name| name == :offset } : (unpack1.arity == 1) + String.prepend( + Module.new { + def unpack1(format, offset: 0) # :nodoc: + offset == 0 ? super(format) : self[offset..].unpack1(format) # steep:ignore + end + } + ) +end diff --git a/lib/prism/prism.gemspec b/lib/prism/prism.gemspec new file mode 100644 index 0000000000..b4504dbf4b --- /dev/null +++ b/lib/prism/prism.gemspec @@ -0,0 +1,161 @@ +# frozen_string_literal: true + +Gem::Specification.new do |spec| + spec.name = "prism" + spec.version = "0.29.0" + spec.authors = ["Shopify"] + spec.email = ["ruby@shopify.com"] + + spec.summary = "Prism Ruby parser" + spec.homepage = "https://github.com/ruby/prism" + spec.license = "MIT" + + spec.required_ruby_version = ">= 2.7.0" + + spec.require_paths = ["lib"] + spec.files = [ + "BSDmakefile", + "CHANGELOG.md", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE.md", + "Makefile", + "README.md", + "config.yml", + "docs/build_system.md", + "docs/configuration.md", + "docs/cruby_compilation.md", + "docs/design.md", + "docs/encoding.md", + "docs/fuzzing.md", + "docs/heredocs.md", + "docs/javascript.md", + "docs/local_variable_depth.md", + "docs/mapping.md", + "docs/parser_translation.md", + "docs/parsing_rules.md", + "docs/releasing.md", + "docs/ripper_translation.md", + "docs/ruby_api.md", + "docs/ruby_parser_translation.md", + "docs/serialization.md", + "docs/testing.md", + "ext/prism/api_node.c", + "ext/prism/api_pack.c", + "ext/prism/extension.c", + "ext/prism/extension.h", + "include/prism.h", + "include/prism/ast.h", + "include/prism/defines.h", + "include/prism/diagnostic.h", + "include/prism/encoding.h", + "include/prism/node.h", + "include/prism/options.h", + "include/prism/pack.h", + "include/prism/parser.h", + "include/prism/prettyprint.h", + "include/prism/regexp.h", + "include/prism/static_literals.h", + "include/prism/util/pm_buffer.h", + "include/prism/util/pm_char.h", + "include/prism/util/pm_constant_pool.h", + "include/prism/util/pm_integer.h", + "include/prism/util/pm_list.h", + "include/prism/util/pm_memchr.h", + "include/prism/util/pm_newline_list.h", + "include/prism/util/pm_strncasecmp.h", + "include/prism/util/pm_string.h", + "include/prism/util/pm_string_list.h", + "include/prism/util/pm_strpbrk.h", + "include/prism/version.h", + "lib/prism.rb", + "lib/prism/compiler.rb", + "lib/prism/desugar_compiler.rb", + "lib/prism/dispatcher.rb", + "lib/prism/dot_visitor.rb", + "lib/prism/dsl.rb", + "lib/prism/ffi.rb", + "lib/prism/inspect_visitor.rb", + "lib/prism/lex_compat.rb", + "lib/prism/mutation_compiler.rb", + "lib/prism/node_ext.rb", + "lib/prism/node.rb", + "lib/prism/pack.rb", + "lib/prism/parse_result.rb", + "lib/prism/parse_result/comments.rb", + "lib/prism/parse_result/newlines.rb", + "lib/prism/pattern.rb", + "lib/prism/polyfill/byteindex.rb", + "lib/prism/polyfill/unpack1.rb", + "lib/prism/reflection.rb", + "lib/prism/serialize.rb", + "lib/prism/translation.rb", + "lib/prism/translation/parser.rb", + "lib/prism/translation/parser33.rb", + "lib/prism/translation/parser34.rb", + "lib/prism/translation/parser/compiler.rb", + "lib/prism/translation/parser/lexer.rb", + "lib/prism/translation/parser/rubocop.rb", + "lib/prism/translation/ripper.rb", + "lib/prism/translation/ripper/sexp.rb", + "lib/prism/translation/ripper/shim.rb", + "lib/prism/translation/ruby_parser.rb", + "lib/prism/visitor.rb", + "prism.gemspec", + "rbi/prism.rbi", + "rbi/prism/compiler.rbi", + "rbi/prism/inspect_visitor.rbi", + "rbi/prism/node_ext.rbi", + "rbi/prism/node.rbi", + "rbi/prism/parse_result.rbi", + "rbi/prism/reflection.rbi", + "rbi/prism/translation/parser.rbi", + "rbi/prism/translation/parser33.rbi", + "rbi/prism/translation/parser34.rbi", + "rbi/prism/translation/ripper.rbi", + "rbi/prism/visitor.rbi", + "sig/prism.rbs", + "sig/prism/compiler.rbs", + "sig/prism/dispatcher.rbs", + "sig/prism/dot_visitor.rbs", + "sig/prism/dsl.rbs", + "sig/prism/inspect_visitor.rbs", + "sig/prism/lex_compat.rbs", + "sig/prism/mutation_compiler.rbs", + "sig/prism/node_ext.rbs", + "sig/prism/node.rbs", + "sig/prism/pack.rbs", + "sig/prism/parse_result.rbs", + "sig/prism/pattern.rbs", + "sig/prism/reflection.rbs", + "sig/prism/serialize.rbs", + "sig/prism/visitor.rbs", + "src/diagnostic.c", + "src/encoding.c", + "src/node.c", + "src/options.c", + "src/pack.c", + "src/prettyprint.c", + "src/prism.c", + "src/regexp.c", + "src/serialize.c", + "src/static_literals.c", + "src/token_type.c", + "src/util/pm_buffer.c", + "src/util/pm_char.c", + "src/util/pm_constant_pool.c", + "src/util/pm_integer.c", + "src/util/pm_list.c", + "src/util/pm_memchr.c", + "src/util/pm_newline_list.c", + "src/util/pm_string_list.c", + "src/util/pm_string.c", + "src/util/pm_strncasecmp.c", + "src/util/pm_strpbrk.c" + ] + + spec.extensions = ["ext/prism/extconf.rb"] + spec.metadata["allowed_push_host"] = "https://rubygems.org" + spec.metadata["source_code_uri"] = "https://github.com/ruby/prism" + spec.metadata["changelog_uri"] = "https://github.com/ruby/prism/blob/main/CHANGELOG.md" +end diff --git a/lib/prism/translation.rb b/lib/prism/translation.rb new file mode 100644 index 0000000000..8b75e8a3ab --- /dev/null +++ b/lib/prism/translation.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Prism + # This module is responsible for converting the prism syntax tree into other + # syntax trees. + module Translation # steep:ignore + autoload :Parser, "prism/translation/parser" + autoload :Parser33, "prism/translation/parser33" + autoload :Parser34, "prism/translation/parser34" + autoload :Ripper, "prism/translation/ripper" + autoload :RubyParser, "prism/translation/ruby_parser" + end +end diff --git a/lib/prism/translation/parser.rb b/lib/prism/translation/parser.rb new file mode 100644 index 0000000000..3748fc896e --- /dev/null +++ b/lib/prism/translation/parser.rb @@ -0,0 +1,307 @@ +# frozen_string_literal: true + +begin + require "parser" +rescue LoadError + warn(%q{Error: Unable to load parser. Add `gem "parser"` to your Gemfile.}) + exit(1) +end + +module Prism + module Translation + # This class is the entry-point for converting a prism syntax tree into the + # whitequark/parser gem's syntax tree. It inherits from the base parser for + # the parser gem, and overrides the parse* methods to parse with prism and + # then translate. + class Parser < ::Parser::Base + Diagnostic = ::Parser::Diagnostic # :nodoc: + private_constant :Diagnostic + + # The parser gem has a list of diagnostics with a hard-coded set of error + # messages. We create our own diagnostic class in order to set our own + # error messages. + class PrismDiagnostic < Diagnostic + # This is the cached message coming from prism. + attr_reader :message + + # Initialize a new diagnostic with the given message and location. + def initialize(message, level, reason, location) + @message = message + super(level, reason, {}, location, []) + end + end + + Racc_debug_parser = false # :nodoc: + + def version # :nodoc: + 34 + end + + # The default encoding for Ruby files is UTF-8. + def default_encoding + Encoding::UTF_8 + end + + def yyerror # :nodoc: + end + + # Parses a source buffer and returns the AST. + def parse(source_buffer) + @source_buffer = source_buffer + source = source_buffer.source + + offset_cache = build_offset_cache(source) + result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache) + + build_ast(result.value, offset_cache) + ensure + @source_buffer = nil + end + + # Parses a source buffer and returns the AST and the source code comments. + def parse_with_comments(source_buffer) + @source_buffer = source_buffer + source = source_buffer.source + + offset_cache = build_offset_cache(source) + result = unwrap(Prism.parse(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache) + + [ + build_ast(result.value, offset_cache), + build_comments(result.comments, offset_cache) + ] + ensure + @source_buffer = nil + end + + # Parses a source buffer and returns the AST, the source code comments, + # and the tokens emitted by the lexer. + def tokenize(source_buffer, recover = false) + @source_buffer = source_buffer + source = source_buffer.source + + offset_cache = build_offset_cache(source) + result = + begin + unwrap(Prism.parse_lex(source, filepath: source_buffer.name, version: convert_for_prism(version), scopes: [[]]), offset_cache) + rescue ::Parser::SyntaxError + raise if !recover + end + + program, tokens = result.value + ast = build_ast(program, offset_cache) if result.success? + + [ + ast, + build_comments(result.comments, offset_cache), + build_tokens(tokens, offset_cache) + ] + ensure + @source_buffer = nil + end + + # Since prism resolves num params for us, we don't need to support this + # kind of logic here. + def try_declare_numparam(node) + node.children[0].match?(/\A_[1-9]\z/) + end + + private + + # This is a hook to allow consumers to disable some errors if they don't + # want them to block creating the syntax tree. + def valid_error?(error) + true + end + + # This is a hook to allow consumers to disable some warnings if they don't + # want them to block creating the syntax tree. + def valid_warning?(warning) + true + end + + # Build a diagnostic from the given prism parse error. + def error_diagnostic(error, offset_cache) + location = error.location + diagnostic_location = build_range(location, offset_cache) + + case error.type + when :argument_block_multi + Diagnostic.new(:error, :block_and_blockarg, {}, diagnostic_location, []) + when :argument_formal_constant + Diagnostic.new(:error, :argument_const, {}, diagnostic_location, []) + when :argument_formal_class + Diagnostic.new(:error, :argument_cvar, {}, diagnostic_location, []) + when :argument_formal_global + Diagnostic.new(:error, :argument_gvar, {}, diagnostic_location, []) + when :argument_formal_ivar + Diagnostic.new(:error, :argument_ivar, {}, diagnostic_location, []) + when :argument_no_forwarding_amp + Diagnostic.new(:error, :no_anonymous_blockarg, {}, diagnostic_location, []) + when :argument_no_forwarding_star + Diagnostic.new(:error, :no_anonymous_restarg, {}, diagnostic_location, []) + when :argument_no_forwarding_star_star + Diagnostic.new(:error, :no_anonymous_kwrestarg, {}, diagnostic_location, []) + when :begin_lonely_else + location = location.copy(length: 4) + diagnostic_location = build_range(location, offset_cache) + Diagnostic.new(:error, :useless_else, {}, diagnostic_location, []) + when :class_name, :module_name + Diagnostic.new(:error, :module_name_const, {}, diagnostic_location, []) + when :class_in_method + Diagnostic.new(:error, :class_in_def, {}, diagnostic_location, []) + when :def_endless_setter + Diagnostic.new(:error, :endless_setter, {}, diagnostic_location, []) + when :embdoc_term + Diagnostic.new(:error, :embedded_document, {}, diagnostic_location, []) + when :incomplete_variable_class, :incomplete_variable_class_3_3 + location = location.copy(length: location.length + 1) + diagnostic_location = build_range(location, offset_cache) + + Diagnostic.new(:error, :cvar_name, { name: location.slice }, diagnostic_location, []) + when :incomplete_variable_instance, :incomplete_variable_instance_3_3 + location = location.copy(length: location.length + 1) + diagnostic_location = build_range(location, offset_cache) + + Diagnostic.new(:error, :ivar_name, { name: location.slice }, diagnostic_location, []) + when :invalid_variable_global, :invalid_variable_global_3_3 + Diagnostic.new(:error, :gvar_name, { name: location.slice }, diagnostic_location, []) + when :module_in_method + Diagnostic.new(:error, :module_in_def, {}, diagnostic_location, []) + when :numbered_parameter_ordinary + Diagnostic.new(:error, :ordinary_param_defined, {}, diagnostic_location, []) + when :numbered_parameter_outer_scope + Diagnostic.new(:error, :numparam_used_in_outer_scope, {}, diagnostic_location, []) + when :parameter_circular + Diagnostic.new(:error, :circular_argument_reference, { var_name: location.slice }, diagnostic_location, []) + when :parameter_name_repeat + Diagnostic.new(:error, :duplicate_argument, {}, diagnostic_location, []) + when :parameter_numbered_reserved + Diagnostic.new(:error, :reserved_for_numparam, { name: location.slice }, diagnostic_location, []) + when :regexp_unknown_options + Diagnostic.new(:error, :regexp_options, { options: location.slice[1..] }, diagnostic_location, []) + when :singleton_for_literals + Diagnostic.new(:error, :singleton_literal, {}, diagnostic_location, []) + when :string_literal_eof + Diagnostic.new(:error, :string_eof, {}, diagnostic_location, []) + when :unexpected_token_ignore + Diagnostic.new(:error, :unexpected_token, { token: location.slice }, diagnostic_location, []) + when :write_target_in_method + Diagnostic.new(:error, :dynamic_const, {}, diagnostic_location, []) + else + PrismDiagnostic.new(error.message, :error, error.type, diagnostic_location) + end + end + + # Build a diagnostic from the given prism parse warning. + def warning_diagnostic(warning, offset_cache) + diagnostic_location = build_range(warning.location, offset_cache) + + case warning.type + when :ambiguous_first_argument_plus + Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "+" }, diagnostic_location, []) + when :ambiguous_first_argument_minus + Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "-" }, diagnostic_location, []) + when :ambiguous_prefix_ampersand + Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "&" }, diagnostic_location, []) + when :ambiguous_prefix_star + Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "*" }, diagnostic_location, []) + when :ambiguous_prefix_star_star + Diagnostic.new(:warning, :ambiguous_prefix, { prefix: "**" }, diagnostic_location, []) + when :ambiguous_slash + Diagnostic.new(:warning, :ambiguous_regexp, {}, diagnostic_location, []) + when :dot_dot_dot_eol + Diagnostic.new(:warning, :triple_dot_at_eol, {}, diagnostic_location, []) + when :duplicated_hash_key + # skip, parser does this on its own + else + PrismDiagnostic.new(warning.message, :warning, warning.type, diagnostic_location) + end + end + + # If there was a error generated during the parse, then raise an + # appropriate syntax error. Otherwise return the result. + def unwrap(result, offset_cache) + result.errors.each do |error| + next unless valid_error?(error) + diagnostics.process(error_diagnostic(error, offset_cache)) + end + + result.warnings.each do |warning| + next unless valid_warning?(warning) + diagnostic = warning_diagnostic(warning, offset_cache) + diagnostics.process(diagnostic) if diagnostic + end + + result + end + + # Prism deals with offsets in bytes, while the parser gem deals with + # offsets in characters. We need to handle this conversion in order to + # build the parser gem AST. + # + # If the bytesize of the source is the same as the length, then we can + # just use the offset directly. Otherwise, we build an array where the + # index is the byte offset and the value is the character offset. + def build_offset_cache(source) + if source.bytesize == source.length + -> (offset) { offset } + else + offset_cache = [] + offset = 0 + + source.each_char do |char| + char.bytesize.times { offset_cache << offset } + offset += 1 + end + + offset_cache << offset + end + end + + # Build the parser gem AST from the prism AST. + def build_ast(program, offset_cache) + program.accept(Compiler.new(self, offset_cache)) + end + + # Build the parser gem comments from the prism comments. + def build_comments(comments, offset_cache) + comments.map do |comment| + ::Parser::Source::Comment.new(build_range(comment.location, offset_cache)) + end + end + + # Build the parser gem tokens from the prism tokens. + def build_tokens(tokens, offset_cache) + Lexer.new(source_buffer, tokens, offset_cache).to_a + end + + # Build a range from a prism location. + def build_range(location, offset_cache) + ::Parser::Source::Range.new( + source_buffer, + offset_cache[location.start_offset], + offset_cache[location.end_offset] + ) + end + + # Converts the version format handled by Parser to the format handled by Prism. + def convert_for_prism(version) + case version + when 33 + "3.3.1" + when 34 + "3.4.0" + else + "latest" + end + end + + require_relative "parser/compiler" + require_relative "parser/lexer" + + private_constant :Compiler + private_constant :Lexer + end + end +end diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb new file mode 100644 index 0000000000..08e54a7bf3 --- /dev/null +++ b/lib/prism/translation/parser/compiler.rb @@ -0,0 +1,2111 @@ +# frozen_string_literal: true + +module Prism + module Translation + class Parser + # A visitor that knows how to convert a prism syntax tree into the + # whitequark/parser gem's syntax tree. + class Compiler < ::Prism::Compiler + # Raised when the tree is malformed or there is a bug in the compiler. + class CompilationError < StandardError + end + + # The Parser::Base instance that is being used to build the AST. + attr_reader :parser + + # The Parser::Builders::Default instance that is being used to build the + # AST. + attr_reader :builder + + # The Parser::Source::Buffer instance that is holding a reference to the + # source code. + attr_reader :source_buffer + + # The offset cache that is used to map between byte and character + # offsets in the file. + attr_reader :offset_cache + + # The types of values that can be forwarded in the current scope. + attr_reader :forwarding + + # Whether or not the current node is in a destructure. + attr_reader :in_destructure + + # Whether or not the current node is in a pattern. + attr_reader :in_pattern + + # Initialize a new compiler with the given parser, offset cache, and + # options. + def initialize(parser, offset_cache, forwarding: [], in_destructure: false, in_pattern: false) + @parser = parser + @builder = parser.builder + @source_buffer = parser.source_buffer + @offset_cache = offset_cache + + @forwarding = forwarding + @in_destructure = in_destructure + @in_pattern = in_pattern + end + + # alias foo bar + # ^^^^^^^^^^^^^ + def visit_alias_method_node(node) + builder.alias(token(node.keyword_loc), visit(node.new_name), visit(node.old_name)) + end + + # alias $foo $bar + # ^^^^^^^^^^^^^^^ + def visit_alias_global_variable_node(node) + builder.alias(token(node.keyword_loc), visit(node.new_name), visit(node.old_name)) + end + + # foo => bar | baz + # ^^^^^^^^^ + def visit_alternation_pattern_node(node) + builder.match_alt(visit(node.left), token(node.operator_loc), visit(node.right)) + end + + # a and b + # ^^^^^^^ + def visit_and_node(node) + builder.logical_op(:and, visit(node.left), token(node.operator_loc), visit(node.right)) + end + + # [] + # ^^ + def visit_array_node(node) + builder.array(token(node.opening_loc), visit_all(node.elements), token(node.closing_loc)) + end + + # foo => [bar] + # ^^^^^ + def visit_array_pattern_node(node) + elements = [*node.requireds] + elements << node.rest if !node.rest.nil? && !node.rest.is_a?(ImplicitRestNode) + elements.concat(node.posts) + visited = visit_all(elements) + + if node.rest.is_a?(ImplicitRestNode) + visited[-1] = builder.match_with_trailing_comma(visited[-1], token(node.rest.location)) + end + + if node.constant + builder.const_pattern(visit(node.constant), token(node.opening_loc), builder.array_pattern(nil, visited, nil), token(node.closing_loc)) + else + builder.array_pattern(token(node.opening_loc), visited, token(node.closing_loc)) + end + end + + # foo(bar) + # ^^^ + def visit_arguments_node(node) + visit_all(node.arguments) + end + + # { a: 1 } + # ^^^^ + def visit_assoc_node(node) + if in_pattern + if node.value.is_a?(ImplicitNode) + if node.key.is_a?(SymbolNode) + builder.match_hash_var([node.key.unescaped, srange(node.key.location)]) + else + builder.match_hash_var_from_str(token(node.key.opening_loc), visit_all(node.key.parts), token(node.key.closing_loc)) + end + else + builder.pair_keyword([node.key.unescaped, srange(node.key.location)], visit(node.value)) + end + elsif node.value.is_a?(ImplicitNode) + if (value = node.value.value).is_a?(LocalVariableReadNode) + builder.pair_keyword( + [node.key.unescaped, srange(node.key)], + builder.ident([value.name, srange(node.key.value_loc)]).updated(:lvar) + ) + else + builder.pair_label([node.key.unescaped, srange(node.key.location)]) + end + elsif node.operator_loc + builder.pair(visit(node.key), token(node.operator_loc), visit(node.value)) + elsif node.key.is_a?(SymbolNode) && node.key.opening_loc.nil? + builder.pair_keyword([node.key.unescaped, srange(node.key.location)], visit(node.value)) + else + parts = + if node.key.is_a?(SymbolNode) + [builder.string_internal([node.key.unescaped, srange(node.key.value_loc)])] + else + visit_all(node.key.parts) + end + + builder.pair_quoted(token(node.key.opening_loc), parts, token(node.key.closing_loc), visit(node.value)) + end + end + + # def foo(**); bar(**); end + # ^^ + # + # { **foo } + # ^^^^^ + def visit_assoc_splat_node(node) + if node.value.nil? && forwarding.include?(:**) + builder.forwarded_kwrestarg(token(node.operator_loc)) + else + builder.kwsplat(token(node.operator_loc), visit(node.value)) + end + end + + # $+ + # ^^ + def visit_back_reference_read_node(node) + builder.back_ref(token(node.location)) + end + + # begin end + # ^^^^^^^^^ + def visit_begin_node(node) + rescue_bodies = [] + + if (rescue_clause = node.rescue_clause) + begin + find_start_offset = (rescue_clause.reference&.location || rescue_clause.exceptions.last&.location || rescue_clause.keyword_loc).end_offset + find_end_offset = (rescue_clause.statements&.location&.start_offset || rescue_clause.consequent&.location&.start_offset || (find_start_offset + 1)) + + rescue_bodies << builder.rescue_body( + token(rescue_clause.keyword_loc), + rescue_clause.exceptions.any? ? builder.array(nil, visit_all(rescue_clause.exceptions), nil) : nil, + token(rescue_clause.operator_loc), + visit(rescue_clause.reference), + srange_find(find_start_offset, find_end_offset, [";"]), + visit(rescue_clause.statements) + ) + end until (rescue_clause = rescue_clause.consequent).nil? + end + + begin_body = + builder.begin_body( + visit(node.statements), + rescue_bodies, + token(node.else_clause&.else_keyword_loc), + visit(node.else_clause), + token(node.ensure_clause&.ensure_keyword_loc), + visit(node.ensure_clause&.statements) + ) + + if node.begin_keyword_loc + builder.begin_keyword(token(node.begin_keyword_loc), begin_body, token(node.end_keyword_loc)) + else + begin_body + end + end + + # foo(&bar) + # ^^^^ + def visit_block_argument_node(node) + builder.block_pass(token(node.operator_loc), visit(node.expression)) + end + + # foo { |; bar| } + # ^^^ + def visit_block_local_variable_node(node) + builder.shadowarg(token(node.location)) + end + + # A block on a keyword or method call. + def visit_block_node(node) + raise CompilationError, "Cannot directly compile block nodes" + end + + # def foo(&bar); end + # ^^^^ + def visit_block_parameter_node(node) + builder.blockarg(token(node.operator_loc), token(node.name_loc)) + end + + # A block's parameters. + def visit_block_parameters_node(node) + [*visit(node.parameters)].concat(visit_all(node.locals)) + end + + # break + # ^^^^^ + # + # break foo + # ^^^^^^^^^ + def visit_break_node(node) + builder.keyword_cmd(:break, token(node.keyword_loc), nil, visit(node.arguments) || [], nil) + end + + # foo + # ^^^ + # + # foo.bar + # ^^^^^^^ + # + # foo.bar() {} + # ^^^^^^^^^^^^ + def visit_call_node(node) + name = node.name + arguments = node.arguments&.arguments || [] + block = node.block + + if block.is_a?(BlockArgumentNode) + arguments = [*arguments, block] + block = nil + end + + if node.call_operator_loc.nil? + case name + when :-@ + case (receiver = node.receiver).type + when :integer_node, :float_node, :rational_node, :imaginary_node + return visit(numeric_negate(node.message_loc, receiver)) + end + when :! + return visit_block(builder.not_op(token(node.message_loc), token(node.opening_loc), visit(node.receiver), token(node.closing_loc)), block) + when :=~ + if (receiver = node.receiver).is_a?(RegularExpressionNode) + return builder.match_op(visit(receiver), token(node.message_loc), visit(node.arguments.arguments.first)) + end + when :[] + return visit_block(builder.index(visit(node.receiver), token(node.opening_loc), visit_all(arguments), token(node.closing_loc)), block) + when :[]= + if node.message != "[]=" && node.arguments && block.nil? && !node.safe_navigation? + arguments = node.arguments.arguments[...-1] + arguments << node.block if node.block + + return visit_block( + builder.assign( + builder.index_asgn( + visit(node.receiver), + token(node.opening_loc), + visit_all(arguments), + token(node.closing_loc), + ), + srange_find(node.message_loc.end_offset, node.arguments.arguments.last.location.start_offset, ["="]), + visit(node.arguments.arguments.last) + ), + block + ) + end + end + end + + message_loc = node.message_loc + call_operator_loc = node.call_operator_loc + call_operator = [{ "." => :dot, "&." => :anddot, "::" => "::" }.fetch(call_operator_loc.slice), srange(call_operator_loc)] if call_operator_loc + + visit_block( + if name.end_with?("=") && !message_loc.slice.end_with?("=") && node.arguments && block.nil? + builder.assign( + builder.attr_asgn(visit(node.receiver), call_operator, token(message_loc)), + srange_find(message_loc.end_offset, node.arguments.location.start_offset, ["="]), + visit(node.arguments.arguments.last) + ) + else + builder.call_method( + visit(node.receiver), + call_operator, + message_loc ? [node.name, srange(message_loc)] : nil, + token(node.opening_loc), + visit_all(arguments), + token(node.closing_loc) + ) + end, + block + ) + end + + # foo.bar += baz + # ^^^^^^^^^^^^^^^ + def visit_call_operator_write_node(node) + call_operator_loc = node.call_operator_loc + + builder.op_assign( + builder.call_method( + visit(node.receiver), + call_operator_loc.nil? ? nil : [{ "." => :dot, "&." => :anddot, "::" => "::" }.fetch(call_operator_loc.slice), srange(call_operator_loc)], + node.message_loc ? [node.read_name, srange(node.message_loc)] : nil, + nil, + [], + nil + ), + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], + visit(node.value) + ) + end + + # foo.bar &&= baz + # ^^^^^^^^^^^^^^^ + def visit_call_and_write_node(node) + call_operator_loc = node.call_operator_loc + + builder.op_assign( + builder.call_method( + visit(node.receiver), + call_operator_loc.nil? ? nil : [{ "." => :dot, "&." => :anddot, "::" => "::" }.fetch(call_operator_loc.slice), srange(call_operator_loc)], + node.message_loc ? [node.read_name, srange(node.message_loc)] : nil, + nil, + [], + nil + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # foo.bar ||= baz + # ^^^^^^^^^^^^^^^ + def visit_call_or_write_node(node) + call_operator_loc = node.call_operator_loc + + builder.op_assign( + builder.call_method( + visit(node.receiver), + call_operator_loc.nil? ? nil : [{ "." => :dot, "&." => :anddot, "::" => "::" }.fetch(call_operator_loc.slice), srange(call_operator_loc)], + node.message_loc ? [node.read_name, srange(node.message_loc)] : nil, + nil, + [], + nil + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # foo.bar, = 1 + # ^^^^^^^ + def visit_call_target_node(node) + call_operator_loc = node.call_operator_loc + + builder.attr_asgn( + visit(node.receiver), + call_operator_loc.nil? ? nil : [{ "." => :dot, "&." => :anddot, "::" => "::" }.fetch(call_operator_loc.slice), srange(call_operator_loc)], + token(node.message_loc) + ) + end + + # foo => bar => baz + # ^^^^^^^^^^ + def visit_capture_pattern_node(node) + builder.match_as(visit(node.value), token(node.operator_loc), visit(node.target)) + end + + # case foo; when bar; end + # ^^^^^^^^^^^^^^^^^^^^^^^ + def visit_case_node(node) + builder.case( + token(node.case_keyword_loc), + visit(node.predicate), + visit_all(node.conditions), + token(node.consequent&.else_keyword_loc), + visit(node.consequent), + token(node.end_keyword_loc) + ) + end + + # case foo; in bar; end + # ^^^^^^^^^^^^^^^^^^^^^ + def visit_case_match_node(node) + builder.case_match( + token(node.case_keyword_loc), + visit(node.predicate), + visit_all(node.conditions), + token(node.consequent&.else_keyword_loc), + visit(node.consequent), + token(node.end_keyword_loc) + ) + end + + # class Foo; end + # ^^^^^^^^^^^^^^ + def visit_class_node(node) + builder.def_class( + token(node.class_keyword_loc), + visit(node.constant_path), + token(node.inheritance_operator_loc), + visit(node.superclass), + node.body&.accept(copy_compiler(forwarding: [])), + token(node.end_keyword_loc) + ) + end + + # @@foo + # ^^^^^ + def visit_class_variable_read_node(node) + builder.cvar(token(node.location)) + end + + # @@foo = 1 + # ^^^^^^^^^ + def visit_class_variable_write_node(node) + builder.assign( + builder.assignable(builder.cvar(token(node.name_loc))), + token(node.operator_loc), + visit(node.value) + ) + end + + # @@foo += bar + # ^^^^^^^^^^^^ + def visit_class_variable_operator_write_node(node) + builder.op_assign( + builder.assignable(builder.cvar(token(node.name_loc))), + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], + visit(node.value) + ) + end + + # @@foo &&= bar + # ^^^^^^^^^^^^^ + def visit_class_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.cvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # @@foo ||= bar + # ^^^^^^^^^^^^^ + def visit_class_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.cvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # @@foo, = bar + # ^^^^^ + def visit_class_variable_target_node(node) + builder.assignable(builder.cvar(token(node.location))) + end + + # Foo + # ^^^ + def visit_constant_read_node(node) + builder.const([node.name, srange(node.location)]) + end + + # Foo = 1 + # ^^^^^^^ + # + # Foo, Bar = 1 + # ^^^ ^^^ + def visit_constant_write_node(node) + builder.assign(builder.assignable(builder.const([node.name, srange(node.name_loc)])), token(node.operator_loc), visit(node.value)) + end + + # Foo += bar + # ^^^^^^^^^^^ + def visit_constant_operator_write_node(node) + builder.op_assign( + builder.assignable(builder.const([node.name, srange(node.name_loc)])), + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], + visit(node.value) + ) + end + + # Foo &&= bar + # ^^^^^^^^^^^^ + def visit_constant_and_write_node(node) + builder.op_assign( + builder.assignable(builder.const([node.name, srange(node.name_loc)])), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # Foo ||= bar + # ^^^^^^^^^^^^ + def visit_constant_or_write_node(node) + builder.op_assign( + builder.assignable(builder.const([node.name, srange(node.name_loc)])), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # Foo, = bar + # ^^^ + def visit_constant_target_node(node) + builder.assignable(builder.const([node.name, srange(node.location)])) + end + + # Foo::Bar + # ^^^^^^^^ + def visit_constant_path_node(node) + if node.parent.nil? + builder.const_global( + token(node.delimiter_loc), + [node.name, srange(node.name_loc)] + ) + else + builder.const_fetch( + visit(node.parent), + token(node.delimiter_loc), + [node.name, srange(node.name_loc)] + ) + end + end + + # Foo::Bar = 1 + # ^^^^^^^^^^^^ + # + # Foo::Foo, Bar::Bar = 1 + # ^^^^^^^^ ^^^^^^^^ + def visit_constant_path_write_node(node) + builder.assign( + builder.assignable(visit(node.target)), + token(node.operator_loc), + visit(node.value) + ) + end + + # Foo::Bar += baz + # ^^^^^^^^^^^^^^^ + def visit_constant_path_operator_write_node(node) + builder.op_assign( + builder.assignable(visit(node.target)), + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], + visit(node.value) + ) + end + + # Foo::Bar &&= baz + # ^^^^^^^^^^^^^^^^ + def visit_constant_path_and_write_node(node) + builder.op_assign( + builder.assignable(visit(node.target)), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # Foo::Bar ||= baz + # ^^^^^^^^^^^^^^^^ + def visit_constant_path_or_write_node(node) + builder.op_assign( + builder.assignable(visit(node.target)), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # Foo::Bar, = baz + # ^^^^^^^^ + def visit_constant_path_target_node(node) + builder.assignable(visit_constant_path_node(node)) + end + + # def foo; end + # ^^^^^^^^^^^^ + # + # def self.foo; end + # ^^^^^^^^^^^^^^^^^ + def visit_def_node(node) + if node.equal_loc + if node.receiver + builder.def_endless_singleton( + token(node.def_keyword_loc), + visit(node.receiver.is_a?(ParenthesesNode) ? node.receiver.body : node.receiver), + token(node.operator_loc), + token(node.name_loc), + builder.args(token(node.lparen_loc), visit(node.parameters) || [], token(node.rparen_loc), false), + token(node.equal_loc), + node.body&.accept(copy_compiler(forwarding: find_forwarding(node.parameters))) + ) + else + builder.def_endless_method( + token(node.def_keyword_loc), + token(node.name_loc), + builder.args(token(node.lparen_loc), visit(node.parameters) || [], token(node.rparen_loc), false), + token(node.equal_loc), + node.body&.accept(copy_compiler(forwarding: find_forwarding(node.parameters))) + ) + end + elsif node.receiver + builder.def_singleton( + token(node.def_keyword_loc), + visit(node.receiver.is_a?(ParenthesesNode) ? node.receiver.body : node.receiver), + token(node.operator_loc), + token(node.name_loc), + builder.args(token(node.lparen_loc), visit(node.parameters) || [], token(node.rparen_loc), false), + node.body&.accept(copy_compiler(forwarding: find_forwarding(node.parameters))), + token(node.end_keyword_loc) + ) + else + builder.def_method( + token(node.def_keyword_loc), + token(node.name_loc), + builder.args(token(node.lparen_loc), visit(node.parameters) || [], token(node.rparen_loc), false), + node.body&.accept(copy_compiler(forwarding: find_forwarding(node.parameters))), + token(node.end_keyword_loc) + ) + end + end + + # defined? a + # ^^^^^^^^^^ + # + # defined?(a) + # ^^^^^^^^^^^ + def visit_defined_node(node) + builder.keyword_cmd( + :defined?, + token(node.keyword_loc), + token(node.lparen_loc), + [visit(node.value)], + token(node.rparen_loc) + ) + end + + # if foo then bar else baz end + # ^^^^^^^^^^^^ + def visit_else_node(node) + visit(node.statements) + end + + # "foo #{bar}" + # ^^^^^^ + def visit_embedded_statements_node(node) + builder.begin( + token(node.opening_loc), + visit(node.statements), + token(node.closing_loc) + ) + end + + # "foo #@bar" + # ^^^^^ + def visit_embedded_variable_node(node) + visit(node.variable) + end + + # begin; foo; ensure; bar; end + # ^^^^^^^^^^^^ + def visit_ensure_node(node) + raise CompilationError, "Cannot directly compile ensure nodes" + end + + # false + # ^^^^^ + def visit_false_node(node) + builder.false(token(node.location)) + end + + # foo => [*, bar, *] + # ^^^^^^^^^^^ + def visit_find_pattern_node(node) + elements = [node.left, *node.requireds, node.right] + + if node.constant + builder.const_pattern(visit(node.constant), token(node.opening_loc), builder.find_pattern(nil, visit_all(elements), nil), token(node.closing_loc)) + else + builder.find_pattern(token(node.opening_loc), visit_all(elements), token(node.closing_loc)) + end + end + + # 1.0 + # ^^^ + def visit_float_node(node) + visit_numeric(node, builder.float([node.value, srange(node.location)])) + end + + # for foo in bar do end + # ^^^^^^^^^^^^^^^^^^^^^ + def visit_for_node(node) + builder.for( + token(node.for_keyword_loc), + visit(node.index), + token(node.in_keyword_loc), + visit(node.collection), + if node.do_keyword_loc + token(node.do_keyword_loc) + else + srange_find(node.collection.location.end_offset, (node.statements&.location || node.end_keyword_loc).start_offset, [";"]) + end, + visit(node.statements), + token(node.end_keyword_loc) + ) + end + + # def foo(...); bar(...); end + # ^^^ + def visit_forwarding_arguments_node(node) + builder.forwarded_args(token(node.location)) + end + + # def foo(...); end + # ^^^ + def visit_forwarding_parameter_node(node) + builder.forward_arg(token(node.location)) + end + + # super + # ^^^^^ + # + # super {} + # ^^^^^^^^ + def visit_forwarding_super_node(node) + visit_block( + builder.keyword_cmd( + :zsuper, + ["super", srange_offsets(node.location.start_offset, node.location.start_offset + 5)] + ), + node.block + ) + end + + # $foo + # ^^^^ + def visit_global_variable_read_node(node) + builder.gvar(token(node.location)) + end + + # $foo = 1 + # ^^^^^^^^ + def visit_global_variable_write_node(node) + builder.assign( + builder.assignable(builder.gvar(token(node.name_loc))), + token(node.operator_loc), + visit(node.value) + ) + end + + # $foo += bar + # ^^^^^^^^^^^ + def visit_global_variable_operator_write_node(node) + builder.op_assign( + builder.assignable(builder.gvar(token(node.name_loc))), + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], + visit(node.value) + ) + end + + # $foo &&= bar + # ^^^^^^^^^^^^ + def visit_global_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.gvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # $foo ||= bar + # ^^^^^^^^^^^^ + def visit_global_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.gvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # $foo, = bar + # ^^^^ + def visit_global_variable_target_node(node) + builder.assignable(builder.gvar([node.slice, srange(node.location)])) + end + + # {} + # ^^ + def visit_hash_node(node) + builder.associate( + token(node.opening_loc), + visit_all(node.elements), + token(node.closing_loc) + ) + end + + # foo => {} + # ^^ + def visit_hash_pattern_node(node) + elements = [*node.elements, *node.rest] + + if node.constant + builder.const_pattern(visit(node.constant), token(node.opening_loc), builder.hash_pattern(nil, visit_all(elements), nil), token(node.closing_loc)) + else + builder.hash_pattern(token(node.opening_loc), visit_all(elements), token(node.closing_loc)) + end + end + + # if foo then bar end + # ^^^^^^^^^^^^^^^^^^^ + # + # bar if foo + # ^^^^^^^^^^ + # + # foo ? bar : baz + # ^^^^^^^^^^^^^^^ + def visit_if_node(node) + if !node.if_keyword_loc + builder.ternary( + visit(node.predicate), + token(node.then_keyword_loc), + visit(node.statements), + token(node.consequent.else_keyword_loc), + visit(node.consequent) + ) + elsif node.if_keyword_loc.start_offset == node.location.start_offset + builder.condition( + token(node.if_keyword_loc), + visit(node.predicate), + if node.then_keyword_loc + token(node.then_keyword_loc) + else + srange_find(node.predicate.location.end_offset, (node.statements&.location || node.consequent&.location || node.end_keyword_loc).start_offset, [";"]) + end, + visit(node.statements), + case node.consequent + when IfNode + token(node.consequent.if_keyword_loc) + when ElseNode + token(node.consequent.else_keyword_loc) + end, + visit(node.consequent), + if node.if_keyword != "elsif" + token(node.end_keyword_loc) + end + ) + else + builder.condition_mod( + visit(node.statements), + visit(node.consequent), + token(node.if_keyword_loc), + visit(node.predicate) + ) + end + end + + # 1i + # ^^ + def visit_imaginary_node(node) + visit_numeric(node, builder.complex([Complex(0, node.numeric.value), srange(node.location)])) + end + + # { foo: } + # ^^^^ + def visit_implicit_node(node) + raise CompilationError, "Cannot directly compile implicit nodes" + end + + # foo { |bar,| } + # ^ + def visit_implicit_rest_node(node) + raise CompilationError, "Cannot compile implicit rest nodes" + end + + # case foo; in bar; end + # ^^^^^^^^^^^^^^^^^^^^^ + def visit_in_node(node) + pattern = nil + guard = nil + + case node.pattern + when IfNode + pattern = within_pattern { |compiler| node.pattern.statements.accept(compiler) } + guard = builder.if_guard(token(node.pattern.if_keyword_loc), visit(node.pattern.predicate)) + when UnlessNode + pattern = within_pattern { |compiler| node.pattern.statements.accept(compiler) } + guard = builder.unless_guard(token(node.pattern.keyword_loc), visit(node.pattern.predicate)) + else + pattern = within_pattern { |compiler| node.pattern.accept(compiler) } + end + + builder.in_pattern( + token(node.in_loc), + pattern, + guard, + srange_find(node.pattern.location.end_offset, node.statements&.location&.start_offset, [";", "then"]), + visit(node.statements) + ) + end + + # foo[bar] += baz + # ^^^^^^^^^^^^^^^ + def visit_index_operator_write_node(node) + arguments = node.arguments&.arguments || [] + arguments << node.block if node.block + + builder.op_assign( + builder.index( + visit(node.receiver), + token(node.opening_loc), + visit_all(arguments), + token(node.closing_loc) + ), + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], + visit(node.value) + ) + end + + # foo[bar] &&= baz + # ^^^^^^^^^^^^^^^^ + def visit_index_and_write_node(node) + arguments = node.arguments&.arguments || [] + arguments << node.block if node.block + + builder.op_assign( + builder.index( + visit(node.receiver), + token(node.opening_loc), + visit_all(arguments), + token(node.closing_loc) + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # foo[bar] ||= baz + # ^^^^^^^^^^^^^^^^ + def visit_index_or_write_node(node) + arguments = node.arguments&.arguments || [] + arguments << node.block if node.block + + builder.op_assign( + builder.index( + visit(node.receiver), + token(node.opening_loc), + visit_all(arguments), + token(node.closing_loc) + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # foo[bar], = 1 + # ^^^^^^^^ + def visit_index_target_node(node) + builder.index_asgn( + visit(node.receiver), + token(node.opening_loc), + visit_all(node.arguments.arguments), + token(node.closing_loc), + ) + end + + # @foo + # ^^^^ + def visit_instance_variable_read_node(node) + builder.ivar(token(node.location)) + end + + # @foo = 1 + # ^^^^^^^^ + def visit_instance_variable_write_node(node) + builder.assign( + builder.assignable(builder.ivar(token(node.name_loc))), + token(node.operator_loc), + visit(node.value) + ) + end + + # @foo += bar + # ^^^^^^^^^^^ + def visit_instance_variable_operator_write_node(node) + builder.op_assign( + builder.assignable(builder.ivar(token(node.name_loc))), + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], + visit(node.value) + ) + end + + # @foo &&= bar + # ^^^^^^^^^^^^ + def visit_instance_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.ivar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # @foo ||= bar + # ^^^^^^^^^^^^ + def visit_instance_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.ivar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # @foo, = bar + # ^^^^ + def visit_instance_variable_target_node(node) + builder.assignable(builder.ivar(token(node.location))) + end + + # 1 + # ^ + def visit_integer_node(node) + visit_numeric(node, builder.integer([node.value, srange(node.location)])) + end + + # /foo #{bar}/ + # ^^^^^^^^^^^^ + def visit_interpolated_regular_expression_node(node) + builder.regexp_compose( + token(node.opening_loc), + visit_all(node.parts), + [node.closing[0], srange_offsets(node.closing_loc.start_offset, node.closing_loc.start_offset + 1)], + builder.regexp_options([node.closing[1..], srange_offsets(node.closing_loc.start_offset + 1, node.closing_loc.end_offset)]) + ) + end + + # if /foo #{bar}/ then end + # ^^^^^^^^^^^^ + alias visit_interpolated_match_last_line_node visit_interpolated_regular_expression_node + + # "foo #{bar}" + # ^^^^^^^^^^^^ + def visit_interpolated_string_node(node) + if node.heredoc? + children, closing = visit_heredoc(node) + opening = token(node.opening_loc) + + start_offset = node.opening_loc.end_offset + 1 + end_offset = node.parts.first.location.start_offset + + # In the below case, the offsets should be the same: + # + # <<~HEREDOC + # a #{b} + # HEREDOC + # + # But in this case, the end_offset would be greater than the start_offset: + # + # <<~HEREDOC + # #{b} + # HEREDOC + # + # So we need to make sure the result node's heredoc range is correct, without updating the children + result = if start_offset < end_offset + # We need to add a padding string to ensure that the heredoc has correct range for its body + padding_string_node = builder.string_internal(["", srange_offsets(start_offset, end_offset)]) + node_with_correct_location = builder.string_compose(opening, [padding_string_node, *children], closing) + # But the padding string should not be included in the final AST, so we need to update the result's children + node_with_correct_location.updated(:dstr, children) + else + builder.string_compose(opening, children, closing) + end + + return result + end + + parts = if node.parts.one? { |part| part.type == :string_node } + node.parts.flat_map do |node| + if node.type == :string_node && node.unescaped.lines.count >= 2 + start_offset = node.content_loc.start_offset + + node.unescaped.lines.map do |line| + end_offset = start_offset + line.length + offsets = srange_offsets(start_offset, end_offset) + start_offset = end_offset + + builder.string_internal([line, offsets]) + end + else + visit(node) + end + end + else + visit_all(node.parts) + end + + builder.string_compose( + token(node.opening_loc), + parts, + token(node.closing_loc) + ) + end + + # :"foo #{bar}" + # ^^^^^^^^^^^^^ + def visit_interpolated_symbol_node(node) + builder.symbol_compose( + token(node.opening_loc), + visit_all(node.parts), + token(node.closing_loc) + ) + end + + # `foo #{bar}` + # ^^^^^^^^^^^^ + def visit_interpolated_x_string_node(node) + if node.heredoc? + children, closing = visit_heredoc(node) + builder.xstring_compose(token(node.opening_loc), children, closing) + else + builder.xstring_compose( + token(node.opening_loc), + visit_all(node.parts), + token(node.closing_loc) + ) + end + end + + # -> { it } + # ^^ + def visit_it_local_variable_read_node(node) + builder.ident([:it, srange(node.location)]).updated(:lvar) + end + + # -> { it } + # ^^^^^^^^^ + def visit_it_parameters_node(node) + builder.args(nil, [], nil, false) + end + + # foo(bar: baz) + # ^^^^^^^^ + def visit_keyword_hash_node(node) + builder.associate(nil, visit_all(node.elements), nil) + end + + # def foo(**bar); end + # ^^^^^ + # + # def foo(**); end + # ^^ + def visit_keyword_rest_parameter_node(node) + builder.kwrestarg( + token(node.operator_loc), + node.name ? [node.name, srange(node.name_loc)] : nil + ) + end + + # -> {} + # ^^^^^ + def visit_lambda_node(node) + parameters = node.parameters + implicit_parameters = parameters.is_a?(NumberedParametersNode) || parameters.is_a?(ItParametersNode) + + builder.block( + builder.call_lambda(token(node.operator_loc)), + [node.opening, srange(node.opening_loc)], + if parameters.nil? + builder.args(nil, [], nil, false) + elsif implicit_parameters + visit(node.parameters) + else + builder.args( + token(node.parameters.opening_loc), + visit(node.parameters), + token(node.parameters.closing_loc), + false + ) + end, + node.body&.accept(copy_compiler(forwarding: implicit_parameters ? [] : find_forwarding(parameters&.parameters))), + [node.closing, srange(node.closing_loc)] + ) + end + + # foo + # ^^^ + def visit_local_variable_read_node(node) + builder.ident([node.name, srange(node.location)]).updated(:lvar) + end + + # foo = 1 + # ^^^^^^^ + def visit_local_variable_write_node(node) + builder.assign( + builder.assignable(builder.ident(token(node.name_loc))), + token(node.operator_loc), + visit(node.value) + ) + end + + # foo += bar + # ^^^^^^^^^^ + def visit_local_variable_operator_write_node(node) + builder.op_assign( + builder.assignable(builder.ident(token(node.name_loc))), + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], + visit(node.value) + ) + end + + # foo &&= bar + # ^^^^^^^^^^^ + def visit_local_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.ident(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # foo ||= bar + # ^^^^^^^^^^^ + def visit_local_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.ident(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end + + # foo, = bar + # ^^^ + def visit_local_variable_target_node(node) + if in_pattern + builder.assignable(builder.match_var([node.name, srange(node.location)])) + else + builder.assignable(builder.ident(token(node.location))) + end + end + + # foo in bar + # ^^^^^^^^^^ + def visit_match_predicate_node(node) + builder.match_pattern_p( + visit(node.value), + token(node.operator_loc), + within_pattern { |compiler| node.pattern.accept(compiler) } + ) + end + + # foo => bar + # ^^^^^^^^^^ + def visit_match_required_node(node) + builder.match_pattern( + visit(node.value), + token(node.operator_loc), + within_pattern { |compiler| node.pattern.accept(compiler) } + ) + end + + # /(?<foo>foo)/ =~ bar + # ^^^^^^^^^^^^^^^^^^^^ + def visit_match_write_node(node) + builder.match_op( + visit(node.call.receiver), + token(node.call.message_loc), + visit(node.call.arguments.arguments.first) + ) + end + + # A node that is missing from the syntax tree. This is only used in the + # case of a syntax error. The parser gem doesn't have such a concept, so + # we invent our own here. + def visit_missing_node(node) + ::AST::Node.new(:missing, [], location: ::Parser::Source::Map.new(srange(node.location))) + end + + # module Foo; end + # ^^^^^^^^^^^^^^^ + def visit_module_node(node) + builder.def_module( + token(node.module_keyword_loc), + visit(node.constant_path), + node.body&.accept(copy_compiler(forwarding: [])), + token(node.end_keyword_loc) + ) + end + + # foo, bar = baz + # ^^^^^^^^ + def visit_multi_target_node(node) + elements = [*node.lefts] + elements << node.rest if !node.rest.nil? && !node.rest.is_a?(ImplicitRestNode) + elements.concat(node.rights) + + builder.multi_lhs( + token(node.lparen_loc), + visit_all(elements), + token(node.rparen_loc) + ) + end + + # foo, bar = baz + # ^^^^^^^^^^^^^^ + def visit_multi_write_node(node) + elements = [*node.lefts] + elements << node.rest if !node.rest.nil? && !node.rest.is_a?(ImplicitRestNode) + elements.concat(node.rights) + + builder.multi_assign( + builder.multi_lhs( + token(node.lparen_loc), + visit_all(elements), + token(node.rparen_loc) + ), + token(node.operator_loc), + visit(node.value) + ) + end + + # next + # ^^^^ + # + # next foo + # ^^^^^^^^ + def visit_next_node(node) + builder.keyword_cmd( + :next, + token(node.keyword_loc), + nil, + visit(node.arguments) || [], + nil + ) + end + + # nil + # ^^^ + def visit_nil_node(node) + builder.nil(token(node.location)) + end + + # def foo(**nil); end + # ^^^^^ + def visit_no_keywords_parameter_node(node) + if in_pattern + builder.match_nil_pattern(token(node.operator_loc), token(node.keyword_loc)) + else + builder.kwnilarg(token(node.operator_loc), token(node.keyword_loc)) + end + end + + # -> { _1 + _2 } + # ^^^^^^^^^^^^^^ + def visit_numbered_parameters_node(node) + builder.numargs(node.maximum) + end + + # $1 + # ^^ + def visit_numbered_reference_read_node(node) + builder.nth_ref([node.number, srange(node.location)]) + end + + # def foo(bar: baz); end + # ^^^^^^^^ + def visit_optional_keyword_parameter_node(node) + builder.kwoptarg([node.name, srange(node.name_loc)], visit(node.value)) + end + + # def foo(bar = 1); end + # ^^^^^^^ + def visit_optional_parameter_node(node) + builder.optarg(token(node.name_loc), token(node.operator_loc), visit(node.value)) + end + + # a or b + # ^^^^^^ + def visit_or_node(node) + builder.logical_op(:or, visit(node.left), token(node.operator_loc), visit(node.right)) + end + + # def foo(bar, *baz); end + # ^^^^^^^^^ + def visit_parameters_node(node) + params = [] + + if node.requireds.any? + node.requireds.each do |required| + if required.is_a?(RequiredParameterNode) + params << visit(required) + else + compiler = copy_compiler(in_destructure: true) + params << required.accept(compiler) + end + end + end + + params.concat(visit_all(node.optionals)) if node.optionals.any? + params << visit(node.rest) if !node.rest.nil? && !node.rest.is_a?(ImplicitRestNode) + + if node.posts.any? + node.posts.each do |post| + if post.is_a?(RequiredParameterNode) + params << visit(post) + else + compiler = copy_compiler(in_destructure: true) + params << post.accept(compiler) + end + end + end + + params.concat(visit_all(node.keywords)) if node.keywords.any? + params << visit(node.keyword_rest) if !node.keyword_rest.nil? + params << visit(node.block) if !node.block.nil? + params + end + + # () + # ^^ + # + # (1) + # ^^^ + def visit_parentheses_node(node) + builder.begin( + token(node.opening_loc), + visit(node.body), + token(node.closing_loc) + ) + end + + # foo => ^(bar) + # ^^^^^^ + def visit_pinned_expression_node(node) + expression = builder.begin(token(node.lparen_loc), visit(node.expression), token(node.rparen_loc)) + builder.pin(token(node.operator_loc), expression) + end + + # foo = 1 and bar => ^foo + # ^^^^ + def visit_pinned_variable_node(node) + builder.pin(token(node.operator_loc), visit(node.variable)) + end + + # END {} + def visit_post_execution_node(node) + builder.postexe( + token(node.keyword_loc), + token(node.opening_loc), + visit(node.statements), + token(node.closing_loc) + ) + end + + # BEGIN {} + def visit_pre_execution_node(node) + builder.preexe( + token(node.keyword_loc), + token(node.opening_loc), + visit(node.statements), + token(node.closing_loc) + ) + end + + # The top-level program node. + def visit_program_node(node) + visit(node.statements) + end + + # 0..5 + # ^^^^ + def visit_range_node(node) + if node.exclude_end? + builder.range_exclusive( + visit(node.left), + token(node.operator_loc), + visit(node.right) + ) + else + builder.range_inclusive( + visit(node.left), + token(node.operator_loc), + visit(node.right) + ) + end + end + + # if foo .. bar; end + # ^^^^^^^^^^ + alias visit_flip_flop_node visit_range_node + + # 1r + # ^^ + def visit_rational_node(node) + visit_numeric(node, builder.rational([node.value, srange(node.location)])) + end + + # redo + # ^^^^ + def visit_redo_node(node) + builder.keyword_cmd(:redo, token(node.location)) + end + + # /foo/ + # ^^^^^ + def visit_regular_expression_node(node) + builder.regexp_compose( + token(node.opening_loc), + [builder.string_internal(token(node.content_loc))], + [node.closing[0], srange_offsets(node.closing_loc.start_offset, node.closing_loc.start_offset + 1)], + builder.regexp_options([node.closing[1..], srange_offsets(node.closing_loc.start_offset + 1, node.closing_loc.end_offset)]) + ) + end + + # if /foo/ then end + # ^^^^^ + alias visit_match_last_line_node visit_regular_expression_node + + # def foo(bar:); end + # ^^^^ + def visit_required_keyword_parameter_node(node) + builder.kwarg([node.name, srange(node.name_loc)]) + end + + # def foo(bar); end + # ^^^ + def visit_required_parameter_node(node) + builder.arg(token(node.location)) + end + + # foo rescue bar + # ^^^^^^^^^^^^^^ + def visit_rescue_modifier_node(node) + builder.begin_body( + visit(node.expression), + [ + builder.rescue_body( + token(node.keyword_loc), + nil, + nil, + nil, + nil, + visit(node.rescue_expression) + ) + ] + ) + end + + # begin; rescue; end + # ^^^^^^^ + def visit_rescue_node(node) + raise CompilationError, "Cannot directly compile rescue nodes" + end + + # def foo(*bar); end + # ^^^^ + # + # def foo(*); end + # ^ + def visit_rest_parameter_node(node) + builder.restarg(token(node.operator_loc), token(node.name_loc)) + end + + # retry + # ^^^^^ + def visit_retry_node(node) + builder.keyword_cmd(:retry, token(node.location)) + end + + # return + # ^^^^^^ + # + # return 1 + # ^^^^^^^^ + def visit_return_node(node) + builder.keyword_cmd( + :return, + token(node.keyword_loc), + nil, + visit(node.arguments) || [], + nil + ) + end + + # self + # ^^^^ + def visit_self_node(node) + builder.self(token(node.location)) + end + + # A shareable constant. + def visit_shareable_constant_node(node) + visit(node.write) + end + + # class << self; end + # ^^^^^^^^^^^^^^^^^^ + def visit_singleton_class_node(node) + builder.def_sclass( + token(node.class_keyword_loc), + token(node.operator_loc), + visit(node.expression), + node.body&.accept(copy_compiler(forwarding: [])), + token(node.end_keyword_loc) + ) + end + + # __ENCODING__ + # ^^^^^^^^^^^^ + def visit_source_encoding_node(node) + builder.accessible(builder.__ENCODING__(token(node.location))) + end + + # __FILE__ + # ^^^^^^^^ + def visit_source_file_node(node) + builder.accessible(builder.__FILE__(token(node.location))) + end + + # __LINE__ + # ^^^^^^^^ + def visit_source_line_node(node) + builder.accessible(builder.__LINE__(token(node.location))) + end + + # foo(*bar) + # ^^^^ + # + # def foo((bar, *baz)); end + # ^^^^ + # + # def foo(*); bar(*); end + # ^ + def visit_splat_node(node) + if node.expression.nil? && forwarding.include?(:*) + builder.forwarded_restarg(token(node.operator_loc)) + elsif in_destructure + builder.restarg(token(node.operator_loc), token(node.expression&.location)) + elsif in_pattern + builder.match_rest(token(node.operator_loc), token(node.expression&.location)) + else + builder.splat(token(node.operator_loc), visit(node.expression)) + end + end + + # A list of statements. + def visit_statements_node(node) + builder.compstmt(visit_all(node.body)) + end + + # "foo" + # ^^^^^ + def visit_string_node(node) + if node.heredoc? + children, closing = visit_heredoc(node.to_interpolated) + builder.string_compose(token(node.opening_loc), children, closing) + elsif node.opening == "?" + builder.character([node.unescaped, srange(node.location)]) + else + content_lines = node.content.lines + unescaped_lines = node.unescaped.lines + + parts = + if content_lines.length <= 1 || unescaped_lines.length <= 1 + [builder.string_internal([node.unescaped, srange(node.content_loc)])] + elsif content_lines.length != unescaped_lines.length + # This occurs when we have line continuations in the string. We + # need to come back and fix this, but for now this stops the + # code from breaking when we encounter it because of trying to + # transpose arrays of different lengths. + [builder.string_internal([node.unescaped, srange(node.content_loc)])] + else + start_offset = node.content_loc.start_offset + + [content_lines, unescaped_lines].transpose.map do |content_line, unescaped_line| + end_offset = start_offset + content_line.length + offsets = srange_offsets(start_offset, end_offset) + start_offset = end_offset + + builder.string_internal([unescaped_line, offsets]) + end + end + + builder.string_compose( + token(node.opening_loc), + parts, + token(node.closing_loc) + ) + end + end + + # super(foo) + # ^^^^^^^^^^ + def visit_super_node(node) + arguments = node.arguments&.arguments || [] + block = node.block + + if block.is_a?(BlockArgumentNode) + arguments = [*arguments, block] + block = nil + end + + visit_block( + builder.keyword_cmd( + :super, + token(node.keyword_loc), + token(node.lparen_loc), + visit_all(arguments), + token(node.rparen_loc) + ), + block + ) + end + + # :foo + # ^^^^ + def visit_symbol_node(node) + if node.closing_loc.nil? + if node.opening_loc.nil? + builder.symbol_internal([node.unescaped, srange(node.location)]) + else + builder.symbol([node.unescaped, srange(node.location)]) + end + else + parts = if node.value.lines.one? + [builder.string_internal([node.unescaped, srange(node.value_loc)])] + else + start_offset = node.value_loc.start_offset + + node.value.lines.map do |line| + end_offset = start_offset + line.length + offsets = srange_offsets(start_offset, end_offset) + start_offset = end_offset + + builder.string_internal([line, offsets]) + end + end + + builder.symbol_compose( + token(node.opening_loc), + parts, + token(node.closing_loc) + ) + end + end + + # true + # ^^^^ + def visit_true_node(node) + builder.true(token(node.location)) + end + + # undef foo + # ^^^^^^^^^ + def visit_undef_node(node) + builder.undef_method(token(node.keyword_loc), visit_all(node.names)) + end + + # unless foo; bar end + # ^^^^^^^^^^^^^^^^^^^ + # + # bar unless foo + # ^^^^^^^^^^^^^^ + def visit_unless_node(node) + if node.keyword_loc.start_offset == node.location.start_offset + builder.condition( + token(node.keyword_loc), + visit(node.predicate), + if node.then_keyword_loc + token(node.then_keyword_loc) + else + srange_find(node.predicate.location.end_offset, (node.statements&.location || node.consequent&.location || node.end_keyword_loc).start_offset, [";"]) + end, + visit(node.consequent), + token(node.consequent&.else_keyword_loc), + visit(node.statements), + token(node.end_keyword_loc) + ) + else + builder.condition_mod( + visit(node.consequent), + visit(node.statements), + token(node.keyword_loc), + visit(node.predicate) + ) + end + end + + # until foo; bar end + # ^^^^^^^^^^^^^^^^^^ + # + # bar until foo + # ^^^^^^^^^^^^^ + def visit_until_node(node) + if node.location.start_offset == node.keyword_loc.start_offset + builder.loop( + :until, + token(node.keyword_loc), + visit(node.predicate), + srange_find(node.predicate.location.end_offset, (node.statements&.location || node.closing_loc).start_offset, [";", "do"]), + visit(node.statements), + token(node.closing_loc) + ) + else + builder.loop_mod( + :until, + visit(node.statements), + token(node.keyword_loc), + visit(node.predicate) + ) + end + end + + # case foo; when bar; end + # ^^^^^^^^^^^^^ + def visit_when_node(node) + builder.when( + token(node.keyword_loc), + visit_all(node.conditions), + if node.then_keyword_loc + token(node.then_keyword_loc) + else + srange_find(node.conditions.last.location.end_offset, node.statements&.location&.start_offset, [";"]) + end, + visit(node.statements) + ) + end + + # while foo; bar end + # ^^^^^^^^^^^^^^^^^^ + # + # bar while foo + # ^^^^^^^^^^^^^ + def visit_while_node(node) + if node.location.start_offset == node.keyword_loc.start_offset + builder.loop( + :while, + token(node.keyword_loc), + visit(node.predicate), + srange_find(node.predicate.location.end_offset, (node.statements&.location || node.closing_loc).start_offset, [";", "do"]), + visit(node.statements), + token(node.closing_loc) + ) + else + builder.loop_mod( + :while, + visit(node.statements), + token(node.keyword_loc), + visit(node.predicate) + ) + end + end + + # `foo` + # ^^^^^ + def visit_x_string_node(node) + if node.heredoc? + children, closing = visit_heredoc(node.to_interpolated) + builder.xstring_compose(token(node.opening_loc), children, closing) + else + parts = if node.unescaped.lines.one? + [builder.string_internal([node.unescaped, srange(node.content_loc)])] + else + start_offset = node.content_loc.start_offset + + node.unescaped.lines.map do |line| + end_offset = start_offset + line.length + offsets = srange_offsets(start_offset, end_offset) + start_offset = end_offset + + builder.string_internal([line, offsets]) + end + end + + builder.xstring_compose( + token(node.opening_loc), + parts, + token(node.closing_loc) + ) + end + end + + # yield + # ^^^^^ + # + # yield 1 + # ^^^^^^^ + def visit_yield_node(node) + builder.keyword_cmd( + :yield, + token(node.keyword_loc), + token(node.lparen_loc), + visit(node.arguments) || [], + token(node.rparen_loc) + ) + end + + private + + # Initialize a new compiler with the given option overrides, used to + # visit a subtree with the given options. + def copy_compiler(forwarding: self.forwarding, in_destructure: self.in_destructure, in_pattern: self.in_pattern) + Compiler.new(parser, offset_cache, forwarding: forwarding, in_destructure: in_destructure, in_pattern: in_pattern) + end + + # When *, **, &, or ... are used as an argument in a method call, we + # check if they were allowed by the current context. To determine that + # we build this lookup table. + def find_forwarding(node) + return [] if node.nil? + + forwarding = [] + forwarding << :* if node.rest.is_a?(RestParameterNode) && node.rest.name.nil? + forwarding << :** if node.keyword_rest.is_a?(KeywordRestParameterNode) && node.keyword_rest.name.nil? + forwarding << :& if !node.block.nil? && node.block.name.nil? + forwarding |= [:&, :"..."] if node.keyword_rest.is_a?(ForwardingParameterNode) + + forwarding + end + + # Negate the value of a numeric node. This is a special case where you + # have a negative sign on one line and then a number on the next line. + # In normal Ruby, this will always be a method call. The parser gem, + # however, marks this as a numeric literal. We have to massage the tree + # here to get it into the correct form. + def numeric_negate(message_loc, receiver) + case receiver.type + when :integer_node, :float_node + receiver.copy(value: -receiver.value, location: message_loc.join(receiver.location)) + when :rational_node + receiver.copy(numerator: -receiver.numerator, location: message_loc.join(receiver.location)) + when :imaginary_node + receiver.copy(numeric: numeric_negate(message_loc, receiver.numeric), location: message_loc.join(receiver.location)) + end + end + + # Blocks can have a special set of parameters that automatically expand + # when given arrays if they have a single required parameter and no + # other parameters. + def procarg0?(parameters) + parameters && + parameters.requireds.length == 1 && + parameters.optionals.empty? && + parameters.rest.nil? && + parameters.posts.empty? && + parameters.keywords.empty? && + parameters.keyword_rest.nil? && + parameters.block.nil? + end + + # Locations in the parser gem AST are generated using this class. We + # store a reference to its constant to make it slightly faster to look + # up. + Range = ::Parser::Source::Range + + # Constructs a new source range from the given start and end offsets. + def srange(location) + Range.new(source_buffer, offset_cache[location.start_offset], offset_cache[location.end_offset]) if location + end + + # Constructs a new source range from the given start and end offsets. + def srange_offsets(start_offset, end_offset) + Range.new(source_buffer, offset_cache[start_offset], offset_cache[end_offset]) + end + + # Constructs a new source range by finding the given tokens between the + # given start offset and end offset. If the needle is not found, it + # returns nil. Importantly it does not search past newlines or comments. + # + # Note that end_offset is allowed to be nil, in which case this will + # search until the end of the string. + def srange_find(start_offset, end_offset, tokens) + if (match = source_buffer.source.byteslice(start_offset...end_offset).match(/(\s*)(#{tokens.join("|")})/)) + _, whitespace, token = *match + token_offset = start_offset + whitespace.bytesize + + [token, Range.new(source_buffer, offset_cache[token_offset], offset_cache[token_offset + token.bytesize])] + end + end + + # Transform a location into a token that the parser gem expects. + def token(location) + [location.slice, Range.new(source_buffer, offset_cache[location.start_offset], offset_cache[location.end_offset])] if location + end + + # Visit a block node on a call. + def visit_block(call, block) + if block + parameters = block.parameters + implicit_parameters = parameters.is_a?(NumberedParametersNode) || parameters.is_a?(ItParametersNode) + + builder.block( + call, + token(block.opening_loc), + if parameters.nil? + builder.args(nil, [], nil, false) + elsif implicit_parameters + visit(parameters) + else + builder.args( + token(parameters.opening_loc), + if procarg0?(parameters.parameters) + parameter = parameters.parameters.requireds.first + [builder.procarg0(visit(parameter))].concat(visit_all(parameters.locals)) + else + visit(parameters) + end, + token(parameters.closing_loc), + false + ) + end, + block.body&.accept(copy_compiler(forwarding: implicit_parameters ? [] : find_forwarding(parameters&.parameters))), + token(block.closing_loc) + ) + else + call + end + end + + # Visit a heredoc that can be either a string or an xstring. + def visit_heredoc(node) + children = Array.new + node.parts.each do |part| + pushing = + if part.is_a?(StringNode) && part.unescaped.include?("\n") + unescaped = part.unescaped.lines(chomp: true) + escaped = part.content.lines(chomp: true) + + escaped_lengths = + if node.opening.end_with?("'") + escaped.map { |line| line.bytesize + 1 } + else + escaped.chunk_while { |before, after| before.match?(/(?<!\\)\\$/) }.map { |line| line.join.bytesize + line.length } + end + + start_offset = part.location.start_offset + end_offset = nil + + unescaped.zip(escaped_lengths).map do |unescaped_line, escaped_length| + end_offset = start_offset + (escaped_length || 0) + inner_part = builder.string_internal(["#{unescaped_line}\n", srange_offsets(start_offset, end_offset)]) + start_offset = end_offset + inner_part + end + else + [visit(part)] + end + + pushing.each do |child| + if child.type == :str && child.children.last == "" + # nothing + elsif child.type == :str && children.last && children.last.type == :str && !children.last.children.first.end_with?("\n") + children.last.children.first << child.children.first + else + children << child + end + end + end + + closing = node.closing + closing_t = [closing.chomp, srange_offsets(node.closing_loc.start_offset, node.closing_loc.end_offset - (closing[/\s+$/]&.length || 0))] + + [children, closing_t] + end + + # Visit a numeric node and account for the optional sign. + def visit_numeric(node, value) + if (slice = node.slice).match?(/^[+-]/) + builder.unary_num( + [slice[0].to_sym, srange_offsets(node.location.start_offset, node.location.start_offset + 1)], + value + ) + else + value + end + end + + # Within the given block, track that we're within a pattern. + def within_pattern + begin + parser.pattern_variables.push + yield copy_compiler(in_pattern: true) + ensure + parser.pattern_variables.pop + end + end + end + end + end +end diff --git a/lib/prism/translation/parser/lexer.rb b/lib/prism/translation/parser/lexer.rb new file mode 100644 index 0000000000..9d7caae0ba --- /dev/null +++ b/lib/prism/translation/parser/lexer.rb @@ -0,0 +1,416 @@ +# frozen_string_literal: true + +module Prism + module Translation + class Parser + # Accepts a list of prism tokens and converts them into the expected + # format for the parser gem. + class Lexer + # The direct translating of types between the two lexers. + TYPES = { + # These tokens should never appear in the output of the lexer. + EOF: nil, + MISSING: nil, + NOT_PROVIDED: nil, + IGNORED_NEWLINE: nil, + EMBDOC_END: nil, + EMBDOC_LINE: nil, + __END__: nil, + + # These tokens have more or less direct mappings. + AMPERSAND: :tAMPER2, + AMPERSAND_AMPERSAND: :tANDOP, + AMPERSAND_AMPERSAND_EQUAL: :tOP_ASGN, + AMPERSAND_DOT: :tANDDOT, + AMPERSAND_EQUAL: :tOP_ASGN, + BACK_REFERENCE: :tBACK_REF, + BACKTICK: :tXSTRING_BEG, + BANG: :tBANG, + BANG_EQUAL: :tNEQ, + BANG_TILDE: :tNMATCH, + BRACE_LEFT: :tLCURLY, + BRACE_RIGHT: :tRCURLY, + BRACKET_LEFT: :tLBRACK2, + BRACKET_LEFT_ARRAY: :tLBRACK, + BRACKET_LEFT_RIGHT: :tAREF, + BRACKET_LEFT_RIGHT_EQUAL: :tASET, + BRACKET_RIGHT: :tRBRACK, + CARET: :tCARET, + CARET_EQUAL: :tOP_ASGN, + CHARACTER_LITERAL: :tCHARACTER, + CLASS_VARIABLE: :tCVAR, + COLON: :tCOLON, + COLON_COLON: :tCOLON2, + COMMA: :tCOMMA, + COMMENT: :tCOMMENT, + CONSTANT: :tCONSTANT, + DOT: :tDOT, + DOT_DOT: :tDOT2, + DOT_DOT_DOT: :tDOT3, + EMBDOC_BEGIN: :tCOMMENT, + EMBEXPR_BEGIN: :tSTRING_DBEG, + EMBEXPR_END: :tSTRING_DEND, + EMBVAR: :tSTRING_DVAR, + EQUAL: :tEQL, + EQUAL_EQUAL: :tEQ, + EQUAL_EQUAL_EQUAL: :tEQQ, + EQUAL_GREATER: :tASSOC, + EQUAL_TILDE: :tMATCH, + FLOAT: :tFLOAT, + FLOAT_IMAGINARY: :tIMAGINARY, + FLOAT_RATIONAL: :tRATIONAL, + FLOAT_RATIONAL_IMAGINARY: :tIMAGINARY, + GLOBAL_VARIABLE: :tGVAR, + GREATER: :tGT, + GREATER_EQUAL: :tGEQ, + GREATER_GREATER: :tRSHFT, + GREATER_GREATER_EQUAL: :tOP_ASGN, + HEREDOC_START: :tSTRING_BEG, + HEREDOC_END: :tSTRING_END, + IDENTIFIER: :tIDENTIFIER, + INSTANCE_VARIABLE: :tIVAR, + INTEGER: :tINTEGER, + INTEGER_IMAGINARY: :tIMAGINARY, + INTEGER_RATIONAL: :tRATIONAL, + INTEGER_RATIONAL_IMAGINARY: :tIMAGINARY, + KEYWORD_ALIAS: :kALIAS, + KEYWORD_AND: :kAND, + KEYWORD_BEGIN: :kBEGIN, + KEYWORD_BEGIN_UPCASE: :klBEGIN, + KEYWORD_BREAK: :kBREAK, + KEYWORD_CASE: :kCASE, + KEYWORD_CLASS: :kCLASS, + KEYWORD_DEF: :kDEF, + KEYWORD_DEFINED: :kDEFINED, + KEYWORD_DO: :kDO, + KEYWORD_DO_LOOP: :kDO_COND, + KEYWORD_END: :kEND, + KEYWORD_END_UPCASE: :klEND, + KEYWORD_ENSURE: :kENSURE, + KEYWORD_ELSE: :kELSE, + KEYWORD_ELSIF: :kELSIF, + KEYWORD_FALSE: :kFALSE, + KEYWORD_FOR: :kFOR, + KEYWORD_IF: :kIF, + KEYWORD_IF_MODIFIER: :kIF_MOD, + KEYWORD_IN: :kIN, + KEYWORD_MODULE: :kMODULE, + KEYWORD_NEXT: :kNEXT, + KEYWORD_NIL: :kNIL, + KEYWORD_NOT: :kNOT, + KEYWORD_OR: :kOR, + KEYWORD_REDO: :kREDO, + KEYWORD_RESCUE: :kRESCUE, + KEYWORD_RESCUE_MODIFIER: :kRESCUE_MOD, + KEYWORD_RETRY: :kRETRY, + KEYWORD_RETURN: :kRETURN, + KEYWORD_SELF: :kSELF, + KEYWORD_SUPER: :kSUPER, + KEYWORD_THEN: :kTHEN, + KEYWORD_TRUE: :kTRUE, + KEYWORD_UNDEF: :kUNDEF, + KEYWORD_UNLESS: :kUNLESS, + KEYWORD_UNLESS_MODIFIER: :kUNLESS_MOD, + KEYWORD_UNTIL: :kUNTIL, + KEYWORD_UNTIL_MODIFIER: :kUNTIL_MOD, + KEYWORD_WHEN: :kWHEN, + KEYWORD_WHILE: :kWHILE, + KEYWORD_WHILE_MODIFIER: :kWHILE_MOD, + KEYWORD_YIELD: :kYIELD, + KEYWORD___ENCODING__: :k__ENCODING__, + KEYWORD___FILE__: :k__FILE__, + KEYWORD___LINE__: :k__LINE__, + LABEL: :tLABEL, + LABEL_END: :tLABEL_END, + LAMBDA_BEGIN: :tLAMBEG, + LESS: :tLT, + LESS_EQUAL: :tLEQ, + LESS_EQUAL_GREATER: :tCMP, + LESS_LESS: :tLSHFT, + LESS_LESS_EQUAL: :tOP_ASGN, + METHOD_NAME: :tFID, + MINUS: :tMINUS, + MINUS_EQUAL: :tOP_ASGN, + MINUS_GREATER: :tLAMBDA, + NEWLINE: :tNL, + NUMBERED_REFERENCE: :tNTH_REF, + PARENTHESIS_LEFT: :tLPAREN, + PARENTHESIS_LEFT_PARENTHESES: :tLPAREN_ARG, + PARENTHESIS_RIGHT: :tRPAREN, + PERCENT: :tPERCENT, + PERCENT_EQUAL: :tOP_ASGN, + PERCENT_LOWER_I: :tQSYMBOLS_BEG, + PERCENT_LOWER_W: :tQWORDS_BEG, + PERCENT_UPPER_I: :tSYMBOLS_BEG, + PERCENT_UPPER_W: :tWORDS_BEG, + PERCENT_LOWER_X: :tXSTRING_BEG, + PLUS: :tPLUS, + PLUS_EQUAL: :tOP_ASGN, + PIPE_EQUAL: :tOP_ASGN, + PIPE: :tPIPE, + PIPE_PIPE: :tOROP, + PIPE_PIPE_EQUAL: :tOP_ASGN, + QUESTION_MARK: :tEH, + REGEXP_BEGIN: :tREGEXP_BEG, + REGEXP_END: :tSTRING_END, + SEMICOLON: :tSEMI, + SLASH: :tDIVIDE, + SLASH_EQUAL: :tOP_ASGN, + STAR: :tSTAR2, + STAR_EQUAL: :tOP_ASGN, + STAR_STAR: :tPOW, + STAR_STAR_EQUAL: :tOP_ASGN, + STRING_BEGIN: :tSTRING_BEG, + STRING_CONTENT: :tSTRING_CONTENT, + STRING_END: :tSTRING_END, + SYMBOL_BEGIN: :tSYMBEG, + TILDE: :tTILDE, + UAMPERSAND: :tAMPER, + UCOLON_COLON: :tCOLON3, + UDOT_DOT: :tBDOT2, + UDOT_DOT_DOT: :tBDOT3, + UMINUS: :tUMINUS, + UMINUS_NUM: :tUNARY_NUM, + UPLUS: :tUPLUS, + USTAR: :tSTAR, + USTAR_STAR: :tPOW, + WORDS_SEP: :tSPACE + } + + # These constants represent flags in our lex state. We really, really + # don't want to be using them and we really, really don't want to be + # exposing them as part of our public API. Unfortunately, we don't have + # another way of matching the exact tokens that the parser gem expects + # without them. We should find another way to do this, but in the + # meantime we'll hide them from the documentation and mark them as + # private constants. + EXPR_BEG = 0x1 # :nodoc: + EXPR_LABEL = 0x400 # :nodoc: + + private_constant :TYPES, :EXPR_BEG, :EXPR_LABEL + + # The Parser::Source::Buffer that the tokens were lexed from. + attr_reader :source_buffer + + # An array of tuples that contain prism tokens and their associated lex + # state when they were lexed. + attr_reader :lexed + + # A hash that maps offsets in bytes to offsets in characters. + attr_reader :offset_cache + + # Initialize the lexer with the given source buffer, prism tokens, and + # offset cache. + def initialize(source_buffer, lexed, offset_cache) + @source_buffer = source_buffer + @lexed = lexed + @offset_cache = offset_cache + end + + Range = ::Parser::Source::Range # :nodoc: + private_constant :Range + + # Convert the prism tokens into the expected format for the parser gem. + def to_a + tokens = [] + + index = 0 + length = lexed.length + + heredoc_identifier_stack = [] + + while index < length + token, state = lexed[index] + index += 1 + next if %i[IGNORED_NEWLINE __END__ EOF].include?(token.type) + + type = TYPES.fetch(token.type) + value = token.value + location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.end_offset]) + + case type + when :tCHARACTER + value.delete_prefix!("?") + when :tCOMMENT + if token.type == :EMBDOC_BEGIN + start_index = index + + while !((next_token = lexed[index][0]) && next_token.type == :EMBDOC_END) && (index < length - 1) + value += next_token.value + index += 1 + end + + if start_index != index + value += next_token.value + location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[lexed[index][0].location.end_offset]) + index += 1 + end + else + value.chomp! + location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.end_offset - 1]) + end + when :tNL + value = nil + when :tFLOAT + value = parse_float(value) + when :tIMAGINARY + value = parse_complex(value) + when :tINTEGER + if value.start_with?("+") + tokens << [:tUNARY_NUM, ["+", Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.start_offset + 1])]] + location = Range.new(source_buffer, offset_cache[token.location.start_offset + 1], offset_cache[token.location.end_offset]) + end + + value = parse_integer(value) + when :tLABEL + value.chomp!(":") + when :tLABEL_END + value.chomp!(":") + when :tLCURLY + type = :tLBRACE if state == EXPR_BEG | EXPR_LABEL + when :tNTH_REF + value = parse_integer(value.delete_prefix("$")) + when :tOP_ASGN + value.chomp!("=") + when :tRATIONAL + value = parse_rational(value) + when :tSPACE + value = nil + when :tSTRING_BEG + if token.type == :HEREDOC_START + heredoc_identifier_stack.push(value.match(/<<[-~]?["'`]?(?<heredoc_identifier>.*?)["'`]?\z/)[:heredoc_identifier]) + end + if ["\"", "'"].include?(value) && (next_token = lexed[index][0]) && next_token.type == :STRING_END + next_location = token.location.join(next_token.location) + type = :tSTRING + value = "" + location = Range.new(source_buffer, offset_cache[next_location.start_offset], offset_cache[next_location.end_offset]) + index += 1 + elsif ["\"", "'"].include?(value) && (next_token = lexed[index][0]) && next_token.type == :STRING_CONTENT && next_token.value.lines.count <= 1 && (next_next_token = lexed[index + 1][0]) && next_next_token.type == :STRING_END + next_location = token.location.join(next_next_token.location) + type = :tSTRING + value = next_token.value.gsub("\\\\", "\\") + location = Range.new(source_buffer, offset_cache[next_location.start_offset], offset_cache[next_location.end_offset]) + index += 2 + elsif value.start_with?("<<") + quote = value[2] == "-" || value[2] == "~" ? value[3] : value[2] + if quote == "`" + type = :tXSTRING_BEG + value = "<<`" + else + value = "<<#{quote == "'" || quote == "\"" ? quote : "\""}" + end + end + when :tSTRING_CONTENT + unless (lines = token.value.lines).one? + start_offset = offset_cache[token.location.start_offset] + lines.map do |line| + newline = line.end_with?("\r\n") ? "\r\n" : "\n" + chomped_line = line.chomp + if match = chomped_line.match(/(?<backslashes>\\+)\z/) + adjustment = match[:backslashes].size / 2 + adjusted_line = chomped_line.delete_suffix("\\" * adjustment) + if match[:backslashes].size.odd? + adjusted_line.delete_suffix!("\\") + adjustment += 2 + else + adjusted_line << newline + end + else + adjusted_line = line + adjustment = 0 + end + + end_offset = start_offset + adjusted_line.length + adjustment + tokens << [:tSTRING_CONTENT, [adjusted_line, Range.new(source_buffer, offset_cache[start_offset], offset_cache[end_offset])]] + start_offset = end_offset + end + next + end + when :tSTRING_DVAR + value = nil + when :tSTRING_END + if token.type == :HEREDOC_END && value.end_with?("\n") + newline_length = value.end_with?("\r\n") ? 2 : 1 + value = heredoc_identifier_stack.pop + location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.end_offset - newline_length]) + elsif token.type == :REGEXP_END + value = value[0] + location = Range.new(source_buffer, offset_cache[token.location.start_offset], offset_cache[token.location.start_offset + 1]) + end + when :tSYMBEG + if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :EMBEXPR_BEGIN && next_token.type != :EMBVAR + next_location = token.location.join(next_token.location) + type = :tSYMBOL + value = next_token.value + value = { "~@" => "~", "!@" => "!" }.fetch(value, value) + location = Range.new(source_buffer, offset_cache[next_location.start_offset], offset_cache[next_location.end_offset]) + index += 1 + end + when :tFID + if !tokens.empty? && tokens.dig(-1, 0) == :kDEF + type = :tIDENTIFIER + end + when :tXSTRING_BEG + if (next_token = lexed[index][0]) && next_token.type != :STRING_CONTENT && next_token.type != :STRING_END + type = :tBACK_REF2 + end + end + + tokens << [type, [value, location]] + + if token.type == :REGEXP_END + tokens << [:tREGEXP_OPT, [token.value[1..], Range.new(source_buffer, offset_cache[token.location.start_offset + 1], offset_cache[token.location.end_offset])]] + end + end + + tokens + end + + private + + # Parse an integer from the string representation. + def parse_integer(value) + Integer(value) + rescue ArgumentError + 0 + end + + # Parse a float from the string representation. + def parse_float(value) + Float(value) + rescue ArgumentError + 0.0 + end + + # Parse a complex from the string representation. + def parse_complex(value) + value.chomp!("i") + + if value.end_with?("r") + Complex(0, parse_rational(value)) + elsif value.start_with?(/0[BbOoDdXx]/) + Complex(0, parse_integer(value)) + else + Complex(0, value) + end + rescue ArgumentError + 0i + end + + # Parse a rational from the string representation. + def parse_rational(value) + value.chomp!("r") + + if value.start_with?(/0[BbOoDdXx]/) + Rational(parse_integer(value)) + else + Rational(value) + end + rescue ArgumentError + 0r + end + end + end + end +end diff --git a/lib/prism/translation/parser/rubocop.rb b/lib/prism/translation/parser/rubocop.rb new file mode 100644 index 0000000000..6c9687a5cc --- /dev/null +++ b/lib/prism/translation/parser/rubocop.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true +# typed: ignore + +warn "WARN: Prism is directly supported since RuboCop 1.62. The `prism/translation/parser/rubocop` file is deprecated." + +require "parser" +require "rubocop" + +require_relative "../../prism" +require_relative "../parser" + +module Prism + module Translation + class Parser + # This is the special version numbers that should be used in RuboCop + # configuration files to trigger using prism. + + # For Ruby 3.3 + VERSION_3_3 = 80_82_73_83_77.33 + + # For Ruby 3.4 + VERSION_3_4 = 80_82_73_83_77.34 + + # This module gets prepended into RuboCop::AST::ProcessedSource. + module ProcessedSource + # This condition is compatible with rubocop-ast versions up to 1.30.0. + if RuboCop::AST::ProcessedSource.instance_method(:parser_class).arity == 1 + # Redefine parser_class so that we can inject the prism parser into the + # list of known parsers. + def parser_class(ruby_version) + if ruby_version == Prism::Translation::Parser::VERSION_3_3 + warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.33` is deprecated. " \ + "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.3` instead." + require_relative "../parser33" + Prism::Translation::Parser33 + elsif ruby_version == Prism::Translation::Parser::VERSION_3_4 + warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.34` is deprecated. " \ + "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.4` instead." + require_relative "../parser34" + Prism::Translation::Parser34 + else + super + end + end + else + # Redefine parser_class so that we can inject the prism parser into the + # list of known parsers. + def parser_class(ruby_version, _parser_engine) + if ruby_version == Prism::Translation::Parser::VERSION_3_3 + warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.33` is deprecated. " \ + "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.3` instead." + require_relative "../parser33" + Prism::Translation::Parser33 + elsif ruby_version == Prism::Translation::Parser::VERSION_3_4 + warn "WARN: Setting `TargetRubyVersion: 80_82_73_83_77.34` is deprecated. " \ + "Set to `ParserEngine: parser_prism` and `TargetRubyVersion: 3.4` instead." + require_relative "../parser34" + Prism::Translation::Parser34 + else + super + end + end + end + end + end + end +end + +# :stopdoc: +RuboCop::AST::ProcessedSource.prepend(Prism::Translation::Parser::ProcessedSource) +known_rubies = RuboCop::TargetRuby.const_get(:KNOWN_RUBIES) +RuboCop::TargetRuby.send(:remove_const, :KNOWN_RUBIES) +RuboCop::TargetRuby::KNOWN_RUBIES = [*known_rubies, Prism::Translation::Parser::VERSION_3_3].freeze diff --git a/lib/prism/translation/parser33.rb b/lib/prism/translation/parser33.rb new file mode 100644 index 0000000000..b09266e06a --- /dev/null +++ b/lib/prism/translation/parser33.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Prism + module Translation + # This class is the entry-point for Ruby 3.3 of `Prism::Translation::Parser`. + class Parser33 < Parser + def version # :nodoc: + 33 + end + end + end +end diff --git a/lib/prism/translation/parser34.rb b/lib/prism/translation/parser34.rb new file mode 100644 index 0000000000..0ead70ad3c --- /dev/null +++ b/lib/prism/translation/parser34.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Prism + module Translation + # This class is the entry-point for Ruby 3.4 of `Prism::Translation::Parser`. + class Parser34 < Parser + def version # :nodoc: + 34 + end + end + end +end diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb new file mode 100644 index 0000000000..79ba0e7ab3 --- /dev/null +++ b/lib/prism/translation/ripper.rb @@ -0,0 +1,3452 @@ +# frozen_string_literal: true + +require "ripper" + +module Prism + module Translation + # This class provides a compatibility layer between prism and Ripper. It + # functions by parsing the entire tree first and then walking it and + # executing each of the Ripper callbacks as it goes. To use this class, you + # treat `Prism::Translation::Ripper` effectively as you would treat the + # `Ripper` class. + # + # Note that this class will serve the most common use cases, but Ripper's + # API is extensive and undocumented. It relies on reporting the state of the + # parser at any given time. We do our best to replicate that here, but + # because it is a different architecture it is not possible to perfectly + # replicate the behavior of Ripper. + # + # The main known difference is that we may omit dispatching some events in + # some cases. This impacts the following events: + # + # - on_assign_error + # - on_comma + # - on_ignored_nl + # - on_ignored_sp + # - on_kw + # - on_label_end + # - on_lbrace + # - on_lbracket + # - on_lparen + # - on_nl + # - on_op + # - on_operator_ambiguous + # - on_rbrace + # - on_rbracket + # - on_rparen + # - on_semicolon + # - on_sp + # - on_symbeg + # - on_tstring_beg + # - on_tstring_end + # + class Ripper < Compiler + # Parses the given Ruby program read from +src+. + # +src+ must be a String or an IO or a object with a #gets method. + def self.parse(src, filename = "(ripper)", lineno = 1) + new(src, filename, lineno).parse + end + + # Tokenizes the Ruby program and returns an array of an array, + # which is formatted like + # <code>[[lineno, column], type, token, state]</code>. + # The +filename+ argument is mostly ignored. + # By default, this method does not handle syntax errors in +src+, + # use the +raise_errors+ keyword to raise a SyntaxError for an error in +src+. + # + # require "ripper" + # require "pp" + # + # pp Ripper.lex("def m(a) nil end") + # #=> [[[1, 0], :on_kw, "def", FNAME ], + # [[1, 3], :on_sp, " ", FNAME ], + # [[1, 4], :on_ident, "m", ENDFN ], + # [[1, 5], :on_lparen, "(", BEG|LABEL], + # [[1, 6], :on_ident, "a", ARG ], + # [[1, 7], :on_rparen, ")", ENDFN ], + # [[1, 8], :on_sp, " ", BEG ], + # [[1, 9], :on_kw, "nil", END ], + # [[1, 12], :on_sp, " ", END ], + # [[1, 13], :on_kw, "end", END ]] + # + def self.lex(src, filename = "-", lineno = 1, raise_errors: false) + result = Prism.lex_compat(src, filepath: filename, line: lineno) + + if result.failure? && raise_errors + raise SyntaxError, result.errors.first.message + else + result.value + end + end + + # This contains a table of all of the parser events and their + # corresponding arity. + PARSER_EVENT_TABLE = { + BEGIN: 1, + END: 1, + alias: 2, + alias_error: 2, + aref: 2, + aref_field: 2, + arg_ambiguous: 1, + arg_paren: 1, + args_add: 2, + args_add_block: 2, + args_add_star: 2, + args_forward: 0, + args_new: 0, + array: 1, + aryptn: 4, + assign: 2, + assign_error: 2, + assoc_new: 2, + assoc_splat: 1, + assoclist_from_args: 1, + bare_assoc_hash: 1, + begin: 1, + binary: 3, + block_var: 2, + blockarg: 1, + bodystmt: 4, + brace_block: 2, + break: 1, + call: 3, + case: 2, + class: 3, + class_name_error: 2, + command: 2, + command_call: 4, + const_path_field: 2, + const_path_ref: 2, + const_ref: 1, + def: 3, + defined: 1, + defs: 5, + do_block: 2, + dot2: 2, + dot3: 2, + dyna_symbol: 1, + else: 1, + elsif: 3, + ensure: 1, + excessed_comma: 0, + fcall: 1, + field: 3, + fndptn: 4, + for: 3, + hash: 1, + heredoc_dedent: 2, + hshptn: 3, + if: 3, + if_mod: 2, + ifop: 3, + in: 3, + kwrest_param: 1, + lambda: 2, + magic_comment: 2, + massign: 2, + method_add_arg: 2, + method_add_block: 2, + mlhs_add: 2, + mlhs_add_post: 2, + mlhs_add_star: 2, + mlhs_new: 0, + mlhs_paren: 1, + module: 2, + mrhs_add: 2, + mrhs_add_star: 2, + mrhs_new: 0, + mrhs_new_from_args: 1, + next: 1, + nokw_param: 1, + opassign: 3, + operator_ambiguous: 2, + param_error: 2, + params: 7, + paren: 1, + parse_error: 1, + program: 1, + qsymbols_add: 2, + qsymbols_new: 0, + qwords_add: 2, + qwords_new: 0, + redo: 0, + regexp_add: 2, + regexp_literal: 2, + regexp_new: 0, + rescue: 4, + rescue_mod: 2, + rest_param: 1, + retry: 0, + return: 1, + return0: 0, + sclass: 2, + stmts_add: 2, + stmts_new: 0, + string_add: 2, + string_concat: 2, + string_content: 0, + string_dvar: 1, + string_embexpr: 1, + string_literal: 1, + super: 1, + symbol: 1, + symbol_literal: 1, + symbols_add: 2, + symbols_new: 0, + top_const_field: 1, + top_const_ref: 1, + unary: 2, + undef: 1, + unless: 3, + unless_mod: 2, + until: 2, + until_mod: 2, + var_alias: 2, + var_field: 1, + var_ref: 1, + vcall: 1, + void_stmt: 0, + when: 3, + while: 2, + while_mod: 2, + word_add: 2, + word_new: 0, + words_add: 2, + words_new: 0, + xstring_add: 2, + xstring_literal: 1, + xstring_new: 0, + yield: 1, + yield0: 0, + zsuper: 0 + } + + # This contains a table of all of the scanner events and their + # corresponding arity. + SCANNER_EVENT_TABLE = { + CHAR: 1, + __end__: 1, + backref: 1, + backtick: 1, + comma: 1, + comment: 1, + const: 1, + cvar: 1, + embdoc: 1, + embdoc_beg: 1, + embdoc_end: 1, + embexpr_beg: 1, + embexpr_end: 1, + embvar: 1, + float: 1, + gvar: 1, + heredoc_beg: 1, + heredoc_end: 1, + ident: 1, + ignored_nl: 1, + imaginary: 1, + int: 1, + ivar: 1, + kw: 1, + label: 1, + label_end: 1, + lbrace: 1, + lbracket: 1, + lparen: 1, + nl: 1, + op: 1, + period: 1, + qsymbols_beg: 1, + qwords_beg: 1, + rational: 1, + rbrace: 1, + rbracket: 1, + regexp_beg: 1, + regexp_end: 1, + rparen: 1, + semicolon: 1, + sp: 1, + symbeg: 1, + symbols_beg: 1, + tlambda: 1, + tlambeg: 1, + tstring_beg: 1, + tstring_content: 1, + tstring_end: 1, + words_beg: 1, + words_sep: 1, + ignored_sp: 1 + } + + # This array contains name of parser events. + PARSER_EVENTS = PARSER_EVENT_TABLE.keys + + # This array contains name of scanner events. + SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys + + # This array contains name of all ripper events. + EVENTS = PARSER_EVENTS + SCANNER_EVENTS + + # A list of all of the Ruby keywords. + KEYWORDS = [ + "alias", + "and", + "begin", + "BEGIN", + "break", + "case", + "class", + "def", + "defined?", + "do", + "else", + "elsif", + "end", + "END", + "ensure", + "false", + "for", + "if", + "in", + "module", + "next", + "nil", + "not", + "or", + "redo", + "rescue", + "retry", + "return", + "self", + "super", + "then", + "true", + "undef", + "unless", + "until", + "when", + "while", + "yield", + "__ENCODING__", + "__FILE__", + "__LINE__" + ] + + # A list of all of the Ruby binary operators. + BINARY_OPERATORS = [ + :!=, + :!~, + :=~, + :==, + :===, + :<=>, + :>, + :>=, + :<, + :<=, + :&, + :|, + :^, + :>>, + :<<, + :-, + :+, + :%, + :/, + :*, + :** + ] + + private_constant :KEYWORDS, :BINARY_OPERATORS + + # Parses +src+ and create S-exp tree. + # Returns more readable tree rather than Ripper.sexp_raw. + # This method is mainly for developer use. + # The +filename+ argument is mostly ignored. + # By default, this method does not handle syntax errors in +src+, + # returning +nil+ in such cases. Use the +raise_errors+ keyword + # to raise a SyntaxError for an error in +src+. + # + # require "ripper" + # require "pp" + # + # pp Ripper.sexp("def m(a) nil end") + # #=> [:program, + # [[:def, + # [:@ident, "m", [1, 4]], + # [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil, nil, nil, nil]], + # [:bodystmt, [[:var_ref, [:@kw, "nil", [1, 9]]]], nil, nil, nil]]]] + # + def self.sexp(src, filename = "-", lineno = 1, raise_errors: false) + builder = SexpBuilderPP.new(src, filename, lineno) + sexp = builder.parse + if builder.error? + if raise_errors + raise SyntaxError, builder.error + end + else + sexp + end + end + + # Parses +src+ and create S-exp tree. + # This method is mainly for developer use. + # The +filename+ argument is mostly ignored. + # By default, this method does not handle syntax errors in +src+, + # returning +nil+ in such cases. Use the +raise_errors+ keyword + # to raise a SyntaxError for an error in +src+. + # + # require "ripper" + # require "pp" + # + # pp Ripper.sexp_raw("def m(a) nil end") + # #=> [:program, + # [:stmts_add, + # [:stmts_new], + # [:def, + # [:@ident, "m", [1, 4]], + # [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil]], + # [:bodystmt, + # [:stmts_add, [:stmts_new], [:var_ref, [:@kw, "nil", [1, 9]]]], + # nil, + # nil, + # nil]]]] + # + def self.sexp_raw(src, filename = "-", lineno = 1, raise_errors: false) + builder = SexpBuilder.new(src, filename, lineno) + sexp = builder.parse + if builder.error? + if raise_errors + raise SyntaxError, builder.error + end + else + sexp + end + end + + autoload :SexpBuilder, "prism/translation/ripper/sexp" + autoload :SexpBuilderPP, "prism/translation/ripper/sexp" + + # The source that is being parsed. + attr_reader :source + + # The filename of the source being parsed. + attr_reader :filename + + # The current line number of the parser. + attr_reader :lineno + + # The current column number of the parser. + attr_reader :column + + # Create a new Translation::Ripper object with the given source. + def initialize(source, filename = "(ripper)", lineno = 1) + @source = source + @filename = filename + @lineno = lineno + @column = 0 + @result = nil + end + + ########################################################################## + # Public interface + ########################################################################## + + # True if the parser encountered an error during parsing. + def error? + result.failure? + end + + # Parse the source and return the result. + def parse + result.comments.each do |comment| + location = comment.location + bounds(location) + + if comment.is_a?(InlineComment) + on_comment(comment.slice) + else + offset = location.start_offset + lines = comment.slice.lines + + lines.each_with_index do |line, index| + bounds(location.copy(start_offset: offset)) + + if index == 0 + on_embdoc_beg(line) + elsif index == lines.size - 1 + on_embdoc_end(line) + else + on_embdoc(line) + end + + offset += line.bytesize + end + end + end + + result.magic_comments.each do |magic_comment| + on_magic_comment(magic_comment.key, magic_comment.value) + end + + unless result.data_loc.nil? + on___end__(result.data_loc.slice.each_line.first) + end + + result.warnings.each do |warning| + bounds(warning.location) + + if warning.level == :default + warning(warning.message) + else + case warning.type + when :ambiguous_first_argument_plus + on_arg_ambiguous("+") + when :ambiguous_first_argument_minus + on_arg_ambiguous("-") + when :ambiguous_slash + on_arg_ambiguous("/") + else + warn(warning.message) + end + end + end + + if error? + result.errors.each do |error| + location = error.location + bounds(location) + + case error.type + when :alias_argument + on_alias_error("can't make alias for the number variables", location.slice) + when :argument_formal_class + on_param_error("formal argument cannot be a class variable", location.slice) + when :argument_format_constant + on_param_error("formal argument cannot be a constant", location.slice) + when :argument_formal_global + on_param_error("formal argument cannot be a global variable", location.slice) + when :argument_formal_ivar + on_param_error("formal argument cannot be an instance variable", location.slice) + when :class_name, :module_name + on_class_name_error("class/module name must be CONSTANT", location.slice) + else + on_parse_error(error.message) + end + end + + nil + else + result.value.accept(self) + end + end + + ########################################################################## + # Visitor methods + ########################################################################## + + # alias foo bar + # ^^^^^^^^^^^^^ + def visit_alias_method_node(node) + new_name = visit(node.new_name) + old_name = visit(node.old_name) + + bounds(node.location) + on_alias(new_name, old_name) + end + + # alias $foo $bar + # ^^^^^^^^^^^^^^^ + def visit_alias_global_variable_node(node) + new_name = visit_alias_global_variable_node_value(node.new_name) + old_name = visit_alias_global_variable_node_value(node.old_name) + + bounds(node.location) + on_var_alias(new_name, old_name) + end + + # Visit one side of an alias global variable node. + private def visit_alias_global_variable_node_value(node) + bounds(node.location) + + case node + when BackReferenceReadNode + on_backref(node.slice) + when GlobalVariableReadNode + on_gvar(node.name.to_s) + else + raise + end + end + + # foo => bar | baz + # ^^^^^^^^^ + def visit_alternation_pattern_node(node) + left = visit_pattern_node(node.left) + right = visit_pattern_node(node.right) + + bounds(node.location) + on_binary(left, :|, right) + end + + # Visit a pattern within a pattern match. This is used to bypass the + # parenthesis node that can be used to wrap patterns. + private def visit_pattern_node(node) + if node.is_a?(ParenthesesNode) + visit(node.body) + else + visit(node) + end + end + + # a and b + # ^^^^^^^ + def visit_and_node(node) + left = visit(node.left) + right = visit(node.right) + + bounds(node.location) + on_binary(left, node.operator.to_sym, right) + end + + # [] + # ^^ + def visit_array_node(node) + case (opening = node.opening) + when /^%w/ + opening_loc = node.opening_loc + bounds(opening_loc) + on_qwords_beg(opening) + + elements = on_qwords_new + previous = nil + + node.elements.each do |element| + visit_words_sep(opening_loc, previous, element) + + bounds(element.location) + elements = on_qwords_add(elements, on_tstring_content(element.content)) + + previous = element + end + + bounds(node.closing_loc) + on_tstring_end(node.closing) + when /^%i/ + opening_loc = node.opening_loc + bounds(opening_loc) + on_qsymbols_beg(opening) + + elements = on_qsymbols_new + previous = nil + + node.elements.each do |element| + visit_words_sep(opening_loc, previous, element) + + bounds(element.location) + elements = on_qsymbols_add(elements, on_tstring_content(element.value)) + + previous = element + end + + bounds(node.closing_loc) + on_tstring_end(node.closing) + when /^%W/ + opening_loc = node.opening_loc + bounds(opening_loc) + on_words_beg(opening) + + elements = on_words_new + previous = nil + + node.elements.each do |element| + visit_words_sep(opening_loc, previous, element) + + bounds(element.location) + elements = + on_words_add( + elements, + if element.is_a?(StringNode) + on_word_add(on_word_new, on_tstring_content(element.content)) + else + element.parts.inject(on_word_new) do |word, part| + word_part = + if part.is_a?(StringNode) + bounds(part.location) + on_tstring_content(part.content) + else + visit(part) + end + + on_word_add(word, word_part) + end + end + ) + + previous = element + end + + bounds(node.closing_loc) + on_tstring_end(node.closing) + when /^%I/ + opening_loc = node.opening_loc + bounds(opening_loc) + on_symbols_beg(opening) + + elements = on_symbols_new + previous = nil + + node.elements.each do |element| + visit_words_sep(opening_loc, previous, element) + + bounds(element.location) + elements = + on_symbols_add( + elements, + if element.is_a?(SymbolNode) + on_word_add(on_word_new, on_tstring_content(element.value)) + else + element.parts.inject(on_word_new) do |word, part| + word_part = + if part.is_a?(StringNode) + bounds(part.location) + on_tstring_content(part.content) + else + visit(part) + end + + on_word_add(word, word_part) + end + end + ) + + previous = element + end + + bounds(node.closing_loc) + on_tstring_end(node.closing) + else + bounds(node.opening_loc) + on_lbracket(opening) + + elements = visit_arguments(node.elements) unless node.elements.empty? + + bounds(node.closing_loc) + on_rbracket(node.closing) + end + + bounds(node.location) + on_array(elements) + end + + # Dispatch a words_sep event that contains the space between the elements + # of list literals. + private def visit_words_sep(opening_loc, previous, current) + end_offset = (previous.nil? ? opening_loc : previous.location).end_offset + start_offset = current.location.start_offset + + if end_offset != start_offset + bounds(current.location.copy(start_offset: end_offset)) + on_words_sep(source.byteslice(end_offset...start_offset)) + end + end + + # Visit a list of elements, like the elements of an array or arguments. + private def visit_arguments(elements) + bounds(elements.first.location) + elements.inject(on_args_new) do |args, element| + arg = visit(element) + bounds(element.location) + + case element + when BlockArgumentNode + on_args_add_block(args, arg) + when SplatNode + on_args_add_star(args, arg) + else + on_args_add(args, arg) + end + end + end + + # foo => [bar] + # ^^^^^ + def visit_array_pattern_node(node) + constant = visit(node.constant) + requireds = visit_all(node.requireds) if node.requireds.any? + rest = + if (rest_node = node.rest).is_a?(SplatNode) + if rest_node.expression.nil? + bounds(rest_node.location) + on_var_field(nil) + else + visit(rest_node.expression) + end + end + + posts = visit_all(node.posts) if node.posts.any? + + bounds(node.location) + on_aryptn(constant, requireds, rest, posts) + end + + # foo(bar) + # ^^^ + def visit_arguments_node(node) + arguments, _ = visit_call_node_arguments(node, nil, false) + arguments + end + + # { a: 1 } + # ^^^^ + def visit_assoc_node(node) + key = visit(node.key) + value = visit(node.value) + + bounds(node.location) + on_assoc_new(key, value) + end + + # def foo(**); bar(**); end + # ^^ + # + # { **foo } + # ^^^^^ + def visit_assoc_splat_node(node) + value = visit(node.value) + + bounds(node.location) + on_assoc_splat(value) + end + + # $+ + # ^^ + def visit_back_reference_read_node(node) + bounds(node.location) + on_backref(node.slice) + end + + # begin end + # ^^^^^^^^^ + def visit_begin_node(node) + clauses = visit_begin_node_clauses(node.begin_keyword_loc, node, false) + + bounds(node.location) + on_begin(clauses) + end + + # Visit the clauses of a begin node to form an on_bodystmt call. + private def visit_begin_node_clauses(location, node, allow_newline) + statements = + if node.statements.nil? + on_stmts_add(on_stmts_new, on_void_stmt) + else + body = node.statements.body + body.unshift(nil) if void_stmt?(location, node.statements.body[0].location, allow_newline) + + bounds(node.statements.location) + visit_statements_node_body(body) + end + + rescue_clause = visit(node.rescue_clause) + else_clause = + unless (else_clause_node = node.else_clause).nil? + else_statements = + if else_clause_node.statements.nil? + [nil] + else + body = else_clause_node.statements.body + body.unshift(nil) if void_stmt?(else_clause_node.else_keyword_loc, else_clause_node.statements.body[0].location, allow_newline) + body + end + + bounds(else_clause_node.location) + visit_statements_node_body(else_statements) + end + ensure_clause = visit(node.ensure_clause) + + bounds(node.location) + on_bodystmt(statements, rescue_clause, else_clause, ensure_clause) + end + + # Visit the body of a structure that can have either a set of statements + # or statements wrapped in rescue/else/ensure. + private def visit_body_node(location, node, allow_newline = false) + case node + when nil + bounds(location) + on_bodystmt(visit_statements_node_body([nil]), nil, nil, nil) + when StatementsNode + body = [*node.body] + body.unshift(nil) if void_stmt?(location, body[0].location, allow_newline) + stmts = visit_statements_node_body(body) + + bounds(node.body.first.location) + on_bodystmt(stmts, nil, nil, nil) + when BeginNode + visit_begin_node_clauses(location, node, allow_newline) + else + raise + end + end + + # foo(&bar) + # ^^^^ + def visit_block_argument_node(node) + visit(node.expression) + end + + # foo { |; bar| } + # ^^^ + def visit_block_local_variable_node(node) + bounds(node.location) + on_ident(node.name.to_s) + end + + # Visit a BlockNode. + def visit_block_node(node) + braces = node.opening == "{" + parameters = visit(node.parameters) + + body = + case node.body + when nil + bounds(node.location) + stmts = on_stmts_add(on_stmts_new, on_void_stmt) + + bounds(node.location) + braces ? stmts : on_bodystmt(stmts, nil, nil, nil) + when StatementsNode + stmts = node.body.body + stmts.unshift(nil) if void_stmt?(node.parameters&.location || node.opening_loc, node.body.location, false) + stmts = visit_statements_node_body(stmts) + + bounds(node.body.location) + braces ? stmts : on_bodystmt(stmts, nil, nil, nil) + when BeginNode + visit_body_node(node.parameters&.location || node.opening_loc, node.body) + else + raise + end + + if braces + bounds(node.location) + on_brace_block(parameters, body) + else + bounds(node.location) + on_do_block(parameters, body) + end + end + + # def foo(&bar); end + # ^^^^ + def visit_block_parameter_node(node) + if node.name_loc.nil? + bounds(node.location) + on_blockarg(nil) + else + bounds(node.name_loc) + name = visit_token(node.name.to_s) + + bounds(node.location) + on_blockarg(name) + end + end + + # A block's parameters. + def visit_block_parameters_node(node) + parameters = + if node.parameters.nil? + on_params(nil, nil, nil, nil, nil, nil, nil) + else + visit(node.parameters) + end + + locals = + if node.locals.any? + visit_all(node.locals) + else + false + end + + bounds(node.location) + on_block_var(parameters, locals) + end + + # break + # ^^^^^ + # + # break foo + # ^^^^^^^^^ + def visit_break_node(node) + if node.arguments.nil? + bounds(node.location) + on_break(on_args_new) + else + arguments = visit(node.arguments) + + bounds(node.location) + on_break(arguments) + end + end + + # foo + # ^^^ + # + # foo.bar + # ^^^^^^^ + # + # foo.bar() {} + # ^^^^^^^^^^^^ + def visit_call_node(node) + if node.call_operator_loc.nil? + case node.name + when :[] + receiver = visit(node.receiver) + arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.location) + call = on_aref(receiver, arguments) + + if block.nil? + call + else + bounds(node.location) + on_method_add_block(call, block) + end + when :[]= + receiver = visit(node.receiver) + + *arguments, last_argument = node.arguments.arguments + arguments << node.block if !node.block.nil? + + arguments = + if arguments.any? + args = visit_arguments(arguments) + + if !node.block.nil? + args + else + bounds(arguments.first.location) + on_args_add_block(args, false) + end + end + + bounds(node.location) + call = on_aref_field(receiver, arguments) + value = visit_write_value(last_argument) + + bounds(last_argument.location) + on_assign(call, value) + when :-@, :+@, :~ + receiver = visit(node.receiver) + + bounds(node.location) + on_unary(node.name, receiver) + when :! + receiver = visit(node.receiver) + + bounds(node.location) + on_unary(node.message == "not" ? :not : :!, receiver) + when *BINARY_OPERATORS + receiver = visit(node.receiver) + value = visit(node.arguments.arguments.first) + + bounds(node.location) + on_binary(receiver, node.name, value) + else + bounds(node.message_loc) + message = visit_token(node.message, false) + + if node.variable_call? + on_vcall(message) + else + arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location)) + call = + if node.opening_loc.nil? && arguments&.any? + bounds(node.location) + on_command(message, arguments) + elsif !node.opening_loc.nil? + bounds(node.location) + on_method_add_arg(on_fcall(message), on_arg_paren(arguments)) + else + bounds(node.location) + on_method_add_arg(on_fcall(message), on_args_new) + end + + if block.nil? + call + else + bounds(node.block.location) + on_method_add_block(call, block) + end + end + end + else + receiver = visit(node.receiver) + + bounds(node.call_operator_loc) + call_operator = visit_token(node.call_operator) + + message = + if node.message_loc.nil? + :call + else + bounds(node.message_loc) + visit_token(node.message, false) + end + + if node.name.end_with?("=") && !node.message.end_with?("=") && !node.arguments.nil? && node.block.nil? + value = visit_write_value(node.arguments.arguments.first) + + bounds(node.location) + on_assign(on_field(receiver, call_operator, message), value) + else + arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc || node.location)) + call = + if node.opening_loc.nil? + bounds(node.location) + + if node.arguments.nil? && !node.block.is_a?(BlockArgumentNode) + on_call(receiver, call_operator, message) + else + on_command_call(receiver, call_operator, message, arguments) + end + else + bounds(node.opening_loc) + arguments = on_arg_paren(arguments) + + bounds(node.location) + on_method_add_arg(on_call(receiver, call_operator, message), arguments) + end + + if block.nil? + call + else + bounds(node.block.location) + on_method_add_block(call, block) + end + end + end + end + + # Visit the arguments and block of a call node and return the arguments + # and block as they should be used. + private def visit_call_node_arguments(arguments_node, block_node, trailing_comma) + arguments = arguments_node&.arguments || [] + block = block_node + + if block.is_a?(BlockArgumentNode) + arguments << block + block = nil + end + + [ + if arguments.length == 1 && arguments.first.is_a?(ForwardingArgumentsNode) + visit(arguments.first) + elsif arguments.any? + args = visit_arguments(arguments) + + if block_node.is_a?(BlockArgumentNode) || arguments.last.is_a?(ForwardingArgumentsNode) || command?(arguments.last) || trailing_comma + args + else + bounds(arguments.first.location) + on_args_add_block(args, false) + end + end, + visit(block) + ] + end + + # Returns true if the given node is a command node. + private def command?(node) + node.is_a?(CallNode) && + node.opening_loc.nil? && + (!node.arguments.nil? || node.block.is_a?(BlockArgumentNode)) && + !BINARY_OPERATORS.include?(node.name) + end + + # foo.bar += baz + # ^^^^^^^^^^^^^^^ + def visit_call_operator_write_node(node) + receiver = visit(node.receiver) + + bounds(node.call_operator_loc) + call_operator = visit_token(node.call_operator) + + bounds(node.message_loc) + message = visit_token(node.message) + + bounds(node.location) + target = on_field(receiver, call_operator, message) + + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # foo.bar &&= baz + # ^^^^^^^^^^^^^^^ + def visit_call_and_write_node(node) + receiver = visit(node.receiver) + + bounds(node.call_operator_loc) + call_operator = visit_token(node.call_operator) + + bounds(node.message_loc) + message = visit_token(node.message) + + bounds(node.location) + target = on_field(receiver, call_operator, message) + + bounds(node.operator_loc) + operator = on_op("&&=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # foo.bar ||= baz + # ^^^^^^^^^^^^^^^ + def visit_call_or_write_node(node) + receiver = visit(node.receiver) + + bounds(node.call_operator_loc) + call_operator = visit_token(node.call_operator) + + bounds(node.message_loc) + message = visit_token(node.message) + + bounds(node.location) + target = on_field(receiver, call_operator, message) + + bounds(node.operator_loc) + operator = on_op("||=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # foo.bar, = 1 + # ^^^^^^^ + def visit_call_target_node(node) + if node.call_operator == "::" + receiver = visit(node.receiver) + + bounds(node.message_loc) + message = visit_token(node.message) + + bounds(node.location) + on_const_path_field(receiver, message) + else + receiver = visit(node.receiver) + + bounds(node.call_operator_loc) + call_operator = visit_token(node.call_operator) + + bounds(node.message_loc) + message = visit_token(node.message) + + bounds(node.location) + on_field(receiver, call_operator, message) + end + end + + # foo => bar => baz + # ^^^^^^^^^^ + def visit_capture_pattern_node(node) + value = visit(node.value) + target = visit(node.target) + + bounds(node.location) + on_binary(value, :"=>", target) + end + + # case foo; when bar; end + # ^^^^^^^^^^^^^^^^^^^^^^^ + def visit_case_node(node) + predicate = visit(node.predicate) + clauses = + node.conditions.reverse_each.inject(visit(node.consequent)) do |consequent, condition| + on_when(*visit(condition), consequent) + end + + bounds(node.location) + on_case(predicate, clauses) + end + + # case foo; in bar; end + # ^^^^^^^^^^^^^^^^^^^^^ + def visit_case_match_node(node) + predicate = visit(node.predicate) + clauses = + node.conditions.reverse_each.inject(visit(node.consequent)) do |consequent, condition| + on_in(*visit(condition), consequent) + end + + bounds(node.location) + on_case(predicate, clauses) + end + + # class Foo; end + # ^^^^^^^^^^^^^^ + def visit_class_node(node) + constant_path = + if node.constant_path.is_a?(ConstantReadNode) + bounds(node.constant_path.location) + on_const_ref(on_const(node.constant_path.name.to_s)) + else + visit(node.constant_path) + end + + superclass = visit(node.superclass) + bodystmt = visit_body_node(node.superclass&.location || node.constant_path.location, node.body, node.superclass.nil?) + + bounds(node.location) + on_class(constant_path, superclass, bodystmt) + end + + # @@foo + # ^^^^^ + def visit_class_variable_read_node(node) + bounds(node.location) + on_var_ref(on_cvar(node.slice)) + end + + # @@foo = 1 + # ^^^^^^^^^ + # + # @@foo, @@bar = 1 + # ^^^^^ ^^^^^ + def visit_class_variable_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_cvar(node.name.to_s)) + value = visit_write_value(node.value) + + bounds(node.location) + on_assign(target, value) + end + + # @@foo += bar + # ^^^^^^^^^^^^ + def visit_class_variable_operator_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_cvar(node.name.to_s)) + + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # @@foo &&= bar + # ^^^^^^^^^^^^^ + def visit_class_variable_and_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_cvar(node.name.to_s)) + + bounds(node.operator_loc) + operator = on_op("&&=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # @@foo ||= bar + # ^^^^^^^^^^^^^ + def visit_class_variable_or_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_cvar(node.name.to_s)) + + bounds(node.operator_loc) + operator = on_op("||=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # @@foo, = bar + # ^^^^^ + def visit_class_variable_target_node(node) + bounds(node.location) + on_var_field(on_cvar(node.name.to_s)) + end + + # Foo + # ^^^ + def visit_constant_read_node(node) + bounds(node.location) + on_var_ref(on_const(node.name.to_s)) + end + + # Foo = 1 + # ^^^^^^^ + # + # Foo, Bar = 1 + # ^^^ ^^^ + def visit_constant_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_const(node.name.to_s)) + value = visit_write_value(node.value) + + bounds(node.location) + on_assign(target, value) + end + + # Foo += bar + # ^^^^^^^^^^^ + def visit_constant_operator_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_const(node.name.to_s)) + + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # Foo &&= bar + # ^^^^^^^^^^^^ + def visit_constant_and_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_const(node.name.to_s)) + + bounds(node.operator_loc) + operator = on_op("&&=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # Foo ||= bar + # ^^^^^^^^^^^^ + def visit_constant_or_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_const(node.name.to_s)) + + bounds(node.operator_loc) + operator = on_op("||=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # Foo, = bar + # ^^^ + def visit_constant_target_node(node) + bounds(node.location) + on_var_field(on_const(node.name.to_s)) + end + + # Foo::Bar + # ^^^^^^^^ + def visit_constant_path_node(node) + if node.parent.nil? + bounds(node.name_loc) + child = on_const(node.name.to_s) + + bounds(node.location) + on_top_const_ref(child) + else + parent = visit(node.parent) + + bounds(node.name_loc) + child = on_const(node.name.to_s) + + bounds(node.location) + on_const_path_ref(parent, child) + end + end + + # Foo::Bar = 1 + # ^^^^^^^^^^^^ + # + # Foo::Foo, Bar::Bar = 1 + # ^^^^^^^^ ^^^^^^^^ + def visit_constant_path_write_node(node) + target = visit_constant_path_write_node_target(node.target) + value = visit_write_value(node.value) + + bounds(node.location) + on_assign(target, value) + end + + # Visit a constant path that is part of a write node. + private def visit_constant_path_write_node_target(node) + if node.parent.nil? + bounds(node.name_loc) + child = on_const(node.name.to_s) + + bounds(node.location) + on_top_const_field(child) + else + parent = visit(node.parent) + + bounds(node.name_loc) + child = on_const(node.name.to_s) + + bounds(node.location) + on_const_path_field(parent, child) + end + end + + # Foo::Bar += baz + # ^^^^^^^^^^^^^^^ + def visit_constant_path_operator_write_node(node) + target = visit_constant_path_write_node_target(node.target) + value = visit(node.value) + + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # Foo::Bar &&= baz + # ^^^^^^^^^^^^^^^^ + def visit_constant_path_and_write_node(node) + target = visit_constant_path_write_node_target(node.target) + value = visit(node.value) + + bounds(node.operator_loc) + operator = on_op("&&=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # Foo::Bar ||= baz + # ^^^^^^^^^^^^^^^^ + def visit_constant_path_or_write_node(node) + target = visit_constant_path_write_node_target(node.target) + value = visit(node.value) + + bounds(node.operator_loc) + operator = on_op("||=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # Foo::Bar, = baz + # ^^^^^^^^ + def visit_constant_path_target_node(node) + visit_constant_path_write_node_target(node) + end + + # def foo; end + # ^^^^^^^^^^^^ + # + # def self.foo; end + # ^^^^^^^^^^^^^^^^^ + def visit_def_node(node) + receiver = visit(node.receiver) + operator = + if !node.operator_loc.nil? + bounds(node.operator_loc) + visit_token(node.operator) + end + + bounds(node.name_loc) + name = visit_token(node.name_loc.slice) + + parameters = + if node.parameters.nil? + bounds(node.location) + on_params(nil, nil, nil, nil, nil, nil, nil) + else + visit(node.parameters) + end + + if !node.lparen_loc.nil? + bounds(node.lparen_loc) + parameters = on_paren(parameters) + end + + bodystmt = + if node.equal_loc.nil? + visit_body_node(node.rparen_loc || node.end_keyword_loc, node.body) + else + body = visit(node.body.body.first) + + bounds(node.body.location) + on_bodystmt(body, nil, nil, nil) + end + + bounds(node.location) + if receiver.nil? + on_def(name, parameters, bodystmt) + else + on_defs(receiver, operator, name, parameters, bodystmt) + end + end + + # defined? a + # ^^^^^^^^^^ + # + # defined?(a) + # ^^^^^^^^^^^ + def visit_defined_node(node) + bounds(node.location) + on_defined(visit(node.value)) + end + + # if foo then bar else baz end + # ^^^^^^^^^^^^ + def visit_else_node(node) + statements = + if node.statements.nil? + [nil] + else + body = node.statements.body + body.unshift(nil) if void_stmt?(node.else_keyword_loc, node.statements.body[0].location, false) + body + end + + bounds(node.location) + on_else(visit_statements_node_body(statements)) + end + + # "foo #{bar}" + # ^^^^^^ + def visit_embedded_statements_node(node) + bounds(node.opening_loc) + on_embexpr_beg(node.opening) + + statements = + if node.statements.nil? + bounds(node.location) + on_stmts_add(on_stmts_new, on_void_stmt) + else + visit(node.statements) + end + + bounds(node.closing_loc) + on_embexpr_end(node.closing) + + bounds(node.location) + on_string_embexpr(statements) + end + + # "foo #@bar" + # ^^^^^ + def visit_embedded_variable_node(node) + bounds(node.operator_loc) + on_embvar(node.operator) + + variable = visit(node.variable) + + bounds(node.location) + on_string_dvar(variable) + end + + # Visit an EnsureNode node. + def visit_ensure_node(node) + statements = + if node.statements.nil? + [nil] + else + body = node.statements.body + body.unshift(nil) if void_stmt?(node.ensure_keyword_loc, body[0].location, false) + body + end + + statements = visit_statements_node_body(statements) + + bounds(node.location) + on_ensure(statements) + end + + # false + # ^^^^^ + def visit_false_node(node) + bounds(node.location) + on_var_ref(on_kw("false")) + end + + # foo => [*, bar, *] + # ^^^^^^^^^^^ + def visit_find_pattern_node(node) + constant = visit(node.constant) + left = + if node.left.expression.nil? + bounds(node.left.location) + on_var_field(nil) + else + visit(node.left.expression) + end + + requireds = visit_all(node.requireds) if node.requireds.any? + right = + if node.right.expression.nil? + bounds(node.right.location) + on_var_field(nil) + else + visit(node.right.expression) + end + + bounds(node.location) + on_fndptn(constant, left, requireds, right) + end + + # if foo .. bar; end + # ^^^^^^^^^^ + def visit_flip_flop_node(node) + left = visit(node.left) + right = visit(node.right) + + bounds(node.location) + if node.exclude_end? + on_dot3(left, right) + else + on_dot2(left, right) + end + end + + # 1.0 + # ^^^ + def visit_float_node(node) + visit_number_node(node) { |text| on_float(text) } + end + + # for foo in bar do end + # ^^^^^^^^^^^^^^^^^^^^^ + def visit_for_node(node) + index = visit(node.index) + collection = visit(node.collection) + statements = + if node.statements.nil? + bounds(node.location) + on_stmts_add(on_stmts_new, on_void_stmt) + else + visit(node.statements) + end + + bounds(node.location) + on_for(index, collection, statements) + end + + # def foo(...); bar(...); end + # ^^^ + def visit_forwarding_arguments_node(node) + bounds(node.location) + on_args_forward + end + + # def foo(...); end + # ^^^ + def visit_forwarding_parameter_node(node) + bounds(node.location) + on_args_forward + end + + # super + # ^^^^^ + # + # super {} + # ^^^^^^^^ + def visit_forwarding_super_node(node) + if node.block.nil? + bounds(node.location) + on_zsuper + else + block = visit(node.block) + + bounds(node.location) + on_method_add_block(on_zsuper, block) + end + end + + # $foo + # ^^^^ + def visit_global_variable_read_node(node) + bounds(node.location) + on_var_ref(on_gvar(node.name.to_s)) + end + + # $foo = 1 + # ^^^^^^^^ + # + # $foo, $bar = 1 + # ^^^^ ^^^^ + def visit_global_variable_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_gvar(node.name.to_s)) + value = visit_write_value(node.value) + + bounds(node.location) + on_assign(target, value) + end + + # $foo += bar + # ^^^^^^^^^^^ + def visit_global_variable_operator_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_gvar(node.name.to_s)) + + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # $foo &&= bar + # ^^^^^^^^^^^^ + def visit_global_variable_and_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_gvar(node.name.to_s)) + + bounds(node.operator_loc) + operator = on_op("&&=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # $foo ||= bar + # ^^^^^^^^^^^^ + def visit_global_variable_or_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_gvar(node.name.to_s)) + + bounds(node.operator_loc) + operator = on_op("||=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # $foo, = bar + # ^^^^ + def visit_global_variable_target_node(node) + bounds(node.location) + on_var_field(on_gvar(node.name.to_s)) + end + + # {} + # ^^ + def visit_hash_node(node) + elements = + if node.elements.any? + args = visit_all(node.elements) + + bounds(node.elements.first.location) + on_assoclist_from_args(args) + end + + bounds(node.location) + on_hash(elements) + end + + # foo => {} + # ^^ + def visit_hash_pattern_node(node) + constant = visit(node.constant) + elements = + if node.elements.any? || !node.rest.nil? + node.elements.map do |element| + [ + if (key = element.key).opening_loc.nil? + visit(key) + else + bounds(key.value_loc) + if (value = key.value).empty? + on_string_content + else + on_string_add(on_string_content, on_tstring_content(value)) + end + end, + visit(element.value) + ] + end + end + + rest = + case node.rest + when AssocSplatNode + visit(node.rest.value) + when NoKeywordsParameterNode + bounds(node.rest.location) + on_var_field(visit(node.rest)) + end + + bounds(node.location) + on_hshptn(constant, elements, rest) + end + + # if foo then bar end + # ^^^^^^^^^^^^^^^^^^^ + # + # bar if foo + # ^^^^^^^^^^ + # + # foo ? bar : baz + # ^^^^^^^^^^^^^^^ + def visit_if_node(node) + if node.then_keyword == "?" + predicate = visit(node.predicate) + truthy = visit(node.statements.body.first) + falsy = visit(node.consequent.statements.body.first) + + bounds(node.location) + on_ifop(predicate, truthy, falsy) + elsif node.statements.nil? || (node.predicate.location.start_offset < node.statements.location.start_offset) + predicate = visit(node.predicate) + statements = + if node.statements.nil? + bounds(node.location) + on_stmts_add(on_stmts_new, on_void_stmt) + else + visit(node.statements) + end + consequent = visit(node.consequent) + + bounds(node.location) + if node.if_keyword == "if" + on_if(predicate, statements, consequent) + else + on_elsif(predicate, statements, consequent) + end + else + statements = visit(node.statements.body.first) + predicate = visit(node.predicate) + + bounds(node.location) + on_if_mod(predicate, statements) + end + end + + # 1i + # ^^ + def visit_imaginary_node(node) + visit_number_node(node) { |text| on_imaginary(text) } + end + + # { foo: } + # ^^^^ + def visit_implicit_node(node) + end + + # foo { |bar,| } + # ^ + def visit_implicit_rest_node(node) + bounds(node.location) + on_excessed_comma + end + + # case foo; in bar; end + # ^^^^^^^^^^^^^^^^^^^^^ + def visit_in_node(node) + # This is a special case where we're not going to call on_in directly + # because we don't have access to the consequent. Instead, we'll return + # the component parts and let the parent node handle it. + pattern = visit_pattern_node(node.pattern) + statements = + if node.statements.nil? + bounds(node.location) + on_stmts_add(on_stmts_new, on_void_stmt) + else + visit(node.statements) + end + + [pattern, statements] + end + + # foo[bar] += baz + # ^^^^^^^^^^^^^^^ + def visit_index_operator_write_node(node) + receiver = visit(node.receiver) + arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.location) + target = on_aref_field(receiver, arguments) + + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # foo[bar] &&= baz + # ^^^^^^^^^^^^^^^^ + def visit_index_and_write_node(node) + receiver = visit(node.receiver) + arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.location) + target = on_aref_field(receiver, arguments) + + bounds(node.operator_loc) + operator = on_op("&&=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # foo[bar] ||= baz + # ^^^^^^^^^^^^^^^^ + def visit_index_or_write_node(node) + receiver = visit(node.receiver) + arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.location) + target = on_aref_field(receiver, arguments) + + bounds(node.operator_loc) + operator = on_op("||=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # foo[bar], = 1 + # ^^^^^^^^ + def visit_index_target_node(node) + receiver = visit(node.receiver) + arguments, _ = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.closing_loc)) + + bounds(node.location) + on_aref_field(receiver, arguments) + end + + # @foo + # ^^^^ + def visit_instance_variable_read_node(node) + bounds(node.location) + on_var_ref(on_ivar(node.name.to_s)) + end + + # @foo = 1 + # ^^^^^^^^ + def visit_instance_variable_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_ivar(node.name.to_s)) + value = visit_write_value(node.value) + + bounds(node.location) + on_assign(target, value) + end + + # @foo += bar + # ^^^^^^^^^^^ + def visit_instance_variable_operator_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_ivar(node.name.to_s)) + + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # @foo &&= bar + # ^^^^^^^^^^^^ + def visit_instance_variable_and_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_ivar(node.name.to_s)) + + bounds(node.operator_loc) + operator = on_op("&&=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # @foo ||= bar + # ^^^^^^^^^^^^ + def visit_instance_variable_or_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_ivar(node.name.to_s)) + + bounds(node.operator_loc) + operator = on_op("||=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # @foo, = bar + # ^^^^ + def visit_instance_variable_target_node(node) + bounds(node.location) + on_var_field(on_ivar(node.name.to_s)) + end + + # 1 + # ^ + def visit_integer_node(node) + visit_number_node(node) { |text| on_int(text) } + end + + # if /foo #{bar}/ then end + # ^^^^^^^^^^^^ + def visit_interpolated_match_last_line_node(node) + bounds(node.opening_loc) + on_regexp_beg(node.opening) + + bounds(node.parts.first.location) + parts = + node.parts.inject(on_regexp_new) do |content, part| + on_regexp_add(content, visit_string_content(part)) + end + + bounds(node.closing_loc) + closing = on_regexp_end(node.closing) + + bounds(node.location) + on_regexp_literal(parts, closing) + end + + # /foo #{bar}/ + # ^^^^^^^^^^^^ + def visit_interpolated_regular_expression_node(node) + bounds(node.opening_loc) + on_regexp_beg(node.opening) + + bounds(node.parts.first.location) + parts = + node.parts.inject(on_regexp_new) do |content, part| + on_regexp_add(content, visit_string_content(part)) + end + + bounds(node.closing_loc) + closing = on_regexp_end(node.closing) + + bounds(node.location) + on_regexp_literal(parts, closing) + end + + # "foo #{bar}" + # ^^^^^^^^^^^^ + def visit_interpolated_string_node(node) + if node.opening&.start_with?("<<~") + heredoc = visit_heredoc_string_node(node) + + bounds(node.location) + on_string_literal(heredoc) + elsif !node.heredoc? && node.parts.length > 1 && node.parts.any? { |part| (part.is_a?(StringNode) || part.is_a?(InterpolatedStringNode)) && !part.opening_loc.nil? } + first, *rest = node.parts + rest.inject(visit(first)) do |content, part| + concat = visit(part) + + bounds(part.location) + on_string_concat(content, concat) + end + else + bounds(node.parts.first.location) + parts = + node.parts.inject(on_string_content) do |content, part| + on_string_add(content, visit_string_content(part)) + end + + bounds(node.location) + on_string_literal(parts) + end + end + + # :"foo #{bar}" + # ^^^^^^^^^^^^^ + def visit_interpolated_symbol_node(node) + bounds(node.parts.first.location) + parts = + node.parts.inject(on_string_content) do |content, part| + on_string_add(content, visit_string_content(part)) + end + + bounds(node.location) + on_dyna_symbol(parts) + end + + # `foo #{bar}` + # ^^^^^^^^^^^^ + def visit_interpolated_x_string_node(node) + if node.opening.start_with?("<<~") + heredoc = visit_heredoc_x_string_node(node) + + bounds(node.location) + on_xstring_literal(heredoc) + else + bounds(node.parts.first.location) + parts = + node.parts.inject(on_xstring_new) do |content, part| + on_xstring_add(content, visit_string_content(part)) + end + + bounds(node.location) + on_xstring_literal(parts) + end + end + + # Visit an individual part of a string-like node. + private def visit_string_content(part) + if part.is_a?(StringNode) + bounds(part.content_loc) + on_tstring_content(part.content) + else + visit(part) + end + end + + # -> { it } + # ^^ + def visit_it_local_variable_read_node(node) + bounds(node.location) + on_vcall(on_ident(node.slice)) + end + + # -> { it } + # ^^^^^^^^^ + def visit_it_parameters_node(node) + end + + # foo(bar: baz) + # ^^^^^^^^ + def visit_keyword_hash_node(node) + elements = visit_all(node.elements) + + bounds(node.location) + on_bare_assoc_hash(elements) + end + + # def foo(**bar); end + # ^^^^^ + # + # def foo(**); end + # ^^ + def visit_keyword_rest_parameter_node(node) + if node.name_loc.nil? + bounds(node.location) + on_kwrest_param(nil) + else + bounds(node.name_loc) + name = on_ident(node.name.to_s) + + bounds(node.location) + on_kwrest_param(name) + end + end + + # -> {} + def visit_lambda_node(node) + bounds(node.operator_loc) + on_tlambda(node.operator) + + parameters = + if node.parameters.is_a?(BlockParametersNode) + # Ripper does not track block-locals within lambdas, so we skip + # directly to the parameters here. + params = + if node.parameters.parameters.nil? + bounds(node.location) + on_params(nil, nil, nil, nil, nil, nil, nil) + else + visit(node.parameters.parameters) + end + + if node.parameters.opening_loc.nil? + params + else + bounds(node.parameters.opening_loc) + on_paren(params) + end + else + bounds(node.location) + on_params(nil, nil, nil, nil, nil, nil, nil) + end + + braces = node.opening == "{" + if braces + bounds(node.opening_loc) + on_tlambeg(node.opening) + end + + body = + case node.body + when nil + bounds(node.location) + stmts = on_stmts_add(on_stmts_new, on_void_stmt) + + bounds(node.location) + braces ? stmts : on_bodystmt(stmts, nil, nil, nil) + when StatementsNode + stmts = node.body.body + stmts.unshift(nil) if void_stmt?(node.parameters&.location || node.opening_loc, node.body.location, false) + stmts = visit_statements_node_body(stmts) + + bounds(node.body.location) + braces ? stmts : on_bodystmt(stmts, nil, nil, nil) + when BeginNode + visit_body_node(node.opening_loc, node.body) + else + raise + end + + bounds(node.location) + on_lambda(parameters, body) + end + + # foo + # ^^^ + def visit_local_variable_read_node(node) + bounds(node.location) + on_var_ref(on_ident(node.slice)) + end + + # foo = 1 + # ^^^^^^^ + def visit_local_variable_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_ident(node.name_loc.slice)) + value = visit_write_value(node.value) + + bounds(node.location) + on_assign(target, value) + end + + # foo += bar + # ^^^^^^^^^^ + def visit_local_variable_operator_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_ident(node.name_loc.slice)) + + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # foo &&= bar + # ^^^^^^^^^^^ + def visit_local_variable_and_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_ident(node.name_loc.slice)) + + bounds(node.operator_loc) + operator = on_op("&&=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # foo ||= bar + # ^^^^^^^^^^^ + def visit_local_variable_or_write_node(node) + bounds(node.name_loc) + target = on_var_field(on_ident(node.name_loc.slice)) + + bounds(node.operator_loc) + operator = on_op("||=") + value = visit_write_value(node.value) + + bounds(node.location) + on_opassign(target, operator, value) + end + + # foo, = bar + # ^^^ + def visit_local_variable_target_node(node) + bounds(node.location) + on_var_field(on_ident(node.name.to_s)) + end + + # if /foo/ then end + # ^^^^^ + def visit_match_last_line_node(node) + bounds(node.opening_loc) + on_regexp_beg(node.opening) + + bounds(node.content_loc) + tstring_content = on_tstring_content(node.content) + + bounds(node.closing_loc) + closing = on_regexp_end(node.closing) + + on_regexp_literal(on_regexp_add(on_regexp_new, tstring_content), closing) + end + + # foo in bar + # ^^^^^^^^^^ + def visit_match_predicate_node(node) + value = visit(node.value) + pattern = on_in(visit_pattern_node(node.pattern), nil, nil) + + on_case(value, pattern) + end + + # foo => bar + # ^^^^^^^^^^ + def visit_match_required_node(node) + value = visit(node.value) + pattern = on_in(visit_pattern_node(node.pattern), nil, nil) + + on_case(value, pattern) + end + + # /(?<foo>foo)/ =~ bar + # ^^^^^^^^^^^^^^^^^^^^ + def visit_match_write_node(node) + visit(node.call) + end + + # A node that is missing from the syntax tree. This is only used in the + # case of a syntax error. + def visit_missing_node(node) + raise "Cannot visit missing nodes directly." + end + + # module Foo; end + # ^^^^^^^^^^^^^^^ + def visit_module_node(node) + constant_path = + if node.constant_path.is_a?(ConstantReadNode) + bounds(node.constant_path.location) + on_const_ref(on_const(node.constant_path.name.to_s)) + else + visit(node.constant_path) + end + + bodystmt = visit_body_node(node.constant_path.location, node.body, true) + + bounds(node.location) + on_module(constant_path, bodystmt) + end + + # (foo, bar), bar = qux + # ^^^^^^^^^^ + def visit_multi_target_node(node) + bounds(node.location) + targets = visit_multi_target_node_targets(node.lefts, node.rest, node.rights, true) + + if node.lparen_loc.nil? + targets + else + bounds(node.lparen_loc) + on_mlhs_paren(targets) + end + end + + # Visit the targets of a multi-target node. + private def visit_multi_target_node_targets(lefts, rest, rights, skippable) + if skippable && lefts.length == 1 && lefts.first.is_a?(MultiTargetNode) && rest.nil? && rights.empty? + return visit(lefts.first) + end + + mlhs = on_mlhs_new + + lefts.each do |left| + bounds(left.location) + mlhs = on_mlhs_add(mlhs, visit(left)) + end + + case rest + when nil + # do nothing + when ImplicitRestNode + # these do not get put into the generated tree + bounds(rest.location) + on_excessed_comma + else + bounds(rest.location) + mlhs = on_mlhs_add_star(mlhs, visit(rest)) + end + + if rights.any? + bounds(rights.first.location) + post = on_mlhs_new + + rights.each do |right| + bounds(right.location) + post = on_mlhs_add(post, visit(right)) + end + + mlhs = on_mlhs_add_post(mlhs, post) + end + + mlhs + end + + # foo, bar = baz + # ^^^^^^^^^^^^^^ + def visit_multi_write_node(node) + bounds(node.location) + targets = visit_multi_target_node_targets(node.lefts, node.rest, node.rights, true) + + unless node.lparen_loc.nil? + bounds(node.lparen_loc) + targets = on_mlhs_paren(targets) + end + + value = visit_write_value(node.value) + + bounds(node.location) + on_massign(targets, value) + end + + # next + # ^^^^ + # + # next foo + # ^^^^^^^^ + def visit_next_node(node) + if node.arguments.nil? + bounds(node.location) + on_next(on_args_new) + else + arguments = visit(node.arguments) + + bounds(node.location) + on_next(arguments) + end + end + + # nil + # ^^^ + def visit_nil_node(node) + bounds(node.location) + on_var_ref(on_kw("nil")) + end + + # def foo(**nil); end + # ^^^^^ + def visit_no_keywords_parameter_node(node) + bounds(node.location) + on_nokw_param(nil) + + :nil + end + + # -> { _1 + _2 } + # ^^^^^^^^^^^^^^ + def visit_numbered_parameters_node(node) + end + + # $1 + # ^^ + def visit_numbered_reference_read_node(node) + bounds(node.location) + on_backref(node.slice) + end + + # def foo(bar: baz); end + # ^^^^^^^^ + def visit_optional_keyword_parameter_node(node) + bounds(node.name_loc) + name = on_label("#{node.name}:") + value = visit(node.value) + + [name, value] + end + + # def foo(bar = 1); end + # ^^^^^^^ + def visit_optional_parameter_node(node) + bounds(node.name_loc) + name = visit_token(node.name.to_s) + value = visit(node.value) + + [name, value] + end + + # a or b + # ^^^^^^ + def visit_or_node(node) + left = visit(node.left) + right = visit(node.right) + + bounds(node.location) + on_binary(left, node.operator.to_sym, right) + end + + # def foo(bar, *baz); end + # ^^^^^^^^^ + def visit_parameters_node(node) + requireds = node.requireds.map { |required| required.is_a?(MultiTargetNode) ? visit_destructured_parameter_node(required) : visit(required) } if node.requireds.any? + optionals = visit_all(node.optionals) if node.optionals.any? + rest = visit(node.rest) + posts = node.posts.map { |post| post.is_a?(MultiTargetNode) ? visit_destructured_parameter_node(post) : visit(post) } if node.posts.any? + keywords = visit_all(node.keywords) if node.keywords.any? + keyword_rest = visit(node.keyword_rest) + block = visit(node.block) + + bounds(node.location) + on_params(requireds, optionals, rest, posts, keywords, keyword_rest, block) + end + + # Visit a destructured positional parameter node. + private def visit_destructured_parameter_node(node) + bounds(node.location) + targets = visit_multi_target_node_targets(node.lefts, node.rest, node.rights, false) + + bounds(node.lparen_loc) + on_mlhs_paren(targets) + end + + # () + # ^^ + # + # (1) + # ^^^ + def visit_parentheses_node(node) + body = + if node.body.nil? + on_stmts_add(on_stmts_new, on_void_stmt) + else + visit(node.body) + end + + bounds(node.location) + on_paren(body) + end + + # foo => ^(bar) + # ^^^^^^ + def visit_pinned_expression_node(node) + expression = visit(node.expression) + + bounds(node.location) + on_begin(expression) + end + + # foo = 1 and bar => ^foo + # ^^^^ + def visit_pinned_variable_node(node) + visit(node.variable) + end + + # END {} + # ^^^^^^ + def visit_post_execution_node(node) + statements = + if node.statements.nil? + bounds(node.location) + on_stmts_add(on_stmts_new, on_void_stmt) + else + visit(node.statements) + end + + bounds(node.location) + on_END(statements) + end + + # BEGIN {} + # ^^^^^^^^ + def visit_pre_execution_node(node) + statements = + if node.statements.nil? + bounds(node.location) + on_stmts_add(on_stmts_new, on_void_stmt) + else + visit(node.statements) + end + + bounds(node.location) + on_BEGIN(statements) + end + + # The top-level program node. + def visit_program_node(node) + body = node.statements.body + body << nil if body.empty? + statements = visit_statements_node_body(body) + + bounds(node.location) + on_program(statements) + end + + # 0..5 + # ^^^^ + def visit_range_node(node) + left = visit(node.left) + right = visit(node.right) + + bounds(node.location) + if node.exclude_end? + on_dot3(left, right) + else + on_dot2(left, right) + end + end + + # 1r + # ^^ + def visit_rational_node(node) + visit_number_node(node) { |text| on_rational(text) } + end + + # redo + # ^^^^ + def visit_redo_node(node) + bounds(node.location) + on_redo + end + + # /foo/ + # ^^^^^ + def visit_regular_expression_node(node) + bounds(node.opening_loc) + on_regexp_beg(node.opening) + + if node.content.empty? + bounds(node.closing_loc) + closing = on_regexp_end(node.closing) + + on_regexp_literal(on_regexp_new, closing) + else + bounds(node.content_loc) + tstring_content = on_tstring_content(node.content) + + bounds(node.closing_loc) + closing = on_regexp_end(node.closing) + + on_regexp_literal(on_regexp_add(on_regexp_new, tstring_content), closing) + end + end + + # def foo(bar:); end + # ^^^^ + def visit_required_keyword_parameter_node(node) + bounds(node.name_loc) + [on_label("#{node.name}:"), false] + end + + # def foo(bar); end + # ^^^ + def visit_required_parameter_node(node) + bounds(node.location) + on_ident(node.name.to_s) + end + + # foo rescue bar + # ^^^^^^^^^^^^^^ + def visit_rescue_modifier_node(node) + expression = visit_write_value(node.expression) + rescue_expression = visit(node.rescue_expression) + + bounds(node.location) + on_rescue_mod(expression, rescue_expression) + end + + # begin; rescue; end + # ^^^^^^^ + def visit_rescue_node(node) + exceptions = + case node.exceptions.length + when 0 + nil + when 1 + if (exception = node.exceptions.first).is_a?(SplatNode) + bounds(exception.location) + on_mrhs_add_star(on_mrhs_new, visit(exception)) + else + [visit(node.exceptions.first)] + end + else + bounds(node.location) + length = node.exceptions.length + + node.exceptions.each_with_index.inject(on_args_new) do |mrhs, (exception, index)| + arg = visit(exception) + + bounds(exception.location) + mrhs = on_mrhs_new_from_args(mrhs) if index == length - 1 + + if exception.is_a?(SplatNode) + if index == length - 1 + on_mrhs_add_star(mrhs, arg) + else + on_args_add_star(mrhs, arg) + end + else + if index == length - 1 + on_mrhs_add(mrhs, arg) + else + on_args_add(mrhs, arg) + end + end + end + end + + reference = visit(node.reference) + statements = + if node.statements.nil? + bounds(node.location) + on_stmts_add(on_stmts_new, on_void_stmt) + else + visit(node.statements) + end + + consequent = visit(node.consequent) + + bounds(node.location) + on_rescue(exceptions, reference, statements, consequent) + end + + # def foo(*bar); end + # ^^^^ + # + # def foo(*); end + # ^ + def visit_rest_parameter_node(node) + if node.name_loc.nil? + bounds(node.location) + on_rest_param(nil) + else + bounds(node.name_loc) + on_rest_param(visit_token(node.name.to_s)) + end + end + + # retry + # ^^^^^ + def visit_retry_node(node) + bounds(node.location) + on_retry + end + + # return + # ^^^^^^ + # + # return 1 + # ^^^^^^^^ + def visit_return_node(node) + if node.arguments.nil? + bounds(node.location) + on_return0 + else + arguments = visit(node.arguments) + + bounds(node.location) + on_return(arguments) + end + end + + # self + # ^^^^ + def visit_self_node(node) + bounds(node.location) + on_var_ref(on_kw("self")) + end + + # A shareable constant. + def visit_shareable_constant_node(node) + visit(node.write) + end + + # class << self; end + # ^^^^^^^^^^^^^^^^^^ + def visit_singleton_class_node(node) + expression = visit(node.expression) + bodystmt = visit_body_node(node.body&.location || node.end_keyword_loc, node.body) + + bounds(node.location) + on_sclass(expression, bodystmt) + end + + # __ENCODING__ + # ^^^^^^^^^^^^ + def visit_source_encoding_node(node) + bounds(node.location) + on_var_ref(on_kw("__ENCODING__")) + end + + # __FILE__ + # ^^^^^^^^ + def visit_source_file_node(node) + bounds(node.location) + on_var_ref(on_kw("__FILE__")) + end + + # __LINE__ + # ^^^^^^^^ + def visit_source_line_node(node) + bounds(node.location) + on_var_ref(on_kw("__LINE__")) + end + + # foo(*bar) + # ^^^^ + # + # def foo((bar, *baz)); end + # ^^^^ + # + # def foo(*); bar(*); end + # ^ + def visit_splat_node(node) + visit(node.expression) + end + + # A list of statements. + def visit_statements_node(node) + bounds(node.location) + visit_statements_node_body(node.body) + end + + # Visit the list of statements of a statements node. We support nil + # statements in the list. This would normally not be allowed by the + # structure of the prism parse tree, but we manually add them here so that + # we can mirror Ripper's void stmt. + private def visit_statements_node_body(body) + body.inject(on_stmts_new) do |stmts, stmt| + on_stmts_add(stmts, stmt.nil? ? on_void_stmt : visit(stmt)) + end + end + + # "foo" + # ^^^^^ + def visit_string_node(node) + if (content = node.content).empty? + bounds(node.location) + on_string_literal(on_string_content) + elsif (opening = node.opening) == "?" + bounds(node.location) + on_CHAR("?#{node.content}") + elsif opening.start_with?("<<~") + heredoc = visit_heredoc_string_node(node.to_interpolated) + + bounds(node.location) + on_string_literal(heredoc) + else + bounds(node.content_loc) + tstring_content = on_tstring_content(content) + + bounds(node.location) + on_string_literal(on_string_add(on_string_content, tstring_content)) + end + end + + # Ripper gives back the escaped string content but strips out the common + # leading whitespace. Prism gives back the unescaped string content and + # a location for the escaped string content. Unfortunately these don't + # work well together, so here we need to re-derive the common leading + # whitespace. + private def visit_heredoc_node_whitespace(parts) + common_whitespace = nil + dedent_next = true + + parts.each do |part| + if part.is_a?(StringNode) + if dedent_next && !(content = part.content).chomp.empty? + common_whitespace = [ + common_whitespace || Float::INFINITY, + content[/\A\s*/].each_char.inject(0) do |part_whitespace, char| + char == "\t" ? ((part_whitespace / 8 + 1) * 8) : (part_whitespace + 1) + end + ].min + end + + dedent_next = true + else + dedent_next = false + end + end + + common_whitespace || 0 + end + + # Visit a string that is expressed using a <<~ heredoc. + private def visit_heredoc_node(parts, base) + common_whitespace = visit_heredoc_node_whitespace(parts) + + if common_whitespace == 0 + bounds(parts.first.location) + + string = [] + result = base + + parts.each do |part| + if part.is_a?(StringNode) + if string.empty? + string = [part] + else + string << part + end + else + unless string.empty? + bounds(string[0].location) + result = yield result, on_tstring_content(string.map(&:content).join) + string = [] + end + + result = yield result, visit(part) + end + end + + unless string.empty? + bounds(string[0].location) + result = yield result, on_tstring_content(string.map(&:content).join) + end + + result + else + bounds(parts.first.location) + result = + parts.inject(base) do |string_content, part| + yield string_content, visit_string_content(part) + end + + bounds(parts.first.location) + on_heredoc_dedent(result, common_whitespace) + end + end + + # Visit a heredoc node that is representing a string. + private def visit_heredoc_string_node(node) + bounds(node.opening_loc) + on_heredoc_beg(node.opening) + + bounds(node.location) + result = + visit_heredoc_node(node.parts, on_string_content) do |parts, part| + on_string_add(parts, part) + end + + bounds(node.closing_loc) + on_heredoc_end(node.closing) + + result + end + + # Visit a heredoc node that is representing an xstring. + private def visit_heredoc_x_string_node(node) + bounds(node.opening_loc) + on_heredoc_beg(node.opening) + + bounds(node.location) + result = + visit_heredoc_node(node.parts, on_xstring_new) do |parts, part| + on_xstring_add(parts, part) + end + + bounds(node.closing_loc) + on_heredoc_end(node.closing) + + result + end + + # super(foo) + # ^^^^^^^^^^ + def visit_super_node(node) + arguments, block = visit_call_node_arguments(node.arguments, node.block, trailing_comma?(node.arguments&.location || node.location, node.rparen_loc || node.location)) + + if !node.lparen_loc.nil? + bounds(node.lparen_loc) + arguments = on_arg_paren(arguments) + end + + bounds(node.location) + call = on_super(arguments) + + if block.nil? + call + else + bounds(node.block.location) + on_method_add_block(call, block) + end + end + + # :foo + # ^^^^ + def visit_symbol_node(node) + if (opening = node.opening)&.match?(/^%s|['"]:?$/) + bounds(node.value_loc) + content = on_string_content + + if !(value = node.value).empty? + content = on_string_add(content, on_tstring_content(value)) + end + + on_dyna_symbol(content) + elsif (closing = node.closing) == ":" + bounds(node.location) + on_label("#{node.value}:") + elsif opening.nil? && node.closing_loc.nil? + bounds(node.value_loc) + on_symbol_literal(visit_token(node.value)) + else + bounds(node.value_loc) + on_symbol_literal(on_symbol(visit_token(node.value))) + end + end + + # true + # ^^^^ + def visit_true_node(node) + bounds(node.location) + on_var_ref(on_kw("true")) + end + + # undef foo + # ^^^^^^^^^ + def visit_undef_node(node) + names = visit_all(node.names) + + bounds(node.location) + on_undef(names) + end + + # unless foo; bar end + # ^^^^^^^^^^^^^^^^^^^ + # + # bar unless foo + # ^^^^^^^^^^^^^^ + def visit_unless_node(node) + if node.statements.nil? || (node.predicate.location.start_offset < node.statements.location.start_offset) + predicate = visit(node.predicate) + statements = + if node.statements.nil? + bounds(node.location) + on_stmts_add(on_stmts_new, on_void_stmt) + else + visit(node.statements) + end + consequent = visit(node.consequent) + + bounds(node.location) + on_unless(predicate, statements, consequent) + else + statements = visit(node.statements.body.first) + predicate = visit(node.predicate) + + bounds(node.location) + on_unless_mod(predicate, statements) + end + end + + # until foo; bar end + # ^^^^^^^^^^^^^^^^^ + # + # bar until foo + # ^^^^^^^^^^^^^ + def visit_until_node(node) + if node.statements.nil? || (node.predicate.location.start_offset < node.statements.location.start_offset) + predicate = visit(node.predicate) + statements = + if node.statements.nil? + bounds(node.location) + on_stmts_add(on_stmts_new, on_void_stmt) + else + visit(node.statements) + end + + bounds(node.location) + on_until(predicate, statements) + else + statements = visit(node.statements.body.first) + predicate = visit(node.predicate) + + bounds(node.location) + on_until_mod(predicate, statements) + end + end + + # case foo; when bar; end + # ^^^^^^^^^^^^^ + def visit_when_node(node) + # This is a special case where we're not going to call on_when directly + # because we don't have access to the consequent. Instead, we'll return + # the component parts and let the parent node handle it. + conditions = visit_arguments(node.conditions) + statements = + if node.statements.nil? + bounds(node.location) + on_stmts_add(on_stmts_new, on_void_stmt) + else + visit(node.statements) + end + + [conditions, statements] + end + + # while foo; bar end + # ^^^^^^^^^^^^^^^^^^ + # + # bar while foo + # ^^^^^^^^^^^^^ + def visit_while_node(node) + if node.statements.nil? || (node.predicate.location.start_offset < node.statements.location.start_offset) + predicate = visit(node.predicate) + statements = + if node.statements.nil? + bounds(node.location) + on_stmts_add(on_stmts_new, on_void_stmt) + else + visit(node.statements) + end + + bounds(node.location) + on_while(predicate, statements) + else + statements = visit(node.statements.body.first) + predicate = visit(node.predicate) + + bounds(node.location) + on_while_mod(predicate, statements) + end + end + + # `foo` + # ^^^^^ + def visit_x_string_node(node) + if node.unescaped.empty? + bounds(node.location) + on_xstring_literal(on_xstring_new) + elsif node.opening.start_with?("<<~") + heredoc = visit_heredoc_x_string_node(node.to_interpolated) + + bounds(node.location) + on_xstring_literal(heredoc) + else + bounds(node.content_loc) + content = on_tstring_content(node.content) + + bounds(node.location) + on_xstring_literal(on_xstring_add(on_xstring_new, content)) + end + end + + # yield + # ^^^^^ + # + # yield 1 + # ^^^^^^^ + def visit_yield_node(node) + if node.arguments.nil? && node.lparen_loc.nil? + bounds(node.location) + on_yield0 + else + arguments = + if node.arguments.nil? + bounds(node.location) + on_args_new + else + visit(node.arguments) + end + + unless node.lparen_loc.nil? + bounds(node.lparen_loc) + arguments = on_paren(arguments) + end + + bounds(node.location) + on_yield(arguments) + end + end + + private + + # Lazily initialize the parse result. + def result + @result ||= + begin + scopes = RUBY_VERSION >= "3.3.0" ? [] : [[]] + Prism.parse(source, scopes: scopes) + end + end + + ########################################################################## + # Helpers + ########################################################################## + + # Returns true if there is a comma between the two locations. + def trailing_comma?(left, right) + source.byteslice(left.end_offset...right.start_offset).include?(",") + end + + # Returns true if there is a semicolon between the two locations. + def void_stmt?(left, right, allow_newline) + pattern = allow_newline ? /[;\n]/ : /;/ + source.byteslice(left.end_offset...right.start_offset).match?(pattern) + end + + # Visit the string content of a particular node. This method is used to + # split into the various token types. + def visit_token(token, allow_keywords = true) + case token + when "." + on_period(token) + when "`" + on_backtick(token) + when *(allow_keywords ? KEYWORDS : []) + on_kw(token) + when /^_/ + on_ident(token) + when /^[[:upper:]]\w*$/ + on_const(token) + when /^@@/ + on_cvar(token) + when /^@/ + on_ivar(token) + when /^\$/ + on_gvar(token) + when /^[[:punct:]]/ + on_op(token) + else + on_ident(token) + end + end + + # Visit a node that represents a number. We need to explicitly handle the + # unary - operator. + def visit_number_node(node) + slice = node.slice + location = node.location + + if slice[0] == "-" + bounds(location.copy(start_offset: location.start_offset + 1)) + value = yield slice[1..-1] + + bounds(node.location) + on_unary(:-@, value) + else + bounds(location) + yield slice + end + end + + # Visit a node that represents a write value. This is used to handle the + # special case of an implicit array that is generated without brackets. + def visit_write_value(node) + if node.is_a?(ArrayNode) && node.opening_loc.nil? + elements = node.elements + length = elements.length + + bounds(elements.first.location) + elements.each_with_index.inject((elements.first.is_a?(SplatNode) && length == 1) ? on_mrhs_new : on_args_new) do |args, (element, index)| + arg = visit(element) + bounds(element.location) + + if index == length - 1 + if element.is_a?(SplatNode) + mrhs = index == 0 ? args : on_mrhs_new_from_args(args) + on_mrhs_add_star(mrhs, arg) + else + on_mrhs_add(on_mrhs_new_from_args(args), arg) + end + else + case element + when BlockArgumentNode + on_args_add_block(args, arg) + when SplatNode + on_args_add_star(args, arg) + else + on_args_add(args, arg) + end + end + end + else + visit(node) + end + end + + # This method is responsible for updating lineno and column information + # to reflect the current node. + # + # This method could be drastically improved with some caching on the start + # of every line, but for now it's good enough. + def bounds(location) + @lineno = location.start_line + @column = location.start_column + end + + ########################################################################## + # Ripper interface + ########################################################################## + + # :stopdoc: + def _dispatch_0; end + def _dispatch_1(_); end + def _dispatch_2(_, _); end + def _dispatch_3(_, _, _); end + def _dispatch_4(_, _, _, _); end + def _dispatch_5(_, _, _, _, _); end + def _dispatch_7(_, _, _, _, _, _, _); end + # :startdoc: + + # + # Parser Events + # + + PARSER_EVENT_TABLE.each do |id, arity| + alias_method "on_#{id}", "_dispatch_#{arity}" + end + + # This method is called when weak warning is produced by the parser. + # +fmt+ and +args+ is printf style. + def warn(fmt, *args) + end + + # This method is called when strong warning is produced by the parser. + # +fmt+ and +args+ is printf style. + def warning(fmt, *args) + end + + # This method is called when the parser found syntax error. + def compile_error(msg) + end + + # + # Scanner Events + # + + SCANNER_EVENTS.each do |id| + alias_method "on_#{id}", :_dispatch_1 + end + + # This method is provided by the Ripper C extension. It is called when a + # string needs to be dedented because of a tilde heredoc. It is expected + # that it will modify the string in place and return the number of bytes + # that were removed. + def dedent_string(string, width) + whitespace = 0 + cursor = 0 + + while cursor < string.length && string[cursor].match?(/\s/) && whitespace < width + if string[cursor] == "\t" + whitespace = ((whitespace / 8 + 1) * 8) + break if whitespace > width + else + whitespace += 1 + end + + cursor += 1 + end + + string.replace(string[cursor..]) + cursor + end + end + end +end diff --git a/lib/prism/translation/ripper/sexp.rb b/lib/prism/translation/ripper/sexp.rb new file mode 100644 index 0000000000..dc26a639a3 --- /dev/null +++ b/lib/prism/translation/ripper/sexp.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +require_relative "../ripper" + +module Prism + module Translation + class Ripper + # This class mirrors the ::Ripper::SexpBuilder subclass of ::Ripper that + # returns the arrays of [type, *children]. + class SexpBuilder < Ripper + # :stopdoc: + + attr_reader :error + + private + + def dedent_element(e, width) + if (n = dedent_string(e[1], width)) > 0 + e[2][1] += n + end + e + end + + def on_heredoc_dedent(val, width) + sub = proc do |cont| + cont.map! do |e| + if Array === e + case e[0] + when :@tstring_content + e = dedent_element(e, width) + when /_add\z/ + e[1] = sub[e[1]] + end + elsif String === e + dedent_string(e, width) + end + e + end + end + sub[val] + val + end + + events = private_instance_methods(false).grep(/\Aon_/) {$'.to_sym} + (PARSER_EVENTS - events).each do |event| + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{event}(*args) + args.unshift :#{event} + end + End + end + + SCANNER_EVENTS.each do |event| + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{event}(tok) + [:@#{event}, tok, [lineno(), column()]] + end + End + end + + def on_error(mesg) + @error = mesg + end + remove_method :on_parse_error + alias on_parse_error on_error + alias compile_error on_error + + # :startdoc: + end + + # This class mirrors the ::Ripper::SexpBuilderPP subclass of ::Ripper that + # returns the same values as ::Ripper::SexpBuilder except with a couple of + # niceties that flatten linked lists into arrays. + class SexpBuilderPP < SexpBuilder + # :stopdoc: + + private + + def on_heredoc_dedent(val, width) + val.map! do |e| + next e if Symbol === e and /_content\z/ =~ e + if Array === e and e[0] == :@tstring_content + e = dedent_element(e, width) + elsif String === e + dedent_string(e, width) + end + e + end + val + end + + def _dispatch_event_new + [] + end + + def _dispatch_event_push(list, item) + list.push item + list + end + + def on_mlhs_paren(list) + [:mlhs, *list] + end + + def on_mlhs_add_star(list, star) + list.push([:rest_param, star]) + end + + def on_mlhs_add_post(list, post) + list.concat(post) + end + + PARSER_EVENT_TABLE.each do |event, arity| + if /_new\z/ =~ event and arity == 0 + alias_method "on_#{event}", :_dispatch_event_new + elsif /_add\z/ =~ event + alias_method "on_#{event}", :_dispatch_event_push + end + end + + # :startdoc: + end + end + end +end diff --git a/lib/prism/translation/ripper/shim.rb b/lib/prism/translation/ripper/shim.rb new file mode 100644 index 0000000000..10e21cd16a --- /dev/null +++ b/lib/prism/translation/ripper/shim.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +# This writes the prism ripper translation into the Ripper constant so that +# users can transparently use Ripper without any changes. +Ripper = Prism::Translation::Ripper diff --git a/lib/prism/translation/ruby_parser.rb b/lib/prism/translation/ruby_parser.rb new file mode 100644 index 0000000000..ec458a3b63 --- /dev/null +++ b/lib/prism/translation/ruby_parser.rb @@ -0,0 +1,1594 @@ +# frozen_string_literal: true + +begin + require "ruby_parser" +rescue LoadError + warn(%q{Error: Unable to load ruby_parser. Add `gem "ruby_parser"` to your Gemfile.}) + exit(1) +end + +module Prism + module Translation + # This module is the entry-point for converting a prism syntax tree into the + # seattlerb/ruby_parser gem's syntax tree. + class RubyParser + # A prism visitor that builds Sexp objects. + class Compiler < ::Prism::Compiler + # This is the name of the file that we are compiling. We set it on every + # Sexp object that is generated, and also use it to compile __FILE__ + # nodes. + attr_reader :file + + # Class variables will change their type based on if they are inside of + # a method definition or not, so we need to track that state. + attr_reader :in_def + + # Some nodes will change their representation if they are inside of a + # pattern, so we need to track that state. + attr_reader :in_pattern + + # Initialize a new compiler with the given file name. + def initialize(file, in_def: false, in_pattern: false) + @file = file + @in_def = in_def + @in_pattern = in_pattern + end + + # alias foo bar + # ^^^^^^^^^^^^^ + def visit_alias_method_node(node) + s(node, :alias, visit(node.new_name), visit(node.old_name)) + end + + # alias $foo $bar + # ^^^^^^^^^^^^^^^ + def visit_alias_global_variable_node(node) + s(node, :valias, node.new_name.name, node.old_name.name) + end + + # foo => bar | baz + # ^^^^^^^^^ + def visit_alternation_pattern_node(node) + s(node, :or, visit(node.left), visit(node.right)) + end + + # a and b + # ^^^^^^^ + def visit_and_node(node) + s(node, :and, visit(node.left), visit(node.right)) + end + + # [] + # ^^ + def visit_array_node(node) + if in_pattern + s(node, :array_pat, nil).concat(visit_all(node.elements)) + else + s(node, :array).concat(visit_all(node.elements)) + end + end + + # foo => [bar] + # ^^^^^ + def visit_array_pattern_node(node) + if node.constant.nil? && node.requireds.empty? && node.rest.nil? && node.posts.empty? + s(node, :array_pat) + else + result = s(node, :array_pat, visit_pattern_constant(node.constant)).concat(visit_all(node.requireds)) + + case node.rest + when SplatNode + result << :"*#{node.rest.expression&.name}" + when ImplicitRestNode + result << :* + + # This doesn't make any sense at all, but since we're trying to + # replicate the behavior directly, we'll copy it. + result.line(666) + end + + result.concat(visit_all(node.posts)) + end + end + + # foo(bar) + # ^^^ + def visit_arguments_node(node) + raise "Cannot visit arguments directly" + end + + # { a: 1 } + # ^^^^ + def visit_assoc_node(node) + [visit(node.key), visit(node.value)] + end + + # def foo(**); bar(**); end + # ^^ + # + # { **foo } + # ^^^^^ + def visit_assoc_splat_node(node) + if node.value.nil? + [s(node, :kwsplat)] + else + [s(node, :kwsplat, visit(node.value))] + end + end + + # $+ + # ^^ + def visit_back_reference_read_node(node) + s(node, :back_ref, node.name.name.delete_prefix("$").to_sym) + end + + # begin end + # ^^^^^^^^^ + def visit_begin_node(node) + result = node.statements.nil? ? s(node, :nil) : visit(node.statements) + + if !node.rescue_clause.nil? + if !node.statements.nil? + result = s(node.statements, :rescue, result, visit(node.rescue_clause)) + else + result = s(node.rescue_clause, :rescue, visit(node.rescue_clause)) + end + + current = node.rescue_clause + until (current = current.consequent).nil? + result << visit(current) + end + end + + if !node.else_clause&.statements.nil? + result << visit(node.else_clause) + end + + if !node.ensure_clause.nil? + if !node.statements.nil? || !node.rescue_clause.nil? || !node.else_clause.nil? + result = s(node.statements || node.rescue_clause || node.else_clause || node.ensure_clause, :ensure, result, visit(node.ensure_clause)) + else + result = s(node.ensure_clause, :ensure, visit(node.ensure_clause)) + end + end + + result + end + + # foo(&bar) + # ^^^^ + def visit_block_argument_node(node) + s(node, :block_pass).tap do |result| + result << visit(node.expression) unless node.expression.nil? + end + end + + # foo { |; bar| } + # ^^^ + def visit_block_local_variable_node(node) + node.name + end + + # A block on a keyword or method call. + def visit_block_node(node) + s(node, :block_pass, visit(node.expression)) + end + + # def foo(&bar); end + # ^^^^ + def visit_block_parameter_node(node) + :"&#{node.name}" + end + + # A block's parameters. + def visit_block_parameters_node(node) + # If this block parameters has no parameters and is using pipes, then + # it inherits its location from its shadow locals, even if they're not + # on the same lines as the pipes. + shadow_loc = true + + result = + if node.parameters.nil? + s(node, :args) + else + shadow_loc = false + visit(node.parameters) + end + + if node.opening == "(" + result.line = node.opening_loc.start_line + result.line_max = node.closing_loc.end_line + shadow_loc = false + end + + if node.locals.any? + shadow = s(node, :shadow).concat(visit_all(node.locals)) + shadow.line = node.locals.first.location.start_line + shadow.line_max = node.locals.last.location.end_line + result << shadow + + if shadow_loc + result.line = shadow.line + result.line_max = shadow.line_max + end + end + + result + end + + # break + # ^^^^^ + # + # break foo + # ^^^^^^^^^ + def visit_break_node(node) + if node.arguments.nil? + s(node, :break) + elsif node.arguments.arguments.length == 1 + s(node, :break, visit(node.arguments.arguments.first)) + else + s(node, :break, s(node.arguments, :array).concat(visit_all(node.arguments.arguments))) + end + end + + # foo + # ^^^ + # + # foo.bar + # ^^^^^^^ + # + # foo.bar() {} + # ^^^^^^^^^^^^ + def visit_call_node(node) + case node.name + when :!~ + return s(node, :not, visit(node.copy(name: :"=~"))) + when :=~ + if node.arguments&.arguments&.length == 1 && node.block.nil? + case node.receiver + when StringNode + return s(node, :match3, visit(node.arguments.arguments.first), visit(node.receiver)) + when RegularExpressionNode, InterpolatedRegularExpressionNode + return s(node, :match2, visit(node.receiver), visit(node.arguments.arguments.first)) + end + end + end + + type = node.attribute_write? ? :attrasgn : :call + type = :"safe_#{type}" if node.safe_navigation? + + arguments = node.arguments&.arguments || [] + write_value = arguments.pop if type == :attrasgn + block = node.block + + if block.is_a?(BlockArgumentNode) + arguments << block + block = nil + end + + result = s(node, type, visit(node.receiver), node.name).concat(visit_all(arguments)) + result << visit_write_value(write_value) unless write_value.nil? + + visit_block(node, result, block) + end + + # foo.bar += baz + # ^^^^^^^^^^^^^^^ + def visit_call_operator_write_node(node) + if op_asgn?(node) + s(node, op_asgn_type(node, :op_asgn), visit(node.receiver), visit_write_value(node.value), node.read_name, node.binary_operator) + else + s(node, op_asgn_type(node, :op_asgn2), visit(node.receiver), node.write_name, node.binary_operator, visit_write_value(node.value)) + end + end + + # foo.bar &&= baz + # ^^^^^^^^^^^^^^^ + def visit_call_and_write_node(node) + if op_asgn?(node) + s(node, op_asgn_type(node, :op_asgn), visit(node.receiver), visit_write_value(node.value), node.read_name, :"&&") + else + s(node, op_asgn_type(node, :op_asgn2), visit(node.receiver), node.write_name, :"&&", visit_write_value(node.value)) + end + end + + # foo.bar ||= baz + # ^^^^^^^^^^^^^^^ + def visit_call_or_write_node(node) + if op_asgn?(node) + s(node, op_asgn_type(node, :op_asgn), visit(node.receiver), visit_write_value(node.value), node.read_name, :"||") + else + s(node, op_asgn_type(node, :op_asgn2), visit(node.receiver), node.write_name, :"||", visit_write_value(node.value)) + end + end + + # Call nodes with operators following them will either be op_asgn or + # op_asgn2 nodes. That is determined by their call operator and their + # right-hand side. + private def op_asgn?(node) + node.call_operator == "::" || (node.value.is_a?(CallNode) && node.value.opening_loc.nil? && !node.value.arguments.nil?) + end + + # Call nodes with operators following them can use &. as an operator, + # which changes their type by prefixing "safe_". + private def op_asgn_type(node, type) + node.safe_navigation? ? :"safe_#{type}" : type + end + + # foo.bar, = 1 + # ^^^^^^^ + def visit_call_target_node(node) + s(node, :attrasgn, visit(node.receiver), node.name) + end + + # foo => bar => baz + # ^^^^^^^^^^ + def visit_capture_pattern_node(node) + visit(node.target) << visit(node.value) + end + + # case foo; when bar; end + # ^^^^^^^^^^^^^^^^^^^^^^^ + def visit_case_node(node) + s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.consequent) + end + + # case foo; in bar; end + # ^^^^^^^^^^^^^^^^^^^^^ + def visit_case_match_node(node) + s(node, :case, visit(node.predicate)).concat(visit_all(node.conditions)) << visit(node.consequent) + end + + # class Foo; end + # ^^^^^^^^^^^^^^ + def visit_class_node(node) + name = + if node.constant_path.is_a?(ConstantReadNode) + node.name + else + visit(node.constant_path) + end + + if node.body.nil? + s(node, :class, name, visit(node.superclass)) + elsif node.body.is_a?(StatementsNode) + compiler = copy_compiler(in_def: false) + s(node, :class, name, visit(node.superclass)).concat(node.body.body.map { |child| child.accept(compiler) }) + else + s(node, :class, name, visit(node.superclass), node.body.accept(copy_compiler(in_def: false))) + end + end + + # @@foo + # ^^^^^ + def visit_class_variable_read_node(node) + s(node, :cvar, node.name) + end + + # @@foo = 1 + # ^^^^^^^^^ + # + # @@foo, @@bar = 1 + # ^^^^^ ^^^^^ + def visit_class_variable_write_node(node) + s(node, class_variable_write_type, node.name, visit_write_value(node.value)) + end + + # @@foo += bar + # ^^^^^^^^^^^^ + def visit_class_variable_operator_write_node(node) + s(node, class_variable_write_type, node.name, s(node, :call, s(node, :cvar, node.name), node.binary_operator, visit_write_value(node.value))) + end + + # @@foo &&= bar + # ^^^^^^^^^^^^^ + def visit_class_variable_and_write_node(node) + s(node, :op_asgn_and, s(node, :cvar, node.name), s(node, class_variable_write_type, node.name, visit_write_value(node.value))) + end + + # @@foo ||= bar + # ^^^^^^^^^^^^^ + def visit_class_variable_or_write_node(node) + s(node, :op_asgn_or, s(node, :cvar, node.name), s(node, class_variable_write_type, node.name, visit_write_value(node.value))) + end + + # @@foo, = bar + # ^^^^^ + def visit_class_variable_target_node(node) + s(node, class_variable_write_type, node.name) + end + + # If a class variable is written within a method definition, it has a + # different type than everywhere else. + private def class_variable_write_type + in_def ? :cvasgn : :cvdecl + end + + # Foo + # ^^^ + def visit_constant_read_node(node) + s(node, :const, node.name) + end + + # Foo = 1 + # ^^^^^^^ + # + # Foo, Bar = 1 + # ^^^ ^^^ + def visit_constant_write_node(node) + s(node, :cdecl, node.name, visit_write_value(node.value)) + end + + # Foo += bar + # ^^^^^^^^^^^ + def visit_constant_operator_write_node(node) + s(node, :cdecl, node.name, s(node, :call, s(node, :const, node.name), node.binary_operator, visit_write_value(node.value))) + end + + # Foo &&= bar + # ^^^^^^^^^^^^ + def visit_constant_and_write_node(node) + s(node, :op_asgn_and, s(node, :const, node.name), s(node, :cdecl, node.name, visit(node.value))) + end + + # Foo ||= bar + # ^^^^^^^^^^^^ + def visit_constant_or_write_node(node) + s(node, :op_asgn_or, s(node, :const, node.name), s(node, :cdecl, node.name, visit(node.value))) + end + + # Foo, = bar + # ^^^ + def visit_constant_target_node(node) + s(node, :cdecl, node.name) + end + + # Foo::Bar + # ^^^^^^^^ + def visit_constant_path_node(node) + if node.parent.nil? + s(node, :colon3, node.name) + else + s(node, :colon2, visit(node.parent), node.name) + end + end + + # Foo::Bar = 1 + # ^^^^^^^^^^^^ + # + # Foo::Foo, Bar::Bar = 1 + # ^^^^^^^^ ^^^^^^^^ + def visit_constant_path_write_node(node) + s(node, :cdecl, visit(node.target), visit_write_value(node.value)) + end + + # Foo::Bar += baz + # ^^^^^^^^^^^^^^^ + def visit_constant_path_operator_write_node(node) + s(node, :op_asgn, visit(node.target), node.binary_operator, visit_write_value(node.value)) + end + + # Foo::Bar &&= baz + # ^^^^^^^^^^^^^^^^ + def visit_constant_path_and_write_node(node) + s(node, :op_asgn_and, visit(node.target), visit_write_value(node.value)) + end + + # Foo::Bar ||= baz + # ^^^^^^^^^^^^^^^^ + def visit_constant_path_or_write_node(node) + s(node, :op_asgn_or, visit(node.target), visit_write_value(node.value)) + end + + # Foo::Bar, = baz + # ^^^^^^^^ + def visit_constant_path_target_node(node) + inner = + if node.parent.nil? + s(node, :colon3, node.child.name) + else + s(node, :colon2, visit(node.parent), node.child.name) + end + + s(node, :const, inner) + end + + # def foo; end + # ^^^^^^^^^^^^ + # + # def self.foo; end + # ^^^^^^^^^^^^^^^^^ + def visit_def_node(node) + name = node.name_loc.slice.to_sym + result = + if node.receiver.nil? + s(node, :defn, name) + else + s(node, :defs, visit(node.receiver), name) + end + + result.line(node.name_loc.start_line) + if node.parameters.nil? + result << s(node, :args).line(node.name_loc.start_line) + else + result << visit(node.parameters) + end + + if node.body.nil? + result << s(node, :nil) + elsif node.body.is_a?(StatementsNode) + compiler = copy_compiler(in_def: true) + result.concat(node.body.body.map { |child| child.accept(compiler) }) + else + result << node.body.accept(copy_compiler(in_def: true)) + end + end + + # defined? a + # ^^^^^^^^^^ + # + # defined?(a) + # ^^^^^^^^^^^ + def visit_defined_node(node) + s(node, :defined, visit(node.value)) + end + + # if foo then bar else baz end + # ^^^^^^^^^^^^ + def visit_else_node(node) + visit(node.statements) + end + + # "foo #{bar}" + # ^^^^^^ + def visit_embedded_statements_node(node) + result = s(node, :evstr) + result << visit(node.statements) unless node.statements.nil? + result + end + + # "foo #@bar" + # ^^^^^ + def visit_embedded_variable_node(node) + s(node, :evstr, visit(node.variable)) + end + + # begin; foo; ensure; bar; end + # ^^^^^^^^^^^^ + def visit_ensure_node(node) + node.statements.nil? ? s(node, :nil) : visit(node.statements) + end + + # false + # ^^^^^ + def visit_false_node(node) + s(node, :false) + end + + # foo => [*, bar, *] + # ^^^^^^^^^^^ + def visit_find_pattern_node(node) + s(node, :find_pat, visit_pattern_constant(node.constant), :"*#{node.left.expression&.name}", *visit_all(node.requireds), :"*#{node.right.expression&.name}") + end + + # if foo .. bar; end + # ^^^^^^^^^^ + def visit_flip_flop_node(node) + if node.left.is_a?(IntegerNode) && node.right.is_a?(IntegerNode) + s(node, :lit, Range.new(node.left.value, node.right.value, node.exclude_end?)) + else + s(node, node.exclude_end? ? :flip3 : :flip2, visit(node.left), visit(node.right)) + end + end + + # 1.0 + # ^^^ + def visit_float_node(node) + s(node, :lit, node.value) + end + + # for foo in bar do end + # ^^^^^^^^^^^^^^^^^^^^^ + def visit_for_node(node) + s(node, :for, visit(node.collection), visit(node.index), visit(node.statements)) + end + + # def foo(...); bar(...); end + # ^^^ + def visit_forwarding_arguments_node(node) + s(node, :forward_args) + end + + # def foo(...); end + # ^^^ + def visit_forwarding_parameter_node(node) + s(node, :forward_args) + end + + # super + # ^^^^^ + # + # super {} + # ^^^^^^^^ + def visit_forwarding_super_node(node) + visit_block(node, s(node, :zsuper), node.block) + end + + # $foo + # ^^^^ + def visit_global_variable_read_node(node) + s(node, :gvar, node.name) + end + + # $foo = 1 + # ^^^^^^^^ + # + # $foo, $bar = 1 + # ^^^^ ^^^^ + def visit_global_variable_write_node(node) + s(node, :gasgn, node.name, visit_write_value(node.value)) + end + + # $foo += bar + # ^^^^^^^^^^^ + def visit_global_variable_operator_write_node(node) + s(node, :gasgn, node.name, s(node, :call, s(node, :gvar, node.name), node.binary_operator, visit(node.value))) + end + + # $foo &&= bar + # ^^^^^^^^^^^^ + def visit_global_variable_and_write_node(node) + s(node, :op_asgn_and, s(node, :gvar, node.name), s(node, :gasgn, node.name, visit_write_value(node.value))) + end + + # $foo ||= bar + # ^^^^^^^^^^^^ + def visit_global_variable_or_write_node(node) + s(node, :op_asgn_or, s(node, :gvar, node.name), s(node, :gasgn, node.name, visit_write_value(node.value))) + end + + # $foo, = bar + # ^^^^ + def visit_global_variable_target_node(node) + s(node, :gasgn, node.name) + end + + # {} + # ^^ + def visit_hash_node(node) + s(node, :hash).concat(node.elements.flat_map { |element| visit(element) }) + end + + # foo => {} + # ^^ + def visit_hash_pattern_node(node) + result = s(node, :hash_pat, visit_pattern_constant(node.constant)).concat(node.elements.flat_map { |element| visit(element) }) + + case node.rest + when AssocSplatNode + result << s(node.rest, :kwrest, :"**#{node.rest.value&.name}") + when NoKeywordsParameterNode + result << visit(node.rest) + end + + result + end + + # if foo then bar end + # ^^^^^^^^^^^^^^^^^^^ + # + # bar if foo + # ^^^^^^^^^^ + # + # foo ? bar : baz + # ^^^^^^^^^^^^^^^ + def visit_if_node(node) + s(node, :if, visit(node.predicate), visit(node.statements), visit(node.consequent)) + end + + # 1i + def visit_imaginary_node(node) + s(node, :lit, node.value) + end + + # { foo: } + # ^^^^ + def visit_implicit_node(node) + end + + # foo { |bar,| } + # ^ + def visit_implicit_rest_node(node) + end + + # case foo; in bar; end + # ^^^^^^^^^^^^^^^^^^^^^ + def visit_in_node(node) + pattern = + if node.pattern.is_a?(ConstantPathNode) + s(node.pattern, :const, visit(node.pattern)) + else + node.pattern.accept(copy_compiler(in_pattern: true)) + end + + s(node, :in, pattern).concat(node.statements.nil? ? [nil] : visit_all(node.statements.body)) + end + + # foo[bar] += baz + # ^^^^^^^^^^^^^^^ + def visit_index_operator_write_node(node) + arglist = nil + + if !node.arguments.nil? || !node.block.nil? + arglist = s(node, :arglist).concat(visit_all(node.arguments&.arguments || [])) + arglist << visit(node.block) if !node.block.nil? + end + + s(node, :op_asgn1, visit(node.receiver), arglist, node.binary_operator, visit_write_value(node.value)) + end + + # foo[bar] &&= baz + # ^^^^^^^^^^^^^^^^ + def visit_index_and_write_node(node) + arglist = nil + + if !node.arguments.nil? || !node.block.nil? + arglist = s(node, :arglist).concat(visit_all(node.arguments&.arguments || [])) + arglist << visit(node.block) if !node.block.nil? + end + + s(node, :op_asgn1, visit(node.receiver), arglist, :"&&", visit_write_value(node.value)) + end + + # foo[bar] ||= baz + # ^^^^^^^^^^^^^^^^ + def visit_index_or_write_node(node) + arglist = nil + + if !node.arguments.nil? || !node.block.nil? + arglist = s(node, :arglist).concat(visit_all(node.arguments&.arguments || [])) + arglist << visit(node.block) if !node.block.nil? + end + + s(node, :op_asgn1, visit(node.receiver), arglist, :"||", visit_write_value(node.value)) + end + + # foo[bar], = 1 + # ^^^^^^^^ + def visit_index_target_node(node) + arguments = visit_all(node.arguments&.arguments || []) + arguments << visit(node.block) unless node.block.nil? + + s(node, :attrasgn, visit(node.receiver), :[]=).concat(arguments) + end + + # @foo + # ^^^^ + def visit_instance_variable_read_node(node) + s(node, :ivar, node.name) + end + + # @foo = 1 + # ^^^^^^^^ + # + # @foo, @bar = 1 + # ^^^^ ^^^^ + def visit_instance_variable_write_node(node) + s(node, :iasgn, node.name, visit_write_value(node.value)) + end + + # @foo += bar + # ^^^^^^^^^^^ + def visit_instance_variable_operator_write_node(node) + s(node, :iasgn, node.name, s(node, :call, s(node, :ivar, node.name), node.binary_operator, visit_write_value(node.value))) + end + + # @foo &&= bar + # ^^^^^^^^^^^^ + def visit_instance_variable_and_write_node(node) + s(node, :op_asgn_and, s(node, :ivar, node.name), s(node, :iasgn, node.name, visit(node.value))) + end + + # @foo ||= bar + # ^^^^^^^^^^^^ + def visit_instance_variable_or_write_node(node) + s(node, :op_asgn_or, s(node, :ivar, node.name), s(node, :iasgn, node.name, visit(node.value))) + end + + # @foo, = bar + # ^^^^ + def visit_instance_variable_target_node(node) + s(node, :iasgn, node.name) + end + + # 1 + # ^ + def visit_integer_node(node) + s(node, :lit, node.value) + end + + # if /foo #{bar}/ then end + # ^^^^^^^^^^^^ + def visit_interpolated_match_last_line_node(node) + parts = visit_interpolated_parts(node.parts) + regexp = + if parts.length == 1 + s(node, :lit, Regexp.new(parts.first, node.options)) + else + s(node, :dregx).concat(parts).tap do |result| + options = node.options + result << options if options != 0 + end + end + + s(node, :match, regexp) + end + + # /foo #{bar}/ + # ^^^^^^^^^^^^ + def visit_interpolated_regular_expression_node(node) + parts = visit_interpolated_parts(node.parts) + + if parts.length == 1 + s(node, :lit, Regexp.new(parts.first, node.options)) + else + s(node, :dregx).concat(parts).tap do |result| + options = node.options + result << options if options != 0 + end + end + end + + # "foo #{bar}" + # ^^^^^^^^^^^^ + def visit_interpolated_string_node(node) + parts = visit_interpolated_parts(node.parts) + parts.length == 1 ? s(node, :str, parts.first) : s(node, :dstr).concat(parts) + end + + # :"foo #{bar}" + # ^^^^^^^^^^^^^ + def visit_interpolated_symbol_node(node) + parts = visit_interpolated_parts(node.parts) + parts.length == 1 ? s(node, :lit, parts.first.to_sym) : s(node, :dsym).concat(parts) + end + + # `foo #{bar}` + # ^^^^^^^^^^^^ + def visit_interpolated_x_string_node(node) + source = node.heredoc? ? node.parts.first : node + parts = visit_interpolated_parts(node.parts) + parts.length == 1 ? s(source, :xstr, parts.first) : s(source, :dxstr).concat(parts) + end + + # Visit the interpolated content of the string-like node. + private def visit_interpolated_parts(parts) + visited = [] + parts.each do |part| + result = visit(part) + + if result[0] == :evstr && result[1] + if result[1][0] == :str + visited << result[1] + elsif result[1][0] == :dstr + visited.concat(result[1][1..-1]) + else + visited << result + end + elsif result[0] == :dstr + visited.concat(result[1..-1]) + else + visited << result + end + end + + state = :beginning #: :beginning | :string_content | :interpolated_content + + visited.each_with_object([]) do |result, results| + case state + when :beginning + if result.is_a?(String) + results << result + state = :string_content + elsif result.is_a?(Array) && result[0] == :str + results << result[1] + state = :string_content + else + results << "" + results << result + state = :interpolated_content + end + when :string_content + if result.is_a?(String) + results[0] << result + elsif result.is_a?(Array) && result[0] == :str + results[0] << result[1] + else + results << result + state = :interpolated_content + end + when :interpolated_content + if result.is_a?(Array) && result[0] == :str && results[-1][0] == :str && (results[-1].line_max == result.line) + results[-1][1] << result[1] + results[-1].line_max = result.line_max + else + results << result + end + end + end + end + + # -> { it } + # ^^ + def visit_it_local_variable_read_node(node) + s(node, :call, nil, :it) + end + + # foo(bar: baz) + # ^^^^^^^^ + def visit_keyword_hash_node(node) + s(node, :hash).concat(node.elements.flat_map { |element| visit(element) }) + end + + # def foo(**bar); end + # ^^^^^ + # + # def foo(**); end + # ^^ + def visit_keyword_rest_parameter_node(node) + :"**#{node.name}" + end + + # -> {} + def visit_lambda_node(node) + parameters = + case node.parameters + when nil, NumberedParametersNode + s(node, :args) + else + visit(node.parameters) + end + + if node.body.nil? + s(node, :iter, s(node, :lambda), parameters) + else + s(node, :iter, s(node, :lambda), parameters, visit(node.body)) + end + end + + # foo + # ^^^ + def visit_local_variable_read_node(node) + if node.name.match?(/^_\d$/) + s(node, :call, nil, node.name) + else + s(node, :lvar, node.name) + end + end + + # foo = 1 + # ^^^^^^^ + # + # foo, bar = 1 + # ^^^ ^^^ + def visit_local_variable_write_node(node) + s(node, :lasgn, node.name, visit_write_value(node.value)) + end + + # foo += bar + # ^^^^^^^^^^ + def visit_local_variable_operator_write_node(node) + s(node, :lasgn, node.name, s(node, :call, s(node, :lvar, node.name), node.binary_operator, visit_write_value(node.value))) + end + + # foo &&= bar + # ^^^^^^^^^^^ + def visit_local_variable_and_write_node(node) + s(node, :op_asgn_and, s(node, :lvar, node.name), s(node, :lasgn, node.name, visit_write_value(node.value))) + end + + # foo ||= bar + # ^^^^^^^^^^^ + def visit_local_variable_or_write_node(node) + s(node, :op_asgn_or, s(node, :lvar, node.name), s(node, :lasgn, node.name, visit_write_value(node.value))) + end + + # foo, = bar + # ^^^ + def visit_local_variable_target_node(node) + s(node, :lasgn, node.name) + end + + # if /foo/ then end + # ^^^^^ + def visit_match_last_line_node(node) + s(node, :match, s(node, :lit, Regexp.new(node.unescaped, node.options))) + end + + # foo in bar + # ^^^^^^^^^^ + def visit_match_predicate_node(node) + s(node, :case, visit(node.value), s(node, :in, node.pattern.accept(copy_compiler(in_pattern: true)), nil), nil) + end + + # foo => bar + # ^^^^^^^^^^ + def visit_match_required_node(node) + s(node, :case, visit(node.value), s(node, :in, node.pattern.accept(copy_compiler(in_pattern: true)), nil), nil) + end + + # /(?<foo>foo)/ =~ bar + # ^^^^^^^^^^^^^^^^^^^^ + def visit_match_write_node(node) + s(node, :match2, visit(node.call.receiver), visit(node.call.arguments.arguments.first)) + end + + # A node that is missing from the syntax tree. This is only used in the + # case of a syntax error. The parser gem doesn't have such a concept, so + # we invent our own here. + def visit_missing_node(node) + raise "Cannot visit missing node directly" + end + + # module Foo; end + # ^^^^^^^^^^^^^^^ + def visit_module_node(node) + name = + if node.constant_path.is_a?(ConstantReadNode) + node.name + else + visit(node.constant_path) + end + + if node.body.nil? + s(node, :module, name) + elsif node.body.is_a?(StatementsNode) + compiler = copy_compiler(in_def: false) + s(node, :module, name).concat(node.body.body.map { |child| child.accept(compiler) }) + else + s(node, :module, name, node.body.accept(copy_compiler(in_def: false))) + end + end + + # foo, bar = baz + # ^^^^^^^^ + def visit_multi_target_node(node) + targets = [*node.lefts] + targets << node.rest if !node.rest.nil? && !node.rest.is_a?(ImplicitRestNode) + targets.concat(node.rights) + + s(node, :masgn, s(node, :array).concat(visit_all(targets))) + end + + # foo, bar = baz + # ^^^^^^^^^^^^^^ + def visit_multi_write_node(node) + targets = [*node.lefts] + targets << node.rest if !node.rest.nil? && !node.rest.is_a?(ImplicitRestNode) + targets.concat(node.rights) + + value = + if node.value.is_a?(ArrayNode) && node.value.opening_loc.nil? + if node.value.elements.length == 1 && node.value.elements.first.is_a?(SplatNode) + visit(node.value.elements.first) + else + visit(node.value) + end + else + s(node.value, :to_ary, visit(node.value)) + end + + s(node, :masgn, s(node, :array).concat(visit_all(targets)), value) + end + + # next + # ^^^^ + # + # next foo + # ^^^^^^^^ + def visit_next_node(node) + if node.arguments.nil? + s(node, :next) + elsif node.arguments.arguments.length == 1 + argument = node.arguments.arguments.first + s(node, :next, argument.is_a?(SplatNode) ? s(node, :svalue, visit(argument)) : visit(argument)) + else + s(node, :next, s(node, :array).concat(visit_all(node.arguments.arguments))) + end + end + + # nil + # ^^^ + def visit_nil_node(node) + s(node, :nil) + end + + # def foo(**nil); end + # ^^^^^ + def visit_no_keywords_parameter_node(node) + in_pattern ? s(node, :kwrest, :"**nil") : :"**nil" + end + + # -> { _1 + _2 } + # ^^^^^^^^^^^^^^ + def visit_numbered_parameters_node(node) + raise "Cannot visit numbered parameters directly" + end + + # $1 + # ^^ + def visit_numbered_reference_read_node(node) + s(node, :nth_ref, node.number) + end + + # def foo(bar: baz); end + # ^^^^^^^^ + def visit_optional_keyword_parameter_node(node) + s(node, :kwarg, node.name, visit(node.value)) + end + + # def foo(bar = 1); end + # ^^^^^^^ + def visit_optional_parameter_node(node) + s(node, :lasgn, node.name, visit(node.value)) + end + + # a or b + # ^^^^^^ + def visit_or_node(node) + s(node, :or, visit(node.left), visit(node.right)) + end + + # def foo(bar, *baz); end + # ^^^^^^^^^ + def visit_parameters_node(node) + children = + node.compact_child_nodes.map do |element| + if element.is_a?(MultiTargetNode) + visit_destructured_parameter(element) + else + visit(element) + end + end + + s(node, :args).concat(children) + end + + # def foo((bar, baz)); end + # ^^^^^^^^^^ + private def visit_destructured_parameter(node) + children = + [*node.lefts, *node.rest, *node.rights].map do |child| + case child + when RequiredParameterNode + visit(child) + when MultiTargetNode + visit_destructured_parameter(child) + when SplatNode + :"*#{child.expression&.name}" + else + raise + end + end + + s(node, :masgn).concat(children) + end + + # () + # ^^ + # + # (1) + # ^^^ + def visit_parentheses_node(node) + if node.body.nil? + s(node, :nil) + else + visit(node.body) + end + end + + # foo => ^(bar) + # ^^^^^^ + def visit_pinned_expression_node(node) + node.expression.accept(copy_compiler(in_pattern: false)) + end + + # foo = 1 and bar => ^foo + # ^^^^ + def visit_pinned_variable_node(node) + if node.variable.is_a?(LocalVariableReadNode) && node.variable.name.match?(/^_\d$/) + s(node, :lvar, node.variable.name) + else + visit(node.variable) + end + end + + # END {} + def visit_post_execution_node(node) + s(node, :iter, s(node, :postexe), 0, visit(node.statements)) + end + + # BEGIN {} + def visit_pre_execution_node(node) + s(node, :iter, s(node, :preexe), 0, visit(node.statements)) + end + + # The top-level program node. + def visit_program_node(node) + visit(node.statements) + end + + # 0..5 + # ^^^^ + def visit_range_node(node) + if !in_pattern && !node.left.nil? && !node.right.nil? && ([node.left.type, node.right.type] - %i[nil_node integer_node]).empty? + left = node.left.value if node.left.is_a?(IntegerNode) + right = node.right.value if node.right.is_a?(IntegerNode) + s(node, :lit, Range.new(left, right, node.exclude_end?)) + else + s(node, node.exclude_end? ? :dot3 : :dot2, visit_range_bounds_node(node.left), visit_range_bounds_node(node.right)) + end + end + + # If the bounds of a range node are empty parentheses, then they do not + # get replaced by their usual s(:nil), but instead are s(:begin). + private def visit_range_bounds_node(node) + if node.is_a?(ParenthesesNode) && node.body.nil? + s(node, :begin) + else + visit(node) + end + end + + # 1r + # ^^ + def visit_rational_node(node) + s(node, :lit, node.value) + end + + # redo + # ^^^^ + def visit_redo_node(node) + s(node, :redo) + end + + # /foo/ + # ^^^^^ + def visit_regular_expression_node(node) + s(node, :lit, Regexp.new(node.unescaped, node.options)) + end + + # def foo(bar:); end + # ^^^^ + def visit_required_keyword_parameter_node(node) + s(node, :kwarg, node.name) + end + + # def foo(bar); end + # ^^^ + def visit_required_parameter_node(node) + node.name + end + + # foo rescue bar + # ^^^^^^^^^^^^^^ + def visit_rescue_modifier_node(node) + s(node, :rescue, visit(node.expression), s(node.rescue_expression, :resbody, s(node.rescue_expression, :array), visit(node.rescue_expression))) + end + + # begin; rescue; end + # ^^^^^^^ + def visit_rescue_node(node) + exceptions = + if node.exceptions.length == 1 && node.exceptions.first.is_a?(SplatNode) + visit(node.exceptions.first) + else + s(node, :array).concat(visit_all(node.exceptions)) + end + + if !node.reference.nil? + exceptions << (visit(node.reference) << s(node.reference, :gvar, :"$!")) + end + + s(node, :resbody, exceptions).concat(node.statements.nil? ? [nil] : visit_all(node.statements.body)) + end + + # def foo(*bar); end + # ^^^^ + # + # def foo(*); end + # ^ + def visit_rest_parameter_node(node) + :"*#{node.name}" + end + + # retry + # ^^^^^ + def visit_retry_node(node) + s(node, :retry) + end + + # return + # ^^^^^^ + # + # return 1 + # ^^^^^^^^ + def visit_return_node(node) + if node.arguments.nil? + s(node, :return) + elsif node.arguments.arguments.length == 1 + argument = node.arguments.arguments.first + s(node, :return, argument.is_a?(SplatNode) ? s(node, :svalue, visit(argument)) : visit(argument)) + else + s(node, :return, s(node, :array).concat(visit_all(node.arguments.arguments))) + end + end + + # self + # ^^^^ + def visit_self_node(node) + s(node, :self) + end + + # A shareable constant. + def visit_shareable_constant_node(node) + visit(node.write) + end + + # class << self; end + # ^^^^^^^^^^^^^^^^^^ + def visit_singleton_class_node(node) + s(node, :sclass, visit(node.expression)).tap do |sexp| + sexp << node.body.accept(copy_compiler(in_def: false)) unless node.body.nil? + end + end + + # __ENCODING__ + # ^^^^^^^^^^^^ + def visit_source_encoding_node(node) + # TODO + s(node, :colon2, s(node, :const, :Encoding), :UTF_8) + end + + # __FILE__ + # ^^^^^^^^ + def visit_source_file_node(node) + s(node, :str, node.filepath) + end + + # __LINE__ + # ^^^^^^^^ + def visit_source_line_node(node) + s(node, :lit, node.location.start_line) + end + + # foo(*bar) + # ^^^^ + # + # def foo((bar, *baz)); end + # ^^^^ + # + # def foo(*); bar(*); end + # ^ + def visit_splat_node(node) + if node.expression.nil? + s(node, :splat) + else + s(node, :splat, visit(node.expression)) + end + end + + # A list of statements. + def visit_statements_node(node) + first, *rest = node.body + + if rest.empty? + visit(first) + else + s(node, :block).concat(visit_all(node.body)) + end + end + + # "foo" + # ^^^^^ + def visit_string_node(node) + s(node, :str, node.unescaped) + end + + # super(foo) + # ^^^^^^^^^^ + def visit_super_node(node) + arguments = node.arguments&.arguments || [] + block = node.block + + if block.is_a?(BlockArgumentNode) + arguments << block + block = nil + end + + visit_block(node, s(node, :super).concat(visit_all(arguments)), block) + end + + # :foo + # ^^^^ + def visit_symbol_node(node) + node.value == "!@" ? s(node, :lit, :"!@") : s(node, :lit, node.unescaped.to_sym) + end + + # true + # ^^^^ + def visit_true_node(node) + s(node, :true) + end + + # undef foo + # ^^^^^^^^^ + def visit_undef_node(node) + names = node.names.map { |name| s(node, :undef, visit(name)) } + names.length == 1 ? names.first : s(node, :block).concat(names) + end + + # unless foo; bar end + # ^^^^^^^^^^^^^^^^^^^ + # + # bar unless foo + # ^^^^^^^^^^^^^^ + def visit_unless_node(node) + s(node, :if, visit(node.predicate), visit(node.consequent), visit(node.statements)) + end + + # until foo; bar end + # ^^^^^^^^^^^^^^^^^ + # + # bar until foo + # ^^^^^^^^^^^^^ + def visit_until_node(node) + s(node, :until, visit(node.predicate), visit(node.statements), !node.begin_modifier?) + end + + # case foo; when bar; end + # ^^^^^^^^^^^^^ + def visit_when_node(node) + s(node, :when, s(node, :array).concat(visit_all(node.conditions))).concat(node.statements.nil? ? [nil] : visit_all(node.statements.body)) + end + + # while foo; bar end + # ^^^^^^^^^^^^^^^^^^ + # + # bar while foo + # ^^^^^^^^^^^^^ + def visit_while_node(node) + s(node, :while, visit(node.predicate), visit(node.statements), !node.begin_modifier?) + end + + # `foo` + # ^^^^^ + def visit_x_string_node(node) + result = s(node, :xstr, node.unescaped) + + if node.heredoc? + result.line = node.content_loc.start_line + result.line_max = node.content_loc.end_line + end + + result + end + + # yield + # ^^^^^ + # + # yield 1 + # ^^^^^^^ + def visit_yield_node(node) + s(node, :yield).concat(visit_all(node.arguments&.arguments || [])) + end + + private + + # Create a new compiler with the given options. + def copy_compiler(in_def: self.in_def, in_pattern: self.in_pattern) + Compiler.new(file, in_def: in_def, in_pattern: in_pattern) + end + + # Create a new Sexp object from the given prism node and arguments. + def s(node, *arguments) + result = Sexp.new(*arguments) + result.file = file + result.line = node.location.start_line + result.line_max = node.location.end_line + result + end + + # Visit a block node, which will modify the AST by wrapping the given + # visited node in an iter node. + def visit_block(node, sexp, block) + if block.nil? + sexp + else + parameters = + case block.parameters + when nil, NumberedParametersNode + 0 + else + visit(block.parameters) + end + + if block.body.nil? + s(node, :iter, sexp, parameters) + else + s(node, :iter, sexp, parameters, visit(block.body)) + end + end + end + + # Pattern constants get wrapped in another layer of :const. + def visit_pattern_constant(node) + case node + when nil + # nothing + when ConstantReadNode + visit(node) + else + s(node, :const, visit(node)) + end + end + + # Visit the value of a write, which will be on the right-hand side of + # a write operator. Because implicit arrays can have splats, those could + # potentially be wrapped in an svalue node. + def visit_write_value(node) + if node.is_a?(ArrayNode) && node.opening_loc.nil? + if node.elements.length == 1 && node.elements.first.is_a?(SplatNode) + s(node, :svalue, visit(node.elements.first)) + else + s(node, :svalue, visit(node)) + end + else + visit(node) + end + end + end + + private_constant :Compiler + + # Parse the given source and translate it into the seattlerb/ruby_parser + # gem's Sexp format. + def parse(source, filepath = "(string)") + translate(Prism.parse(source, filepath: filepath, scopes: [[]]), filepath) + end + + # Parse the given file and translate it into the seattlerb/ruby_parser + # gem's Sexp format. + def parse_file(filepath) + translate(Prism.parse_file(filepath, scopes: [[]]), filepath) + end + + class << self + # Parse the given source and translate it into the seattlerb/ruby_parser + # gem's Sexp format. + def parse(source, filepath = "(string)") + new.parse(source, filepath) + end + + # Parse the given file and translate it into the seattlerb/ruby_parser + # gem's Sexp format. + def parse_file(filepath) + new.parse_file(filepath) + end + end + + private + + # Translate the given parse result and filepath into the + # seattlerb/ruby_parser gem's Sexp format. + def translate(result, filepath) + if result.failure? + error = result.errors.first + raise ::RubyParser::SyntaxError, "#{filepath}:#{error.location.start_line} :: #{error.message}" + end + + result.value.accept(Compiler.new(filepath)) + end + end + end +end diff --git a/lib/pstore.rb b/lib/pstore.rb index 97df3c1509..57ecb0ef5c 100644 --- a/lib/pstore.rb +++ b/lib/pstore.rb @@ -326,7 +326,7 @@ require "digest" # end # class PStore - VERSION = "0.1.2" + VERSION = "0.1.3" RDWR_ACCESS = {mode: IO::RDWR | IO::CREAT | IO::BINARY, encoding: Encoding::ASCII_8BIT}.freeze RD_ACCESS = {mode: IO::RDONLY | IO::BINARY, encoding: Encoding::ASCII_8BIT}.freeze @@ -437,7 +437,7 @@ class PStore in_transaction unless @table.key? key if default == PStore::Error - raise PStore::Error, format("undefined key `%s'", key) + raise PStore::Error, format("undefined key '%s'", key) else return default end @@ -517,8 +517,8 @@ class PStore end # Exits the current transaction block, committing any changes - # specified in the transaction block. - # See {Committing or Aborting}[rdoc-ref:PStore@Committing+or+Aborting]. + # specified in the + # {transaction block}[rdoc-ref:PStore@The+Transaction+Block]. # # Raises an exception if called outside a transaction block. def commit @@ -528,8 +528,8 @@ class PStore end # Exits the current transaction block, discarding any changes - # specified in the transaction block. - # See {Committing or Aborting}[rdoc-ref:PStore@Committing+or+Aborting]. + # specified in the + # {transaction block}[rdoc-ref:PStore@The+Transaction+Block]. # # Raises an exception if called outside a transaction block. def abort diff --git a/lib/racc.rb b/lib/racc.rb deleted file mode 100644 index f6e4ac03a8..0000000000 --- a/lib/racc.rb +++ /dev/null @@ -1,6 +0,0 @@ -require 'racc/compat' -require 'racc/debugflags' -require 'racc/grammar' -require 'racc/state' -require 'racc/exception' -require 'racc/info' diff --git a/lib/racc/compat.rb b/lib/racc/compat.rb deleted file mode 100644 index 62f4f630be..0000000000 --- a/lib/racc/compat.rb +++ /dev/null @@ -1,33 +0,0 @@ -#-- -# -# -# -# Copyright (c) 1999-2006 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the same terms of ruby. -# see the file "COPYING". -# -#++ - -unless Object.method_defined?(:__send) - class Object - alias __send __send__ - end -end - -unless Object.method_defined?(:__send!) - class Object - alias __send! __send__ - end -end - -unless Array.method_defined?(:map!) - class Array - if Array.method_defined?(:collect!) - alias map! collect! - else - alias map! filter - end - end -end diff --git a/lib/racc/debugflags.rb b/lib/racc/debugflags.rb deleted file mode 100644 index ee34cf2314..0000000000 --- a/lib/racc/debugflags.rb +++ /dev/null @@ -1,60 +0,0 @@ -#-- -# -# -# -# Copyright (c) 1999-2006 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the same terms of ruby. -# see the file "COPYING". -# -#++ - -module Racc - - class DebugFlags - def DebugFlags.parse_option_string(s) - parse = rule = token = state = la = prec = conf = false - s.split(//).each do |ch| - case ch - when 'p' then parse = true - when 'r' then rule = true - when 't' then token = true - when 's' then state = true - when 'l' then la = true - when 'c' then prec = true - when 'o' then conf = true - else - raise "unknown debug flag char: #{ch.inspect}" - end - end - new(parse, rule, token, state, la, prec, conf) - end - - def initialize(parse = false, rule = false, token = false, state = false, - la = false, prec = false, conf = false) - @parse = parse - @rule = rule - @token = token - @state = state - @la = la - @prec = prec - @any = (parse || rule || token || state || la || prec) - @status_logging = conf - end - - attr_reader :parse - attr_reader :rule - attr_reader :token - attr_reader :state - attr_reader :la - attr_reader :prec - - def any? - @any - end - - attr_reader :status_logging - end - -end diff --git a/lib/racc/exception.rb b/lib/racc/exception.rb deleted file mode 100644 index c11dc2e43e..0000000000 --- a/lib/racc/exception.rb +++ /dev/null @@ -1,16 +0,0 @@ -#-- -# -# -# -# Copyright (c) 1999-2006 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the same terms of ruby. -# see the file "COPYING". -# -#++ - -module Racc - class Error < StandardError; end - class CompileError < Error; end -end diff --git a/lib/racc/grammar.rb b/lib/racc/grammar.rb deleted file mode 100644 index 7159862ffb..0000000000 --- a/lib/racc/grammar.rb +++ /dev/null @@ -1,1114 +0,0 @@ -#-- -# -# -# -# Copyright (c) 1999-2006 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the same terms of ruby. -# see the file "COPYING". -# -#++ - -require 'racc/compat' -require 'racc/iset' -require 'racc/sourcetext' -require 'racc/logfilegenerator' -require 'racc/exception' -require 'forwardable' - -module Racc - - class Grammar - - def initialize(debug_flags = DebugFlags.new) - @symboltable = SymbolTable.new - @debug_symbol = debug_flags.token - @rules = [] # :: [Rule] - @start = nil - @n_expected_srconflicts = nil - @prec_table = [] - @prec_table_closed = false - @closed = false - @states = nil - end - - attr_reader :start - attr_reader :symboltable - attr_accessor :n_expected_srconflicts - - def [](x) - @rules[x] - end - - def each_rule(&block) - @rules.each(&block) - end - - alias each each_rule - - def each_index(&block) - @rules.each_index(&block) - end - - def each_with_index(&block) - @rules.each_with_index(&block) - end - - def size - @rules.size - end - - def to_s - "<Racc::Grammar>" - end - - extend Forwardable - - def_delegator "@symboltable", :each, :each_symbol - def_delegator "@symboltable", :each_terminal - def_delegator "@symboltable", :each_nonterminal - - def intern(value, dummy = false) - @symboltable.intern(value, dummy) - end - - def symbols - @symboltable.symbols - end - - def nonterminal_base - @symboltable.nt_base - end - - def useless_nonterminal_exist? - n_useless_nonterminals() != 0 - end - - def n_useless_nonterminals - @n_useless_nonterminals ||= each_useless_nonterminal.count - end - - def each_useless_nonterminal - return to_enum __method__ unless block_given? - - @symboltable.each_nonterminal do |sym| - yield sym if sym.useless? - end - end - - def useless_rule_exist? - n_useless_rules() != 0 - end - - def n_useless_rules - @n_useless_rules ||= each_useless_rule.count - end - - def each_useless_rule - return to_enum __method__ unless block_given? - - each do |r| - yield r if r.useless? - end - end - - def nfa - (@states ||= States.new(self)).nfa - end - - def dfa - (@states ||= States.new(self)).dfa - end - - alias states dfa - - def state_transition_table - states().state_transition_table - end - - def parser_class - states = states() # cache - if $DEBUG - srcfilename = caller(1).first.slice(/\A(.*?):/, 1) - begin - write_log srcfilename + ".output" - rescue SystemCallError - end - report = lambda {|s| $stderr.puts "racc: #{srcfilename}: #{s}" } - if states.should_report_srconflict? - report["#{states.n_srconflicts} shift/reduce conflicts"] - end - if states.rrconflict_exist? - report["#{states.n_rrconflicts} reduce/reduce conflicts"] - end - g = states.grammar - if g.useless_nonterminal_exist? - report["#{g.n_useless_nonterminals} useless nonterminals"] - end - if g.useless_rule_exist? - report["#{g.n_useless_rules} useless rules"] - end - end - states.state_transition_table.parser_class - end - - def write_log(path) - File.open(path, 'w') {|f| - LogFileGenerator.new(states()).output f - } - end - - # - # Grammar Definition Interface - # - - def add(rule) - raise ArgumentError, "rule added after the Grammar closed" if @closed - @rules.push rule - end - - def added?(sym) - @rules.detect {|r| r.target == sym } - end - - def start_symbol=(s) - raise CompileError, "start symbol set twice'" if @start - @start = s - end - - def declare_precedence(assoc, syms) - raise CompileError, "precedence table defined twice" if @prec_table_closed - @prec_table.push [assoc, syms] - end - - def end_precedence_declaration(reverse) - @prec_table_closed = true - return if @prec_table.empty? - table = reverse ? @prec_table.reverse : @prec_table - table.each_with_index do |(assoc, syms), idx| - syms.each do |sym| - sym.assoc = assoc - sym.precedence = idx - end - end - end - - # - # Dynamic Generation Interface - # - - def Grammar.define(&block) - env = DefinitionEnv.new - env.instance_eval(&block) - env.grammar - end - - class DefinitionEnv - def initialize - @grammar = Grammar.new - @seqs = Hash.new(0) - @delayed = [] - end - - def grammar - flush_delayed - @grammar.each do |rule| - if rule.specified_prec - rule.specified_prec = @grammar.intern(rule.specified_prec) - end - end - @grammar.init - @grammar - end - - def precedence_table(&block) - env = PrecedenceDefinitionEnv.new(@grammar) - env.instance_eval(&block) - @grammar.end_precedence_declaration env.reverse - end - - def method_missing(mid, *args, &block) - unless mid.to_s[-1,1] == '=' - super # raises NoMethodError - end - target = @grammar.intern(mid.to_s.chop.intern) - unless args.size == 1 - raise ArgumentError, "too many arguments for #{mid} (#{args.size} for 1)" - end - _add target, args.first - end - - def _add(target, x) - case x - when Sym - @delayed.each do |rule| - rule.replace x, target if rule.target == x - end - @grammar.symboltable.delete x - else - x.each_rule do |r| - r.target = target - @grammar.add r - end - end - flush_delayed - end - - def _delayed_add(rule) - @delayed.push rule - end - - def _added?(sym) - @grammar.added?(sym) or @delayed.detect {|r| r.target == sym } - end - - def flush_delayed - return if @delayed.empty? - @delayed.each do |rule| - @grammar.add rule - end - @delayed.clear - end - - def seq(*list, &block) - Rule.new(nil, list.map {|x| _intern(x) }, UserAction.proc(block)) - end - - def null(&block) - seq(&block) - end - - def action(&block) - id = "@#{@seqs["action"] += 1}".intern - _delayed_add Rule.new(@grammar.intern(id), [], UserAction.proc(block)) - id - end - - alias _ action - - def option(sym, default = nil, &block) - _defmetasyntax("option", _intern(sym), block) {|target| - seq() { default } | seq(sym) - } - end - - def many(sym, &block) - _defmetasyntax("many", _intern(sym), block) {|target| - seq() { [] }\ - | seq(target, sym) {|list, x| list.push x; list } - } - end - - def many1(sym, &block) - _defmetasyntax("many1", _intern(sym), block) {|target| - seq(sym) {|x| [x] }\ - | seq(target, sym) {|list, x| list.push x; list } - } - end - - def separated_by(sep, sym, &block) - option(separated_by1(sep, sym), [], &block) - end - - def separated_by1(sep, sym, &block) - _defmetasyntax("separated_by1", _intern(sym), block) {|target| - seq(sym) {|x| [x] }\ - | seq(target, sep, sym) {|list, _, x| list.push x; list } - } - end - - def _intern(x) - case x - when Symbol, String - @grammar.intern(x) - when Racc::Sym - x - else - raise TypeError, "wrong type #{x.class} (expected Symbol/String/Racc::Sym)" - end - end - - private - - def _defmetasyntax(type, id, action, &block) - if action - idbase = "#{type}@#{id}-#{@seqs[type] += 1}" - target = _wrap(idbase, "#{idbase}-core", action) - _register("#{idbase}-core", &block) - else - target = _register("#{type}@#{id}", &block) - end - @grammar.intern(target) - end - - def _register(target_name) - target = target_name.intern - unless _added?(@grammar.intern(target)) - yield(target).each_rule do |rule| - rule.target = @grammar.intern(target) - _delayed_add rule - end - end - target - end - - def _wrap(target_name, sym, block) - target = target_name.intern - _delayed_add Rule.new(@grammar.intern(target), - [@grammar.intern(sym.intern)], - UserAction.proc(block)) - target - end - end - - class PrecedenceDefinitionEnv - def initialize(g) - @grammar = g - @prechigh_seen = false - @preclow_seen = false - @reverse = false - end - - attr_reader :reverse - - def higher - if @prechigh_seen - raise CompileError, "prechigh used twice" - end - @prechigh_seen = true - end - - def lower - if @preclow_seen - raise CompileError, "preclow used twice" - end - if @prechigh_seen - @reverse = true - end - @preclow_seen = true - end - - def left(*syms) - @grammar.declare_precedence :Left, syms.map {|s| @grammar.intern(s) } - end - - def right(*syms) - @grammar.declare_precedence :Right, syms.map {|s| @grammar.intern(s) } - end - - def nonassoc(*syms) - @grammar.declare_precedence :Nonassoc, syms.map {|s| @grammar.intern(s)} - end - end - - # - # Computation - # - - def init - return if @closed - @closed = true - @start ||= @rules.map {|r| r.target }.detect {|sym| not sym.dummy? } - raise CompileError, 'no rule in input' if @rules.empty? - add_start_rule - @rules.freeze - fix_ident - compute_hash - compute_heads - determine_terminals - compute_nullable_0 - @symboltable.fix - compute_locate - @symboltable.each_nonterminal {|t| compute_expand t } - compute_nullable - compute_useless - end - - private - - def add_start_rule - r = Rule.new(@symboltable.dummy, - [@start, @symboltable.anchor, @symboltable.anchor], - UserAction.empty) - r.ident = 0 - r.hash = 0 - r.precedence = nil - @rules.unshift r - end - - # Rule#ident - # LocationPointer#ident - def fix_ident - @rules.each_with_index do |rule, idx| - rule.ident = idx - end - end - - # Rule#hash - def compute_hash - hash = 4 # size of dummy rule - @rules.each do |rule| - rule.hash = hash - hash += (rule.size + 1) - end - end - - # Sym#heads - def compute_heads - @rules.each do |rule| - rule.target.heads.push rule.ptrs[0] - end - end - - # Sym#terminal? - def determine_terminals - @symboltable.each do |s| - s.term = s.heads.empty? - end - end - - # Sym#self_null? - def compute_nullable_0 - @symboltable.each do |s| - if s.terminal? - s.snull = false - else - s.snull = s.heads.any? {|loc| loc.reduce? } - end - end - end - - # Sym#locate - def compute_locate - @rules.each do |rule| - t = nil - rule.ptrs.each do |ptr| - unless ptr.reduce? - tok = ptr.dereference - tok.locate.push ptr - t = tok if tok.terminal? - end - end - rule.precedence = t - end - end - - # Sym#expand - def compute_expand(t) - puts "expand> #{t.to_s}" if @debug_symbol - t.expand = _compute_expand(t, ISet.new, []) - puts "expand< #{t.to_s}: #{t.expand.to_s}" if @debug_symbol - end - - def _compute_expand(t, set, lock) - if tmp = t.expand - set.update tmp - return set - end - tok = nil - set.update_a t.heads - t.heads.each do |ptr| - tok = ptr.dereference - if tok and tok.nonterminal? - unless lock[tok.ident] - lock[tok.ident] = true - _compute_expand tok, set, lock - end - end - end - set - end - - # Sym#nullable?, Rule#nullable? - def compute_nullable - @rules.each {|r| r.null = false } - @symboltable.each {|t| t.null = false } - r = @rules.dup - s = @symboltable.nonterminals - begin - rs = r.size - ss = s.size - check_rules_nullable r - check_symbols_nullable s - end until rs == r.size and ss == s.size - end - - def check_rules_nullable(rules) - rules.delete_if do |rule| - rule.null = true - rule.symbols.each do |t| - unless t.nullable? - rule.null = false - break - end - end - rule.nullable? - end - end - - def check_symbols_nullable(symbols) - symbols.delete_if do |sym| - sym.heads.each do |ptr| - if ptr.rule.nullable? - sym.null = true - break - end - end - sym.nullable? - end - end - - # Sym#useless?, Rule#useless? - # FIXME: what means "useless"? - def compute_useless - @symboltable.each_terminal {|sym| sym.useless = false } - @symboltable.each_nonterminal {|sym| sym.useless = true } - @rules.each {|rule| rule.useless = true } - r = @rules.dup - s = @symboltable.nonterminals - begin - rs = r.size - ss = s.size - check_rules_useless r - check_symbols_useless s - end until r.size == rs and s.size == ss - end - - def check_rules_useless(rules) - rules.delete_if do |rule| - rule.useless = false - rule.symbols.each do |sym| - if sym.useless? - rule.useless = true - break - end - end - not rule.useless? - end - end - - def check_symbols_useless(s) - s.delete_if do |t| - t.heads.each do |ptr| - unless ptr.rule.useless? - t.useless = false - break - end - end - not t.useless? - end - end - - end # class Grammar - - - class Rule - - def initialize(target, syms, act) - @target = target - @symbols = syms - @action = act - @alternatives = [] - - @ident = nil - @hash = nil - @ptrs = nil - @precedence = nil - @specified_prec = nil - @null = nil - @useless = nil - end - - attr_accessor :target - attr_reader :symbols - attr_reader :action - - def |(x) - @alternatives.push x.rule - self - end - - def rule - self - end - - def each_rule(&block) - yield self - @alternatives.each(&block) - end - - attr_accessor :ident - - attr_reader :hash - attr_reader :ptrs - - def hash=(n) - @hash = n - ptrs = [] - @symbols.each_with_index do |sym, idx| - ptrs.push LocationPointer.new(self, idx, sym) - end - ptrs.push LocationPointer.new(self, @symbols.size, nil) - @ptrs = ptrs - end - - def precedence - @specified_prec || @precedence - end - - def precedence=(sym) - @precedence ||= sym - end - - def prec(sym, &block) - @specified_prec = sym - if block - unless @action.empty? - raise CompileError, 'both of rule action block and prec block given' - end - @action = UserAction.proc(block) - end - self - end - - attr_accessor :specified_prec - - def nullable?() @null end - def null=(n) @null = n end - - def useless?() @useless end - def useless=(u) @useless = u end - - def inspect - "#<Racc::Rule id=#{@ident} (#{@target})>" - end - - def ==(other) - other.kind_of?(Rule) and @ident == other.ident - end - - def [](idx) - @symbols[idx] - end - - def size - @symbols.size - end - - def empty? - @symbols.empty? - end - - def to_s - "#<rule#{@ident}>" - end - - def accept? - if tok = @symbols[-1] - tok.anchor? - else - false - end - end - - def each(&block) - @symbols.each(&block) - end - - def replace(src, dest) - @target = dest - @symbols = @symbols.map {|s| s == src ? dest : s } - end - - end # class Rule - - - class UserAction - - def UserAction.source_text(src) - new(src, nil) - end - - def UserAction.proc(pr = nil, &block) - if pr and block - raise ArgumentError, "both of argument and block given" - end - new(nil, pr || block) - end - - def UserAction.empty - new(nil, nil) - end - - private_class_method :new - - def initialize(src, proc) - @source = src - @proc = proc - end - - attr_reader :source - attr_reader :proc - - def source? - not @proc - end - - def proc? - not @source - end - - def empty? - not @proc and not @source - end - - def name - "{action type=#{@source || @proc || 'nil'}}" - end - - alias inspect name - - end - - - class OrMark - def initialize(lineno) - @lineno = lineno - end - - def name - '|' - end - - alias inspect name - - attr_reader :lineno - end - - - class Prec - def initialize(symbol, lineno) - @symbol = symbol - @lineno = lineno - end - - def name - "=#{@symbol}" - end - - alias inspect name - - attr_reader :symbol - attr_reader :lineno - end - - - # - # A set of rule and position in it's RHS. - # Note that the number of pointers is more than rule's RHS array, - # because pointer points right edge of the final symbol when reducing. - # - class LocationPointer - - def initialize(rule, i, sym) - @rule = rule - @index = i - @symbol = sym - @ident = @rule.hash + i - @reduce = sym.nil? - end - - attr_reader :rule - attr_reader :index - attr_reader :symbol - - alias dereference symbol - - attr_reader :ident - alias hash ident - attr_reader :reduce - alias reduce? reduce - - def to_s - sprintf('(%d,%d %s)', - @rule.ident, @index, (reduce?() ? '#' : @symbol.to_s)) - end - - alias inspect to_s - - def eql?(ot) - @hash == ot.hash - end - - alias == eql? - - def head? - @index == 0 - end - - def next - @rule.ptrs[@index + 1] or ptr_bug! - end - - alias increment next - - def before(len) - @rule.ptrs[@index - len] or ptr_bug! - end - - private - - def ptr_bug! - raise "racc: fatal: pointer not exist: self: #{to_s}" - end - - end # class LocationPointer - - - class SymbolTable - - include Enumerable - - def initialize - @symbols = [] # :: [Racc::Sym] - @cache = {} # :: {(String|Symbol) => Racc::Sym} - @dummy = intern(:$start, true) - @anchor = intern(false, true) # Symbol ID = 0 - @error = intern(:error, false) # Symbol ID = 1 - end - - attr_reader :dummy - attr_reader :anchor - attr_reader :error - - def [](id) - @symbols[id] - end - - def intern(val, dummy = false) - @cache[val] ||= - begin - sym = Sym.new(val, dummy) - @symbols.push sym - sym - end - end - - attr_reader :symbols - alias to_a symbols - - def delete(sym) - @symbols.delete sym - @cache.delete sym.value - end - - attr_reader :nt_base - - def nt_max - @symbols.size - end - - def each(&block) - @symbols.each(&block) - end - - def terminals(&block) - @symbols[0, @nt_base] - end - - def each_terminal(&block) - @terms.each(&block) - end - - def nonterminals - @symbols[@nt_base, @symbols.size - @nt_base] - end - - def each_nonterminal(&block) - @nterms.each(&block) - end - - def fix - terms, nterms = @symbols.partition {|s| s.terminal? } - @symbols = terms + nterms - @terms = terms - @nterms = nterms - @nt_base = terms.size - fix_ident - check_terminals - end - - private - - def fix_ident - @symbols.each_with_index do |t, i| - t.ident = i - end - end - - def check_terminals - return unless @symbols.any? {|s| s.should_terminal? } - @anchor.should_terminal - @error.should_terminal - each_terminal do |t| - t.should_terminal if t.string_symbol? - end - each do |s| - s.should_terminal if s.assoc - end - terminals().reject {|t| t.should_terminal? }.each do |t| - raise CompileError, "terminal #{t} not declared as terminal" - end - nonterminals().select {|n| n.should_terminal? }.each do |n| - raise CompileError, "symbol #{n} declared as terminal but is not terminal" - end - end - - end # class SymbolTable - - - # Stands terminal and nonterminal symbols. - class Sym - - def initialize(value, dummyp) - @ident = nil - @value = value - @dummyp = dummyp - - @term = nil - @nterm = nil - @should_terminal = false - @precedence = nil - case value - when Symbol - @to_s = value.to_s - @serialized = value.inspect - @string = false - when String - @to_s = value.inspect - @serialized = value.dump - @string = true - when false - @to_s = '$end' - @serialized = 'false' - @string = false - else - raise ArgumentError, "unknown symbol value: #{value.class}" - end - - @heads = [] - @locate = [] - @snull = nil - @null = nil - @expand = nil - @useless = nil - end - - class << self - def once_writer(nm) - nm = nm.id2name - module_eval(<<-EOS) - def #{nm}=(v) - raise 'racc: fatal: @#{nm} != nil' unless @#{nm}.nil? - @#{nm} = v - end - EOS - end - end - - once_writer :ident - attr_reader :ident - - alias hash ident - - attr_reader :value - - def dummy? - @dummyp - end - - def terminal? - @term - end - - def nonterminal? - @nterm - end - - def term=(t) - raise 'racc: fatal: term= called twice' unless @term.nil? - @term = t - @nterm = !t - end - - def should_terminal - @should_terminal = true - end - - def should_terminal? - @should_terminal - end - - def string_symbol? - @string - end - - def serialize - @serialized - end - - attr_writer :serialized - - attr_accessor :precedence - attr_accessor :assoc - - def to_s - @to_s.dup - end - - alias inspect to_s - - def |(x) - rule() | x.rule - end - - def rule - Rule.new(nil, [self], UserAction.empty) - end - - # - # cache - # - - attr_reader :heads - attr_reader :locate - - def self_null? - @snull - end - - once_writer :snull - - def nullable? - @null - end - - def null=(n) - @null = n - end - - attr_reader :expand - once_writer :expand - - def useless? - @useless - end - - def useless=(f) - @useless = f - end - - end # class Sym - -end # module Racc diff --git a/lib/racc/grammarfileparser.rb b/lib/racc/grammarfileparser.rb deleted file mode 100644 index c7d1207f0b..0000000000 --- a/lib/racc/grammarfileparser.rb +++ /dev/null @@ -1,561 +0,0 @@ -#-- -# -# -# -# Copyright (c) 1999-2006 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the same terms of ruby. -# see the file "COPYING". -# -#++ - -require 'racc' -require 'racc/compat' -require 'racc/grammar' -require 'racc/parserfilegenerator' -require 'racc/sourcetext' -require 'stringio' - -module Racc - - grammar = Grammar.define { - g = self - - g.class = seq(:CLASS, :cname, many(:param), :RULE, :rules, option(:END)) - - g.cname = seq(:rubyconst) {|name| - @result.params.classname = name - }\ - | seq(:rubyconst, "<", :rubyconst) {|c, _, s| - @result.params.classname = c - @result.params.superclass = s - } - - g.rubyconst = separated_by1(:colon2, :SYMBOL) {|syms| - syms.map {|s| s.to_s }.join('::') - } - - g.colon2 = seq(':', ':') - - g.param = seq(:CONV, many1(:convdef), :END) {|*| - #@grammar.end_convert_block # FIXME - }\ - | seq(:PRECHIGH, many1(:precdef), :PRECLOW) {|*| - @grammar.end_precedence_declaration true - }\ - | seq(:PRECLOW, many1(:precdef), :PRECHIGH) {|*| - @grammar.end_precedence_declaration false - }\ - | seq(:START, :symbol) {|_, sym| - @grammar.start_symbol = sym - }\ - | seq(:TOKEN, :symbols) {|_, syms| - syms.each do |s| - s.should_terminal - end - }\ - | seq(:OPTION, :options) {|_, syms| - syms.each do |opt| - case opt - when 'result_var' - @result.params.result_var = true - when 'no_result_var' - @result.params.result_var = false - when 'omit_action_call' - @result.params.omit_action_call = true - when 'no_omit_action_call' - @result.params.omit_action_call = false - else - raise CompileError, "unknown option: #{opt}" - end - end - }\ - | seq(:EXPECT, :DIGIT) {|_, num| - if @grammar.n_expected_srconflicts - raise CompileError, "`expect' seen twice" - end - @grammar.n_expected_srconflicts = num - } - - g.convdef = seq(:symbol, :STRING) {|sym, code| - sym.serialized = code - } - - g.precdef = seq(:LEFT, :symbols) {|_, syms| - @grammar.declare_precedence :Left, syms - }\ - | seq(:RIGHT, :symbols) {|_, syms| - @grammar.declare_precedence :Right, syms - }\ - | seq(:NONASSOC, :symbols) {|_, syms| - @grammar.declare_precedence :Nonassoc, syms - } - - g.symbols = seq(:symbol) {|sym| - [sym] - }\ - | seq(:symbols, :symbol) {|list, sym| - list.push sym - list - }\ - | seq(:symbols, "|") - - g.symbol = seq(:SYMBOL) {|sym| @grammar.intern(sym) }\ - | seq(:STRING) {|str| @grammar.intern(str) } - - g.options = many(:SYMBOL) {|syms| syms.map {|s| s.to_s } } - - g.rules = option(:rules_core) {|list| - add_rule_block list unless list.empty? - nil - } - - g.rules_core = seq(:symbol) {|sym| - [sym] - }\ - | seq(:rules_core, :rule_item) {|list, i| - list.push i - list - }\ - | seq(:rules_core, ';') {|list, *| - add_rule_block list unless list.empty? - list.clear - list - }\ - | seq(:rules_core, ':') {|list, *| - next_target = list.pop - add_rule_block list unless list.empty? - [next_target] - } - - g.rule_item = seq(:symbol)\ - | seq("|") {|*| - OrMark.new(@scanner.lineno) - }\ - | seq("=", :symbol) {|_, sym| - Prec.new(sym, @scanner.lineno) - }\ - | seq(:ACTION) {|src| - UserAction.source_text(src) - } - } - - GrammarFileParser = grammar.parser_class - - if grammar.states.srconflict_exist? - raise 'Racc boot script fatal: S/R conflict in build' - end - if grammar.states.rrconflict_exist? - raise 'Racc boot script fatal: R/R conflict in build' - end - - class GrammarFileParser # reopen - - class Result - def initialize(grammar) - @grammar = grammar - @params = ParserFileGenerator::Params.new - end - - attr_reader :grammar - attr_reader :params - end - - def GrammarFileParser.parse_file(filename) - parse(File.read(filename), filename, 1) - end - - def GrammarFileParser.parse(src, filename = '-', lineno = 1) - new().parse(src, filename, lineno) - end - - def initialize(debug_flags = DebugFlags.new) - @yydebug = debug_flags.parse - end - - def parse(src, filename = '-', lineno = 1) - @filename = filename - @lineno = lineno - @scanner = GrammarFileScanner.new(src, @filename) - @scanner.debug = @yydebug - @grammar = Grammar.new - @result = Result.new(@grammar) - @embedded_action_seq = 0 - yyparse @scanner, :yylex - parse_user_code - @result.grammar.init - @result - end - - private - - def next_token - @scanner.scan - end - - def on_error(tok, val, _values) - if val.respond_to?(:id2name) - v = val.id2name - elsif val.kind_of?(String) - v = val - else - v = val.inspect - end - raise CompileError, "#{location()}: unexpected token '#{v}'" - end - - def location - "#{@filename}:#{@lineno - 1 + @scanner.lineno}" - end - - def add_rule_block(list) - sprec = nil - target = list.shift - case target - when OrMark, UserAction, Prec - raise CompileError, "#{target.lineno}: unexpected symbol #{target.name}" - end - curr = [] - list.each do |i| - case i - when OrMark - add_rule target, curr, sprec - curr = [] - sprec = nil - when Prec - raise CompileError, "'=<prec>' used twice in one rule" if sprec - sprec = i.symbol - else - curr.push i - end - end - add_rule target, curr, sprec - end - - def add_rule(target, list, sprec) - if list.last.kind_of?(UserAction) - act = list.pop - else - act = UserAction.empty - end - list.map! {|s| s.kind_of?(UserAction) ? embedded_action(s) : s } - rule = Rule.new(target, list, act) - rule.specified_prec = sprec - @grammar.add rule - end - - def embedded_action(act) - sym = @grammar.intern("@#{@embedded_action_seq += 1}".intern, true) - @grammar.add Rule.new(sym, [], act) - sym - end - - # - # User Code Block - # - - def parse_user_code - line = @scanner.lineno - _, *blocks = *@scanner.epilogue.split(/^----/) - blocks.each do |block| - header, *body = block.lines.to_a - label0, paths = *header.sub(/\A-+/, '').split('=', 2) - label = canonical_label(label0) - (paths ? paths.strip.split(' ') : []).each do |path| - add_user_code label, SourceText.new(File.read(path), path, 1) - end - add_user_code label, SourceText.new(body.join(''), @filename, line + 1) - line += (1 + body.size) - end - end - - USER_CODE_LABELS = { - 'header' => :header, - 'prepare' => :header, # obsolete - 'inner' => :inner, - 'footer' => :footer, - 'driver' => :footer # obsolete - } - - def canonical_label(src) - label = src.to_s.strip.downcase.slice(/\w+/) - unless USER_CODE_LABELS.key?(label) - raise CompileError, "unknown user code type: #{label.inspect}" - end - label - end - - def add_user_code(label, src) - @result.params.public_send(USER_CODE_LABELS[label]).push src - end - - end - - - class GrammarFileScanner - - def initialize(str, filename = '-') - @lines = str.b.split(/\n|\r\n|\r/) - @filename = filename - @lineno = -1 - @line_head = true - @in_rule_blk = false - @in_conv_blk = false - @in_block = nil - @epilogue = '' - @debug = false - next_line - end - - attr_reader :epilogue - - def lineno - @lineno + 1 - end - - attr_accessor :debug - - def yylex(&block) - unless @debug - yylex0(&block) - else - yylex0 do |sym, tok| - $stderr.printf "%7d %-10s %s\n", lineno(), sym.inspect, tok.inspect - yield [sym, tok] - end - end - end - - private - - def yylex0 - begin - until @line.empty? - @line.sub!(/\A\s+/, '') - if /\A\#/ =~ @line - break - elsif /\A\/\*/ =~ @line - skip_comment - elsif s = reads(/\A[a-zA-Z_]\w*/) - yield [atom_symbol(s), s.intern] - elsif s = reads(/\A\d+/) - yield [:DIGIT, s.to_i] - elsif ch = reads(/\A./) - case ch - when '"', "'" - yield [:STRING, eval(scan_quoted(ch))] - when '{' - lineno = lineno() - yield [:ACTION, SourceText.new(scan_action(), @filename, lineno)] - else - if ch == '|' - @line_head = false - end - yield [ch, ch] - end - else - end - end - end while next_line() - yield nil - end - - def next_line - @lineno += 1 - @line = @lines[@lineno] - if not @line or /\A----/ =~ @line - @epilogue = @lines.join("\n") - @lines.clear - @line = nil - if @in_block - @lineno -= 1 - scan_error! sprintf('unterminated %s', @in_block) - end - false - else - @line.sub!(/(?:\n|\r\n|\r)\z/, '') - @line_head = true - true - end - end - - ReservedWord = { - 'right' => :RIGHT, - 'left' => :LEFT, - 'nonassoc' => :NONASSOC, - 'preclow' => :PRECLOW, - 'prechigh' => :PRECHIGH, - 'token' => :TOKEN, - 'convert' => :CONV, - 'options' => :OPTION, - 'start' => :START, - 'expect' => :EXPECT, - 'class' => :CLASS, - 'rule' => :RULE, - 'end' => :END - } - - def atom_symbol(token) - if token == 'end' - symbol = :END - @in_conv_blk = false - @in_rule_blk = false - else - if @line_head and not @in_conv_blk and not @in_rule_blk - symbol = ReservedWord[token] || :SYMBOL - else - symbol = :SYMBOL - end - case symbol - when :RULE then @in_rule_blk = true - when :CONV then @in_conv_blk = true - end - end - @line_head = false - symbol - end - - def skip_comment - @in_block = 'comment' - until m = /\*\//.match(@line) - next_line - end - @line = m.post_match - @in_block = nil - end - - $raccs_print_type = false - - def scan_action - buf = String.new - nest = 1 - pre = nil - @in_block = 'action' - begin - pre = nil - if s = reads(/\A\s+/) - # does not set 'pre' - buf << s - end - until @line.empty? - if s = reads(/\A[^'"`{}%#\/\$]+/) - buf << (pre = s) - next - end - case ch = read(1) - when '{' - nest += 1 - buf << (pre = ch) - when '}' - nest -= 1 - if nest == 0 - @in_block = nil - buf.sub!(/[ \t\f]+\z/, '') - return buf - end - buf << (pre = ch) - when '#' # comment - buf << ch << @line - break - when "'", '"', '`' - buf << (pre = scan_quoted(ch)) - when '%' - if literal_head? pre, @line - # % string, regexp, array - buf << ch - case ch = read(1) - when /[qQx]/n - buf << ch << (pre = scan_quoted(read(1), '%string')) - when /wW/n - buf << ch << (pre = scan_quoted(read(1), '%array')) - when /s/n - buf << ch << (pre = scan_quoted(read(1), '%symbol')) - when /r/n - buf << ch << (pre = scan_quoted(read(1), '%regexp')) - when /[a-zA-Z0-9= ]/n # does not include "_" - scan_error! "unknown type of % literal '%#{ch}'" - else - buf << (pre = scan_quoted(ch, '%string')) - end - else - # operator - buf << '||op->' if $raccs_print_type - buf << (pre = ch) - end - when '/' - if literal_head? pre, @line - # regexp - buf << (pre = scan_quoted(ch, 'regexp')) - else - # operator - buf << '||op->' if $raccs_print_type - buf << (pre = ch) - end - when '$' # gvar - buf << ch << (pre = read(1)) - else - raise 'racc: fatal: must not happen' - end - end - buf << "\n" - end while next_line() - raise 'racc: fatal: scan finished before parser finished' - end - - def literal_head?(pre, post) - (!pre || /[a-zA-Z_0-9]/n !~ pre[-1,1]) && - !post.empty? && /\A[\s\=]/n !~ post - end - - def read(len) - s = @line[0, len] - @line = @line[len .. -1] - s - end - - def reads(re) - m = re.match(@line) or return nil - @line = m.post_match - m[0] - end - - def scan_quoted(left, tag = 'string') - buf = left.dup - buf = "||#{tag}->" + buf if $raccs_print_type - re = get_quoted_re(left) - sv, @in_block = @in_block, tag - begin - if s = reads(re) - buf << s - break - else - buf << @line - end - end while next_line() - @in_block = sv - buf << "<-#{tag}||" if $raccs_print_type - buf - end - - LEFT_TO_RIGHT = { - '(' => ')', - '{' => '}', - '[' => ']', - '<' => '>' - } - - CACHE = {} - - def get_quoted_re(left) - term = Regexp.quote(LEFT_TO_RIGHT[left] || left) - CACHE[left] ||= /\A[^#{term}\\]*(?:\\.[^\\#{term}]*)*#{term}/ - end - - def scan_error!(msg) - raise CompileError, "#{lineno()}: #{msg}" - end - - end - -end # module Racc diff --git a/lib/racc/info.rb b/lib/racc/info.rb deleted file mode 100644 index 37bff7edba..0000000000 --- a/lib/racc/info.rb +++ /dev/null @@ -1,17 +0,0 @@ -#-- -# -# -# -# Copyright (c) 1999-2006 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the same terms of ruby. -# see the file "COPYING". -# -#++ - -module Racc - VERSION = '1.6.2' - Version = VERSION - Copyright = 'Copyright (c) 1999-2006 Minero Aoki' -end diff --git a/lib/racc/iset.rb b/lib/racc/iset.rb deleted file mode 100644 index 339221d21b..0000000000 --- a/lib/racc/iset.rb +++ /dev/null @@ -1,92 +0,0 @@ -#-- -# -# -# -# Copyright (c) 1999-2006 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the same terms of ruby. -# see the file "COPYING". -# -#++ - -module Racc - - # An "indexed" set. All items must respond to :ident. - class ISet - - def initialize(a = []) - @set = a - end - - attr_reader :set - - def add(i) - @set[i.ident] = i - end - - def [](key) - @set[key.ident] - end - - def []=(key, val) - @set[key.ident] = val - end - - alias include? [] - alias key? [] - - def update(other) - s = @set - o = other.set - o.each_index do |idx| - if t = o[idx] - s[idx] = t - end - end - end - - def update_a(a) - s = @set - a.each {|i| s[i.ident] = i } - end - - def delete(key) - i = @set[key.ident] - @set[key.ident] = nil - i - end - - def each(&block) - @set.compact.each(&block) - end - - def to_a - @set.compact - end - - def to_s - "[#{@set.compact.join(' ')}]" - end - - alias inspect to_s - - def size - @set.nitems - end - - def empty? - @set.nitems == 0 - end - - def clear - @set.clear - end - - def dup - ISet.new(@set.dup) - end - - end # class ISet - -end # module Racc diff --git a/lib/racc/logfilegenerator.rb b/lib/racc/logfilegenerator.rb deleted file mode 100644 index 2f5aa0c8b0..0000000000 --- a/lib/racc/logfilegenerator.rb +++ /dev/null @@ -1,212 +0,0 @@ -#-- -# -# -# -# Copyright (c) 1999-2006 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the same terms of ruby. -# see the file "COPYING". -# -#++ - -module Racc - - class LogFileGenerator - - def initialize(states, debug_flags = DebugFlags.new) - @states = states - @grammar = states.grammar - @debug_flags = debug_flags - end - - def output(out) - output_conflict out; out.puts - output_useless out; out.puts - output_rule out; out.puts - output_token out; out.puts - output_state out - end - - # - # Warnings - # - - def output_conflict(out) - @states.each do |state| - if state.srconf - out.printf "state %d contains %d shift/reduce conflicts\n", - state.stateid, state.srconf.size - end - if state.rrconf - out.printf "state %d contains %d reduce/reduce conflicts\n", - state.stateid, state.rrconf.size - end - end - end - - def output_useless(out) - @grammar.each do |rl| - if rl.useless? - out.printf "rule %d (%s) never reduced\n", - rl.ident, rl.target.to_s - end - end - @grammar.each_nonterminal do |t| - if t.useless? - out.printf "useless nonterminal %s\n", t.to_s - end - end - end - - # - # States - # - - def output_state(out) - out << "--------- State ---------\n" - - showall = @debug_flags.la || @debug_flags.state - @states.each do |state| - out << "\nstate #{state.ident}\n\n" - - (showall ? state.closure : state.core).each do |ptr| - pointer_out(out, ptr) if ptr.rule.ident != 0 or showall - end - out << "\n" - - action_out out, state - end - end - - def pointer_out(out, ptr) - buf = sprintf("%4d) %s :", ptr.rule.ident, ptr.rule.target.to_s) - ptr.rule.symbols.each_with_index do |tok, idx| - buf << ' _' if idx == ptr.index - buf << ' ' << tok.to_s - end - buf << ' _' if ptr.reduce? - out.puts buf - end - - def action_out(f, state) - sr = state.srconf && state.srconf.dup - rr = state.rrconf && state.rrconf.dup - acts = state.action - keys = acts.keys - keys.sort! {|a,b| a.ident <=> b.ident } - - [ Shift, Reduce, Error, Accept ].each do |klass| - keys.delete_if do |tok| - act = acts[tok] - if act.kind_of?(klass) - outact f, tok, act - if sr and c = sr.delete(tok) - outsrconf f, c - end - if rr and c = rr.delete(tok) - outrrconf f, c - end - - true - else - false - end - end - end - sr.each {|tok, c| outsrconf f, c } if sr - rr.each {|tok, c| outrrconf f, c } if rr - - act = state.defact - if not act.kind_of?(Error) or @debug_flags.any? - outact f, '$default', act - end - - f.puts - state.goto_table.each do |t, st| - if t.nonterminal? - f.printf " %-12s go to state %d\n", t.to_s, st.ident - end - end - end - - def outact(f, t, act) - case act - when Shift - f.printf " %-12s shift, and go to state %d\n", - t.to_s, act.goto_id - when Reduce - f.printf " %-12s reduce using rule %d (%s)\n", - t.to_s, act.ruleid, act.rule.target.to_s - when Accept - f.printf " %-12s accept\n", t.to_s - when Error - f.printf " %-12s error\n", t.to_s - else - raise "racc: fatal: wrong act for outact: act=#{act}(#{act.class})" - end - end - - def outsrconf(f, confs) - confs.each do |c| - r = c.reduce - f.printf " %-12s [reduce using rule %d (%s)]\n", - c.shift.to_s, r.ident, r.target.to_s - end - end - - def outrrconf(f, confs) - confs.each do |c| - r = c.low_prec - f.printf " %-12s [reduce using rule %d (%s)]\n", - c.token.to_s, r.ident, r.target.to_s - end - end - - # - # Rules - # - - def output_rule(out) - out.print "-------- Grammar --------\n\n" - @grammar.each do |rl| - if @debug_flags.any? or rl.ident != 0 - out.printf "rule %d %s: %s\n", - rl.ident, rl.target.to_s, rl.symbols.join(' ') - end - end - end - - # - # Tokens - # - - def output_token(out) - out.print "------- Symbols -------\n\n" - - out.print "**Nonterminals, with rules where they appear\n\n" - @grammar.each_nonterminal do |t| - tmp = <<SRC - %s (%d) - on right: %s - on left : %s -SRC - out.printf tmp, t.to_s, t.ident, - symbol_locations(t.locate).join(' '), - symbol_locations(t.heads).join(' ') - end - - out.print "\n**Terminals, with rules where they appear\n\n" - @grammar.each_terminal do |t| - out.printf " %s (%d) %s\n", - t.to_s, t.ident, symbol_locations(t.locate).join(' ') - end - end - - def symbol_locations(locs) - locs.map {|loc| loc.rule.ident }.reject {|n| n == 0 }.uniq - end - - end - -end # module Racc diff --git a/lib/racc/parser-text.rb b/lib/racc/parser-text.rb deleted file mode 100644 index 63ac2e368b..0000000000 --- a/lib/racc/parser-text.rb +++ /dev/null @@ -1,637 +0,0 @@ -module Racc - PARSER_TEXT = <<'__end_of_file__' -# frozen_string_literal: false -#-- -# Copyright (c) 1999-2006 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the same terms of ruby. -# -# As a special exception, when this code is copied by Racc -# into a Racc output file, you may use that output file -# without restriction. -#++ - -require 'racc/info' - -unless defined?(NotImplementedError) - NotImplementedError = NotImplementError # :nodoc: -end - -module Racc - class ParseError < StandardError; end -end -unless defined?(::ParseError) - ParseError = Racc::ParseError # :nodoc: -end - -# Racc is a LALR(1) parser generator. -# It is written in Ruby itself, and generates Ruby programs. -# -# == Command-line Reference -# -# racc [-o<var>filename</var>] [--output-file=<var>filename</var>] -# [-e<var>rubypath</var>] [--executable=<var>rubypath</var>] -# [-v] [--verbose] -# [-O<var>filename</var>] [--log-file=<var>filename</var>] -# [-g] [--debug] -# [-E] [--embedded] -# [-l] [--no-line-convert] -# [-c] [--line-convert-all] -# [-a] [--no-omit-actions] -# [-C] [--check-only] -# [-S] [--output-status] -# [--version] [--copyright] [--help] <var>grammarfile</var> -# -# [+grammarfile+] -# Racc grammar file. Any extension is permitted. -# [-o+outfile+, --output-file=+outfile+] -# A filename for output. default is <+filename+>.tab.rb -# [-O+filename+, --log-file=+filename+] -# Place logging output in file +filename+. -# Default log file name is <+filename+>.output. -# [-e+rubypath+, --executable=+rubypath+] -# output executable file(mode 755). where +path+ is the Ruby interpreter. -# [-v, --verbose] -# verbose mode. create +filename+.output file, like yacc's y.output file. -# [-g, --debug] -# add debug code to parser class. To display debuggin information, -# use this '-g' option and set @yydebug true in parser class. -# [-E, --embedded] -# Output parser which doesn't need runtime files (racc/parser.rb). -# [-C, --check-only] -# Check syntax of racc grammar file and quit. -# [-S, --output-status] -# Print messages time to time while compiling. -# [-l, --no-line-convert] -# turns off line number converting. -# [-c, --line-convert-all] -# Convert line number of actions, inner, header and footer. -# [-a, --no-omit-actions] -# Call all actions, even if an action is empty. -# [--version] -# print Racc version and quit. -# [--copyright] -# Print copyright and quit. -# [--help] -# Print usage and quit. -# -# == Generating Parser Using Racc -# -# To compile Racc grammar file, simply type: -# -# $ racc parse.y -# -# This creates Ruby script file "parse.tab.y". The -o option can change the output filename. -# -# == Writing A Racc Grammar File -# -# If you want your own parser, you have to write a grammar file. -# A grammar file contains the name of your parser class, grammar for the parser, -# user code, and anything else. -# When writing a grammar file, yacc's knowledge is helpful. -# If you have not used yacc before, Racc is not too difficult. -# -# Here's an example Racc grammar file. -# -# class Calcparser -# rule -# target: exp { print val[0] } -# -# exp: exp '+' exp -# | exp '*' exp -# | '(' exp ')' -# | NUMBER -# end -# -# Racc grammar files resemble yacc files. -# But (of course), this is Ruby code. -# yacc's $$ is the 'result', $0, $1... is -# an array called 'val', and $-1, $-2... is an array called '_values'. -# -# See the {Grammar File Reference}[rdoc-ref:lib/racc/rdoc/grammar.en.rdoc] for -# more information on grammar files. -# -# == Parser -# -# Then you must prepare the parse entry method. There are two types of -# parse methods in Racc, Racc::Parser#do_parse and Racc::Parser#yyparse -# -# Racc::Parser#do_parse is simple. -# -# It's yyparse() of yacc, and Racc::Parser#next_token is yylex(). -# This method must returns an array like [TOKENSYMBOL, ITS_VALUE]. -# EOF is [false, false]. -# (TOKENSYMBOL is a Ruby symbol (taken from String#intern) by default. -# If you want to change this, see the grammar reference. -# -# Racc::Parser#yyparse is little complicated, but useful. -# It does not use Racc::Parser#next_token, instead it gets tokens from any iterator. -# -# For example, <code>yyparse(obj, :scan)</code> causes -# calling +obj#scan+, and you can return tokens by yielding them from +obj#scan+. -# -# == Debugging -# -# When debugging, "-v" or/and the "-g" option is helpful. -# -# "-v" creates verbose log file (.output). -# "-g" creates a "Verbose Parser". -# Verbose Parser prints the internal status when parsing. -# But it's _not_ automatic. -# You must use -g option and set +@yydebug+ to +true+ in order to get output. -# -g option only creates the verbose parser. -# -# === Racc reported syntax error. -# -# Isn't there too many "end"? -# grammar of racc file is changed in v0.10. -# -# Racc does not use '%' mark, while yacc uses huge number of '%' marks.. -# -# === Racc reported "XXXX conflicts". -# -# Try "racc -v xxxx.y". -# It causes producing racc's internal log file, xxxx.output. -# -# === Generated parsers does not work correctly -# -# Try "racc -g xxxx.y". -# This command let racc generate "debugging parser". -# Then set @yydebug=true in your parser. -# It produces a working log of your parser. -# -# == Re-distributing Racc runtime -# -# A parser, which is created by Racc, requires the Racc runtime module; -# racc/parser.rb. -# -# Ruby 1.8.x comes with Racc runtime module, -# you need NOT distribute Racc runtime files. -# -# If you want to include the Racc runtime module with your parser. -# This can be done by using '-E' option: -# -# $ racc -E -omyparser.rb myparser.y -# -# This command creates myparser.rb which `includes' Racc runtime. -# Only you must do is to distribute your parser file (myparser.rb). -# -# Note: parser.rb is ruby license, but your parser is not. -# Your own parser is completely yours. -module Racc - - unless defined?(Racc_No_Extensions) - Racc_No_Extensions = false # :nodoc: - end - - class Parser - - Racc_Runtime_Version = ::Racc::VERSION - Racc_Runtime_Core_Version_R = ::Racc::VERSION - - begin - if Object.const_defined?(:RUBY_ENGINE) and RUBY_ENGINE == 'jruby' - require 'jruby' - require 'racc/cparse-jruby.jar' - com.headius.racc.Cparse.new.load(JRuby.runtime, false) - else - require 'racc/cparse' - end - - unless new.respond_to?(:_racc_do_parse_c, true) - raise LoadError, 'old cparse.so' - end - if Racc_No_Extensions - raise LoadError, 'selecting ruby version of racc runtime core' - end - - Racc_Main_Parsing_Routine = :_racc_do_parse_c # :nodoc: - Racc_YY_Parse_Method = :_racc_yyparse_c # :nodoc: - Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_C # :nodoc: - Racc_Runtime_Type = 'c' # :nodoc: - rescue LoadError - Racc_Main_Parsing_Routine = :_racc_do_parse_rb - Racc_YY_Parse_Method = :_racc_yyparse_rb - Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_R - Racc_Runtime_Type = 'ruby' - end - - def Parser.racc_runtime_type # :nodoc: - Racc_Runtime_Type - end - - def _racc_setup - @yydebug = false unless self.class::Racc_debug_parser - @yydebug = false unless defined?(@yydebug) - if @yydebug - @racc_debug_out = $stderr unless defined?(@racc_debug_out) - @racc_debug_out ||= $stderr - end - arg = self.class::Racc_arg - arg[13] = true if arg.size < 14 - arg - end - - def _racc_init_sysvars - @racc_state = [0] - @racc_tstack = [] - @racc_vstack = [] - - @racc_t = nil - @racc_val = nil - - @racc_read_next = true - - @racc_user_yyerror = false - @racc_error_status = 0 - end - - # The entry point of the parser. This method is used with #next_token. - # If Racc wants to get token (and its value), calls next_token. - # - # Example: - # def parse - # @q = [[1,1], - # [2,2], - # [3,3], - # [false, '$']] - # do_parse - # end - # - # def next_token - # @q.shift - # end - class_eval <<~RUBY, __FILE__, __LINE__ + 1 - def do_parse - #{Racc_Main_Parsing_Routine}(_racc_setup(), false) - end - RUBY - - # The method to fetch next token. - # If you use #do_parse method, you must implement #next_token. - # - # The format of return value is [TOKEN_SYMBOL, VALUE]. - # +token-symbol+ is represented by Ruby's symbol by default, e.g. :IDENT - # for 'IDENT'. ";" (String) for ';'. - # - # The final symbol (End of file) must be false. - def next_token - raise NotImplementedError, "#{self.class}\#next_token is not defined" - end - - def _racc_do_parse_rb(arg, in_debug) - action_table, action_check, action_default, action_pointer, - _, _, _, _, - _, _, token_table, * = arg - - _racc_init_sysvars - tok = act = i = nil - - catch(:racc_end_parse) { - while true - if i = action_pointer[@racc_state[-1]] - if @racc_read_next - if @racc_t != 0 # not EOF - tok, @racc_val = next_token() - unless tok # EOF - @racc_t = 0 - else - @racc_t = (token_table[tok] or 1) # error token - end - racc_read_token(@racc_t, tok, @racc_val) if @yydebug - @racc_read_next = false - end - end - i += @racc_t - unless i >= 0 and - act = action_table[i] and - action_check[i] == @racc_state[-1] - act = action_default[@racc_state[-1]] - end - else - act = action_default[@racc_state[-1]] - end - while act = _racc_evalact(act, arg) - ; - end - end - } - end - - # Another entry point for the parser. - # If you use this method, you must implement RECEIVER#METHOD_ID method. - # - # RECEIVER#METHOD_ID is a method to get next token. - # It must 'yield' the token, which format is [TOKEN-SYMBOL, VALUE]. - class_eval <<~RUBY, __FILE__, __LINE__ + 1 - def yyparse(recv, mid) - #{Racc_YY_Parse_Method}(recv, mid, _racc_setup(), false) - end - RUBY - - def _racc_yyparse_rb(recv, mid, arg, c_debug) - action_table, action_check, action_default, action_pointer, - _, _, _, _, - _, _, token_table, * = arg - - _racc_init_sysvars - - catch(:racc_end_parse) { - until i = action_pointer[@racc_state[-1]] - while act = _racc_evalact(action_default[@racc_state[-1]], arg) - ; - end - end - recv.__send__(mid) do |tok, val| - unless tok - @racc_t = 0 - else - @racc_t = (token_table[tok] or 1) # error token - end - @racc_val = val - @racc_read_next = false - - i += @racc_t - unless i >= 0 and - act = action_table[i] and - action_check[i] == @racc_state[-1] - act = action_default[@racc_state[-1]] - end - while act = _racc_evalact(act, arg) - ; - end - - while !(i = action_pointer[@racc_state[-1]]) || - ! @racc_read_next || - @racc_t == 0 # $ - unless i and i += @racc_t and - i >= 0 and - act = action_table[i] and - action_check[i] == @racc_state[-1] - act = action_default[@racc_state[-1]] - end - while act = _racc_evalact(act, arg) - ; - end - end - end - } - end - - ### - ### common - ### - - def _racc_evalact(act, arg) - action_table, action_check, _, action_pointer, - _, _, _, _, - _, _, _, shift_n, - reduce_n, * = arg - nerr = 0 # tmp - - if act > 0 and act < shift_n - # - # shift - # - if @racc_error_status > 0 - @racc_error_status -= 1 unless @racc_t <= 1 # error token or EOF - end - @racc_vstack.push @racc_val - @racc_state.push act - @racc_read_next = true - if @yydebug - @racc_tstack.push @racc_t - racc_shift @racc_t, @racc_tstack, @racc_vstack - end - - elsif act < 0 and act > -reduce_n - # - # reduce - # - code = catch(:racc_jump) { - @racc_state.push _racc_do_reduce(arg, act) - false - } - if code - case code - when 1 # yyerror - @racc_user_yyerror = true # user_yyerror - return -reduce_n - when 2 # yyaccept - return shift_n - else - raise '[Racc Bug] unknown jump code' - end - end - - elsif act == shift_n - # - # accept - # - racc_accept if @yydebug - throw :racc_end_parse, @racc_vstack[0] - - elsif act == -reduce_n - # - # error - # - case @racc_error_status - when 0 - unless arg[21] # user_yyerror - nerr += 1 - on_error @racc_t, @racc_val, @racc_vstack - end - when 3 - if @racc_t == 0 # is $ - # We're at EOF, and another error occurred immediately after - # attempting auto-recovery - throw :racc_end_parse, nil - end - @racc_read_next = true - end - @racc_user_yyerror = false - @racc_error_status = 3 - while true - if i = action_pointer[@racc_state[-1]] - i += 1 # error token - if i >= 0 and - (act = action_table[i]) and - action_check[i] == @racc_state[-1] - break - end - end - throw :racc_end_parse, nil if @racc_state.size <= 1 - @racc_state.pop - @racc_vstack.pop - if @yydebug - @racc_tstack.pop - racc_e_pop @racc_state, @racc_tstack, @racc_vstack - end - end - return act - - else - raise "[Racc Bug] unknown action #{act.inspect}" - end - - racc_next_state(@racc_state[-1], @racc_state) if @yydebug - - nil - end - - def _racc_do_reduce(arg, act) - _, _, _, _, - goto_table, goto_check, goto_default, goto_pointer, - nt_base, reduce_table, _, _, - _, use_result, * = arg - - state = @racc_state - vstack = @racc_vstack - tstack = @racc_tstack - - i = act * -3 - len = reduce_table[i] - reduce_to = reduce_table[i+1] - method_id = reduce_table[i+2] - void_array = [] - - tmp_t = tstack[-len, len] if @yydebug - tmp_v = vstack[-len, len] - tstack[-len, len] = void_array if @yydebug - vstack[-len, len] = void_array - state[-len, len] = void_array - - # tstack must be updated AFTER method call - if use_result - vstack.push __send__(method_id, tmp_v, vstack, tmp_v[0]) - else - vstack.push __send__(method_id, tmp_v, vstack) - end - tstack.push reduce_to - - racc_reduce(tmp_t, reduce_to, tstack, vstack) if @yydebug - - k1 = reduce_to - nt_base - if i = goto_pointer[k1] - i += state[-1] - if i >= 0 and (curstate = goto_table[i]) and goto_check[i] == k1 - return curstate - end - end - goto_default[k1] - end - - # This method is called when a parse error is found. - # - # ERROR_TOKEN_ID is an internal ID of token which caused error. - # You can get string representation of this ID by calling - # #token_to_str. - # - # ERROR_VALUE is a value of error token. - # - # value_stack is a stack of symbol values. - # DO NOT MODIFY this object. - # - # This method raises ParseError by default. - # - # If this method returns, parsers enter "error recovering mode". - def on_error(t, val, vstack) - raise ParseError, sprintf("\nparse error on value %s (%s)", - val.inspect, token_to_str(t) || '?') - end - - # Enter error recovering mode. - # This method does not call #on_error. - def yyerror - throw :racc_jump, 1 - end - - # Exit parser. - # Return value is +Symbol_Value_Stack[0]+. - def yyaccept - throw :racc_jump, 2 - end - - # Leave error recovering mode. - def yyerrok - @racc_error_status = 0 - end - - # For debugging output - def racc_read_token(t, tok, val) - @racc_debug_out.print 'read ' - @racc_debug_out.print tok.inspect, '(', racc_token2str(t), ') ' - @racc_debug_out.puts val.inspect - @racc_debug_out.puts - end - - def racc_shift(tok, tstack, vstack) - @racc_debug_out.puts "shift #{racc_token2str tok}" - racc_print_stacks tstack, vstack - @racc_debug_out.puts - end - - def racc_reduce(toks, sim, tstack, vstack) - out = @racc_debug_out - out.print 'reduce ' - if toks.empty? - out.print ' <none>' - else - toks.each {|t| out.print ' ', racc_token2str(t) } - end - out.puts " --> #{racc_token2str(sim)}" - racc_print_stacks tstack, vstack - @racc_debug_out.puts - end - - def racc_accept - @racc_debug_out.puts 'accept' - @racc_debug_out.puts - end - - def racc_e_pop(state, tstack, vstack) - @racc_debug_out.puts 'error recovering mode: pop token' - racc_print_states state - racc_print_stacks tstack, vstack - @racc_debug_out.puts - end - - def racc_next_state(curstate, state) - @racc_debug_out.puts "goto #{curstate}" - racc_print_states state - @racc_debug_out.puts - end - - def racc_print_stacks(t, v) - out = @racc_debug_out - out.print ' [' - t.each_index do |i| - out.print ' (', racc_token2str(t[i]), ' ', v[i].inspect, ')' - end - out.puts ' ]' - end - - def racc_print_states(s) - out = @racc_debug_out - out.print ' [' - s.each {|st| out.print ' ', st } - out.puts ' ]' - end - - def racc_token2str(tok) - self.class::Racc_token_to_s_table[tok] or - raise "[Racc Bug] can't convert token #{tok} to string" - end - - # Convert internal ID of token symbol to the string. - def token_to_str(t) - self.class::Racc_token_to_s_table[t] - end - - end - -end - -__end_of_file__ -end diff --git a/lib/racc/parser.rb b/lib/racc/parser.rb deleted file mode 100644 index f2cc050826..0000000000 --- a/lib/racc/parser.rb +++ /dev/null @@ -1,632 +0,0 @@ -# frozen_string_literal: false -#-- -# Copyright (c) 1999-2006 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the same terms of ruby. -# -# As a special exception, when this code is copied by Racc -# into a Racc output file, you may use that output file -# without restriction. -#++ - -require 'racc/info' - -unless defined?(NotImplementedError) - NotImplementedError = NotImplementError # :nodoc: -end - -module Racc - class ParseError < StandardError; end -end -unless defined?(::ParseError) - ParseError = Racc::ParseError # :nodoc: -end - -# Racc is a LALR(1) parser generator. -# It is written in Ruby itself, and generates Ruby programs. -# -# == Command-line Reference -# -# racc [-o<var>filename</var>] [--output-file=<var>filename</var>] -# [-e<var>rubypath</var>] [--executable=<var>rubypath</var>] -# [-v] [--verbose] -# [-O<var>filename</var>] [--log-file=<var>filename</var>] -# [-g] [--debug] -# [-E] [--embedded] -# [-l] [--no-line-convert] -# [-c] [--line-convert-all] -# [-a] [--no-omit-actions] -# [-C] [--check-only] -# [-S] [--output-status] -# [--version] [--copyright] [--help] <var>grammarfile</var> -# -# [+grammarfile+] -# Racc grammar file. Any extension is permitted. -# [-o+outfile+, --output-file=+outfile+] -# A filename for output. default is <+filename+>.tab.rb -# [-O+filename+, --log-file=+filename+] -# Place logging output in file +filename+. -# Default log file name is <+filename+>.output. -# [-e+rubypath+, --executable=+rubypath+] -# output executable file(mode 755). where +path+ is the Ruby interpreter. -# [-v, --verbose] -# verbose mode. create +filename+.output file, like yacc's y.output file. -# [-g, --debug] -# add debug code to parser class. To display debuggin information, -# use this '-g' option and set @yydebug true in parser class. -# [-E, --embedded] -# Output parser which doesn't need runtime files (racc/parser.rb). -# [-C, --check-only] -# Check syntax of racc grammar file and quit. -# [-S, --output-status] -# Print messages time to time while compiling. -# [-l, --no-line-convert] -# turns off line number converting. -# [-c, --line-convert-all] -# Convert line number of actions, inner, header and footer. -# [-a, --no-omit-actions] -# Call all actions, even if an action is empty. -# [--version] -# print Racc version and quit. -# [--copyright] -# Print copyright and quit. -# [--help] -# Print usage and quit. -# -# == Generating Parser Using Racc -# -# To compile Racc grammar file, simply type: -# -# $ racc parse.y -# -# This creates Ruby script file "parse.tab.y". The -o option can change the output filename. -# -# == Writing A Racc Grammar File -# -# If you want your own parser, you have to write a grammar file. -# A grammar file contains the name of your parser class, grammar for the parser, -# user code, and anything else. -# When writing a grammar file, yacc's knowledge is helpful. -# If you have not used yacc before, Racc is not too difficult. -# -# Here's an example Racc grammar file. -# -# class Calcparser -# rule -# target: exp { print val[0] } -# -# exp: exp '+' exp -# | exp '*' exp -# | '(' exp ')' -# | NUMBER -# end -# -# Racc grammar files resemble yacc files. -# But (of course), this is Ruby code. -# yacc's $$ is the 'result', $0, $1... is -# an array called 'val', and $-1, $-2... is an array called '_values'. -# -# See the {Grammar File Reference}[rdoc-ref:lib/racc/rdoc/grammar.en.rdoc] for -# more information on grammar files. -# -# == Parser -# -# Then you must prepare the parse entry method. There are two types of -# parse methods in Racc, Racc::Parser#do_parse and Racc::Parser#yyparse -# -# Racc::Parser#do_parse is simple. -# -# It's yyparse() of yacc, and Racc::Parser#next_token is yylex(). -# This method must returns an array like [TOKENSYMBOL, ITS_VALUE]. -# EOF is [false, false]. -# (TOKENSYMBOL is a Ruby symbol (taken from String#intern) by default. -# If you want to change this, see the grammar reference. -# -# Racc::Parser#yyparse is little complicated, but useful. -# It does not use Racc::Parser#next_token, instead it gets tokens from any iterator. -# -# For example, <code>yyparse(obj, :scan)</code> causes -# calling +obj#scan+, and you can return tokens by yielding them from +obj#scan+. -# -# == Debugging -# -# When debugging, "-v" or/and the "-g" option is helpful. -# -# "-v" creates verbose log file (.output). -# "-g" creates a "Verbose Parser". -# Verbose Parser prints the internal status when parsing. -# But it's _not_ automatic. -# You must use -g option and set +@yydebug+ to +true+ in order to get output. -# -g option only creates the verbose parser. -# -# === Racc reported syntax error. -# -# Isn't there too many "end"? -# grammar of racc file is changed in v0.10. -# -# Racc does not use '%' mark, while yacc uses huge number of '%' marks.. -# -# === Racc reported "XXXX conflicts". -# -# Try "racc -v xxxx.y". -# It causes producing racc's internal log file, xxxx.output. -# -# === Generated parsers does not work correctly -# -# Try "racc -g xxxx.y". -# This command let racc generate "debugging parser". -# Then set @yydebug=true in your parser. -# It produces a working log of your parser. -# -# == Re-distributing Racc runtime -# -# A parser, which is created by Racc, requires the Racc runtime module; -# racc/parser.rb. -# -# Ruby 1.8.x comes with Racc runtime module, -# you need NOT distribute Racc runtime files. -# -# If you want to include the Racc runtime module with your parser. -# This can be done by using '-E' option: -# -# $ racc -E -omyparser.rb myparser.y -# -# This command creates myparser.rb which `includes' Racc runtime. -# Only you must do is to distribute your parser file (myparser.rb). -# -# Note: parser.rb is ruby license, but your parser is not. -# Your own parser is completely yours. -module Racc - - unless defined?(Racc_No_Extensions) - Racc_No_Extensions = false # :nodoc: - end - - class Parser - - Racc_Runtime_Version = ::Racc::VERSION - Racc_Runtime_Core_Version_R = ::Racc::VERSION - - begin - if Object.const_defined?(:RUBY_ENGINE) and RUBY_ENGINE == 'jruby' - require 'jruby' - require 'racc/cparse-jruby.jar' - com.headius.racc.Cparse.new.load(JRuby.runtime, false) - else - require 'racc/cparse' - end - - unless new.respond_to?(:_racc_do_parse_c, true) - raise LoadError, 'old cparse.so' - end - if Racc_No_Extensions - raise LoadError, 'selecting ruby version of racc runtime core' - end - - Racc_Main_Parsing_Routine = :_racc_do_parse_c # :nodoc: - Racc_YY_Parse_Method = :_racc_yyparse_c # :nodoc: - Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_C # :nodoc: - Racc_Runtime_Type = 'c' # :nodoc: - rescue LoadError - Racc_Main_Parsing_Routine = :_racc_do_parse_rb - Racc_YY_Parse_Method = :_racc_yyparse_rb - Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_R - Racc_Runtime_Type = 'ruby' - end - - def Parser.racc_runtime_type # :nodoc: - Racc_Runtime_Type - end - - def _racc_setup - @yydebug = false unless self.class::Racc_debug_parser - @yydebug = false unless defined?(@yydebug) - if @yydebug - @racc_debug_out = $stderr unless defined?(@racc_debug_out) - @racc_debug_out ||= $stderr - end - arg = self.class::Racc_arg - arg[13] = true if arg.size < 14 - arg - end - - def _racc_init_sysvars - @racc_state = [0] - @racc_tstack = [] - @racc_vstack = [] - - @racc_t = nil - @racc_val = nil - - @racc_read_next = true - - @racc_user_yyerror = false - @racc_error_status = 0 - end - - # The entry point of the parser. This method is used with #next_token. - # If Racc wants to get token (and its value), calls next_token. - # - # Example: - # def parse - # @q = [[1,1], - # [2,2], - # [3,3], - # [false, '$']] - # do_parse - # end - # - # def next_token - # @q.shift - # end - class_eval <<~RUBY, __FILE__, __LINE__ + 1 - def do_parse - #{Racc_Main_Parsing_Routine}(_racc_setup(), false) - end - RUBY - - # The method to fetch next token. - # If you use #do_parse method, you must implement #next_token. - # - # The format of return value is [TOKEN_SYMBOL, VALUE]. - # +token-symbol+ is represented by Ruby's symbol by default, e.g. :IDENT - # for 'IDENT'. ";" (String) for ';'. - # - # The final symbol (End of file) must be false. - def next_token - raise NotImplementedError, "#{self.class}\#next_token is not defined" - end - - def _racc_do_parse_rb(arg, in_debug) - action_table, action_check, action_default, action_pointer, - _, _, _, _, - _, _, token_table, * = arg - - _racc_init_sysvars - tok = act = i = nil - - catch(:racc_end_parse) { - while true - if i = action_pointer[@racc_state[-1]] - if @racc_read_next - if @racc_t != 0 # not EOF - tok, @racc_val = next_token() - unless tok # EOF - @racc_t = 0 - else - @racc_t = (token_table[tok] or 1) # error token - end - racc_read_token(@racc_t, tok, @racc_val) if @yydebug - @racc_read_next = false - end - end - i += @racc_t - unless i >= 0 and - act = action_table[i] and - action_check[i] == @racc_state[-1] - act = action_default[@racc_state[-1]] - end - else - act = action_default[@racc_state[-1]] - end - while act = _racc_evalact(act, arg) - ; - end - end - } - end - - # Another entry point for the parser. - # If you use this method, you must implement RECEIVER#METHOD_ID method. - # - # RECEIVER#METHOD_ID is a method to get next token. - # It must 'yield' the token, which format is [TOKEN-SYMBOL, VALUE]. - class_eval <<~RUBY, __FILE__, __LINE__ + 1 - def yyparse(recv, mid) - #{Racc_YY_Parse_Method}(recv, mid, _racc_setup(), false) - end - RUBY - - def _racc_yyparse_rb(recv, mid, arg, c_debug) - action_table, action_check, action_default, action_pointer, - _, _, _, _, - _, _, token_table, * = arg - - _racc_init_sysvars - - catch(:racc_end_parse) { - until i = action_pointer[@racc_state[-1]] - while act = _racc_evalact(action_default[@racc_state[-1]], arg) - ; - end - end - recv.__send__(mid) do |tok, val| - unless tok - @racc_t = 0 - else - @racc_t = (token_table[tok] or 1) # error token - end - @racc_val = val - @racc_read_next = false - - i += @racc_t - unless i >= 0 and - act = action_table[i] and - action_check[i] == @racc_state[-1] - act = action_default[@racc_state[-1]] - end - while act = _racc_evalact(act, arg) - ; - end - - while !(i = action_pointer[@racc_state[-1]]) || - ! @racc_read_next || - @racc_t == 0 # $ - unless i and i += @racc_t and - i >= 0 and - act = action_table[i] and - action_check[i] == @racc_state[-1] - act = action_default[@racc_state[-1]] - end - while act = _racc_evalact(act, arg) - ; - end - end - end - } - end - - ### - ### common - ### - - def _racc_evalact(act, arg) - action_table, action_check, _, action_pointer, - _, _, _, _, - _, _, _, shift_n, - reduce_n, * = arg - nerr = 0 # tmp - - if act > 0 and act < shift_n - # - # shift - # - if @racc_error_status > 0 - @racc_error_status -= 1 unless @racc_t <= 1 # error token or EOF - end - @racc_vstack.push @racc_val - @racc_state.push act - @racc_read_next = true - if @yydebug - @racc_tstack.push @racc_t - racc_shift @racc_t, @racc_tstack, @racc_vstack - end - - elsif act < 0 and act > -reduce_n - # - # reduce - # - code = catch(:racc_jump) { - @racc_state.push _racc_do_reduce(arg, act) - false - } - if code - case code - when 1 # yyerror - @racc_user_yyerror = true # user_yyerror - return -reduce_n - when 2 # yyaccept - return shift_n - else - raise '[Racc Bug] unknown jump code' - end - end - - elsif act == shift_n - # - # accept - # - racc_accept if @yydebug - throw :racc_end_parse, @racc_vstack[0] - - elsif act == -reduce_n - # - # error - # - case @racc_error_status - when 0 - unless arg[21] # user_yyerror - nerr += 1 - on_error @racc_t, @racc_val, @racc_vstack - end - when 3 - if @racc_t == 0 # is $ - # We're at EOF, and another error occurred immediately after - # attempting auto-recovery - throw :racc_end_parse, nil - end - @racc_read_next = true - end - @racc_user_yyerror = false - @racc_error_status = 3 - while true - if i = action_pointer[@racc_state[-1]] - i += 1 # error token - if i >= 0 and - (act = action_table[i]) and - action_check[i] == @racc_state[-1] - break - end - end - throw :racc_end_parse, nil if @racc_state.size <= 1 - @racc_state.pop - @racc_vstack.pop - if @yydebug - @racc_tstack.pop - racc_e_pop @racc_state, @racc_tstack, @racc_vstack - end - end - return act - - else - raise "[Racc Bug] unknown action #{act.inspect}" - end - - racc_next_state(@racc_state[-1], @racc_state) if @yydebug - - nil - end - - def _racc_do_reduce(arg, act) - _, _, _, _, - goto_table, goto_check, goto_default, goto_pointer, - nt_base, reduce_table, _, _, - _, use_result, * = arg - - state = @racc_state - vstack = @racc_vstack - tstack = @racc_tstack - - i = act * -3 - len = reduce_table[i] - reduce_to = reduce_table[i+1] - method_id = reduce_table[i+2] - void_array = [] - - tmp_t = tstack[-len, len] if @yydebug - tmp_v = vstack[-len, len] - tstack[-len, len] = void_array if @yydebug - vstack[-len, len] = void_array - state[-len, len] = void_array - - # tstack must be updated AFTER method call - if use_result - vstack.push __send__(method_id, tmp_v, vstack, tmp_v[0]) - else - vstack.push __send__(method_id, tmp_v, vstack) - end - tstack.push reduce_to - - racc_reduce(tmp_t, reduce_to, tstack, vstack) if @yydebug - - k1 = reduce_to - nt_base - if i = goto_pointer[k1] - i += state[-1] - if i >= 0 and (curstate = goto_table[i]) and goto_check[i] == k1 - return curstate - end - end - goto_default[k1] - end - - # This method is called when a parse error is found. - # - # ERROR_TOKEN_ID is an internal ID of token which caused error. - # You can get string representation of this ID by calling - # #token_to_str. - # - # ERROR_VALUE is a value of error token. - # - # value_stack is a stack of symbol values. - # DO NOT MODIFY this object. - # - # This method raises ParseError by default. - # - # If this method returns, parsers enter "error recovering mode". - def on_error(t, val, vstack) - raise ParseError, sprintf("\nparse error on value %s (%s)", - val.inspect, token_to_str(t) || '?') - end - - # Enter error recovering mode. - # This method does not call #on_error. - def yyerror - throw :racc_jump, 1 - end - - # Exit parser. - # Return value is +Symbol_Value_Stack[0]+. - def yyaccept - throw :racc_jump, 2 - end - - # Leave error recovering mode. - def yyerrok - @racc_error_status = 0 - end - - # For debugging output - def racc_read_token(t, tok, val) - @racc_debug_out.print 'read ' - @racc_debug_out.print tok.inspect, '(', racc_token2str(t), ') ' - @racc_debug_out.puts val.inspect - @racc_debug_out.puts - end - - def racc_shift(tok, tstack, vstack) - @racc_debug_out.puts "shift #{racc_token2str tok}" - racc_print_stacks tstack, vstack - @racc_debug_out.puts - end - - def racc_reduce(toks, sim, tstack, vstack) - out = @racc_debug_out - out.print 'reduce ' - if toks.empty? - out.print ' <none>' - else - toks.each {|t| out.print ' ', racc_token2str(t) } - end - out.puts " --> #{racc_token2str(sim)}" - racc_print_stacks tstack, vstack - @racc_debug_out.puts - end - - def racc_accept - @racc_debug_out.puts 'accept' - @racc_debug_out.puts - end - - def racc_e_pop(state, tstack, vstack) - @racc_debug_out.puts 'error recovering mode: pop token' - racc_print_states state - racc_print_stacks tstack, vstack - @racc_debug_out.puts - end - - def racc_next_state(curstate, state) - @racc_debug_out.puts "goto #{curstate}" - racc_print_states state - @racc_debug_out.puts - end - - def racc_print_stacks(t, v) - out = @racc_debug_out - out.print ' [' - t.each_index do |i| - out.print ' (', racc_token2str(t[i]), ' ', v[i].inspect, ')' - end - out.puts ' ]' - end - - def racc_print_states(s) - out = @racc_debug_out - out.print ' [' - s.each {|st| out.print ' ', st } - out.puts ' ]' - end - - def racc_token2str(tok) - self.class::Racc_token_to_s_table[tok] or - raise "[Racc Bug] can't convert token #{tok} to string" - end - - # Convert internal ID of token symbol to the string. - def token_to_str(t) - self.class::Racc_token_to_s_table[t] - end - - end - -end diff --git a/lib/racc/parserfilegenerator.rb b/lib/racc/parserfilegenerator.rb deleted file mode 100644 index 7846b584fa..0000000000 --- a/lib/racc/parserfilegenerator.rb +++ /dev/null @@ -1,470 +0,0 @@ -#-- -# -# -# -# Copyright (c) 1999-2006 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the same terms of ruby. -# see the file "COPYING". -# -#++ - -require 'racc/compat' -require 'racc/sourcetext' -require 'racc/parser-text' -require 'rbconfig' - -module Racc - - class ParserFileGenerator - - class Params - def self.bool_attr(name) - module_eval(<<-End) - def #{name}? - @#{name} - end - - def #{name}=(b) - @#{name} = b - end - End - end - - attr_accessor :filename - attr_accessor :classname - attr_accessor :superclass - bool_attr :omit_action_call - bool_attr :result_var - attr_accessor :header - attr_accessor :inner - attr_accessor :footer - - bool_attr :debug_parser - bool_attr :convert_line - bool_attr :convert_line_all - bool_attr :embed_runtime - bool_attr :make_executable - attr_accessor :interpreter - - def initialize - # Parameters derived from parser - self.filename = nil - self.classname = nil - self.superclass = 'Racc::Parser' - self.omit_action_call = true - self.result_var = true - self.header = [] - self.inner = [] - self.footer = [] - - # Parameters derived from command line options - self.debug_parser = false - self.convert_line = true - self.convert_line_all = false - self.embed_runtime = false - self.make_executable = false - self.interpreter = nil - end - end - - def initialize(states, params) - @states = states - @grammar = states.grammar - @params = params - end - - def generate_parser - string_io = StringIO.new - - init_line_conversion_system - @f = string_io - parser_file - - string_io.rewind - string_io.read - end - - def generate_parser_file(destpath) - init_line_conversion_system - File.open(destpath, 'w') {|f| - @f = f - parser_file - } - File.chmod 0755, destpath if @params.make_executable? - end - - private - - def parser_file - shebang @params.interpreter if @params.make_executable? - notice - line - if @params.embed_runtime? - embed_library runtime_source() - else - require 'racc/parser.rb' - end - header - parser_class(@params.classname, @params.superclass) { - inner - state_transition_table - } - footer - end - - c = ::RbConfig::CONFIG - RUBY_PATH = "#{c['bindir']}/#{c['ruby_install_name']}#{c['EXEEXT']}" - - def shebang(path) - line '#!' + (path == 'ruby' ? RUBY_PATH : path) - end - - def notice - line %q[#] - line %q[# DO NOT MODIFY!!!!] - line %Q[# This file is automatically generated by Racc #{Racc::Version}] - line %Q[# from Racc grammar file "#{@params.filename}".] - line %q[#] - end - - def runtime_source - SourceText.new(::Racc::PARSER_TEXT, 'racc/parser.rb', 1) - end - - def embed_library(src) - line %[###### #{src.filename} begin] - line %[unless $".index '#{src.filename}'] - line %[$".push '#{src.filename}'] - put src, @params.convert_line? - line %[end] - line %[###### #{src.filename} end] - end - - def require(feature) - line "require '#{feature}'" - end - - def parser_class(classname, superclass) - mods = classname.split('::') - classid = mods.pop - mods.each do |mod| - indent; line "module #{mod}" - cref_push mod - end - indent; line "class #{classid} < #{superclass}" - cref_push classid - yield - cref_pop - indent; line "end \# class #{classid}" - mods.reverse_each do |mod| - cref_pop - indent; line "end \# module #{mod}" - end - end - - def header - @params.header.each do |src| - line - put src, @params.convert_line_all? - end - end - - def inner - @params.inner.each do |src| - line - put src, @params.convert_line? - end - end - - def footer - @params.footer.each do |src| - line - put src, @params.convert_line_all? - end - end - - # Low Level Routines - - def put(src, convert_line = false) - if convert_line - replace_location(src) { - @f.puts src.text - } - else - @f.puts src.text - end - end - - def line(str = '') - @f.puts str - end - - def init_line_conversion_system - @cref = [] - @used_separator = {} - end - - def cref_push(name) - @cref.push name - end - - def cref_pop - @cref.pop - end - - def indent - @f.print ' ' * @cref.size - end - - def toplevel? - @cref.empty? - end - - def replace_location(src) - sep = make_separator(src) - @f.print 'self.class.' if toplevel? - @f.puts "module_eval(<<'#{sep}', '#{src.filename}', #{src.lineno})" - yield - @f.puts sep - end - - def make_separator(src) - sep = unique_separator(src.filename) - sep *= 2 while src.text.index(sep) - sep - end - - def unique_separator(id) - sep = String.new "...end #{id}/module_eval..." - while @used_separator.key?(sep) - sep.concat sprintf('%02x', rand(255)) - end - @used_separator[sep] = true - sep - end - - # - # State Transition Table Serialization - # - - public - - def put_state_transition_table(f) - @f = f - state_transition_table - end - - private - - def state_transition_table - table = @states.state_transition_table - table.use_result_var = @params.result_var? - table.debug_parser = @params.debug_parser? - - line "##### State transition tables begin ###" - line - integer_list 'racc_action_table', table.action_table - line - integer_list 'racc_action_check', table.action_check - line - integer_list 'racc_action_pointer', table.action_pointer - line - integer_list 'racc_action_default', table.action_default - line - integer_list 'racc_goto_table', table.goto_table - line - integer_list 'racc_goto_check', table.goto_check - line - integer_list 'racc_goto_pointer', table.goto_pointer - line - integer_list 'racc_goto_default', table.goto_default - line - i_i_sym_list 'racc_reduce_table', table.reduce_table - line - line "racc_reduce_n = #{table.reduce_n}" - line - line "racc_shift_n = #{table.shift_n}" - line - sym_int_hash 'racc_token_table', table.token_table - line - line "racc_nt_base = #{table.nt_base}" - line - line "racc_use_result_var = #{table.use_result_var}" - line - @f.print(unindent_auto(<<-End)) - Racc_arg = [ - racc_action_table, - racc_action_check, - racc_action_default, - racc_action_pointer, - racc_goto_table, - racc_goto_check, - racc_goto_default, - racc_goto_pointer, - racc_nt_base, - racc_reduce_table, - racc_token_table, - racc_shift_n, - racc_reduce_n, - racc_use_result_var ] - End - line "Ractor.make_shareable(Racc_arg) if defined?(Ractor)" - line - string_list 'Racc_token_to_s_table', table.token_to_s_table - line "Ractor.make_shareable(Racc_token_to_s_table) if defined?(Ractor)" - line - line "Racc_debug_parser = #{table.debug_parser}" - line - line '##### State transition tables end #####' - actions - end - - def integer_list(name, table) - sep = '' - line "#{name} = [" - table.each_slice(10) do |ns| - @f.print sep; sep = ",\n" - @f.print ns.map {|n| sprintf('%6s', n ? n.to_s : 'nil') }.join(',') - end - line ' ]' - end - - def i_i_sym_list(name, table) - sep = '' - line "#{name} = [" - table.each_slice(3) do |len, target, mid| - @f.print sep; sep = ",\n" - @f.printf ' %d, %d, %s', len, target, mid.inspect - end - line " ]" - end - - def sym_int_hash(name, h) - sep = "\n" - @f.print "#{name} = {" - h.to_a.sort_by {|sym, i| i }.each do |sym, i| - @f.print sep; sep = ",\n" - @f.printf " %s => %d", sym.serialize, i - end - line " }" - end - - def string_list(name, list) - sep = " " - line "#{name} = [" - list.each do |s| - @f.print sep; sep = ",\n " - @f.print s.dump - end - line ' ]' - end - - def actions - @grammar.each do |rule| - unless rule.action.source? - raise "racc: fatal: cannot generate parser file when any action is a Proc" - end - end - - if @params.result_var? - decl = ', result' - retval = "\n result" - default_body = '' - else - decl = '' - retval = '' - default_body = 'val[0]' - end - @grammar.each do |rule| - line - if rule.action.empty? and @params.omit_action_call? - line "# reduce #{rule.ident} omitted" - else - src0 = rule.action.source || SourceText.new(default_body, __FILE__, 0) - if @params.convert_line? - src = remove_blank_lines(src0) - delim = make_delimiter(src.text) - @f.printf unindent_auto(<<-End), - module_eval(<<'%s', '%s', %d) - def _reduce_%d(val, _values%s) - %s%s - end - %s - End - delim, src.filename, src.lineno - 1, - rule.ident, decl, - src.text, retval, - delim - else - src = remove_blank_lines(src0) - @f.printf unindent_auto(<<-End), - def _reduce_%d(val, _values%s) - %s%s - end - End - rule.ident, decl, - src.text, retval - end - end - end - line - @f.printf unindent_auto(<<-'End'), decl - def _reduce_none(val, _values%s) - val[0] - end - End - line - end - - def remove_blank_lines(src) - body = src.text.dup - line = src.lineno - while body.slice!(/\A[ \t\f]*(?:\n|\r\n|\r)/) - line += 1 - end - SourceText.new(body, src.filename, line) - end - - def make_delimiter(body) - delim = '.,.,' - while body.index(delim) - delim *= 2 - end - delim - end - - def unindent_auto(str) - lines = str.lines.to_a - n = minimum_indent(lines) - lines.map {|line| detab(line).sub(indent_re(n), '').rstrip + "\n" }.join('') - end - - def minimum_indent(lines) - lines.map {|line| n_indent(line) }.min - end - - def n_indent(line) - line.slice(/\A\s+/).size - end - - RE_CACHE = {} - - def indent_re(n) - RE_CACHE[n] ||= /\A {#{n}}/ - end - - def detab(str, ts = 8) - add = 0 - len = nil - str.gsub(/\t/) { - len = ts - ($`.size + add) % ts - add += len - 1 - ' ' * len - } - end - - end - -end diff --git a/lib/racc/racc.gemspec b/lib/racc/racc.gemspec deleted file mode 100644 index 1095c8f47e..0000000000 --- a/lib/racc/racc.gemspec +++ /dev/null @@ -1,58 +0,0 @@ -# -*- encoding: utf-8 -*- - -begin - require_relative "lib/racc/info" -rescue LoadError # Fallback to load version file in ruby core repository - require_relative "info" -end - -Gem::Specification.new do |s| - s.name = "racc" - s.version = Racc::VERSION - s.summary = "Racc is a LALR(1) parser generator" - s.description = <<DESC -Racc is a LALR(1) parser generator. - It is written in Ruby itself, and generates Ruby program. - - NOTE: Ruby 1.8.x comes with Racc runtime module. You - can run your parsers generated by racc 1.4.x out of the - box. -DESC - s.authors = ["Minero Aoki", "Aaron Patterson"] - s.email = [nil, "aaron@tenderlovemaking.com"] - s.homepage = "https://github.com/ruby/racc" - s.licenses = ["Ruby", "BSD-2-Clause"] - s.executables = ["racc"] - s.files = [ - "COPYING", "ChangeLog", "TODO", - "README.ja.rdoc", "README.rdoc", "bin/racc", - "ext/racc/MANIFEST", - "ext/racc/cparse/cparse.c", - "ext/racc/cparse/extconf.rb", - "lib/racc.rb", "lib/racc/compat.rb", - "lib/racc/debugflags.rb", "lib/racc/exception.rb", - "lib/racc/grammar.rb", "lib/racc/grammarfileparser.rb", - "lib/racc/info.rb", "lib/racc/iset.rb", - "lib/racc/logfilegenerator.rb", "lib/racc/parser-text.rb", - "lib/racc/parser.rb", "lib/racc/parserfilegenerator.rb", - "lib/racc/sourcetext.rb", - "lib/racc/state.rb", "lib/racc/statetransitiontable.rb", - "lib/racc/static.rb", - "doc/en/NEWS.en.rdoc", "doc/en/grammar2.en.rdoc", - "doc/en/grammar.en.rdoc", "doc/ja/NEWS.ja.rdoc", - "doc/ja/command.ja.html", "doc/ja/debug.ja.rdoc", - "doc/ja/grammar.ja.rdoc", "doc/ja/index.ja.html", - "doc/ja/parser.ja.rdoc", "doc/ja/usage.ja.html", - ] - s.require_paths = ["lib"] - s.required_ruby_version = ">= 2.5" - s.rdoc_options = ["--main", "README.rdoc"] - s.extra_rdoc_files = ["README.ja.rdoc", "README.rdoc"] - - if RUBY_PLATFORM =~ /java/ - s.files << 'lib/racc/cparse-jruby.jar' - s.platform = 'java' - else - s.extensions = ["ext/racc/cparse/extconf.rb"] - end -end diff --git a/lib/racc/sourcetext.rb b/lib/racc/sourcetext.rb deleted file mode 100644 index de52dcae9b..0000000000 --- a/lib/racc/sourcetext.rb +++ /dev/null @@ -1,35 +0,0 @@ -#-- -# -# -# -# Copyright (c) 1999-2006 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the same terms of ruby. -# see the file "COPYING". -# -#++ - -module Racc - - class SourceText - def initialize(text, filename, lineno) - @text = text - @filename = filename - @lineno = lineno - end - - attr_reader :text - attr_reader :filename - attr_reader :lineno - - def to_s - "#<SourceText #{location()}>" - end - - def location - "#{@filename}:#{@lineno}" - end - end - -end diff --git a/lib/racc/state.rb b/lib/racc/state.rb deleted file mode 100644 index f85809fbeb..0000000000 --- a/lib/racc/state.rb +++ /dev/null @@ -1,972 +0,0 @@ -#-- -# -# -# -# Copyright (c) 1999-2006 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the same terms of ruby. -# see the file "COPYING". -# -#++ - -require 'racc/iset' -require 'racc/statetransitiontable' -require 'racc/exception' -require 'forwardable' - -module Racc - - # A table of LALR states. - class States - - include Enumerable - - def initialize(grammar, debug_flags = DebugFlags.new) - @grammar = grammar - @symboltable = grammar.symboltable - @d_state = debug_flags.state - @d_la = debug_flags.la - @d_prec = debug_flags.prec - @states = [] - @statecache = {} - @actions = ActionTable.new(@grammar, self) - @nfa_computed = false - @dfa_computed = false - end - - attr_reader :grammar - attr_reader :actions - - def size - @states.size - end - - def inspect - '#<state table>' - end - - alias to_s inspect - - def [](i) - @states[i] - end - - def each_state(&block) - @states.each(&block) - end - - alias each each_state - - def each_index(&block) - @states.each_index(&block) - end - - extend Forwardable - - def_delegator "@actions", :shift_n - def_delegator "@actions", :reduce_n - def_delegator "@actions", :nt_base - - def should_report_srconflict? - srconflict_exist? and - (n_srconflicts() != @grammar.n_expected_srconflicts) - end - - def srconflict_exist? - n_srconflicts() != 0 - end - - def n_srconflicts - @n_srconflicts ||= inject(0) {|sum, st| sum + st.n_srconflicts } - end - - def rrconflict_exist? - n_rrconflicts() != 0 - end - - def n_rrconflicts - @n_rrconflicts ||= inject(0) {|sum, st| sum + st.n_rrconflicts } - end - - def state_transition_table - @state_transition_table ||= StateTransitionTable.generate(self.dfa) - end - - # - # NFA (Non-deterministic Finite Automaton) Computation - # - - public - - def nfa - return self if @nfa_computed - compute_nfa - @nfa_computed = true - self - end - - private - - def compute_nfa - @grammar.init - # add state 0 - core_to_state [ @grammar[0].ptrs[0] ] - # generate LALR states - cur = 0 - @gotos = [] - while cur < @states.size - generate_states @states[cur] # state is added here - cur += 1 - end - @actions.init - end - - def generate_states(state) - puts "dstate: #{state}" if @d_state - - table = {} - state.closure.each do |ptr| - if sym = ptr.dereference - addsym table, sym, ptr.next - end - end - table.each do |sym, core| - puts "dstate: sym=#{sym} ncore=#{core}" if @d_state - - dest = core_to_state(core.to_a) - state.goto_table[sym] = dest - id = sym.nonterminal?() ? @gotos.size : nil - g = Goto.new(id, sym, state, dest) - @gotos.push g if sym.nonterminal? - state.gotos[sym] = g - puts "dstate: #{state.ident} --#{sym}--> #{dest.ident}" if @d_state - - # check infinite recursion - if state.ident == dest.ident and state.closure.size == 1 - raise CompileError, - sprintf("Infinite recursion: state %d, with rule %d", - state.ident, state.ptrs[0].rule.ident) - end - end - end - - def addsym(table, sym, ptr) - unless s = table[sym] - table[sym] = s = ISet.new - end - s.add ptr - end - - def core_to_state(core) - # - # convert CORE to a State object. - # If matching state does not exist, create it and add to the table. - # - - k = fingerprint(core) - unless dest = @statecache[k] - # not registered yet - dest = State.new(@states.size, core) - @states.push dest - - @statecache[k] = dest - - puts "core_to_state: create state ID #{dest.ident}" if @d_state - else - if @d_state - puts "core_to_state: dest is cached ID #{dest.ident}" - puts "core_to_state: dest core #{dest.core.join(' ')}" - end - end - - dest - end - - def fingerprint(arr) - arr.map {|i| i.ident }.pack('L*') - end - - # - # DFA (Deterministic Finite Automaton) Generation - # - - public - - def dfa - return self if @dfa_computed - nfa - compute_dfa - @dfa_computed = true - self - end - - private - - def compute_dfa - la = lookahead() - @states.each do |state| - state.la = la - resolve state - end - set_accept - @states.each do |state| - pack state - end - check_useless - end - - def lookahead - # - # lookahead algorithm ver.3 -- from bison 1.26 - # - - gotos = @gotos - if @d_la - puts "\n--- goto ---" - gotos.each_with_index {|g, i| print i, ' '; p g } - end - - ### initialize_LA() - ### set_goto_map() - la_rules = [] - @states.each do |state| - state.check_la la_rules - end - - ### initialize_F() - f = create_tmap(gotos.size) - reads = [] - edge = [] - gotos.each do |goto| - goto.to_state.goto_table.each do |t, st| - if t.terminal? - f[goto.ident] |= (1 << t.ident) - elsif t.nullable? - edge.push goto.to_state.gotos[t].ident - end - end - if edge.empty? - reads.push nil - else - reads.push edge - edge = [] - end - end - digraph f, reads - if @d_la - puts "\n--- F1 (reads) ---" - print_tab gotos, reads, f - end - - ### build_relations() - ### compute_FOLLOWS - path = nil - edge = [] - lookback = Array.new(la_rules.size, nil) - includes = [] - gotos.each do |goto| - goto.symbol.heads.each do |ptr| - path = record_path(goto.from_state, ptr.rule) - lastgoto = path.last - st = lastgoto ? lastgoto.to_state : goto.from_state - if st.conflict? - addrel lookback, st.rruleid(ptr.rule), goto - end - path.reverse_each do |g| - break if g.symbol.terminal? - edge.push g.ident - break unless g.symbol.nullable? - end - end - if edge.empty? - includes.push nil - else - includes.push edge - edge = [] - end - end - includes = transpose(includes) - digraph f, includes - if @d_la - puts "\n--- F2 (includes) ---" - print_tab gotos, includes, f - end - - ### compute_lookaheads - la = create_tmap(la_rules.size) - lookback.each_with_index do |arr, i| - if arr - arr.each do |g| - la[i] |= f[g.ident] - end - end - end - if @d_la - puts "\n--- LA (lookback) ---" - print_tab la_rules, lookback, la - end - - la - end - - def create_tmap(size) - Array.new(size, 0) # use Integer as bitmap - end - - def addrel(tbl, i, item) - if a = tbl[i] - a.push item - else - tbl[i] = [item] - end - end - - def record_path(begst, rule) - st = begst - path = [] - rule.symbols.each do |t| - goto = st.gotos[t] - path.push goto - st = goto.to_state - end - path - end - - def transpose(rel) - new = Array.new(rel.size, nil) - rel.each_with_index do |arr, idx| - if arr - arr.each do |i| - addrel new, i, idx - end - end - end - new - end - - def digraph(map, relation) - n = relation.size - index = Array.new(n, nil) - vertices = [] - @infinity = n + 2 - - index.each_index do |i| - if not index[i] and relation[i] - traverse i, index, vertices, map, relation - end - end - end - - def traverse(i, index, vertices, map, relation) - vertices.push i - index[i] = height = vertices.size - - if rp = relation[i] - rp.each do |proci| - unless index[proci] - traverse proci, index, vertices, map, relation - end - if index[i] > index[proci] - # circulative recursion !!! - index[i] = index[proci] - end - map[i] |= map[proci] - end - end - - if index[i] == height - while true - proci = vertices.pop - index[proci] = @infinity - break if i == proci - - map[proci] |= map[i] - end - end - end - - # for debug - def print_atab(idx, tab) - tab.each_with_index do |i,ii| - printf '%-20s', idx[ii].inspect - p i - end - end - - def print_tab(idx, rel, tab) - tab.each_with_index do |bin,i| - print i, ' ', idx[i].inspect, ' << '; p rel[i] - print ' ' - each_t(@symboltable, bin) {|t| print ' ', t } - puts - end - end - - # for debug - def print_tab_i(idx, rel, tab, i) - bin = tab[i] - print i, ' ', idx[i].inspect, ' << '; p rel[i] - print ' ' - each_t(@symboltable, bin) {|t| print ' ', t } - end - - # for debug - def printb(i) - each_t(@symboltable, i) do |t| - print t, ' ' - end - puts - end - - def each_t(tbl, set) - 0.upto( set.size ) do |i| - (0..7).each do |ii| - if set[idx = i * 8 + ii] == 1 - yield tbl[idx] - end - end - end - end - - # - # resolve - # - - def resolve(state) - if state.conflict? - resolve_rr state, state.ritems - resolve_sr state, state.stokens - else - if state.rrules.empty? - # shift - state.stokens.each do |t| - state.action[t] = @actions.shift(state.goto_table[t]) - end - else - # reduce - state.defact = @actions.reduce(state.rrules[0]) - end - end - end - - def resolve_rr(state, r) - r.each do |item| - item.each_la(@symboltable) do |t| - act = state.action[t] - if act - unless act.kind_of?(Reduce) - raise "racc: fatal: #{act.class} in action table" - end - # Cannot resolve R/R conflict (on t). - # Reduce with upper rule as default. - state.rr_conflict act.rule, item.rule, t - else - # No conflict. - state.action[t] = @actions.reduce(item.rule) - end - end - end - end - - def resolve_sr(state, s) - s.each do |stok| - goto = state.goto_table[stok] - act = state.action[stok] - - unless act - # no conflict - state.action[stok] = @actions.shift(goto) - else - unless act.kind_of?(Reduce) - puts 'DEBUG -------------------------------' - p stok - p act - state.action.each do |k,v| - print k.inspect, ' ', v.inspect, "\n" - end - raise "racc: fatal: #{act.class} in action table" - end - - # conflict on stok - - rtok = act.rule.precedence - case do_resolve_sr(stok, rtok) - when :Reduce - # action is already set - - when :Shift - # overwrite - act.decref - state.action[stok] = @actions.shift(goto) - - when :Error - act.decref - state.action[stok] = @actions.error - - when :CantResolve - # shift as default - act.decref - state.action[stok] = @actions.shift(goto) - state.sr_conflict stok, act.rule - end - end - end - end - - ASSOC = { - :Left => :Reduce, - :Right => :Shift, - :Nonassoc => :Error - } - - def do_resolve_sr(stok, rtok) - puts "resolve_sr: s/r conflict: rtok=#{rtok}, stok=#{stok}" if @d_prec - - unless rtok and rtok.precedence - puts "resolve_sr: no prec for #{rtok}(R)" if @d_prec - return :CantResolve - end - rprec = rtok.precedence - - unless stok and stok.precedence - puts "resolve_sr: no prec for #{stok}(S)" if @d_prec - return :CantResolve - end - sprec = stok.precedence - - ret = if rprec == sprec - ASSOC[rtok.assoc] or - raise "racc: fatal: #{rtok}.assoc is not Left/Right/Nonassoc" - else - (rprec > sprec) ? (:Reduce) : (:Shift) - end - - puts "resolve_sr: resolved as #{ret.id2name}" if @d_prec - ret - end - - # - # complete - # - - def set_accept - anch = @symboltable.anchor - init_state = @states[0].goto_table[@grammar.start] - targ_state = init_state.action[anch].goto_state - acc_state = targ_state.action[anch].goto_state - - acc_state.action.clear - acc_state.goto_table.clear - acc_state.defact = @actions.accept - end - - def pack(state) - ### find most frequently used reduce rule - act = state.action - arr = Array.new(@grammar.size, 0) - act.each do |t, a| - arr[a.ruleid] += 1 if a.kind_of?(Reduce) - end - i = arr.max - s = (i > 0) ? arr.index(i) : nil - - ### set & delete default action - if s - r = @actions.reduce(s) - if not state.defact or state.defact == r - act.delete_if {|t, a| a == r } - state.defact = r - end - else - state.defact ||= @actions.error - end - end - - def check_useless - used = [] - @actions.each_reduce do |act| - if not act or act.refn == 0 - act.rule.useless = true - else - t = act.rule.target - used[t.ident] = t - end - end - @symboltable.nt_base.upto(@symboltable.nt_max - 1) do |n| - unless used[n] - @symboltable[n].useless = true - end - end - end - - end # class StateTable - - - # A LALR state. - class State - - def initialize(ident, core) - @ident = ident - @core = core - @goto_table = {} - @gotos = {} - @stokens = nil - @ritems = nil - @action = {} - @defact = nil - @rrconf = nil - @srconf = nil - - @closure = make_closure(@core) - end - - attr_reader :ident - alias stateid ident - alias hash ident - - attr_reader :core - attr_reader :closure - - attr_reader :goto_table - attr_reader :gotos - - attr_reader :stokens - attr_reader :ritems - attr_reader :rrules - - attr_reader :action - attr_accessor :defact # default action - - attr_reader :rrconf - attr_reader :srconf - - def inspect - "<state #{@ident}>" - end - - alias to_s inspect - - def ==(oth) - @ident == oth.ident - end - - alias eql? == - - def make_closure(core) - set = ISet.new - core.each do |ptr| - set.add ptr - if t = ptr.dereference and t.nonterminal? - set.update_a t.expand - end - end - set.to_a - end - - def check_la(la_rules) - @conflict = false - s = [] - r = [] - @closure.each do |ptr| - if t = ptr.dereference - if t.terminal? - s[t.ident] = t - if t.ident == 1 # $error - @conflict = true - end - end - else - r.push ptr.rule - end - end - unless r.empty? - if not s.empty? or r.size > 1 - @conflict = true - end - end - s.compact! - @stokens = s - @rrules = r - - if @conflict - @la_rules_i = la_rules.size - @la_rules = r.map {|i| i.ident } - la_rules.concat r - else - @la_rules_i = @la_rules = nil - end - end - - def conflict? - @conflict - end - - def rruleid(rule) - if i = @la_rules.index(rule.ident) - @la_rules_i + i - else - puts '/// rruleid' - p self - p rule - p @rrules - p @la_rules_i - raise 'racc: fatal: cannot get reduce rule id' - end - end - - def la=(la) - return unless @conflict - i = @la_rules_i - @ritems = r = [] - @rrules.each do |rule| - r.push Item.new(rule, la[i]) - i += 1 - end - end - - def rr_conflict(high, low, ctok) - c = RRconflict.new(@ident, high, low, ctok) - - @rrconf ||= {} - if a = @rrconf[ctok] - a.push c - else - @rrconf[ctok] = [c] - end - end - - def sr_conflict(shift, reduce) - c = SRconflict.new(@ident, shift, reduce) - - @srconf ||= {} - if a = @srconf[shift] - a.push c - else - @srconf[shift] = [c] - end - end - - def n_srconflicts - @srconf ? @srconf.size : 0 - end - - def n_rrconflicts - @rrconf ? @rrconf.size : 0 - end - - end # class State - - - # - # Represents a transition on the grammar. - # "Real goto" means a transition by nonterminal, - # but this class treats also terminal's. - # If one is a terminal transition, .ident returns nil. - # - class Goto - def initialize(ident, sym, from, to) - @ident = ident - @symbol = sym - @from_state = from - @to_state = to - end - - attr_reader :ident - attr_reader :symbol - attr_reader :from_state - attr_reader :to_state - - def inspect - "(#{@from_state.ident}-#{@symbol}->#{@to_state.ident})" - end - end - - - # LALR item. A set of rule and its lookahead tokens. - class Item - def initialize(rule, la) - @rule = rule - @la = la - end - - attr_reader :rule - attr_reader :la - - def each_la(tbl) - la = @la - 0.upto(la.size - 1) do |i| - (0..7).each do |ii| - if la[idx = i * 8 + ii] == 1 - yield tbl[idx] - end - end - end - end - end - - - # The table of LALR actions. Actions are either of - # Shift, Reduce, Accept and Error. - class ActionTable - - def initialize(rt, st) - @grammar = rt - @statetable = st - - @reduce = [] - @shift = [] - @accept = nil - @error = nil - end - - def init - @grammar.each do |rule| - @reduce.push Reduce.new(rule) - end - @statetable.each do |state| - @shift.push Shift.new(state) - end - @accept = Accept.new - @error = Error.new - end - - def reduce_n - @reduce.size - end - - def reduce(i) - case i - when Rule then i = i.ident - when Integer then ; - else - raise "racc: fatal: wrong class #{i.class} for reduce" - end - - r = @reduce[i] or raise "racc: fatal: reduce action #{i.inspect} not exist" - r.incref - r - end - - def each_reduce(&block) - @reduce.each(&block) - end - - def shift_n - @shift.size - end - - def shift(i) - case i - when State then i = i.ident - when Integer then ; - else - raise "racc: fatal: wrong class #{i.class} for shift" - end - - @shift[i] or raise "racc: fatal: shift action #{i} does not exist" - end - - def each_shift(&block) - @shift.each(&block) - end - - attr_reader :accept - attr_reader :error - - end - - - class Shift - def initialize(goto) - @goto_state = goto - end - - attr_reader :goto_state - - def goto_id - @goto_state.ident - end - - def inspect - "<shift #{@goto_state.ident}>" - end - end - - - class Reduce - def initialize(rule) - @rule = rule - @refn = 0 - end - - attr_reader :rule - attr_reader :refn - - def ruleid - @rule.ident - end - - def inspect - "<reduce #{@rule.ident}>" - end - - def incref - @refn += 1 - end - - def decref - @refn -= 1 - raise 'racc: fatal: act.refn < 0' if @refn < 0 - end - end - - class Accept - def inspect - "<accept>" - end - end - - class Error - def inspect - "<error>" - end - end - - class SRconflict - def initialize(sid, shift, reduce) - @stateid = sid - @shift = shift - @reduce = reduce - end - - attr_reader :stateid - attr_reader :shift - attr_reader :reduce - - def to_s - sprintf('state %d: S/R conflict rule %d reduce and shift %s', - @stateid, @reduce.ruleid, @shift.to_s) - end - end - - class RRconflict - def initialize(sid, high, low, tok) - @stateid = sid - @high_prec = high - @low_prec = low - @token = tok - end - - attr_reader :stateid - attr_reader :high_prec - attr_reader :low_prec - attr_reader :token - - def to_s - sprintf('state %d: R/R conflict with rule %d and %d on %s', - @stateid, @high_prec.ident, @low_prec.ident, @token.to_s) - end - end - -end diff --git a/lib/racc/statetransitiontable.rb b/lib/racc/statetransitiontable.rb deleted file mode 100644 index d75fa1657a..0000000000 --- a/lib/racc/statetransitiontable.rb +++ /dev/null @@ -1,311 +0,0 @@ -#-- -# -# -# -# Copyright (c) 1999-2006 Minero Aoki -# -# This program is free software. -# You can distribute/modify this program under the same terms of ruby. -# see the file "COPYING". -# -#++ - -require 'racc/parser' - -module Racc - - StateTransitionTable = Struct.new(:action_table, - :action_check, - :action_default, - :action_pointer, - :goto_table, - :goto_check, - :goto_default, - :goto_pointer, - :token_table, - :reduce_table, - :reduce_n, - :shift_n, - :nt_base, - :token_to_s_table, - :use_result_var, - :debug_parser) - class StateTransitionTable # reopen - def StateTransitionTable.generate(states) - StateTransitionTableGenerator.new(states).generate - end - - def initialize(states) - super() - @states = states - @grammar = states.grammar - self.use_result_var = true - self.debug_parser = true - end - - attr_reader :states - attr_reader :grammar - - def parser_class - ParserClassGenerator.new(@states).generate - end - - def token_value_table - h = {} - token_table().each do |sym, i| - h[sym.value] = i - end - h - end - end - - - class StateTransitionTableGenerator - - def initialize(states) - @states = states - @grammar = states.grammar - end - - def generate - t = StateTransitionTable.new(@states) - gen_action_tables t, @states - gen_goto_tables t, @grammar - t.token_table = token_table(@grammar) - t.reduce_table = reduce_table(@grammar) - t.reduce_n = @states.reduce_n - t.shift_n = @states.shift_n - t.nt_base = @grammar.nonterminal_base - t.token_to_s_table = @grammar.symbols.map {|sym| sym.to_s } - t - end - - def reduce_table(grammar) - t = [0, 0, :racc_error] - grammar.each_with_index do |rule, idx| - next if idx == 0 - t.push rule.size - t.push rule.target.ident - t.push(if rule.action.empty? # and @params.omit_action_call? - then :_reduce_none - else "_reduce_#{idx}".intern - end) - end - t - end - - def token_table(grammar) - h = {} - grammar.symboltable.terminals.each do |t| - h[t] = t.ident - end - h - end - - def gen_action_tables(t, states) - t.action_table = yytable = [] - t.action_check = yycheck = [] - t.action_default = yydefact = [] - t.action_pointer = yypact = [] - e1 = [] - e2 = [] - states.each do |state| - yydefact.push act2actid(state.defact) - if state.action.empty? - yypact.push nil - next - end - vector = [] - state.action.each do |tok, act| - vector[tok.ident] = act2actid(act) - end - addent e1, vector, state.ident, yypact - end - set_table e1, e2, yytable, yycheck, yypact - end - - def gen_goto_tables(t, grammar) - t.goto_table = yytable2 = [] - t.goto_check = yycheck2 = [] - t.goto_pointer = yypgoto = [] - t.goto_default = yydefgoto = [] - e1 = [] - e2 = [] - grammar.each_nonterminal do |tok| - tmp = [] - - # decide default - freq = Array.new(@states.size, 0) - @states.each do |state| - st = state.goto_table[tok] - if st - st = st.ident - freq[st] += 1 - end - tmp[state.ident] = st - end - max = freq.max - if max > 1 - default = freq.index(max) - tmp.map! {|i| default == i ? nil : i } - else - default = nil - end - yydefgoto.push default - - # delete default value - tmp.pop until tmp.last or tmp.empty? - if tmp.compact.empty? - # only default - yypgoto.push nil - next - end - - addent e1, tmp, (tok.ident - grammar.nonterminal_base), yypgoto - end - set_table e1, e2, yytable2, yycheck2, yypgoto - end - - def addent(all, arr, chkval, ptr) - max = arr.size - min = nil - arr.each_with_index do |item, idx| - if item - min ||= idx - end - end - ptr.push(-7777) # mark - arr = arr[min...max] - all.push [arr, chkval, mkmapexp(arr), min, ptr.size - 1] - end - - n = 2 ** 16 - begin - Regexp.compile("a{#{n}}") - RE_DUP_MAX = n - rescue RegexpError - n /= 2 - retry - end - - def mkmapexp(arr) - i = ii = 0 - as = arr.size - map = String.new - maxdup = RE_DUP_MAX - curr = nil - while i < as - ii = i + 1 - if arr[i] - ii += 1 while ii < as and arr[ii] - curr = '-' - else - ii += 1 while ii < as and not arr[ii] - curr = '.' - end - - offset = ii - i - if offset == 1 - map << curr - else - while offset > maxdup - map << "#{curr}{#{maxdup}}" - offset -= maxdup - end - map << "#{curr}{#{offset}}" if offset > 1 - end - i = ii - end - Regexp.compile(map, Regexp::NOENCODING) - end - - def set_table(entries, dummy, tbl, chk, ptr) - upper = 0 - map = '-' * 10240 - - # sort long to short - entries.sort_by!.with_index {|a,i| [-a[0].size, i] } - - entries.each do |arr, chkval, expr, min, ptri| - if upper + arr.size > map.size - map << '-' * (arr.size + 1024) - end - idx = map.index(expr) - ptr[ptri] = idx - min - arr.each_with_index do |item, i| - if item - i += idx - tbl[i] = item - chk[i] = chkval - map[i] = ?o - end - end - upper = idx + arr.size - end - end - - def act2actid(act) - case act - when Shift then act.goto_id - when Reduce then -act.ruleid - when Accept then @states.shift_n - when Error then @states.reduce_n * -1 - else - raise "racc: fatal: wrong act type #{act.class} in action table" - end - end - - end - - - class ParserClassGenerator - - def initialize(states) - @states = states - @grammar = states.grammar - end - - def generate - table = @states.state_transition_table - c = Class.new(::Racc::Parser) - c.const_set :Racc_arg, [table.action_table, - table.action_check, - table.action_default, - table.action_pointer, - table.goto_table, - table.goto_check, - table.goto_default, - table.goto_pointer, - table.nt_base, - table.reduce_table, - table.token_value_table, - table.shift_n, - table.reduce_n, - false] - c.const_set :Racc_token_to_s_table, table.token_to_s_table - c.const_set :Racc_debug_parser, true - define_actions c - c - end - - private - - def define_actions(c) - c.module_eval "def _reduce_none(vals, vstack) vals[0] end" - @grammar.each do |rule| - if rule.action.empty? - c.alias_method("_reduce_#{rule.ident}", :_reduce_none) - else - c.define_method("_racc_action_#{rule.ident}", &rule.action.proc) - c.module_eval(<<-End, __FILE__, __LINE__ + 1) - def _reduce_#{rule.ident}(vals, vstack) - _racc_action_#{rule.ident}(*vals) - end - End - end - end - end - - end - -end # module Racc diff --git a/lib/racc/static.rb b/lib/racc/static.rb deleted file mode 100644 index bebbeb5aa6..0000000000 --- a/lib/racc/static.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'racc' -require 'racc/parser' -require 'racc/grammarfileparser' -require 'racc/parserfilegenerator' -require 'racc/logfilegenerator' diff --git a/lib/random/formatter.rb b/lib/random/formatter.rb index 4dea61c16c..037f9d8748 100644 --- a/lib/random/formatter.rb +++ b/lib/random/formatter.rb @@ -165,15 +165,134 @@ module Random::Formatter # # The result contains 122 random bits (15.25 random bytes). # - # See RFC4122[https://datatracker.ietf.org/doc/html/rfc4122] for details of UUID. + # See RFC4122[https://www.rfc-editor.org/rfc/rfc4122] for details of UUID. # def uuid - ary = random_bytes(16).unpack("NnnnnN") - ary[2] = (ary[2] & 0x0fff) | 0x4000 - ary[3] = (ary[3] & 0x3fff) | 0x8000 - "%08x-%04x-%04x-%04x-%04x%08x" % ary + ary = random_bytes(16) + ary.setbyte(6, (ary.getbyte(6) & 0x0f) | 0x40) + ary.setbyte(8, (ary.getbyte(8) & 0x3f) | 0x80) + ary.unpack("H8H4H4H4H12").join(?-) end + alias uuid_v4 uuid + + # Generate a random v7 UUID (Universally Unique IDentifier). + # + # require 'random/formatter' + # + # Random.uuid_v7 # => "0188d4c3-1311-7f96-85c7-242a7aa58f1e" + # Random.uuid_v7 # => "0188d4c3-16fe-744f-86af-38fa04c62bb5" + # Random.uuid_v7 # => "0188d4c3-1af8-764f-b049-c204ce0afa23" + # Random.uuid_v7 # => "0188d4c3-1e74-7085-b14f-ef6415dc6f31" + # # |<--sorted-->| |<----- random ---->| + # + # # or + # prng = Random.new + # prng.uuid_v7 # => "0188ca51-5e72-7950-a11d-def7ff977c98" + # + # The version 7 UUID starts with the least significant 48 bits of a 64 bit + # Unix timestamp (milliseconds since the epoch) and fills the remaining bits + # with random data, excluding the version and variant bits. + # + # This allows version 7 UUIDs to be sorted by creation time. Time ordered + # UUIDs can be used for better database index locality of newly inserted + # records, which may have a significant performance benefit compared to random + # data inserts. + # + # The result contains 74 random bits (9.25 random bytes). + # + # Note that this method cannot be made reproducible because its output + # includes not only random bits but also timestamp. + # + # See draft-ietf-uuidrev-rfc4122bis[https://datatracker.ietf.org/doc/draft-ietf-uuidrev-rfc4122bis/] + # for details of UUIDv7. + # + # ==== Monotonicity + # + # UUIDv7 has millisecond precision by default, so multiple UUIDs created + # within the same millisecond are not issued in monotonically increasing + # order. To create UUIDs that are time-ordered with sub-millisecond + # precision, up to 12 bits of additional timestamp may added with + # +extra_timestamp_bits+. The extra timestamp precision comes at the expense + # of random bits. Setting <tt>extra_timestamp_bits: 12</tt> provides ~244ns + # of precision, but only 62 random bits (7.75 random bytes). + # + # prng = Random.new + # Array.new(4) { prng.uuid_v7(extra_timestamp_bits: 12) } + # # => + # ["0188d4c7-13da-74f9-8b53-22a786ffdd5a", + # "0188d4c7-13da-753b-83a5-7fb9b2afaeea", + # "0188d4c7-13da-754a-88ea-ac0baeedd8db", + # "0188d4c7-13da-7557-83e1-7cad9cda0d8d"] + # # |<--- sorted --->| |<-- random --->| + # + # Array.new(4) { prng.uuid_v7(extra_timestamp_bits: 8) } + # # => + # ["0188d4c7-3333-7a95-850a-de6edb858f7e", + # "0188d4c7-3333-7ae8-842e-bc3a8b7d0cf9", # <- out of order + # "0188d4c7-3333-7ae2-995a-9f135dc44ead", # <- out of order + # "0188d4c7-3333-7af9-87c3-8f612edac82e"] + # # |<--- sorted -->||<---- random --->| + # + # Any rollbacks of the system clock will break monotonicity. UUIDv7 is based + # on UTC, which excludes leap seconds and can rollback the clock. To avoid + # this, the system clock can synchronize with an NTP server configured to use + # a "leap smear" approach. NTP or PTP will also be needed to synchronize + # across distributed nodes. + # + # Counters and other mechanisms for stronger guarantees of monotonicity are + # not implemented. Applications with stricter requirements should follow + # {Section 6.2}[https://www.ietf.org/archive/id/draft-ietf-uuidrev-rfc4122bis-07.html#monotonicity_counters] + # of the specification. + # + def uuid_v7(extra_timestamp_bits: 0) + case (extra_timestamp_bits = Integer(extra_timestamp_bits)) + when 0 # min timestamp precision + ms = Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond) + rand = random_bytes(10) + rand.setbyte(0, rand.getbyte(0) & 0x0f | 0x70) # version + rand.setbyte(2, rand.getbyte(2) & 0x3f | 0x80) # variant + "%08x-%04x-%s" % [ + (ms & 0x0000_ffff_ffff_0000) >> 16, + (ms & 0x0000_0000_0000_ffff), + rand.unpack("H4H4H12").join("-") + ] + + when 12 # max timestamp precision + ms, ns = Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond) + .divmod(1_000_000) + extra_bits = ns * 4096 / 1_000_000 + rand = random_bytes(8) + rand.setbyte(0, rand.getbyte(0) & 0x3f | 0x80) # variant + "%08x-%04x-7%03x-%s" % [ + (ms & 0x0000_ffff_ffff_0000) >> 16, + (ms & 0x0000_0000_0000_ffff), + extra_bits, + rand.unpack("H4H12").join("-") + ] + + when (0..12) # the generic version is slower than the special cases above + rand_a, rand_b1, rand_b2, rand_b3 = random_bytes(10).unpack("nnnN") + rand_mask_bits = 12 - extra_timestamp_bits + ms, ns = Process.clock_gettime(Process::CLOCK_REALTIME, :nanosecond) + .divmod(1_000_000) + "%08x-%04x-%04x-%04x-%04x%08x" % [ + (ms & 0x0000_ffff_ffff_0000) >> 16, + (ms & 0x0000_0000_0000_ffff), + 0x7000 | + ((ns * (1 << extra_timestamp_bits) / 1_000_000) << rand_mask_bits) | + rand_a & ((1 << rand_mask_bits) - 1), + 0x8000 | (rand_b1 & 0x3fff), + rand_b2, + rand_b3 + ] + + else + raise ArgumentError, "extra_timestamp_bits must be in 0..12" + end + end + + # Internal interface to Random; Generate random data _n_ bytes. private def gen_random(n) self.bytes(n) end @@ -221,16 +340,20 @@ module Random::Formatter result end + # The default character list for #alphanumeric. ALPHANUMERIC = [*'A'..'Z', *'a'..'z', *'0'..'9'] + # Generate a random alphanumeric string. # # The argument _n_ specifies the length, in characters, of the alphanumeric # string to be generated. + # The argument _chars_ specifies the character list which the result is + # consist of. # # If _n_ is not specified or is nil, 16 is assumed. # It may be larger in the future. # - # The result may contain A-Z, a-z and 0-9. + # The result may contain A-Z, a-z and 0-9, unless _chars_ is specified. # # require 'random/formatter' # @@ -238,8 +361,13 @@ module Random::Formatter # # or # prng = Random.new # prng.alphanumeric(10) #=> "i6K93NdqiH" - def alphanumeric(n=nil) + # + # Random.alphanumeric(4, chars: [*"0".."9"]) #=> "2952" + # # or + # prng = Random.new + # prng.alphanumeric(10, chars: [*"!".."/"]) #=> ",.,++%/''." + def alphanumeric(n = nil, chars: ALPHANUMERIC) n = 16 if n.nil? - choose(ALPHANUMERIC, n) + choose(chars, n) end end diff --git a/lib/rdoc.rb b/lib/rdoc.rb index b62c22576d..9dc4595324 100644 --- a/lib/rdoc.rb +++ b/lib/rdoc.rb @@ -120,6 +120,17 @@ module RDoc end end + ## + # Searches and returns the directory for settings. + # + # 1. <tt>$HOME/.rdoc</tt> directory, if it exists. + # 2. The +rdoc+ directory under the path specified by the + # +XDG_DATA_HOME+ environment variable, if it is set. + # 3. <tt>$HOME/.local/share/rdoc</tt> directory. + # + # Other than the home directory, the containing directory will be + # created automatically. + def self.home rdoc_dir = begin File.expand_path('~/.rdoc') @@ -129,6 +140,7 @@ module RDoc if File.directory?(rdoc_dir) rdoc_dir else + require 'fileutils' begin # XDG xdg_data_home = ENV["XDG_DATA_HOME"] || File.join(File.expand_path("~"), '.local', 'share') diff --git a/lib/rdoc/any_method.rb b/lib/rdoc/any_method.rb index 051f946a10..465c4a4fb2 100644 --- a/lib/rdoc/any_method.rb +++ b/lib/rdoc/any_method.rb @@ -116,6 +116,13 @@ class RDoc::AnyMethod < RDoc::MethodAttr end ## + # Whether the method has a call-seq. + + def has_call_seq? + !!(@call_seq || is_alias_for&._call_seq) + end + + ## # Loads is_alias_for from the internal name. Returns nil if the alias # cannot be found. @@ -297,6 +304,14 @@ class RDoc::AnyMethod < RDoc::MethodAttr end ## + # Whether to skip the method description, true for methods that have + # aliases with a call-seq that doesn't include the method name. + + def skip_description? + has_call_seq? && call_seq.nil? && !!(is_alias_for || !aliases.empty?) + end + + ## # Sets the store for this method and its referenced code objects. def store= store diff --git a/lib/rdoc/comment.rb b/lib/rdoc/comment.rb index 63197492c4..04ec226436 100644 --- a/lib/rdoc/comment.rb +++ b/lib/rdoc/comment.rb @@ -6,8 +6,8 @@ # Each comment may have a different markup format set by #format=. By default # 'rdoc' is used. The :markup: directive tells RDoc which format to use. # -# See RDoc::Markup@Other+directives for instructions on adding an alternate -# format. +# See RDoc::MarkupReference@Directive+for+Specifying+RDoc+Source+Format. + class RDoc::Comment @@ -133,12 +133,7 @@ class RDoc::Comment # HACK dubious def encode! encoding - # TODO: Remove this condition after Ruby 2.2 EOL - if RUBY_VERSION < '2.3.0' - @text = @text.force_encoding encoding - else - @text = String.new @text, encoding: encoding - end + @text = String.new @text, encoding: encoding self end diff --git a/lib/rdoc/context.rb b/lib/rdoc/context.rb index c6edfb473c..c688d562c3 100644 --- a/lib/rdoc/context.rb +++ b/lib/rdoc/context.rb @@ -710,7 +710,7 @@ class RDoc::Context < RDoc::CodeObject # This method exists to make it easy to work with Context subclasses that # aren't part of RDoc. - def each_ancestor # :nodoc: + def each_ancestor(&_) # :nodoc: end ## diff --git a/lib/rdoc/cross_reference.rb b/lib/rdoc/cross_reference.rb index f3b703a559..4e011219e8 100644 --- a/lib/rdoc/cross_reference.rb +++ b/lib/rdoc/cross_reference.rb @@ -131,6 +131,9 @@ class RDoc::CrossReference @seen = {} end + ## + # Returns a method reference to +name+. + def resolve_method name ref = nil diff --git a/lib/rdoc/encoding.rb b/lib/rdoc/encoding.rb index cf60badd24..67e190f782 100644 --- a/lib/rdoc/encoding.rb +++ b/lib/rdoc/encoding.rb @@ -86,17 +86,6 @@ module RDoc::Encoding nil end - def self.remove_frozen_string_literal string - string =~ /\A(?:#!.*\n)?(.*\n)/ - first_line = $1 - - if first_line =~ /\A# +frozen[-_]string[-_]literal[=:].+$/i - string = string.sub first_line, '' - end - - string - end - ## # Detects the encoding of +string+ based on the magic comment @@ -124,12 +113,7 @@ module RDoc::Encoding if text.kind_of? RDoc::Comment text.encode! encoding else - # TODO: Remove this condition after Ruby 2.2 EOL - if RUBY_VERSION < '2.3.0' - text.force_encoding encoding - else - String.new text, encoding: encoding - end + String.new text, encoding: encoding end end diff --git a/lib/rdoc/erbio.rb b/lib/rdoc/erbio.rb index 56c0511a3d..0f98eaedee 100644 --- a/lib/rdoc/erbio.rb +++ b/lib/rdoc/erbio.rb @@ -20,12 +20,8 @@ class RDoc::ERBIO < ERB ## # Defaults +eoutvar+ to 'io', otherwise is identical to ERB's initialize - def initialize str, safe_level = nil, legacy_trim_mode = nil, legacy_eoutvar = 'io', trim_mode: nil, eoutvar: 'io' - if RUBY_VERSION >= '2.6' - super(str, trim_mode: trim_mode, eoutvar: eoutvar) - else - super(str, safe_level, legacy_trim_mode, legacy_eoutvar) - end + def initialize str, trim_mode: nil, eoutvar: 'io' + super(str, trim_mode: trim_mode, eoutvar: eoutvar) end ## diff --git a/lib/rdoc/generator/darkfish.rb b/lib/rdoc/generator/darkfish.rb index 60e0265e8c..1b408a6f8e 100644 --- a/lib/rdoc/generator/darkfish.rb +++ b/lib/rdoc/generator/darkfish.rb @@ -610,7 +610,7 @@ class RDoc::Generator::Darkfish @classes = @store.all_classes_and_modules.sort @files = @store.all_files.sort - @methods = @classes.map { |m| m.method_list }.flatten.sort + @methods = @classes.flat_map { |m| m.method_list }.sort @modsort = get_sorted_module_list @classes end @@ -778,11 +778,7 @@ class RDoc::Generator::Darkfish erbout = "_erbout_#{file_var}" end - if RUBY_VERSION >= '2.6' - template = klass.new template, trim_mode: '-', eoutvar: erbout - else - template = klass.new template, nil, '-', erbout - end + template = klass.new template, trim_mode: '-', eoutvar: erbout @template_cache[file] = template template end diff --git a/lib/rdoc/generator/json_index.rb b/lib/rdoc/generator/json_index.rb index 3a1000033d..c454910d5c 100644 --- a/lib/rdoc/generator/json_index.rb +++ b/lib/rdoc/generator/json_index.rb @@ -230,9 +230,9 @@ class RDoc::Generator::JsonIndex def index_methods debug_msg " generating method search index" - list = @classes.uniq.map do |klass| + list = @classes.uniq.flat_map do |klass| klass.method_list - end.flatten.sort_by do |method| + end.sort_by do |method| [method.name, method.parent.full_name] end diff --git a/lib/rdoc/generator/pot.rb b/lib/rdoc/generator/pot.rb index bee1133b07..b0b7c07179 100644 --- a/lib/rdoc/generator/pot.rb +++ b/lib/rdoc/generator/pot.rb @@ -81,6 +81,7 @@ class RDoc::Generator::POT end end + # :nodoc: def class_dir nil end diff --git a/lib/rdoc/generator/template/darkfish/_sidebar_search.rhtml b/lib/rdoc/generator/template/darkfish/_sidebar_search.rhtml index 9c49b31376..afc7f7b88d 100644 --- a/lib/rdoc/generator/template/darkfish/_sidebar_search.rhtml +++ b/lib/rdoc/generator/template/darkfish/_sidebar_search.rhtml @@ -3,7 +3,7 @@ <div id="search-field-wrapper"> <input id="search-field" role="combobox" aria-label="Search" aria-autocomplete="list" aria-controls="search-results" - type="text" name="search" placeholder="Search" spellcheck="false" + type="text" name="search" placeholder="Search (/) for a class, method, ..." spellcheck="false" title="Type to search, Up and Down to navigate, Enter to load"> </div> diff --git a/lib/rdoc/generator/template/darkfish/class.rhtml b/lib/rdoc/generator/template/darkfish/class.rhtml index 97d175dddc..d6510336df 100644 --- a/lib/rdoc/generator/template/darkfish/class.rhtml +++ b/lib/rdoc/generator/template/darkfish/class.rhtml @@ -112,6 +112,10 @@ <%- end -%> </div> <%- end -%> + <%- elsif method.has_call_seq? then -%> + <div class="method-heading"> + <span class="method-name"><%= h method.name %></span> + </div> <%- else -%> <div class="method-heading"> <span class="method-name"><%= h method.name %></span><span @@ -123,6 +127,7 @@ <%- end -%> </div> + <%- unless method.skip_description? then -%> <div class="method-description"> <%- if method.comment then -%> <%= method.description.strip %> @@ -145,6 +150,7 @@ </div> <%- end -%> </div> + <%- end -%> <%- unless method.aliases.empty? then -%> <div class="aliases"> diff --git a/lib/rdoc/generator/template/darkfish/css/rdoc.css b/lib/rdoc/generator/template/darkfish/css/rdoc.css index ddaf4d47c6..2cc55e03b1 100644 --- a/lib/rdoc/generator/template/darkfish/css/rdoc.css +++ b/lib/rdoc/generator/template/darkfish/css/rdoc.css @@ -87,6 +87,17 @@ pre { border-radius: 0.2em; } +em { + text-decoration-color: rgba(52, 48, 64, 0.25); + text-decoration-line: underline; + text-decoration-style: dotted; +} + +strong, +em { + background-color: rgba(158, 178, 255, 0.1); +} + table { margin: 0; border-spacing: 0; @@ -567,7 +578,7 @@ main .method-click-advice { line-height: 20px; background: url(../images/zoom.png) no-repeat right top; } -main .method-heading:hover .method-click-advice { +main .method-header:hover .method-click-advice { visibility: visible; } diff --git a/lib/rdoc/generator/template/darkfish/js/darkfish.js b/lib/rdoc/generator/template/darkfish/js/darkfish.js index d0c9467751..19a85c54e1 100644 --- a/lib/rdoc/generator/template/darkfish/js/darkfish.js +++ b/lib/rdoc/generator/template/darkfish/js/darkfish.js @@ -78,7 +78,20 @@ function hookSearch() { search.scrollIntoView = search.scrollInWindow; }; +function hookFocus() { + document.addEventListener("keydown", (event) => { + if (document.activeElement.tagName === 'INPUT') { + return; + } + if (event.key === "/") { + event.preventDefault(); + document.querySelector('#search-field').focus(); + } + }); +} + document.addEventListener('DOMContentLoaded', function() { hookSourceViews(); hookSearch(); + hookFocus(); }); diff --git a/lib/rdoc/generator/template/darkfish/js/search.js b/lib/rdoc/generator/template/darkfish/js/search.js index 58e52afecf..d3cded1d57 100644 --- a/lib/rdoc/generator/template/darkfish/js/search.js +++ b/lib/rdoc/generator/template/darkfish/js/search.js @@ -15,9 +15,9 @@ Search.prototype = Object.assign({}, Navigation, new function() { this.init = function() { var _this = this; var observer = function(e) { - switch(e.keyCode) { - case 38: // Event.KEY_UP - case 40: // Event.KEY_DOWN + switch(e.key) { + case 'ArrowUp': + case 'ArrowDown': return; } _this.search(_this.input.value); diff --git a/lib/rdoc/generator/template/darkfish/table_of_contents.rhtml b/lib/rdoc/generator/template/darkfish/table_of_contents.rhtml index c8a422a521..54a376c9e5 100644 --- a/lib/rdoc/generator/template/darkfish/table_of_contents.rhtml +++ b/lib/rdoc/generator/template/darkfish/table_of_contents.rhtml @@ -47,9 +47,9 @@ <h2 id="methods">Methods</h2> <ul> -<%- @store.all_classes_and_modules.map do |mod| +<%- @store.all_classes_and_modules.flat_map do |mod| mod.method_list - end.flatten.sort.each do |method| %> + end.sort.each do |method| %> <li class="method"> <a href="<%= method.path %>"><%= h method.pretty_name %></a> — diff --git a/lib/rdoc/generator/template/json_index/js/navigation.js b/lib/rdoc/generator/template/json_index/js/navigation.js index dfad74b1ae..137e3a0038 100644 --- a/lib/rdoc/generator/template/json_index/js/navigation.js +++ b/lib/rdoc/generator/template/json_index/js/navigation.js @@ -23,24 +23,24 @@ Navigation = new function() { this.onkeydown = function(e) { if (!this.navigationActive) return; - switch(e.keyCode) { - case 37: //Event.KEY_LEFT: + switch(e.key) { + case 'ArrowLeft': if (this.moveLeft()) e.preventDefault(); break; - case 38: //Event.KEY_UP: - if (e.keyCode == 38 || e.ctrlKey) { + case 'ArrowUp': + if (e.key == 'ArrowUp' || e.ctrlKey) { if (this.moveUp()) e.preventDefault(); } break; - case 39: //Event.KEY_RIGHT: + case 'ArrowRight': if (this.moveRight()) e.preventDefault(); break; - case 40: //Event.KEY_DOWN: - if (e.keyCode == 40 || e.ctrlKey) { + case 'ArrowDown': + if (e.key == 'ArrowDown' || e.ctrlKey) { if (this.moveDown()) e.preventDefault(); } break; - case 13: //Event.KEY_RETURN: + case 'Enter': if (this.current) e.preventDefault(); this.select(this.current); break; diff --git a/lib/rdoc/markdown.rb b/lib/rdoc/markdown.rb index a0709b6352..5c72a5f224 100644 --- a/lib/rdoc/markdown.rb +++ b/lib/rdoc/markdown.rb @@ -175,7 +175,7 @@ # [dingus]: http://daringfireball.net/projects/markdown/dingus # [GFM]: https://github.github.com/gfm/ # [pegmarkdown]: https://github.com/jgm/peg-markdown -# [PHPE]: http://michelf.com/projects/php-markdown/extra/#def-list +# [PHPE]: https://michelf.ca/projects/php-markdown/extra/#def-list # [syntax]: http://daringfireball.net/projects/markdown/syntax #-- # Last updated to jgm/peg-markdown commit 8f8fc22ef0 @@ -16445,12 +16445,12 @@ class RDoc::Markdown return _tmp end - # DefinitionListLabel = StrChunk:label @Sp @Newline { label } + # DefinitionListLabel = Inline:label @Sp @Newline { label } def _DefinitionListLabel _save = self.pos while true # sequence - _tmp = apply(:_StrChunk) + _tmp = apply(:_Inline) label = @result unless _tmp self.pos = _save @@ -16777,7 +16777,7 @@ class RDoc::Markdown Rules[:_TableAlign] = rule_info("TableAlign", "< /:?-+:?/ > @Sp { text.start_with?(\":\") ? (text.end_with?(\":\") ? :center : :left) : (text.end_with?(\":\") ? :right : nil) }") Rules[:_DefinitionList] = rule_info("DefinitionList", "&{ definition_lists? } DefinitionListItem+:list { RDoc::Markup::List.new :NOTE, *list.flatten }") Rules[:_DefinitionListItem] = rule_info("DefinitionListItem", "DefinitionListLabel+:label DefinitionListDefinition+:defns { list_items = [] list_items << RDoc::Markup::ListItem.new(label, defns.shift) list_items.concat defns.map { |defn| RDoc::Markup::ListItem.new nil, defn } unless list_items.empty? list_items }") - Rules[:_DefinitionListLabel] = rule_info("DefinitionListLabel", "StrChunk:label @Sp @Newline { label }") + Rules[:_DefinitionListLabel] = rule_info("DefinitionListLabel", "Inline:label @Sp @Newline { label }") Rules[:_DefinitionListDefinition] = rule_info("DefinitionListDefinition", "@NonindentSpace \":\" @Space Inlines:a @BlankLine+ { paragraph a }") # :startdoc: end diff --git a/lib/rdoc/markdown/literals.rb b/lib/rdoc/markdown/literals.rb index 37659b7ae0..c5c15d3100 100644 --- a/lib/rdoc/markdown/literals.rb +++ b/lib/rdoc/markdown/literals.rb @@ -3,7 +3,6 @@ # :markup: markdown ## -#-- # This set of literals is for Ruby 1.9 regular expressions and gives full # unicode support. # diff --git a/lib/rdoc/markup/attribute_manager.rb b/lib/rdoc/markup/attribute_manager.rb index 1162f27284..f6eb06da95 100644 --- a/lib/rdoc/markup/attribute_manager.rb +++ b/lib/rdoc/markup/attribute_manager.rb @@ -138,6 +138,7 @@ class RDoc::Markup::AttributeManager res end + # :nodoc: def exclusive?(attr) (attr & @exclusive_bitmap) != 0 end @@ -155,6 +156,7 @@ class RDoc::Markup::AttributeManager convert_attrs_word_pair_map(str, attrs, exclusive) end + # :nodoc: def convert_attrs_matching_word_pairs(str, attrs, exclusive) # first do matching ones tags = @matching_word_pairs.select { |start, bitmap| @@ -179,6 +181,7 @@ class RDoc::Markup::AttributeManager str.delete!(NON_PRINTING_START + NON_PRINTING_END) end + # :nodoc: def convert_attrs_word_pair_map(str, attrs, exclusive) # then non-matching unless @word_pair_map.empty? then diff --git a/lib/rdoc/markup/formatter.rb b/lib/rdoc/markup/formatter.rb index 37a2c2751a..9daffaabb8 100644 --- a/lib/rdoc/markup/formatter.rb +++ b/lib/rdoc/markup/formatter.rb @@ -90,7 +90,7 @@ class RDoc::Markup::Formatter def add_regexp_handling_TIDYLINK @markup.add_regexp_handling(/(?: - \{.*?\} | # multi-word label + \{[^{}]*\} | # multi-word label \b[^\s{}]+? # single-word label ) diff --git a/lib/rdoc/markup/parser.rb b/lib/rdoc/markup/parser.rb index 0029df7e65..9c77048591 100644 --- a/lib/rdoc/markup/parser.rb +++ b/lib/rdoc/markup/parser.rb @@ -218,7 +218,7 @@ class RDoc::Markup::Parser break if peek_token.first == :BREAK - data << ' ' if skip :NEWLINE + data << ' ' if skip :NEWLINE and /#{SPACE_SEPARATED_LETTER_CLASS}\z/o.match?(data) else unget break @@ -420,6 +420,8 @@ class RDoc::Markup::Parser # A simple wrapper of StringScanner that is aware of the current column and lineno class MyStringScanner + # :stopdoc: + def initialize(input) @line = @column = 0 @s = StringScanner.new input @@ -456,6 +458,8 @@ class RDoc::Markup::Parser def [](i) @s[i] end + + #:startdoc: end ## diff --git a/lib/rdoc/markup/table.rb b/lib/rdoc/markup/table.rb index 7bcb10aff3..27a20f073a 100644 --- a/lib/rdoc/markup/table.rb +++ b/lib/rdoc/markup/table.rb @@ -3,12 +3,21 @@ # A section of table class RDoc::Markup::Table - attr_accessor :header, :align, :body + # headers of each column + attr_accessor :header + # alignments of each column + attr_accessor :align + + # body texts of each column + attr_accessor :body + + # Creates new instance def initialize header, align, body @header, @align, @body = header, align, body end + # :stopdoc: def == other self.class == other.class and @header == other.header and @@ -20,7 +29,7 @@ class RDoc::Markup::Table visitor.accept_table @header, @body, @align end - def pretty_print q # :nodoc: + def pretty_print q q.group 2, '[Table: ', ']' do q.group 2, '[Head: ', ']' do q.seplist @header.zip(@align) do |text, align| diff --git a/lib/rdoc/markup/to_bs.rb b/lib/rdoc/markup/to_bs.rb index f9b86487db..afd9d6e981 100644 --- a/lib/rdoc/markup/to_bs.rb +++ b/lib/rdoc/markup/to_bs.rb @@ -41,6 +41,31 @@ class RDoc::Markup::ToBs < RDoc::Markup::ToRdoc end ## + # Prepares the visitor for consuming +list_item+ + + def accept_list_item_start list_item + type = @list_type.last + + case type + when :NOTE, :LABEL then + bullets = Array(list_item.label).map do |label| + attributes(label).strip + end.join "\n" + + bullets << ":\n" unless bullets.empty? + + @prefix = ' ' * @indent + @indent += 2 + @prefix << bullets + (' ' * @indent) + else + bullet = type == :BULLET ? '*' : @list_index.last.to_s + '.' + @prefix = (' ' * @indent) + bullet.ljust(bullet.length + 1) + width = bullet.length + 1 + @indent += width + end + end + + ## # Turns on or off regexp handling for +convert_string+ def annotate tag diff --git a/lib/rdoc/markup/to_html.rb b/lib/rdoc/markup/to_html.rb index b33cb620ec..91cadf9d16 100644 --- a/lib/rdoc/markup/to_html.rb +++ b/lib/rdoc/markup/to_html.rb @@ -61,6 +61,7 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter # # These methods are used by regexp handling markup added by RDoc::Markup#add_regexp_handling. + # :nodoc: URL_CHARACTERS_REGEXP_STR = /[A-Za-z0-9\-._~:\/\?#\[\]@!$&'\(\)*+,;%=]/.source ## @@ -202,7 +203,9 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter def accept_paragraph paragraph @res << "\n<p>" text = paragraph.text @hard_break - text = text.gsub(/\r?\n/, ' ') + text = text.gsub(/(#{SPACE_SEPARATED_LETTER_CLASS})?\K\r?\n(?=(?(1)(#{SPACE_SEPARATED_LETTER_CLASS})?))/o) { + defined?($2) && ' ' + } @res << to_html(text) @res << "</p>\n" end @@ -430,7 +433,9 @@ class RDoc::Markup::ToHtml < RDoc::Markup::Formatter def parseable? text verbose, $VERBOSE = $VERBOSE, nil - eval("BEGIN {return true}\n#{text}") + catch(:valid) do + eval("BEGIN { throw :valid, true }\n#{text}") + end rescue SyntaxError false ensure diff --git a/lib/rdoc/markup/to_html_crossref.rb b/lib/rdoc/markup/to_html_crossref.rb index 434b622495..9b5de62fd6 100644 --- a/lib/rdoc/markup/to_html_crossref.rb +++ b/lib/rdoc/markup/to_html_crossref.rb @@ -42,6 +42,7 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml @cross_reference = RDoc::CrossReference.new @context end + # :nodoc: def init_link_notation_regexp_handlings add_regexp_handling_RDOCLINK @@ -62,8 +63,8 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml name = name[1..-1] unless @show_hash if name[0, 1] == '#' - if !(name.end_with?('+@', '-@')) and name =~ /(.*[^#:])@/ - text ||= "#{CGI.unescape $'} at <code>#{$1}</code>" + if !(name.end_with?('+@', '-@')) and name =~ /(.*[^#:])?@/ + text ||= [CGI.unescape($'), (" at <code>#{$1}</code>" if $~.begin(1))].join("") code = false else text ||= name @@ -138,35 +139,34 @@ class RDoc::Markup::ToHtmlCrossref < RDoc::Markup::ToHtml # Creates an HTML link to +name+ with the given +text+. def link name, text, code = true - if !(name.end_with?('+@', '-@')) and name =~ /(.*[^#:])@/ + if !(name.end_with?('+@', '-@')) and name =~ /(.*[^#:])?@/ name = $1 label = $' end - ref = @cross_reference.resolve name, text + ref = @cross_reference.resolve name, text if name case ref when String then ref else - path = ref.as_href @from_path + path = ref ? ref.as_href(@from_path) : +"" if code and RDoc::CodeObject === ref and !(RDoc::TopLevel === ref) text = "<code>#{CGI.escapeHTML text}</code>" end - if path =~ /#/ then - path << "-label-#{label}" - elsif ref.sections and - ref.sections.any? { |section| label == section.title } then - path << "##{label}" - else - if ref.respond_to?(:aref) + if label + if path =~ /#/ + path << "-label-#{label}" + elsif ref&.sections&.any? { |section| label == section.title } + path << "##{label}" + elsif ref.respond_to?(:aref) path << "##{ref.aref}-label-#{label}" else path << "#label-#{label}" end - end if label + end "<a href=\"#{path}\">#{text}</a>" end diff --git a/lib/rdoc/markup/to_html_snippet.rb b/lib/rdoc/markup/to_html_snippet.rb index 8219f0b169..f471395a3a 100644 --- a/lib/rdoc/markup/to_html_snippet.rb +++ b/lib/rdoc/markup/to_html_snippet.rb @@ -66,6 +66,9 @@ class RDoc::Markup::ToHtmlSnippet < RDoc::Markup::ToHtml alias accept_rule ignore + ## + # Adds +paragraph+ to the output + def accept_paragraph paragraph para = @in_list_entry.last || "<p>" diff --git a/lib/rdoc/markup/to_joined_paragraph.rb b/lib/rdoc/markup/to_joined_paragraph.rb index aaa343157e..31cbe0853c 100644 --- a/lib/rdoc/markup/to_joined_paragraph.rb +++ b/lib/rdoc/markup/to_joined_paragraph.rb @@ -25,9 +25,9 @@ class RDoc::Markup::ToJoinedParagraph < RDoc::Markup::Formatter def accept_paragraph paragraph parts = paragraph.parts.chunk do |part| String === part - end.map do |string, chunk| + end.flat_map do |string, chunk| string ? chunk.join.rstrip : chunk - end.flatten + end paragraph.parts.replace parts end diff --git a/lib/rdoc/markup/to_markdown.rb b/lib/rdoc/markup/to_markdown.rb index 5dd60e18f5..b915fab60b 100644 --- a/lib/rdoc/markup/to_markdown.rb +++ b/lib/rdoc/markup/to_markdown.rb @@ -45,8 +45,6 @@ class RDoc::Markup::ToMarkdown < RDoc::Markup::ToRdoc # Finishes consumption of `list` def accept_list_end list - @res << "\n" - super end @@ -60,6 +58,8 @@ class RDoc::Markup::ToMarkdown < RDoc::Markup::ToRdoc when :NOTE, :LABEL then use_prefix + @res << "\n" + 4 else @list_index[-1] = @list_index.last.succ @@ -81,11 +81,11 @@ class RDoc::Markup::ToMarkdown < RDoc::Markup::ToRdoc attributes(label).strip end.join "\n" - bullets << "\n:" + bullets << "\n" unless bullets.empty? @prefix = ' ' * @indent @indent += 4 - @prefix << bullets + (' ' * (@indent - 1)) + @prefix << bullets << ":" << (' ' * (@indent - 1)) else bullet = type == :BULLET ? '*' : @list_index.last.to_s + '.' @prefix = (' ' * @indent) + bullet.ljust(4) diff --git a/lib/rdoc/markup/to_rdoc.rb b/lib/rdoc/markup/to_rdoc.rb index 6929049582..88234f5096 100644 --- a/lib/rdoc/markup/to_rdoc.rb +++ b/lib/rdoc/markup/to_rdoc.rb @@ -145,11 +145,19 @@ class RDoc::Markup::ToRdoc < RDoc::Markup::Formatter case type when :NOTE, :LABEL then - bullets = Array(list_item.label).map do |label| + stripped_labels = Array(list_item.label).map do |label| attributes(label).strip - end.join "\n" + end + + bullets = case type + when :NOTE + stripped_labels.map { |b| "#{b}::" } + when :LABEL + stripped_labels.map { |b| "[#{b}]" } + end - bullets << ":\n" unless bullets.empty? + bullets = bullets.join("\n") + bullets << "\n" unless stripped_labels.empty? @prefix = ' ' * @indent @indent += 2 diff --git a/lib/rdoc/options.rb b/lib/rdoc/options.rb index 6ee29fd071..7518e6cc54 100644 --- a/lib/rdoc/options.rb +++ b/lib/rdoc/options.rb @@ -105,6 +105,7 @@ class RDoc::Options generator_name generator_options generators + locale op_dir page_dir option_parser diff --git a/lib/rdoc/parser.rb b/lib/rdoc/parser.rb index 3bb6f5d1f2..425105effa 100644 --- a/lib/rdoc/parser.rb +++ b/lib/rdoc/parser.rb @@ -125,9 +125,11 @@ class RDoc::Parser return parser if ext_name.empty? if parser == RDoc::Parser::Simple and ext_name !~ /txt|rdoc/ then - case check_modeline file_name + case mode = check_modeline(file_name) when nil, 'rdoc' then # continue - else return nil + else + RDoc::Parser.parsers.find { |_, p| return p if mode.casecmp?(p.name[/\w+\z/]) } + return nil end end diff --git a/lib/rdoc/parser/c.rb b/lib/rdoc/parser/c.rb index 5695bf1acb..f8f238fd74 100644 --- a/lib/rdoc/parser/c.rb +++ b/lib/rdoc/parser/c.rb @@ -575,19 +575,18 @@ class RDoc::Parser::C < RDoc::Parser table = {} file_content.scan(%r{ ((?>/\*.*?\*/\s*)?) - ((?:(?:\w+)\s+)? - (?:intern\s+)?VALUE\s+(\w+) - \s*(?:\([^)]*\))(?:[^\);]|$)) + ((?:\w+\s+){0,2} VALUE\s+(\w+) + \s*(?:\([^\)]*\))(?:[^\);]|$)) | ((?>/\*.*?\*/\s*))^\s*(\#\s*define\s+(\w+)\s+(\w+)) | ^\s*\#\s*define\s+(\w+)\s+(\w+) }xm) do case - when $1 - table[$3] = [:func_def, $1, $2, $~.offset(2)] if !table[$3] || table[$3][0] != :func_def - when $4 - table[$6] = [:macro_def, $4, $5, $~.offset(5), $7] if !table[$6] || table[$6][0] == :macro_alias - when $8 - table[$8] ||= [:macro_alias, $9] + when name = $3 + table[name] = [:func_def, $1, $2, $~.offset(2)] if !(t = table[name]) || t[0] != :func_def + when name = $6 + table[name] = [:macro_def, $4, $5, $~.offset(5), $7] if !(t = table[name]) || t[0] == :macro_alias + when name = $8 + table[name] ||= [:macro_alias, $9] end end table @@ -757,17 +756,27 @@ class RDoc::Parser::C < RDoc::Parser def gen_const_table file_content table = {} @content.scan(%r{ - ((?>^\s*/\*.*?\*/\s+)) - rb_define_(\w+)\((?:\s*(?:\w+),)?\s* - "(\w+)"\s*, + (?<doc>(?>^\s*/\*.*?\*/\s+)) + rb_define_(?<type>\w+)\(\s*(?:\w+),\s* + "(?<name>\w+)"\s*, .*?\)\s*; + | (?<doc>(?>^\s*/\*.*?\*/\s+)) + rb_file_(?<type>const)\(\s* + "(?<name>\w+)"\s*, + .*?\)\s*; + | (?<doc>(?>^\s*/\*.*?\*/\s+)) + rb_curses_define_(?<type>const)\(\s* + (?<name>\w+) + \s*\)\s*; | Document-(?:const|global|variable):\s - ((?:\w+::)*\w+) - \s*?\n((?>.*?\*/)) + (?<name>(?:\w+::)*\w+) + \s*?\n(?<doc>(?>.*?\*/)) }mxi) do - case - when $1 then table[[$2, $3]] = $1 - when $4 then table[$4] = "/*\n" + $5 + name, doc, type = $~.values_at(:name, :doc, :type) + if type + table[[type, name]] = doc + else + table[name] = "/*\n" + doc end end table @@ -939,14 +948,13 @@ class RDoc::Parser::C < RDoc::Parser # "/* definition: comment */" form. The literal ':' and '\' characters # can be escaped with a backslash. if type.downcase == 'const' then - no_match, new_definition, new_comment = comment.text.split(/(\A.*):/) + if /\A(.+?)?:(?!\S)/ =~ comment.text + new_definition, new_comment = $1, $' - if no_match and no_match.empty? then - if new_definition.empty? then # Default to literal C definition + if !new_definition # Default to literal C definition new_definition = definition else - new_definition = new_definition.gsub("\:", ":") - new_definition = new_definition.gsub("\\", '\\') + new_definition = new_definition.gsub(/\\([\\:])/, '\1') end new_definition.sub!(/\A(\s+)/, '') @@ -1217,6 +1225,9 @@ class RDoc::Parser::C < RDoc::Parser @top_level end + ## + # Creates a RDoc::Comment instance. + def new_comment text = nil, location = nil, language = nil RDoc::Comment.new(text, location, language).tap do |comment| comment.format = @markup diff --git a/lib/rdoc/parser/changelog.rb b/lib/rdoc/parser/changelog.rb index 2a685d1d79..a046241870 100644 --- a/lib/rdoc/parser/changelog.rb +++ b/lib/rdoc/parser/changelog.rb @@ -216,12 +216,22 @@ class RDoc::Parser::ChangeLog < RDoc::Parser @top_level end + ## + # The extension for Git commit log + module Git + ## + # Parses auxiliary info. Currentry `base-url` to expand + # references is effective. + def parse_info(info) /^\s*base-url\s*=\s*(.*\S)/ =~ info @base_url = $1 end + ## + # Parses the entries in the Git commit logs + def parse_entries entries = [] @@ -244,6 +254,11 @@ class RDoc::Parser::ChangeLog < RDoc::Parser entries end + ## + # Returns a list of ChangeLog entries as + # RDoc::Parser::ChangeLog::Git::LogEntry list for the given + # +entries+. + def create_entries entries # git log entries have no strictly itemized style like the old # style, just assume Markdown. diff --git a/lib/rdoc/parser/ripper_state_lex.rb b/lib/rdoc/parser/ripper_state_lex.rb index 5492f08726..f6cefd0305 100644 --- a/lib/rdoc/parser/ripper_state_lex.rb +++ b/lib/rdoc/parser/ripper_state_lex.rb @@ -1,7 +1,12 @@ # frozen_string_literal: true require 'ripper' +## +# Wrapper for Ripper lex states + class RDoc::Parser::RipperStateLex + # :stopdoc: + # TODO: Remove this constants after Ruby 2.4 EOL RIPPER_HAS_LEX_STATE = Ripper::Filter.method_defined?(:state) @@ -368,7 +373,7 @@ class RDoc::Parser::RipperStateLex private def get_symbol_tk(tk) is_symbol = true symbol_tk = Token.new(tk.line_no, tk.char_no, :on_symbol) - if ":'" == tk[:text] or ':"' == tk[:text] + if ":'" == tk[:text] or ':"' == tk[:text] or tk[:text].start_with?('%s') tk1 = get_string_tk(tk) symbol_tk[:text] = tk1[:text] symbol_tk[:state] = tk1[:state] @@ -565,6 +570,9 @@ class RDoc::Parser::RipperStateLex tk end + # :startdoc: + + # New lexer for +code+. def initialize(code) @buf = [] @heredoc_queue = [] @@ -572,6 +580,7 @@ class RDoc::Parser::RipperStateLex @tokens = @inner_lex.parse([]) end + # Returns tokens parsed from +code+. def self.parse(code) lex = self.new(code) tokens = [] @@ -584,6 +593,7 @@ class RDoc::Parser::RipperStateLex tokens end + # Returns +true+ if lex state will be +END+ after +token+. def self.end?(token) (token[:state] & EXPR_END) end diff --git a/lib/rdoc/parser/ruby.rb b/lib/rdoc/parser/ruby.rb index 2c8a19f608..85f1cd0391 100644 --- a/lib/rdoc/parser/ruby.rb +++ b/lib/rdoc/parser/ruby.rb @@ -180,6 +180,9 @@ class RDoc::Parser::Ruby < RDoc::Parser reset end + ## + # Return +true+ if +tk+ is a newline. + def tk_nl?(tk) :on_nl == tk[:kind] or :on_ignored_nl == tk[:kind] end @@ -786,8 +789,10 @@ class RDoc::Parser::Ruby < RDoc::Parser al.line = line_no read_documentation_modifiers al, RDoc::ATTR_MODIFIERS - context.add_alias al - @stats.add_alias al + if al.document_self or not @track_visibility + context.add_alias al + @stats.add_alias al + end al end @@ -1450,6 +1455,12 @@ class RDoc::Parser::Ruby < RDoc::Parser meth = RDoc::AnyMethod.new get_tkread, name look_for_directives_in meth, comment meth.singleton = single == SINGLE ? true : singleton + if singleton + # `current_line_visibility' is useless because it works against + # the normal method named as same as the singleton method, after + # the latter was defined. Of course these are different things. + container.current_line_visibility = :public + end record_location meth meth.line = line_no @@ -1773,6 +1784,7 @@ class RDoc::Parser::Ruby < RDoc::Parser nest = 1 save_visibility = container.visibility + container.visibility = :public unless current_method non_comment_seen = true diff --git a/lib/rdoc/rd/block_parser.rb b/lib/rdoc/rd/block_parser.rb index 52ea1467d8..527147d91d 100644 --- a/lib/rdoc/rd/block_parser.rb +++ b/lib/rdoc/rd/block_parser.rb @@ -1,11 +1,659 @@ # frozen_string_literal: true # # DO NOT MODIFY!!!! -# This file is automatically generated by Racc 1.6.2 -# from Racc grammar file "". +# This file is automatically generated by Racc 1.7.3 +# from Racc grammar file "block_parser.ry". # -require 'racc/parser.rb' +###### racc/parser.rb begin +unless $".find {|p| p.end_with?('/racc/parser.rb')} +$".push "#{__dir__}/racc/parser.rb" +#-- +# Copyright (c) 1999-2006 Minero Aoki +# +# This program is free software. +# You can distribute/modify this program under the same terms of ruby. +# +# As a special exception, when this code is copied by Racc +# into a Racc output file, you may use that output file +# without restriction. +#++ + +unless $".find {|p| p.end_with?('/racc/info.rb')} +$".push "#{__dir__}/racc/info.rb" + +module Racc + VERSION = '1.7.3' + Version = VERSION + Copyright = 'Copyright (c) 1999-2006 Minero Aoki' +end + +end + + +unless defined?(NotImplementedError) + NotImplementedError = NotImplementError # :nodoc: +end + +module Racc + class ParseError < StandardError; end +end +unless defined?(::ParseError) + ParseError = Racc::ParseError # :nodoc: +end + +# Racc is a LALR(1) parser generator. +# It is written in Ruby itself, and generates Ruby programs. +# +# == Command-line Reference +# +# racc [-o<var>filename</var>] [--output-file=<var>filename</var>] +# [-e<var>rubypath</var>] [--executable=<var>rubypath</var>] +# [-v] [--verbose] +# [-O<var>filename</var>] [--log-file=<var>filename</var>] +# [-g] [--debug] +# [-E] [--embedded] +# [-l] [--no-line-convert] +# [-c] [--line-convert-all] +# [-a] [--no-omit-actions] +# [-C] [--check-only] +# [-S] [--output-status] +# [--version] [--copyright] [--help] <var>grammarfile</var> +# +# [+grammarfile+] +# Racc grammar file. Any extension is permitted. +# [-o+outfile+, --output-file=+outfile+] +# A filename for output. default is <+filename+>.tab.rb +# [-O+filename+, --log-file=+filename+] +# Place logging output in file +filename+. +# Default log file name is <+filename+>.output. +# [-e+rubypath+, --executable=+rubypath+] +# output executable file(mode 755). where +path+ is the Ruby interpreter. +# [-v, --verbose] +# verbose mode. create +filename+.output file, like yacc's y.output file. +# [-g, --debug] +# add debug code to parser class. To display debugging information, +# use this '-g' option and set @yydebug true in parser class. +# [-E, --embedded] +# Output parser which doesn't need runtime files (racc/parser.rb). +# [-F, --frozen] +# Output parser which declares frozen_string_literals: true +# [-C, --check-only] +# Check syntax of racc grammar file and quit. +# [-S, --output-status] +# Print messages time to time while compiling. +# [-l, --no-line-convert] +# turns off line number converting. +# [-c, --line-convert-all] +# Convert line number of actions, inner, header and footer. +# [-a, --no-omit-actions] +# Call all actions, even if an action is empty. +# [--version] +# print Racc version and quit. +# [--copyright] +# Print copyright and quit. +# [--help] +# Print usage and quit. +# +# == Generating Parser Using Racc +# +# To compile Racc grammar file, simply type: +# +# $ racc parse.y +# +# This creates Ruby script file "parse.tab.y". The -o option can change the output filename. +# +# == Writing A Racc Grammar File +# +# If you want your own parser, you have to write a grammar file. +# A grammar file contains the name of your parser class, grammar for the parser, +# user code, and anything else. +# When writing a grammar file, yacc's knowledge is helpful. +# If you have not used yacc before, Racc is not too difficult. +# +# Here's an example Racc grammar file. +# +# class Calcparser +# rule +# target: exp { print val[0] } +# +# exp: exp '+' exp +# | exp '*' exp +# | '(' exp ')' +# | NUMBER +# end +# +# Racc grammar files resemble yacc files. +# But (of course), this is Ruby code. +# yacc's $$ is the 'result', $0, $1... is +# an array called 'val', and $-1, $-2... is an array called '_values'. +# +# See the {Grammar File Reference}[rdoc-ref:lib/racc/rdoc/grammar.en.rdoc] for +# more information on grammar files. +# +# == Parser +# +# Then you must prepare the parse entry method. There are two types of +# parse methods in Racc, Racc::Parser#do_parse and Racc::Parser#yyparse +# +# Racc::Parser#do_parse is simple. +# +# It's yyparse() of yacc, and Racc::Parser#next_token is yylex(). +# This method must returns an array like [TOKENSYMBOL, ITS_VALUE]. +# EOF is [false, false]. +# (TOKENSYMBOL is a Ruby symbol (taken from String#intern) by default. +# If you want to change this, see the grammar reference. +# +# Racc::Parser#yyparse is little complicated, but useful. +# It does not use Racc::Parser#next_token, instead it gets tokens from any iterator. +# +# For example, <code>yyparse(obj, :scan)</code> causes +# calling +obj#scan+, and you can return tokens by yielding them from +obj#scan+. +# +# == Debugging +# +# When debugging, "-v" or/and the "-g" option is helpful. +# +# "-v" creates verbose log file (.output). +# "-g" creates a "Verbose Parser". +# Verbose Parser prints the internal status when parsing. +# But it's _not_ automatic. +# You must use -g option and set +@yydebug+ to +true+ in order to get output. +# -g option only creates the verbose parser. +# +# === Racc reported syntax error. +# +# Isn't there too many "end"? +# grammar of racc file is changed in v0.10. +# +# Racc does not use '%' mark, while yacc uses huge number of '%' marks.. +# +# === Racc reported "XXXX conflicts". +# +# Try "racc -v xxxx.y". +# It causes producing racc's internal log file, xxxx.output. +# +# === Generated parsers does not work correctly +# +# Try "racc -g xxxx.y". +# This command let racc generate "debugging parser". +# Then set @yydebug=true in your parser. +# It produces a working log of your parser. +# +# == Re-distributing Racc runtime +# +# A parser, which is created by Racc, requires the Racc runtime module; +# racc/parser.rb. +# +# Ruby 1.8.x comes with Racc runtime module, +# you need NOT distribute Racc runtime files. +# +# If you want to include the Racc runtime module with your parser. +# This can be done by using '-E' option: +# +# $ racc -E -omyparser.rb myparser.y +# +# This command creates myparser.rb which `includes' Racc runtime. +# Only you must do is to distribute your parser file (myparser.rb). +# +# Note: parser.rb is ruby license, but your parser is not. +# Your own parser is completely yours. +module Racc + + unless defined?(Racc_No_Extensions) + Racc_No_Extensions = false # :nodoc: + end + + class Parser + + Racc_Runtime_Version = ::Racc::VERSION + Racc_Runtime_Core_Version_R = ::Racc::VERSION + + begin + if Object.const_defined?(:RUBY_ENGINE) and RUBY_ENGINE == 'jruby' + require 'jruby' + require 'racc/cparse-jruby.jar' + com.headius.racc.Cparse.new.load(JRuby.runtime, false) + else + require 'racc/cparse' + end + + unless new.respond_to?(:_racc_do_parse_c, true) + raise LoadError, 'old cparse.so' + end + if Racc_No_Extensions + raise LoadError, 'selecting ruby version of racc runtime core' + end + + Racc_Main_Parsing_Routine = :_racc_do_parse_c # :nodoc: + Racc_YY_Parse_Method = :_racc_yyparse_c # :nodoc: + Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_C # :nodoc: + Racc_Runtime_Type = 'c' # :nodoc: + rescue LoadError + Racc_Main_Parsing_Routine = :_racc_do_parse_rb + Racc_YY_Parse_Method = :_racc_yyparse_rb + Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_R + Racc_Runtime_Type = 'ruby' + end + + def Parser.racc_runtime_type # :nodoc: + Racc_Runtime_Type + end + + def _racc_setup + @yydebug = false unless self.class::Racc_debug_parser + @yydebug = false unless defined?(@yydebug) + if @yydebug + @racc_debug_out = $stderr unless defined?(@racc_debug_out) + @racc_debug_out ||= $stderr + end + arg = self.class::Racc_arg + arg[13] = true if arg.size < 14 + arg + end + + def _racc_init_sysvars + @racc_state = [0] + @racc_tstack = [] + @racc_vstack = [] + + @racc_t = nil + @racc_val = nil + + @racc_read_next = true + + @racc_user_yyerror = false + @racc_error_status = 0 + end + + # The entry point of the parser. This method is used with #next_token. + # If Racc wants to get token (and its value), calls next_token. + # + # Example: + # def parse + # @q = [[1,1], + # [2,2], + # [3,3], + # [false, '$']] + # do_parse + # end + # + # def next_token + # @q.shift + # end + class_eval <<~RUBY, __FILE__, __LINE__ + 1 + def do_parse + #{Racc_Main_Parsing_Routine}(_racc_setup(), false) + end + RUBY + + # The method to fetch next token. + # If you use #do_parse method, you must implement #next_token. + # + # The format of return value is [TOKEN_SYMBOL, VALUE]. + # +token-symbol+ is represented by Ruby's symbol by default, e.g. :IDENT + # for 'IDENT'. ";" (String) for ';'. + # + # The final symbol (End of file) must be false. + def next_token + raise NotImplementedError, "#{self.class}\#next_token is not defined" + end + + def _racc_do_parse_rb(arg, in_debug) + action_table, action_check, action_default, action_pointer, + _, _, _, _, + _, _, token_table, * = arg + + _racc_init_sysvars + tok = act = i = nil + + catch(:racc_end_parse) { + while true + if i = action_pointer[@racc_state[-1]] + if @racc_read_next + if @racc_t != 0 # not EOF + tok, @racc_val = next_token() + unless tok # EOF + @racc_t = 0 + else + @racc_t = (token_table[tok] or 1) # error token + end + racc_read_token(@racc_t, tok, @racc_val) if @yydebug + @racc_read_next = false + end + end + i += @racc_t + unless i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + else + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + end + } + end + + # Another entry point for the parser. + # If you use this method, you must implement RECEIVER#METHOD_ID method. + # + # RECEIVER#METHOD_ID is a method to get next token. + # It must 'yield' the token, which format is [TOKEN-SYMBOL, VALUE]. + class_eval <<~RUBY, __FILE__, __LINE__ + 1 + def yyparse(recv, mid) + #{Racc_YY_Parse_Method}(recv, mid, _racc_setup(), false) + end + RUBY + + def _racc_yyparse_rb(recv, mid, arg, c_debug) + action_table, action_check, action_default, action_pointer, + _, _, _, _, + _, _, token_table, * = arg + + _racc_init_sysvars + + catch(:racc_end_parse) { + until i = action_pointer[@racc_state[-1]] + while act = _racc_evalact(action_default[@racc_state[-1]], arg) + ; + end + end + recv.__send__(mid) do |tok, val| + unless tok + @racc_t = 0 + else + @racc_t = (token_table[tok] or 1) # error token + end + @racc_val = val + @racc_read_next = false + + i += @racc_t + unless i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + + while !(i = action_pointer[@racc_state[-1]]) || + ! @racc_read_next || + @racc_t == 0 # $ + unless i and i += @racc_t and + i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + end + end + } + end + + ### + ### common + ### + + def _racc_evalact(act, arg) + action_table, action_check, _, action_pointer, + _, _, _, _, + _, _, _, shift_n, + reduce_n, * = arg + nerr = 0 # tmp + + if act > 0 and act < shift_n + # + # shift + # + if @racc_error_status > 0 + @racc_error_status -= 1 unless @racc_t <= 1 # error token or EOF + end + @racc_vstack.push @racc_val + @racc_state.push act + @racc_read_next = true + if @yydebug + @racc_tstack.push @racc_t + racc_shift @racc_t, @racc_tstack, @racc_vstack + end + + elsif act < 0 and act > -reduce_n + # + # reduce + # + code = catch(:racc_jump) { + @racc_state.push _racc_do_reduce(arg, act) + false + } + if code + case code + when 1 # yyerror + @racc_user_yyerror = true # user_yyerror + return -reduce_n + when 2 # yyaccept + return shift_n + else + raise '[Racc Bug] unknown jump code' + end + end + + elsif act == shift_n + # + # accept + # + racc_accept if @yydebug + throw :racc_end_parse, @racc_vstack[0] + + elsif act == -reduce_n + # + # error + # + case @racc_error_status + when 0 + unless arg[21] # user_yyerror + nerr += 1 + on_error @racc_t, @racc_val, @racc_vstack + end + when 3 + if @racc_t == 0 # is $ + # We're at EOF, and another error occurred immediately after + # attempting auto-recovery + throw :racc_end_parse, nil + end + @racc_read_next = true + end + @racc_user_yyerror = false + @racc_error_status = 3 + while true + if i = action_pointer[@racc_state[-1]] + i += 1 # error token + if i >= 0 and + (act = action_table[i]) and + action_check[i] == @racc_state[-1] + break + end + end + throw :racc_end_parse, nil if @racc_state.size <= 1 + @racc_state.pop + @racc_vstack.pop + if @yydebug + @racc_tstack.pop + racc_e_pop @racc_state, @racc_tstack, @racc_vstack + end + end + return act + + else + raise "[Racc Bug] unknown action #{act.inspect}" + end + + racc_next_state(@racc_state[-1], @racc_state) if @yydebug + + nil + end + + def _racc_do_reduce(arg, act) + _, _, _, _, + goto_table, goto_check, goto_default, goto_pointer, + nt_base, reduce_table, _, _, + _, use_result, * = arg + + state = @racc_state + vstack = @racc_vstack + tstack = @racc_tstack + + i = act * -3 + len = reduce_table[i] + reduce_to = reduce_table[i+1] + method_id = reduce_table[i+2] + void_array = [] + + tmp_t = tstack[-len, len] if @yydebug + tmp_v = vstack[-len, len] + tstack[-len, len] = void_array if @yydebug + vstack[-len, len] = void_array + state[-len, len] = void_array + + # tstack must be updated AFTER method call + if use_result + vstack.push __send__(method_id, tmp_v, vstack, tmp_v[0]) + else + vstack.push __send__(method_id, tmp_v, vstack) + end + tstack.push reduce_to + + racc_reduce(tmp_t, reduce_to, tstack, vstack) if @yydebug + + k1 = reduce_to - nt_base + if i = goto_pointer[k1] + i += state[-1] + if i >= 0 and (curstate = goto_table[i]) and goto_check[i] == k1 + return curstate + end + end + goto_default[k1] + end + + # This method is called when a parse error is found. + # + # ERROR_TOKEN_ID is an internal ID of token which caused error. + # You can get string representation of this ID by calling + # #token_to_str. + # + # ERROR_VALUE is a value of error token. + # + # value_stack is a stack of symbol values. + # DO NOT MODIFY this object. + # + # This method raises ParseError by default. + # + # If this method returns, parsers enter "error recovering mode". + def on_error(t, val, vstack) + raise ParseError, sprintf("parse error on value %s (%s)", + val.inspect, token_to_str(t) || '?') + end + + # Enter error recovering mode. + # This method does not call #on_error. + def yyerror + throw :racc_jump, 1 + end + + # Exit parser. + # Return value is +Symbol_Value_Stack[0]+. + def yyaccept + throw :racc_jump, 2 + end + + # Leave error recovering mode. + def yyerrok + @racc_error_status = 0 + end + + # For debugging output + def racc_read_token(t, tok, val) + @racc_debug_out.print 'read ' + @racc_debug_out.print tok.inspect, '(', racc_token2str(t), ') ' + @racc_debug_out.puts val.inspect + @racc_debug_out.puts + end + + def racc_shift(tok, tstack, vstack) + @racc_debug_out.puts "shift #{racc_token2str tok}" + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_reduce(toks, sim, tstack, vstack) + out = @racc_debug_out + out.print 'reduce ' + if toks.empty? + out.print ' <none>' + else + toks.each {|t| out.print ' ', racc_token2str(t) } + end + out.puts " --> #{racc_token2str(sim)}" + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_accept + @racc_debug_out.puts 'accept' + @racc_debug_out.puts + end + + def racc_e_pop(state, tstack, vstack) + @racc_debug_out.puts 'error recovering mode: pop token' + racc_print_states state + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_next_state(curstate, state) + @racc_debug_out.puts "goto #{curstate}" + racc_print_states state + @racc_debug_out.puts + end + + def racc_print_stacks(t, v) + out = @racc_debug_out + out.print ' [' + t.each_index do |i| + out.print ' (', racc_token2str(t[i]), ' ', v[i].inspect, ')' + end + out.puts ' ]' + end + + def racc_print_states(s) + out = @racc_debug_out + out.print ' [' + s.each {|st| out.print ' ', st } + out.puts ' ]' + end + + def racc_token2str(tok) + self.class::Racc_token_to_s_table[tok] or + raise "[Racc Bug] can't convert token #{tok} to string" + end + + # Convert internal ID of token symbol to the string. + def token_to_str(t) + self.class::Racc_token_to_s_table[t] + end + + end + +end + +end +###### racc/parser.rb end class RDoc::RD diff --git a/lib/rdoc/rd/inline_parser.rb b/lib/rdoc/rd/inline_parser.rb index 0f0becefef..adacf64d5b 100644 --- a/lib/rdoc/rd/inline_parser.rb +++ b/lib/rdoc/rd/inline_parser.rb @@ -1,11 +1,659 @@ # frozen_string_literal: true # # DO NOT MODIFY!!!! -# This file is automatically generated by Racc 1.6.2 -# from Racc grammar file "". +# This file is automatically generated by Racc 1.7.3 +# from Racc grammar file "inline_parser.ry". # -require 'racc/parser.rb' +###### racc/parser.rb begin +unless $".find {|p| p.end_with?('/racc/parser.rb')} +$".push "#{__dir__}/racc/parser.rb" +#-- +# Copyright (c) 1999-2006 Minero Aoki +# +# This program is free software. +# You can distribute/modify this program under the same terms of ruby. +# +# As a special exception, when this code is copied by Racc +# into a Racc output file, you may use that output file +# without restriction. +#++ + +unless $".find {|p| p.end_with?('/racc/info.rb')} +$".push "#{__dir__}/racc/info.rb" + +module Racc + VERSION = '1.7.3' + Version = VERSION + Copyright = 'Copyright (c) 1999-2006 Minero Aoki' +end + +end + + +unless defined?(NotImplementedError) + NotImplementedError = NotImplementError # :nodoc: +end + +module Racc + class ParseError < StandardError; end +end +unless defined?(::ParseError) + ParseError = Racc::ParseError # :nodoc: +end + +# Racc is a LALR(1) parser generator. +# It is written in Ruby itself, and generates Ruby programs. +# +# == Command-line Reference +# +# racc [-o<var>filename</var>] [--output-file=<var>filename</var>] +# [-e<var>rubypath</var>] [--executable=<var>rubypath</var>] +# [-v] [--verbose] +# [-O<var>filename</var>] [--log-file=<var>filename</var>] +# [-g] [--debug] +# [-E] [--embedded] +# [-l] [--no-line-convert] +# [-c] [--line-convert-all] +# [-a] [--no-omit-actions] +# [-C] [--check-only] +# [-S] [--output-status] +# [--version] [--copyright] [--help] <var>grammarfile</var> +# +# [+grammarfile+] +# Racc grammar file. Any extension is permitted. +# [-o+outfile+, --output-file=+outfile+] +# A filename for output. default is <+filename+>.tab.rb +# [-O+filename+, --log-file=+filename+] +# Place logging output in file +filename+. +# Default log file name is <+filename+>.output. +# [-e+rubypath+, --executable=+rubypath+] +# output executable file(mode 755). where +path+ is the Ruby interpreter. +# [-v, --verbose] +# verbose mode. create +filename+.output file, like yacc's y.output file. +# [-g, --debug] +# add debug code to parser class. To display debugging information, +# use this '-g' option and set @yydebug true in parser class. +# [-E, --embedded] +# Output parser which doesn't need runtime files (racc/parser.rb). +# [-F, --frozen] +# Output parser which declares frozen_string_literals: true +# [-C, --check-only] +# Check syntax of racc grammar file and quit. +# [-S, --output-status] +# Print messages time to time while compiling. +# [-l, --no-line-convert] +# turns off line number converting. +# [-c, --line-convert-all] +# Convert line number of actions, inner, header and footer. +# [-a, --no-omit-actions] +# Call all actions, even if an action is empty. +# [--version] +# print Racc version and quit. +# [--copyright] +# Print copyright and quit. +# [--help] +# Print usage and quit. +# +# == Generating Parser Using Racc +# +# To compile Racc grammar file, simply type: +# +# $ racc parse.y +# +# This creates Ruby script file "parse.tab.y". The -o option can change the output filename. +# +# == Writing A Racc Grammar File +# +# If you want your own parser, you have to write a grammar file. +# A grammar file contains the name of your parser class, grammar for the parser, +# user code, and anything else. +# When writing a grammar file, yacc's knowledge is helpful. +# If you have not used yacc before, Racc is not too difficult. +# +# Here's an example Racc grammar file. +# +# class Calcparser +# rule +# target: exp { print val[0] } +# +# exp: exp '+' exp +# | exp '*' exp +# | '(' exp ')' +# | NUMBER +# end +# +# Racc grammar files resemble yacc files. +# But (of course), this is Ruby code. +# yacc's $$ is the 'result', $0, $1... is +# an array called 'val', and $-1, $-2... is an array called '_values'. +# +# See the {Grammar File Reference}[rdoc-ref:lib/racc/rdoc/grammar.en.rdoc] for +# more information on grammar files. +# +# == Parser +# +# Then you must prepare the parse entry method. There are two types of +# parse methods in Racc, Racc::Parser#do_parse and Racc::Parser#yyparse +# +# Racc::Parser#do_parse is simple. +# +# It's yyparse() of yacc, and Racc::Parser#next_token is yylex(). +# This method must returns an array like [TOKENSYMBOL, ITS_VALUE]. +# EOF is [false, false]. +# (TOKENSYMBOL is a Ruby symbol (taken from String#intern) by default. +# If you want to change this, see the grammar reference. +# +# Racc::Parser#yyparse is little complicated, but useful. +# It does not use Racc::Parser#next_token, instead it gets tokens from any iterator. +# +# For example, <code>yyparse(obj, :scan)</code> causes +# calling +obj#scan+, and you can return tokens by yielding them from +obj#scan+. +# +# == Debugging +# +# When debugging, "-v" or/and the "-g" option is helpful. +# +# "-v" creates verbose log file (.output). +# "-g" creates a "Verbose Parser". +# Verbose Parser prints the internal status when parsing. +# But it's _not_ automatic. +# You must use -g option and set +@yydebug+ to +true+ in order to get output. +# -g option only creates the verbose parser. +# +# === Racc reported syntax error. +# +# Isn't there too many "end"? +# grammar of racc file is changed in v0.10. +# +# Racc does not use '%' mark, while yacc uses huge number of '%' marks.. +# +# === Racc reported "XXXX conflicts". +# +# Try "racc -v xxxx.y". +# It causes producing racc's internal log file, xxxx.output. +# +# === Generated parsers does not work correctly +# +# Try "racc -g xxxx.y". +# This command let racc generate "debugging parser". +# Then set @yydebug=true in your parser. +# It produces a working log of your parser. +# +# == Re-distributing Racc runtime +# +# A parser, which is created by Racc, requires the Racc runtime module; +# racc/parser.rb. +# +# Ruby 1.8.x comes with Racc runtime module, +# you need NOT distribute Racc runtime files. +# +# If you want to include the Racc runtime module with your parser. +# This can be done by using '-E' option: +# +# $ racc -E -omyparser.rb myparser.y +# +# This command creates myparser.rb which `includes' Racc runtime. +# Only you must do is to distribute your parser file (myparser.rb). +# +# Note: parser.rb is ruby license, but your parser is not. +# Your own parser is completely yours. +module Racc + + unless defined?(Racc_No_Extensions) + Racc_No_Extensions = false # :nodoc: + end + + class Parser + + Racc_Runtime_Version = ::Racc::VERSION + Racc_Runtime_Core_Version_R = ::Racc::VERSION + + begin + if Object.const_defined?(:RUBY_ENGINE) and RUBY_ENGINE == 'jruby' + require 'jruby' + require 'racc/cparse-jruby.jar' + com.headius.racc.Cparse.new.load(JRuby.runtime, false) + else + require 'racc/cparse' + end + + unless new.respond_to?(:_racc_do_parse_c, true) + raise LoadError, 'old cparse.so' + end + if Racc_No_Extensions + raise LoadError, 'selecting ruby version of racc runtime core' + end + + Racc_Main_Parsing_Routine = :_racc_do_parse_c # :nodoc: + Racc_YY_Parse_Method = :_racc_yyparse_c # :nodoc: + Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_C # :nodoc: + Racc_Runtime_Type = 'c' # :nodoc: + rescue LoadError + Racc_Main_Parsing_Routine = :_racc_do_parse_rb + Racc_YY_Parse_Method = :_racc_yyparse_rb + Racc_Runtime_Core_Version = Racc_Runtime_Core_Version_R + Racc_Runtime_Type = 'ruby' + end + + def Parser.racc_runtime_type # :nodoc: + Racc_Runtime_Type + end + + def _racc_setup + @yydebug = false unless self.class::Racc_debug_parser + @yydebug = false unless defined?(@yydebug) + if @yydebug + @racc_debug_out = $stderr unless defined?(@racc_debug_out) + @racc_debug_out ||= $stderr + end + arg = self.class::Racc_arg + arg[13] = true if arg.size < 14 + arg + end + + def _racc_init_sysvars + @racc_state = [0] + @racc_tstack = [] + @racc_vstack = [] + + @racc_t = nil + @racc_val = nil + + @racc_read_next = true + + @racc_user_yyerror = false + @racc_error_status = 0 + end + + # The entry point of the parser. This method is used with #next_token. + # If Racc wants to get token (and its value), calls next_token. + # + # Example: + # def parse + # @q = [[1,1], + # [2,2], + # [3,3], + # [false, '$']] + # do_parse + # end + # + # def next_token + # @q.shift + # end + class_eval <<~RUBY, __FILE__, __LINE__ + 1 + def do_parse + #{Racc_Main_Parsing_Routine}(_racc_setup(), false) + end + RUBY + + # The method to fetch next token. + # If you use #do_parse method, you must implement #next_token. + # + # The format of return value is [TOKEN_SYMBOL, VALUE]. + # +token-symbol+ is represented by Ruby's symbol by default, e.g. :IDENT + # for 'IDENT'. ";" (String) for ';'. + # + # The final symbol (End of file) must be false. + def next_token + raise NotImplementedError, "#{self.class}\#next_token is not defined" + end + + def _racc_do_parse_rb(arg, in_debug) + action_table, action_check, action_default, action_pointer, + _, _, _, _, + _, _, token_table, * = arg + + _racc_init_sysvars + tok = act = i = nil + + catch(:racc_end_parse) { + while true + if i = action_pointer[@racc_state[-1]] + if @racc_read_next + if @racc_t != 0 # not EOF + tok, @racc_val = next_token() + unless tok # EOF + @racc_t = 0 + else + @racc_t = (token_table[tok] or 1) # error token + end + racc_read_token(@racc_t, tok, @racc_val) if @yydebug + @racc_read_next = false + end + end + i += @racc_t + unless i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + else + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + end + } + end + + # Another entry point for the parser. + # If you use this method, you must implement RECEIVER#METHOD_ID method. + # + # RECEIVER#METHOD_ID is a method to get next token. + # It must 'yield' the token, which format is [TOKEN-SYMBOL, VALUE]. + class_eval <<~RUBY, __FILE__, __LINE__ + 1 + def yyparse(recv, mid) + #{Racc_YY_Parse_Method}(recv, mid, _racc_setup(), false) + end + RUBY + + def _racc_yyparse_rb(recv, mid, arg, c_debug) + action_table, action_check, action_default, action_pointer, + _, _, _, _, + _, _, token_table, * = arg + + _racc_init_sysvars + + catch(:racc_end_parse) { + until i = action_pointer[@racc_state[-1]] + while act = _racc_evalact(action_default[@racc_state[-1]], arg) + ; + end + end + recv.__send__(mid) do |tok, val| + unless tok + @racc_t = 0 + else + @racc_t = (token_table[tok] or 1) # error token + end + @racc_val = val + @racc_read_next = false + + i += @racc_t + unless i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + + while !(i = action_pointer[@racc_state[-1]]) || + ! @racc_read_next || + @racc_t == 0 # $ + unless i and i += @racc_t and + i >= 0 and + act = action_table[i] and + action_check[i] == @racc_state[-1] + act = action_default[@racc_state[-1]] + end + while act = _racc_evalact(act, arg) + ; + end + end + end + } + end + + ### + ### common + ### + + def _racc_evalact(act, arg) + action_table, action_check, _, action_pointer, + _, _, _, _, + _, _, _, shift_n, + reduce_n, * = arg + nerr = 0 # tmp + + if act > 0 and act < shift_n + # + # shift + # + if @racc_error_status > 0 + @racc_error_status -= 1 unless @racc_t <= 1 # error token or EOF + end + @racc_vstack.push @racc_val + @racc_state.push act + @racc_read_next = true + if @yydebug + @racc_tstack.push @racc_t + racc_shift @racc_t, @racc_tstack, @racc_vstack + end + + elsif act < 0 and act > -reduce_n + # + # reduce + # + code = catch(:racc_jump) { + @racc_state.push _racc_do_reduce(arg, act) + false + } + if code + case code + when 1 # yyerror + @racc_user_yyerror = true # user_yyerror + return -reduce_n + when 2 # yyaccept + return shift_n + else + raise '[Racc Bug] unknown jump code' + end + end + + elsif act == shift_n + # + # accept + # + racc_accept if @yydebug + throw :racc_end_parse, @racc_vstack[0] + + elsif act == -reduce_n + # + # error + # + case @racc_error_status + when 0 + unless arg[21] # user_yyerror + nerr += 1 + on_error @racc_t, @racc_val, @racc_vstack + end + when 3 + if @racc_t == 0 # is $ + # We're at EOF, and another error occurred immediately after + # attempting auto-recovery + throw :racc_end_parse, nil + end + @racc_read_next = true + end + @racc_user_yyerror = false + @racc_error_status = 3 + while true + if i = action_pointer[@racc_state[-1]] + i += 1 # error token + if i >= 0 and + (act = action_table[i]) and + action_check[i] == @racc_state[-1] + break + end + end + throw :racc_end_parse, nil if @racc_state.size <= 1 + @racc_state.pop + @racc_vstack.pop + if @yydebug + @racc_tstack.pop + racc_e_pop @racc_state, @racc_tstack, @racc_vstack + end + end + return act + + else + raise "[Racc Bug] unknown action #{act.inspect}" + end + + racc_next_state(@racc_state[-1], @racc_state) if @yydebug + + nil + end + + def _racc_do_reduce(arg, act) + _, _, _, _, + goto_table, goto_check, goto_default, goto_pointer, + nt_base, reduce_table, _, _, + _, use_result, * = arg + + state = @racc_state + vstack = @racc_vstack + tstack = @racc_tstack + + i = act * -3 + len = reduce_table[i] + reduce_to = reduce_table[i+1] + method_id = reduce_table[i+2] + void_array = [] + + tmp_t = tstack[-len, len] if @yydebug + tmp_v = vstack[-len, len] + tstack[-len, len] = void_array if @yydebug + vstack[-len, len] = void_array + state[-len, len] = void_array + + # tstack must be updated AFTER method call + if use_result + vstack.push __send__(method_id, tmp_v, vstack, tmp_v[0]) + else + vstack.push __send__(method_id, tmp_v, vstack) + end + tstack.push reduce_to + + racc_reduce(tmp_t, reduce_to, tstack, vstack) if @yydebug + + k1 = reduce_to - nt_base + if i = goto_pointer[k1] + i += state[-1] + if i >= 0 and (curstate = goto_table[i]) and goto_check[i] == k1 + return curstate + end + end + goto_default[k1] + end + + # This method is called when a parse error is found. + # + # ERROR_TOKEN_ID is an internal ID of token which caused error. + # You can get string representation of this ID by calling + # #token_to_str. + # + # ERROR_VALUE is a value of error token. + # + # value_stack is a stack of symbol values. + # DO NOT MODIFY this object. + # + # This method raises ParseError by default. + # + # If this method returns, parsers enter "error recovering mode". + def on_error(t, val, vstack) + raise ParseError, sprintf("parse error on value %s (%s)", + val.inspect, token_to_str(t) || '?') + end + + # Enter error recovering mode. + # This method does not call #on_error. + def yyerror + throw :racc_jump, 1 + end + + # Exit parser. + # Return value is +Symbol_Value_Stack[0]+. + def yyaccept + throw :racc_jump, 2 + end + + # Leave error recovering mode. + def yyerrok + @racc_error_status = 0 + end + + # For debugging output + def racc_read_token(t, tok, val) + @racc_debug_out.print 'read ' + @racc_debug_out.print tok.inspect, '(', racc_token2str(t), ') ' + @racc_debug_out.puts val.inspect + @racc_debug_out.puts + end + + def racc_shift(tok, tstack, vstack) + @racc_debug_out.puts "shift #{racc_token2str tok}" + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_reduce(toks, sim, tstack, vstack) + out = @racc_debug_out + out.print 'reduce ' + if toks.empty? + out.print ' <none>' + else + toks.each {|t| out.print ' ', racc_token2str(t) } + end + out.puts " --> #{racc_token2str(sim)}" + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_accept + @racc_debug_out.puts 'accept' + @racc_debug_out.puts + end + + def racc_e_pop(state, tstack, vstack) + @racc_debug_out.puts 'error recovering mode: pop token' + racc_print_states state + racc_print_stacks tstack, vstack + @racc_debug_out.puts + end + + def racc_next_state(curstate, state) + @racc_debug_out.puts "goto #{curstate}" + racc_print_states state + @racc_debug_out.puts + end + + def racc_print_stacks(t, v) + out = @racc_debug_out + out.print ' [' + t.each_index do |i| + out.print ' (', racc_token2str(t[i]), ' ', v[i].inspect, ')' + end + out.puts ' ]' + end + + def racc_print_states(s) + out = @racc_debug_out + out.print ' [' + s.each {|st| out.print ' ', st } + out.puts ' ]' + end + + def racc_token2str(tok) + self.class::Racc_token_to_s_table[tok] or + raise "[Racc Bug] can't convert token #{tok} to string" + end + + # Convert internal ID of token symbol to the string. + def token_to_str(t) + self.class::Racc_token_to_s_table[t] + end + + end + +end + +end +###### racc/parser.rb end require 'strscan' diff --git a/lib/rdoc/rdoc.gemspec b/lib/rdoc/rdoc.gemspec index 9e925832ea..93a281c8ae 100644 --- a/lib/rdoc/rdoc.gemspec +++ b/lib/rdoc/rdoc.gemspec @@ -28,6 +28,10 @@ RDoc includes the +rdoc+ and +ri+ tools for generating and displaying documentat s.homepage = "https://ruby.github.io/rdoc" s.licenses = ["Ruby"] + s.metadata["homepage_uri"] = s.homepage + s.metadata["source_code_uri"] = "https://github.com/ruby/rdoc" + s.metadata["changelog_uri"] = "#{s.metadata["source_code_uri"]}/releases" + s.bindir = "exe" s.executables = ["rdoc", "ri"] s.require_paths = ["lib"] diff --git a/lib/rdoc/ri/driver.rb b/lib/rdoc/ri/driver.rb index 819cff8aa3..64783dc163 100644 --- a/lib/rdoc/ri/driver.rb +++ b/lib/rdoc/ri/driver.rb @@ -34,9 +34,9 @@ class RDoc::RI::Driver class NotFoundError < Error - def initialize(klass, suggestions = nil) # :nodoc: + def initialize(klass, suggestion_proc = nil) # :nodoc: @klass = klass - @suggestions = suggestions + @suggestion_proc = suggestion_proc end ## @@ -48,8 +48,9 @@ class RDoc::RI::Driver def message # :nodoc: str = "Nothing known about #{@klass}" - if @suggestions and !@suggestions.empty? - str += "\nDid you mean? #{@suggestions.join("\n ")}" + suggestions = @suggestion_proc&.call + if suggestions and !suggestions.empty? + str += "\nDid you mean? #{suggestions.join("\n ")}" end str end @@ -948,8 +949,8 @@ or the PAGER environment variable. ary = class_names.grep(Regexp.new("\\A#{klass.gsub(/(?=::|\z)/, '[^:]*')}\\z")) if ary.length != 1 && ary.first != klass if check_did_you_mean - suggestions = DidYouMean::SpellChecker.new(dictionary: class_names).correct(klass) - raise NotFoundError.new(klass, suggestions) + suggestion_proc = -> { DidYouMean::SpellChecker.new(dictionary: class_names).correct(klass) } + raise NotFoundError.new(klass, suggestion_proc) else raise NotFoundError, klass end @@ -1087,7 +1088,7 @@ or the PAGER environment variable. loop do name = if defined? Readline then - Readline.readline ">> " + Readline.readline ">> ", true else print ">> " $stdin.gets @@ -1237,8 +1238,8 @@ or the PAGER environment variable. methods.push(*store.instance_methods[klass]) if [:instance, :both].include? types end methods = methods.uniq - suggestions = DidYouMean::SpellChecker.new(dictionary: methods).correct(method_name) - raise NotFoundError.new(name, suggestions) + suggestion_proc = -> { DidYouMean::SpellChecker.new(dictionary: methods).correct(method_name) } + raise NotFoundError.new(name, suggestion_proc) else raise NotFoundError, name end diff --git a/lib/rdoc/store.rb b/lib/rdoc/store.rb index 9fc540d317..cd27d47dd1 100644 --- a/lib/rdoc/store.rb +++ b/lib/rdoc/store.rb @@ -197,6 +197,9 @@ class RDoc::Store top_level end + ## + # Sets the parser of +absolute_name+, unless it from a source code file. + def update_parser_of_file(absolute_name, parser) if top_level = @files_hash[absolute_name] then @text_files_hash[absolute_name] = top_level if top_level.text? @@ -556,9 +559,7 @@ class RDoc::Store def load_cache #orig_enc = @encoding - File.open cache_path, 'rb' do |io| - @cache = Marshal.load io - end + @cache = marshal_load(cache_path) load_enc = @cache[:encoding] @@ -615,9 +616,7 @@ class RDoc::Store def load_class_data klass_name file = class_file klass_name - File.open file, 'rb' do |io| - Marshal.load io - end + marshal_load(file) rescue Errno::ENOENT => e error = MissingFileError.new(self, file, klass_name) error.set_backtrace e.backtrace @@ -630,14 +629,10 @@ class RDoc::Store def load_method klass_name, method_name file = method_file klass_name, method_name - File.open file, 'rb' do |io| - obj = Marshal.load io - obj.store = self - obj.parent = - find_class_or_module(klass_name) || load_class(klass_name) unless - obj.parent - obj - end + obj = marshal_load(file) + obj.store = self + obj.parent ||= find_class_or_module(klass_name) || load_class(klass_name) + obj rescue Errno::ENOENT => e error = MissingFileError.new(self, file, klass_name + method_name) error.set_backtrace e.backtrace @@ -650,11 +645,9 @@ class RDoc::Store def load_page page_name file = page_file page_name - File.open file, 'rb' do |io| - obj = Marshal.load io - obj.store = self - obj - end + obj = marshal_load(file) + obj.store = self + obj rescue Errno::ENOENT => e error = MissingFileError.new(self, file, page_name) error.set_backtrace e.backtrace @@ -976,4 +969,21 @@ class RDoc::Store @unique_modules end + private + def marshal_load(file) + File.open(file, 'rb') {|io| Marshal.load(io, MarshalFilter)} + end + + MarshalFilter = proc do |obj| + case obj + when true, false, nil, Array, Class, Encoding, Hash, Integer, String, Symbol, RDoc::Text + else + unless obj.class.name.start_with?("RDoc::") + raise TypeError, "not permitted class: #{obj.class.name}" + end + end + obj + end + private_constant :MarshalFilter + end diff --git a/lib/rdoc/text.rb b/lib/rdoc/text.rb index 0bc4aba428..9804f81abe 100644 --- a/lib/rdoc/text.rb +++ b/lib/rdoc/text.rb @@ -10,6 +10,10 @@ require 'strscan' module RDoc::Text + ## + # The language for this text. This affects stripping comments + # markers. + attr_accessor :language ## @@ -309,4 +313,10 @@ module RDoc::Text res.join.strip end + ## + # Character class to be separated by a space when concatenating + # lines. + + SPACE_SEPARATED_LETTER_CLASS = /[\p{Nd}\p{Lc}\p{Pc}]|[!-~&&\W]/ + end diff --git a/lib/rdoc/token_stream.rb b/lib/rdoc/token_stream.rb index 8fc6eadd85..19ca7ed248 100644 --- a/lib/rdoc/token_stream.rb +++ b/lib/rdoc/token_stream.rb @@ -112,7 +112,7 @@ module RDoc::TokenStream # Returns a string representation of the token stream def tokens_to_s - token_stream.compact.map { |token| token[:text] }.join '' + (token_stream or return '').compact.map { |token| token[:text] }.join '' end end diff --git a/lib/rdoc/top_level.rb b/lib/rdoc/top_level.rb index e6caa34ee3..3864f66431 100644 --- a/lib/rdoc/top_level.rb +++ b/lib/rdoc/top_level.rb @@ -52,6 +52,9 @@ class RDoc::TopLevel < RDoc::Context @classes_or_modules = [] end + ## + # Sets the parser for this toplevel context, also the store. + def parser=(val) @parser = val @store.update_parser_of_file(absolute_name, val) if @store diff --git a/lib/rdoc/version.rb b/lib/rdoc/version.rb index 04322a1970..427d4ae232 100644 --- a/lib/rdoc/version.rb +++ b/lib/rdoc/version.rb @@ -5,6 +5,6 @@ module RDoc ## # RDoc version you are using - VERSION = '6.5.0' + VERSION = '6.7.0' end diff --git a/lib/readline.gemspec b/lib/readline.gemspec index 3a18f9edb6..9221c29263 100644 --- a/lib/readline.gemspec +++ b/lib/readline.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |spec| spec.name = 'readline' - spec.version = '0.0.3' + spec.version = '0.0.4' spec.authors = ['aycabta'] spec.email = ['aycabta@gmail.com'] diff --git a/lib/readline.rb b/lib/readline.rb index 29cdf3a14f..d1c9d3a955 100644 --- a/lib/readline.rb +++ b/lib/readline.rb @@ -1,5 +1,5 @@ begin - require 'readline.so' + require "readline.#{RbConfig::CONFIG["DLEXT"]}" rescue LoadError require 'reline' unless defined? Reline Object.send(:remove_const, :Readline) if Object.const_defined?(:Readline) diff --git a/lib/reline.rb b/lib/reline.rb index fd70ffc9d0..fb00b96531 100644 --- a/lib/reline.rb +++ b/lib/reline.rb @@ -1,5 +1,4 @@ require 'io/console' -require 'timeout' require 'forwardable' require 'reline/version' require 'reline/config' @@ -8,6 +7,7 @@ require 'reline/key_stroke' require 'reline/line_editor' require 'reline/history' require 'reline/terminfo' +require 'reline/face' require 'rbconfig' module Reline @@ -17,7 +17,7 @@ module Reline class ConfigEncodingConversionError < StandardError; end - Key = Struct.new('Key', :char, :combined_char, :with_meta) do + Key = Struct.new(:char, :combined_char, :with_meta) do def match?(other) case other when Reline::Key @@ -37,10 +37,8 @@ module Reline DialogRenderInfo = Struct.new( :pos, :contents, - :bg_color, - :pointer_bg_color, - :fg_color, - :pointer_fg_color, + :face, + :bg_color, # For the time being, this line should stay here for the compatibility with IRB. :width, :height, :scrollbar, @@ -77,50 +75,54 @@ module Reline def initialize self.output = STDOUT + @mutex = Mutex.new @dialog_proc_list = {} yield self @completion_quote_character = nil - @bracketed_paste_finished = false + end + + def io_gate + Reline::IOGate end def encoding - Reline::IOGate.encoding + io_gate.encoding end def completion_append_character=(val) if val.nil? @completion_append_character = nil elsif val.size == 1 - @completion_append_character = val.encode(Reline::IOGate.encoding) + @completion_append_character = val.encode(encoding) elsif val.size > 1 - @completion_append_character = val[0].encode(Reline::IOGate.encoding) + @completion_append_character = val[0].encode(encoding) else @completion_append_character = nil end end def basic_word_break_characters=(v) - @basic_word_break_characters = v.encode(Reline::IOGate.encoding) + @basic_word_break_characters = v.encode(encoding) end def completer_word_break_characters=(v) - @completer_word_break_characters = v.encode(Reline::IOGate.encoding) + @completer_word_break_characters = v.encode(encoding) end def basic_quote_characters=(v) - @basic_quote_characters = v.encode(Reline::IOGate.encoding) + @basic_quote_characters = v.encode(encoding) end def completer_quote_characters=(v) - @completer_quote_characters = v.encode(Reline::IOGate.encoding) + @completer_quote_characters = v.encode(encoding) end def filename_quote_characters=(v) - @filename_quote_characters = v.encode(Reline::IOGate.encoding) + @filename_quote_characters = v.encode(encoding) end def special_prefixes=(v) - @special_prefixes = v.encode(Reline::IOGate.encoding) + @special_prefixes = v.encode(encoding) end def completion_case_fold=(v) @@ -181,20 +183,16 @@ module Reline def input=(val) raise TypeError unless val.respond_to?(:getc) or val.nil? - if val.respond_to?(:getc) - if defined?(Reline::ANSI) and Reline::IOGate == Reline::ANSI - Reline::ANSI.input = val - elsif Reline::IOGate == Reline::GeneralIO - Reline::GeneralIO.input = val - end + if val.respond_to?(:getc) && io_gate.respond_to?(:input=) + io_gate.input = val end end def output=(val) raise TypeError unless val.respond_to?(:write) or val.nil? @output = val - if defined?(Reline::ANSI) and Reline::IOGate == Reline::ANSI - Reline::ANSI.output = val + if io_gate.respond_to?(:output=) + io_gate.output = val end end @@ -217,37 +215,30 @@ module Reline end def get_screen_size - Reline::IOGate.get_screen_size + io_gate.get_screen_size end Reline::DEFAULT_DIALOG_PROC_AUTOCOMPLETE = ->() { # autocomplete - return nil unless config.autocompletion - if just_cursor_moving and completion_journey_data.nil? - # Auto complete starts only when edited - return nil - end - pre, target, post = retrieve_completion_block(true) - if target.nil? or target.empty? or (completion_journey_data&.pointer == -1 and target.size <= 3) - return nil - end - if completion_journey_data and completion_journey_data.list - result = completion_journey_data.list.dup - result.shift - pointer = completion_journey_data.pointer - 1 - else - result = call_completion_proc_with_checking_args(pre, target, post) - pointer = nil - end - if result and result.size == 1 and result[0] == target and pointer != 0 - result = nil - end + return unless config.autocompletion + + journey_data = completion_journey_data + return unless journey_data + + target = journey_data.list.first + completed = journey_data.list[journey_data.pointer] + result = journey_data.list.drop(1) + pointer = journey_data.pointer - 1 + return if completed.empty? || (result == [completed] && pointer < 0) + target_width = Reline::Unicode.calculate_width(target) - x = cursor_pos.x - target_width - if x < 0 - x = screen_width + x + completed_width = Reline::Unicode.calculate_width(completed) + if cursor_pos.x <= completed_width - target_width + # When target is rendered on the line above cursor position + x = screen_width - completed_width y = -1 else + x = [cursor_pos.x - completed_width, 0].max y = 0 end cursor_pos_to_render = Reline::CursorPos.new(x, y) @@ -261,20 +252,21 @@ module Reline contents: result, scrollbar: true, height: [15, preferred_dialog_height].min, - bg_color: 46, - pointer_bg_color: 45, - fg_color: 37, - pointer_fg_color: 37 + face: :completion_dialog ) } Reline::DEFAULT_DIALOG_CONTEXT = Array.new def readmultiline(prompt = '', add_hist = false, &confirm_multiline_termination) - Reline::IOGate.with_raw_input do + @mutex.synchronize do unless confirm_multiline_termination raise ArgumentError.new('#readmultiline needs block to confirm multiline termination') end - inner_readline(prompt, add_hist, true, &confirm_multiline_termination) + + Reline.update_iogate + io_gate.with_raw_input do + inner_readline(prompt, add_hist, true, &confirm_multiline_termination) + end whole_buffer = line_editor.whole_buffer.dup whole_buffer.taint if RUBY_VERSION < '2.7' @@ -282,27 +274,37 @@ module Reline Reline::HISTORY << whole_buffer end - line_editor.reset_line if line_editor.whole_buffer.nil? - whole_buffer + if line_editor.eof? + line_editor.reset_line + # Return nil if the input is aborted by C-d. + nil + else + whole_buffer + end end end def readline(prompt = '', add_hist = false) - inner_readline(prompt, add_hist, false) + @mutex.synchronize do + Reline.update_iogate + io_gate.with_raw_input do + inner_readline(prompt, add_hist, false) + end - line = line_editor.line.dup - line.taint if RUBY_VERSION < '2.7' - if add_hist and line and line.chomp("\n").size > 0 - Reline::HISTORY << line.chomp("\n") - end + line = line_editor.line.dup + line.taint if RUBY_VERSION < '2.7' + if add_hist and line and line.chomp("\n").size > 0 + Reline::HISTORY << line.chomp("\n") + end - line_editor.reset_line if line_editor.line.nil? - line + line_editor.reset_line if line_editor.line.nil? + line + end end private def inner_readline(prompt, add_hist, multiline, &confirm_multiline_termination) if ENV['RELINE_STDERR_TTY'] - if Reline::IOGate.win? + if io_gate.win? $stderr = File.open(ENV['RELINE_STDERR_TTY'], 'a') else $stderr.reopen(ENV['RELINE_STDERR_TTY'], 'w') @@ -310,10 +312,14 @@ module Reline $stderr.sync = true $stderr.puts "Reline is used by #{Process.pid}" end - otio = Reline::IOGate.prep + unless config.test_mode or config.loaded? + config.read + io_gate.set_default_key_bindings(config) + end + otio = io_gate.prep may_req_ambiguous_char_width - line_editor.reset(prompt, encoding: Reline::IOGate.encoding) + line_editor.reset(prompt, encoding: encoding) if multiline line_editor.multiline_on if block_given? @@ -329,58 +335,47 @@ module Reline line_editor.prompt_proc = prompt_proc line_editor.auto_indent_proc = auto_indent_proc line_editor.dig_perfect_match_proc = dig_perfect_match_proc - line_editor.pre_input_hook = pre_input_hook - @dialog_proc_list.each_pair do |name_sym, d| - line_editor.add_dialog_proc(name_sym, d.dialog_proc, d.context) - end - - unless config.test_mode - config.read - config.reset_default_key_bindings - Reline::IOGate.set_default_key_bindings(config) + pre_input_hook&.call + unless Reline::IOGate == Reline::GeneralIO + @dialog_proc_list.each_pair do |name_sym, d| + line_editor.add_dialog_proc(name_sym, d.dialog_proc, d.context) + end end + line_editor.print_nomultiline_prompt(prompt) + line_editor.update_dialogs line_editor.rerender begin line_editor.set_signal_handlers - prev_pasting_state = false loop do - prev_pasting_state = Reline::IOGate.in_pasting? read_io(config.keyseq_timeout) { |inputs| - line_editor.set_pasting_state(Reline::IOGate.in_pasting?) - inputs.each { |c| - line_editor.input_key(c) - line_editor.rerender - } - if @bracketed_paste_finished - line_editor.rerender_all - @bracketed_paste_finished = false + line_editor.set_pasting_state(io_gate.in_pasting?) + inputs.each do |key| + if key.char == :bracketed_paste_start + text = io_gate.read_bracketed_paste + line_editor.insert_pasted_text(text) + line_editor.scroll_into_view + else + line_editor.update(key) + end end } - if prev_pasting_state == true and not Reline::IOGate.in_pasting? and not line_editor.finished? - line_editor.set_pasting_state(false) - prev_pasting_state = false - line_editor.rerender_all + if line_editor.finished? + line_editor.render_finished + break + else + line_editor.set_pasting_state(io_gate.in_pasting?) + line_editor.rerender end - break if line_editor.finished? end - Reline::IOGate.move_cursor_column(0) + io_gate.move_cursor_column(0) rescue Errno::EIO # Maybe the I/O has been closed. - rescue StandardError => e - line_editor.finalize - Reline::IOGate.deprep(otio) - raise e - rescue Exception - # Including Interrupt + ensure line_editor.finalize - Reline::IOGate.deprep(otio) - raise + io_gate.deprep(otio) end - - line_editor.finalize - Reline::IOGate.deprep(otio) end # GNU Readline waits for "keyseq-timeout" milliseconds to see if the ESC @@ -395,10 +390,9 @@ module Reline private def read_io(keyseq_timeout, &block) buffer = [] loop do - c = Reline::IOGate.getc + c = io_gate.getc(Float::INFINITY) if c == -1 result = :unmatched - @bracketed_paste_finished = true else buffer << c result = key_stroke.match_status(buffer) @@ -432,15 +426,8 @@ module Reline end private def read_2nd_character_of_key_sequence(keyseq_timeout, buffer, c, block) - begin - succ_c = nil - Timeout.timeout(keyseq_timeout / 1000.0) { - succ_c = Reline::IOGate.getc - } - rescue Timeout::Error # cancel matching only when first byte - block.([Reline::Key.new(c, c, false)]) - return :break - else + succ_c = io_gate.getc(keyseq_timeout.fdiv(1000)) + if succ_c case key_stroke.match_status(buffer.dup.push(succ_c)) when :unmatched if c == "\e".ord @@ -450,7 +437,7 @@ module Reline end return :break when :matching - Reline::IOGate.ungetc(succ_c) + io_gate.ungetc(succ_c) return :next when :matched buffer << succ_c @@ -460,27 +447,23 @@ module Reline block.(expanded) return :break end + else + block.([Reline::Key.new(c, c, false)]) + return :break end end private def read_escaped_key(keyseq_timeout, c, block) - begin - escaped_c = nil - Timeout.timeout(keyseq_timeout / 1000.0) { - escaped_c = Reline::IOGate.getc - } - rescue Timeout::Error # independent ESC + escaped_c = io_gate.getc(keyseq_timeout.fdiv(1000)) + + if escaped_c.nil? block.([Reline::Key.new(c, c, false)]) + elsif escaped_c >= 128 # maybe, first byte of multi byte + block.([Reline::Key.new(c, c, false), Reline::Key.new(escaped_c, escaped_c, false)]) + elsif escaped_c == "\e".ord # escape twice + block.([Reline::Key.new(c, c, false), Reline::Key.new(c, c, false)]) else - if escaped_c.nil? - block.([Reline::Key.new(c, c, false)]) - elsif escaped_c >= 128 # maybe, first byte of multi byte - block.([Reline::Key.new(c, c, false), Reline::Key.new(escaped_c, escaped_c, false)]) - elsif escaped_c == "\e".ord # escape twice - block.([Reline::Key.new(c, c, false), Reline::Key.new(c, c, false)]) - else - block.([Reline::Key.new(escaped_c, escaped_c | 0b10000000, true)]) - end + block.([Reline::Key.new(escaped_c, escaped_c | 0b10000000, true)]) end end @@ -490,19 +473,19 @@ module Reline end private def may_req_ambiguous_char_width - @ambiguous_width = 2 if Reline::IOGate == Reline::GeneralIO or !STDOUT.tty? + @ambiguous_width = 2 if io_gate == Reline::GeneralIO or !STDOUT.tty? return if defined? @ambiguous_width - Reline::IOGate.move_cursor_column(0) + io_gate.move_cursor_column(0) begin output.write "\u{25bd}" rescue Encoding::UndefinedConversionError # LANG=C @ambiguous_width = 1 else - @ambiguous_width = Reline::IOGate.cursor_pos.x + @ambiguous_width = io_gate.cursor_pos.x end - Reline::IOGate.move_cursor_column(0) - Reline::IOGate.erase_after_cursor + io_gate.move_cursor_column(0) + io_gate.erase_after_cursor end end @@ -565,7 +548,7 @@ module Reline @core ||= Core.new { |core| core.config = Reline::Config.new core.key_stroke = Reline::KeyStroke.new(core.config) - core.line_editor = Reline::LineEditor.new(core.config, Reline::IOGate.encoding) + core.line_editor = Reline::LineEditor.new(core.config, core.encoding) core.basic_word_break_characters = " \t\n`><=;|&{(" core.completer_word_break_characters = " \t\n`><=;|&{(" @@ -578,12 +561,24 @@ module Reline end def self.ungetc(c) - Reline::IOGate.ungetc(c) + core.io_gate.ungetc(c) end def self.line_editor core.line_editor end + + def self.update_iogate + return if core.config.test_mode + + # Need to change IOGate when `$stdout.tty?` change from false to true by `$stdout.reopen` + # Example: rails/spring boot the application in non-tty, then run console in tty. + if ENV['TERM'] != 'dumb' && core.io_gate == Reline::GeneralIO && $stdout.tty? + require 'reline/ansi' + remove_const(:IOGate) + const_set(:IOGate, Reline::ANSI) + end + end end require 'reline/general_io' @@ -604,4 +599,6 @@ else io end +Reline::Face.load_initial_configs + Reline::HISTORY = Reline::History.new(Reline.core.config) diff --git a/lib/reline/ansi.rb b/lib/reline/ansi.rb index 06ea9efd09..fa9feb2630 100644 --- a/lib/reline/ansi.rb +++ b/lib/reline/ansi.rb @@ -1,9 +1,10 @@ require 'io/console' require 'io/wait' -require 'timeout' require_relative 'terminfo' class Reline::ANSI + RESET_COLOR = "\e[0m" + CAPNAME_KEY_BINDINGS = { 'khome' => :ed_move_to_beg, 'kend' => :ed_move_to_end, @@ -14,10 +15,21 @@ class Reline::ANSI 'kcud1' => :ed_next_history, 'kcuf1' => :ed_next_char, 'kcub1' => :ed_prev_char, - 'cuu' => :ed_prev_history, - 'cud' => :ed_next_history, - 'cuf' => :ed_next_char, - 'cub' => :ed_prev_char, + } + + ANSI_CURSOR_KEY_BINDINGS = { + # Up + 'A' => [:ed_prev_history, {}], + # Down + 'B' => [:ed_next_history, {}], + # Right + 'C' => [:ed_next_char, { ctrl: :em_next_word, meta: :em_next_word }], + # Left + 'D' => [:ed_prev_char, { ctrl: :ed_prev_word, meta: :ed_prev_word }], + # End + 'F' => [:ed_move_to_end, {}], + # Home + 'H' => [:ed_move_to_beg, {}], } if Reline::Terminfo.enabled? @@ -33,23 +45,14 @@ class Reline::ANSI end def self.set_default_key_bindings(config, allow_terminfo: true) + set_bracketed_paste_key_bindings(config) + set_default_key_bindings_ansi_cursor(config) if allow_terminfo && Reline::Terminfo.enabled? set_default_key_bindings_terminfo(config) else set_default_key_bindings_comprehensive_list(config) end { - # extended entries of terminfo - [27, 91, 49, 59, 53, 67] => :em_next_word, # Ctrl+→, extended entry - [27, 91, 49, 59, 53, 68] => :ed_prev_word, # Ctrl+←, extended entry - [27, 91, 49, 59, 51, 67] => :em_next_word, # Meta+→, extended entry - [27, 91, 49, 59, 51, 68] => :ed_prev_word, # Meta+←, extended entry - }.each_pair do |key, func| - config.add_default_key_binding_by_keymap(:emacs, key, func) - config.add_default_key_binding_by_keymap(:vi_insert, key, func) - config.add_default_key_binding_by_keymap(:vi_command, key, func) - end - { [27, 91, 90] => :completion_journey_up, # S-Tab }.each_pair do |key, func| config.add_default_key_binding_by_keymap(:emacs, key, func) @@ -64,18 +67,39 @@ class Reline::ANSI end end + def self.set_bracketed_paste_key_bindings(config) + [:emacs, :vi_insert, :vi_command].each do |keymap| + config.add_default_key_binding_by_keymap(keymap, START_BRACKETED_PASTE.bytes, :bracketed_paste_start) + end + end + + def self.set_default_key_bindings_ansi_cursor(config) + ANSI_CURSOR_KEY_BINDINGS.each do |char, (default_func, modifiers)| + bindings = [["\e[#{char}", default_func]] # CSI + char + if modifiers[:ctrl] + # CSI + ctrl_key_modifier + char + bindings << ["\e[1;5#{char}", modifiers[:ctrl]] + end + if modifiers[:meta] + # CSI + meta_key_modifier + char + bindings << ["\e[1;3#{char}", modifiers[:meta]] + # Meta(ESC) + CSI + char + bindings << ["\e\e[#{char}", modifiers[:meta]] + end + bindings.each do |sequence, func| + key = sequence.bytes + config.add_default_key_binding_by_keymap(:emacs, key, func) + config.add_default_key_binding_by_keymap(:vi_insert, key, func) + config.add_default_key_binding_by_keymap(:vi_command, key, func) + end + end + end + def self.set_default_key_bindings_terminfo(config) key_bindings = CAPNAME_KEY_BINDINGS.map do |capname, key_binding| begin key_code = Reline::Terminfo.tigetstr(capname) - case capname - # Escape sequences that omit the move distance and are set to defaults - # value 1 may be sometimes sent by pressing the arrow-key. - when 'cuu', 'cud', 'cuf', 'cub' - [ key_code.sub(/%p1%d/, '').bytes, key_binding ] - else - [ key_code.bytes, key_binding ] - end + [ key_code.bytes, key_binding ] rescue Reline::Terminfo::TerminfoError # capname is undefined end @@ -94,14 +118,8 @@ class Reline::ANSI [27, 91, 49, 126] => :ed_move_to_beg, # Home [27, 91, 52, 126] => :ed_move_to_end, # End [27, 91, 51, 126] => :key_delete, # Del - [27, 91, 65] => :ed_prev_history, # ↑ - [27, 91, 66] => :ed_next_history, # ↓ - [27, 91, 67] => :ed_next_char, # → - [27, 91, 68] => :ed_prev_char, # ← # KDE - [27, 91, 72] => :ed_move_to_beg, # Home - [27, 91, 70] => :ed_move_to_end, # End # Del is 0x08 [27, 71, 65] => :ed_prev_history, # ↑ [27, 71, 66] => :ed_next_history, # ↓ @@ -118,12 +136,6 @@ class Reline::ANSI # Del is 0x08 # Arrow keys are the same of KDE - # iTerm2 - [27, 27, 91, 67] => :em_next_word, # Option+→, extended entry - [27, 27, 91, 68] => :ed_prev_word, # Option+←, extended entry - [195, 166] => :em_next_word, # Option+f - [195, 162] => :ed_prev_word, # Option+b - [27, 79, 65] => :ed_prev_history, # ↑ [27, 79, 66] => :ed_next_history, # ↓ [27, 79, 67] => :ed_next_char, # → @@ -146,17 +158,25 @@ class Reline::ANSI end def self.with_raw_input - @@input.raw { yield } + if @@input.tty? + @@input.raw(intr: true) { yield } + else + yield + end end @@buf = [] - def self.inner_getc + def self.inner_getc(timeout_second) unless @@buf.empty? return @@buf.shift end - until c = @@input.raw(intr: true) { @@input.wait_readable(0.1) && @@input.getbyte } - Reline.core.line_editor.resize + until @@input.wait_readable(0.01) + timeout_second -= 0.01 + return nil if timeout_second <= 0 + + Reline.core.line_editor.handle_signal end + c = @@input.getbyte (c == 0x16 && @@input.raw(min: 0, time: 0, &:getbyte)) || c rescue Errno::EIO # Maybe the I/O has been closed. @@ -165,48 +185,26 @@ class Reline::ANSI nil end - @@in_bracketed_paste_mode = false - START_BRACKETED_PASTE = String.new("\e[200~,", encoding: Encoding::ASCII_8BIT) - END_BRACKETED_PASTE = String.new("\e[200~.", encoding: Encoding::ASCII_8BIT) - def self.getc_with_bracketed_paste + START_BRACKETED_PASTE = String.new("\e[200~", encoding: Encoding::ASCII_8BIT) + END_BRACKETED_PASTE = String.new("\e[201~", encoding: Encoding::ASCII_8BIT) + def self.read_bracketed_paste buffer = String.new(encoding: Encoding::ASCII_8BIT) - buffer << inner_getc - while START_BRACKETED_PASTE.start_with?(buffer) or END_BRACKETED_PASTE.start_with?(buffer) do - if START_BRACKETED_PASTE == buffer - @@in_bracketed_paste_mode = true - return inner_getc - elsif END_BRACKETED_PASTE == buffer - @@in_bracketed_paste_mode = false - ungetc(-1) - return inner_getc - end - begin - succ_c = nil - Timeout.timeout(Reline.core.config.keyseq_timeout * 100) { - succ_c = inner_getc - } - rescue Timeout::Error - break - else - buffer << succ_c - end - end - buffer.bytes.reverse_each do |ch| - ungetc ch + until buffer.end_with?(END_BRACKETED_PASTE) + c = inner_getc(Float::INFINITY) + break unless c + buffer << c end - inner_getc + string = buffer.delete_suffix(END_BRACKETED_PASTE).force_encoding(encoding) + string.valid_encoding? ? string : '' end - def self.getc - if Reline.core.config.enable_bracketed_paste - getc_with_bracketed_paste - else - inner_getc - end + # if the usage expects to wait indefinitely, use Float::INFINITY for timeout_second + def self.getc(timeout_second) + inner_getc(timeout_second) end def self.in_pasting? - @@in_bracketed_paste_mode or (not Reline::IOGate.empty_buffer?) + not empty_buffer? end def self.empty_buffer? @@ -237,7 +235,7 @@ class Reline::ANSI s = [ENV["LINES"].to_i, ENV["COLUMNS"].to_i] return s if s[0] > 0 && s[1] > 0 [24, 80] - rescue Errno::ENOTTY + rescue Errno::ENOTTY, Errno::ENODEV [24, 80] end @@ -273,7 +271,7 @@ class Reline::ANSI buf = @@output.pread(@@output.pos, 0) row = buf.count("\n") column = buf.rindex("\n") ? (buf.size - buf.rindex("\n")) - 1 : 0 - rescue Errno::ESPIPE + rescue Errno::ESPIPE, IOError # Just returns column 1 for ambiguous width because this I/O is not # tty and can't seek. row = 0 @@ -304,7 +302,7 @@ class Reline::ANSI end def self.hide_cursor - if Reline::Terminfo.enabled? + if Reline::Terminfo.enabled? && Reline::Terminfo.term_supported? begin @@output.write Reline::Terminfo.tigetstr('civis') rescue Reline::Terminfo::TerminfoError @@ -316,7 +314,7 @@ class Reline::ANSI end def self.show_cursor - if Reline::Terminfo.enabled? + if Reline::Terminfo.enabled? && Reline::Terminfo.term_supported? begin @@output.write Reline::Terminfo.tigetstr('cnorm') rescue Reline::Terminfo::TerminfoError @@ -331,9 +329,12 @@ class Reline::ANSI @@output.write "\e[K" end + # This only works when the cursor is at the bottom of the scroll range + # For more details, see https://github.com/ruby/reline/pull/577#issuecomment-1646679623 def self.scroll_down(x) return if x.zero? - @@output.write "\e[#{x}S" + # We use `\n` instead of CSI + S because CSI + S would cause https://github.com/ruby/reline/issues/576 + @@output.write "\n" * x end def self.clear_screen @@ -347,11 +348,15 @@ class Reline::ANSI end def self.prep + # Enable bracketed paste + @@output.write "\e[?2004h" if Reline.core.config.enable_bracketed_paste retrieve_keybuffer nil end def self.deprep(otio) + # Disable bracketed paste + @@output.write "\e[?2004l" if Reline.core.config.enable_bracketed_paste Signal.trap('WINCH', @@old_winch_handler) if @@old_winch_handler end end diff --git a/lib/reline/config.rb b/lib/reline/config.rb index 87726393a6..62c6d105b3 100644 --- a/lib/reline/config.rb +++ b/lib/reline/config.rb @@ -8,31 +8,12 @@ class Reline::Config end VARIABLE_NAMES = %w{ - bind-tty-special-chars - blink-matching-paren - byte-oriented completion-ignore-case convert-meta disable-completion - enable-keypad - expand-tilde - history-preserve-point history-size - horizontal-scroll-mode - input-meta keyseq-timeout - mark-directories - mark-modified-lines - mark-symlinked-directories - match-hidden-files - meta-flag - output-meta - page-completions - prefer-visible-bell - print-completions-horizontally show-all-if-ambiguous - show-all-if-unmodified - visible-stats show-mode-in-prompt vi-cmd-mode-string vi-ins-mode-string @@ -53,8 +34,6 @@ class Reline::Config @additional_key_bindings[:vi_insert] = {} @additional_key_bindings[:vi_command] = {} @oneshot_key_bindings = {} - @skip_section = nil - @if_stack = nil @editing_mode_label = :emacs @keymap_label = :emacs @keymap_prefix = [] @@ -71,17 +50,15 @@ class Reline::Config @test_mode = false @autocompletion = false @convert_meta = true if seven_bit_encoding?(Reline::IOGate.encoding) + @loaded = false + @enable_bracketed_paste = true end def reset if editing_mode_is?(:vi_command) @editing_mode_label = :vi_insert end - @additional_key_bindings.keys.each do |key| - @additional_key_bindings[key].clear - end @oneshot_key_bindings.clear - reset_default_key_bindings end def editing_mode @@ -100,6 +77,10 @@ class Reline::Config @key_actors[@keymap_label] end + def loaded? + @loaded + end + def inputrc_path case ENV['INPUTRC'] when nil, '' @@ -131,6 +112,7 @@ class Reline::Config end def read(file = nil) + @loaded = true file ||= default_inputrc_path begin if file.respond_to?(:readlines) @@ -173,12 +155,6 @@ class Reline::Config @key_actors[@keymap_label].default_key_bindings[keystroke] = target end - def reset_default_key_bindings - @key_actors.values.each do |ka| - ka.reset_default_key_bindings - end - end - def read_lines(lines, file = nil) if not lines.empty? and lines.first.encoding != Reline.encoding_system_needs begin @@ -190,9 +166,7 @@ class Reline::Config end end end - conditions = [@skip_section, @if_stack] - @skip_section = nil - @if_stack = [] + if_stack = [] lines.each_with_index do |line, no| next if line.match(/\A\s*#/) @@ -201,62 +175,67 @@ class Reline::Config line = line.chomp.lstrip if line.start_with?('$') - handle_directive(line[1..-1], file, no) + handle_directive(line[1..-1], file, no, if_stack) next end - next if @skip_section + next if if_stack.any? { |_no, skip| skip } case line - when /^set +([^ ]+) +([^ ]+)/i - var, value = $1.downcase, $2 - bind_variable(var, value) + when /^set +([^ ]+) +(.+)/i + # value ignores everything after a space, raw_value does not. + var, value, raw_value = $1.downcase, $2.partition(' ').first, $2 + bind_variable(var, value, raw_value) next when /\s*("#{KEYSEQ_PATTERN}+")\s*:\s*(.*)\s*$/o key, func_name = $1, $2 + func_name = func_name.split.first keystroke, func = bind_key(key, func_name) next unless keystroke @additional_key_bindings[@keymap_label][@keymap_prefix + keystroke] = func end end - unless @if_stack.empty? - raise InvalidInputrc, "#{file}:#{@if_stack.last[1]}: unclosed if" + unless if_stack.empty? + raise InvalidInputrc, "#{file}:#{if_stack.last[0]}: unclosed if" end - ensure - @skip_section, @if_stack = conditions end - def handle_directive(directive, file, no) + def handle_directive(directive, file, no, if_stack) directive, args = directive.split(' ') case directive when 'if' condition = false case args - when 'mode' + when /^mode=(vi|emacs)$/i + mode = $1.downcase + # NOTE: mode=vi means vi-insert mode + mode = 'vi_insert' if mode == 'vi' + if @editing_mode_label == mode.to_sym + condition = true + end when 'term' when 'version' else # application name condition = true if args == 'Ruby' condition = true if args == 'Reline' end - @if_stack << [file, no, @skip_section] - @skip_section = !condition + if_stack << [no, !condition] when 'else' - if @if_stack.empty? + if if_stack.empty? raise InvalidInputrc, "#{file}:#{no}: unmatched else" end - @skip_section = !@skip_section + if_stack.last[1] = !if_stack.last[1] when 'endif' - if @if_stack.empty? + if if_stack.empty? raise InvalidInputrc, "#{file}:#{no}: unmatched endif" end - @skip_section = @if_stack.pop + if_stack.pop when 'include' - read(args) + read(File.expand_path(args)) end end - def bind_variable(name, value) + def bind_variable(name, value, raw_value) case name when 'history-size' begin @@ -281,7 +260,7 @@ class Reline::Config when 'completion-query-items' @completion_query_items = value.to_i when 'isearch-terminators' - @isearch_terminators = retrieve_string(value) + @isearch_terminators = retrieve_string(raw_value) when 'editing-mode' case value when 'emacs' @@ -323,11 +302,11 @@ class Reline::Config @show_mode_in_prompt = false end when 'vi-cmd-mode-string' - @vi_cmd_mode_string = retrieve_string(value) + @vi_cmd_mode_string = retrieve_string(raw_value) when 'vi-ins-mode-string' - @vi_ins_mode_string = retrieve_string(value) + @vi_ins_mode_string = retrieve_string(raw_value) when 'emacs-mode-string' - @emacs_mode_string = retrieve_string(value) + @emacs_mode_string = retrieve_string(raw_value) when *VARIABLE_NAMES then variable_name = :"@#{name.tr(?-, ?_)}" instance_variable_set(variable_name, value.nil? || value == '1' || value == 'on') diff --git a/lib/reline/face.rb b/lib/reline/face.rb new file mode 100644 index 0000000000..d07196e2e7 --- /dev/null +++ b/lib/reline/face.rb @@ -0,0 +1,199 @@ +# frozen_string_literal: true + +class Reline::Face + SGR_PARAMETERS = { + foreground: { + black: 30, + red: 31, + green: 32, + yellow: 33, + blue: 34, + magenta: 35, + cyan: 36, + white: 37, + bright_black: 90, + gray: 90, + bright_red: 91, + bright_green: 92, + bright_yellow: 93, + bright_blue: 94, + bright_magenta: 95, + bright_cyan: 96, + bright_white: 97 + }, + background: { + black: 40, + red: 41, + green: 42, + yellow: 43, + blue: 44, + magenta: 45, + cyan: 46, + white: 47, + bright_black: 100, + gray: 100, + bright_red: 101, + bright_green: 102, + bright_yellow: 103, + bright_blue: 104, + bright_magenta: 105, + bright_cyan: 106, + bright_white: 107, + }, + style: { + reset: 0, + bold: 1, + faint: 2, + italicized: 3, + underlined: 4, + slowly_blinking: 5, + blinking: 5, + rapidly_blinking: 6, + negative: 7, + concealed: 8, + crossed_out: 9 + } + }.freeze + + class Config + ESSENTIAL_DEFINE_NAMES = %i(default enhanced scrollbar).freeze + RESET_SGR = "\e[0m".freeze + + def initialize(name, &block) + @definition = {} + block.call(self) + ESSENTIAL_DEFINE_NAMES.each do |name| + @definition[name] ||= { style: :reset, escape_sequence: RESET_SGR } + end + end + + attr_reader :definition + + def define(name, **values) + values[:escape_sequence] = format_to_sgr(values.to_a).freeze + @definition[name] = values + end + + def reconfigure + @definition.each_value do |values| + values.delete(:escape_sequence) + values[:escape_sequence] = format_to_sgr(values.to_a).freeze + end + end + + def [](name) + @definition.dig(name, :escape_sequence) or raise ArgumentError, "unknown face: #{name}" + end + + private + + def sgr_rgb(key, value) + return nil unless rgb_expression?(value) + if Reline::Face.truecolor? + sgr_rgb_truecolor(key, value) + else + sgr_rgb_256color(key, value) + end + end + + def sgr_rgb_truecolor(key, value) + case key + when :foreground + "38;2;" + when :background + "48;2;" + end + value[1, 6].scan(/../).map(&:hex).join(";") + end + + def sgr_rgb_256color(key, value) + # 256 colors are + # 0..15: standard colors, hight intensity colors + # 16..232: 216 colors (R, G, B each 6 steps) + # 233..255: grayscale colors (24 steps) + # This methods converts rgb_expression to 216 colors + rgb = value[1, 6].scan(/../).map(&:hex) + # Color steps are [0, 95, 135, 175, 215, 255] + r, g, b = rgb.map { |v| v <= 95 ? v / 48 : (v - 35) / 40 } + color = (16 + 36 * r + 6 * g + b) + case key + when :foreground + "38;5;#{color}" + when :background + "48;5;#{color}" + end + end + + def format_to_sgr(ordered_values) + sgr = "\e[" + ordered_values.map do |key_value| + key, value = key_value + case key + when :foreground, :background + case value + when Symbol + SGR_PARAMETERS[key][value] + when String + sgr_rgb(key, value) + end + when :style + [ value ].flatten.map do |style_name| + SGR_PARAMETERS[:style][style_name] + end.then do |sgr_parameters| + sgr_parameters.include?(nil) ? nil : sgr_parameters + end + end.then do |rendition_expression| + unless rendition_expression + raise ArgumentError, "invalid SGR parameter: #{value.inspect}" + end + rendition_expression + end + end.join(';') + "m" + sgr == RESET_SGR ? RESET_SGR : RESET_SGR + sgr + end + + def rgb_expression?(color) + color.respond_to?(:match?) and color.match?(/\A#[0-9a-fA-F]{6}\z/) + end + end + + private_constant :SGR_PARAMETERS, :Config + + def self.truecolor? + @force_truecolor || %w[truecolor 24bit].include?(ENV['COLORTERM']) + end + + def self.force_truecolor + @force_truecolor = true + @configs&.each_value(&:reconfigure) + end + + def self.[](name) + @configs[name] + end + + def self.config(name, &block) + @configs ||= {} + @configs[name] = Config.new(name, &block) + end + + def self.configs + @configs.transform_values(&:definition) + end + + def self.load_initial_configs + config(:default) do |conf| + conf.define :default, style: :reset + conf.define :enhanced, style: :reset + conf.define :scrollbar, style: :reset + end + config(:completion_dialog) do |conf| + conf.define :default, foreground: :bright_white, background: :gray + conf.define :enhanced, foreground: :black, background: :white + conf.define :scrollbar, foreground: :white, background: :gray + end + end + + def self.reset_to_initial_configs + @configs = {} + load_initial_configs + end +end diff --git a/lib/reline/general_io.rb b/lib/reline/general_io.rb index 9929846568..d52151ad3c 100644 --- a/lib/reline/general_io.rb +++ b/lib/reline/general_io.rb @@ -1,10 +1,15 @@ -require 'timeout' require 'io/wait' class Reline::GeneralIO + RESET_COLOR = '' # Do not send color reset sequence + def self.reset(encoding: nil) @@pasting = false - @@encoding = encoding + if encoding + @@encoding = encoding + elsif defined?(@@encoding) + remove_class_variable(:@@encoding) + end end def self.encoding @@ -35,12 +40,13 @@ class Reline::GeneralIO yield end - def self.getc + def self.getc(_timeout_second) unless @@buf.empty? return @@buf.shift end c = nil loop do + Reline.core.line_editor.handle_signal result = @@input.wait_readable(0.1) next if result.nil? c = @@input.read(1) @@ -54,7 +60,7 @@ class Reline::GeneralIO end def self.get_screen_size - [1, 1] + [24, 80] end def self.cursor_pos @@ -97,14 +103,6 @@ class Reline::GeneralIO @@pasting end - def self.start_pasting - @@pasting = true - end - - def self.finish_pasting - @@pasting = false - end - def self.prep end diff --git a/lib/reline/history.rb b/lib/reline/history.rb index 7a1ed6b90b..3f3b65fea6 100644 --- a/lib/reline/history.rb +++ b/lib/reline/history.rb @@ -62,7 +62,7 @@ class Reline::History < Array private def check_index(index) index += size if index < 0 if index < -2147483648 or 2147483647 < index - raise RangeError.new("integer #{index} too big to convert to `int'") + raise RangeError.new("integer #{index} too big to convert to 'int'") end # If history_size is negative, history size is unlimited. if @config.history_size.positive? diff --git a/lib/reline/key_actor/base.rb b/lib/reline/key_actor/base.rb index a1cd7fb2a1..194e98938c 100644 --- a/lib/reline/key_actor/base.rb +++ b/lib/reline/key_actor/base.rb @@ -12,8 +12,4 @@ class Reline::KeyActor::Base def default_key_bindings @default_key_bindings end - - def reset_default_key_bindings - @default_key_bindings.clear - end end diff --git a/lib/reline/key_actor/emacs.rb b/lib/reline/key_actor/emacs.rb index a561feee57..edd88289a3 100644 --- a/lib/reline/key_actor/emacs.rb +++ b/lib/reline/key_actor/emacs.rb @@ -19,7 +19,7 @@ class Reline::KeyActor::Emacs < Reline::KeyActor::Base # 8 ^H :em_delete_prev_char, # 9 ^I - :ed_unassigned, + :complete, # 10 ^J :ed_newline, # 11 ^K @@ -49,13 +49,13 @@ class Reline::KeyActor::Emacs < Reline::KeyActor::Base # 23 ^W :em_kill_region, # 24 ^X - :ed_sequence_lead_in, + :ed_unassigned, # 25 ^Y :em_yank, # 26 ^Z :ed_ignore, # 27 ^[ - :em_meta_next, + :ed_unassigned, # 28 ^\ :ed_ignore, # 29 ^] @@ -63,7 +63,7 @@ class Reline::KeyActor::Emacs < Reline::KeyActor::Base # 30 ^^ :ed_unassigned, # 31 ^_ - :ed_unassigned, + :undo, # 32 SPACE :ed_insert, # 33 ! @@ -319,9 +319,9 @@ class Reline::KeyActor::Emacs < Reline::KeyActor::Base # 158 M-^^ :ed_unassigned, # 159 M-^_ - :em_copy_prev_word, - # 160 M-SPACE :ed_unassigned, + # 160 M-SPACE + :em_set_mark, # 161 M-! :ed_unassigned, # 162 M-" @@ -415,7 +415,7 @@ class Reline::KeyActor::Emacs < Reline::KeyActor::Base # 206 M-N :vi_search_next, # 207 M-O - :ed_sequence_lead_in, + :ed_unassigned, # 208 M-P :vi_search_prev, # 209 M-Q @@ -431,15 +431,15 @@ class Reline::KeyActor::Emacs < Reline::KeyActor::Base # 214 M-V :ed_unassigned, # 215 M-W - :em_copy_region, + :ed_unassigned, # 216 M-X - :ed_command, - # 217 M-Y :ed_unassigned, + # 217 M-Y + :em_yank_pop, # 218 M-Z :ed_unassigned, # 219 M-[ - :ed_sequence_lead_in, + :ed_unassigned, # 220 M-\ :ed_unassigned, # 221 M-] @@ -495,9 +495,9 @@ class Reline::KeyActor::Emacs < Reline::KeyActor::Base # 246 M-v :ed_unassigned, # 247 M-w - :em_copy_region, + :ed_unassigned, # 248 M-x - :ed_command, + :ed_unassigned, # 249 M-y :ed_unassigned, # 250 M-z diff --git a/lib/reline/key_actor/vi_command.rb b/lib/reline/key_actor/vi_command.rb index 98146d2f77..06bb0ba8e4 100644 --- a/lib/reline/key_actor/vi_command.rb +++ b/lib/reline/key_actor/vi_command.rb @@ -17,7 +17,7 @@ class Reline::KeyActor::ViCommand < Reline::KeyActor::Base # 7 ^G :ed_unassigned, # 8 ^H - :ed_unassigned, + :ed_prev_char, # 9 ^I :ed_unassigned, # 10 ^J @@ -41,7 +41,7 @@ class Reline::KeyActor::ViCommand < Reline::KeyActor::Base # 19 ^S :ed_ignore, # 20 ^T - :ed_unassigned, + :ed_transpose_chars, # 21 ^U :vi_kill_line_prev, # 22 ^V @@ -51,7 +51,7 @@ class Reline::KeyActor::ViCommand < Reline::KeyActor::Base # 24 ^X :ed_unassigned, # 25 ^Y - :ed_unassigned, + :em_yank, # 26 ^Z :ed_unassigned, # 27 ^[ @@ -75,7 +75,7 @@ class Reline::KeyActor::ViCommand < Reline::KeyActor::Base # 36 $ :ed_move_to_end, # 37 % - :vi_match, + :ed_unassigned, # 38 & :ed_unassigned, # 39 ' @@ -89,11 +89,11 @@ class Reline::KeyActor::ViCommand < Reline::KeyActor::Base # 43 + :ed_next_history, # 44 , - :vi_repeat_prev_char, + :ed_unassigned, # 45 - :ed_prev_history, # 46 . - :vi_redo, + :ed_unassigned, # 47 / :vi_search_prev, # 48 0 @@ -117,9 +117,9 @@ class Reline::KeyActor::ViCommand < Reline::KeyActor::Base # 57 9 :ed_argument_digit, # 58 : - :ed_command, + :ed_unassigned, # 59 ; - :vi_repeat_next_char, + :ed_unassigned, # 60 < :ed_unassigned, # 61 = @@ -157,21 +157,21 @@ class Reline::KeyActor::ViCommand < Reline::KeyActor::Base # 77 M :ed_unassigned, # 78 N - :vi_repeat_search_prev, + :ed_unassigned, # 79 O - :ed_sequence_lead_in, + :ed_unassigned, # 80 P :vi_paste_prev, # 81 Q :ed_unassigned, # 82 R - :vi_replace_mode, + :ed_unassigned, # 83 S - :vi_substitute_line, + :ed_unassigned, # 84 T :vi_to_prev_char, # 85 U - :vi_undo_line, + :ed_unassigned, # 86 V :ed_unassigned, # 87 W @@ -179,11 +179,11 @@ class Reline::KeyActor::ViCommand < Reline::KeyActor::Base # 88 X :ed_delete_prev_char, # 89 Y - :vi_yank_end, + :ed_unassigned, # 90 Z :ed_unassigned, # 91 [ - :ed_sequence_lead_in, + :ed_unassigned, # 92 \ :ed_unassigned, # 93 ] @@ -191,7 +191,7 @@ class Reline::KeyActor::ViCommand < Reline::KeyActor::Base # 94 ^ :vi_first_print, # 95 _ - :vi_history_word, + :ed_unassigned, # 96 ` :ed_unassigned, # 97 a @@ -221,7 +221,7 @@ class Reline::KeyActor::ViCommand < Reline::KeyActor::Base # 109 m :ed_unassigned, # 110 n - :vi_repeat_search_next, + :ed_unassigned, # 111 o :ed_unassigned, # 112 p @@ -231,11 +231,11 @@ class Reline::KeyActor::ViCommand < Reline::KeyActor::Base # 114 r :vi_replace_char, # 115 s - :vi_substitute_char, + :ed_unassigned, # 116 t :vi_to_next_char, # 117 u - :vi_undo, + :ed_unassigned, # 118 v :vi_histedit, # 119 w @@ -253,9 +253,9 @@ class Reline::KeyActor::ViCommand < Reline::KeyActor::Base # 125 } :ed_unassigned, # 126 ~ - :vi_change_case, - # 127 ^? :ed_unassigned, + # 127 ^? + :em_delete_prev_char, # 128 M-^@ :ed_unassigned, # 129 M-^A @@ -415,7 +415,7 @@ class Reline::KeyActor::ViCommand < Reline::KeyActor::Base # 206 M-N :ed_unassigned, # 207 M-O - :ed_sequence_lead_in, + :ed_unassigned, # 208 M-P :ed_unassigned, # 209 M-Q @@ -439,7 +439,7 @@ class Reline::KeyActor::ViCommand < Reline::KeyActor::Base # 218 M-Z :ed_unassigned, # 219 M-[ - :ed_sequence_lead_in, + :ed_unassigned, # 220 M-\ :ed_unassigned, # 221 M-] diff --git a/lib/reline/key_actor/vi_insert.rb b/lib/reline/key_actor/vi_insert.rb index b8e89f81d8..f8ccf468c6 100644 --- a/lib/reline/key_actor/vi_insert.rb +++ b/lib/reline/key_actor/vi_insert.rb @@ -19,7 +19,7 @@ class Reline::KeyActor::ViInsert < Reline::KeyActor::Base # 8 ^H :vi_delete_prev_char, # 9 ^I - :ed_insert, + :complete, # 10 ^J :ed_newline, # 11 ^K @@ -29,11 +29,11 @@ class Reline::KeyActor::ViInsert < Reline::KeyActor::Base # 13 ^M :ed_newline, # 14 ^N - :ed_insert, + :menu_complete, # 15 ^O :ed_insert, # 16 ^P - :ed_insert, + :menu_complete_backward, # 17 ^Q :ed_ignore, # 18 ^R @@ -41,7 +41,7 @@ class Reline::KeyActor::ViInsert < Reline::KeyActor::Base # 19 ^S :vi_search_next, # 20 ^T - :ed_insert, + :ed_transpose_chars, # 21 ^U :vi_kill_line_prev, # 22 ^V @@ -51,7 +51,7 @@ class Reline::KeyActor::ViInsert < Reline::KeyActor::Base # 24 ^X :ed_insert, # 25 ^Y - :ed_insert, + :em_yank, # 26 ^Z :ed_insert, # 27 ^[ diff --git a/lib/reline/key_stroke.rb b/lib/reline/key_stroke.rb index c1c61513a9..bceffbb53f 100644 --- a/lib/reline/key_stroke.rb +++ b/lib/reline/key_stroke.rb @@ -1,4 +1,8 @@ class Reline::KeyStroke + ESC_BYTE = 27 + CSI_PARAMETER_BYTES_RANGE = 0x30..0x3f + CSI_INTERMEDIATE_BYTES_RANGE = (0x20..0x2f) + def initialize(config) @config = config end @@ -73,17 +77,26 @@ class Reline::KeyStroke return :matched if it.max_by(&:size)&.size&.< input.size return :matching if it.size > 1 } - key_mapping.keys.select { |lhs| - start_with?(input, lhs) - }.tap { |it| - return it.size > 0 ? :matched : :unmatched - } + if key_mapping.keys.any? { |lhs| start_with?(input, lhs) } + :matched + else + match_unknown_escape_sequence(input).first + end end def expand(input) - input = compress_meta_key(input) lhs = key_mapping.keys.select { |item| start_with?(input, item) }.sort_by(&:size).last - return input unless lhs + unless lhs + status, size = match_unknown_escape_sequence(input) + case status + when :matched + return [:ed_unassigned] + expand(input.drop(size)) + when :matching + return [:ed_unassigned] + else + return input + end + end rhs = key_mapping[lhs] case rhs @@ -99,6 +112,36 @@ class Reline::KeyStroke private + # returns match status of CSI/SS3 sequence and matched length + def match_unknown_escape_sequence(input) + idx = 0 + return [:unmatched, nil] unless input[idx] == ESC_BYTE + idx += 1 + idx += 1 if input[idx] == ESC_BYTE + + case input[idx] + when nil + return [:matching, nil] + when 91 # == '['.ord + # CSI sequence + idx += 1 + idx += 1 while idx < input.size && CSI_PARAMETER_BYTES_RANGE.cover?(input[idx]) + idx += 1 while idx < input.size && CSI_INTERMEDIATE_BYTES_RANGE.cover?(input[idx]) + input[idx] ? [:matched, idx + 1] : [:matching, nil] + when 79 # == 'O'.ord + # SS3 sequence + input[idx + 1] ? [:matched, idx + 2] : [:matching, nil] + else + if idx == 1 + # `ESC char`, make it :unmatched so that it will be handled correctly in `read_2nd_character_of_key_sequence` + [:unmatched, nil] + else + # `ESC ESC char` + [:matched, idx + 1] + end + end + end + def key_mapping @config.key_bindings end diff --git a/lib/reline/kill_ring.rb b/lib/reline/kill_ring.rb index bb3684b42b..201f6f3ca0 100644 --- a/lib/reline/kill_ring.rb +++ b/lib/reline/kill_ring.rb @@ -14,7 +14,7 @@ class Reline::KillRing end def ==(other) - object_id == other.object_id + equal?(other) end end @@ -68,7 +68,7 @@ class Reline::KillRing def append(string, before_p = false) case @state when State::FRESH, State::YANK - @ring << RingPoint.new(string) + @ring << RingPoint.new(+string) @state = State::CONTINUED when State::CONTINUED, State::PROCESSED if before_p diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb index e8b83e2a6e..23ece60220 100644 --- a/lib/reline/line_editor.rb +++ b/lib/reline/line_editor.rb @@ -4,9 +4,7 @@ require 'reline/unicode' require 'tempfile' class Reline::LineEditor - # TODO: undo # TODO: Use "private alias_method" idiom after drop Ruby 2.5. - attr_reader :line attr_reader :byte_pointer attr_accessor :confirm_multiline_termination_proc attr_accessor :completion_proc @@ -14,7 +12,6 @@ class Reline::LineEditor attr_accessor :output_modifier_proc attr_accessor :prompt_proc attr_accessor :auto_indent_proc - attr_accessor :pre_input_hook attr_accessor :dig_perfect_match_proc attr_writer :output @@ -35,99 +32,93 @@ class Reline::LineEditor vi_next_big_word vi_prev_big_word vi_end_big_word - vi_repeat_next_char - vi_repeat_prev_char } module CompletionState NORMAL = :normal COMPLETION = :completion MENU = :menu - JOURNEY = :journey MENU_WITH_PERFECT_MATCH = :menu_with_perfect_match PERFECT_MATCH = :perfect_match end - CompletionJourneyData = Struct.new('CompletionJourneyData', :preposing, :postposing, :list, :pointer) - MenuInfo = Struct.new('MenuInfo', :target, :list) + RenderedScreen = Struct.new(:base_y, :lines, :cursor_y, keyword_init: true) + + CompletionJourneyState = Struct.new(:line_index, :pre, :target, :post, :list, :pointer) + + class MenuInfo + attr_reader :list + + def initialize(list) + @list = list + end + + def lines(screen_width) + return [] if @list.empty? + + list = @list.sort + sizes = list.map { |item| Reline::Unicode.calculate_width(item) } + item_width = sizes.max + 2 + num_cols = [screen_width / item_width, 1].max + num_rows = list.size.fdiv(num_cols).ceil + list_with_padding = list.zip(sizes).map { |item, size| item + ' ' * (item_width - size) } + aligned = (list_with_padding + [nil] * (num_rows * num_cols - list_with_padding.size)).each_slice(num_rows).to_a.transpose + aligned.map do |row| + row.join.rstrip + end + end + end - PROMPT_LIST_CACHE_TIMEOUT = 0.5 MINIMUM_SCROLLBAR_HEIGHT = 1 def initialize(config, encoding) @config = config @completion_append_character = '' + @screen_size = [0, 0] # Should be initialized with actual winsize in LineEditor#reset reset_variables(encoding: encoding) end - def set_pasting_state(in_pasting) - @in_pasting = in_pasting + def io_gate + Reline::IOGate end - def simplified_rendering? - if finished? - false - elsif @just_cursor_moving and not @rerender_all - true - else - not @rerender_all and not finished? and @in_pasting - end + def set_pasting_state(in_pasting) + # While pasting, text to be inserted is stored to @continuous_insertion_buffer. + # After pasting, this buffer should be force inserted. + process_insert(force: true) if @in_pasting && !in_pasting + @in_pasting = in_pasting end private def check_mode_string - mode_string = nil if @config.show_mode_in_prompt if @config.editing_mode_is?(:vi_command) - mode_string = @config.vi_cmd_mode_string + @config.vi_cmd_mode_string elsif @config.editing_mode_is?(:vi_insert) - mode_string = @config.vi_ins_mode_string + @config.vi_ins_mode_string elsif @config.editing_mode_is?(:emacs) - mode_string = @config.emacs_mode_string + @config.emacs_mode_string else - mode_string = '?' + '?' end end - if mode_string != @prev_mode_string - @rerender_all = true - end - @prev_mode_string = mode_string - mode_string end - private def check_multiline_prompt(buffer, force_recalc: false) + private def check_multiline_prompt(buffer, mode_string) if @vi_arg prompt = "(arg: #{@vi_arg}) " - @rerender_all = true elsif @searching_prompt prompt = @searching_prompt - @rerender_all = true else prompt = @prompt end - if simplified_rendering? && !force_recalc + if !@is_multiline mode_string = check_mode_string prompt = mode_string + prompt if mode_string - return [prompt, calculate_width(prompt, true), [prompt] * buffer.size] - end - if @prompt_proc - use_cached_prompt_list = false - if @cached_prompt_list - if @just_cursor_moving - use_cached_prompt_list = true - elsif Time.now.to_f < (@prompt_cache_time + PROMPT_LIST_CACHE_TIMEOUT) and buffer.size == @cached_prompt_list.size - use_cached_prompt_list = true - end - end - use_cached_prompt_list = false if @rerender_all - if use_cached_prompt_list - prompt_list = @cached_prompt_list - else - prompt_list = @cached_prompt_list = @prompt_proc.(buffer).map { |pr| pr.gsub("\n", "\\n") } - @prompt_cache_time = Time.now.to_f - end + [prompt] + [''] * (buffer.size - 1) + elsif @prompt_proc + prompt_list = @prompt_proc.(buffer).map { |pr| pr.gsub("\n", "\\n") } prompt_list.map!{ prompt } if @vi_arg or @searching_prompt prompt_list = [prompt] if prompt_list.empty? - mode_string = check_mode_string prompt_list = prompt_list.map{ |pr| mode_string + pr } if mode_string prompt = prompt_list[@line_index] prompt = prompt_list[0] if prompt.nil? @@ -137,24 +128,17 @@ class Reline::LineEditor prompt_list << prompt_list.last end end - prompt_width = calculate_width(prompt, true) - [prompt, prompt_width, prompt_list] + prompt_list else - mode_string = check_mode_string prompt = mode_string + prompt if mode_string - prompt_width = calculate_width(prompt, true) - [prompt, prompt_width, nil] + [prompt] * buffer.size end end def reset(prompt = '', encoding:) - @rest_height = (Reline::IOGate.get_screen_size.first - 1) - Reline::IOGate.cursor_pos.y @screen_size = Reline::IOGate.get_screen_size - @screen_height = @screen_size.first reset_variables(prompt, encoding: encoding) - Reline::IOGate.set_winch_handler do - @resized = true - end + @rendered_screen.base_y = Reline::IOGate.cursor_pos.y if ENV.key?('RELINE_ALT_SCROLLBAR') @full_block = '::' @upper_half_block = "''" @@ -178,67 +162,53 @@ class Reline::LineEditor end end - def resize + def handle_signal + handle_interrupted + handle_resized + end + + private def handle_resized return unless @resized - @resized = false - @rest_height = (Reline::IOGate.get_screen_size.first - 1) - Reline::IOGate.cursor_pos.y - old_screen_size = @screen_size + @screen_size = Reline::IOGate.get_screen_size - @screen_height = @screen_size.first - if old_screen_size.last < @screen_size.last # columns increase - @rerender_all = true - rerender + @resized = false + scroll_into_view + Reline::IOGate.move_cursor_up @rendered_screen.cursor_y + @rendered_screen.base_y = Reline::IOGate.cursor_pos.y + @rendered_screen.lines = [] + @rendered_screen.cursor_y = 0 + render_differential + end + + private def handle_interrupted + return unless @interrupted + + @interrupted = false + clear_dialogs + scrolldown = render_differential + Reline::IOGate.scroll_down scrolldown + Reline::IOGate.move_cursor_column 0 + @rendered_screen.lines = [] + @rendered_screen.cursor_y = 0 + case @old_trap + when 'DEFAULT', 'SYSTEM_DEFAULT' + raise Interrupt + when 'IGNORE' + # Do nothing + when 'EXIT' + exit else - back = 0 - new_buffer = whole_lines - prompt, prompt_width, prompt_list = check_multiline_prompt(new_buffer) - new_buffer.each_with_index do |line, index| - prompt_width = calculate_width(prompt_list[index], true) if @prompt_proc - width = prompt_width + calculate_width(line) - height = calculate_height_by_width(width) - back += height - end - @highest_in_all = back - @highest_in_this = calculate_height_by_width(prompt_width + @cursor_max) - @first_line_started_from = - if @line_index.zero? - 0 - else - calculate_height_by_lines(@buffer_of_lines[0..(@line_index - 1)], prompt_list || prompt) - end - if @prompt_proc - prompt = prompt_list[@line_index] - prompt_width = calculate_width(prompt, true) - end - calculate_nearest_cursor - @started_from = calculate_height_by_width(prompt_width + @cursor) - 1 - Reline::IOGate.move_cursor_column((prompt_width + @cursor) % @screen_size.last) - @highest_in_this = calculate_height_by_width(prompt_width + @cursor_max) - @rerender_all = true + @old_trap.call if @old_trap.respond_to?(:call) end end def set_signal_handlers - @old_trap = Signal.trap('INT') { - clear_dialog(0) - if @scroll_partial_screen - move_cursor_down(@screen_height - (@line_index - @scroll_partial_screen) - 1) - else - move_cursor_down(@highest_in_all - @line_index - 1) - end - Reline::IOGate.move_cursor_column(0) - scroll_down(1) - case @old_trap - when 'DEFAULT', 'SYSTEM_DEFAULT' - raise Interrupt - when 'IGNORE' - # Do nothing - when 'EXIT' - exit - else - @old_trap.call if @old_trap.respond_to?(:call) - end - } + Reline::IOGate.set_winch_handler do + @resized = true + end + @old_trap = Signal.trap('INT') do + @interrupted = true + end end def finalize @@ -255,56 +225,43 @@ class Reline::LineEditor @encoding = encoding @is_multiline = false @finished = false - @cleared = false - @rerender_all = false @history_pointer = nil @kill_ring ||= Reline::KillRing.new @vi_clipboard = '' @vi_arg = nil @waiting_proc = nil - @waiting_operator_proc = nil - @waiting_operator_vi_arg = nil - @completion_journey_data = nil + @vi_waiting_operator = nil + @vi_waiting_operator_arg = nil + @completion_journey_state = nil @completion_state = CompletionState::NORMAL @perfect_matched = nil @menu_info = nil - @first_prompt = true @searching_prompt = nil @first_char = true - @add_newline_to_end_of_buffer = false - @just_cursor_moving = nil - @cached_prompt_list = nil - @prompt_cache_time = nil + @just_cursor_moving = false @eof = false @continuous_insertion_buffer = String.new(encoding: @encoding) - @scroll_partial_screen = nil - @prev_mode_string = nil + @scroll_partial_screen = 0 @drop_terminate_spaces = false @in_pasting = false @auto_indent_proc = nil @dialogs = [] - @previous_rendered_dialog_y = 0 - @last_key = nil + @interrupted = false @resized = false + @cache = {} + @rendered_screen = RenderedScreen.new(base_y: 0, lines: [], cursor_y: 0) + @past_lines = [] + @undoing = false reset_line end def reset_line - @cursor = 0 - @cursor_max = 0 @byte_pointer = 0 @buffer_of_lines = [String.new(encoding: @encoding)] @line_index = 0 - @previous_line_index = nil - @line = @buffer_of_lines[0] - @first_line_started_from = 0 - @move_up = 0 - @started_from = 0 - @highest_in_this = 1 - @highest_in_all = 1 + @cache.clear @line_backup_in_history = nil @multibyte_buffer = String.new(encoding: 'ASCII-8BIT') - @check_new_auto_indent = false end def multiline_on @@ -315,68 +272,44 @@ class Reline::LineEditor @is_multiline = false end - private def calculate_height_by_lines(lines, prompt) - result = 0 - prompt_list = prompt.is_a?(Array) ? prompt : nil - lines.each_with_index { |line, i| - prompt = prompt_list[i] if prompt_list and prompt_list[i] - result += calculate_height_by_width(calculate_width(prompt, true) + calculate_width(line)) - } - result - end - private def insert_new_line(cursor_line, next_line) - @line = cursor_line @buffer_of_lines.insert(@line_index + 1, String.new(next_line, encoding: @encoding)) - @previous_line_index = @line_index + @buffer_of_lines[@line_index] = cursor_line @line_index += 1 - @just_cursor_moving = false - end - - private def calculate_height_by_width(width) - width.div(@screen_size.last) + 1 - end - - private def split_by_width(str, max_width) - Reline::Unicode.split_by_width(str, max_width, @encoding) - end - - private def scroll_down(val) - if val <= @rest_height - Reline::IOGate.move_cursor_down(val) - @rest_height -= val - else - Reline::IOGate.move_cursor_down(@rest_height) - Reline::IOGate.scroll_down(val - @rest_height) - @rest_height = 0 + @byte_pointer = 0 + if @auto_indent_proc && !@in_pasting + if next_line.empty? + ( + # For compatibility, use this calculation instead of just `process_auto_indent @line_index - 1, cursor_dependent: false` + indent1 = @auto_indent_proc.(@buffer_of_lines.take(@line_index - 1).push(''), @line_index - 1, 0, true) + indent2 = @auto_indent_proc.(@buffer_of_lines.take(@line_index), @line_index - 1, @buffer_of_lines[@line_index - 1].bytesize, false) + indent = indent2 || indent1 + @buffer_of_lines[@line_index - 1] = ' ' * indent + @buffer_of_lines[@line_index - 1].gsub(/\A\s*/, '') + ) + process_auto_indent @line_index, add_newline: true + else + process_auto_indent @line_index - 1, cursor_dependent: false + process_auto_indent @line_index, add_newline: true # Need for compatibility + process_auto_indent @line_index, cursor_dependent: false + end end end - private def move_cursor_up(val) - if val > 0 - Reline::IOGate.move_cursor_up(val) - @rest_height += val - elsif val < 0 - move_cursor_down(-val) - end + private def split_by_width(str, max_width, offset: 0) + Reline::Unicode.split_by_width(str, max_width, @encoding, offset: offset) end - private def move_cursor_down(val) - if val > 0 - Reline::IOGate.move_cursor_down(val) - @rest_height -= val - @rest_height = 0 if @rest_height < 0 - elsif val < 0 - move_cursor_up(-val) - end + def current_byte_pointer_cursor + calculate_width(current_line.byteslice(0, @byte_pointer)) end - private def calculate_nearest_cursor(line_to_calc = @line, cursor = @cursor, started_from = @started_from, byte_pointer = @byte_pointer, update = true) + private def calculate_nearest_cursor(cursor) + line_to_calc = current_line new_cursor_max = calculate_width(line_to_calc) new_cursor = 0 new_byte_pointer = 0 height = 1 - max_width = @screen_size.last + max_width = screen_width if @config.editing_mode_is?(:vi_command) last_byte_size = Reline::Unicode.get_prev_mbchar_size(line_to_calc, line_to_calc.bytesize) if last_byte_size > 0 @@ -402,113 +335,242 @@ class Reline::LineEditor end new_byte_pointer += gc.bytesize end - new_started_from = height - 1 - if update - @cursor = new_cursor - @cursor_max = new_cursor_max - @started_from = new_started_from - @byte_pointer = new_byte_pointer - else - [new_cursor, new_cursor_max, new_started_from, new_byte_pointer] + @byte_pointer = new_byte_pointer + end + + def with_cache(key, *deps) + cached_deps, value = @cache[key] + if cached_deps != deps + @cache[key] = [deps, value = yield(*deps, cached_deps, value)] end + value end - def rerender_all - @rerender_all = true - process_insert(force: true) - rerender + def modified_lines + with_cache(__method__, whole_lines, finished?) do |whole, complete| + modify_lines(whole, complete) + end end - def rerender - return if @line.nil? - if @menu_info - scroll_down(@highest_in_all - @first_line_started_from) - @rerender_all = true + def prompt_list + with_cache(__method__, whole_lines, check_mode_string, @vi_arg, @searching_prompt) do |lines, mode_string| + check_multiline_prompt(lines, mode_string) end - if @menu_info - show_menu - @menu_info = nil - end - prompt, prompt_width, prompt_list = check_multiline_prompt(whole_lines) - cursor_column = (prompt_width + @cursor) % @screen_size.last - if @cleared - clear_screen_buffer(prompt, prompt_list, prompt_width) - @cleared = false - return + end + + def screen_height + @screen_size.first + end + + def screen_width + @screen_size.last + end + + def screen_scroll_top + @scroll_partial_screen + end + + def wrapped_prompt_and_input_lines + with_cache(__method__, @buffer_of_lines.size, modified_lines, prompt_list, screen_width) do |n, lines, prompts, width, prev_cache_key, cached_value| + prev_n, prev_lines, prev_prompts, prev_width = prev_cache_key + cached_wraps = {} + if prev_width == width + prev_n.times do |i| + cached_wraps[[prev_prompts[i], prev_lines[i]]] = cached_value[i] + end + end + + n.times.map do |i| + prompt = prompts[i] || '' + line = lines[i] || '' + if (cached = cached_wraps[[prompt, line]]) + next cached + end + *wrapped_prompts, code_line_prompt = split_by_width(prompt, width).first.compact + wrapped_lines = split_by_width(line, width, offset: calculate_width(code_line_prompt, true)).first.compact + wrapped_prompts.map { |p| [p, ''] } + [[code_line_prompt, wrapped_lines.first]] + wrapped_lines.drop(1).map { |c| ['', c] } + end end - if @is_multiline and finished? and @scroll_partial_screen - # Re-output all code higher than the screen when finished. - Reline::IOGate.move_cursor_up(@first_line_started_from + @started_from - @scroll_partial_screen) - Reline::IOGate.move_cursor_column(0) - @scroll_partial_screen = nil - new_lines = whole_lines - prompt, prompt_width, prompt_list = check_multiline_prompt(new_lines) - modify_lines(new_lines).each_with_index do |line, index| - @output.write "#{prompt_list ? prompt_list[index] : prompt}#{line}\r\n" - Reline::IOGate.erase_after_cursor - end - @output.flush - clear_dialog(cursor_column) - return + end + + def calculate_overlay_levels(overlay_levels) + levels = [] + overlay_levels.each do |x, w, l| + levels.fill(l, x, w) end - new_highest_in_this = calculate_height_by_width(prompt_width + calculate_width(@line.nil? ? '' : @line)) - rendered = false - if @add_newline_to_end_of_buffer - clear_dialog_with_trap_key(cursor_column) - rerender_added_newline(prompt, prompt_width, prompt_list) - @add_newline_to_end_of_buffer = false - else - if @just_cursor_moving and not @rerender_all - clear_dialog_with_trap_key(cursor_column) - rendered = just_move_cursor - @just_cursor_moving = false - return - elsif @previous_line_index or new_highest_in_this != @highest_in_this - clear_dialog_with_trap_key(cursor_column) - rerender_changed_current_line - @previous_line_index = nil - rendered = true - elsif @rerender_all - rerender_all_lines - @rerender_all = false - rendered = true + levels + end + + def render_line_differential(old_items, new_items) + old_levels = calculate_overlay_levels(old_items.zip(new_items).each_with_index.map {|((x, w, c), (nx, _nw, nc)), i| [x, w, c == nc && x == nx ? i : -1] if x }.compact) + new_levels = calculate_overlay_levels(new_items.each_with_index.map { |(x, w), i| [x, w, i] if x }.compact).take(screen_width) + base_x = 0 + new_levels.zip(old_levels).chunk { |n, o| n == o ? :skip : n || :blank }.each do |level, chunk| + width = chunk.size + if level == :skip + # do nothing + elsif level == :blank + Reline::IOGate.move_cursor_column base_x + @output.write "#{Reline::IOGate::RESET_COLOR}#{' ' * width}" else + x, w, content = new_items[level] + cover_begin = base_x != 0 && new_levels[base_x - 1] == level + cover_end = new_levels[base_x + width] == level + pos = 0 + unless x == base_x && w == width + content, pos = Reline::Unicode.take_mbchar_range(content, base_x - x, width, cover_begin: cover_begin, cover_end: cover_end, padding: true) + end + Reline::IOGate.move_cursor_column x + pos + @output.write "#{Reline::IOGate::RESET_COLOR}#{content}#{Reline::IOGate::RESET_COLOR}" end + base_x += width end - if @is_multiline - if finished? - # Always rerender on finish because output_modifier_proc may return a different output. - new_lines = whole_lines - line = modify_lines(new_lines)[@line_index] - clear_dialog(cursor_column) - prompt, prompt_width, prompt_list = check_multiline_prompt(new_lines) - render_partial(prompt, prompt_width, line, @first_line_started_from) - move_cursor_down(@highest_in_all - (@first_line_started_from + @highest_in_this - 1) - 1) - scroll_down(1) - Reline::IOGate.move_cursor_column(0) - Reline::IOGate.erase_after_cursor - else - if not rendered and not @in_pasting - line = modify_lines(whole_lines)[@line_index] - prompt, prompt_width, prompt_list = check_multiline_prompt(whole_lines) - render_partial(prompt, prompt_width, line, @first_line_started_from) - end - render_dialog(cursor_column) + if old_levels.size > new_levels.size + Reline::IOGate.move_cursor_column new_levels.size + Reline::IOGate.erase_after_cursor + end + end + + # Calculate cursor position in word wrapped content. + def wrapped_cursor_position + prompt_width = calculate_width(prompt_list[@line_index], true) + line_before_cursor = whole_lines[@line_index].byteslice(0, @byte_pointer) + wrapped_line_before_cursor = split_by_width(' ' * prompt_width + line_before_cursor, screen_width).first.compact + wrapped_cursor_y = wrapped_prompt_and_input_lines[0...@line_index].sum(&:size) + wrapped_line_before_cursor.size - 1 + wrapped_cursor_x = calculate_width(wrapped_line_before_cursor.last) + [wrapped_cursor_x, wrapped_cursor_y] + end + + def clear_dialogs + @dialogs.each do |dialog| + dialog.contents = nil + dialog.trap_key = nil + end + end + + def update_dialogs(key = nil) + wrapped_cursor_x, wrapped_cursor_y = wrapped_cursor_position + @dialogs.each do |dialog| + dialog.trap_key = nil + update_each_dialog(dialog, wrapped_cursor_x, wrapped_cursor_y - screen_scroll_top, key) + end + end + + def render_finished + clear_rendered_lines + render_full_content + end + + def clear_rendered_lines + Reline::IOGate.move_cursor_up @rendered_screen.cursor_y + Reline::IOGate.move_cursor_column 0 + + num_lines = @rendered_screen.lines.size + return unless num_lines && num_lines >= 1 + + Reline::IOGate.move_cursor_down num_lines - 1 + (num_lines - 1).times do + Reline::IOGate.erase_after_cursor + Reline::IOGate.move_cursor_up 1 + end + Reline::IOGate.erase_after_cursor + @rendered_screen.lines = [] + @rendered_screen.cursor_y = 0 + end + + def render_full_content + lines = @buffer_of_lines.size.times.map do |i| + line = prompt_list[i] + modified_lines[i] + wrapped_lines, = split_by_width(line, screen_width) + wrapped_lines.last.empty? ? "#{line} " : line + end + @output.puts lines.map { |l| "#{l}\r\n" }.join + end + + def print_nomultiline_prompt(prompt) + return unless prompt && !@is_multiline + + # Readline's test `TestRelineAsReadline#test_readline` requires first output to be prompt, not cursor reset escape sequence. + @rendered_screen.lines = [[[0, Reline::Unicode.calculate_width(prompt, true), prompt]]] + @rendered_screen.cursor_y = 0 + @output.write prompt + end + + def render_differential + wrapped_cursor_x, wrapped_cursor_y = wrapped_cursor_position + + rendered_lines = @rendered_screen.lines + new_lines = wrapped_prompt_and_input_lines.flatten(1)[screen_scroll_top, screen_height].map do |prompt, line| + prompt_width = Reline::Unicode.calculate_width(prompt, true) + [[0, prompt_width, prompt], [prompt_width, Reline::Unicode.calculate_width(line, true), line]] + end + if @menu_info + @menu_info.lines(screen_width).each do |item| + new_lines << [[0, Reline::Unicode.calculate_width(item), item]] end - @buffer_of_lines[@line_index] = @line - @rest_height = 0 if @scroll_partial_screen - else - line = modify_lines(whole_lines)[@line_index] - render_partial(prompt, prompt_width, line, 0) - if finished? - scroll_down(1) - Reline::IOGate.move_cursor_column(0) - Reline::IOGate.erase_after_cursor + @menu_info = nil # TODO: do not change state here + end + + @dialogs.each_with_index do |dialog, index| + next unless dialog.contents + + x_range, y_range = dialog_range dialog, wrapped_cursor_y - screen_scroll_top + y_range.each do |row| + next if row < 0 || row >= screen_height + dialog_rows = new_lines[row] ||= [] + # index 0 is for prompt, index 1 is for line, index 2.. is for dialog + dialog_rows[index + 2] = [x_range.begin, dialog.width, dialog.contents[row - y_range.begin]] end end + + cursor_y = @rendered_screen.cursor_y + if new_lines != rendered_lines + # Hide cursor while rendering to avoid cursor flickering. + Reline::IOGate.hide_cursor + num_lines = [[new_lines.size, rendered_lines.size].max, screen_height].min + if @rendered_screen.base_y + num_lines > screen_height + Reline::IOGate.scroll_down(num_lines - cursor_y - 1) + @rendered_screen.base_y = screen_height - num_lines + cursor_y = num_lines - 1 + end + num_lines.times do |i| + rendered_line = rendered_lines[i] || [] + line_to_render = new_lines[i] || [] + next if rendered_line == line_to_render + + Reline::IOGate.move_cursor_down i - cursor_y + cursor_y = i + unless rendered_lines[i] + Reline::IOGate.move_cursor_column 0 + Reline::IOGate.erase_after_cursor + end + render_line_differential(rendered_line, line_to_render) + end + @rendered_screen.lines = new_lines + Reline::IOGate.show_cursor + end + y = wrapped_cursor_y - screen_scroll_top + Reline::IOGate.move_cursor_column wrapped_cursor_x + Reline::IOGate.move_cursor_down y - cursor_y + @rendered_screen.cursor_y = y + new_lines.size - y + end + + def upper_space_height(wrapped_cursor_y) + wrapped_cursor_y - screen_scroll_top + end + + def rest_height(wrapped_cursor_y) + screen_height - wrapped_cursor_y + screen_scroll_top - @rendered_screen.base_y - 1 + end + + def rerender + render_differential unless @in_pasting end class DialogProcScope + CompletionJourneyData = Struct.new(:preposing, :postposing, :list, :pointer) + def initialize(line_editor, config, proc_to_exec, context) @line_editor = line_editor @config = config @@ -559,21 +621,20 @@ class Reline::LineEditor end def screen_width - @line_editor.instance_variable_get(:@screen_size).last + @line_editor.screen_width end def screen_height - @line_editor.instance_variable_get(:@screen_size).first + @line_editor.screen_height end def preferred_dialog_height - rest_height = @line_editor.instance_variable_get(:@rest_height) - scroll_partial_screen = @line_editor.instance_variable_get(:@scroll_partial_screen) || 0 - [cursor_pos.y - scroll_partial_screen, rest_height, (screen_height + 6) / 5].max + _wrapped_cursor_x, wrapped_cursor_y = @line_editor.wrapped_cursor_position + [@line_editor.upper_space_height(wrapped_cursor_y), @line_editor.rest_height(wrapped_cursor_y), (screen_height + 6) / 5].max end def completion_journey_data - @line_editor.instance_variable_get(:@completion_journey_data) + @line_editor.dialog_proc_scope_completion_journey_data end def config @@ -642,27 +703,6 @@ class Reline::LineEditor end DIALOG_DEFAULT_HEIGHT = 20 - private def render_dialog(cursor_column) - changes = @dialogs.map do |dialog| - old_dialog = dialog.dup - update_each_dialog(dialog, cursor_column) - [old_dialog, dialog] - end - render_dialog_changes(changes, cursor_column) - end - - private def padding_space_with_escape_sequences(str, width) - padding_width = width - calculate_width(str, true) - # padding_width should be only positive value. But macOS and Alacritty returns negative value. - padding_width = 0 if padding_width < 0 - str + (' ' * padding_width) - end - - private def range_subtract(base_ranges, subtract_ranges) - indices = base_ranges.flat_map(&:to_a).uniq.sort - subtract_ranges.flat_map(&:to_a) - chunks = indices.chunk_while { |a, b| a + 1 == b } - chunks.map { |a| a.first...a.last + 1 } - end private def dialog_range(dialog, dialog_y) x_range = dialog.column...dialog.column + dialog.width @@ -670,106 +710,9 @@ class Reline::LineEditor [x_range, y_range] end - private def render_dialog_changes(changes, cursor_column) - # Collect x-coordinate range and content of previous and current dialogs for each line - old_dialog_ranges = {} - new_dialog_ranges = {} - new_dialog_contents = {} - changes.each do |old_dialog, new_dialog| - if old_dialog.contents - x_range, y_range = dialog_range(old_dialog, @previous_rendered_dialog_y) - y_range.each do |y| - (old_dialog_ranges[y] ||= []) << x_range - end - end - if new_dialog.contents - x_range, y_range = dialog_range(new_dialog, @first_line_started_from + @started_from) - y_range.each do |y| - (new_dialog_ranges[y] ||= []) << x_range - (new_dialog_contents[y] ||= []) << [x_range, new_dialog.contents[y - y_range.begin]] - end - end - end - return if old_dialog_ranges.empty? && new_dialog_ranges.empty? - - # Calculate x-coordinate ranges to restore text that was hidden behind dialogs for each line - ranges_to_restore = {} - subtract_cache = {} - old_dialog_ranges.each do |y, old_x_ranges| - new_x_ranges = new_dialog_ranges[y] || [] - ranges = subtract_cache[[old_x_ranges, new_x_ranges]] ||= range_subtract(old_x_ranges, new_x_ranges) - ranges_to_restore[y] = ranges if ranges.any? - end - - # Create visual_lines for restoring text hidden behind dialogs - if ranges_to_restore.any? - lines = whole_lines - prompt, _prompt_width, prompt_list = check_multiline_prompt(lines, force_recalc: true) - modified_lines = modify_lines(lines, force_recalc: true) - visual_lines = [] - modified_lines.each_with_index { |l, i| - pr = prompt_list ? prompt_list[i] : prompt - vl, = split_by_width(pr + l, @screen_size.last) - vl.compact! - visual_lines.concat(vl) - } - end - - # Clear and rerender all dialogs line by line - Reline::IOGate.hide_cursor - ymin, ymax = (ranges_to_restore.keys + new_dialog_ranges.keys).minmax - scroll_partial_screen = @scroll_partial_screen || 0 - screen_y_range = scroll_partial_screen..(scroll_partial_screen + @screen_height - 1) - ymin = ymin.clamp(screen_y_range.begin, screen_y_range.end) - ymax = ymax.clamp(screen_y_range.begin, screen_y_range.end) - dialog_y = @first_line_started_from + @started_from - cursor_y = dialog_y - if @highest_in_all <= ymax - scroll_down(ymax - cursor_y) - move_cursor_up(ymax - cursor_y) - end - (ymin..ymax).each do |y| - move_cursor_down(y - cursor_y) - cursor_y = y - new_x_ranges = new_dialog_ranges[y] - restore_ranges = ranges_to_restore[y] - # Restore text that was hidden behind dialogs - if restore_ranges - line = visual_lines[y] || '' - restore_ranges.each do |range| - col = range.begin - width = range.end - range.begin - s = padding_space_with_escape_sequences(Reline::Unicode.take_range(line, col, width), width) - Reline::IOGate.move_cursor_column(col) - @output.write "\e[0m#{s}\e[0m" - end - max_column = [calculate_width(line, true), new_x_ranges&.map(&:end)&.max || 0].max - if max_column < restore_ranges.map(&:end).max - Reline::IOGate.move_cursor_column(max_column) - Reline::IOGate.erase_after_cursor - end - end - # Render dialog contents - new_dialog_contents[y]&.each do |x_range, content| - Reline::IOGate.move_cursor_column(x_range.begin) - @output.write "\e[0m#{content}\e[0m" - end - end - move_cursor_up(cursor_y - dialog_y) - Reline::IOGate.move_cursor_column(cursor_column) - Reline::IOGate.show_cursor - - @previous_rendered_dialog_y = dialog_y - end - - private def update_each_dialog(dialog, cursor_column) - if @in_pasting - dialog.contents = nil - dialog.trap_key = nil - return - end - dialog.set_cursor_pos(cursor_column, @first_line_started_from + @started_from) - dialog_render_info = dialog.call(@last_key) + private def update_each_dialog(dialog, cursor_column, cursor_row, key = nil) + dialog.set_cursor_pos(cursor_column, cursor_row) + dialog_render_info = dialog.call(key) if dialog_render_info.nil? or dialog_render_info.contents.nil? or dialog_render_info.contents.empty? dialog.contents = nil dialog.trap_key = nil @@ -809,45 +752,41 @@ class Reline::LineEditor else scrollbar_pos = nil end - upper_space = @first_line_started_from - @started_from dialog.column = dialog_render_info.pos.x dialog.width += @block_elem_width if scrollbar_pos - diff = (dialog.column + dialog.width) - (@screen_size.last) + diff = (dialog.column + dialog.width) - screen_width if diff > 0 dialog.column -= diff end - if (@rest_height - dialog_render_info.pos.y) >= height + if rest_height(screen_scroll_top + cursor_row) - dialog_render_info.pos.y >= height dialog.vertical_offset = dialog_render_info.pos.y + 1 - elsif upper_space >= height + elsif cursor_row >= height dialog.vertical_offset = dialog_render_info.pos.y - height else dialog.vertical_offset = dialog_render_info.pos.y + 1 end if dialog.column < 0 dialog.column = 0 - dialog.width = @screen_size.last + dialog.width = screen_width end + face = Reline::Face[dialog_render_info.face || :default] + scrollbar_sgr = face[:scrollbar] + default_sgr = face[:default] + enhanced_sgr = face[:enhanced] dialog.contents = contents.map.with_index do |item, i| - if i == pointer - fg_color = dialog_render_info.pointer_fg_color - bg_color = dialog_render_info.pointer_bg_color - else - fg_color = dialog_render_info.fg_color - bg_color = dialog_render_info.bg_color - end + line_sgr = i == pointer ? enhanced_sgr : default_sgr str_width = dialog.width - (scrollbar_pos.nil? ? 0 : @block_elem_width) - str = padding_space_with_escape_sequences(Reline::Unicode.take_range(item, 0, str_width), str_width) - colored_content = "\e[#{bg_color}m\e[#{fg_color}m#{str}" + str, = Reline::Unicode.take_mbchar_range(item, 0, str_width, padding: true) + colored_content = "#{line_sgr}#{str}" if scrollbar_pos - color_seq = "\e[37m" if scrollbar_pos <= (i * 2) and (i * 2 + 1) < (scrollbar_pos + bar_height) - colored_content + color_seq + @full_block + colored_content + scrollbar_sgr + @full_block elsif scrollbar_pos <= (i * 2) and (i * 2) < (scrollbar_pos + bar_height) - colored_content + color_seq + @upper_half_block + colored_content + scrollbar_sgr + @upper_half_block elsif scrollbar_pos <= (i * 2 + 1) and (i * 2) < (scrollbar_pos + bar_height) - colored_content + color_seq + @lower_half_block + colored_content + scrollbar_sgr + @lower_half_block else - colored_content + color_seq + ' ' * @block_elem_width + colored_content + scrollbar_sgr + ' ' * @block_elem_width end else colored_content @@ -855,379 +794,20 @@ class Reline::LineEditor end end - private def clear_dialog(cursor_column) - changes = @dialogs.map do |dialog| - old_dialog = dialog.dup - dialog.contents = nil - [old_dialog, dialog] - end - render_dialog_changes(changes, cursor_column) - end - - private def clear_dialog_with_trap_key(cursor_column) - clear_dialog(cursor_column) - @dialogs.each do |dialog| - dialog.trap_key = nil - end - end - - private def calculate_scroll_partial_screen(highest_in_all, cursor_y) - if @screen_height < highest_in_all - old_scroll_partial_screen = @scroll_partial_screen - if cursor_y == 0 - @scroll_partial_screen = 0 - elsif cursor_y == (highest_in_all - 1) - @scroll_partial_screen = highest_in_all - @screen_height - else - if @scroll_partial_screen - if cursor_y <= @scroll_partial_screen - @scroll_partial_screen = cursor_y - elsif (@scroll_partial_screen + @screen_height - 1) < cursor_y - @scroll_partial_screen = cursor_y - (@screen_height - 1) - end - else - if cursor_y > (@screen_height - 1) - @scroll_partial_screen = cursor_y - (@screen_height - 1) - else - @scroll_partial_screen = 0 - end - end - end - if @scroll_partial_screen != old_scroll_partial_screen - @rerender_all = true - end - else - if @scroll_partial_screen - @rerender_all = true - end - @scroll_partial_screen = nil - end - end - - private def rerender_added_newline(prompt, prompt_width, prompt_list) - @buffer_of_lines[@previous_line_index] = @line - @line = @buffer_of_lines[@line_index] - @previous_line_index = nil - if @in_pasting - scroll_down(1) - else - lines = whole_lines - prev_line_prompt = @prompt_proc ? prompt_list[@line_index - 1] : prompt - prev_line_prompt_width = @prompt_proc ? calculate_width(prev_line_prompt, true) : prompt_width - prev_line = modify_lines(lines)[@line_index - 1] - move_cursor_up(@started_from) - render_partial(prev_line_prompt, prev_line_prompt_width, prev_line, @first_line_started_from + @started_from, with_control: false) - scroll_down(1) - render_partial(prompt, prompt_width, @line, @first_line_started_from + @started_from + 1, with_control: false) - end - @cursor = @cursor_max = calculate_width(@line) - @byte_pointer = @line.bytesize - @highest_in_all += @highest_in_this - @highest_in_this = calculate_height_by_width(prompt_width + @cursor_max) - @first_line_started_from += @started_from + 1 - @started_from = calculate_height_by_width(prompt_width + @cursor) - 1 - end - - def just_move_cursor - prompt, prompt_width, prompt_list = check_multiline_prompt(@buffer_of_lines) - move_cursor_up(@started_from) - new_first_line_started_from = - if @line_index.zero? - 0 - else - calculate_height_by_lines(@buffer_of_lines[0..(@line_index - 1)], prompt_list || prompt) - end - first_line_diff = new_first_line_started_from - @first_line_started_from - @cursor, @cursor_max, _, @byte_pointer = calculate_nearest_cursor(@buffer_of_lines[@line_index], @cursor, @started_from, @byte_pointer, false) - new_started_from = calculate_height_by_width(prompt_width + @cursor) - 1 - calculate_scroll_partial_screen(@highest_in_all, new_first_line_started_from + new_started_from) - @previous_line_index = nil - @line = @buffer_of_lines[@line_index] - if @rerender_all - rerender_all_lines - @rerender_all = false - true - else - @first_line_started_from = new_first_line_started_from - @started_from = new_started_from - move_cursor_down(first_line_diff + @started_from) - Reline::IOGate.move_cursor_column((prompt_width + @cursor) % @screen_size.last) - false - end - end - - private def rerender_changed_current_line - new_lines = whole_lines - prompt, prompt_width, prompt_list = check_multiline_prompt(new_lines) - all_height = calculate_height_by_lines(new_lines, prompt_list || prompt) - diff = all_height - @highest_in_all - move_cursor_down(@highest_in_all - @first_line_started_from - @started_from - 1) - if diff > 0 - scroll_down(diff) - move_cursor_up(all_height - 1) - elsif diff < 0 - (-diff).times do - Reline::IOGate.move_cursor_column(0) - Reline::IOGate.erase_after_cursor - move_cursor_up(1) - end - move_cursor_up(all_height - 1) - else - move_cursor_up(all_height - 1) - end - @highest_in_all = all_height - back = render_whole_lines(new_lines, prompt_list || prompt, prompt_width) - move_cursor_up(back) - if @previous_line_index - @buffer_of_lines[@previous_line_index] = @line - @line = @buffer_of_lines[@line_index] - end - @first_line_started_from = - if @line_index.zero? - 0 - else - calculate_height_by_lines(@buffer_of_lines[0..(@line_index - 1)], prompt_list || prompt) - end - if @prompt_proc - prompt = prompt_list[@line_index] - prompt_width = calculate_width(prompt, true) - end - move_cursor_down(@first_line_started_from) - calculate_nearest_cursor - @started_from = calculate_height_by_width(prompt_width + @cursor) - 1 - move_cursor_down(@started_from) - Reline::IOGate.move_cursor_column((prompt_width + @cursor) % @screen_size.last) - @highest_in_this = calculate_height_by_width(prompt_width + @cursor_max) - end - - private def rerender_all_lines - move_cursor_up(@first_line_started_from + @started_from) - Reline::IOGate.move_cursor_column(0) - back = 0 - new_buffer = whole_lines - prompt, prompt_width, prompt_list = check_multiline_prompt(new_buffer) - new_buffer.each_with_index do |line, index| - prompt_width = calculate_width(prompt_list[index], true) if @prompt_proc - width = prompt_width + calculate_width(line) - height = calculate_height_by_width(width) - back += height - end - old_highest_in_all = @highest_in_all - if @line_index.zero? - new_first_line_started_from = 0 - else - new_first_line_started_from = calculate_height_by_lines(new_buffer[0..(@line_index - 1)], prompt_list || prompt) - end - new_started_from = calculate_height_by_width(prompt_width + @cursor) - 1 - calculate_scroll_partial_screen(back, new_first_line_started_from + new_started_from) - if @scroll_partial_screen - move_cursor_up(@first_line_started_from + @started_from) - scroll_down(@screen_height - 1) - move_cursor_up(@screen_height) - Reline::IOGate.move_cursor_column(0) - elsif back > old_highest_in_all - scroll_down(back - 1) - move_cursor_up(back - 1) - elsif back < old_highest_in_all - scroll_down(back) - Reline::IOGate.erase_after_cursor - (old_highest_in_all - back - 1).times do - scroll_down(1) - Reline::IOGate.erase_after_cursor - end - move_cursor_up(old_highest_in_all - 1) - end - render_whole_lines(new_buffer, prompt_list || prompt, prompt_width) - if @prompt_proc - prompt = prompt_list[@line_index] - prompt_width = calculate_width(prompt, true) - end - @highest_in_this = calculate_height_by_width(prompt_width + @cursor_max) - @highest_in_all = back - @first_line_started_from = new_first_line_started_from - @started_from = new_started_from - if @scroll_partial_screen - Reline::IOGate.move_cursor_up(@screen_height - (@first_line_started_from + @started_from - @scroll_partial_screen) - 1) - Reline::IOGate.move_cursor_column((prompt_width + @cursor) % @screen_size.last) - else - move_cursor_down(@first_line_started_from + @started_from - back + 1) - Reline::IOGate.move_cursor_column((prompt_width + @cursor) % @screen_size.last) - end - end - - private def render_whole_lines(lines, prompt, prompt_width) - rendered_height = 0 - modify_lines(lines).each_with_index do |line, index| - if prompt.is_a?(Array) - line_prompt = prompt[index] - prompt_width = calculate_width(line_prompt, true) - else - line_prompt = prompt - end - height = render_partial(line_prompt, prompt_width, line, rendered_height, with_control: false) - if index < (lines.size - 1) - if @scroll_partial_screen - if (@scroll_partial_screen - height) < rendered_height and (@scroll_partial_screen + @screen_height - 1) >= (rendered_height + height) - move_cursor_down(1) - end - else - scroll_down(1) - end - rendered_height += height - else - rendered_height += height - 1 - end - end - rendered_height - end - - private def render_partial(prompt, prompt_width, line_to_render, this_started_from, with_control: true) - visual_lines, height = split_by_width(line_to_render.nil? ? prompt : prompt + line_to_render, @screen_size.last) - cursor_up_from_last_line = 0 - if @scroll_partial_screen - last_visual_line = this_started_from + (height - 1) - last_screen_line = @scroll_partial_screen + (@screen_height - 1) - if (@scroll_partial_screen - this_started_from) >= height - # Render nothing because this line is before the screen. - visual_lines = [] - elsif this_started_from > last_screen_line - # Render nothing because this line is after the screen. - visual_lines = [] - else - deleted_lines_before_screen = [] - if @scroll_partial_screen > this_started_from and last_visual_line >= @scroll_partial_screen - # A part of visual lines are before the screen. - deleted_lines_before_screen = visual_lines.shift((@scroll_partial_screen - this_started_from) * 2) - deleted_lines_before_screen.compact! - end - if this_started_from <= last_screen_line and last_screen_line < last_visual_line - # A part of visual lines are after the screen. - visual_lines.pop((last_visual_line - last_screen_line) * 2) - end - move_cursor_up(deleted_lines_before_screen.size - @started_from) - cursor_up_from_last_line = @started_from - deleted_lines_before_screen.size - end - end - if with_control - if height > @highest_in_this - diff = height - @highest_in_this - scroll_down(diff) - @highest_in_all += diff - @highest_in_this = height - move_cursor_up(diff) - elsif height < @highest_in_this - diff = @highest_in_this - height - @highest_in_all -= diff - @highest_in_this = height - end - move_cursor_up(@started_from) - @started_from = calculate_height_by_width(prompt_width + @cursor) - 1 - cursor_up_from_last_line = height - 1 - @started_from - end - if Reline::Unicode::CSI_REGEXP.match?(prompt + line_to_render) - @output.write "\e[0m" # clear character decorations - end - visual_lines.each_with_index do |line, index| - Reline::IOGate.move_cursor_column(0) - if line.nil? - if calculate_width(visual_lines[index - 1], true) == Reline::IOGate.get_screen_size.last - # reaches the end of line - if Reline::IOGate.win? and Reline::IOGate.win_legacy_console? - # A newline is automatically inserted if a character is rendered at - # eol on command prompt. - else - # When the cursor is at the end of the line and erases characters - # after the cursor, some terminals delete the character at the - # cursor position. - move_cursor_down(1) - Reline::IOGate.move_cursor_column(0) - end - else - Reline::IOGate.erase_after_cursor - move_cursor_down(1) - Reline::IOGate.move_cursor_column(0) - end - next - end - @output.write line - if Reline::IOGate.win? and Reline::IOGate.win_legacy_console? and calculate_width(line, true) == Reline::IOGate.get_screen_size.last - # A newline is automatically inserted if a character is rendered at eol on command prompt. - @rest_height -= 1 if @rest_height > 0 - end - @output.flush - if @first_prompt - @first_prompt = false - @pre_input_hook&.call - end - end - unless visual_lines.empty? - Reline::IOGate.erase_after_cursor - Reline::IOGate.move_cursor_column(0) - end - if with_control - # Just after rendring, so the cursor is on the last line. - if finished? - Reline::IOGate.move_cursor_column(0) - else - # Moves up from bottom of lines to the cursor position. - move_cursor_up(cursor_up_from_last_line) - # This logic is buggy if a fullwidth char is wrapped because there is only one halfwidth at end of a line. - Reline::IOGate.move_cursor_column((prompt_width + @cursor) % @screen_size.last) - end - end - height - end - - private def modify_lines(before, force_recalc: false) - return before if !force_recalc && (before.nil? || before.empty? || simplified_rendering?) - - if after = @output_modifier_proc&.call("#{before.join("\n")}\n", complete: finished?) + private def modify_lines(before, complete) + if after = @output_modifier_proc&.call("#{before.join("\n")}\n", complete: complete) after.lines("\n").map { |l| l.chomp('') } else - before - end - end - - private def show_menu - scroll_down(@highest_in_all - @first_line_started_from) - @rerender_all = true - @menu_info.list.sort!.each do |item| - Reline::IOGate.move_cursor_column(0) - @output.write item - @output.flush - scroll_down(1) + before.map { |l| Reline::Unicode.escape_for_print(l) } end - scroll_down(@highest_in_all - 1) - move_cursor_up(@highest_in_all - 1 - @first_line_started_from) - end - - private def clear_screen_buffer(prompt, prompt_list, prompt_width) - Reline::IOGate.clear_screen - back = 0 - modify_lines(whole_lines).each_with_index do |line, index| - if @prompt_proc - pr = prompt_list[index] - height = render_partial(pr, calculate_width(pr), line, back, with_control: false) - else - height = render_partial(prompt, prompt_width, line, back, with_control: false) - end - if index < (@buffer_of_lines.size - 1) - move_cursor_down(1) - back += height - end - end - move_cursor_up(back) - move_cursor_down(@first_line_started_from + @started_from) - @rest_height = (Reline::IOGate.get_screen_size.first - 1) - Reline::IOGate.cursor_pos.y - Reline::IOGate.move_cursor_column((prompt_width + @cursor) % @screen_size.last) end def editing_mode @config.editing_mode end - private def menu(target, list) - @menu_info = MenuInfo.new(target, list) + private def menu(_target, list) + @menu_info = MenuInfo.new(list) end private def complete_internal_proc(list, is_menu) @@ -1255,7 +835,7 @@ class Reline::LineEditor item_mbchars = item.grapheme_clusters end size = [memo_mbchars.size, item_mbchars.size].min - result = '' + result = +'' size.times do |i| if @config.completion_ignore_case if memo_mbchars[i].casecmp?(item_mbchars[i]) @@ -1276,9 +856,9 @@ class Reline::LineEditor [target, preposing, completed, postposing] end - private def complete(list, just_show_list = false) + private def perform_completion(list, just_show_list) case @completion_state - when CompletionState::NORMAL, CompletionState::JOURNEY + when CompletionState::NORMAL @completion_state = CompletionState::COMPLETION when CompletionState::PERFECT_MATCH @dig_perfect_match_proc&.(@perfect_matched) @@ -1305,100 +885,80 @@ class Reline::LineEditor @completion_state = CompletionState::PERFECT_MATCH else @completion_state = CompletionState::MENU_WITH_PERFECT_MATCH + perform_completion(list, true) if @config.show_all_if_ambiguous end @perfect_matched = completed else @completion_state = CompletionState::MENU + perform_completion(list, true) if @config.show_all_if_ambiguous end if not just_show_list and target < completed - @line = (preposing + completed + completion_append_character.to_s + postposing).split("\n")[@line_index] || String.new(encoding: @encoding) - line_to_pointer = (preposing + completed + completion_append_character.to_s).split("\n").last || String.new(encoding: @encoding) - @cursor_max = calculate_width(@line) - @cursor = calculate_width(line_to_pointer) + @buffer_of_lines[@line_index] = (preposing + completed + completion_append_character.to_s + postposing).split("\n")[@line_index] || String.new(encoding: @encoding) + line_to_pointer = (preposing + completed + completion_append_character.to_s).split("\n")[@line_index] || String.new(encoding: @encoding) @byte_pointer = line_to_pointer.bytesize end end end - private def move_completed_list(list, direction) - case @completion_state - when CompletionState::NORMAL, CompletionState::COMPLETION, - CompletionState::MENU, CompletionState::MENU_WITH_PERFECT_MATCH - @completion_state = CompletionState::JOURNEY - result = retrieve_completion_block - return if result.nil? - preposing, target, postposing = result - @completion_journey_data = CompletionJourneyData.new( - preposing, postposing, - [target] + list.select{ |item| item.start_with?(target) }, 0) - if @completion_journey_data.list.size == 1 - @completion_journey_data.pointer = 0 - else - case direction - when :up - @completion_journey_data.pointer = @completion_journey_data.list.size - 1 - when :down - @completion_journey_data.pointer = 1 - end - end - @completion_state = CompletionState::JOURNEY - else - case direction - when :up - @completion_journey_data.pointer -= 1 - if @completion_journey_data.pointer < 0 - @completion_journey_data.pointer = @completion_journey_data.list.size - 1 - end - when :down - @completion_journey_data.pointer += 1 - if @completion_journey_data.pointer >= @completion_journey_data.list.size - @completion_journey_data.pointer = 0 - end - end + def dialog_proc_scope_completion_journey_data + return nil unless @completion_journey_state + line_index = @completion_journey_state.line_index + pre_lines = @buffer_of_lines[0...line_index].map { |line| line + "\n" } + post_lines = @buffer_of_lines[(line_index + 1)..-1].map { |line| line + "\n" } + DialogProcScope::CompletionJourneyData.new( + pre_lines.join + @completion_journey_state.pre, + @completion_journey_state.post + post_lines.join, + @completion_journey_state.list, + @completion_journey_state.pointer + ) + end + + private def move_completed_list(direction) + @completion_journey_state ||= retrieve_completion_journey_state + return false unless @completion_journey_state + + if (delta = { up: -1, down: +1 }[direction]) + @completion_journey_state.pointer = (@completion_journey_state.pointer + delta) % @completion_journey_state.list.size end - completed = @completion_journey_data.list[@completion_journey_data.pointer] - new_line = (@completion_journey_data.preposing + completed + @completion_journey_data.postposing).split("\n")[@line_index] - @line = new_line.nil? ? String.new(encoding: @encoding) : new_line - line_to_pointer = (@completion_journey_data.preposing + completed).split("\n").last - line_to_pointer = String.new(encoding: @encoding) if line_to_pointer.nil? - @cursor_max = calculate_width(@line) - @cursor = calculate_width(line_to_pointer) - @byte_pointer = line_to_pointer.bytesize + completed = @completion_journey_state.list[@completion_journey_state.pointer] + set_current_line(@completion_journey_state.pre + completed + @completion_journey_state.post, @completion_journey_state.pre.bytesize + completed.bytesize) + true + end + + private def retrieve_completion_journey_state + preposing, target, postposing = retrieve_completion_block + list = call_completion_proc + return unless list.is_a?(Array) + + candidates = list.select{ |item| item.start_with?(target) } + return if candidates.empty? + + pre = preposing.split("\n", -1).last || '' + post = postposing.split("\n", -1).first || '' + CompletionJourneyState.new( + @line_index, pre, target, post, [target] + candidates, 0 + ) end private def run_for_operators(key, method_symbol, &block) - if @waiting_operator_proc + if @vi_waiting_operator if VI_MOTIONS.include?(method_symbol) - old_cursor, old_byte_pointer = @cursor, @byte_pointer - @vi_arg = @waiting_operator_vi_arg if @waiting_operator_vi_arg&.> 1 + old_byte_pointer = @byte_pointer + @vi_arg = (@vi_arg || 1) * @vi_waiting_operator_arg block.(true) unless @waiting_proc - cursor_diff, byte_pointer_diff = @cursor - old_cursor, @byte_pointer - old_byte_pointer - @cursor, @byte_pointer = old_cursor, old_byte_pointer - @waiting_operator_proc.(cursor_diff, byte_pointer_diff) - else - old_waiting_proc = @waiting_proc - old_waiting_operator_proc = @waiting_operator_proc - current_waiting_operator_proc = @waiting_operator_proc - @waiting_proc = proc { |k| - old_cursor, old_byte_pointer = @cursor, @byte_pointer - old_waiting_proc.(k) - cursor_diff, byte_pointer_diff = @cursor - old_cursor, @byte_pointer - old_byte_pointer - @cursor, @byte_pointer = old_cursor, old_byte_pointer - current_waiting_operator_proc.(cursor_diff, byte_pointer_diff) - @waiting_operator_proc = old_waiting_operator_proc - } + byte_pointer_diff = @byte_pointer - old_byte_pointer + @byte_pointer = old_byte_pointer + method_obj = method(@vi_waiting_operator) + wrap_method_call(@vi_waiting_operator, method_obj, byte_pointer_diff) + cleanup_waiting end else # Ignores operator when not motion is given. block.(false) + cleanup_waiting end - @waiting_operator_proc = nil - @waiting_operator_vi_arg = nil - if @vi_arg - @rerender_all = true - @vi_arg = nil - end + @vi_arg = nil else block.(false) end @@ -1415,7 +975,7 @@ class Reline::LineEditor end def wrap_method_call(method_symbol, method_obj, key, with_operator = false) - if @config.editing_mode_is?(:emacs, :vi_insert) and @waiting_proc.nil? and @waiting_operator_proc.nil? + if @config.editing_mode_is?(:emacs, :vi_insert) and @vi_waiting_operator.nil? not_insertion = method_symbol != :ed_insert process_insert(force: not_insertion) end @@ -1434,11 +994,33 @@ class Reline::LineEditor end end + private def cleanup_waiting + @waiting_proc = nil + @vi_waiting_operator = nil + @vi_waiting_operator_arg = nil + @searching_prompt = nil + @drop_terminate_spaces = false + end + private def process_key(key, method_symbol) + if key.is_a?(Symbol) + cleanup_waiting + elsif @waiting_proc + old_byte_pointer = @byte_pointer + @waiting_proc.call(key) + if @vi_waiting_operator + byte_pointer_diff = @byte_pointer - old_byte_pointer + @byte_pointer = old_byte_pointer + method_obj = method(@vi_waiting_operator) + wrap_method_call(@vi_waiting_operator, method_obj, byte_pointer_diff) + cleanup_waiting + end + @kill_ring.process + return + end + if method_symbol and respond_to?(method_symbol, true) method_obj = method(method_symbol) - else - method_obj = nil end if method_symbol and key.is_a?(Symbol) if @vi_arg and argumentable?(method_obj) @@ -1450,7 +1032,6 @@ class Reline::LineEditor end @kill_ring.process if @vi_arg - @rerender_al = true @vi_arg = nil end elsif @vi_arg @@ -1461,8 +1042,6 @@ class Reline::LineEditor run_for_operators(key, method_symbol) do |with_operator| wrap_method_call(method_symbol, method_obj, key, with_operator) end - elsif @waiting_proc - @waiting_proc.(key) elsif method_obj wrap_method_call(method_symbol, method_obj, key) else @@ -1470,13 +1049,9 @@ class Reline::LineEditor end @kill_ring.process if @vi_arg - @rerender_all = true @vi_arg = nil end end - elsif @waiting_proc - @waiting_proc.(key) - @kill_ring.process elsif method_obj if method_symbol == :ed_argument_digit wrap_method_call(method_symbol, method_obj, key) @@ -1492,11 +1067,6 @@ class Reline::LineEditor end private def normal_char(key) - method_symbol = method_obj = nil - if key.combined_char.is_a?(Symbol) - process_key(key.combined_char, key.combined_char) - return - end @multibyte_buffer << key.combined_char if @multibyte_buffer.size > 1 if @multibyte_buffer.dup.force_encoding(@encoding).valid_encoding? @@ -1510,97 +1080,107 @@ class Reline::LineEditor return if key.char >= 128 # maybe, first byte of multi byte method_symbol = @config.editing_mode.get_method(key.combined_char) if key.with_meta and method_symbol == :ed_unassigned - # split ESC + key - method_symbol = @config.editing_mode.get_method("\e".ord) - process_key("\e".ord, method_symbol) - method_symbol = @config.editing_mode.get_method(key.char) - process_key(key.char, method_symbol) + if @config.editing_mode_is?(:vi_command, :vi_insert) + # split ESC + key in vi mode + method_symbol = @config.editing_mode.get_method("\e".ord) + process_key("\e".ord, method_symbol) + method_symbol = @config.editing_mode.get_method(key.char) + process_key(key.char, method_symbol) + end else process_key(key.combined_char, method_symbol) end @multibyte_buffer.clear end - if @config.editing_mode_is?(:vi_command) and @cursor > 0 and @cursor == @cursor_max - byte_size = Reline::Unicode.get_prev_mbchar_size(@line, @byte_pointer) + if @config.editing_mode_is?(:vi_command) and @byte_pointer > 0 and @byte_pointer == current_line.bytesize + byte_size = Reline::Unicode.get_prev_mbchar_size(@buffer_of_lines[@line_index], @byte_pointer) @byte_pointer -= byte_size - mbchar = @line.byteslice(@byte_pointer, byte_size) - width = Reline::Unicode.get_mbchar_width(mbchar) - @cursor -= width + end + end + + def update(key) + modified = input_key(key) + unless @in_pasting + scroll_into_view + @just_cursor_moving = !modified + update_dialogs(key) + @just_cursor_moving = false end end def input_key(key) - @last_key = key + save_old_buffer @config.reset_oneshot_key_bindings @dialogs.each do |dialog| if key.char.instance_of?(Symbol) and key.char == dialog.name return end end - @just_cursor_moving = nil if key.char.nil? + process_insert(force: true) if @first_char - @line = nil + @eof = true end finish return end - old_line = @line.dup @first_char = false - completion_occurs = false - if @config.editing_mode_is?(:emacs, :vi_insert) and key.char == "\C-i".ord - unless @config.disable_completion - result = call_completion_proc - if result.is_a?(Array) - completion_occurs = true - process_insert - if @config.autocompletion - move_completed_list(result, :down) - else - complete(result) - end - end - end - elsif @config.editing_mode_is?(:emacs, :vi_insert) and key.char == :completion_journey_up - if not @config.disable_completion and @config.autocompletion - result = call_completion_proc - if result.is_a?(Array) - completion_occurs = true - process_insert - move_completed_list(result, :up) - end - end - elsif not @config.disable_completion and @config.editing_mode_is?(:vi_insert) and ["\C-p".ord, "\C-n".ord].include?(key.char) - unless @config.disable_completion - result = call_completion_proc - if result.is_a?(Array) - completion_occurs = true - process_insert - move_completed_list(result, "\C-p".ord == key.char ? :up : :down) - end - end - elsif Symbol === key.char and respond_to?(key.char, true) + @completion_occurs = false + + if key.char.is_a?(Symbol) process_key(key.char, key.char) else normal_char(key) end - unless completion_occurs + unless @completion_occurs @completion_state = CompletionState::NORMAL - @completion_journey_data = nil + @completion_journey_state = nil end - if not @in_pasting and @just_cursor_moving.nil? - if @previous_line_index and @buffer_of_lines[@previous_line_index] == @line - @just_cursor_moving = true - elsif @previous_line_index.nil? and @buffer_of_lines[@line_index] == @line and old_line == @line - @just_cursor_moving = true - else - @just_cursor_moving = false - end - else - @just_cursor_moving = false + + push_past_lines unless @undoing + @undoing = false + + if @in_pasting + clear_dialogs + return + end + + modified = @old_buffer_of_lines != @buffer_of_lines + if !@completion_occurs && modified && !@config.disable_completion && @config.autocompletion + # Auto complete starts only when edited + process_insert(force: true) + @completion_journey_state = retrieve_completion_journey_state + end + modified + end + + def save_old_buffer + @old_buffer_of_lines = @buffer_of_lines.dup + @old_byte_pointer = @byte_pointer.dup + @old_line_index = @line_index.dup + end + + def push_past_lines + if @old_buffer_of_lines != @buffer_of_lines + @past_lines.push([@old_buffer_of_lines, @old_byte_pointer, @old_line_index]) end - if @is_multiline and @auto_indent_proc and not simplified_rendering? - process_auto_indent + trim_past_lines + end + + MAX_PAST_LINES = 100 + def trim_past_lines + if @past_lines.size > MAX_PAST_LINES + @past_lines.shift + end + end + + def scroll_into_view + _wrapped_cursor_x, wrapped_cursor_y = wrapped_cursor_position + if wrapped_cursor_y < screen_scroll_top + @scroll_partial_screen = wrapped_cursor_y + end + if wrapped_cursor_y >= screen_scroll_top + screen_height + @scroll_partial_screen = wrapped_cursor_y - screen_height + 1 end end @@ -1634,43 +1214,52 @@ class Reline::LineEditor result end - private def process_auto_indent - return if not @check_new_auto_indent and @previous_line_index # move cursor up or down - if @check_new_auto_indent and @previous_line_index and @previous_line_index > 0 and @line_index > @previous_line_index - # Fix indent of a line when a newline is inserted to the next - new_lines = whole_lines - new_indent = @auto_indent_proc.(new_lines[0..-3].push(''), @line_index - 1, 0, true) - md = @line.match(/\A */) - prev_indent = md[0].count(' ') - @line = ' ' * new_indent + @line.lstrip - - new_indent = nil - result = @auto_indent_proc.(new_lines[0..-2], @line_index - 1, (new_lines[-2].size + 1), false) - if result - new_indent = result - end - if new_indent&.>= 0 - @line = ' ' * new_indent + @line.lstrip - end - end - new_lines = whole_lines - new_indent = @auto_indent_proc.(new_lines, @line_index, @byte_pointer, @check_new_auto_indent) - if new_indent&.>= 0 - md = new_lines[@line_index].match(/\A */) - prev_indent = md[0].count(' ') - if @check_new_auto_indent - line = @buffer_of_lines[@line_index] = ' ' * new_indent + @buffer_of_lines[@line_index].lstrip - @cursor = new_indent - @cursor_max = calculate_width(line) - @byte_pointer = new_indent - else - @line = ' ' * new_indent + @line.lstrip - @cursor += new_indent - prev_indent - @cursor_max = calculate_width(@line) - @byte_pointer += new_indent - prev_indent - end + private def process_auto_indent(line_index = @line_index, cursor_dependent: true, add_newline: false) + return if @in_pasting + return unless @auto_indent_proc + + line = @buffer_of_lines[line_index] + byte_pointer = cursor_dependent && @line_index == line_index ? @byte_pointer : line.bytesize + new_indent = @auto_indent_proc.(@buffer_of_lines.take(line_index + 1).push(''), line_index, byte_pointer, add_newline) + return unless new_indent + + new_line = ' ' * new_indent + line.lstrip + @buffer_of_lines[line_index] = new_line + if @line_index == line_index + indent_diff = new_line.bytesize - line.bytesize + @byte_pointer = [@byte_pointer + indent_diff, 0].max end - @check_new_auto_indent = false + end + + def line() + @buffer_of_lines.join("\n") unless eof? + end + + def current_line + @buffer_of_lines[@line_index] + end + + def set_current_line(line, byte_pointer = nil) + cursor = current_byte_pointer_cursor + @buffer_of_lines[@line_index] = line + if byte_pointer + @byte_pointer = byte_pointer + else + calculate_nearest_cursor(cursor) + end + process_auto_indent + end + + def set_current_lines(lines, byte_pointer = nil, line_index = 0) + cursor = current_byte_pointer_cursor + @buffer_of_lines = lines + @line_index = line_index + if byte_pointer + @byte_pointer = byte_pointer + else + calculate_nearest_cursor(cursor) + end + process_auto_indent end def retrieve_completion_block(set_completion_quote_character = false) @@ -1684,7 +1273,7 @@ class Reline::LineEditor else quote_characters_regexp = /\A[#{Regexp.escape(Reline.completer_quote_characters)}]/ end - before = @line.byteslice(0, @byte_pointer) + before = current_line.byteslice(0, @byte_pointer) rest = nil break_pointer = nil quote = nil @@ -1692,7 +1281,7 @@ class Reline::LineEditor escaped_quote = nil i = 0 while i < @byte_pointer do - slice = @line.byteslice(i, @byte_pointer - i) + slice = current_line.byteslice(i, @byte_pointer - i) unless slice.valid_encoding? i += 1 next @@ -1714,15 +1303,15 @@ class Reline::LineEditor elsif word_break_regexp and not quote and slice =~ word_break_regexp rest = $' i += 1 - before = @line.byteslice(i, @byte_pointer - i) + before = current_line.byteslice(i, @byte_pointer - i) break_pointer = i else i += 1 end end - postposing = @line.byteslice(@byte_pointer, @line.bytesize - @byte_pointer) + postposing = current_line.byteslice(@byte_pointer, current_line.bytesize - @byte_pointer) if rest - preposing = @line.byteslice(0, break_pointer) + preposing = current_line.byteslice(0, break_pointer) target = rest if set_completion_quote_character and quote Reline.core.instance_variable_set(:@completion_quote_character, quote) @@ -1733,126 +1322,93 @@ class Reline::LineEditor else preposing = '' if break_pointer - preposing = @line.byteslice(0, break_pointer) + preposing = current_line.byteslice(0, break_pointer) else preposing = '' end target = before end - if @is_multiline - lines = whole_lines - if @line_index > 0 - preposing = lines[0..(@line_index - 1)].join("\n") + "\n" + preposing - end - if (lines.size - 1) > @line_index - postposing = postposing + "\n" + lines[(@line_index + 1)..-1].join("\n") - end + lines = whole_lines + if @line_index > 0 + preposing = lines[0..(@line_index - 1)].join("\n") + "\n" + preposing + end + if (lines.size - 1) > @line_index + postposing = postposing + "\n" + lines[(@line_index + 1)..-1].join("\n") end [preposing.encode(@encoding), target.encode(@encoding), postposing.encode(@encoding)] end def confirm_multiline_termination temp_buffer = @buffer_of_lines.dup - if @previous_line_index and @line_index == (@buffer_of_lines.size - 1) - temp_buffer[@previous_line_index] = @line - else - temp_buffer[@line_index] = @line - end @confirm_multiline_termination_proc.(temp_buffer.join("\n") + "\n") end + def insert_pasted_text(text) + save_old_buffer + pre = @buffer_of_lines[@line_index].byteslice(0, @byte_pointer) + post = @buffer_of_lines[@line_index].byteslice(@byte_pointer..) + lines = (pre + text.gsub(/\r\n?/, "\n") + post).split("\n", -1) + lines << '' if lines.empty? + @buffer_of_lines[@line_index, 1] = lines + @line_index += lines.size - 1 + @byte_pointer = @buffer_of_lines[@line_index].bytesize - post.bytesize + push_past_lines + end + def insert_text(text) - width = calculate_width(text) - if @cursor == @cursor_max - @line += text + if @buffer_of_lines[@line_index].bytesize == @byte_pointer + @buffer_of_lines[@line_index] += text else - @line = byteinsert(@line, @byte_pointer, text) + @buffer_of_lines[@line_index] = byteinsert(@buffer_of_lines[@line_index], @byte_pointer, text) end @byte_pointer += text.bytesize - @cursor += width - @cursor_max += width + process_auto_indent end def delete_text(start = nil, length = nil) if start.nil? and length.nil? - if @is_multiline - if @buffer_of_lines.size == 1 - @line&.clear - @byte_pointer = 0 - @cursor = 0 - @cursor_max = 0 - elsif @line_index == (@buffer_of_lines.size - 1) and @line_index > 0 - @buffer_of_lines.pop - @line_index -= 1 - @line = @buffer_of_lines[@line_index] - @byte_pointer = 0 - @cursor = 0 - @cursor_max = calculate_width(@line) - elsif @line_index < (@buffer_of_lines.size - 1) - @buffer_of_lines.delete_at(@line_index) - @line = @buffer_of_lines[@line_index] - @byte_pointer = 0 - @cursor = 0 - @cursor_max = calculate_width(@line) - end - else - @line&.clear + if @buffer_of_lines.size == 1 + @buffer_of_lines[@line_index] = '' + @byte_pointer = 0 + elsif @line_index == (@buffer_of_lines.size - 1) and @line_index > 0 + @buffer_of_lines.pop + @line_index -= 1 + @byte_pointer = 0 + elsif @line_index < (@buffer_of_lines.size - 1) + @buffer_of_lines.delete_at(@line_index) @byte_pointer = 0 - @cursor = 0 - @cursor_max = 0 end elsif not start.nil? and not length.nil? - if @line - before = @line.byteslice(0, start) - after = @line.byteslice(start + length, @line.bytesize) - @line = before + after - @byte_pointer = @line.bytesize if @byte_pointer > @line.bytesize - str = @line.byteslice(0, @byte_pointer) - @cursor = calculate_width(str) - @cursor_max = calculate_width(@line) + if current_line + before = current_line.byteslice(0, start) + after = current_line.byteslice(start + length, current_line.bytesize) + set_current_line(before + after) end elsif start.is_a?(Range) range = start first = range.first last = range.last - last = @line.bytesize - 1 if last > @line.bytesize - last += @line.bytesize if last < 0 - first += @line.bytesize if first < 0 + last = current_line.bytesize - 1 if last > current_line.bytesize + last += current_line.bytesize if last < 0 + first += current_line.bytesize if first < 0 range = range.exclude_end? ? first...last : first..last - @line = @line.bytes.reject.with_index{ |c, i| range.include?(i) }.map{ |c| c.chr(Encoding::ASCII_8BIT) }.join.force_encoding(@encoding) - @byte_pointer = @line.bytesize if @byte_pointer > @line.bytesize - str = @line.byteslice(0, @byte_pointer) - @cursor = calculate_width(str) - @cursor_max = calculate_width(@line) + line = current_line.bytes.reject.with_index{ |c, i| range.include?(i) }.map{ |c| c.chr(Encoding::ASCII_8BIT) }.join.force_encoding(@encoding) + set_current_line(line) else - @line = @line.byteslice(0, start) - @byte_pointer = @line.bytesize if @byte_pointer > @line.bytesize - str = @line.byteslice(0, @byte_pointer) - @cursor = calculate_width(str) - @cursor_max = calculate_width(@line) + set_current_line(current_line.byteslice(0, start)) end end def byte_pointer=(val) @byte_pointer = val - str = @line.byteslice(0, @byte_pointer) - @cursor = calculate_width(str) - @cursor_max = calculate_width(@line) end def whole_lines - index = @previous_line_index || @line_index - temp_lines = @buffer_of_lines.dup - temp_lines[index] = @line - temp_lines + @buffer_of_lines.dup end def whole_buffer - if @buffer_of_lines.size == 1 and @line.nil? - nil - else - whole_lines.join("\n") - end + whole_lines.join("\n") end def finished? @@ -1861,7 +1417,6 @@ class Reline::LineEditor def finish @finished = true - @rerender_all = true @config.reset end @@ -1892,33 +1447,56 @@ class Reline::LineEditor private def key_newline(key) if @is_multiline - if (@buffer_of_lines.size - 1) == @line_index and @line.bytesize == @byte_pointer - @add_newline_to_end_of_buffer = true - end - next_line = @line.byteslice(@byte_pointer, @line.bytesize - @byte_pointer) - cursor_line = @line.byteslice(0, @byte_pointer) + next_line = current_line.byteslice(@byte_pointer, current_line.bytesize - @byte_pointer) + cursor_line = current_line.byteslice(0, @byte_pointer) insert_new_line(cursor_line, next_line) - @cursor = 0 - @check_new_auto_indent = true unless @in_pasting end end + private def complete(_key) + return if @config.disable_completion + + process_insert(force: true) + if @config.autocompletion + @completion_state = CompletionState::NORMAL + @completion_occurs = move_completed_list(:down) + else + @completion_journey_state = nil + result = call_completion_proc + if result.is_a?(Array) + @completion_occurs = true + perform_completion(result, false) + end + end + end + + private def completion_journey_move(direction) + return if @config.disable_completion + + process_insert(force: true) + @completion_state = CompletionState::NORMAL + @completion_occurs = move_completed_list(direction) + end + + private def menu_complete(_key) + completion_journey_move(:down) + end + + private def menu_complete_backward(_key) + completion_journey_move(:up) + end + + private def completion_journey_up(_key) + completion_journey_move(:up) if @config.autocompletion + end + # Editline:: +ed-unassigned+ This editor command always results in an error. # GNU Readline:: There is no corresponding macro. private def ed_unassigned(key) end # do nothing private def process_insert(force: false) return if @continuous_insertion_buffer.empty? or (@in_pasting and not force) - width = Reline::Unicode.calculate_width(@continuous_insertion_buffer) - bytesize = @continuous_insertion_buffer.bytesize - if @cursor == @cursor_max - @line += @continuous_insertion_buffer - else - @line = byteinsert(@line, @byte_pointer, @continuous_insertion_buffer) - end - @byte_pointer += bytesize - @cursor += width - @cursor_max += width + insert_text(@continuous_insertion_buffer) @continuous_insertion_buffer.clear end @@ -1936,9 +1514,6 @@ class Reline::LineEditor # million. # GNU Readline:: +self-insert+ (a, b, A, 1, !, …) Insert yourself. private def ed_insert(key) - str = nil - width = nil - bytesize = nil if key.instance_of?(String) begin key.encode(Encoding::UTF_8) @@ -1946,7 +1521,6 @@ class Reline::LineEditor return end str = key - bytesize = key.bytesize else begin key.chr.encode(Encoding::UTF_8) @@ -1954,7 +1528,6 @@ class Reline::LineEditor return end str = key.chr - bytesize = 1 end if @in_pasting @continuous_insertion_buffer << str @@ -1962,28 +1535,8 @@ class Reline::LineEditor elsif not @continuous_insertion_buffer.empty? process_insert end - width = Reline::Unicode.get_mbchar_width(str) - if @cursor == @cursor_max - @line += str - else - @line = byteinsert(@line, @byte_pointer, str) - end - last_byte_size = Reline::Unicode.get_prev_mbchar_size(@line, @byte_pointer) - @byte_pointer += bytesize - last_mbchar = @line.byteslice((@byte_pointer - bytesize - last_byte_size), last_byte_size) - combined_char = last_mbchar + str - if last_byte_size != 0 and combined_char.grapheme_clusters.size == 1 - # combined char - last_mbchar_width = Reline::Unicode.get_mbchar_width(last_mbchar) - combined_char_width = Reline::Unicode.get_mbchar_width(combined_char) - if combined_char_width > last_mbchar_width - width = combined_char_width - last_mbchar_width - else - width = 0 - end - end - @cursor += width - @cursor_max += width + + insert_text(str) end alias_method :ed_digit, :ed_insert alias_method :self_insert, :ed_insert @@ -2005,18 +1558,11 @@ class Reline::LineEditor alias_method :quoted_insert, :ed_quoted_insert private def ed_next_char(key, arg: 1) - byte_size = Reline::Unicode.get_next_mbchar_size(@line, @byte_pointer) - if (@byte_pointer < @line.bytesize) - mbchar = @line.byteslice(@byte_pointer, byte_size) - width = Reline::Unicode.get_mbchar_width(mbchar) - @cursor += width if width + byte_size = Reline::Unicode.get_next_mbchar_size(current_line, @byte_pointer) + if (@byte_pointer < current_line.bytesize) @byte_pointer += byte_size - elsif @is_multiline and @config.editing_mode_is?(:emacs) and @byte_pointer == @line.bytesize and @line_index < @buffer_of_lines.size - 1 - next_line = @buffer_of_lines[@line_index + 1] - @cursor = 0 + elsif @config.editing_mode_is?(:emacs) and @byte_pointer == current_line.bytesize and @line_index < @buffer_of_lines.size - 1 @byte_pointer = 0 - @cursor_max = calculate_width(next_line) - @previous_line_index = @line_index @line_index += 1 end arg -= 1 @@ -2025,19 +1571,12 @@ class Reline::LineEditor alias_method :forward_char, :ed_next_char private def ed_prev_char(key, arg: 1) - if @cursor > 0 - byte_size = Reline::Unicode.get_prev_mbchar_size(@line, @byte_pointer) + if @byte_pointer > 0 + byte_size = Reline::Unicode.get_prev_mbchar_size(current_line, @byte_pointer) @byte_pointer -= byte_size - mbchar = @line.byteslice(@byte_pointer, byte_size) - width = Reline::Unicode.get_mbchar_width(mbchar) - @cursor -= width - elsif @is_multiline and @config.editing_mode_is?(:emacs) and @byte_pointer == 0 and @line_index > 0 - prev_line = @buffer_of_lines[@line_index - 1] - @cursor = calculate_width(prev_line) - @byte_pointer = prev_line.bytesize - @cursor_max = calculate_width(prev_line) - @previous_line_index = @line_index + elsif @config.editing_mode_is?(:emacs) and @byte_pointer == 0 and @line_index > 0 @line_index -= 1 + @byte_pointer = current_line.bytesize end arg -= 1 ed_prev_char(key, arg: arg) if arg > 0 @@ -2045,157 +1584,109 @@ class Reline::LineEditor alias_method :backward_char, :ed_prev_char private def vi_first_print(key) - @byte_pointer, @cursor = Reline::Unicode.vi_first_print(@line) + @byte_pointer, = Reline::Unicode.vi_first_print(current_line) end private def ed_move_to_beg(key) - @byte_pointer = @cursor = 0 + @byte_pointer = 0 end alias_method :beginning_of_line, :ed_move_to_beg + alias_method :vi_zero, :ed_move_to_beg private def ed_move_to_end(key) - @byte_pointer = 0 - @cursor = 0 - byte_size = 0 - while @byte_pointer < @line.bytesize - byte_size = Reline::Unicode.get_next_mbchar_size(@line, @byte_pointer) - if byte_size > 0 - mbchar = @line.byteslice(@byte_pointer, byte_size) - @cursor += Reline::Unicode.get_mbchar_width(mbchar) - end - @byte_pointer += byte_size - end + @byte_pointer = current_line.bytesize end alias_method :end_of_line, :ed_move_to_end - private def generate_searcher - Fiber.new do |first_key| - prev_search_key = first_key - search_word = String.new(encoding: @encoding) - multibyte_buf = String.new(encoding: 'ASCII-8BIT') - last_hit = nil - case first_key - when "\C-r".ord - prompt_name = 'reverse-i-search' - when "\C-s".ord - prompt_name = 'i-search' + private def generate_searcher(search_key) + search_word = String.new(encoding: @encoding) + multibyte_buf = String.new(encoding: 'ASCII-8BIT') + hit_pointer = nil + lambda do |key| + search_again = false + case key + when "\C-h".ord, "\C-?".ord + grapheme_clusters = search_word.grapheme_clusters + if grapheme_clusters.size > 0 + grapheme_clusters.pop + search_word = grapheme_clusters.join + end + when "\C-r".ord, "\C-s".ord + search_again = true if search_key == key + search_key = key + else + multibyte_buf << key + if multibyte_buf.dup.force_encoding(@encoding).valid_encoding? + search_word << multibyte_buf.dup.force_encoding(@encoding) + multibyte_buf.clear + end end - loop do - key = Fiber.yield(search_word) - search_again = false - case key - when -1 # determined - Reline.last_incremental_search = search_word - break - when "\C-h".ord, "\C-?".ord - grapheme_clusters = search_word.grapheme_clusters - if grapheme_clusters.size > 0 - grapheme_clusters.pop - search_word = grapheme_clusters.join - end - when "\C-r".ord, "\C-s".ord - search_again = true if prev_search_key == key - prev_search_key = key - else - multibyte_buf << key - if multibyte_buf.dup.force_encoding(@encoding).valid_encoding? - search_word << multibyte_buf.dup.force_encoding(@encoding) - multibyte_buf.clear + hit = nil + if not search_word.empty? and @line_backup_in_history&.include?(search_word) + hit_pointer = Reline::HISTORY.size + hit = @line_backup_in_history + else + if search_again + if search_word.empty? and Reline.last_incremental_search + search_word = Reline.last_incremental_search end - end - hit = nil - if not search_word.empty? and @line_backup_in_history&.include?(search_word) - @history_pointer = nil - hit = @line_backup_in_history - else - if search_again - if search_word.empty? and Reline.last_incremental_search - search_word = Reline.last_incremental_search - end - if @history_pointer - case prev_search_key - when "\C-r".ord - history_pointer_base = 0 - history = Reline::HISTORY[0..(@history_pointer - 1)] - when "\C-s".ord - history_pointer_base = @history_pointer + 1 - history = Reline::HISTORY[(@history_pointer + 1)..-1] - end - else - history_pointer_base = 0 - history = Reline::HISTORY - end - elsif @history_pointer - case prev_search_key + if @history_pointer + case search_key when "\C-r".ord history_pointer_base = 0 - history = Reline::HISTORY[0..@history_pointer] + history = Reline::HISTORY[0..(@history_pointer - 1)] when "\C-s".ord - history_pointer_base = @history_pointer - history = Reline::HISTORY[@history_pointer..-1] + history_pointer_base = @history_pointer + 1 + history = Reline::HISTORY[(@history_pointer + 1)..-1] end else history_pointer_base = 0 history = Reline::HISTORY end - case prev_search_key + elsif @history_pointer + case search_key when "\C-r".ord - hit_index = history.rindex { |item| - item.include?(search_word) - } + history_pointer_base = 0 + history = Reline::HISTORY[0..@history_pointer] when "\C-s".ord - hit_index = history.index { |item| - item.include?(search_word) - } - end - if hit_index - @history_pointer = history_pointer_base + hit_index - hit = Reline::HISTORY[@history_pointer] + history_pointer_base = @history_pointer + history = Reline::HISTORY[@history_pointer..-1] end + else + history_pointer_base = 0 + history = Reline::HISTORY end - case prev_search_key + case search_key when "\C-r".ord - prompt_name = 'reverse-i-search' + hit_index = history.rindex { |item| + item.include?(search_word) + } when "\C-s".ord - prompt_name = 'i-search' + hit_index = history.index { |item| + item.include?(search_word) + } end - if hit - if @is_multiline - @buffer_of_lines = hit.split("\n") - @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? - @line_index = @buffer_of_lines.size - 1 - @line = @buffer_of_lines.last - @byte_pointer = @line.bytesize - @cursor = @cursor_max = calculate_width(@line) - @rerender_all = true - @searching_prompt = "(%s)`%s'" % [prompt_name, search_word] - else - @line = hit - @searching_prompt = "(%s)`%s': %s" % [prompt_name, search_word, hit] - end - last_hit = hit - else - if @is_multiline - @rerender_all = true - @searching_prompt = "(failed %s)`%s'" % [prompt_name, search_word] - else - @searching_prompt = "(failed %s)`%s': %s" % [prompt_name, search_word, last_hit] - end + if hit_index + hit_pointer = history_pointer_base + hit_index + hit = Reline::HISTORY[hit_pointer] end end + case search_key + when "\C-r".ord + prompt_name = 'reverse-i-search' + when "\C-s".ord + prompt_name = 'i-search' + end + prompt_name = "failed #{prompt_name}" unless hit + [search_word, prompt_name, hit_pointer] end end private def incremental_search_history(key) unless @history_pointer - if @is_multiline - @line_backup_in_history = whole_buffer - else - @line_backup_in_history = @line - end + @line_backup_in_history = whole_buffer end - searcher = generate_searcher - searcher.resume(key) + searcher = generate_searcher(key) @searching_prompt = "(reverse-i-search)`': " termination_keys = ["\C-j".ord] termination_keys.concat(@config.isearch_terminators&.chars&.map(&:ord)) if @config.isearch_terminators @@ -2207,67 +1698,41 @@ class Reline::LineEditor else buffer = @line_backup_in_history end - if @is_multiline - @buffer_of_lines = buffer.split("\n") - @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? - @line_index = @buffer_of_lines.size - 1 - @line = @buffer_of_lines.last - @rerender_all = true - else - @line = buffer - end + @buffer_of_lines = buffer.split("\n") + @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? + @line_index = @buffer_of_lines.size - 1 @searching_prompt = nil @waiting_proc = nil - @cursor_max = calculate_width(@line) - @cursor = @byte_pointer = 0 - @rerender_all = true - @cached_prompt_list = nil - searcher.resume(-1) + @byte_pointer = 0 when "\C-g".ord - if @is_multiline - @buffer_of_lines = @line_backup_in_history.split("\n") - @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? - @line_index = @buffer_of_lines.size - 1 - @line = @buffer_of_lines.last - @rerender_all = true - else - @line = @line_backup_in_history - end - @history_pointer = nil + @buffer_of_lines = @line_backup_in_history.split("\n") + @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? + @line_index = @buffer_of_lines.size - 1 + move_history(nil, line: :end, cursor: :end, save_buffer: false) @searching_prompt = nil @waiting_proc = nil - @line_backup_in_history = nil - @cursor_max = calculate_width(@line) - @cursor = @byte_pointer = 0 - @rerender_all = true + @byte_pointer = 0 else chr = k.is_a?(String) ? k : k.chr(Encoding::ASCII_8BIT) if chr.match?(/[[:print:]]/) or k == "\C-h".ord or k == "\C-?".ord or k == "\C-r".ord or k == "\C-s".ord - searcher.resume(k) + search_word, prompt_name, hit_pointer = searcher.call(k) + Reline.last_incremental_search = search_word + @searching_prompt = "(%s)`%s'" % [prompt_name, search_word] + @searching_prompt += ': ' unless @is_multiline + move_history(hit_pointer, line: :end, cursor: :end, save_buffer: false) if hit_pointer else if @history_pointer line = Reline::HISTORY[@history_pointer] else line = @line_backup_in_history end - if @is_multiline - @line_backup_in_history = whole_buffer - @buffer_of_lines = line.split("\n") - @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? - @line_index = @buffer_of_lines.size - 1 - @line = @buffer_of_lines.last - @rerender_all = true - else - @line_backup_in_history = @line - @line = line - end + @line_backup_in_history = whole_buffer + @buffer_of_lines = line.split("\n") + @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? + @line_index = @buffer_of_lines.size - 1 @searching_prompt = nil @waiting_proc = nil - @cursor_max = calculate_width(@line) - @cursor = @byte_pointer = 0 - @rerender_all = true - @cached_prompt_list = nil - searcher.resume(-1) + @byte_pointer = 0 end end } @@ -2283,199 +1748,95 @@ class Reline::LineEditor end alias_method :forward_search_history, :vi_search_next - private def ed_search_prev_history(key, arg: 1) - history = nil - h_pointer = nil - line_no = nil - substr = @line.slice(0, @byte_pointer) - if @history_pointer.nil? - return if not @line.empty? and substr.empty? - history = Reline::HISTORY - elsif @history_pointer.zero? - history = nil - h_pointer = nil - else - history = Reline::HISTORY.slice(0, @history_pointer) - end - return if history.nil? - if @is_multiline - h_pointer = history.rindex { |h| - h.split("\n").each_with_index { |l, i| - if l.start_with?(substr) - line_no = i - break - end - } - not line_no.nil? - } - else - h_pointer = history.rindex { |l| - l.start_with?(substr) - } - end - return if h_pointer.nil? - @history_pointer = h_pointer - if @is_multiline - @buffer_of_lines = Reline::HISTORY[@history_pointer].split("\n") - @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? - @line_index = line_no - @line = @buffer_of_lines[@line_index] - @rerender_all = true - else - @line = Reline::HISTORY[@history_pointer] + private def search_history(prefix, pointer_range) + pointer_range.each do |pointer| + lines = Reline::HISTORY[pointer].split("\n") + lines.each_with_index do |line, index| + return [pointer, index] if line.start_with?(prefix) + end end - @cursor_max = calculate_width(@line) + nil + end + + private def ed_search_prev_history(key, arg: 1) + substr = current_line.byteslice(0, @byte_pointer) + return if @history_pointer == 0 + return if @history_pointer.nil? && substr.empty? && !current_line.empty? + + history_range = 0...(@history_pointer || Reline::HISTORY.size) + h_pointer, line_index = search_history(substr, history_range.reverse_each) + return unless h_pointer + move_history(h_pointer, line: line_index || :start, cursor: @byte_pointer) arg -= 1 ed_search_prev_history(key, arg: arg) if arg > 0 end alias_method :history_search_backward, :ed_search_prev_history private def ed_search_next_history(key, arg: 1) - substr = @line.slice(0, @byte_pointer) - if @history_pointer.nil? - return - elsif @history_pointer == (Reline::HISTORY.size - 1) and not substr.empty? - return - end - history = Reline::HISTORY.slice((@history_pointer + 1)..-1) - h_pointer = nil - line_no = nil - if @is_multiline - h_pointer = history.index { |h| - h.split("\n").each_with_index { |l, i| - if l.start_with?(substr) - line_no = i - break - end - } - not line_no.nil? - } - else - h_pointer = history.index { |l| - l.start_with?(substr) - } - end - h_pointer += @history_pointer + 1 if h_pointer and @history_pointer + substr = current_line.byteslice(0, @byte_pointer) + return if @history_pointer.nil? + + history_range = @history_pointer + 1...Reline::HISTORY.size + h_pointer, line_index = search_history(substr, history_range) return if h_pointer.nil? and not substr.empty? - @history_pointer = h_pointer - if @is_multiline - if @history_pointer.nil? and substr.empty? - @buffer_of_lines = [] - @line_index = 0 - else - @buffer_of_lines = Reline::HISTORY[@history_pointer].split("\n") - @line_index = line_no - end - @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? - @line = @buffer_of_lines[@line_index] - @rerender_all = true - else - if @history_pointer.nil? and substr.empty? - @line = '' - else - @line = Reline::HISTORY[@history_pointer] - end - end - @cursor_max = calculate_width(@line) + + move_history(h_pointer, line: line_index || :start, cursor: @byte_pointer) arg -= 1 ed_search_next_history(key, arg: arg) if arg > 0 end alias_method :history_search_forward, :ed_search_next_history - private def ed_prev_history(key, arg: 1) - if @is_multiline and @line_index > 0 - @previous_line_index = @line_index - @line_index -= 1 - return - end - if Reline::HISTORY.empty? - return + private def move_history(history_pointer, line:, cursor:, save_buffer: true) + history_pointer ||= Reline::HISTORY.size + return if history_pointer < 0 || history_pointer > Reline::HISTORY.size + old_history_pointer = @history_pointer || Reline::HISTORY.size + if old_history_pointer == Reline::HISTORY.size + @line_backup_in_history = save_buffer ? whole_buffer : '' + else + Reline::HISTORY[old_history_pointer] = whole_buffer if save_buffer end - if @history_pointer.nil? - @history_pointer = Reline::HISTORY.size - 1 - if @is_multiline - @line_backup_in_history = whole_buffer - @buffer_of_lines = Reline::HISTORY[@history_pointer].split("\n") - @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? - @line_index = @buffer_of_lines.size - 1 - @line = @buffer_of_lines.last - @rerender_all = true - else - @line_backup_in_history = @line - @line = Reline::HISTORY[@history_pointer] - end - elsif @history_pointer.zero? - return + if history_pointer == Reline::HISTORY.size + buf = @line_backup_in_history + @history_pointer = @line_backup_in_history = nil else - if @is_multiline - Reline::HISTORY[@history_pointer] = whole_buffer - @history_pointer -= 1 - @buffer_of_lines = Reline::HISTORY[@history_pointer].split("\n") - @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? - @line_index = @buffer_of_lines.size - 1 - @line = @buffer_of_lines.last - @rerender_all = true - else - Reline::HISTORY[@history_pointer] = @line - @history_pointer -= 1 - @line = Reline::HISTORY[@history_pointer] - end + buf = Reline::HISTORY[history_pointer] + @history_pointer = history_pointer end - if @config.editing_mode_is?(:emacs, :vi_insert) - @cursor_max = @cursor = calculate_width(@line) - @byte_pointer = @line.bytesize - elsif @config.editing_mode_is?(:vi_command) - @byte_pointer = @cursor = 0 - @cursor_max = calculate_width(@line) + @buffer_of_lines = buf.split("\n") + @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? + @line_index = line == :start ? 0 : line == :end ? @buffer_of_lines.size - 1 : line + @byte_pointer = cursor == :start ? 0 : cursor == :end ? current_line.bytesize : cursor + end + + private def ed_prev_history(key, arg: 1) + if @line_index > 0 + cursor = current_byte_pointer_cursor + @line_index -= 1 + calculate_nearest_cursor(cursor) + return end + move_history( + (@history_pointer || Reline::HISTORY.size) - 1, + line: :end, + cursor: @config.editing_mode_is?(:vi_command) ? :start : :end, + ) arg -= 1 ed_prev_history(key, arg: arg) if arg > 0 end alias_method :previous_history, :ed_prev_history private def ed_next_history(key, arg: 1) - if @is_multiline and @line_index < (@buffer_of_lines.size - 1) - @previous_line_index = @line_index + if @line_index < (@buffer_of_lines.size - 1) + cursor = current_byte_pointer_cursor @line_index += 1 + calculate_nearest_cursor(cursor) return end - if @history_pointer.nil? - return - elsif @history_pointer == (Reline::HISTORY.size - 1) - if @is_multiline - @history_pointer = nil - @buffer_of_lines = @line_backup_in_history.split("\n") - @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? - @line_index = 0 - @line = @buffer_of_lines.first - @rerender_all = true - else - @history_pointer = nil - @line = @line_backup_in_history - end - else - if @is_multiline - Reline::HISTORY[@history_pointer] = whole_buffer - @history_pointer += 1 - @buffer_of_lines = Reline::HISTORY[@history_pointer].split("\n") - @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? - @line_index = 0 - @line = @buffer_of_lines.first - @rerender_all = true - else - Reline::HISTORY[@history_pointer] = @line - @history_pointer += 1 - @line = Reline::HISTORY[@history_pointer] - end - end - @line = '' unless @line - if @config.editing_mode_is?(:emacs, :vi_insert) - @cursor_max = @cursor = calculate_width(@line) - @byte_pointer = @line.bytesize - elsif @config.editing_mode_is?(:vi_command) - @byte_pointer = @cursor = 0 - @cursor_max = calculate_width(@line) - end + move_history( + (@history_pointer || Reline::HISTORY.size) + 1, + line: :start, + cursor: @config.editing_mode_is?(:vi_command) ? :start : :end, + ) arg -= 1 ed_next_history(key, arg: arg) if arg > 0 end @@ -2500,40 +1861,29 @@ class Reline::LineEditor end else # should check confirm_multiline_termination to finish? - @previous_line_index = @line_index @line_index = @buffer_of_lines.size - 1 + @byte_pointer = current_line.bytesize finish end end else - if @history_pointer - Reline::HISTORY[@history_pointer] = @line - @history_pointer = nil - end finish end end private def em_delete_prev_char(key, arg: 1) - if @is_multiline and @cursor == 0 and @line_index > 0 - @buffer_of_lines[@line_index] = @line - @cursor = calculate_width(@buffer_of_lines[@line_index - 1]) - @byte_pointer = @buffer_of_lines[@line_index - 1].bytesize - @buffer_of_lines[@line_index - 1] += @buffer_of_lines.delete_at(@line_index) - @line_index -= 1 - @line = @buffer_of_lines[@line_index] - @cursor_max = calculate_width(@line) - @rerender_all = true - elsif @cursor > 0 - byte_size = Reline::Unicode.get_prev_mbchar_size(@line, @byte_pointer) - @byte_pointer -= byte_size - @line, mbchar = byteslice!(@line, @byte_pointer, byte_size) - width = Reline::Unicode.get_mbchar_width(mbchar) - @cursor -= width - @cursor_max -= width + arg.times do + if @byte_pointer == 0 and @line_index > 0 + @byte_pointer = @buffer_of_lines[@line_index - 1].bytesize + @buffer_of_lines[@line_index - 1] += @buffer_of_lines.delete_at(@line_index) + @line_index -= 1 + elsif @byte_pointer > 0 + byte_size = Reline::Unicode.get_prev_mbchar_size(current_line, @byte_pointer) + line, = byteslice!(current_line, @byte_pointer - byte_size, byte_size) + set_current_line(line, @byte_pointer - byte_size) + end end - arg -= 1 - em_delete_prev_char(key, arg: arg) if arg > 0 + process_auto_indent end alias_method :backward_delete_char, :em_delete_prev_char @@ -2543,23 +1893,23 @@ class Reline::LineEditor # the line. With a negative numeric argument, kill backward # from the cursor to the beginning of the current line. private def ed_kill_line(key) - if @line.bytesize > @byte_pointer - @line, deleted = byteslice!(@line, @byte_pointer, @line.bytesize - @byte_pointer) - @byte_pointer = @line.bytesize - @cursor = @cursor_max = calculate_width(@line) + if current_line.bytesize > @byte_pointer + line, deleted = byteslice!(current_line, @byte_pointer, current_line.bytesize - @byte_pointer) + set_current_line(line, line.bytesize) @kill_ring.append(deleted) - elsif @is_multiline and @byte_pointer == @line.bytesize and @buffer_of_lines.size > @line_index + 1 - @cursor = calculate_width(@line) - @byte_pointer = @line.bytesize - @line += @buffer_of_lines.delete_at(@line_index + 1) - @cursor_max = calculate_width(@line) - @buffer_of_lines[@line_index] = @line - @rerender_all = true - @rest_height += 1 + elsif @byte_pointer == current_line.bytesize and @buffer_of_lines.size > @line_index + 1 + set_current_line(current_line + @buffer_of_lines.delete_at(@line_index + 1), current_line.bytesize) end end alias_method :kill_line, :ed_kill_line + # Editline:: +vi_change_to_eol+ (vi command: +C+) + Kill and change from the cursor to the end of the line. + private def vi_change_to_eol(key) + ed_kill_line(key) + + @config.editing_mode = :vi_insert + end + # Editline:: +vi-kill-line-prev+ (vi: +Ctrl-U+) Delete the string from the # beginning of the edit buffer to the cursor and save it to the # cut buffer. @@ -2567,11 +1917,9 @@ class Reline::LineEditor # to the beginning of the current line. private def vi_kill_line_prev(key) if @byte_pointer > 0 - @line, deleted = byteslice!(@line, 0, @byte_pointer) - @byte_pointer = 0 + line, deleted = byteslice!(current_line, 0, @byte_pointer) + set_current_line(line, 0) @kill_ring.append(deleted, true) - @cursor_max = calculate_width(@line) - @cursor = 0 end end alias_method :unix_line_discard, :vi_kill_line_prev @@ -2581,50 +1929,35 @@ class Reline::LineEditor # GNU Readline:: +kill-whole-line+ (not bound) Kill all characters on the # current line, no matter where point is. private def em_kill_line(key) - if @line.size > 0 - @kill_ring.append(@line.dup, true) - @line.clear - @byte_pointer = 0 - @cursor_max = 0 - @cursor = 0 + if current_line.size > 0 + @kill_ring.append(current_line.dup, true) + set_current_line('', 0) end end alias_method :kill_whole_line, :em_kill_line private def em_delete(key) - if @line.empty? and (not @is_multiline or @buffer_of_lines.size == 1) and key == "\C-d".ord - @line = nil - if @buffer_of_lines.size > 1 - scroll_down(@highest_in_all - @first_line_started_from) - end - Reline::IOGate.move_cursor_column(0) + if current_line.empty? and @buffer_of_lines.size == 1 and key == "\C-d".ord @eof = true finish - elsif @byte_pointer < @line.bytesize - splitted_last = @line.byteslice(@byte_pointer, @line.bytesize) + elsif @byte_pointer < current_line.bytesize + splitted_last = current_line.byteslice(@byte_pointer, current_line.bytesize) mbchar = splitted_last.grapheme_clusters.first - width = Reline::Unicode.get_mbchar_width(mbchar) - @cursor_max -= width - @line, = byteslice!(@line, @byte_pointer, mbchar.bytesize) - elsif @is_multiline and @byte_pointer == @line.bytesize and @buffer_of_lines.size > @line_index + 1 - @cursor = calculate_width(@line) - @byte_pointer = @line.bytesize - @line += @buffer_of_lines.delete_at(@line_index + 1) - @cursor_max = calculate_width(@line) - @buffer_of_lines[@line_index] = @line - @rerender_all = true - @rest_height += 1 + line, = byteslice!(current_line, @byte_pointer, mbchar.bytesize) + set_current_line(line) + elsif @byte_pointer == current_line.bytesize and @buffer_of_lines.size > @line_index + 1 + set_current_line(current_line + @buffer_of_lines.delete_at(@line_index + 1), current_line.bytesize) end end alias_method :delete_char, :em_delete private def em_delete_or_list(key) - if @line.empty? or @byte_pointer < @line.bytesize + if current_line.empty? or @byte_pointer < current_line.bytesize em_delete(key) - else # show completed list + elsif !@config.autocompletion # show completed list result = call_completion_proc if result.is_a?(Array) - complete(result, true) + perform_completion(result, true) end end end @@ -2632,162 +1965,136 @@ class Reline::LineEditor private def em_yank(key) yanked = @kill_ring.yank - if yanked - @line = byteinsert(@line, @byte_pointer, yanked) - yanked_width = calculate_width(yanked) - @cursor += yanked_width - @cursor_max += yanked_width - @byte_pointer += yanked.bytesize - end + insert_text(yanked) if yanked end alias_method :yank, :em_yank private def em_yank_pop(key) yanked, prev_yank = @kill_ring.yank_pop if yanked - prev_yank_width = calculate_width(prev_yank) - @cursor -= prev_yank_width - @cursor_max -= prev_yank_width - @byte_pointer -= prev_yank.bytesize - @line, = byteslice!(@line, @byte_pointer, prev_yank.bytesize) - @line = byteinsert(@line, @byte_pointer, yanked) - yanked_width = calculate_width(yanked) - @cursor += yanked_width - @cursor_max += yanked_width - @byte_pointer += yanked.bytesize + line, = byteslice!(current_line, @byte_pointer - prev_yank.bytesize, prev_yank.bytesize) + set_current_line(line, @byte_pointer - prev_yank.bytesize) + insert_text(yanked) end end alias_method :yank_pop, :em_yank_pop private def ed_clear_screen(key) - @cleared = true + Reline::IOGate.clear_screen + @screen_size = Reline::IOGate.get_screen_size + @rendered_screen.lines = [] + @rendered_screen.base_y = 0 + @rendered_screen.cursor_y = 0 end alias_method :clear_screen, :ed_clear_screen private def em_next_word(key) - if @line.bytesize > @byte_pointer - byte_size, width = Reline::Unicode.em_forward_word(@line, @byte_pointer) + if current_line.bytesize > @byte_pointer + byte_size, _ = Reline::Unicode.em_forward_word(current_line, @byte_pointer) @byte_pointer += byte_size - @cursor += width end end alias_method :forward_word, :em_next_word private def ed_prev_word(key) if @byte_pointer > 0 - byte_size, width = Reline::Unicode.em_backward_word(@line, @byte_pointer) + byte_size, _ = Reline::Unicode.em_backward_word(current_line, @byte_pointer) @byte_pointer -= byte_size - @cursor -= width end end alias_method :backward_word, :ed_prev_word private def em_delete_next_word(key) - if @line.bytesize > @byte_pointer - byte_size, width = Reline::Unicode.em_forward_word(@line, @byte_pointer) - @line, word = byteslice!(@line, @byte_pointer, byte_size) + if current_line.bytesize > @byte_pointer + byte_size, _ = Reline::Unicode.em_forward_word(current_line, @byte_pointer) + line, word = byteslice!(current_line, @byte_pointer, byte_size) + set_current_line(line) @kill_ring.append(word) - @cursor_max -= width end end + alias_method :kill_word, :em_delete_next_word private def ed_delete_prev_word(key) if @byte_pointer > 0 - byte_size, width = Reline::Unicode.em_backward_word(@line, @byte_pointer) - @line, word = byteslice!(@line, @byte_pointer - byte_size, byte_size) + byte_size, _ = Reline::Unicode.em_backward_word(current_line, @byte_pointer) + line, word = byteslice!(current_line, @byte_pointer - byte_size, byte_size) + set_current_line(line, @byte_pointer - byte_size) @kill_ring.append(word, true) - @byte_pointer -= byte_size - @cursor -= width - @cursor_max -= width end end + alias_method :backward_kill_word, :ed_delete_prev_word private def ed_transpose_chars(key) if @byte_pointer > 0 - if @cursor_max > @cursor - byte_size = Reline::Unicode.get_next_mbchar_size(@line, @byte_pointer) - mbchar = @line.byteslice(@byte_pointer, byte_size) - width = Reline::Unicode.get_mbchar_width(mbchar) - @cursor += width + if @byte_pointer < current_line.bytesize + byte_size = Reline::Unicode.get_next_mbchar_size(current_line, @byte_pointer) @byte_pointer += byte_size end - back1_byte_size = Reline::Unicode.get_prev_mbchar_size(@line, @byte_pointer) + back1_byte_size = Reline::Unicode.get_prev_mbchar_size(current_line, @byte_pointer) if (@byte_pointer - back1_byte_size) > 0 - back2_byte_size = Reline::Unicode.get_prev_mbchar_size(@line, @byte_pointer - back1_byte_size) + back2_byte_size = Reline::Unicode.get_prev_mbchar_size(current_line, @byte_pointer - back1_byte_size) back2_pointer = @byte_pointer - back1_byte_size - back2_byte_size - @line, back2_mbchar = byteslice!(@line, back2_pointer, back2_byte_size) - @line = byteinsert(@line, @byte_pointer - back2_byte_size, back2_mbchar) + line, back2_mbchar = byteslice!(current_line, back2_pointer, back2_byte_size) + set_current_line(byteinsert(line, @byte_pointer - back2_byte_size, back2_mbchar)) end end end alias_method :transpose_chars, :ed_transpose_chars private def ed_transpose_words(key) - left_word_start, middle_start, right_word_start, after_start = Reline::Unicode.ed_transpose_words(@line, @byte_pointer) - before = @line.byteslice(0, left_word_start) - left_word = @line.byteslice(left_word_start, middle_start - left_word_start) - middle = @line.byteslice(middle_start, right_word_start - middle_start) - right_word = @line.byteslice(right_word_start, after_start - right_word_start) - after = @line.byteslice(after_start, @line.bytesize - after_start) + left_word_start, middle_start, right_word_start, after_start = Reline::Unicode.ed_transpose_words(current_line, @byte_pointer) + before = current_line.byteslice(0, left_word_start) + left_word = current_line.byteslice(left_word_start, middle_start - left_word_start) + middle = current_line.byteslice(middle_start, right_word_start - middle_start) + right_word = current_line.byteslice(right_word_start, after_start - right_word_start) + after = current_line.byteslice(after_start, current_line.bytesize - after_start) return if left_word.empty? or right_word.empty? - @line = before + right_word + middle + left_word + after from_head_to_left_word = before + right_word + middle + left_word - @byte_pointer = from_head_to_left_word.bytesize - @cursor = calculate_width(from_head_to_left_word) + set_current_line(from_head_to_left_word + after, from_head_to_left_word.bytesize) end alias_method :transpose_words, :ed_transpose_words private def em_capitol_case(key) - if @line.bytesize > @byte_pointer - byte_size, _, new_str = Reline::Unicode.em_forward_word_with_capitalization(@line, @byte_pointer) - before = @line.byteslice(0, @byte_pointer) - after = @line.byteslice((@byte_pointer + byte_size)..-1) - @line = before + new_str + after - @byte_pointer += new_str.bytesize - @cursor += calculate_width(new_str) + if current_line.bytesize > @byte_pointer + byte_size, _, new_str = Reline::Unicode.em_forward_word_with_capitalization(current_line, @byte_pointer) + before = current_line.byteslice(0, @byte_pointer) + after = current_line.byteslice((@byte_pointer + byte_size)..-1) + set_current_line(before + new_str + after, @byte_pointer + new_str.bytesize) end end alias_method :capitalize_word, :em_capitol_case private def em_lower_case(key) - if @line.bytesize > @byte_pointer - byte_size, = Reline::Unicode.em_forward_word(@line, @byte_pointer) - part = @line.byteslice(@byte_pointer, byte_size).grapheme_clusters.map { |mbchar| + if current_line.bytesize > @byte_pointer + byte_size, = Reline::Unicode.em_forward_word(current_line, @byte_pointer) + part = current_line.byteslice(@byte_pointer, byte_size).grapheme_clusters.map { |mbchar| mbchar =~ /[A-Z]/ ? mbchar.downcase : mbchar }.join - rest = @line.byteslice((@byte_pointer + byte_size)..-1) - @line = @line.byteslice(0, @byte_pointer) + part - @byte_pointer = @line.bytesize - @cursor = calculate_width(@line) - @cursor_max = @cursor + calculate_width(rest) - @line += rest + rest = current_line.byteslice((@byte_pointer + byte_size)..-1) + line = current_line.byteslice(0, @byte_pointer) + part + set_current_line(line + rest, line.bytesize) end end alias_method :downcase_word, :em_lower_case private def em_upper_case(key) - if @line.bytesize > @byte_pointer - byte_size, = Reline::Unicode.em_forward_word(@line, @byte_pointer) - part = @line.byteslice(@byte_pointer, byte_size).grapheme_clusters.map { |mbchar| + if current_line.bytesize > @byte_pointer + byte_size, = Reline::Unicode.em_forward_word(current_line, @byte_pointer) + part = current_line.byteslice(@byte_pointer, byte_size).grapheme_clusters.map { |mbchar| mbchar =~ /[a-z]/ ? mbchar.upcase : mbchar }.join - rest = @line.byteslice((@byte_pointer + byte_size)..-1) - @line = @line.byteslice(0, @byte_pointer) + part - @byte_pointer = @line.bytesize - @cursor = calculate_width(@line) - @cursor_max = @cursor + calculate_width(rest) - @line += rest + rest = current_line.byteslice((@byte_pointer + byte_size)..-1) + line = current_line.byteslice(0, @byte_pointer) + part + set_current_line(line + rest, line.bytesize) end end alias_method :upcase_word, :em_upper_case private def em_kill_region(key) if @byte_pointer > 0 - byte_size, width = Reline::Unicode.em_big_backward_word(@line, @byte_pointer) - @line, deleted = byteslice!(@line, @byte_pointer - byte_size, byte_size) - @byte_pointer -= byte_size - @cursor -= width - @cursor_max -= width + byte_size, _ = Reline::Unicode.em_big_backward_word(current_line, @byte_pointer) + line, deleted = byteslice!(current_line, @byte_pointer - byte_size, byte_size) + set_current_line(line, @byte_pointer - byte_size) @kill_ring.append(deleted, true) end end @@ -2815,10 +2122,9 @@ class Reline::LineEditor alias_method :vi_movement_mode, :vi_command_mode private def vi_next_word(key, arg: 1) - if @line.bytesize > @byte_pointer - byte_size, width = Reline::Unicode.vi_forward_word(@line, @byte_pointer, @drop_terminate_spaces) + if current_line.bytesize > @byte_pointer + byte_size, _ = Reline::Unicode.vi_forward_word(current_line, @byte_pointer, @drop_terminate_spaces) @byte_pointer += byte_size - @cursor += width end arg -= 1 vi_next_word(key, arg: arg) if arg > 0 @@ -2826,38 +2132,32 @@ class Reline::LineEditor private def vi_prev_word(key, arg: 1) if @byte_pointer > 0 - byte_size, width = Reline::Unicode.vi_backward_word(@line, @byte_pointer) + byte_size, _ = Reline::Unicode.vi_backward_word(current_line, @byte_pointer) @byte_pointer -= byte_size - @cursor -= width end arg -= 1 vi_prev_word(key, arg: arg) if arg > 0 end private def vi_end_word(key, arg: 1, inclusive: false) - if @line.bytesize > @byte_pointer - byte_size, width = Reline::Unicode.vi_forward_end_word(@line, @byte_pointer) + if current_line.bytesize > @byte_pointer + byte_size, _ = Reline::Unicode.vi_forward_end_word(current_line, @byte_pointer) @byte_pointer += byte_size - @cursor += width end arg -= 1 if inclusive and arg.zero? - byte_size = Reline::Unicode.get_next_mbchar_size(@line, @byte_pointer) + byte_size = Reline::Unicode.get_next_mbchar_size(current_line, @byte_pointer) if byte_size > 0 - c = @line.byteslice(@byte_pointer, byte_size) - width = Reline::Unicode.get_mbchar_width(c) @byte_pointer += byte_size - @cursor += width end end vi_end_word(key, arg: arg) if arg > 0 end private def vi_next_big_word(key, arg: 1) - if @line.bytesize > @byte_pointer - byte_size, width = Reline::Unicode.vi_big_forward_word(@line, @byte_pointer) + if current_line.bytesize > @byte_pointer + byte_size, _ = Reline::Unicode.vi_big_forward_word(current_line, @byte_pointer) @byte_pointer += byte_size - @cursor += width end arg -= 1 vi_next_big_word(key, arg: arg) if arg > 0 @@ -2865,50 +2165,39 @@ class Reline::LineEditor private def vi_prev_big_word(key, arg: 1) if @byte_pointer > 0 - byte_size, width = Reline::Unicode.vi_big_backward_word(@line, @byte_pointer) + byte_size, _ = Reline::Unicode.vi_big_backward_word(current_line, @byte_pointer) @byte_pointer -= byte_size - @cursor -= width end arg -= 1 vi_prev_big_word(key, arg: arg) if arg > 0 end private def vi_end_big_word(key, arg: 1, inclusive: false) - if @line.bytesize > @byte_pointer - byte_size, width = Reline::Unicode.vi_big_forward_end_word(@line, @byte_pointer) + if current_line.bytesize > @byte_pointer + byte_size, _ = Reline::Unicode.vi_big_forward_end_word(current_line, @byte_pointer) @byte_pointer += byte_size - @cursor += width end arg -= 1 if inclusive and arg.zero? - byte_size = Reline::Unicode.get_next_mbchar_size(@line, @byte_pointer) + byte_size = Reline::Unicode.get_next_mbchar_size(current_line, @byte_pointer) if byte_size > 0 - c = @line.byteslice(@byte_pointer, byte_size) - width = Reline::Unicode.get_mbchar_width(c) @byte_pointer += byte_size - @cursor += width end end vi_end_big_word(key, arg: arg) if arg > 0 end private def vi_delete_prev_char(key) - if @is_multiline and @cursor == 0 and @line_index > 0 - @buffer_of_lines[@line_index] = @line - @cursor = calculate_width(@buffer_of_lines[@line_index - 1]) + if @byte_pointer == 0 and @line_index > 0 @byte_pointer = @buffer_of_lines[@line_index - 1].bytesize @buffer_of_lines[@line_index - 1] += @buffer_of_lines.delete_at(@line_index) @line_index -= 1 - @line = @buffer_of_lines[@line_index] - @cursor_max = calculate_width(@line) - @rerender_all = true - elsif @cursor > 0 - byte_size = Reline::Unicode.get_prev_mbchar_size(@line, @byte_pointer) + process_auto_indent cursor_dependent: false + elsif @byte_pointer > 0 + byte_size = Reline::Unicode.get_prev_mbchar_size(current_line, @byte_pointer) @byte_pointer -= byte_size - @line, mbchar = byteslice!(@line, @byte_pointer, byte_size) - width = Reline::Unicode.get_mbchar_width(mbchar) - @cursor -= width - @cursor_max -= width + line, _ = byteslice!(current_line, @byte_pointer, byte_size) + set_current_line(line) end end @@ -2923,78 +2212,81 @@ class Reline::LineEditor end private def ed_delete_prev_char(key, arg: 1) - deleted = '' + deleted = +'' arg.times do - if @cursor > 0 - byte_size = Reline::Unicode.get_prev_mbchar_size(@line, @byte_pointer) + if @byte_pointer > 0 + byte_size = Reline::Unicode.get_prev_mbchar_size(current_line, @byte_pointer) @byte_pointer -= byte_size - @line, mbchar = byteslice!(@line, @byte_pointer, byte_size) + line, mbchar = byteslice!(current_line, @byte_pointer, byte_size) + set_current_line(line) deleted.prepend(mbchar) - width = Reline::Unicode.get_mbchar_width(mbchar) - @cursor -= width - @cursor_max -= width end end copy_for_vi(deleted) end - private def vi_zero(key) - @byte_pointer = 0 - @cursor = 0 - end - - private def vi_change_meta(key, arg: 1) - @drop_terminate_spaces = true - @waiting_operator_proc = proc { |cursor_diff, byte_pointer_diff| - if byte_pointer_diff > 0 - @line, cut = byteslice!(@line, @byte_pointer, byte_pointer_diff) - elsif byte_pointer_diff < 0 - @line, cut = byteslice!(@line, @byte_pointer + byte_pointer_diff, -byte_pointer_diff) - end - copy_for_vi(cut) - @cursor += cursor_diff if cursor_diff < 0 - @cursor_max -= cursor_diff.abs - @byte_pointer += byte_pointer_diff if byte_pointer_diff < 0 - @config.editing_mode = :vi_insert - @drop_terminate_spaces = false - } - @waiting_operator_vi_arg = arg + private def vi_change_meta(key, arg: nil) + if @vi_waiting_operator + set_current_line('', 0) if @vi_waiting_operator == :vi_change_meta_confirm && arg.nil? + @vi_waiting_operator = nil + @vi_waiting_operator_arg = nil + else + @drop_terminate_spaces = true + @vi_waiting_operator = :vi_change_meta_confirm + @vi_waiting_operator_arg = arg || 1 + end end - private def vi_delete_meta(key, arg: 1) - @waiting_operator_proc = proc { |cursor_diff, byte_pointer_diff| - if byte_pointer_diff > 0 - @line, cut = byteslice!(@line, @byte_pointer, byte_pointer_diff) - elsif byte_pointer_diff < 0 - @line, cut = byteslice!(@line, @byte_pointer + byte_pointer_diff, -byte_pointer_diff) - end - copy_for_vi(cut) - @cursor += cursor_diff if cursor_diff < 0 - @cursor_max -= cursor_diff.abs - @byte_pointer += byte_pointer_diff if byte_pointer_diff < 0 - } - @waiting_operator_vi_arg = arg + private def vi_change_meta_confirm(byte_pointer_diff) + vi_delete_meta_confirm(byte_pointer_diff) + @config.editing_mode = :vi_insert + @drop_terminate_spaces = false end - private def vi_yank(key, arg: 1) - @waiting_operator_proc = proc { |cursor_diff, byte_pointer_diff| - if byte_pointer_diff > 0 - cut = @line.byteslice(@byte_pointer, byte_pointer_diff) - elsif byte_pointer_diff < 0 - cut = @line.byteslice(@byte_pointer + byte_pointer_diff, -byte_pointer_diff) - end - copy_for_vi(cut) - } - @waiting_operator_vi_arg = arg + private def vi_delete_meta(key, arg: nil) + if @vi_waiting_operator + set_current_line('', 0) if @vi_waiting_operator == :vi_delete_meta_confirm && arg.nil? + @vi_waiting_operator = nil + @vi_waiting_operator_arg = nil + else + @vi_waiting_operator = :vi_delete_meta_confirm + @vi_waiting_operator_arg = arg || 1 + end + end + + private def vi_delete_meta_confirm(byte_pointer_diff) + if byte_pointer_diff > 0 + line, cut = byteslice!(current_line, @byte_pointer, byte_pointer_diff) + elsif byte_pointer_diff < 0 + line, cut = byteslice!(current_line, @byte_pointer + byte_pointer_diff, -byte_pointer_diff) + end + copy_for_vi(cut) + set_current_line(line || '', @byte_pointer + (byte_pointer_diff < 0 ? byte_pointer_diff : 0)) + end + + private def vi_yank(key, arg: nil) + if @vi_waiting_operator + copy_for_vi(current_line) if @vi_waiting_operator == :vi_yank_confirm && arg.nil? + @vi_waiting_operator = nil + @vi_waiting_operator_arg = nil + else + @vi_waiting_operator = :vi_yank_confirm + @vi_waiting_operator_arg = arg || 1 + end + end + + private def vi_yank_confirm(byte_pointer_diff) + if byte_pointer_diff > 0 + cut = current_line.byteslice(@byte_pointer, byte_pointer_diff) + elsif byte_pointer_diff < 0 + cut = current_line.byteslice(@byte_pointer + byte_pointer_diff, -byte_pointer_diff) + end + copy_for_vi(cut) end private def vi_list_or_eof(key) - if (not @is_multiline and @line.empty?) or (@is_multiline and @line.empty? and @buffer_of_lines.size == 1) - @line = nil - if @buffer_of_lines.size > 1 - scroll_down(@highest_in_all - @first_line_started_from) - end - Reline::IOGate.move_cursor_column(0) + if current_line.empty? and @buffer_of_lines.size == 1 + set_current_line('', 0) @eof = true finish else @@ -3005,18 +2297,15 @@ class Reline::LineEditor alias_method :vi_eof_maybe, :vi_list_or_eof private def ed_delete_next_char(key, arg: 1) - byte_size = Reline::Unicode.get_next_mbchar_size(@line, @byte_pointer) - unless @line.empty? || byte_size == 0 - @line, mbchar = byteslice!(@line, @byte_pointer, byte_size) + byte_size = Reline::Unicode.get_next_mbchar_size(current_line, @byte_pointer) + unless current_line.empty? || byte_size == 0 + line, mbchar = byteslice!(current_line, @byte_pointer, byte_size) copy_for_vi(mbchar) - width = Reline::Unicode.get_mbchar_width(mbchar) - @cursor_max -= width - if @cursor > 0 and @cursor >= @cursor_max - byte_size = Reline::Unicode.get_prev_mbchar_size(@line, @byte_pointer) - mbchar = @line.byteslice(@byte_pointer - byte_size, byte_size) - width = Reline::Unicode.get_mbchar_width(mbchar) - @byte_pointer -= byte_size - @cursor -= width + if @byte_pointer > 0 && current_line.bytesize == @byte_pointer + byte_size + byte_size = Reline::Unicode.get_prev_mbchar_size(line, @byte_pointer) + set_current_line(line, @byte_pointer - byte_size) + else + set_current_line(line, @byte_pointer) end end arg -= 1 @@ -3027,54 +2316,25 @@ class Reline::LineEditor if Reline::HISTORY.empty? return end - if @history_pointer.nil? - @history_pointer = 0 - @line_backup_in_history = @line - @line = Reline::HISTORY[@history_pointer] - @cursor_max = calculate_width(@line) - @cursor = 0 - @byte_pointer = 0 - elsif @history_pointer.zero? - return - else - Reline::HISTORY[@history_pointer] = @line - @history_pointer = 0 - @line = Reline::HISTORY[@history_pointer] - @cursor_max = calculate_width(@line) - @cursor = 0 - @byte_pointer = 0 - end + move_history(0, line: :start, cursor: :start) end private def vi_histedit(key) path = Tempfile.open { |fp| - if @is_multiline - fp.write whole_lines.join("\n") - else - fp.write @line - end + fp.write whole_lines.join("\n") fp.path } system("#{ENV['EDITOR']} #{path}") - if @is_multiline - @buffer_of_lines = File.read(path).split("\n") - @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? - @line_index = 0 - @line = @buffer_of_lines[@line_index] - @rerender_all = true - else - @line = File.read(path) - end + @buffer_of_lines = File.read(path).split("\n") + @buffer_of_lines = [String.new(encoding: @encoding)] if @buffer_of_lines.empty? + @line_index = 0 finish end private def vi_paste_prev(key, arg: 1) if @vi_clipboard.size > 0 - @line = byteinsert(@line, @byte_pointer, @vi_clipboard) - @cursor_max += calculate_width(@vi_clipboard) cursor_point = @vi_clipboard.grapheme_clusters[0..-2].join - @cursor += calculate_width(cursor_point) - @byte_pointer += cursor_point.bytesize + set_current_line(byteinsert(current_line, @byte_pointer, @vi_clipboard), @byte_pointer + cursor_point.bytesize) end arg -= 1 vi_paste_prev(key, arg: arg) if arg > 0 @@ -3082,11 +2342,9 @@ class Reline::LineEditor private def vi_paste_next(key, arg: 1) if @vi_clipboard.size > 0 - byte_size = Reline::Unicode.get_next_mbchar_size(@line, @byte_pointer) - @line = byteinsert(@line, @byte_pointer + byte_size, @vi_clipboard) - @cursor_max += calculate_width(@vi_clipboard) - @cursor += calculate_width(@vi_clipboard) - @byte_pointer += @vi_clipboard.bytesize + byte_size = Reline::Unicode.get_next_mbchar_size(current_line, @byte_pointer) + line = byteinsert(current_line, @byte_pointer + byte_size, @vi_clipboard) + set_current_line(line, @byte_pointer + @vi_clipboard.bytesize) end arg -= 1 vi_paste_next(key, arg: arg) if arg > 0 @@ -3110,43 +2368,33 @@ class Reline::LineEditor end private def vi_to_column(key, arg: 0) - @byte_pointer, @cursor = @line.grapheme_clusters.inject([0, 0]) { |total, gc| - # total has [byte_size, cursor] + # Implementing behavior of vi, not Readline's vi-mode. + @byte_pointer, = current_line.grapheme_clusters.inject([0, 0]) { |(total_byte_size, total_width), gc| mbchar_width = Reline::Unicode.get_mbchar_width(gc) - if (total.last + mbchar_width) >= arg - break total - elsif (total.last + mbchar_width) >= @cursor_max - break total - else - total = [total.first + gc.bytesize, total.last + mbchar_width] - total - end + break [total_byte_size, total_width] if (total_width + mbchar_width) >= arg + [total_byte_size + gc.bytesize, total_width + mbchar_width] } end private def vi_replace_char(key, arg: 1) @waiting_proc = ->(k) { if arg == 1 - byte_size = Reline::Unicode.get_next_mbchar_size(@line, @byte_pointer) - before = @line.byteslice(0, @byte_pointer) + byte_size = Reline::Unicode.get_next_mbchar_size(current_line, @byte_pointer) + before = current_line.byteslice(0, @byte_pointer) remaining_point = @byte_pointer + byte_size - after = @line.byteslice(remaining_point, @line.bytesize - remaining_point) - @line = before + k.chr + after - @cursor_max = calculate_width(@line) + after = current_line.byteslice(remaining_point, current_line.bytesize - remaining_point) + set_current_line(before + k.chr + after) @waiting_proc = nil elsif arg > 1 byte_size = 0 arg.times do - byte_size += Reline::Unicode.get_next_mbchar_size(@line, @byte_pointer + byte_size) + byte_size += Reline::Unicode.get_next_mbchar_size(current_line, @byte_pointer + byte_size) end - before = @line.byteslice(0, @byte_pointer) + before = current_line.byteslice(0, @byte_pointer) remaining_point = @byte_pointer + byte_size - after = @line.byteslice(remaining_point, @line.bytesize - remaining_point) + after = current_line.byteslice(remaining_point, current_line.bytesize - remaining_point) replaced = k.chr * arg - @line = before + replaced + after - @byte_pointer += replaced.bytesize - @cursor += calculate_width(replaced) - @cursor_max = calculate_width(@line) + set_current_line(before + replaced + after, @byte_pointer + replaced.bytesize) @waiting_proc = nil end } @@ -3169,7 +2417,7 @@ class Reline::LineEditor prev_total = nil total = nil found = false - @line.byteslice(@byte_pointer..-1).grapheme_clusters.each do |mbchar| + current_line.byteslice(@byte_pointer..-1).grapheme_clusters.each do |mbchar| # total has [byte_size, cursor] unless total # skip cursor point @@ -3189,21 +2437,16 @@ class Reline::LineEditor end end if not need_prev_char and found and total - byte_size, width = total + byte_size, _ = total @byte_pointer += byte_size - @cursor += width elsif need_prev_char and found and prev_total - byte_size, width = prev_total + byte_size, _ = prev_total @byte_pointer += byte_size - @cursor += width end if inclusive - byte_size = Reline::Unicode.get_next_mbchar_size(@line, @byte_pointer) + byte_size = Reline::Unicode.get_next_mbchar_size(current_line, @byte_pointer) if byte_size > 0 - c = @line.byteslice(@byte_pointer, byte_size) - width = Reline::Unicode.get_mbchar_width(c) @byte_pointer += byte_size - @cursor += width end end @waiting_proc = nil @@ -3226,7 +2469,7 @@ class Reline::LineEditor prev_total = nil total = nil found = false - @line.byteslice(0..@byte_pointer).grapheme_clusters.reverse_each do |mbchar| + current_line.byteslice(0..@byte_pointer).grapheme_clusters.reverse_each do |mbchar| # total has [byte_size, cursor] unless total # skip cursor point @@ -3246,26 +2489,19 @@ class Reline::LineEditor end end if not need_next_char and found and total - byte_size, width = total + byte_size, _ = total @byte_pointer -= byte_size - @cursor -= width elsif need_next_char and found and prev_total - byte_size, width = prev_total + byte_size, _ = prev_total @byte_pointer -= byte_size - @cursor -= width end @waiting_proc = nil end private def vi_join_lines(key, arg: 1) - if @is_multiline and @buffer_of_lines.size > @line_index + 1 - @cursor = calculate_width(@line) - @byte_pointer = @line.bytesize - @line += ' ' + @buffer_of_lines.delete_at(@line_index + 1).lstrip - @cursor_max = calculate_width(@line) - @buffer_of_lines[@line_index] = @line - @rerender_all = true - @rest_height += 1 + if @buffer_of_lines.size > @line_index + 1 + next_line = @buffer_of_lines.delete_at(@line_index + 1).lstrip + set_current_line(current_line + ' ' + next_line, current_line.bytesize) end arg -= 1 vi_join_lines(key, arg: arg) if arg > 0 @@ -3279,11 +2515,27 @@ class Reline::LineEditor private def em_exchange_mark(key) return unless @mark_pointer new_pointer = [@byte_pointer, @line_index] - @previous_line_index = @line_index @byte_pointer, @line_index = @mark_pointer - @cursor = calculate_width(@line.byteslice(0, @byte_pointer)) - @cursor_max = calculate_width(@line) @mark_pointer = new_pointer end alias_method :exchange_point_and_mark, :em_exchange_mark + + private def emacs_editing_mode(key) + @config.editing_mode = :emacs + end + + private def vi_editing_mode(key) + @config.editing_mode = :vi_insert + end + + private def undo(_key) + return if @past_lines.empty? + + @undoing = true + + target_lines, target_cursor_x, target_cursor_y = @past_lines.last + set_current_lines(target_lines, target_cursor_x, target_cursor_y) + + @past_lines.pop + end end diff --git a/lib/reline/reline.gemspec b/lib/reline/reline.gemspec index 7bf1f8758b..dfaf966728 100644 --- a/lib/reline/reline.gemspec +++ b/lib/reline/reline.gemspec @@ -18,6 +18,11 @@ Gem::Specification.new do |spec| spec.files = Dir['BSDL', 'COPYING', 'README.md', 'license_of_rb-readline', 'lib/**/*'] spec.require_paths = ['lib'] + spec.metadata = { + "bug_tracker_uri" => "https://github.com/ruby/reline/issues", + "changelog_uri" => "https://github.com/ruby/reline/releases", + "source_code_uri" => "https://github.com/ruby/reline" + } spec.required_ruby_version = Gem::Requirement.new('>= 2.6') diff --git a/lib/reline/terminfo.rb b/lib/reline/terminfo.rb index 2cfa32b9f7..6885a0c6be 100644 --- a/lib/reline/terminfo.rb +++ b/lib/reline/terminfo.rb @@ -80,23 +80,11 @@ module Reline::Terminfo def self.setupterm(term, fildes) errret_int = Fiddle::Pointer.malloc(Fiddle::SIZEOF_INT) ret = @setupterm.(term, fildes, errret_int) - errret = errret_int[0, Fiddle::SIZEOF_INT].unpack1('i') case ret when 0 # OK - 0 + @term_supported = true when -1 # ERR - case errret - when 1 - raise TerminfoError.new('The terminal is hardcopy, cannot be used for curses applications.') - when 0 - raise TerminfoError.new('The terminal could not be found, or that it is a generic type, having too little information for curses applications to run.') - when -1 - raise TerminfoError.new('The terminfo database could not be found.') - else # unknown - -1 - end - else # unknown - -2 + @term_supported = false end end @@ -148,9 +136,14 @@ module Reline::Terminfo num end + # NOTE: This means Fiddle and curses are enabled. def self.enabled? true end + + def self.term_supported? + @term_supported + end end if Reline::Terminfo.curses_dl module Reline::Terminfo diff --git a/lib/reline/unicode.rb b/lib/reline/unicode.rb index 0a7f59cf06..d7460d6d4a 100644 --- a/lib/reline/unicode.rb +++ b/lib/reline/unicode.rb @@ -41,33 +41,15 @@ class Reline::Unicode OSC_REGEXP = /\e\]\d+(?:;[^;\a\e]+)*(?:\a|\e\\)/ WIDTH_SCANNER = /\G(?:(#{NON_PRINTING_START})|(#{NON_PRINTING_END})|(#{CSI_REGEXP})|(#{OSC_REGEXP})|(\X))/o - def self.get_mbchar_byte_size_by_first_char(c) - # Checks UTF-8 character byte size - case c.ord - # 0b0xxxxxxx - when ->(code) { (code ^ 0b10000000).allbits?(0b10000000) } then 1 - # 0b110xxxxx - when ->(code) { (code ^ 0b00100000).allbits?(0b11100000) } then 2 - # 0b1110xxxx - when ->(code) { (code ^ 0b00010000).allbits?(0b11110000) } then 3 - # 0b11110xxx - when ->(code) { (code ^ 0b00001000).allbits?(0b11111000) } then 4 - # 0b111110xx - when ->(code) { (code ^ 0b00000100).allbits?(0b11111100) } then 5 - # 0b1111110x - when ->(code) { (code ^ 0b00000010).allbits?(0b11111110) } then 6 - # successor of mbchar - else 0 - end - end - def self.escape_for_print(str) str.chars.map! { |gr| - escaped = EscapedPairs[gr.ord] - if escaped && gr != -"\n" && gr != -"\t" - escaped - else + case gr + when -"\n" gr + when -"\t" + -' ' + else + EscapedPairs[gr.ord] || gr end }.join end @@ -148,10 +130,10 @@ class Reline::Unicode end end - def self.split_by_width(str, max_width, encoding = str.encoding) + def self.split_by_width(str, max_width, encoding = str.encoding, offset: 0) lines = [String.new(encoding: encoding)] height = 1 - width = 0 + width = offset rest = str.encode(Encoding::UTF_8) in_zero_width = false seq = String.new(encoding: encoding) @@ -165,7 +147,13 @@ class Reline::Unicode lines.last << NON_PRINTING_END when csi lines.last << csi - seq << csi + unless in_zero_width + if csi == -"\e[m" || csi == -"\e[0m" + seq.clear + else + seq << csi + end + end when osc lines.last << osc seq << osc @@ -193,32 +181,78 @@ class Reline::Unicode # Take a chunk of a String cut by width with escape sequences. def self.take_range(str, start_col, max_width) + take_mbchar_range(str, start_col, max_width).first + end + + def self.take_mbchar_range(str, start_col, width, cover_begin: false, cover_end: false, padding: false) chunk = String.new(encoding: str.encoding) + + end_col = start_col + width total_width = 0 rest = str.encode(Encoding::UTF_8) in_zero_width = false + chunk_start_col = nil + chunk_end_col = nil + has_csi = false rest.scan(WIDTH_SCANNER) do |non_printing_start, non_printing_end, csi, osc, gc| case when non_printing_start in_zero_width = true + chunk << NON_PRINTING_START when non_printing_end in_zero_width = false + chunk << NON_PRINTING_END when csi + has_csi = true chunk << csi when osc chunk << osc when gc if in_zero_width chunk << gc + next + end + + mbchar_width = get_mbchar_width(gc) + prev_width = total_width + total_width += mbchar_width + + if (cover_begin || padding ? total_width <= start_col : prev_width < start_col) + # Current character haven't reached start_col yet + next + elsif padding && !cover_begin && prev_width < start_col && start_col < total_width + # Add preceding padding. This padding might have background color. + chunk << ' ' + chunk_start_col ||= start_col + chunk_end_col = total_width + next + elsif (cover_end ? prev_width < end_col : total_width <= end_col) + # Current character is in the range + chunk << gc + chunk_start_col ||= prev_width + chunk_end_col = total_width + break if total_width >= end_col else - mbchar_width = get_mbchar_width(gc) - total_width += mbchar_width - break if (start_col + max_width) < total_width - chunk << gc if start_col < total_width + # Current character exceeds end_col + if padding && end_col < total_width + # Add succeeding padding. This padding might have background color. + chunk << ' ' + chunk_start_col ||= prev_width + chunk_end_col = end_col + end + break end end end - chunk + chunk_start_col ||= start_col + chunk_end_col ||= start_col + if padding && chunk_end_col < end_col + # Append padding. This padding should not include background color. + chunk << "\e[0m" if has_csi + chunk << ' ' * (end_col - chunk_end_col) + chunk_end_col = end_col + end + [chunk, chunk_start_col, chunk_end_col - chunk_start_col] end def self.get_next_mbchar_size(line, byte_pointer) diff --git a/lib/reline/unicode/east_asian_width.rb b/lib/reline/unicode/east_asian_width.rb index 97dfec4c52..fa16a1bb56 100644 --- a/lib/reline/unicode/east_asian_width.rb +++ b/lib/reline/unicode/east_asian_width.rb @@ -1,6 +1,6 @@ class Reline::Unicode::EastAsianWidth # This is based on EastAsianWidth.txt - # EastAsianWidth.txt + # UNICODE_VERSION = '15.1.0' # Fullwidth TYPE_F = /^[#{ %W( @@ -60,14 +60,14 @@ class Reline::Unicode::EastAsianWidth \u{2E80}-\u{2E99} \u{2E9B}-\u{2EF3} \u{2F00}-\u{2FD5} - \u{2FF0}-\u{2FFB} + \u{2FF0}-\u{2FFF} \u{3001}-\u{303E} \u{3041}-\u{3096} \u{3099}-\u{30FF} \u{3105}-\u{312F} \u{3131}-\u{318E} \u{3190}-\u{31E3} - \u{31F0}-\u{321E} + \u{31EF}-\u{321E} \u{3220}-\u{3247} \u{3250}-\u{4DBF} \u{4E00}-\u{A48C} diff --git a/lib/reline/version.rb b/lib/reline/version.rb index abb200526c..46613a5952 100644 --- a/lib/reline/version.rb +++ b/lib/reline/version.rb @@ -1,3 +1,3 @@ module Reline - VERSION = '0.3.5' + VERSION = '0.5.7' end diff --git a/lib/reline/windows.rb b/lib/reline/windows.rb index 7ea2a00f63..ee3f73e383 100644 --- a/lib/reline/windows.rb +++ b/lib/reline/windows.rb @@ -1,6 +1,8 @@ require 'fiddle/import' class Reline::Windows + RESET_COLOR = "\e[0m" + def self.encoding Encoding::UTF_8 end @@ -85,7 +87,7 @@ class Reline::Windows def call(*args) import = @proto.split("") args.each_with_index do |x, i| - args[i], = [x == 0 ? nil : x].pack("p").unpack(POINTER_TYPE) if import[i] == "S" + args[i], = [x == 0 ? nil : +x].pack("p").unpack(POINTER_TYPE) if import[i] == "S" args[i], = [x].pack("I").unpack("i") if import[i] == "I" end ret, = @func.call(*args) @@ -257,7 +259,7 @@ class Reline::Windows def self.check_input_event num_of_events = 0.chr * 8 while @@output_buf.empty? - Reline.core.line_editor.resize + Reline.core.line_editor.handle_signal if @@WaitForSingleObject.(@@hConsoleInputHandle, 100) != 0 # max 0.1 sec # prevent for background consolemode change @@legacy_console = (getconsolemode() & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0) @@ -295,7 +297,7 @@ class Reline::Windows yield end - def self.getc + def self.getc(_timeout_second) check_input_event @@output_buf.shift end diff --git a/lib/resolv-replace.gemspec b/lib/resolv-replace.gemspec deleted file mode 100644 index 48f7108a8e..0000000000 --- a/lib/resolv-replace.gemspec +++ /dev/null @@ -1,22 +0,0 @@ -Gem::Specification.new do |spec| - spec.name = "resolv-replace" - spec.version = "0.1.1" - spec.authors = ["Tanaka Akira"] - spec.email = ["akr@fsij.org"] - - spec.summary = %q{Replace Socket DNS with Resolv.} - spec.description = %q{Replace Socket DNS with Resolv.} - spec.homepage = "https://github.com/ruby/resolv-replace" - spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0") - spec.licenses = ["Ruby", "BSD-2-Clause"] - - spec.metadata["homepage_uri"] = spec.homepage - spec.metadata["source_code_uri"] = spec.homepage - - spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do - `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } - end - spec.require_paths = ["lib"] - - spec.add_dependency "resolv" -end diff --git a/lib/resolv-replace.rb b/lib/resolv-replace.rb deleted file mode 100644 index a83e79d996..0000000000 --- a/lib/resolv-replace.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -require 'socket' -require 'resolv' - -class << IPSocket - # :stopdoc: - alias original_resolv_getaddress getaddress - # :startdoc: - def getaddress(host) - begin - return Resolv.getaddress(host).to_s - rescue Resolv::ResolvError - raise SocketError, "Hostname not known: #{host}" - end - end -end - -class TCPSocket < IPSocket - # :stopdoc: - alias original_resolv_initialize initialize - # :startdoc: - def initialize(host, serv, *rest) - rest[0] = IPSocket.getaddress(rest[0]) if rest[0] - original_resolv_initialize(IPSocket.getaddress(host), serv, *rest) - end -end - -class UDPSocket < IPSocket - # :stopdoc: - alias original_resolv_bind bind - # :startdoc: - def bind(host, port) - host = IPSocket.getaddress(host) if host != "" - original_resolv_bind(host, port) - end - - # :stopdoc: - alias original_resolv_connect connect - # :startdoc: - def connect(host, port) - original_resolv_connect(IPSocket.getaddress(host), port) - end - - # :stopdoc: - alias original_resolv_send send - # :startdoc: - def send(mesg, flags, *rest) - if rest.length == 2 - host, port = rest - begin - addrs = Resolv.getaddresses(host) - rescue Resolv::ResolvError - raise SocketError, "Hostname not known: #{host}" - end - addrs[0...-1].each {|addr| - begin - return original_resolv_send(mesg, flags, addr, port) - rescue SystemCallError - end - } - original_resolv_send(mesg, flags, addrs[-1], port) - else - original_resolv_send(mesg, flags, *rest) - end - end -end - -class SOCKSSocket < TCPSocket - # :stopdoc: - alias original_resolv_initialize initialize - # :startdoc: - def initialize(host, serv) - original_resolv_initialize(IPSocket.getaddress(host), port) - end -end if defined? SOCKSSocket diff --git a/lib/resolv.rb b/lib/resolv.rb index 47c4ef6b3b..e36dbce259 100644 --- a/lib/resolv.rb +++ b/lib/resolv.rb @@ -37,7 +37,7 @@ end class Resolv - VERSION = "0.2.2" + VERSION = "0.4.0" ## # Looks up the first IP address for +name+. @@ -84,8 +84,8 @@ class Resolv ## # Creates a new Resolv using +resolvers+. - def initialize(resolvers=[Hosts.new, DNS.new]) - @resolvers = resolvers + def initialize(resolvers=nil, use_ipv6: nil) + @resolvers = resolvers || [Hosts.new, DNS.new(DNS::Config.default_config_hash.merge(use_ipv6: use_ipv6))] end ## @@ -194,17 +194,10 @@ class Resolv File.open(@filename, 'rb') {|f| f.each {|line| line.sub!(/#.*/, '') - addr, hostname, *aliases = line.split(/\s+/) + addr, *hostnames = line.split(/\s+/) next unless addr - @addr2name[addr] = [] unless @addr2name.include? addr - @addr2name[addr] << hostname - @addr2name[addr].concat(aliases) - @name2addr[hostname] = [] unless @name2addr.include? hostname - @name2addr[hostname] << addr - aliases.each {|n| - @name2addr[n] = [] unless @name2addr.include? n - @name2addr[n] << addr - } + (@addr2name[addr] ||= []).concat(hostnames) + hostnames.each {|hostname| (@name2addr[hostname] ||= []) << addr} } } @name2addr.each {|name, arr| arr.reverse!} @@ -312,6 +305,8 @@ class Resolv # String:: Path to a file using /etc/resolv.conf's format. # Hash:: Must contain :nameserver, :search and :ndots keys. # :nameserver_port can be used to specify port number of nameserver address. + # :raise_timeout_errors can be used to raise timeout errors + # as exceptions instead of treating the same as an NXDOMAIN response. # # The value of :nameserver should be an address string or # an array of address strings. @@ -408,6 +403,11 @@ class Resolv end def use_ipv6? # :nodoc: + use_ipv6 = @config.use_ipv6? + unless use_ipv6.nil? + return use_ipv6 + end + begin list = Socket.ip_address_list rescue NotImplementedError @@ -750,7 +750,7 @@ class Resolv next if @socks_hash[bind_host] begin sock = UDPSocket.new(af) - rescue Errno::EAFNOSUPPORT + rescue Errno::EAFNOSUPPORT, Errno::EPROTONOSUPPORT next # The kernel doesn't support the address family. end @socks << sock @@ -1006,6 +1006,7 @@ class Resolv @mutex.synchronize { unless @initialized @nameserver_port = [] + @use_ipv6 = nil @search = nil @ndots = 1 case @config_info @@ -1030,8 +1031,12 @@ class Resolv if config_hash.include? :nameserver_port @nameserver_port = config_hash[:nameserver_port].map {|ns, port| [ns, (port || Port)] } end + if config_hash.include? :use_ipv6 + @use_ipv6 = config_hash[:use_ipv6] + end @search = config_hash[:search] if config_hash.include? :search @ndots = config_hash[:ndots] if config_hash.include? :ndots + @raise_timeout_errors = config_hash[:raise_timeout_errors] if @nameserver_port.empty? @nameserver_port << ['0.0.0.0', Port] @@ -1085,6 +1090,10 @@ class Resolv @nameserver_port end + def use_ipv6? + @use_ipv6 + end + def generate_candidates(name) candidates = nil name = Name.create(name) @@ -1118,6 +1127,7 @@ class Resolv def resolv(name) candidates = generate_candidates(name) timeouts = @timeouts || generate_timeouts + timeout_error = false begin candidates.each {|candidate| begin @@ -1129,11 +1139,13 @@ class Resolv end } } + timeout_error = true raise ResolvError.new("DNS resolv timeout: #{name}") rescue NXDomain end } rescue ResolvError + raise if @raise_timeout_errors && timeout_error end end @@ -1520,13 +1532,15 @@ class Resolv id, flag, qdcount, ancount, nscount, arcount = msg.get_unpack('nnnnnn') o.id = id + o.tc = (flag >> 9) & 1 + o.rcode = flag & 15 + return o unless o.tc.zero? + o.qr = (flag >> 15) & 1 o.opcode = (flag >> 11) & 15 o.aa = (flag >> 10) & 1 - o.tc = (flag >> 9) & 1 o.rd = (flag >> 8) & 1 o.ra = (flag >> 7) & 1 - o.rcode = flag & 15 (1..qdcount).each { name, typeclass = msg.get_question o.add_question(name, typeclass) @@ -1618,6 +1632,14 @@ class Resolv strings end + def get_list + [].tap do |values| + while @index < @limit + values << yield + end + end + end + def get_name return Name.new(self.get_labels) end @@ -1679,6 +1701,378 @@ class Resolv end ## + # SvcParams for service binding RRs. [RFC9460] + + class SvcParams + include Enumerable + + ## + # Create a list of SvcParams with the given initial content. + # + # +params+ has to be an enumerable of +SvcParam+s. + # If its content has +SvcParam+s with the duplicate key, + # the one appears last takes precedence. + + def initialize(params = []) + @params = {} + + params.each do |param| + add param + end + end + + ## + # Get SvcParam for the given +key+ in this list. + + def [](key) + @params[canonical_key(key)] + end + + ## + # Get the number of SvcParams in this list. + + def count + @params.count + end + + ## + # Get whether this list is empty. + + def empty? + @params.empty? + end + + ## + # Add the SvcParam +param+ to this list, overwriting the existing one with the same key. + + def add(param) + @params[param.class.key_number] = param + end + + ## + # Remove the +SvcParam+ with the given +key+ and return it. + + def delete(key) + @params.delete(canonical_key(key)) + end + + ## + # Enumerate the +SvcParam+s in this list. + + def each(&block) + return enum_for(:each) unless block + @params.each_value(&block) + end + + def encode(msg) # :nodoc: + @params.keys.sort.each do |key| + msg.put_pack('n', key) + msg.put_length16 do + @params.fetch(key).encode(msg) + end + end + end + + def self.decode(msg) # :nodoc: + params = msg.get_list do + key, = msg.get_unpack('n') + msg.get_length16 do + SvcParam::ClassHash[key].decode(msg) + end + end + + return self.new(params) + end + + private + + def canonical_key(key) # :nodoc: + case key + when Integer + key + when /\Akey(\d+)\z/ + Integer($1) + when Symbol + SvcParam::ClassHash[key].key_number + else + raise TypeError, 'key must be either String or Symbol' + end + end + end + + + ## + # Base class for SvcParam. [RFC9460] + + class SvcParam + + ## + # Get the presentation name of the SvcParamKey. + + def self.key_name + const_get(:KeyName) + end + + ## + # Get the registered number of the SvcParamKey. + + def self.key_number + const_get(:KeyNumber) + end + + ClassHash = Hash.new do |h, key| # :nodoc: + case key + when Integer + Generic.create(key) + when /\Akey(?<key>\d+)\z/ + Generic.create(key.to_int) + when Symbol + raise KeyError, "unknown key #{key}" + else + raise TypeError, 'key must be either String or Symbol' + end + end + + ## + # Generic SvcParam abstract class. + + class Generic < SvcParam + + ## + # SvcParamValue in wire-format byte string. + + attr_reader :value + + ## + # Create generic SvcParam + + def initialize(value) + @value = value + end + + def encode(msg) # :nodoc: + msg.put_bytes(@value) + end + + def self.decode(msg) # :nodoc: + return self.new(msg.get_bytes) + end + + def self.create(key_number) + c = Class.new(Generic) + key_name = :"key#{key_number}" + c.const_set(:KeyName, key_name) + c.const_set(:KeyNumber, key_number) + self.const_set(:"Key#{key_number}", c) + ClassHash[key_name] = ClassHash[key_number] = c + return c + end + end + + ## + # "mandatory" SvcParam -- Mandatory keys in service binding RR + + class Mandatory < SvcParam + KeyName = :mandatory + KeyNumber = 0 + ClassHash[KeyName] = ClassHash[KeyNumber] = self # :nodoc: + + ## + # Mandatory keys. + + attr_reader :keys + + ## + # Initialize "mandatory" ScvParam. + + def initialize(keys) + @keys = keys.map(&:to_int) + end + + def encode(msg) # :nodoc: + @keys.sort.each do |key| + msg.put_pack('n', key) + end + end + + def self.decode(msg) # :nodoc: + keys = msg.get_list { msg.get_unpack('n')[0] } + return self.new(keys) + end + end + + ## + # "alpn" SvcParam -- Additional supported protocols + + class ALPN < SvcParam + KeyName = :alpn + KeyNumber = 1 + ClassHash[KeyName] = ClassHash[KeyNumber] = self # :nodoc: + + ## + # Supported protocol IDs. + + attr_reader :protocol_ids + + ## + # Initialize "alpn" ScvParam. + + def initialize(protocol_ids) + @protocol_ids = protocol_ids.map(&:to_str) + end + + def encode(msg) # :nodoc: + msg.put_string_list(@protocol_ids) + end + + def self.decode(msg) # :nodoc: + return self.new(msg.get_string_list) + end + end + + ## + # "no-default-alpn" SvcParam -- No support for default protocol + + class NoDefaultALPN < SvcParam + KeyName = :'no-default-alpn' + KeyNumber = 2 + ClassHash[KeyName] = ClassHash[KeyNumber] = self # :nodoc: + + def encode(msg) # :nodoc: + # no payload + end + + def self.decode(msg) # :nodoc: + return self.new + end + end + + ## + # "port" SvcParam -- Port for alternative endpoint + + class Port < SvcParam + KeyName = :port + KeyNumber = 3 + ClassHash[KeyName] = ClassHash[KeyNumber] = self # :nodoc: + + ## + # Port number. + + attr_reader :port + + ## + # Initialize "port" ScvParam. + + def initialize(port) + @port = port.to_int + end + + def encode(msg) # :nodoc: + msg.put_pack('n', @port) + end + + def self.decode(msg) # :nodoc: + port, = msg.get_unpack('n') + return self.new(port) + end + end + + ## + # "ipv4hint" SvcParam -- IPv4 address hints + + class IPv4Hint < SvcParam + KeyName = :ipv4hint + KeyNumber = 4 + ClassHash[KeyName] = ClassHash[KeyNumber] = self # :nodoc: + + ## + # Set of IPv4 addresses. + + attr_reader :addresses + + ## + # Initialize "ipv4hint" ScvParam. + + def initialize(addresses) + @addresses = addresses.map {|address| IPv4.create(address) } + end + + def encode(msg) # :nodoc: + @addresses.each do |address| + msg.put_bytes(address.address) + end + end + + def self.decode(msg) # :nodoc: + addresses = msg.get_list { IPv4.new(msg.get_bytes(4)) } + return self.new(addresses) + end + end + + ## + # "ipv6hint" SvcParam -- IPv6 address hints + + class IPv6Hint < SvcParam + KeyName = :ipv6hint + KeyNumber = 6 + ClassHash[KeyName] = ClassHash[KeyNumber] = self # :nodoc: + + ## + # Set of IPv6 addresses. + + attr_reader :addresses + + ## + # Initialize "ipv6hint" ScvParam. + + def initialize(addresses) + @addresses = addresses.map {|address| IPv6.create(address) } + end + + def encode(msg) # :nodoc: + @addresses.each do |address| + msg.put_bytes(address.address) + end + end + + def self.decode(msg) # :nodoc: + addresses = msg.get_list { IPv6.new(msg.get_bytes(16)) } + return self.new(addresses) + end + end + + ## + # "dohpath" SvcParam -- DNS over HTTPS path template [RFC9461] + + class DoHPath < SvcParam + KeyName = :dohpath + KeyNumber = 7 + ClassHash[KeyName] = ClassHash[KeyNumber] = self # :nodoc: + + ## + # URI template for DoH queries. + + attr_reader :template + + ## + # Initialize "dohpath" ScvParam. + + def initialize(template) + @template = template.encode('utf-8') + end + + def encode(msg) # :nodoc: + msg.put_bytes(@template) + end + + def self.decode(msg) # :nodoc: + template = msg.get_bytes.force_encoding('utf-8') + return self.new(template) + end + end + end + + ## # A DNS query abstract class. class Query @@ -2143,8 +2537,70 @@ class Resolv TypeValue = 255 # :nodoc: end + ## + # CAA resource record defined in RFC 8659 + # + # These records identify certificate authority allowed to issue + # certificates for the given domain. + + class CAA < Resource + TypeValue = 257 + + ## + # Creates a new CAA for +flags+, +tag+ and +value+. + + def initialize(flags, tag, value) + unless (0..255) === flags + raise ArgumentError.new('flags must be an Integer between 0 and 255') + end + unless (1..15) === tag.bytesize + raise ArgumentError.new('length of tag must be between 1 and 15') + end + + @flags = flags + @tag = tag + @value = value + end + + ## + # Flags for this proprty: + # - Bit 0 : 0 = not critical, 1 = critical + + attr_reader :flags + + ## + # Property tag ("issue", "issuewild", "iodef"...). + + attr_reader :tag + + ## + # Property value. + + attr_reader :value + + ## + # Whether the critical flag is set on this property. + + def critical? + flags & 0x80 != 0 + end + + def encode_rdata(msg) # :nodoc: + msg.put_pack('C', @flags) + msg.put_string(@tag) + msg.put_bytes(@value) + end + + def self.decode_rdata(msg) # :nodoc: + flags, = msg.get_unpack('C') + tag = msg.get_string + value = msg.get_bytes + self.new flags, tag, value + end + end + ClassInsensitiveTypes = [ # :nodoc: - NS, CNAME, SOA, PTR, HINFO, MINFO, MX, TXT, LOC, ANY + NS, CNAME, SOA, PTR, HINFO, MINFO, MX, TXT, LOC, ANY, CAA ] ## @@ -2341,6 +2797,84 @@ class Resolv return self.new(priority, weight, port, target) end end + + ## + # Common implementation for SVCB-compatible resource records. + + class ServiceBinding + + ## + # Create a service binding resource record. + + def initialize(priority, target, params = []) + @priority = priority.to_int + @target = Name.create(target) + @params = SvcParams.new(params) + end + + ## + # The priority of this target host. + # + # The range is 0-65535. + # If set to 0, this RR is in AliasMode. Otherwise, it is in ServiceMode. + + attr_reader :priority + + ## + # The domain name of the target host. + + attr_reader :target + + ## + # The service parameters for the target host. + + attr_reader :params + + ## + # Whether this RR is in AliasMode. + + def alias_mode? + self.priority == 0 + end + + ## + # Whether this RR is in ServiceMode. + + def service_mode? + !alias_mode? + end + + def encode_rdata(msg) # :nodoc: + msg.put_pack("n", @priority) + msg.put_name(@target, compress: false) + @params.encode(msg) + end + + def self.decode_rdata(msg) # :nodoc: + priority, = msg.get_unpack("n") + target = msg.get_name + params = SvcParams.decode(msg) + return self.new(priority, target, params) + end + end + + ## + # SVCB resource record [RFC9460] + + class SVCB < ServiceBinding + TypeValue = 64 + ClassValue = IN::ClassValue + ClassHash[[TypeValue, ClassValue]] = self # :nodoc: + end + + ## + # HTTPS resource record [RFC9460] + + class HTTPS < ServiceBinding + TypeValue = 65 + ClassValue = IN::ClassValue + ClassHash[[TypeValue, ClassValue]] = self # :nodoc: + end end end end @@ -2560,11 +3094,7 @@ class Resolv attr_reader :address def to_s # :nodoc: - address = sprintf("%x:%x:%x:%x:%x:%x:%x:%x", *@address.unpack("nnnnnnnn")) - unless address.sub!(/(^|:)0(:0)+(:|$)/, '::') - address.sub!(/(^|:)0(:|$)/, '::') - end - return address + sprintf("%x:%x:%x:%x:%x:%x:%x:%x", *@address.unpack("nnnnnnnn")).sub(/(^|:)0(:0)+(:|$)/, '::') end def inspect # :nodoc: diff --git a/lib/rinda/rinda.gemspec b/lib/rinda/rinda.gemspec deleted file mode 100644 index e9f53dd864..0000000000 --- a/lib/rinda/rinda.gemspec +++ /dev/null @@ -1,35 +0,0 @@ -name = File.basename(__FILE__, ".gemspec") -version = ["lib/rinda", "."].find do |dir| - break File.foreach(File.join(__dir__, dir, "#{name}.rb")) do |line| - /^\s*VERSION\s*=\s*"(.*)"/ =~ line and break $1 - end rescue nil -end - -Gem::Specification.new do |spec| - spec.name = name - spec.version = version - spec.authors = ["Masatoshi SEKI"] - spec.email = ["seki@ruby-lang.org"] - - spec.summary = %q{The Linda distributed computing paradigm in Ruby.} - spec.description = %q{The Linda distributed computing paradigm in Ruby.} - spec.homepage = "https://github.com/ruby/rinda" - spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0") - spec.licenses = ["Ruby", "BSD-2-Clause"] - - spec.metadata["homepage_uri"] = spec.homepage - spec.metadata["source_code_uri"] = spec.homepage - - # Specify which files should be added to the gem when it is released. - # The `git ls-files -z` loads the files in the RubyGem that have been added into git. - spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do - `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } - end - spec.bindir = "exe" - spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } - spec.require_paths = ["lib"] - - spec.add_dependency "drb" - spec.add_dependency "ipaddr" - spec.add_dependency "forwardable" -end diff --git a/lib/rinda/rinda.rb b/lib/rinda/rinda.rb deleted file mode 100644 index ac3444244d..0000000000 --- a/lib/rinda/rinda.rb +++ /dev/null @@ -1,329 +0,0 @@ -# frozen_string_literal: false -require 'drb/drb' - -## -# A module to implement the Linda distributed computing paradigm in Ruby. -# -# Rinda is part of DRb (dRuby). -# -# == Example(s) -# -# See the sample/drb/ directory in the Ruby distribution, from 1.8.2 onwards. -# -#-- -# TODO -# == Introduction to Linda/rinda? -# -# == Why is this library separate from DRb? - -module Rinda - - VERSION = "0.1.1" - - ## - # Rinda error base class - - class RindaError < RuntimeError; end - - ## - # Raised when a hash-based tuple has an invalid key. - - class InvalidHashTupleKey < RindaError; end - - ## - # Raised when trying to use a canceled tuple. - - class RequestCanceledError < ThreadError; end - - ## - # Raised when trying to use an expired tuple. - - class RequestExpiredError < ThreadError; end - - ## - # A tuple is the elementary object in Rinda programming. - # Tuples may be matched against templates if the tuple and - # the template are the same size. - - class Tuple - - ## - # Creates a new Tuple from +ary_or_hash+ which must be an Array or Hash. - - def initialize(ary_or_hash) - if hash?(ary_or_hash) - init_with_hash(ary_or_hash) - else - init_with_ary(ary_or_hash) - end - end - - ## - # The number of elements in the tuple. - - def size - @tuple.size - end - - ## - # Accessor method for elements of the tuple. - - def [](k) - @tuple[k] - end - - ## - # Fetches item +k+ from the tuple. - - def fetch(k) - @tuple.fetch(k) - end - - ## - # Iterate through the tuple, yielding the index or key, and the - # value, thus ensuring arrays are iterated similarly to hashes. - - def each # FIXME - if Hash === @tuple - @tuple.each { |k, v| yield(k, v) } - else - @tuple.each_with_index { |v, k| yield(k, v) } - end - end - - ## - # Return the tuple itself - def value - @tuple - end - - private - - def hash?(ary_or_hash) - ary_or_hash.respond_to?(:keys) - end - - ## - # Munges +ary+ into a valid Tuple. - - def init_with_ary(ary) - @tuple = Array.new(ary.size) - @tuple.size.times do |i| - @tuple[i] = ary[i] - end - end - - ## - # Ensures +hash+ is a valid Tuple. - - def init_with_hash(hash) - @tuple = Hash.new - hash.each do |k, v| - raise InvalidHashTupleKey unless String === k - @tuple[k] = v - end - end - - end - - ## - # Templates are used to match tuples in Rinda. - - class Template < Tuple - - ## - # Matches this template against +tuple+. The +tuple+ must be the same - # size as the template. An element with a +nil+ value in a template acts - # as a wildcard, matching any value in the corresponding position in the - # tuple. Elements of the template match the +tuple+ if the are #== or - # #===. - # - # Template.new([:foo, 5]).match Tuple.new([:foo, 5]) # => true - # Template.new([:foo, nil]).match Tuple.new([:foo, 5]) # => true - # Template.new([String]).match Tuple.new(['hello']) # => true - # - # Template.new([:foo]).match Tuple.new([:foo, 5]) # => false - # Template.new([:foo, 6]).match Tuple.new([:foo, 5]) # => false - # Template.new([:foo, nil]).match Tuple.new([:foo]) # => false - # Template.new([:foo, 6]).match Tuple.new([:foo]) # => false - - def match(tuple) - return false unless tuple.respond_to?(:size) - return false unless tuple.respond_to?(:fetch) - return false unless self.size == tuple.size - each do |k, v| - begin - it = tuple.fetch(k) - rescue - return false - end - next if v.nil? - next if v == it - next if v === it - return false - end - return true - end - - ## - # Alias for #match. - - def ===(tuple) - match(tuple) - end - - end - - ## - # <i>Documentation?</i> - - class DRbObjectTemplate - - ## - # Creates a new DRbObjectTemplate that will match against +uri+ and +ref+. - - def initialize(uri=nil, ref=nil) - @drb_uri = uri - @drb_ref = ref - end - - ## - # This DRbObjectTemplate matches +ro+ if the remote object's drburi and - # drbref are the same. +nil+ is used as a wildcard. - - def ===(ro) - return true if super(ro) - unless @drb_uri.nil? - return false unless (@drb_uri === ro.__drburi rescue false) - end - unless @drb_ref.nil? - return false unless (@drb_ref === ro.__drbref rescue false) - end - true - end - - end - - ## - # TupleSpaceProxy allows a remote Tuplespace to appear as local. - - class TupleSpaceProxy - ## - # A Port ensures that a moved tuple arrives properly at its destination - # and does not get lost. - # - # See https://bugs.ruby-lang.org/issues/8125 - - class Port # :nodoc: - attr_reader :value - - def self.deliver - port = new - - begin - yield(port) - ensure - port.close - end - - port.value - end - - def initialize - @open = true - @value = nil - end - - ## - # Don't let the DRb thread push to it when remote sends tuple - - def close - @open = false - end - - ## - # Stores +value+ and ensure it does not get marshaled multiple times. - - def push value - raise 'port closed' unless @open - - @value = value - - nil # avoid Marshal - end - end - - ## - # Creates a new TupleSpaceProxy to wrap +ts+. - - def initialize(ts) - @ts = ts - end - - ## - # Adds +tuple+ to the proxied TupleSpace. See TupleSpace#write. - - def write(tuple, sec=nil) - @ts.write(tuple, sec) - end - - ## - # Takes +tuple+ from the proxied TupleSpace. See TupleSpace#take. - - def take(tuple, sec=nil, &block) - Port.deliver do |port| - @ts.move(DRbObject.new(port), tuple, sec, &block) - end - end - - ## - # Reads +tuple+ from the proxied TupleSpace. See TupleSpace#read. - - def read(tuple, sec=nil, &block) - @ts.read(tuple, sec, &block) - end - - ## - # Reads all tuples matching +tuple+ from the proxied TupleSpace. See - # TupleSpace#read_all. - - def read_all(tuple) - @ts.read_all(tuple) - end - - ## - # Registers for notifications of event +ev+ on the proxied TupleSpace. - # See TupleSpace#notify - - def notify(ev, tuple, sec=nil) - @ts.notify(ev, tuple, sec) - end - - end - - ## - # An SimpleRenewer allows a TupleSpace to check if a TupleEntry is still - # alive. - - class SimpleRenewer - - include DRbUndumped - - ## - # Creates a new SimpleRenewer that keeps an object alive for another +sec+ - # seconds. - - def initialize(sec=180) - @sec = sec - end - - ## - # Called by the TupleSpace to check if the object is still alive. - - def renew - @sec - end - end - -end - diff --git a/lib/rinda/ring.rb b/lib/rinda/ring.rb deleted file mode 100644 index 948cfaf208..0000000000 --- a/lib/rinda/ring.rb +++ /dev/null @@ -1,484 +0,0 @@ -# frozen_string_literal: false -# -# Note: Rinda::Ring API is unstable. -# -require 'drb/drb' -require_relative 'rinda' -require 'ipaddr' - -module Rinda - - ## - # The default port Ring discovery will use. - - Ring_PORT = 7647 - - ## - # A RingServer allows a Rinda::TupleSpace to be located via UDP broadcasts. - # Default service location uses the following steps: - # - # 1. A RingServer begins listening on the network broadcast UDP address. - # 2. A RingFinger sends a UDP packet containing the DRb URI where it will - # listen for a reply. - # 3. The RingServer receives the UDP packet and connects back to the - # provided DRb URI with the DRb service. - # - # A RingServer requires a TupleSpace: - # - # ts = Rinda::TupleSpace.new - # rs = Rinda::RingServer.new - # - # RingServer can also listen on multicast addresses for announcements. This - # allows multiple RingServers to run on the same host. To use network - # broadcast and multicast: - # - # ts = Rinda::TupleSpace.new - # rs = Rinda::RingServer.new ts, %w[Socket::INADDR_ANY, 239.0.0.1 ff02::1] - - class RingServer - - include DRbUndumped - - ## - # Special renewer for the RingServer to allow shutdown - - class Renewer # :nodoc: - include DRbUndumped - - ## - # Set to false to shutdown future requests using this Renewer - - attr_writer :renew - - def initialize # :nodoc: - @renew = true - end - - def renew # :nodoc: - @renew ? 1 : true - end - end - - ## - # Advertises +ts+ on the given +addresses+ at +port+. - # - # If +addresses+ is omitted only the UDP broadcast address is used. - # - # +addresses+ can contain multiple addresses. If a multicast address is - # given in +addresses+ then the RingServer will listen for multicast - # queries. - # - # If you use IPv4 multicast you may need to set an address of the inbound - # interface which joins a multicast group. - # - # ts = Rinda::TupleSpace.new - # rs = Rinda::RingServer.new(ts, [['239.0.0.1', '9.5.1.1']]) - # - # You can set addresses as an Array Object. The first element of the - # Array is a multicast address and the second is an inbound interface - # address. If the second is omitted then '0.0.0.0' is used. - # - # If you use IPv6 multicast you may need to set both the local interface - # address and the inbound interface index: - # - # rs = Rinda::RingServer.new(ts, [['ff02::1', '::1', 1]]) - # - # The first element is a multicast address and the second is an inbound - # interface address. The third is an inbound interface index. - # - # At this time there is no easy way to get an interface index by name. - # - # If the second is omitted then '::1' is used. - # If the third is omitted then 0 (default interface) is used. - - def initialize(ts, addresses=[Socket::INADDR_ANY], port=Ring_PORT) - @port = port - - if Integer === addresses then - addresses, @port = [Socket::INADDR_ANY], addresses - end - - @renewer = Renewer.new - - @ts = ts - @sockets = [] - addresses.each do |address| - if Array === address - make_socket(*address) - else - make_socket(address) - end - end - - @w_services = write_services - @r_service = reply_service - end - - ## - # Creates a socket at +address+ - # - # If +address+ is multicast address then +interface_address+ and - # +multicast_interface+ can be set as optional. - # - # A created socket is bound to +interface_address+. If you use IPv4 - # multicast then the interface of +interface_address+ is used as the - # inbound interface. If +interface_address+ is omitted or nil then - # '0.0.0.0' or '::1' is used. - # - # If you use IPv6 multicast then +multicast_interface+ is used as the - # inbound interface. +multicast_interface+ is a network interface index. - # If +multicast_interface+ is omitted then 0 (default interface) is used. - - def make_socket(address, interface_address=nil, multicast_interface=0) - addrinfo = Addrinfo.udp(address, @port) - - socket = Socket.new(addrinfo.pfamily, addrinfo.socktype, - addrinfo.protocol) - - if addrinfo.ipv4_multicast? or addrinfo.ipv6_multicast? then - if Socket.const_defined?(:SO_REUSEPORT) then - socket.setsockopt(:SOCKET, :SO_REUSEPORT, true) - else - socket.setsockopt(:SOCKET, :SO_REUSEADDR, true) - end - - if addrinfo.ipv4_multicast? then - interface_address = '0.0.0.0' if interface_address.nil? - socket.bind(Addrinfo.udp(interface_address, @port)) - - mreq = IPAddr.new(addrinfo.ip_address).hton + - IPAddr.new(interface_address).hton - - socket.setsockopt(:IPPROTO_IP, :IP_ADD_MEMBERSHIP, mreq) - else - interface_address = '::1' if interface_address.nil? - socket.bind(Addrinfo.udp(interface_address, @port)) - - mreq = IPAddr.new(addrinfo.ip_address).hton + - [multicast_interface].pack('I') - - socket.setsockopt(:IPPROTO_IPV6, :IPV6_JOIN_GROUP, mreq) - end - else - socket.bind(addrinfo) - end - - socket - rescue - socket = socket.close if socket - raise - ensure - @sockets << socket if socket - end - - ## - # Creates threads that pick up UDP packets and passes them to do_write for - # decoding. - - def write_services - @sockets.map do |s| - Thread.new(s) do |socket| - loop do - msg = socket.recv(1024) - do_write(msg) - end - end - end - end - - ## - # Extracts the response URI from +msg+ and adds it to TupleSpace where it - # will be picked up by +reply_service+ for notification. - - def do_write(msg) - Thread.new do - begin - tuple, sec = Marshal.load(msg) - @ts.write(tuple, sec) - rescue - end - end - end - - ## - # Creates a thread that notifies waiting clients from the TupleSpace. - - def reply_service - Thread.new do - loop do - do_reply - end - end - end - - ## - # Pulls lookup tuples out of the TupleSpace and sends their DRb object the - # address of the local TupleSpace. - - def do_reply - tuple = @ts.take([:lookup_ring, nil], @renewer) - Thread.new { tuple[1].call(@ts) rescue nil} - rescue - end - - ## - # Shuts down the RingServer - - def shutdown - @renewer.renew = false - - @w_services.each do |thread| - thread.kill - thread.join - end - - @sockets.each do |socket| - socket.close - end - - @r_service.kill - @r_service.join - end - - end - - ## - # RingFinger is used by RingServer clients to discover the RingServer's - # TupleSpace. Typically, all a client needs to do is call - # RingFinger.primary to retrieve the remote TupleSpace, which it can then - # begin using. - # - # To find the first available remote TupleSpace: - # - # Rinda::RingFinger.primary - # - # To create a RingFinger that broadcasts to a custom list: - # - # rf = Rinda::RingFinger.new ['localhost', '192.0.2.1'] - # rf.primary - # - # Rinda::RingFinger also understands multicast addresses and sets them up - # properly. This allows you to run multiple RingServers on the same host: - # - # rf = Rinda::RingFinger.new ['239.0.0.1'] - # rf.primary - # - # You can set the hop count (or TTL) for multicast searches using - # #multicast_hops. - # - # If you use IPv6 multicast you may need to set both an address and the - # outbound interface index: - # - # rf = Rinda::RingFinger.new ['ff02::1'] - # rf.multicast_interface = 1 - # rf.primary - # - # At this time there is no easy way to get an interface index by name. - - class RingFinger - - @@broadcast_list = ['<broadcast>', 'localhost'] - - @@finger = nil - - ## - # Creates a singleton RingFinger and looks for a RingServer. Returns the - # created RingFinger. - - def self.finger - unless @@finger - @@finger = self.new - @@finger.lookup_ring_any - end - @@finger - end - - ## - # Returns the first advertised TupleSpace. - - def self.primary - finger.primary - end - - ## - # Contains all discovered TupleSpaces except for the primary. - - def self.to_a - finger.to_a - end - - ## - # The list of addresses where RingFinger will send query packets. - - attr_accessor :broadcast_list - - ## - # Maximum number of hops for sent multicast packets (if using a multicast - # address in the broadcast list). The default is 1 (same as UDP - # broadcast). - - attr_accessor :multicast_hops - - ## - # The interface index to send IPv6 multicast packets from. - - attr_accessor :multicast_interface - - ## - # The port that RingFinger will send query packets to. - - attr_accessor :port - - ## - # Contain the first advertised TupleSpace after lookup_ring_any is called. - - attr_accessor :primary - - ## - # Creates a new RingFinger that will look for RingServers at +port+ on - # the addresses in +broadcast_list+. - # - # If +broadcast_list+ contains a multicast address then multicast queries - # will be made using the given multicast_hops and multicast_interface. - - def initialize(broadcast_list=@@broadcast_list, port=Ring_PORT) - @broadcast_list = broadcast_list || ['localhost'] - @port = port - @primary = nil - @rings = [] - - @multicast_hops = 1 - @multicast_interface = 0 - end - - ## - # Contains all discovered TupleSpaces except for the primary. - - def to_a - @rings - end - - ## - # Iterates over all discovered TupleSpaces starting with the primary. - - def each - lookup_ring_any unless @primary - return unless @primary - yield(@primary) - @rings.each { |x| yield(x) } - end - - ## - # Looks up RingServers waiting +timeout+ seconds. RingServers will be - # given +block+ as a callback, which will be called with the remote - # TupleSpace. - - def lookup_ring(timeout=5, &block) - return lookup_ring_any(timeout) unless block_given? - - msg = Marshal.dump([[:lookup_ring, DRbObject.new(block)], timeout]) - @broadcast_list.each do |it| - send_message(it, msg) - end - sleep(timeout) - end - - ## - # Returns the first found remote TupleSpace. Any further recovered - # TupleSpaces can be found by calling +to_a+. - - def lookup_ring_any(timeout=5) - queue = Thread::Queue.new - - Thread.new do - self.lookup_ring(timeout) do |ts| - queue.push(ts) - end - queue.push(nil) - end - - @primary = queue.pop - raise('RingNotFound') if @primary.nil? - - Thread.new do - while it = queue.pop - @rings.push(it) - end - end - - @primary - end - - ## - # Creates a socket for +address+ with the appropriate multicast options - # for multicast addresses. - - def make_socket(address) # :nodoc: - addrinfo = Addrinfo.udp(address, @port) - - soc = Socket.new(addrinfo.pfamily, addrinfo.socktype, addrinfo.protocol) - begin - if addrinfo.ipv4_multicast? then - soc.setsockopt(Socket::Option.ipv4_multicast_loop(1)) - soc.setsockopt(Socket::Option.ipv4_multicast_ttl(@multicast_hops)) - elsif addrinfo.ipv6_multicast? then - soc.setsockopt(:IPPROTO_IPV6, :IPV6_MULTICAST_LOOP, true) - soc.setsockopt(:IPPROTO_IPV6, :IPV6_MULTICAST_HOPS, - [@multicast_hops].pack('I')) - soc.setsockopt(:IPPROTO_IPV6, :IPV6_MULTICAST_IF, - [@multicast_interface].pack('I')) - else - soc.setsockopt(:SOL_SOCKET, :SO_BROADCAST, true) - end - - soc.connect(addrinfo) - rescue Exception - soc.close - raise - end - - soc - end - - def send_message(address, message) # :nodoc: - soc = make_socket(address) - - soc.send(message, 0) - rescue - nil - ensure - soc.close if soc - end - - end - - ## - # RingProvider uses a RingServer advertised TupleSpace as a name service. - # TupleSpace clients can register themselves with the remote TupleSpace and - # look up other provided services via the remote TupleSpace. - # - # Services are registered with a tuple of the format [:name, klass, - # DRbObject, description]. - - class RingProvider - - ## - # Creates a RingProvider that will provide a +klass+ service running on - # +front+, with a +description+. +renewer+ is optional. - - def initialize(klass, front, desc, renewer = nil) - @tuple = [:name, klass, front, desc] - @renewer = renewer || Rinda::SimpleRenewer.new - end - - ## - # Advertises this service on the primary remote TupleSpace. - - def provide - ts = Rinda::RingFinger.primary - ts.write(@tuple, @renewer) - end - - end - -end diff --git a/lib/rinda/tuplespace.rb b/lib/rinda/tuplespace.rb deleted file mode 100644 index 6a41a7ba75..0000000000 --- a/lib/rinda/tuplespace.rb +++ /dev/null @@ -1,641 +0,0 @@ -# frozen_string_literal: false -require 'monitor' -require 'drb/drb' -require_relative 'rinda' -require 'forwardable' - -module Rinda - - ## - # A TupleEntry is a Tuple (i.e. a possible entry in some Tuplespace) - # together with expiry and cancellation data. - - class TupleEntry - - include DRbUndumped - - attr_accessor :expires - - ## - # Creates a TupleEntry based on +ary+ with an optional renewer or expiry - # time +sec+. - # - # A renewer must implement the +renew+ method which returns a Numeric, - # nil, or true to indicate when the tuple has expired. - - def initialize(ary, sec=nil) - @cancel = false - @expires = nil - @tuple = make_tuple(ary) - @renewer = nil - renew(sec) - end - - ## - # Marks this TupleEntry as canceled. - - def cancel - @cancel = true - end - - ## - # A TupleEntry is dead when it is canceled or expired. - - def alive? - !canceled? && !expired? - end - - ## - # Return the object which makes up the tuple itself: the Array - # or Hash. - - def value; @tuple.value; end - - ## - # Returns the canceled status. - - def canceled?; @cancel; end - - ## - # Has this tuple expired? (true/false). - # - # A tuple has expired when its expiry timer based on the +sec+ argument to - # #initialize runs out. - - def expired? - return true unless @expires - return false if @expires > Time.now - return true if @renewer.nil? - renew(@renewer) - return true unless @expires - return @expires < Time.now - end - - ## - # Reset the expiry time according to +sec_or_renewer+. - # - # +nil+:: it is set to expire in the far future. - # +true+:: it has expired. - # Numeric:: it will expire in that many seconds. - # - # Otherwise the argument refers to some kind of renewer object - # which will reset its expiry time. - - def renew(sec_or_renewer) - sec, @renewer = get_renewer(sec_or_renewer) - @expires = make_expires(sec) - end - - ## - # Returns an expiry Time based on +sec+ which can be one of: - # Numeric:: +sec+ seconds into the future - # +true+:: the expiry time is the start of 1970 (i.e. expired) - # +nil+:: it is Tue Jan 19 03:14:07 GMT Standard Time 2038 (i.e. when - # UNIX clocks will die) - - def make_expires(sec=nil) - case sec - when Numeric - Time.now + sec - when true - Time.at(1) - when nil - Time.at(2**31-1) - end - end - - ## - # Retrieves +key+ from the tuple. - - def [](key) - @tuple[key] - end - - ## - # Fetches +key+ from the tuple. - - def fetch(key) - @tuple.fetch(key) - end - - ## - # The size of the tuple. - - def size - @tuple.size - end - - ## - # Creates a Rinda::Tuple for +ary+. - - def make_tuple(ary) - Rinda::Tuple.new(ary) - end - - private - - ## - # Returns a valid argument to make_expires and the renewer or nil. - # - # Given +true+, +nil+, or Numeric, returns that value and +nil+ (no actual - # renewer). Otherwise it returns an expiry value from calling +it.renew+ - # and the renewer. - - def get_renewer(it) - case it - when Numeric, true, nil - return it, nil - else - begin - return it.renew, it - rescue Exception - return it, nil - end - end - end - - end - - ## - # A TemplateEntry is a Template together with expiry and cancellation data. - - class TemplateEntry < TupleEntry - ## - # Matches this TemplateEntry against +tuple+. See Template#match for - # details on how a Template matches a Tuple. - - def match(tuple) - @tuple.match(tuple) - end - - alias === match - - def make_tuple(ary) # :nodoc: - Rinda::Template.new(ary) - end - - end - - ## - # <i>Documentation?</i> - - class WaitTemplateEntry < TemplateEntry - - attr_reader :found - - def initialize(place, ary, expires=nil) - super(ary, expires) - @place = place - @cond = place.new_cond - @found = nil - end - - def cancel - super - signal - end - - def wait - @cond.wait - end - - def read(tuple) - @found = tuple - signal - end - - def signal - @place.synchronize do - @cond.signal - end - end - - end - - ## - # A NotifyTemplateEntry is returned by TupleSpace#notify and is notified of - # TupleSpace changes. You may receive either your subscribed event or the - # 'close' event when iterating over notifications. - # - # See TupleSpace#notify_event for valid notification types. - # - # == Example - # - # ts = Rinda::TupleSpace.new - # observer = ts.notify 'write', [nil] - # - # Thread.start do - # observer.each { |t| p t } - # end - # - # 3.times { |i| ts.write [i] } - # - # Outputs: - # - # ['write', [0]] - # ['write', [1]] - # ['write', [2]] - - class NotifyTemplateEntry < TemplateEntry - - ## - # Creates a new NotifyTemplateEntry that watches +place+ for +event+s that - # match +tuple+. - - def initialize(place, event, tuple, expires=nil) - ary = [event, Rinda::Template.new(tuple)] - super(ary, expires) - @queue = Thread::Queue.new - @done = false - end - - ## - # Called by TupleSpace to notify this NotifyTemplateEntry of a new event. - - def notify(ev) - @queue.push(ev) - end - - ## - # Retrieves a notification. Raises RequestExpiredError when this - # NotifyTemplateEntry expires. - - def pop - raise RequestExpiredError if @done - it = @queue.pop - @done = true if it[0] == 'close' - return it - end - - ## - # Yields event/tuple pairs until this NotifyTemplateEntry expires. - - def each # :yields: event, tuple - while !@done - it = pop - yield(it) - end - rescue - ensure - cancel - end - - end - - ## - # TupleBag is an unordered collection of tuples. It is the basis - # of Tuplespace. - - class TupleBag - class TupleBin - extend Forwardable - def_delegators '@bin', :find_all, :delete_if, :each, :empty? - - def initialize - @bin = [] - end - - def add(tuple) - @bin.push(tuple) - end - - def delete(tuple) - idx = @bin.rindex(tuple) - @bin.delete_at(idx) if idx - end - - def find - @bin.reverse_each do |x| - return x if yield(x) - end - nil - end - end - - def initialize # :nodoc: - @hash = {} - @enum = enum_for(:each_entry) - end - - ## - # +true+ if the TupleBag to see if it has any expired entries. - - def has_expires? - @enum.find do |tuple| - tuple.expires - end - end - - ## - # Add +tuple+ to the TupleBag. - - def push(tuple) - key = bin_key(tuple) - @hash[key] ||= TupleBin.new - @hash[key].add(tuple) - end - - ## - # Removes +tuple+ from the TupleBag. - - def delete(tuple) - key = bin_key(tuple) - bin = @hash[key] - return nil unless bin - bin.delete(tuple) - @hash.delete(key) if bin.empty? - tuple - end - - ## - # Finds all live tuples that match +template+. - def find_all(template) - bin_for_find(template).find_all do |tuple| - tuple.alive? && template.match(tuple) - end - end - - ## - # Finds a live tuple that matches +template+. - - def find(template) - bin_for_find(template).find do |tuple| - tuple.alive? && template.match(tuple) - end - end - - ## - # Finds all tuples in the TupleBag which when treated as templates, match - # +tuple+ and are alive. - - def find_all_template(tuple) - @enum.find_all do |template| - template.alive? && template.match(tuple) - end - end - - ## - # Delete tuples which dead tuples from the TupleBag, returning the deleted - # tuples. - - def delete_unless_alive - deleted = [] - @hash.each do |key, bin| - bin.delete_if do |tuple| - if tuple.alive? - false - else - deleted.push(tuple) - true - end - end - end - deleted - end - - private - def each_entry(&blk) - @hash.each do |k, v| - v.each(&blk) - end - end - - def bin_key(tuple) - head = tuple[0] - if head.class == Symbol - return head - else - false - end - end - - def bin_for_find(template) - key = bin_key(template) - key ? @hash.fetch(key, []) : @enum - end - end - - ## - # The Tuplespace manages access to the tuples it contains, - # ensuring mutual exclusion requirements are met. - # - # The +sec+ option for the write, take, move, read and notify methods may - # either be a number of seconds or a Renewer object. - - class TupleSpace - - include DRbUndumped - include MonitorMixin - - ## - # Creates a new TupleSpace. +period+ is used to control how often to look - # for dead tuples after modifications to the TupleSpace. - # - # If no dead tuples are found +period+ seconds after the last - # modification, the TupleSpace will stop looking for dead tuples. - - def initialize(period=60) - super() - @bag = TupleBag.new - @read_waiter = TupleBag.new - @take_waiter = TupleBag.new - @notify_waiter = TupleBag.new - @period = period - @keeper = nil - end - - ## - # Adds +tuple+ - - def write(tuple, sec=nil) - entry = create_entry(tuple, sec) - synchronize do - if entry.expired? - @read_waiter.find_all_template(entry).each do |template| - template.read(tuple) - end - notify_event('write', entry.value) - notify_event('delete', entry.value) - else - @bag.push(entry) - start_keeper if entry.expires - @read_waiter.find_all_template(entry).each do |template| - template.read(tuple) - end - @take_waiter.find_all_template(entry).each do |template| - template.signal - end - notify_event('write', entry.value) - end - end - entry - end - - ## - # Removes +tuple+ - - def take(tuple, sec=nil, &block) - move(nil, tuple, sec, &block) - end - - ## - # Moves +tuple+ to +port+. - - def move(port, tuple, sec=nil) - template = WaitTemplateEntry.new(self, tuple, sec) - yield(template) if block_given? - synchronize do - entry = @bag.find(template) - if entry - port.push(entry.value) if port - @bag.delete(entry) - notify_event('take', entry.value) - return port ? nil : entry.value - end - raise RequestExpiredError if template.expired? - - begin - @take_waiter.push(template) - start_keeper if template.expires - while true - raise RequestCanceledError if template.canceled? - raise RequestExpiredError if template.expired? - entry = @bag.find(template) - if entry - port.push(entry.value) if port - @bag.delete(entry) - notify_event('take', entry.value) - return port ? nil : entry.value - end - template.wait - end - ensure - @take_waiter.delete(template) - end - end - end - - ## - # Reads +tuple+, but does not remove it. - - def read(tuple, sec=nil) - template = WaitTemplateEntry.new(self, tuple, sec) - yield(template) if block_given? - synchronize do - entry = @bag.find(template) - return entry.value if entry - raise RequestExpiredError if template.expired? - - begin - @read_waiter.push(template) - start_keeper if template.expires - template.wait - raise RequestCanceledError if template.canceled? - raise RequestExpiredError if template.expired? - return template.found - ensure - @read_waiter.delete(template) - end - end - end - - ## - # Returns all tuples matching +tuple+. Does not remove the found tuples. - - def read_all(tuple) - template = WaitTemplateEntry.new(self, tuple, nil) - synchronize do - entry = @bag.find_all(template) - entry.collect do |e| - e.value - end - end - end - - ## - # Registers for notifications of +event+. Returns a NotifyTemplateEntry. - # See NotifyTemplateEntry for examples of how to listen for notifications. - # - # +event+ can be: - # 'write':: A tuple was added - # 'take':: A tuple was taken or moved - # 'delete':: A tuple was lost after being overwritten or expiring - # - # The TupleSpace will also notify you of the 'close' event when the - # NotifyTemplateEntry has expired. - - def notify(event, tuple, sec=nil) - template = NotifyTemplateEntry.new(self, event, tuple, sec) - synchronize do - @notify_waiter.push(template) - end - template - end - - private - - def create_entry(tuple, sec) - TupleEntry.new(tuple, sec) - end - - ## - # Removes dead tuples. - - def keep_clean - synchronize do - @read_waiter.delete_unless_alive.each do |e| - e.signal - end - @take_waiter.delete_unless_alive.each do |e| - e.signal - end - @notify_waiter.delete_unless_alive.each do |e| - e.notify(['close']) - end - @bag.delete_unless_alive.each do |e| - notify_event('delete', e.value) - end - end - end - - ## - # Notifies all registered listeners for +event+ of a status change of - # +tuple+. - - def notify_event(event, tuple) - ev = [event, tuple] - @notify_waiter.find_all_template(ev).each do |template| - template.notify(ev) - end - end - - ## - # Creates a thread that scans the tuplespace for expired tuples. - - def start_keeper - return if @keeper && @keeper.alive? - @keeper = Thread.new do - while true - sleep(@period) - synchronize do - break unless need_keeper? - keep_clean - end - end - end - end - - ## - # Checks the tuplespace to see if it needs cleaning. - - def need_keeper? - return true if @bag.has_expires? - return true if @read_waiter.has_expires? - return true if @take_waiter.has_expires? - return true if @notify_waiter.has_expires? - end - - end - -end - diff --git a/lib/ruby_vm/rjit/.document b/lib/ruby_vm/rjit/.document new file mode 100644 index 0000000000..0a603afe3d --- /dev/null +++ b/lib/ruby_vm/rjit/.document @@ -0,0 +1 @@ +stats.rb diff --git a/lib/ruby_vm/rjit/assembler.rb b/lib/ruby_vm/rjit/assembler.rb index 64f663ec0e..645072d11b 100644 --- a/lib/ruby_vm/rjit/assembler.rb +++ b/lib/ruby_vm/rjit/assembler.rb @@ -328,6 +328,17 @@ module RubyVM::RJIT disp: left_disp, imm: imm8(right_imm), ) + # CMP r/m64, imm32 (Mod 01: [reg]+disp8) + in [QwordPtr[R64 => left_reg, IMM8 => left_disp], IMM32 => right_imm] + # REX.W + 81 /7 id + # MI: Operand 1: ModRM:r/m (r), Operand 2: imm8/16/32 + insn( + prefix: REX_W, + opcode: 0x81, + mod_rm: ModRM[mod: Mod01, reg: 7, rm: left_reg], + disp: left_disp, + imm: imm32(right_imm), + ) # CMP r/m64, imm8 (Mod 10: [reg]+disp32) in [QwordPtr[R64 => left_reg, IMM32 => left_disp], IMM8 => right_imm] # REX.W + 83 /7 ib diff --git a/lib/ruby_vm/rjit/c_pointer.rb b/lib/ruby_vm/rjit/c_pointer.rb index d65d5a93a5..db00c4cd11 100644 --- a/lib/ruby_vm/rjit/c_pointer.rb +++ b/lib/ruby_vm/rjit/c_pointer.rb @@ -238,6 +238,40 @@ module RubyVM::RJIT end end + # Basically Immediate but without #* to skip auto-dereference of structs. + class Array + attr_reader :type + + # @param addr [Integer] + # @param type [Class] RubyVM::RJIT::CType::* + def initialize(addr, type) + @addr = addr + @type = type + end + + # Array access + def [](index) + @type.new(@addr)[index] + end + + # Array set + # @param index [Integer] + # @param value [Integer, RubyVM::RJIT::CPointer::Struct] an address itself or an object that return an address with to_i + def []=(index, value) + @type.new(@addr)[index] = value + end + + private + + def self.define(block) + Class.new(self) do + define_method(:initialize) do |addr| + super(addr, block.call) + end + end + end + end + class Pointer attr_reader :type @@ -338,7 +372,7 @@ module RubyVM::RJIT # Give a name to a dynamic CPointer class to see it on inspect def self.with_class_name(prefix, name, cache: false, &block) - return block.call if name.empty? + return block.call if !name.nil? && name.empty? # Use a cached result only if cache: true class_name = "#{prefix}_#{name}" diff --git a/lib/ruby_vm/rjit/c_type.rb b/lib/ruby_vm/rjit/c_type.rb index bec7e21c38..3b313a658b 100644 --- a/lib/ruby_vm/rjit/c_type.rb +++ b/lib/ruby_vm/rjit/c_type.rb @@ -62,6 +62,14 @@ module RubyVM::RJIT end end + class Array + def self.new(&block) + CPointer.with_class_name('Array', block.object_id.to_s) do + CPointer::Array.define(block) + end + end + end + class Pointer # This takes a block to avoid "stack level too deep" on a cyclic reference # @param block [Proc] diff --git a/lib/ruby_vm/rjit/code_block.rb b/lib/ruby_vm/rjit/code_block.rb index 6260ec8b4b..260bd98671 100644 --- a/lib/ruby_vm/rjit/code_block.rb +++ b/lib/ruby_vm/rjit/code_block.rb @@ -4,7 +4,7 @@ module RubyVM::RJIT # @param mem_size [Integer] JIT buffer size # @param outliend [TrueClass,FalseClass] true for outlined CodeBlock def initialize(mem_block:, mem_size:, outlined: false) - @comments = Hash.new { |h, k| h[k] = [] } + @comments = Hash.new { |h, k| h[k] = [] } if dump_disasm? @mem_block = mem_block @mem_size = mem_size @write_pos = 0 @@ -26,7 +26,7 @@ module RubyVM::RJIT # Convert comment indexes to addresses asm.comments.each do |index, comments| - @comments[start_addr + index] += comments + @comments[start_addr + index] += comments if dump_disasm? end asm.comments.clear @@ -39,7 +39,7 @@ module RubyVM::RJIT def set_write_addr(addr) @write_pos = addr - @mem_block - @comments.delete(addr) # TODO: clean up old comments for all the overwritten range? + @comments.delete(addr) if dump_disasm? end def with_write_addr(addr) @@ -83,5 +83,9 @@ module RubyVM::RJIT def bold(text) "\e[1m#{text}\e[0m" end + + def dump_disasm? + C.rjit_opts.dump_disasm + end end end diff --git a/lib/ruby_vm/rjit/compiler.rb b/lib/ruby_vm/rjit/compiler.rb index 1c024c4c40..e5c3adf0ec 100644 --- a/lib/ruby_vm/rjit/compiler.rb +++ b/lib/ruby_vm/rjit/compiler.rb @@ -59,14 +59,16 @@ module RubyVM::RJIT # @param iseq `RubyVM::RJIT::CPointer::Struct_rb_iseq_t` # @param cfp `RubyVM::RJIT::CPointer::Struct_rb_control_frame_t` def compile(iseq, cfp) + return unless supported_platform? pc = cfp.pc.to_i jit = JITState.new(iseq:, cfp:) asm = Assembler.new compile_prologue(asm, iseq, pc) compile_block(asm, jit:, pc:) - iseq.body.jit_func = @cb.write(asm) + iseq.body.jit_entry = @cb.write(asm) rescue Exception => e - $stderr.puts e.full_message + STDERR.puts "#{e.class}: #{e.message}" + STDERR.puts e.backtrace exit 1 end @@ -108,7 +110,7 @@ module RubyVM::RJIT return block.start_addr rescue Exception => e - $stderr.puts e.full_message + STDERR.puts e.full_message exit 1 end @@ -163,7 +165,7 @@ module RubyVM::RJIT return target.address rescue Exception => e - $stderr.puts e.full_message + STDERR.puts e.full_message exit 1 end @@ -176,8 +178,8 @@ module RubyVM::RJIT # If they were the ISEQ's first blocks, re-compile RJIT entry as well if iseq.body.iseq_encoded.to_i == pc - iseq.body.jit_func = 0 - iseq.body.total_calls = 0 + iseq.body.jit_entry = 0 + iseq.body.jit_entry_calls = 0 end end @@ -505,5 +507,12 @@ module RubyVM::RJIT raise "'#{cond.inspect}' was not true" end end + + def supported_platform? + return @supported_platform if defined?(@supported_platform) + @supported_platform = RUBY_PLATFORM.match?(/x86_64/).tap do |supported| + warn "warning: RJIT does not support #{RUBY_PLATFORM} yet" unless supported + end + end end end diff --git a/lib/ruby_vm/rjit/context.rb b/lib/ruby_vm/rjit/context.rb index 66da478c7f..a2a7ecc6dc 100644 --- a/lib/ruby_vm/rjit/context.rb +++ b/lib/ruby_vm/rjit/context.rb @@ -264,9 +264,9 @@ module RubyVM::RJIT # noop in MapToSelf # noop - in MapToLocal[local_idx] - if stack_idx == local_idx - self.temp_types[stack_idx] = self.local_types[local_idx]; + in MapToLocal[idx] + if idx == local_idx + self.temp_types[stack_idx] = self.local_types[idx] self.temp_mapping[stack_idx] = MapToStack else # noop diff --git a/lib/ruby_vm/rjit/insn_compiler.rb b/lib/ruby_vm/rjit/insn_compiler.rb index 619f5078dc..f9450241c9 100644 --- a/lib/ruby_vm/rjit/insn_compiler.rb +++ b/lib/ruby_vm/rjit/insn_compiler.rb @@ -1,3 +1,4 @@ +# frozen_string_literal: true module RubyVM::RJIT class InsnCompiler # struct rb_calling_info. Storing flags instead of ci. @@ -56,6 +57,7 @@ module RubyVM::RJIT when :putobject then putobject(jit, ctx, asm) when :putspecialobject then putspecialobject(jit, ctx, asm) when :putstring then putstring(jit, ctx, asm) + when :putchilledstring then putchilledstring(jit, ctx, asm) when :concatstrings then concatstrings(jit, ctx, asm) when :anytostring then anytostring(jit, ctx, asm) when :toregexp then toregexp(jit, ctx, asm) @@ -502,30 +504,7 @@ module RubyVM::RJIT shape = C.rb_shape_get_shape_by_id(shape_id) current_capacity = shape.capacity - new_capacity = current_capacity * 2 - - # If the object doesn't have the capacity to store the IV, - # then we'll need to allocate it. - needs_extension = shape.next_iv_index >= current_capacity - - # We can write to the object, but we need to transition the shape - ivar_index = shape.next_iv_index - - capa_shape = - if needs_extension - # We need to add an extended table to the object - # First, create an outgoing transition that increases the capacity - C.rb_shape_transition_shape_capa(shape, new_capacity) - else - nil - end - - dest_shape = - if capa_shape - C.rb_shape_get_next(capa_shape, comptime_receiver, ivar_name) - else - C.rb_shape_get_next(shape, comptime_receiver, ivar_name) - end + dest_shape = C.rb_shape_get_next(shape, comptime_receiver, ivar_name) new_shape_id = C.rb_shape_id(dest_shape) if new_shape_id == C::OBJ_TOO_COMPLEX_SHAPE_ID @@ -533,12 +512,18 @@ module RubyVM::RJIT return CantCompile end + ivar_index = shape.next_iv_index + + # If the new shape has a different capacity, we need to + # reallocate the object. + needs_extension = dest_shape.capacity != shape.capacity + if needs_extension # Generate the C call so that runtime code will increase # the capacity and set the buffer. asm.mov(C_ARGS[0], :rax) asm.mov(C_ARGS[1], current_capacity) - asm.mov(C_ARGS[2], new_capacity) + asm.mov(C_ARGS[2], dest_shape.capacity) asm.call(C.rb_ensure_iv_list_size) # Load the receiver again after the function call @@ -792,9 +777,30 @@ module RubyVM::RJIT asm.mov(C_ARGS[0], EC) asm.mov(C_ARGS[1], to_value(put_val)) + asm.mov(C_ARGS[2], 0) + asm.call(C.rb_ec_str_resurrect) + + stack_top = ctx.stack_push(Type::TString) + asm.mov(stack_top, C_RET) + + KeepCompiling + end + + # @param jit [RubyVM::RJIT::JITState] + # @param ctx [RubyVM::RJIT::Context] + # @param asm [RubyVM::RJIT::Assembler] + def putchilledstring(jit, ctx, asm) + put_val = jit.operand(0, ruby: true) + + # Save the PC and SP because the callee will allocate + jit_prepare_routine_call(jit, ctx, asm) + + asm.mov(C_ARGS[0], EC) + asm.mov(C_ARGS[1], to_value(put_val)) + asm.mov(C_ARGS[2], 1) asm.call(C.rb_ec_str_resurrect) - stack_top = ctx.stack_push(Type::CString) + stack_top = ctx.stack_push(Type::TString) asm.mov(stack_top, C_RET) KeepCompiling @@ -817,7 +823,7 @@ module RubyVM::RJIT asm.call(C.rb_str_concat_literals) ctx.stack_pop(n) - stack_ret = ctx.stack_push(Type::CString) + stack_ret = ctx.stack_push(Type::TString) asm.mov(stack_ret, C_RET) KeepCompiling @@ -932,7 +938,7 @@ module RubyVM::RJIT asm.call(C.rb_ec_ary_new_from_values) ctx.stack_pop(n) - stack_ret = ctx.stack_push(Type::CArray) + stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) KeepCompiling @@ -954,7 +960,7 @@ module RubyVM::RJIT asm.mov(C_ARGS[0], ary) asm.call(C.rb_ary_resurrect) - stack_ret = ctx.stack_push(Type::CArray) + stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) KeepCompiling @@ -1940,26 +1946,30 @@ module RubyVM::RJIT end # Jump to target0 on jnz - branch_stub.compile = proc do |branch_asm| - branch_asm.comment("branchif #{branch_stub.shape}") - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.jnz(branch_stub.target0.address) - branch_asm.jmp(branch_stub.target1.address) - in Next0 - branch_asm.jz(branch_stub.target1.address) - in Next1 - branch_asm.jnz(branch_stub.target0.address) - end - end - end + branch_stub.compile = compile_branchif(branch_stub) branch_stub.compile.call(asm) end EndBlock end + def compile_branchif(branch_stub) # Proc escapes arguments in memory + proc do |branch_asm| + branch_asm.comment("branchif #{branch_stub.shape}") + branch_asm.stub(branch_stub) do + case branch_stub.shape + in Default + branch_asm.jnz(branch_stub.target0.address) + branch_asm.jmp(branch_stub.target1.address) + in Next0 + branch_asm.jz(branch_stub.target1.address) + in Next1 + branch_asm.jnz(branch_stub.target0.address) + end + end + end + end + # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] @@ -2001,26 +2011,30 @@ module RubyVM::RJIT end # Jump to target0 on jz - branch_stub.compile = proc do |branch_asm| - branch_asm.comment("branchunless #{branch_stub.shape}") - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.jz(branch_stub.target0.address) - branch_asm.jmp(branch_stub.target1.address) - in Next0 - branch_asm.jnz(branch_stub.target1.address) - in Next1 - branch_asm.jz(branch_stub.target0.address) - end - end - end + branch_stub.compile = compile_branchunless(branch_stub) branch_stub.compile.call(asm) end EndBlock end + def compile_branchunless(branch_stub) # Proc escapes arguments in memory + proc do |branch_asm| + branch_asm.comment("branchunless #{branch_stub.shape}") + branch_asm.stub(branch_stub) do + case branch_stub.shape + in Default + branch_asm.jz(branch_stub.target0.address) + branch_asm.jmp(branch_stub.target1.address) + in Next0 + branch_asm.jnz(branch_stub.target1.address) + in Next1 + branch_asm.jz(branch_stub.target0.address) + end + end + end + end + # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] @@ -2061,26 +2075,30 @@ module RubyVM::RJIT end # Jump to target0 on je - branch_stub.compile = proc do |branch_asm| - branch_asm.comment("branchnil #{branch_stub.shape}") - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.je(branch_stub.target0.address) - branch_asm.jmp(branch_stub.target1.address) - in Next0 - branch_asm.jne(branch_stub.target1.address) - in Next1 - branch_asm.je(branch_stub.target0.address) - end - end - end + branch_stub.compile = compile_branchnil(branch_stub) branch_stub.compile.call(asm) end EndBlock end + def compile_branchnil(branch_stub) # Proc escapes arguments in memory + proc do |branch_asm| + branch_asm.comment("branchnil #{branch_stub.shape}") + branch_asm.stub(branch_stub) do + case branch_stub.shape + in Default + branch_asm.je(branch_stub.target0.address) + branch_asm.jmp(branch_stub.target1.address) + in Next0 + branch_asm.jne(branch_stub.target1.address) + in Next1 + branch_asm.je(branch_stub.target0.address) + end + end + end + end + # once # @param jit [RubyVM::RJIT::JITState] @@ -2116,7 +2134,7 @@ module RubyVM::RJIT end # Check if the key is the same value - asm.cmp(key_opnd, comptime_key) + asm.cmp(key_opnd, to_value(comptime_key)) side_exit = side_exit(jit, starting_context) jit_chain_guard(:jne, jit, starting_context, asm, side_exit) @@ -2732,7 +2750,7 @@ module RubyVM::RJIT sample_rhs = jit.peek_at_stack(0) sample_lhs = jit.peek_at_stack(1) - # We are not allowing module here because the module hierachy can change at runtime. + # We are not allowing module here because the module hierarchy can change at runtime. if C.RB_TYPE_P(sample_rhs, C::RUBY_T_CLASS) return false end @@ -2994,15 +3012,12 @@ module RubyVM::RJIT # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] def jit_rb_str_empty_p(jit, ctx, asm, argc, known_recv_class) - # Assume same offset to len embedded or not so we can use one code path to read the length - #assert_equal(C.RString.offsetof(:as, :heap, :len), C.RString.offsetof(:as, :embed, :len)) - recv_opnd = ctx.stack_pop(1) out_opnd = ctx.stack_push(Type::UnknownImm) asm.comment('get string length') asm.mov(:rax, recv_opnd) - str_len_opnd = [:rax, C.RString.offsetof(:as, :heap, :len)] + str_len_opnd = [:rax, C.RString.offsetof(:len)] asm.cmp(str_len_opnd, 0) asm.mov(:rax, Qfalse) @@ -3085,7 +3100,7 @@ module RubyVM::RJIT asm.test(recv_reg, C::RUBY_ENCODING_MASK) # Push once, use the resulting operand in both branches below. - stack_ret = ctx.stack_push(Type::CString) + stack_ret = ctx.stack_push(Type::TString) enc_mismatch = asm.new_label('enc_mismatch') asm.jnz(enc_mismatch) @@ -3644,21 +3659,25 @@ module RubyVM::RJIT @exit_compiler.compile_branch_stub(deeper, ocb_asm, branch_stub, true) @ocb.write(ocb_asm) end - branch_stub.compile = proc do |branch_asm| - # Not using `asm.comment` here since it's usually put before cmp/test before this. - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.public_send(opcode, branch_stub.target0.address) - end - end - end + branch_stub.compile = compile_jit_chain_guard(branch_stub, opcode:) branch_stub.compile.call(asm) else asm.public_send(opcode, side_exit) end end + def compile_jit_chain_guard(branch_stub, opcode:) # Proc escapes arguments in memory + proc do |branch_asm| + # Not using `asm.comment` here since it's usually put before cmp/test before this. + branch_asm.stub(branch_stub) do + case branch_stub.shape + in Default + branch_asm.public_send(opcode, branch_stub.target0.address) + end + end + end + end + # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] # @param asm [RubyVM::RJIT::Assembler] @@ -3739,7 +3758,7 @@ module RubyVM::RJIT ctx.upgrade_opnd_type(insn_opnd, Type::Flonum) end - elsif C.FL_TEST(known_klass, C::RUBY_FL_SINGLETON) && comptime_obj == C.rb_class_attached_object(known_klass) + elsif C.RCLASS_SINGLETON_P(known_klass) && comptime_obj == C.rb_class_attached_object(known_klass) # Singleton classes are attached to one specific object, so we can # avoid one memory access (and potentially the is_heap check) by # looking for the expected object directly. @@ -3782,9 +3801,14 @@ module RubyVM::RJIT jit_chain_guard(:jne, jit, ctx, asm, side_exit, limit:) if known_klass == C.rb_cString - ctx.upgrade_opnd_type(insn_opnd, Type::CString) + # Upgrading to Type::CString here is incorrect. + # The guard we put only checks RBASIC_CLASS(obj), + # which adding a singleton class can change. We + # additionally need to know the string is frozen + # to claim Type::CString. + ctx.upgrade_opnd_type(insn_opnd, Type::TString) elsif known_klass == C.rb_cArray - ctx.upgrade_opnd_type(insn_opnd, Type::CArray) + ctx.upgrade_opnd_type(insn_opnd, Type::TArray) end end end @@ -4450,6 +4474,11 @@ module RubyVM::RJIT return CantCompile end + if flags & C::VM_CALL_KW_SPLAT != 0 + asm.incr_counter(:send_iseq_kw_splat) + return CantCompile + end + if iseq_has_rest && opt_num != 0 asm.incr_counter(:send_iseq_has_rest_and_optional) return CantCompile @@ -4556,7 +4585,8 @@ module RubyVM::RJIT # Check if we need the arg0 splat handling of vm_callee_setup_block_arg arg_setup_block = (calling.block_handler == :captured) # arg_setup_type: arg_setup_block (invokeblock) block_arg0_splat = arg_setup_block && argc == 1 && - iseq.body.param.flags.has_lead && !iseq.body.param.flags.ambiguous_param0 + (iseq.body.param.flags.has_lead || opt_num > 1) && + !iseq.body.param.flags.ambiguous_param0 if block_arg0_splat # If block_arg0_splat, we still need side exits after splat, but # doing push_splat_args here disallows it. So bail out. @@ -4570,6 +4600,13 @@ module RubyVM::RJIT asm.incr_counter(:invokeblock_iseq_arg0_has_kw) return CantCompile end + # The block_arg0_splat implementation cannot deal with optional parameters. + # This is a setup_parameters_complex() situation and interacts with the + # starting position of the callee. + if opt_num > 1 + asm.incr_counter(:invokeblock_iseq_arg0_optional) + return CantCompile + end end if flags & C::VM_CALL_ARGS_SPLAT != 0 && !iseq_has_rest array = jit.peek_at_stack(block_arg ? 1 : 0) @@ -4718,7 +4755,7 @@ module RubyVM::RJIT asm.call(C.rb_ec_ary_new_from_values) ctx.stack_pop(n) - stack_ret = ctx.stack_push(Type::CArray) + stack_ret = ctx.stack_push(Type::TArray) asm.mov(stack_ret, C_RET) end end @@ -4914,13 +4951,10 @@ module RubyVM::RJIT asm.comment('inlined leaf builtin') - # Skip this if it doesn't trigger GC - if iseq.body.builtin_attrs & C::BUILTIN_ATTR_NO_GC == 0 - # The callee may allocate, e.g. Integer#abs on a Bignum. - # Save SP for GC, save PC for allocation tracing, and prepare - # for global invalidation after GC's VM lock contention. - jit_prepare_routine_call(jit, ctx, asm) - end + # The callee may allocate, e.g. Integer#abs on a Bignum. + # Save SP for GC, save PC for allocation tracing, and prepare + # for global invalidation after GC's VM lock contention. + jit_prepare_routine_call(jit, ctx, asm) # Call the builtin func (ec, recv, arg1, arg2, ...) asm.mov(C_ARGS[0], EC) @@ -5427,6 +5461,12 @@ module RubyVM::RJIT return CantCompile end + if c_method_tracing_currently_enabled? + # Don't JIT if tracing c_call or c_return + asm.incr_counter(:send_cfunc_tracing) + return CantCompile + end + off = cme.def.body.optimized.index recv_idx = argc # blockarg is not supported @@ -5613,7 +5653,6 @@ module RubyVM::RJIT sp_reg = iseq ? SP : :rax asm.lea(sp_reg, [SP, C.VALUE.size * sp_offset]) asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:sp)], sp_reg) - asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:__bp__)], sp_reg) # TODO: get rid of this!! # cfp->jit_return is used only for ISEQs if iseq @@ -5635,16 +5674,7 @@ module RubyVM::RJIT @exit_compiler.compile_branch_stub(return_ctx, ocb_asm, branch_stub, true) @ocb.write(ocb_asm) end - branch_stub.compile = proc do |branch_asm| - branch_asm.comment('set jit_return to callee CFP') - branch_asm.stub(branch_stub) do - case branch_stub.shape - in Default - branch_asm.mov(:rax, branch_stub.target0.address) - branch_asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:jit_return)], :rax) - end - end - end + branch_stub.compile = compile_jit_return(branch_stub, cfp_offset:) branch_stub.compile.call(asm) end @@ -5655,6 +5685,19 @@ module RubyVM::RJIT asm.mov([EC, C.rb_execution_context_t.offsetof(:cfp)], cfp_reg) end + def compile_jit_return(branch_stub, cfp_offset:) # Proc escapes arguments in memory + proc do |branch_asm| + branch_asm.comment('set jit_return to callee CFP') + branch_asm.stub(branch_stub) do + case branch_stub.shape + in Default + branch_asm.mov(:rax, branch_stub.target0.address) + branch_asm.mov([CFP, cfp_offset + C.rb_control_frame_t.offsetof(:jit_return)], :rax) + end + end + end + end + # CALLER_SETUP_ARG: Return CantCompile if not supported # @param jit [RubyVM::RJIT::JITState] # @param ctx [RubyVM::RJIT::Context] @@ -5805,7 +5848,7 @@ module RubyVM::RJIT asm.cmovz(len_reg, [array_reg, C.RArray.offsetof(:as, :heap, :len)]) end - # Generate RARRAY_CONST_PTR_TRANSIENT (part of RARRAY_AREF) + # Generate RARRAY_CONST_PTR (part of RARRAY_AREF) def jit_array_ptr(asm, array_reg, ary_opnd) # clobbers array_reg asm.comment('get array pointer for embedded or heap') @@ -5875,7 +5918,12 @@ module RubyVM::RJIT @exit_compiler.compile_branch_stub(ctx, ocb_asm, branch_stub, true) @ocb.write(ocb_asm) end - branch_stub.compile = proc do |branch_asm| + branch_stub.compile = compile_jit_direct_jump(branch_stub, comment:) + branch_stub.compile.call(asm) + end + + def compile_jit_direct_jump(branch_stub, comment:) # Proc escapes arguments in memory + proc do |branch_asm| branch_asm.comment(comment) branch_asm.stub(branch_stub) do case branch_stub.shape @@ -5886,7 +5934,6 @@ module RubyVM::RJIT end end end - branch_stub.compile.call(asm) end # @param jit [RubyVM::RJIT::JITState] diff --git a/lib/ruby_vm/rjit/invariants.rb b/lib/ruby_vm/rjit/invariants.rb index db1068b5c2..5b061d1994 100644 --- a/lib/ruby_vm/rjit/invariants.rb +++ b/lib/ruby_vm/rjit/invariants.rb @@ -143,11 +143,11 @@ module RubyVM::RJIT C.rjit_for_each_iseq do |iseq| # Avoid entering past code - iseq.body.jit_func = 0 + iseq.body.jit_entry = 0 # Avoid reusing past code iseq.body.rjit_blocks.clear if iseq.body.rjit_blocks # Compile this again if not converted to trace_* insns - iseq.body.total_calls = 0 + iseq.body.jit_entry_calls = 0 end end end diff --git a/lib/ruby_vm/rjit/stats.rb b/lib/ruby_vm/rjit/stats.rb index 8c4253880a..7e353c698e 100644 --- a/lib/ruby_vm/rjit/stats.rb +++ b/lib/ruby_vm/rjit/stats.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true module RubyVM::RJIT + # Return a Hash for \RJIT statistics. \--rjit-stats makes more information available. def self.runtime_stats stats = {} @@ -15,6 +16,7 @@ module RubyVM::RJIT C.rb_rjit_runtime_counters.members.each do |member| stats[member] = C.rb_rjit_counters.public_send(member) end + stats[:vm_insns_count] = C.rb_vm_insns_count # Other stats are calculated here stats[:side_exit_count] = stats.select { |name, _count| name.start_with?('exit_') }.sum(&:last) @@ -29,6 +31,7 @@ module RubyVM::RJIT stats end + # :nodoc: all class << self private diff --git a/lib/ruby_vm/rjit/type.rb b/lib/ruby_vm/rjit/type.rb index 155e141d63..119692014b 100644 --- a/lib/ruby_vm/rjit/type.rb +++ b/lib/ruby_vm/rjit/type.rb @@ -35,7 +35,6 @@ module RubyVM::RJIT case self in Type::UnknownHeap then true in Type::TArray then true - in Type::CArray then true in Type::Hash then true in Type::HeapSymbol then true in Type::TString then true @@ -45,11 +44,10 @@ module RubyVM::RJIT end end - # Check if it's a T_ARRAY object (both TArray and CArray are T_ARRAY) + # Check if it's a T_ARRAY object def array? case self in Type::TArray then true - in Type::CArray then true else false end end @@ -73,7 +71,6 @@ module RubyVM::RJIT in Type::Flonum then C.rb_cFloat in Type::ImmSymbol | Type::HeapSymbol then C.rb_cSymbol in Type::CString then C.rb_cString - in Type::CArray then C.rb_cArray else nil end end @@ -115,11 +112,6 @@ module RubyVM::RJIT return TypeDiff::Compatible[1] end - # A CArray is also a TArray. - if self == Type::CArray && dst == Type::TArray - return TypeDiff::Compatible[1] - end - # Specific heap type into unknown heap type is imperfect but valid if self.heap? && dst == Type::UnknownHeap return TypeDiff::Compatible[1] @@ -169,12 +161,9 @@ module RubyVM::RJIT end else val_class = C.to_value(C.rb_class_of(val)) - if val_class == C.rb_cString + if val_class == C.rb_cString && C.rb_obj_frozen_p(val) return Type::CString end - if val_class == C.rb_cArray - return Type::CArray - end if C.to_value(val) == C.rb_block_param_proxy return Type::BlockParamProxy end @@ -222,7 +211,6 @@ module RubyVM::RJIT Type::TString = Type[:TString] # An object with the T_STRING flag set, possibly an rb_cString Type::CString = Type[:CString] # An un-subclassed string of type rb_cString (can have instance vars in some cases) Type::TArray = Type[:TArray] # An object with the T_ARRAY flag set, possibly an rb_cArray - Type::CArray = Type[:CArray] # An un-subclassed string of type rb_cArray (can have instance vars in some cases) Type::BlockParamProxy = Type[:BlockParamProxy] # A special sentinel value indicating the block parameter should be read from diff --git a/lib/rubygems.rb b/lib/rubygems.rb index cb71657018..ac225ca70a 100644 --- a/lib/rubygems.rb +++ b/lib/rubygems.rb @@ -9,7 +9,7 @@ require "rbconfig" module Gem - VERSION = "3.5.0.dev" + VERSION = "3.6.0.dev" end # Must be first since it unloads the prelude from 1.9.2 @@ -115,11 +115,6 @@ require_relative "rubygems/errors" module Gem RUBYGEMS_DIR = __dir__ - # Taint support is deprecated in Ruby 2.7. - # This allows switching ".untaint" to ".tap(&Gem::UNTAINT)", - # to avoid deprecation warnings in Ruby 2.7. - UNTAINT = RUBY_VERSION < "2.7" ? :untaint.to_sym : proc {} - ## # An Array of Regexps that match windows Ruby platforms. @@ -496,7 +491,7 @@ An Array (#{env.inspect}) was passed in from #{caller[3]} glob_with_suffixes = "#{glob}#{Gem.suffix_pattern}" $LOAD_PATH.map do |load_path| Gem::Util.glob_files_in_dir(glob_with_suffixes, load_path) - end.flatten.select {|file| File.file? file.tap(&Gem::UNTAINT) } + end.flatten.select {|file| File.file? file } end ## @@ -572,7 +567,7 @@ An Array (#{env.inspect}) was passed in from #{caller[3]} ## # The number of paths in the +$LOAD_PATH+ from activated gems. Used to - # prioritize +-I+ and +ENV['RUBYLIB']+ entries during +require+. + # prioritize +-I+ and <code>ENV['RUBYLIB']</code> entries during +require+. def self.activated_gem_paths @activated_gem_paths ||= 0 @@ -604,6 +599,16 @@ An Array (#{env.inspect}) was passed in from #{caller[3]} @yaml_loaded = true end + @safe_marshal_loaded = false + + def self.load_safe_marshal + return if @safe_marshal_loaded + + require_relative "rubygems/safe_marshal" + + @safe_marshal_loaded = true + end + ## # The file name and line number of the caller of the caller of this method. # @@ -938,6 +943,13 @@ An Array (#{env.inspect}) was passed in from #{caller[3]} end ## + # Suffixes for dynamic library require-able paths. + + def self.dynamic_library_suffixes + @dynamic_library_suffixes ||= suffixes - [".rb"] + end + + ## # Prints the amount of time the supplied block takes to run using the debug # UI output. @@ -1001,6 +1013,13 @@ An Array (#{env.inspect}) was passed in from #{caller[3]} end ## + # Is this platform FreeBSD + + def self.freebsd_platform? + RbConfig::CONFIG["host_os"].to_s.include?("bsd") + end + + ## # Load +plugins+ as Ruby files def self.load_plugin_files(plugins) # :nodoc: @@ -1074,8 +1093,6 @@ An Array (#{env.inspect}) was passed in from #{caller[3]} end end - path.tap(&Gem::UNTAINT) - unless File.file? path return unless raise_exception @@ -1206,9 +1223,16 @@ An Array (#{env.inspect}) was passed in from #{caller[3]} ## # Find a Gem::Specification of default gem from +path+ + def find_default_spec(path) + @path_to_default_spec_map[path] + end + + ## + # Find an unresolved Gem::Specification of default gem from +path+ + def find_unresolved_default_spec(path) default_spec = @path_to_default_spec_map[path] - return default_spec if default_spec && loaded_specs[default_spec.name] != default_spec + default_spec if default_spec && loaded_specs[default_spec.name] != default_spec end ## @@ -1284,9 +1308,10 @@ An Array (#{env.inspect}) was passed in from #{caller[3]} ## # Location of Marshal quick gemspecs on remote repositories - MARSHAL_SPEC_DIR = "quick/Marshal.#{Gem.marshal_version}/" + MARSHAL_SPEC_DIR = "quick/Marshal.#{Gem.marshal_version}/".freeze autoload :ConfigFile, File.expand_path("rubygems/config_file", __dir__) + autoload :CIDetector, File.expand_path("rubygems/ci_detector", __dir__) autoload :Dependency, File.expand_path("rubygems/dependency", __dir__) autoload :DependencyList, File.expand_path("rubygems/dependency_list", __dir__) autoload :Installer, File.expand_path("rubygems/installer", __dir__) @@ -1333,6 +1358,17 @@ begin rescue LoadError end +# TruffleRuby >= 24 defines REUSE_AS_BINARY_ON_TRUFFLERUBY in defaults/truffleruby. +# However, TruffleRuby < 24 defines REUSE_AS_BINARY_ON_TRUFFLERUBY directly in its copy +# of lib/rubygems/platform.rb, so it is not defined if RubyGems is updated (gem update --system). +# Instead, we define it here in that case, similar to bundler/lib/bundler/rubygems_ext.rb. +# We must define it here and not in platform.rb because platform.rb is loaded before defaults/truffleruby. +class Gem::Platform + if RUBY_ENGINE == "truffleruby" && !defined?(REUSE_AS_BINARY_ON_TRUFFLERUBY) + REUSE_AS_BINARY_ON_TRUFFLERUBY = %w[libv8 libv8-node sorbet-static].freeze + end +end + ## # Loads the default specs. Gem::Specification.load_defaults @@ -1342,7 +1378,7 @@ require_relative "rubygems/core_ext/kernel_gem" path = File.join(__dir__, "rubygems/core_ext/kernel_require.rb") # When https://bugs.ruby-lang.org/issues/17259 is available, there is no need to override Kernel#warn if RUBY_ENGINE == "truffleruby" || - (RUBY_ENGINE == "ruby" && RUBY_VERSION >= "3.0") + RUBY_ENGINE == "ruby" file = "<internal:#{path}>" else require_relative "rubygems/core_ext/kernel_warn" diff --git a/lib/rubygems/basic_specification.rb b/lib/rubygems/basic_specification.rb index f9bedc967f..0380fceece 100644 --- a/lib/rubygems/basic_specification.rb +++ b/lib/rubygems/basic_specification.rb @@ -76,7 +76,7 @@ class Gem::BasicSpecification elsif missing_extensions? @ignored = true - if Gem::Platform::RUBY == platform || Gem::Platform.local === platform + if platform == Gem::Platform::RUBY || Gem::Platform.local === platform warn "Ignoring #{full_name} because its extensions are not built. " \ "Try: gem pristine #{name} --version #{version}" end @@ -84,7 +84,13 @@ class Gem::BasicSpecification return false end - have_file? file, Gem.suffixes + is_soext = file.end_with?(".so", ".o") + + if is_soext + have_file? file.delete_suffix(File.extname(file)), Gem.dynamic_library_suffixes + else + have_file? file, Gem.suffixes + end end def default_gem? @@ -96,7 +102,7 @@ class Gem::BasicSpecification # Returns full path to the directory where gem's extensions are installed. def extension_dir - @extension_dir ||= File.expand_path(File.join(extensions_dir, full_name)).tap(&Gem::UNTAINT) + @extension_dir ||= File.expand_path(File.join(extensions_dir, full_name)) end ## @@ -110,9 +116,7 @@ class Gem::BasicSpecification def find_full_gem_path # :nodoc: # TODO: also, shouldn't it default to full_name if it hasn't been written? - path = File.expand_path File.join(gems_dir, full_name) - path.tap(&Gem::UNTAINT) - path + File.expand_path File.join(gems_dir, full_name) end private :find_full_gem_path @@ -133,9 +137,9 @@ class Gem::BasicSpecification def full_name if platform == Gem::Platform::RUBY || platform.nil? - "#{name}-#{version}".dup.tap(&Gem::UNTAINT) + "#{name}-#{version}" else - "#{name}-#{version}-#{platform}".dup.tap(&Gem::UNTAINT) + "#{name}-#{version}-#{platform}" end end @@ -147,7 +151,7 @@ class Gem::BasicSpecification @full_require_paths ||= begin full_paths = raw_require_paths.map do |path| - File.join full_gem_path, path.tap(&Gem::UNTAINT) + File.join full_gem_path, path end full_paths << extension_dir if have_extensions? @@ -161,7 +165,7 @@ class Gem::BasicSpecification def datadir # TODO: drop the extra ", gem_name" which is uselessly redundant - File.expand_path(File.join(gems_dir, full_name, "data", name)).tap(&Gem::UNTAINT) + File.expand_path(File.join(gems_dir, full_name, "data", name)) end ## @@ -270,7 +274,7 @@ class Gem::BasicSpecification def matches_for_glob(glob) # TODO: rename? glob = File.join(lib_dirs_glob, glob) - Dir[glob].map {|f| f.tap(&Gem::UNTAINT) } # FIX our tests are broken, run w/ SAFE=1 + Dir[glob] end ## @@ -295,7 +299,7 @@ class Gem::BasicSpecification "lib" # default value for require_paths for bundler/inline end - "#{full_gem_path}/#{dirs}".dup.tap(&Gem::UNTAINT) + "#{full_gem_path}/#{dirs}" end ## @@ -332,7 +336,7 @@ class Gem::BasicSpecification def have_file?(file, suffixes) return true if raw_require_paths.any? do |path| - base = File.join(gems_dir, full_name, path.tap(&Gem::UNTAINT), file).tap(&Gem::UNTAINT) + base = File.join(gems_dir, full_name, path, file) suffixes.any? {|suf| File.file? base + suf } end diff --git a/lib/rubygems/bundler_version_finder.rb b/lib/rubygems/bundler_version_finder.rb index cd51c37c59..dd2fd77418 100644 --- a/lib/rubygems/bundler_version_finder.rb +++ b/lib/rubygems/bundler_version_finder.rb @@ -52,7 +52,7 @@ module Gem::BundlerVersionFinder unless gemfile begin Gem::Util.traverse_parents(Dir.pwd) do |directory| - next unless gemfile = Gem::GEM_DEP_FILES.find {|f| File.file?(f.tap(&Gem::UNTAINT)) } + next unless gemfile = Gem::GEM_DEP_FILES.find {|f| File.file?(f) } gemfile = File.join directory, gemfile break @@ -67,7 +67,7 @@ module Gem::BundlerVersionFinder lockfile = case gemfile when "gems.rb" then "gems.locked" else "#{gemfile}.lock" - end.dup.tap(&Gem::UNTAINT) + end return unless File.file?(lockfile) diff --git a/lib/rubygems/ci_detector.rb b/lib/rubygems/ci_detector.rb new file mode 100644 index 0000000000..7a2d4ee29a --- /dev/null +++ b/lib/rubygems/ci_detector.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gem + module CIDetector + # NOTE: Any changes made here will need to be made to both lib/rubygems/ci_detector.rb and + # bundler/lib/bundler/ci_detector.rb (which are enforced duplicates). + # TODO: Drop that duplication once bundler drops support for RubyGems 3.4 + # + # ## Recognized CI providers, their signifiers, and the relevant docs ## + # + # Travis CI - CI, TRAVIS https://docs.travis-ci.com/user/environment-variables/#default-environment-variables + # Cirrus CI - CI, CIRRUS_CI https://cirrus-ci.org/guide/writing-tasks/#environment-variables + # Circle CI - CI, CIRCLECI https://circleci.com/docs/variables/#built-in-environment-variables + # Gitlab CI - CI, GITLAB_CI https://docs.gitlab.com/ee/ci/variables/ + # AppVeyor - CI, APPVEYOR https://www.appveyor.com/docs/environment-variables/ + # CodeShip - CI_NAME https://docs.cloudbees.com/docs/cloudbees-codeship/latest/pro-builds-and-configuration/environment-variables#_default_environment_variables + # dsari - CI, DSARI https://github.com/rfinnie/dsari#running + # Jenkins - BUILD_NUMBER https://www.jenkins.io/doc/book/pipeline/jenkinsfile/#using-environment-variables + # TeamCity - TEAMCITY_VERSION https://www.jetbrains.com/help/teamcity/predefined-build-parameters.html#Predefined+Server+Build+Parameters + # Appflow - CI_BUILD_ID https://ionic.io/docs/appflow/automation/environments#predefined-environments + # TaskCluster - TASKCLUSTER_ROOT_URL https://docs.taskcluster.net/docs/manual/design/env-vars + # Semaphore - CI, SEMAPHORE https://docs.semaphoreci.com/ci-cd-environment/environment-variables/ + # BuildKite - CI, BUILDKITE https://buildkite.com/docs/pipelines/environment-variables + # GoCD - GO_SERVER_URL https://docs.gocd.org/current/faq/dev_use_current_revision_in_build.html + # GH Actions - CI, GITHUB_ACTIONS https://docs.github.com/en/actions/learn-github-actions/variables#default-environment-variables + # + # ### Some "standard" ENVs that multiple providers may set ### + # + # * CI - this is set by _most_ (but not all) CI providers now; it's approaching a standard. + # * CI_NAME - Not as frequently used, but some providers set this to specify their own name + + # Any of these being set is a reasonably reliable indicator that we are + # executing in a CI environment. + ENV_INDICATORS = [ + "CI", + "CI_NAME", + "CONTINUOUS_INTEGRATION", + "BUILD_NUMBER", + "CI_APP_ID", + "CI_BUILD_ID", + "CI_BUILD_NUMBER", + "RUN_ID", + "TASKCLUSTER_ROOT_URL", + ].freeze + + # For each CI, this env suffices to indicate that we're on _that_ CI's + # containers. (A few of them only supply a CI_NAME variable, which is also + # nice). And if they set "CI" but we can't tell which one they are, we also + # want to know that - a bare "ci" without another token tells us as much. + ENV_DESCRIPTORS = { + "TRAVIS" => "travis", + "CIRCLECI" => "circle", + "CIRRUS_CI" => "cirrus", + "DSARI" => "dsari", + "SEMAPHORE" => "semaphore", + "JENKINS_URL" => "jenkins", + "BUILDKITE" => "buildkite", + "GO_SERVER_URL" => "go", + "GITLAB_CI" => "gitlab", + "GITHUB_ACTIONS" => "github", + "TASKCLUSTER_ROOT_URL" => "taskcluster", + "CI" => "ci", + }.freeze + + def self.ci? + ENV_INDICATORS.any? {|var| ENV.include?(var) } + end + + def self.ci_strings + matching_names = ENV_DESCRIPTORS.select {|env, _| ENV[env] }.values + matching_names << ENV["CI_NAME"].downcase if ENV["CI_NAME"] + matching_names.reject(&:empty?).sort.uniq + end + end +end diff --git a/lib/rubygems/command.rb b/lib/rubygems/command.rb index 53bb609ae0..ec498a8b94 100644 --- a/lib/rubygems/command.rb +++ b/lib/rubygems/command.rb @@ -6,7 +6,7 @@ # See LICENSE.txt for permissions. #++ -require_relative "optparse" +require_relative "vendored_optparse" require_relative "requirement" require_relative "user_interaction" @@ -190,7 +190,7 @@ class Gem::Command "Please specify at least one gem name (e.g. gem build GEMNAME)" end - args.select {|arg| arg !~ /^-/ } + args.reject {|arg| arg.start_with?("-") } end ## @@ -485,7 +485,7 @@ class Gem::Command @parser.separator nil @parser.separator " Description:" - formatted.split("\n").each do |line| + formatted.each_line do |line| @parser.separator " #{line.rstrip}" end end @@ -512,8 +512,8 @@ class Gem::Command @parser.separator nil @parser.separator " #{title}:" - content.split(/\n/).each do |line| - @parser.separator " #{line}" + content.each_line do |line| + @parser.separator " #{line.rstrip}" end end @@ -522,7 +522,7 @@ class Gem::Command @parser.separator nil @parser.separator " Summary:" - wrap(@summary, 80 - 4).split("\n").each do |line| + wrap(@summary, 80 - 4).each_line do |line| @parser.separator " #{line.strip}" end end diff --git a/lib/rubygems/command_manager.rb b/lib/rubygems/command_manager.rb index c53f0231af..8e578dc196 100644 --- a/lib/rubygems/command_manager.rb +++ b/lib/rubygems/command_manager.rb @@ -60,6 +60,7 @@ class Gem::CommandManager :push, :query, :rdoc, + :rebuild, :search, :server, :signin, @@ -106,7 +107,7 @@ class Gem::CommandManager # Register all the subcommands supported by the gem command. def initialize - require "timeout" + require_relative "vendored_timeout" @commands = {} BUILTIN_COMMANDS.each do |name| @@ -149,7 +150,7 @@ class Gem::CommandManager def run(args, build_args=nil) process_args(args, build_args) - rescue StandardError, Timeout::Error => ex + rescue StandardError, Gem::Timeout::Error => ex if ex.respond_to?(:detailed_message) msg = ex.detailed_message(highlight: false).sub(/\A(.*?)(?: \(.+?\))/) { $1 } else @@ -249,6 +250,7 @@ class Gem::CommandManager def invoke_command(args, build_args) cmd_name = args.shift.downcase cmd = find_command cmd_name + terminate_interaction 1 unless cmd cmd.deprecation_warning if cmd.deprecated? cmd.invoke_with_build_args args, build_args end diff --git a/lib/rubygems/commands/build_command.rb b/lib/rubygems/commands/build_command.rb index 0ebdec565b..2ec8324141 100644 --- a/lib/rubygems/commands/build_command.rb +++ b/lib/rubygems/commands/build_command.rb @@ -1,11 +1,13 @@ # frozen_string_literal: true require_relative "../command" +require_relative "../gemspec_helpers" require_relative "../package" require_relative "../version_option" class Gem::Commands::BuildCommand < Gem::Command include Gem::VersionOption + include Gem::GemspecHelpers def initialize super "build", "Build a gem from a gemspec" @@ -75,17 +77,6 @@ Gems can be saved to a specified filename with the output option: private - def find_gemspec(glob = "*.gemspec") - gemspecs = Dir.glob(glob).sort - - if gemspecs.size > 1 - alert_error "Multiple gemspecs found: #{gemspecs}, please specify one" - terminate_interaction(1) - end - - gemspecs.first - end - def build_gem gemspec = resolve_gem_name diff --git a/lib/rubygems/commands/cert_command.rb b/lib/rubygems/commands/cert_command.rb index 22864a9b29..72dcf1dd17 100644 --- a/lib/rubygems/commands/cert_command.rb +++ b/lib/rubygems/commands/cert_command.rb @@ -6,7 +6,7 @@ require_relative "../security" class Gem::Commands::CertCommand < Gem::Command def initialize super "cert", "Manage RubyGems certificates and signing settings", - :add => [], :remove => [], :list => [], :build => [], :sign => [] + add: [], remove: [], list: [], build: [], sign: [] add_option("-a", "--add CERT", "Add a trusted certificate.") do |cert_file, options| diff --git a/lib/rubygems/commands/check_command.rb b/lib/rubygems/commands/check_command.rb index 6552258552..fb23dd9cb4 100644 --- a/lib/rubygems/commands/check_command.rb +++ b/lib/rubygems/commands/check_command.rb @@ -10,7 +10,7 @@ class Gem::Commands::CheckCommand < Gem::Command def initialize super "check", "Check a gem repository for added or missing files", - :alien => true, :doctor => false, :dry_run => false, :gems => true + alien: true, doctor: false, dry_run: false, gems: true add_option("-a", "--[no-]alien", 'Report "unmanaged" or rogue files in the', diff --git a/lib/rubygems/commands/cleanup_command.rb b/lib/rubygems/commands/cleanup_command.rb index 26beba48df..08fb598cea 100644 --- a/lib/rubygems/commands/cleanup_command.rb +++ b/lib/rubygems/commands/cleanup_command.rb @@ -8,8 +8,8 @@ class Gem::Commands::CleanupCommand < Gem::Command def initialize super "cleanup", "Clean up old versions of installed gems", - :force => false, :install_dir => Gem.dir, - :check_dev => true + force: false, install_dir: Gem.dir, + check_dev: true add_option("-n", "-d", "--dry-run", "Do not uninstall gems") do |_value, options| @@ -166,8 +166,8 @@ If no gems are named all gems in GEM_HOME are cleaned. say "Attempting to uninstall #{spec.full_name}" uninstall_options = { - :executables => false, - :version => "= #{spec.version}", + executables: false, + version: "= #{spec.version}", } uninstall_options[:user_install] = Gem.user_dir == spec.base_dir diff --git a/lib/rubygems/commands/contents_command.rb b/lib/rubygems/commands/contents_command.rb index f3783468aa..807158d9c9 100644 --- a/lib/rubygems/commands/contents_command.rb +++ b/lib/rubygems/commands/contents_command.rb @@ -8,8 +8,8 @@ class Gem::Commands::ContentsCommand < Gem::Command def initialize super "contents", "Display the contents of the installed gems", - :specdirs => [], :lib_only => false, :prefix => true, - :show_install_dir => false + specdirs: [], lib_only: false, prefix: true, + show_install_dir: false add_version_option diff --git a/lib/rubygems/commands/dependency_command.rb b/lib/rubygems/commands/dependency_command.rb index 109bf03aff..9aaefae999 100644 --- a/lib/rubygems/commands/dependency_command.rb +++ b/lib/rubygems/commands/dependency_command.rb @@ -11,7 +11,7 @@ class Gem::Commands::DependencyCommand < Gem::Command def initialize super "dependency", "Show the dependencies of an installed gem", - :version => Gem::Requirement.default, :domain => :local + version: Gem::Requirement.default, domain: :local add_version_option add_platform_option diff --git a/lib/rubygems/commands/fetch_command.rb b/lib/rubygems/commands/fetch_command.rb index 950d4fe30e..f7f5b62306 100644 --- a/lib/rubygems/commands/fetch_command.rb +++ b/lib/rubygems/commands/fetch_command.rb @@ -10,8 +10,8 @@ class Gem::Commands::FetchCommand < Gem::Command def initialize defaults = { - :suggest_alternate => true, - :version => Gem::Requirement.default, + suggest_alternate: true, + version: Gem::Requirement.default, } super "fetch", "Download a gem and place it in the current directory", defaults diff --git a/lib/rubygems/commands/generate_index_command.rb b/lib/rubygems/commands/generate_index_command.rb index ce580cfaf9..13be92593b 100644 --- a/lib/rubygems/commands/generate_index_command.rb +++ b/lib/rubygems/commands/generate_index_command.rb @@ -1,86 +1,51 @@ # frozen_string_literal: true require_relative "../command" -require_relative "../indexer" -## -# Generates a index files for use as a gem server. -# -# See `gem help generate_index` - -class Gem::Commands::GenerateIndexCommand < Gem::Command - def initialize - super "generate_index", - "Generates the index files for a gem server directory", - :directory => ".", :build_modern => true +unless defined? Gem::Commands::GenerateIndexCommand + class Gem::Commands::GenerateIndexCommand < Gem::Command + module RubygemsTrampoline + def description # :nodoc: + <<~EOF + The generate_index command has been moved to the rubygems-generate_index gem. + EOF + end - add_option "-d", "--directory=DIRNAME", - "repository base dir containing gems subdir" do |dir, options| - options[:directory] = File.expand_path dir - end + def execute + alert_error "Install the rubygems-generate_index gem for the generate_index command" + end - add_option "--[no-]modern", - "Generate indexes for RubyGems", - "(always true)" do |value, options| - options[:build_modern] = value + def invoke_with_build_args(args, build_args) + name = "rubygems-generate_index" + spec = begin + Gem::Specification.find_by_name(name) + rescue Gem::LoadError + require "rubygems/dependency_installer" + Gem.install(name, Gem::Requirement.default, Gem::DependencyInstaller::DEFAULT_OPTIONS).find {|s| s.name == name } + end + + # remove the methods defined in this file so that the methods defined in the gem are used instead, + # and without a method redefinition warning + %w[description execute invoke_with_build_args].each do |method| + RubygemsTrampoline.remove_method(method) + end + self.class.singleton_class.remove_method(:new) + + spec.activate + Gem.load_plugin_files spec.matches_for_glob("rubygems_plugin#{Gem.suffix_pattern}") + + self.class.new.invoke_with_build_args(args, build_args) + end end + private_constant :RubygemsTrampoline - deprecate_option("--modern", version: "4.0", extra_msg: "Modern indexes (specs, latest_specs, and prerelease_specs) are always generated, so this option is not needed.") - deprecate_option("--no-modern", version: "4.0", extra_msg: "The `--no-modern` option is currently ignored. Modern indexes (specs, latest_specs, and prerelease_specs) are always generated.") - - add_option "--update", - "Update modern indexes with gems added", - "since the last update" do |value, options| - options[:update] = value + # remove_method(:initialize) warns, but removing new does not warn + def self.new + command = allocate + command.send(:initialize, "generate_index", "Generates the index files for a gem server directory (requires rubygems-generate_index)") + command end - end - - def defaults_str # :nodoc: - "--directory . --modern" - end - - def description # :nodoc: - <<-EOF -The generate_index command creates a set of indexes for serving gems -statically. The command expects a 'gems' directory under the path given to -the --directory option. The given directory will be the directory you serve -as the gem repository. -For `gem generate_index --directory /path/to/repo`, expose /path/to/repo via -your HTTP server configuration (not /path/to/repo/gems). - -When done, it will generate a set of files like this: - - gems/*.gem # .gem files you want to - # index - - specs.<version>.gz # specs index - latest_specs.<version>.gz # latest specs index - prerelease_specs.<version>.gz # prerelease specs index - quick/Marshal.<version>/<gemname>.gemspec.rz # Marshal quick index file - -The .rz extension files are compressed with the inflate algorithm. -The Marshal version number comes from ruby's Marshal::MAJOR_VERSION and -Marshal::MINOR_VERSION constants. It is used to ensure compatibility. - EOF - end - - def execute - # This is always true because it's the only way now. - options[:build_modern] = true - - if !File.exist?(options[:directory]) || - !File.directory?(options[:directory]) - alert_error "unknown directory name #{options[:directory]}." - terminate_interaction 1 - else - indexer = Gem::Indexer.new options.delete(:directory), options - - if options[:update] - indexer.update_index - else - indexer.generate_index - end - end + prepend(RubygemsTrampoline) end end diff --git a/lib/rubygems/commands/help_command.rb b/lib/rubygems/commands/help_command.rb index 043e7d3691..1619b152e7 100644 --- a/lib/rubygems/commands/help_command.rb +++ b/lib/rubygems/commands/help_command.rb @@ -59,7 +59,7 @@ multiple environments. The RubyGems implementation is designed to be compatible with Bundler's Gemfile format. You can see additional documentation on the format at: - http://bundler.io + https://bundler.io RubyGems automatically looks for these gem dependencies files: @@ -172,7 +172,7 @@ and #platforms methods: See the bundler Gemfile manual page for a list of platforms supported in a gem dependencies file.: - http://bundler.io/v1.6/man/gemfile.5.html + https://bundler.io/v2.5/man/gemfile.5.html Ruby Version and Engine Dependency ================================== @@ -333,7 +333,7 @@ platform. @command_manager.command_names.each do |cmd_name| command = @command_manager[cmd_name] - next if command.deprecated? + next if command&.deprecated? summary = if command diff --git a/lib/rubygems/commands/info_command.rb b/lib/rubygems/commands/info_command.rb index ced7751ff5..f65c639662 100644 --- a/lib/rubygems/commands/info_command.rb +++ b/lib/rubygems/commands/info_command.rb @@ -8,8 +8,8 @@ class Gem::Commands::InfoCommand < Gem::Command def initialize super "info", "Show information for the given gem", - :name => //, :domain => :local, :details => false, :versions => true, - :installed => nil, :version => Gem::Requirement.default + name: //, domain: :local, details: false, versions: true, + installed: nil, version: Gem::Requirement.default add_query_options diff --git a/lib/rubygems/commands/install_command.rb b/lib/rubygems/commands/install_command.rb index b8dfc90111..2091634a29 100644 --- a/lib/rubygems/commands/install_command.rb +++ b/lib/rubygems/commands/install_command.rb @@ -23,11 +23,11 @@ class Gem::Commands::InstallCommand < Gem::Command def initialize defaults = Gem::DependencyInstaller::DEFAULT_OPTIONS.merge({ - :format_executable => false, - :lock => true, - :suggest_alternate => true, - :version => Gem::Requirement.default, - :without_groups => [], + format_executable: false, + lock: true, + suggest_alternate: true, + version: Gem::Requirement.default, + without_groups: [], }) defaults.merge!(install_update_options) @@ -136,13 +136,6 @@ You can use `i` command instead of `install`. "#{program_name} [options] GEMNAME [GEMNAME ...] -- --build-flags" end - def check_install_dir # :nodoc: - if options[:install_dir] && options[:user_install] - alert_error "Use --install-dir or --user-install but not both" - terminate_interaction 1 - end - end - def check_version # :nodoc: if options[:version] != Gem::Requirement.default && get_all_gem_names.size > 1 @@ -162,7 +155,6 @@ You can use `i` command instead of `install`. ENV.delete "GEM_PATH" if options[:install_dir].nil? - check_install_dir check_version load_hooks @@ -171,7 +163,7 @@ You can use `i` command instead of `install`. show_installed - say update_suggestion if eglible_for_update? + say update_suggestion if eligible_for_update? terminate_interaction exit_code end diff --git a/lib/rubygems/commands/list_command.rb b/lib/rubygems/commands/list_command.rb index 522c771f90..fab4b73814 100644 --- a/lib/rubygems/commands/list_command.rb +++ b/lib/rubygems/commands/list_command.rb @@ -11,8 +11,8 @@ class Gem::Commands::ListCommand < Gem::Command def initialize super "list", "Display local gems whose name matches REGEXP", - :domain => :local, :details => false, :versions => true, - :installed => nil, :version => Gem::Requirement.default + domain: :local, details: false, versions: true, + installed: nil, version: Gem::Requirement.default add_query_options end diff --git a/lib/rubygems/commands/lock_command.rb b/lib/rubygems/commands/lock_command.rb index 3a9512fe3f..f7fd5ada16 100644 --- a/lib/rubygems/commands/lock_command.rb +++ b/lib/rubygems/commands/lock_command.rb @@ -5,7 +5,7 @@ require_relative "../command" class Gem::Commands::LockCommand < Gem::Command def initialize super "lock", "Generate a lockdown list of gems", - :strict => false + strict: false add_option "-s", "--[no-]strict", "fail if unable to satisfy a dependency" do |strict, options| diff --git a/lib/rubygems/commands/open_command.rb b/lib/rubygems/commands/open_command.rb index 5a13074a1d..0fe90dc8b8 100644 --- a/lib/rubygems/commands/open_command.rb +++ b/lib/rubygems/commands/open_command.rb @@ -70,9 +70,7 @@ class Gem::Commands::OpenCommand < Gem::Command end def open_editor(path) - Dir.chdir(path) do - system(*@editor.split(/\s+/) + [path]) - end + system(*@editor.split(/\s+/) + [path], { chdir: path }) end def spec_for(name) diff --git a/lib/rubygems/commands/owner_command.rb b/lib/rubygems/commands/owner_command.rb index fce32aca3e..12bfe3a834 100644 --- a/lib/rubygems/commands/owner_command.rb +++ b/lib/rubygems/commands/owner_command.rb @@ -39,7 +39,7 @@ permission to. add_proxy_option add_key_option add_otp_option - defaults.merge! :add => [], :remove => [] + defaults.merge! add: [], remove: [] add_option "-a", "--add NEW_OWNER", "Add an owner by user identifier" do |value, options| options[:add] << value diff --git a/lib/rubygems/commands/pristine_command.rb b/lib/rubygems/commands/pristine_command.rb index fff59855f5..b272a15b6c 100644 --- a/lib/rubygems/commands/pristine_command.rb +++ b/lib/rubygems/commands/pristine_command.rb @@ -11,10 +11,10 @@ class Gem::Commands::PristineCommand < Gem::Command def initialize super "pristine", "Restores installed gems to pristine condition from files located in the gem cache", - :version => Gem::Requirement.default, - :extensions => true, - :extensions_set => false, - :all => false + version: Gem::Requirement.default, + extensions: true, + extensions_set: false, + all: false add_option("--all", "Restore all installed gems to pristine", @@ -57,7 +57,7 @@ class Gem::Commands::PristineCommand < Gem::Command end add_option("-i", "--install-dir DIR", - "Gem repository to get binstubs and plugins installed") do |value, options| + "Gem repository to get gems restored") do |value, options| options[:install_dir] = File.expand_path(value) end @@ -103,25 +103,29 @@ extensions will be restored. end def execute + install_dir = options[:install_dir] + + specification_record = install_dir ? Gem::SpecificationRecord.from_path(install_dir) : Gem::Specification.specification_record + specs = if options[:all] - Gem::Specification.map + specification_record.map # `--extensions` must be explicitly given to pristine only gems # with extensions. elsif options[:extensions_set] && options[:extensions] && options[:args].empty? - Gem::Specification.select do |spec| + specification_record.select do |spec| spec.extensions && !spec.extensions.empty? end elsif options[:only_missing_extensions] - Gem::Specification.select(&:missing_extensions?) + specification_record.select(&:missing_extensions?) else get_all_gem_names.sort.map do |gem_name| - Gem::Specification.find_all_by_name(gem_name, options[:version]).reverse + specification_record.find_all_by_name(gem_name, options[:version]).reverse end.flatten end - specs = specs.select {|spec| RUBY_ENGINE == spec.platform || Gem::Platform.local === spec.platform || spec.platform == Gem::Platform::RUBY } + specs = specs.select {|spec| spec.platform == RUBY_ENGINE || Gem::Platform.local === spec.platform || spec.platform == Gem::Platform::RUBY } if specs.to_a.empty? raise Gem::Exception, @@ -176,15 +180,14 @@ extensions will be restored. end bin_dir = options[:bin_dir] if options[:bin_dir] - install_dir = options[:install_dir] if options[:install_dir] installer_options = { - :wrappers => true, - :force => true, - :install_dir => install_dir || spec.base_dir, - :env_shebang => env_shebang, - :build_args => spec.build_args, - :bin_dir => bin_dir, + wrappers: true, + force: true, + install_dir: install_dir || spec.base_dir, + env_shebang: env_shebang, + build_args: spec.build_args, + bin_dir: bin_dir, } if options[:only_executables] diff --git a/lib/rubygems/commands/push_command.rb b/lib/rubygems/commands/push_command.rb index 418b46acc5..591ddc3a80 100644 --- a/lib/rubygems/commands/push_command.rb +++ b/lib/rubygems/commands/push_command.rb @@ -30,7 +30,7 @@ The push command will use ~/.gem/credentials to authenticate to a server, but yo end def initialize - super "push", "Push a gem up to the gem server", :host => host + super "push", "Push a gem up to the gem server", host: host @user_defined_host = false diff --git a/lib/rubygems/commands/query_command.rb b/lib/rubygems/commands/query_command.rb index e7d21f341b..3b527974a3 100644 --- a/lib/rubygems/commands/query_command.rb +++ b/lib/rubygems/commands/query_command.rb @@ -20,8 +20,8 @@ class Gem::Commands::QueryCommand < Gem::Command def initialize(name = "query", summary = "Query gem information in local or remote repositories") super name, summary, - :domain => :local, :details => false, :versions => true, - :installed => nil, :version => Gem::Requirement.default + domain: :local, details: false, versions: true, + installed: nil, version: Gem::Requirement.default add_option("-n", "--name-matches REGEXP", "Name of gem(s) to query on matches the", diff --git a/lib/rubygems/commands/rdoc_command.rb b/lib/rubygems/commands/rdoc_command.rb index e318a52914..977c90b8c4 100644 --- a/lib/rubygems/commands/rdoc_command.rb +++ b/lib/rubygems/commands/rdoc_command.rb @@ -10,8 +10,8 @@ class Gem::Commands::RdocCommand < Gem::Command def initialize super "rdoc", "Generates RDoc for pre-installed gems", - :version => Gem::Requirement.default, - :include_rdoc => false, :include_ri => true, :overwrite => false + version: Gem::Requirement.default, + include_rdoc: false, include_ri: true, overwrite: false add_option("--all", "Generate RDoc/RI documentation for all", @@ -84,14 +84,7 @@ Use --overwrite to force rebuilding of documentation. FileUtils.rm_rf File.join(spec.doc_dir, "rdoc") end - begin - doc.generate - rescue Errno::ENOENT => e - match = / - /.match(e.message) - alert_error "Unable to document #{spec.full_name}, " \ - " #{match.post_match} is missing, skipping" - terminate_interaction 1 if specs.length == 1 - end + doc.generate end end end diff --git a/lib/rubygems/commands/rebuild_command.rb b/lib/rubygems/commands/rebuild_command.rb new file mode 100644 index 0000000000..77a474ef1d --- /dev/null +++ b/lib/rubygems/commands/rebuild_command.rb @@ -0,0 +1,262 @@ +# frozen_string_literal: true + +require "date" +require "digest" +require "fileutils" +require "tmpdir" +require_relative "../gemspec_helpers" +require_relative "../package" + +class Gem::Commands::RebuildCommand < Gem::Command + include Gem::GemspecHelpers + + def initialize + super "rebuild", "Attempt to reproduce a build of a gem." + + add_option "--diff", "If the files don't match, compare them using diffoscope." do |_value, options| + options[:diff] = true + end + + add_option "--force", "Skip validation of the spec." do |_value, options| + options[:force] = true + end + + add_option "--strict", "Consider warnings as errors when validating the spec." do |_value, options| + options[:strict] = true + end + + add_option "--source GEM_SOURCE", "Specify the source to download the gem from." do |value, options| + options[:source] = value + end + + add_option "--original GEM_FILE", "Specify a local file to compare against (instead of downloading it)." do |value, options| + options[:original_gem_file] = value + end + + add_option "--gemspec GEMSPEC_FILE", "Specify the name of the gemspec file." do |value, options| + options[:gemspec_file] = value + end + + add_option "-C PATH", "Run as if gem build was started in <PATH> instead of the current working directory." do |value, options| + options[:build_path] = value + end + end + + def arguments # :nodoc: + "GEM_NAME gem name on gem server\n" \ + "GEM_VERSION gem version you are attempting to rebuild" + end + + def description # :nodoc: + <<-EOF +The rebuild command allows you to (attempt to) reproduce a build of a gem +from a ruby gemspec. + +This command assumes the gemspec can be built with the `gem build` command. +If you use any of `gem build`, `rake build`, or`rake release` in the +build/release process for a gem, it is a potential candidate. + +You will need to match the RubyGems version used, since this is included in +the Gem metadata. + +If the gem includes lockfiles (e.g. Gemfile.lock) and similar, it will +require more effort to reproduce a build. For example, it might require +more precisely matched versions of Ruby and/or Bundler to be used. + EOF + end + + def usage # :nodoc: + "#{program_name} GEM_NAME GEM_VERSION" + end + + def execute + gem_name, gem_version = get_gem_name_and_version + + old_dir, new_dir = prep_dirs + + gem_filename = "#{gem_name}-#{gem_version}.gem" + old_file = File.join(old_dir, gem_filename) + new_file = File.join(new_dir, gem_filename) + + if options[:original_gem_file] + FileUtils.copy_file(options[:original_gem_file], old_file) + else + download_gem(gem_name, gem_version, old_file) + end + + rg_version = rubygems_version(old_file) + unless rg_version == Gem::VERSION + alert_error <<-EOF +You need to use the same RubyGems version #{gem_name} v#{gem_version} was built with. + +#{gem_name} v#{gem_version} was built using RubyGems v#{rg_version}. +Gem files include the version of RubyGems used to build them. +This means in order to reproduce #{gem_filename}, you must also use RubyGems v#{rg_version}. + +You're using RubyGems v#{Gem::VERSION}. + +Please install RubyGems v#{rg_version} and try again. + EOF + terminate_interaction 1 + end + + source_date_epoch = get_timestamp(old_file).to_s + + if build_path = options[:build_path] + Dir.chdir(build_path) { build_gem(gem_name, source_date_epoch, new_file) } + else + build_gem(gem_name, source_date_epoch, new_file) + end + + compare(source_date_epoch, old_file, new_file) + end + + private + + def sha256(file) + Digest::SHA256.hexdigest(Gem.read_binary(file)) + end + + def get_timestamp(file) + mtime = nil + File.open(file, Gem.binary_mode) do |f| + Gem::Package::TarReader.new(f) do |tar| + mtime = tar.seek("metadata.gz") {|tf| tf.header.mtime } + end + end + + mtime + end + + def compare(source_date_epoch, old_file, new_file) + date = Time.at(source_date_epoch.to_i).strftime("%F %T %Z") + + old_hash = sha256(old_file) + new_hash = sha256(new_file) + + say + say "Built at: #{date} (#{source_date_epoch})" + say "Original build saved to: #{old_file}" + say "Reproduced build saved to: #{new_file}" + say "Working directory: #{options[:build_path] || Dir.pwd}" + say + say "Hash comparison:" + say " #{old_hash}\t#{old_file}" + say " #{new_hash}\t#{new_file}" + say + + if old_hash == new_hash + say "SUCCESS - original and rebuild hashes matched" + else + say "FAILURE - original and rebuild hashes did not match" + say + + if options[:diff] + if system("diffoscope", old_file, new_file).nil? + alert_error "error: could not find `diffoscope` executable" + end + else + say "Pass --diff for more details (requires diffoscope to be installed)." + end + + terminate_interaction 1 + end + end + + def prep_dirs + rebuild_dir = Dir.mktmpdir("gem_rebuild") + old_dir = File.join(rebuild_dir, "old") + new_dir = File.join(rebuild_dir, "new") + + FileUtils.mkdir_p(old_dir) + FileUtils.mkdir_p(new_dir) + + [old_dir, new_dir] + end + + def get_gem_name_and_version + args = options[:args] || [] + if args.length == 2 + gem_name, gem_version = args + elsif args.length > 2 + raise Gem::CommandLineError, "Too many arguments" + else + raise Gem::CommandLineError, "Expected GEM_NAME and GEM_VERSION arguments (gem rebuild GEM_NAME GEM_VERSION)" + end + + [gem_name, gem_version] + end + + def build_gem(gem_name, source_date_epoch, output_file) + gemspec = options[:gemspec_file] || find_gemspec("#{gem_name}.gemspec") + + if gemspec + build_package(gemspec, source_date_epoch, output_file) + else + alert_error error_message(gem_name) + terminate_interaction(1) + end + end + + def build_package(gemspec, source_date_epoch, output_file) + with_source_date_epoch(source_date_epoch) do + spec = Gem::Specification.load(gemspec) + if spec + Gem::Package.build( + spec, + options[:force], + options[:strict], + output_file + ) + else + alert_error "Error loading gemspec. Aborting." + terminate_interaction 1 + end + end + end + + def with_source_date_epoch(source_date_epoch) + old_sde = ENV["SOURCE_DATE_EPOCH"] + ENV["SOURCE_DATE_EPOCH"] = source_date_epoch.to_s + + yield + ensure + ENV["SOURCE_DATE_EPOCH"] = old_sde + end + + def error_message(gem_name) + if gem_name + "Couldn't find a gemspec file matching '#{gem_name}' in #{Dir.pwd}" + else + "Couldn't find a gemspec file in #{Dir.pwd}" + end + end + + def download_gem(gem_name, gem_version, old_file) + # This code was based loosely off the `gem fetch` command. + version = "= #{gem_version}" + dep = Gem::Dependency.new gem_name, version + + specs_and_sources, errors = + Gem::SpecFetcher.fetcher.spec_for_dependency dep + + # There should never be more than one item in specs_and_sources, + # since we search for an exact version. + spec, source = specs_and_sources[0] + + if spec.nil? + show_lookup_failure gem_name, version, errors, options[:domain] + terminate_interaction 1 + end + + download_path = source.download spec + + FileUtils.move(download_path, old_file) + + say "Downloaded #{gem_name} version #{gem_version} as #{old_file}." + end + + def rubygems_version(gem_file) + Gem::Package.new(gem_file).spec.rubygems_version + end +end diff --git a/lib/rubygems/commands/search_command.rb b/lib/rubygems/commands/search_command.rb index c7469e1fa8..50e161ac9b 100644 --- a/lib/rubygems/commands/search_command.rb +++ b/lib/rubygems/commands/search_command.rb @@ -8,8 +8,8 @@ class Gem::Commands::SearchCommand < Gem::Command def initialize super "search", "Display remote gems whose name matches REGEXP", - :domain => :remote, :details => false, :versions => true, - :installed => nil, :version => Gem::Requirement.default + domain: :remote, details: false, versions: true, + installed: nil, version: Gem::Requirement.default add_query_options end diff --git a/lib/rubygems/commands/setup_command.rb b/lib/rubygems/commands/setup_command.rb index c35d0f5ccc..9c633d6ef7 100644 --- a/lib/rubygems/commands/setup_command.rb +++ b/lib/rubygems/commands/setup_command.rb @@ -7,19 +7,19 @@ require_relative "../command" # RubyGems checkout or tarball. class Gem::Commands::SetupCommand < Gem::Command - HISTORY_HEADER = %r{^#\s*[\d.a-zA-Z]+\s*/\s*\d{4}-\d{2}-\d{2}\s*$}.freeze - VERSION_MATCHER = %r{^#\s*([\d.a-zA-Z]+)\s*/\s*\d{4}-\d{2}-\d{2}\s*$}.freeze + HISTORY_HEADER = %r{^#\s*[\d.a-zA-Z]+\s*/\s*\d{4}-\d{2}-\d{2}\s*$} + VERSION_MATCHER = %r{^#\s*([\d.a-zA-Z]+)\s*/\s*\d{4}-\d{2}-\d{2}\s*$} ENV_PATHS = %w[/usr/bin/env /bin/env].freeze def initialize super "setup", "Install RubyGems", - :format_executable => false, :document => %w[ri], - :force => true, - :site_or_vendor => "sitelibdir", - :destdir => "", :prefix => "", :previous_version => "", - :regenerate_binstubs => true, - :regenerate_plugins => true + format_executable: false, document: %w[ri], + force: true, + site_or_vendor: "sitelibdir", + destdir: "", prefix: "", previous_version: "", + regenerate_binstubs: true, + regenerate_plugins: true add_option "--previous-version=VERSION", "Previous version of RubyGems", @@ -265,7 +265,7 @@ By default, this RubyGems will install gem as: fp.puts bin.join end - install bin_tmp_file, dest_file, :mode => prog_mode + install bin_tmp_file, dest_file, mode: prog_mode bin_file_names << dest_file ensure rm bin_tmp_file @@ -287,7 +287,7 @@ By default, this RubyGems will install gem as: TEXT end - install bin_cmd_file, "#{dest_file}.bat", :mode => prog_mode + install bin_cmd_file, "#{dest_file}.bat", mode: prog_mode ensure rm bin_cmd_file end @@ -369,18 +369,21 @@ By default, this RubyGems will install gem as: File.dirname(loaded_from) else target_specs_dir = File.join(default_dir, "specifications", "default") - mkdir_p target_specs_dir, :mode => 0o755 + mkdir_p target_specs_dir, mode: 0o755 target_specs_dir end - bundler_spec = Dir.chdir("bundler") { Gem::Specification.load("bundler.gemspec") } - default_spec_path = File.join(specs_dir, "#{bundler_spec.full_name}.gemspec") - Gem.write_binary(default_spec_path, bundler_spec.to_ruby) + new_bundler_spec = Dir.chdir("bundler") { Gem::Specification.load("bundler.gemspec") } + full_name = new_bundler_spec.full_name + gemspec_path = "#{full_name}.gemspec" + + default_spec_path = File.join(specs_dir, gemspec_path) + Gem.write_binary(default_spec_path, new_bundler_spec.to_ruby) bundler_spec = Gem::Specification.load(default_spec_path) # Remove gemspec that was same version of vendored bundler. - normal_gemspec = File.join(default_dir, "specifications", "bundler-#{bundler_spec.version}.gemspec") + normal_gemspec = File.join(default_dir, "specifications", gemspec_path) if File.file? normal_gemspec File.delete normal_gemspec end @@ -388,20 +391,14 @@ By default, this RubyGems will install gem as: # Remove gem files that were same version of vendored bundler. if File.directory? bundler_spec.gems_dir Dir.entries(bundler_spec.gems_dir). - select {|default_gem| File.basename(default_gem) == "bundler-#{bundler_spec.version}" }. + select {|default_gem| File.basename(default_gem) == full_name }. each {|default_gem| rm_r File.join(bundler_spec.gems_dir, default_gem) } end - bundler_bin_dir = bundler_spec.bin_dir - mkdir_p bundler_bin_dir, :mode => 0o755 - bundler_spec.executables.each do |e| - cp File.join("bundler", bundler_spec.bindir, e), File.join(bundler_bin_dir, e) - end - require_relative "../installer" Dir.chdir("bundler") do - built_gem = Gem::Package.build(bundler_spec) + built_gem = Gem::Package.build(new_bundler_spec) begin Gem::Installer.at( built_gem, @@ -418,9 +415,9 @@ By default, this RubyGems will install gem as: end end - bundler_spec.executables.each {|executable| bin_file_names << target_bin_path(bin_dir, executable) } + new_bundler_spec.executables.each {|executable| bin_file_names << target_bin_path(bin_dir, executable) } - say "Bundler #{bundler_spec.version} installed" + say "Bundler #{new_bundler_spec.version} installed" end def make_destination_dirs @@ -430,8 +427,8 @@ By default, this RubyGems will install gem as: lib_dir, bin_dir = generate_default_dirs end - mkdir_p lib_dir, :mode => 0o755 - mkdir_p bin_dir, :mode => 0o755 + mkdir_p lib_dir, mode: 0o755 + mkdir_p bin_dir, mode: 0o755 [lib_dir, bin_dir] end @@ -576,8 +573,8 @@ abort "#{deprecation_message}" def uninstall_old_gemcutter require_relative "../uninstaller" - ui = Gem::Uninstaller.new("gemcutter", :all => true, :ignore => true, - :version => "< 0.4") + ui = Gem::Uninstaller.new("gemcutter", all: true, ignore: true, + version: "< 0.4") ui.uninstall rescue Gem::InstallError end @@ -588,6 +585,8 @@ abort "#{deprecation_message}" args = %w[--all --only-executables --silent] args << "--bindir=#{bindir}" + args << "--install-dir=#{default_dir}" + if options[:env_shebang] args << "--env-shebang" end @@ -639,10 +638,10 @@ abort "#{deprecation_message}" dest_file = File.join dest_dir, file dest_dir = File.dirname dest_file unless File.directory? dest_dir - mkdir_p dest_dir, :mode => 0o755 + mkdir_p dest_dir, mode: 0o755 end - install file, dest_file, :mode => options[:data_mode] || 0o644 + install file, dest_file, mode: options[:data_mode] || 0o644 end def remove_file_list(files, dir) diff --git a/lib/rubygems/commands/sources_command.rb b/lib/rubygems/commands/sources_command.rb index c9c6ee80ed..976f4a4ea2 100644 --- a/lib/rubygems/commands/sources_command.rb +++ b/lib/rubygems/commands/sources_command.rb @@ -59,7 +59,7 @@ class Gem::Commands::SourcesCommand < Gem::Command say "#{source_uri} added to sources" end - rescue URI::Error, ArgumentError + rescue Gem::URI::Error, ArgumentError say "#{source_uri} is not a URI" terminate_interaction 1 rescue Gem::RemoteFetcher::FetchError => e @@ -81,7 +81,7 @@ Do you want to add this source? end def check_rubygems_https(source_uri) # :nodoc: - uri = URI source_uri + uri = Gem::URI source_uri if uri.scheme && uri.scheme.casecmp("http").zero? && uri.host.casecmp("rubygems.org").zero? diff --git a/lib/rubygems/commands/specification_command.rb b/lib/rubygems/commands/specification_command.rb index 938b9507f9..a21ed35be3 100644 --- a/lib/rubygems/commands/specification_command.rb +++ b/lib/rubygems/commands/specification_command.rb @@ -13,8 +13,8 @@ class Gem::Commands::SpecificationCommand < Gem::Command Gem.load_yaml super "specification", "Display gem specification (in yaml)", - :domain => :local, :version => Gem::Requirement.default, - :format => :yaml + domain: :local, version: Gem::Requirement.default, + format: :yaml add_version_option("examine") add_platform_option diff --git a/lib/rubygems/commands/uninstall_command.rb b/lib/rubygems/commands/uninstall_command.rb index c66bbe42e1..283bc96ce3 100644 --- a/lib/rubygems/commands/uninstall_command.rb +++ b/lib/rubygems/commands/uninstall_command.rb @@ -15,8 +15,8 @@ class Gem::Commands::UninstallCommand < Gem::Command def initialize super "uninstall", "Uninstall gems from the local repository", - :version => Gem::Requirement.default, :user_install => true, - :check_dev => false, :vendor => false + version: Gem::Requirement.default, user_install: true, + check_dev: false, vendor: false add_option("-a", "--[no-]all", "Uninstall all matching versions") do |value, options| @@ -168,10 +168,10 @@ that is a dependency of an existing gem. You can use the gems_to_uninstall = {} deps.each do |dep| - next if gems_to_uninstall[dep.name] - gems_to_uninstall[dep.name] = true - - unless original_gem_version[dep.name] == Gem::Requirement.default + if original_gem_version[dep.name] == Gem::Requirement.default + next if gems_to_uninstall[dep.name] + gems_to_uninstall[dep.name] = true + else options[:version] = dep.version end @@ -184,7 +184,7 @@ that is a dependency of an existing gem. You can use the rescue Gem::GemNotInHomeException => e spec = e.spec alert("In order to remove #{spec.name}, please execute:\n" \ - "\tgem uninstall #{spec.name} --install-dir=#{spec.installation_path}") + "\tgem uninstall #{spec.name} --install-dir=#{spec.base_dir}") rescue Gem::UninstallError => e spec = e.spec alert_error("Error: unable to successfully uninstall '#{spec.name}' which is " \ diff --git a/lib/rubygems/commands/unpack_command.rb b/lib/rubygems/commands/unpack_command.rb index 73eefe153c..c2fc720297 100644 --- a/lib/rubygems/commands/unpack_command.rb +++ b/lib/rubygems/commands/unpack_command.rb @@ -21,8 +21,8 @@ class Gem::Commands::UnpackCommand < Gem::Command require "fileutils" super "unpack", "Unpack an installed gem to the current directory", - :version => Gem::Requirement.default, - :target => Dir.pwd + version: Gem::Requirement.default, + target: Dir.pwd add_option("--target=DIR", "target directory for unpacking") do |value, options| @@ -143,12 +143,6 @@ command help for an example. # get_path 'rake', '< 0.1' # nil # get_path 'rak' # nil (exact name required) #-- - # TODO: This should be refactored so that it's a general service. I don't - # think any of our existing classes are the right place though. Just maybe - # 'Cache'? - # - # TODO: It just uses Gem.dir for now. What's an easy way to get the list of - # source directories? def get_path(dependency) return dependency.name if /\.gem$/i.match?(dependency.name) diff --git a/lib/rubygems/commands/update_command.rb b/lib/rubygems/commands/update_command.rb index fb27e755bc..8e80d46856 100644 --- a/lib/rubygems/commands/update_command.rb +++ b/lib/rubygems/commands/update_command.rb @@ -21,7 +21,7 @@ class Gem::Commands::UpdateCommand < Gem::Command def initialize options = { - :force => false, + force: false, } options.merge!(install_update_options) @@ -197,18 +197,17 @@ command to remove old versions. yield else require "tmpdir" - tmpdir = Dir.mktmpdir - FileUtils.mv Gem.plugindir, tmpdir + Dir.mktmpdir("gem_update") do |tmpdir| + FileUtils.mv Gem.plugindir, tmpdir - status = yield + status = yield - if status - FileUtils.rm_rf tmpdir - else - FileUtils.mv File.join(tmpdir, "plugins"), Gem.plugindir - end + unless status + FileUtils.mv File.join(tmpdir, "plugins"), Gem.plugindir + end - status + status + end end end @@ -244,7 +243,7 @@ command to remove old versions. @installer = Gem::DependencyInstaller.new update_options - say "Updating #{name}" unless options[:system] && options[:silent] + say "Updating #{name}" unless options[:system] begin @installer.install name, Gem::Requirement.new(version) rescue Gem::InstallError, Gem::DependencyError => e @@ -282,7 +281,7 @@ command to remove old versions. check_oldest_rubygems version installed_gems = Gem::Specification.find_all_by_name "rubygems-update", requirement - installed_gems = update_gem("rubygems-update", version) if installed_gems.empty? || installed_gems.first.version != version + installed_gems = update_gem("rubygems-update", requirement) if installed_gems.empty? || installed_gems.first.version != version return if installed_gems.empty? install_rubygems installed_gems.first @@ -294,9 +293,7 @@ command to remove old versions. args << "--prefix" << Gem.prefix if Gem.prefix args << "--no-document" unless options[:document].include?("rdoc") || options[:document].include?("ri") args << "--no-format-executable" if options[:no_format_executable] - args << "--previous-version" << Gem::VERSION if - options[:system] == true || - Gem::Version.new(options[:system]) >= Gem::Version.new(2) + args << "--previous-version" << Gem::VERSION args end @@ -328,12 +325,8 @@ command to remove old versions. @oldest_supported_version ||= if Gem.ruby_version > Gem::Version.new("3.1.a") Gem::Version.new("3.3.3") - elsif Gem.ruby_version > Gem::Version.new("3.0.a") - Gem::Version.new("3.2.3") - elsif Gem.ruby_version > Gem::Version.new("2.7.a") - Gem::Version.new("3.1.2") else - Gem::Version.new("3.0.1") + Gem::Version.new("3.2.3") end end end diff --git a/lib/rubygems/commands/which_command.rb b/lib/rubygems/commands/which_command.rb index ec464d9672..5ed4d9d142 100644 --- a/lib/rubygems/commands/which_command.rb +++ b/lib/rubygems/commands/which_command.rb @@ -5,7 +5,7 @@ require_relative "../command" class Gem::Commands::WhichCommand < Gem::Command def initialize super "which", "Find the location of a library file you can require", - :search_gems_first => false, :show_all => false + search_gems_first: false, show_all: false add_option "-a", "--[no-]all", "show all matching files" do |show_all, options| options[:show_all] = show_all diff --git a/lib/rubygems/config_file.rb b/lib/rubygems/config_file.rb index d90ec9f92c..7874ad0dc9 100644 --- a/lib/rubygems/config_file.rb +++ b/lib/rubygems/config_file.rb @@ -47,6 +47,8 @@ class Gem::ConfigFile DEFAULT_CONCURRENT_DOWNLOADS = 8 DEFAULT_CERT_EXPIRATION_LENGTH_DAYS = 365 DEFAULT_IPV4_FALLBACK_ENABLED = false + # TODO: Use false as default value for this option in RubyGems 4.0 + DEFAULT_INSTALL_EXTENSION_IN_LIB = true ## # For Ruby packagers to set configuration defaults. Set in @@ -143,6 +145,11 @@ class Gem::ConfigFile attr_accessor :cert_expiration_length_days ## + # Install extensions into lib as well as into the extension directory. + + attr_accessor :install_extension_in_lib + + ## # == Experimental == # Fallback to IPv4 when IPv6 is not reachable or slow (default: false) @@ -183,12 +190,13 @@ class Gem::ConfigFile @update_sources = DEFAULT_UPDATE_SOURCES @concurrent_downloads = DEFAULT_CONCURRENT_DOWNLOADS @cert_expiration_length_days = DEFAULT_CERT_EXPIRATION_LENGTH_DAYS + @install_extension_in_lib = DEFAULT_INSTALL_EXTENSION_IN_LIB @ipv4_fallback_enabled = ENV["IPV4_FALLBACK_ENABLED"] == "true" || DEFAULT_IPV4_FALLBACK_ENABLED operating_system_config = Marshal.load Marshal.dump(OPERATING_SYSTEM_DEFAULTS) platform_config = Marshal.load Marshal.dump(PLATFORM_DEFAULTS) system_config = load_file SYSTEM_WIDE_CONFIG_FILE - user_config = load_file config_file_name.dup.tap(&Gem::UNTAINT) + user_config = load_file config_file_name environment_config = (ENV["GEMRC"] || ""). split(File::PATH_SEPARATOR).inject({}) do |result, file| @@ -202,21 +210,34 @@ class Gem::ConfigFile @hash = @hash.merge environment_config end + @hash.transform_keys! do |k| + # gemhome and gempath are not working with symbol keys + if %w[backtrace bulk_threshold verbose update_sources cert_expiration_length_days + install_extension_in_lib ipv4_fallback_enabled sources disable_default_gem_server + ssl_verify_mode ssl_ca_cert ssl_client_cert].include?(k) + k.to_sym + else + k + end + end + # HACK: these override command-line args, which is bad @backtrace = @hash[:backtrace] if @hash.key? :backtrace @bulk_threshold = @hash[:bulk_threshold] if @hash.key? :bulk_threshold - @home = @hash[:gemhome] if @hash.key? :gemhome - @path = @hash[:gempath] if @hash.key? :gempath - @update_sources = @hash[:update_sources] if @hash.key? :update_sources @verbose = @hash[:verbose] if @hash.key? :verbose - @disable_default_gem_server = @hash[:disable_default_gem_server] if @hash.key? :disable_default_gem_server - @sources = @hash[:sources] if @hash.key? :sources + @update_sources = @hash[:update_sources] if @hash.key? :update_sources + # TODO: We should handle concurrent_downloads same as other options @cert_expiration_length_days = @hash[:cert_expiration_length_days] if @hash.key? :cert_expiration_length_days + @install_extension_in_lib = @hash[:install_extension_in_lib] if @hash.key? :install_extension_in_lib @ipv4_fallback_enabled = @hash[:ipv4_fallback_enabled] if @hash.key? :ipv4_fallback_enabled - @ssl_verify_mode = @hash[:ssl_verify_mode] if @hash.key? :ssl_verify_mode - @ssl_ca_cert = @hash[:ssl_ca_cert] if @hash.key? :ssl_ca_cert - @ssl_client_cert = @hash[:ssl_client_cert] if @hash.key? :ssl_client_cert + @home = @hash[:gemhome] if @hash.key? :gemhome + @path = @hash[:gempath] if @hash.key? :gempath + @sources = @hash[:sources] if @hash.key? :sources + @disable_default_gem_server = @hash[:disable_default_gem_server] if @hash.key? :disable_default_gem_server + @ssl_verify_mode = @hash[:ssl_verify_mode] if @hash.key? :ssl_verify_mode + @ssl_ca_cert = @hash[:ssl_ca_cert] if @hash.key? :ssl_ca_cert + @ssl_client_cert = @hash[:ssl_client_cert] if @hash.key? :ssl_client_cert @api_keys = nil @rubygems_api_key = nil @@ -464,6 +485,9 @@ if you believe they were disclosed to a third party. yaml_hash[:concurrent_downloads] = @hash.fetch(:concurrent_downloads, DEFAULT_CONCURRENT_DOWNLOADS) + yaml_hash[:install_extension_in_lib] = + @hash.fetch(:install_extension_in_lib, DEFAULT_INSTALL_EXTENSION_IN_LIB) + yaml_hash[:ssl_verify_mode] = @hash[:ssl_verify_mode] if @hash.key? :ssl_verify_mode diff --git a/lib/rubygems/core_ext/kernel_require.rb b/lib/rubygems/core_ext/kernel_require.rb index e967dfc016..073966b696 100644 --- a/lib/rubygems/core_ext/kernel_require.rb +++ b/lib/rubygems/core_ext/kernel_require.rb @@ -36,53 +36,45 @@ module Kernel def require(path) # :doc: return gem_original_require(path) unless Gem.discover_gems_on_require - begin - RUBYGEMS_ACTIVATION_MONITOR.enter + RUBYGEMS_ACTIVATION_MONITOR.synchronize do + path = File.path(path) + + # If +path+ belongs to a default gem, we activate it and then go straight + # to normal require + + if spec = Gem.find_default_spec(path) + name = spec.name - path = path.to_path if path.respond_to? :to_path + next if Gem.loaded_specs[name] - if spec = Gem.find_unresolved_default_spec(path) # Ensure -I beats a default gem resolved_path = begin rp = nil load_path_check_index = Gem.load_path_insert_index - Gem.activated_gem_paths - Gem.suffixes.each do |s| - $LOAD_PATH[0...load_path_check_index].each do |lp| - safe_lp = lp.dup.tap(&Gem::UNTAINT) - begin - if File.symlink? safe_lp # for backward compatibility - next - end - rescue SecurityError - RUBYGEMS_ACTIVATION_MONITOR.exit - raise + Gem.suffixes.find do |s| + $LOAD_PATH[0...load_path_check_index].find do |lp| + if File.symlink? lp # for backward compatibility + next end - full_path = File.expand_path(File.join(safe_lp, "#{path}#{s}")) - if File.file?(full_path) - rp = full_path - break - end + full_path = File.expand_path(File.join(lp, "#{path}#{s}")) + rp = full_path if File.file?(full_path) end - break if rp end rp end - begin - Kernel.send(:gem, spec.name, Gem::Requirement.default_prerelease) - rescue StandardError - RUBYGEMS_ACTIVATION_MONITOR.exit - raise - end unless resolved_path + Kernel.send(:gem, name, Gem::Requirement.default_prerelease) unless + resolved_path + + next end # If there are no unresolved deps, then we can use just try # normal require handle loading a gem from the rescue below. if Gem::Specification.unresolved_deps.empty? - RUBYGEMS_ACTIVATION_MONITOR.exit - return gem_original_require(path) + next end # If +path+ is for a gem that has already been loaded, don't @@ -92,8 +84,7 @@ module Kernel # TODO request access to the C implementation of this to speed up RubyGems if Gem::Specification.find_active_stub_by_path(path) - RUBYGEMS_ACTIVATION_MONITOR.exit - return gem_original_require(path) + next end # Attempt to find +path+ in any unresolved gems... @@ -124,7 +115,6 @@ module Kernel names = found_specs.map(&:name).uniq if names.size > 1 - RUBYGEMS_ACTIVATION_MONITOR.exit raise Gem::LoadError, "#{path} found in multiple gems: #{names.join ", "}" end @@ -135,26 +125,20 @@ module Kernel unless valid le = Gem::LoadError.new "unable to find a version of '#{names.first}' to activate" le.name = names.first - RUBYGEMS_ACTIVATION_MONITOR.exit raise le end valid.activate end + end - RUBYGEMS_ACTIVATION_MONITOR.exit + begin gem_original_require(path) rescue LoadError => load_error - if load_error.path == path - RUBYGEMS_ACTIVATION_MONITOR.enter + if load_error.path == path && + RUBYGEMS_ACTIVATION_MONITOR.synchronize { Gem.try_activate(path) } - begin - require_again = Gem.try_activate(path) - ensure - RUBYGEMS_ACTIVATION_MONITOR.exit - end - - return gem_original_require(path) if require_again + return gem_original_require(path) end raise load_error diff --git a/lib/rubygems/defaults.rb b/lib/rubygems/defaults.rb index 7888e8cee5..1bd208feb9 100644 --- a/lib/rubygems/defaults.rb +++ b/lib/rubygems/defaults.rb @@ -24,7 +24,7 @@ module Gem default_spec_cache_dir = File.join Gem.user_home, ".gem", "specs" unless File.exist?(default_spec_cache_dir) - default_spec_cache_dir = File.join Gem.data_home, "gem", "specs" + default_spec_cache_dir = File.join Gem.cache_home, "gem", "specs" end default_spec_cache_dir @@ -94,7 +94,7 @@ module Gem # The home directory for the user. def self.user_home - @user_home ||= find_home.tap(&Gem::UNTAINT) + @user_home ||= find_home end ## @@ -112,7 +112,7 @@ module Gem # The path to standard location of the user's configuration directory. def self.config_home - @config_home ||= (ENV["XDG_CONFIG_HOME"] || File.join(Gem.user_home, ".config")) + @config_home ||= ENV["XDG_CONFIG_HOME"] || File.join(Gem.user_home, ".config") end ## @@ -131,35 +131,35 @@ module Gem # The path to standard location of the user's .gemrc file. def self.config_file - @config_file ||= find_config_file.tap(&Gem::UNTAINT) + @config_file ||= find_config_file end ## # The path to standard location of the user's state file. def self.state_file - @state_file ||= File.join(Gem.state_home, "gem", "last_update_check").tap(&Gem::UNTAINT) + @state_file ||= File.join(Gem.state_home, "gem", "last_update_check") end ## # The path to standard location of the user's cache directory. def self.cache_home - @cache_home ||= (ENV["XDG_CACHE_HOME"] || File.join(Gem.user_home, ".cache")) + @cache_home ||= ENV["XDG_CACHE_HOME"] || File.join(Gem.user_home, ".cache") end ## # The path to standard location of the user's data directory. def self.data_home - @data_home ||= (ENV["XDG_DATA_HOME"] || File.join(Gem.user_home, ".local", "share")) + @data_home ||= ENV["XDG_DATA_HOME"] || File.join(Gem.user_home, ".local", "share") end ## # The path to standard location of the user's state directory. def self.state_home - @state_home ||= (ENV["XDG_STATE_HOME"] || File.join(Gem.user_home, ".local", "state")) + @state_home ||= ENV["XDG_STATE_HOME"] || File.join(Gem.user_home, ".local", "state") end ## @@ -236,10 +236,22 @@ module Gem end ## + # Enables automatic installation into user directory + + def self.default_user_install # :nodoc: + if !ENV.key?("GEM_HOME") && (File.exist?(Gem.dir) && !File.writable?(Gem.dir)) + Gem.ui.say "Defaulting to user installation because default installation directory (#{Gem.dir}) is not writable." + return true + end + + false + end + + ## # Install extensions into lib as well as into the extension directory. def self.install_extension_in_lib # :nodoc: - true + Gem.configuration.install_extension_in_lib end ## diff --git a/lib/rubygems/dependency.rb b/lib/rubygems/dependency.rb index 00eff2dfe7..5ce9c5e840 100644 --- a/lib/rubygems/dependency.rb +++ b/lib/rubygems/dependency.rb @@ -271,15 +271,7 @@ class Gem::Dependency end def matching_specs(platform_only = false) - env_req = Gem.env_requirement(name) - matches = Gem::Specification.stubs_for(name).find_all do |spec| - requirement.satisfied_by?(spec.version) && env_req.satisfied_by?(spec.version) - end.map(&:to_spec) - - if prioritizes_bundler? - require_relative "bundler_version_finder" - Gem::BundlerVersionFinder.prioritize!(matches) - end + matches = Gem::Specification.find_all_by_name(name, requirement) if platform_only matches.reject! do |spec| @@ -297,10 +289,6 @@ class Gem::Dependency @requirement.specific? end - def prioritizes_bundler? - name == "bundler" && !specific? - end - def to_specs matches = matching_specs true @@ -328,9 +316,9 @@ class Gem::Dependency return active if active unless prerelease? - # Move prereleases to the end of the list for >= 0 requirements + # Consider prereleases only as a fallback pre, matches = matches.partition {|spec| spec.version.prerelease? } - matches += pre if requirement == Gem::Requirement.default + matches = pre if matches.empty? end matches.first diff --git a/lib/rubygems/dependency_installer.rb b/lib/rubygems/dependency_installer.rb index 56a9309a7e..b119dca1cf 100644 --- a/lib/rubygems/dependency_installer.rb +++ b/lib/rubygems/dependency_installer.rb @@ -17,18 +17,18 @@ class Gem::DependencyInstaller extend Gem::Deprecate DEFAULT_OPTIONS = { # :nodoc: - :env_shebang => false, - :document => %w[ri], - :domain => :both, # HACK: dup - :force => false, - :format_executable => false, # HACK: dup - :ignore_dependencies => false, - :prerelease => false, - :security_policy => nil, # HACK: NoSecurity requires OpenSSL. AlmostNo? Low? - :wrappers => true, - :build_args => nil, - :build_docs_in_background => false, - :install_as_default => false, + env_shebang: false, + document: %w[ri], + domain: :both, # HACK: dup + force: false, + format_executable: false, # HACK: dup + ignore_dependencies: false, + prerelease: false, + security_policy: nil, # HACK: NoSecurity requires OpenSSL. AlmostNo? Low? + wrappers: true, + build_args: nil, + build_docs_in_background: false, + install_as_default: false, }.freeze ## @@ -228,22 +228,22 @@ class Gem::DependencyInstaller @installed_gems = [] options = { - :bin_dir => @bin_dir, - :build_args => @build_args, - :document => @document, - :env_shebang => @env_shebang, - :force => @force, - :format_executable => @format_executable, - :ignore_dependencies => @ignore_dependencies, - :prerelease => @prerelease, - :security_policy => @security_policy, - :user_install => @user_install, - :wrappers => @wrappers, - :build_root => @build_root, - :install_as_default => @install_as_default, - :dir_mode => @dir_mode, - :data_mode => @data_mode, - :prog_mode => @prog_mode, + bin_dir: @bin_dir, + build_args: @build_args, + document: @document, + env_shebang: @env_shebang, + force: @force, + format_executable: @format_executable, + ignore_dependencies: @ignore_dependencies, + prerelease: @prerelease, + security_policy: @security_policy, + user_install: @user_install, + wrappers: @wrappers, + build_root: @build_root, + install_as_default: @install_as_default, + dir_mode: @dir_mode, + data_mode: @data_mode, + prog_mode: @prog_mode, } options[:install_dir] = @install_dir if @only_install_dir diff --git a/lib/rubygems/dependency_list.rb b/lib/rubygems/dependency_list.rb index 30098ff0b5..ad5e59e8c1 100644 --- a/lib/rubygems/dependency_list.rb +++ b/lib/rubygems/dependency_list.rb @@ -6,7 +6,7 @@ # See LICENSE.txt for permissions. #++ -require_relative "tsort" +require_relative "vendored_tsort" require_relative "deprecate" ## diff --git a/lib/rubygems/deprecate.rb b/lib/rubygems/deprecate.rb index 58a6c5b7dc..7d24f9cbfc 100644 --- a/lib/rubygems/deprecate.rb +++ b/lib/rubygems/deprecate.rb @@ -69,99 +69,101 @@ # end # end -module Gem::Deprecate - def self.skip # :nodoc: - @skip ||= false - end +module Gem + module Deprecate + def self.skip # :nodoc: + @skip ||= false + end - def self.skip=(v) # :nodoc: - @skip = v - end + def self.skip=(v) # :nodoc: + @skip = v + end - ## - # Temporarily turn off warnings. Intended for tests only. + ## + # Temporarily turn off warnings. Intended for tests only. - def skip_during - original = Gem::Deprecate.skip - Gem::Deprecate.skip = true - yield - ensure - Gem::Deprecate.skip = original - end + def skip_during + original = Gem::Deprecate.skip + Gem::Deprecate.skip = true + yield + ensure + Gem::Deprecate.skip = original + end - def self.next_rubygems_major_version # :nodoc: - Gem::Version.new(Gem.rubygems_version.segments.first).bump - end + def self.next_rubygems_major_version # :nodoc: + Gem::Version.new(Gem.rubygems_version.segments.first).bump + end - ## - # Simple deprecation method that deprecates +name+ by wrapping it up - # in a dummy method. It warns on each call to the dummy method - # telling the user of +repl+ (unless +repl+ is :none) and the - # year/month that it is planned to go away. + ## + # Simple deprecation method that deprecates +name+ by wrapping it up + # in a dummy method. It warns on each call to the dummy method + # telling the user of +repl+ (unless +repl+ is :none) and the + # year/month that it is planned to go away. - def deprecate(name, repl, year, month) - class_eval do - old = "_deprecated_#{name}" - alias_method old, name - define_method name do |*args, &block| - klass = is_a? Module - target = klass ? "#{self}." : "#{self.class}#" - msg = [ - "NOTE: #{target}#{name} is deprecated", - repl == :none ? " with no replacement" : "; use #{repl} instead", - format(". It will be removed on or after %4d-%02d.", year, month), - "\n#{target}#{name} called from #{Gem.location_of_caller.join(":")}", - ] - warn "#{msg.join}." unless Gem::Deprecate.skip - send old, *args, &block + def deprecate(name, repl, year, month) + class_eval do + old = "_deprecated_#{name}" + alias_method old, name + define_method name do |*args, &block| + klass = is_a? Module + target = klass ? "#{self}." : "#{self.class}#" + msg = [ + "NOTE: #{target}#{name} is deprecated", + repl == :none ? " with no replacement" : "; use #{repl} instead", + format(". It will be removed on or after %4d-%02d.", year, month), + "\n#{target}#{name} called from #{Gem.location_of_caller.join(":")}", + ] + warn "#{msg.join}." unless Gem::Deprecate.skip + send old, *args, &block + end + ruby2_keywords name if respond_to?(:ruby2_keywords, true) end - ruby2_keywords name if respond_to?(:ruby2_keywords, true) end - end - ## - # Simple deprecation method that deprecates +name+ by wrapping it up - # in a dummy method. It warns on each call to the dummy method - # telling the user of +repl+ (unless +repl+ is :none) and the - # Rubygems version that it is planned to go away. + ## + # Simple deprecation method that deprecates +name+ by wrapping it up + # in a dummy method. It warns on each call to the dummy method + # telling the user of +repl+ (unless +repl+ is :none) and the + # Rubygems version that it is planned to go away. - def rubygems_deprecate(name, replacement=:none) - class_eval do - old = "_deprecated_#{name}" - alias_method old, name - define_method name do |*args, &block| - klass = is_a? Module - target = klass ? "#{self}." : "#{self.class}#" - msg = [ - "NOTE: #{target}#{name} is deprecated", - replacement == :none ? " with no replacement" : "; use #{replacement} instead", - ". It will be removed in Rubygems #{Gem::Deprecate.next_rubygems_major_version}", - "\n#{target}#{name} called from #{Gem.location_of_caller.join(":")}", - ] - warn "#{msg.join}." unless Gem::Deprecate.skip - send old, *args, &block + def rubygems_deprecate(name, replacement=:none) + class_eval do + old = "_deprecated_#{name}" + alias_method old, name + define_method name do |*args, &block| + klass = is_a? Module + target = klass ? "#{self}." : "#{self.class}#" + msg = [ + "NOTE: #{target}#{name} is deprecated", + replacement == :none ? " with no replacement" : "; use #{replacement} instead", + ". It will be removed in Rubygems #{Gem::Deprecate.next_rubygems_major_version}", + "\n#{target}#{name} called from #{Gem.location_of_caller.join(":")}", + ] + warn "#{msg.join}." unless Gem::Deprecate.skip + send old, *args, &block + end + ruby2_keywords name if respond_to?(:ruby2_keywords, true) end - ruby2_keywords name if respond_to?(:ruby2_keywords, true) end - end - # Deprecation method to deprecate Rubygems commands - def rubygems_deprecate_command(version = Gem::Deprecate.next_rubygems_major_version) - class_eval do - define_method "deprecated?" do - true - end + # Deprecation method to deprecate Rubygems commands + def rubygems_deprecate_command(version = Gem::Deprecate.next_rubygems_major_version) + class_eval do + define_method "deprecated?" do + true + end - define_method "deprecation_warning" do - msg = [ - "#{command} command is deprecated", - ". It will be removed in Rubygems #{version}.\n", - ] + define_method "deprecation_warning" do + msg = [ + "#{command} command is deprecated", + ". It will be removed in Rubygems #{version}.\n", + ] - alert_warning msg.join.to_s unless Gem::Deprecate.skip + alert_warning msg.join.to_s unless Gem::Deprecate.skip + end end end - end - module_function :rubygems_deprecate, :rubygems_deprecate_command, :skip_during + module_function :rubygems_deprecate, :rubygems_deprecate_command, :skip_during + end end diff --git a/lib/rubygems/exceptions.rb b/lib/rubygems/exceptions.rb index 65caaab8b1..0308b4687f 100644 --- a/lib/rubygems/exceptions.rb +++ b/lib/rubygems/exceptions.rb @@ -292,9 +292,3 @@ class Gem::UnsatisfiableDependencyError < Gem::DependencyError @dependency.requirement end end - -## -# Backwards compatible typo'd exception class for early RubyGems 2.0.x - -Gem::UnsatisfiableDepedencyError = Gem::UnsatisfiableDependencyError # :nodoc: -Gem.deprecate_constant :UnsatisfiableDepedencyError diff --git a/lib/rubygems/ext/builder.rb b/lib/rubygems/ext/builder.rb index c39afc9c9d..be1ba3031c 100644 --- a/lib/rubygems/ext/builder.rb +++ b/lib/rubygems/ext/builder.rb @@ -7,6 +7,7 @@ #++ require_relative "../user_interaction" +require_relative "../shellwords" class Gem::Ext::Builder include Gem::UserInteraction @@ -55,9 +56,8 @@ class Gem::Ext::Builder end def self.ruby - require "shellwords" # Gem.ruby is quoted if it contains whitespace - cmd = Gem.ruby.shellsplit + cmd = Shellwords.split(Gem.ruby) # This load_path is only needed when running rubygems test without a proper installation. # Prepending it in a normal installation will cause problem with order of $LOAD_PATH. @@ -82,20 +82,26 @@ class Gem::Ext::Builder p(command) end results << "current directory: #{dir}" - require "shellwords" - results << command.shelljoin + results << Shellwords.join(command) require "open3" # Set $SOURCE_DATE_EPOCH for the subprocess. build_env = { "SOURCE_DATE_EPOCH" => Gem.source_date_epoch_string }.merge(env) output, status = begin - Open3.capture2e(build_env, *command, :chdir => dir) + Open3.popen2e(build_env, *command, chdir: dir) do |_stdin, stdouterr, wait_thread| + output = String.new + while line = stdouterr.gets + output << line + if verbose + print line + end + end + [output, wait_thread.value] + end rescue StandardError => error raise Gem::InstallError, "#{command_name || class_name} failed#{error.message}" end - if verbose - puts output - else + unless verbose results << output end ensure diff --git a/lib/rubygems/ext/cargo_builder.rb b/lib/rubygems/ext/cargo_builder.rb index fcead1577b..09ad1407c2 100644 --- a/lib/rubygems/ext/cargo_builder.rb +++ b/lib/rubygems/ext/cargo_builder.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative "../shellwords" + # This class is used by rubygems to build Rust extensions. It is a thin-wrapper # over the `cargo rustc` command which takes care of building Rust code in a way # that Ruby can use. @@ -45,7 +47,6 @@ class Gem::Ext::CargoBuilder < Gem::Ext::Builder nesting = extension_nesting(extension) - # TODO: remove in RubyGems 4 if Gem.install_extension_in_lib && lib_dir nested_lib_dir = File.join(lib_dir, nesting) FileUtils.mkdir_p nested_lib_dir @@ -73,8 +74,6 @@ class Gem::Ext::CargoBuilder < Gem::Ext::Builder end def cargo_command(cargo_toml, dest_path, args = [], crate_name = nil) - require "shellwords" - cmd = [] cmd += [cargo, "rustc"] cmd += ["--crate-type", "cdylib"] @@ -185,6 +184,7 @@ class Gem::Ext::CargoBuilder < Gem::Ext::Builder end def cargo_dylib_path(dest_path, crate_name) + so_ext = RbConfig::CONFIG["SOEXT"] prefix = so_ext == "dll" ? "" : "lib" path_parts = [dest_path] path_parts << ENV["CARGO_BUILD_TARGET"] if ENV["CARGO_BUILD_TARGET"] @@ -198,7 +198,7 @@ class Gem::Ext::CargoBuilder < Gem::Ext::Builder output, status = begin - Open3.capture2e(cargo, "metadata", "--no-deps", "--format-version", "1", :chdir => cargo_dir) + Open3.capture2e(cargo, "metadata", "--no-deps", "--format-version", "1", chdir: cargo_dir) rescue StandardError => error raise Gem::InstallError, "cargo metadata failed #{error.message}" end @@ -293,7 +293,7 @@ EOF case var_name # On windows, it is assumed that mkmf has setup an exports file for the - # extension, so we have to to create one ourselves. + # extension, so we have to create one ourselves. when "DEFFILE" write_deffile(dest_dir, crate_name) else @@ -313,22 +313,6 @@ EOF deffile_path end - # We have to basically reimplement RbConfig::CONFIG['SOEXT'] here to support - # Ruby < 2.5 - # - # @see https://github.com/ruby/ruby/blob/c87c027f18c005460746a74c07cd80ee355b16e4/configure.ac#L3185 - def so_ext - return RbConfig::CONFIG["SOEXT"] if RbConfig::CONFIG.key?("SOEXT") - - if win_target? - "dll" - elsif darwin_target? - "dylib" - else - "so" - end - end - # Corresponds to $(LIBPATH) in mkmf def mkmf_libpath ["-L", "native=#{makefile_config("libdir")}"] diff --git a/lib/rubygems/ext/ext_conf_builder.rb b/lib/rubygems/ext/ext_conf_builder.rb index 7efd4a06b7..fb68a7a8cc 100644 --- a/lib/rubygems/ext/ext_conf_builder.rb +++ b/lib/rubygems/ext/ext_conf_builder.rb @@ -43,12 +43,11 @@ class Gem::Ext::ExtConfBuilder < Gem::Ext::Builder full_tmp_dest = File.join(extension_dir, tmp_dest_relative) - # TODO: remove in RubyGems 4 if Gem.install_extension_in_lib && lib_dir FileUtils.mkdir_p lib_dir entries = Dir.entries(full_tmp_dest) - %w[. ..] entries = entries.map {|entry| File.join full_tmp_dest, entry } - FileUtils.cp_r entries, lib_dir, :remove_destination => true + FileUtils.cp_r entries, lib_dir, remove_destination: true end FileUtils::Entry_.new(full_tmp_dest).traverse do |ent| diff --git a/lib/rubygems/ext/rake_builder.rb b/lib/rubygems/ext/rake_builder.rb index 2b9a23a23f..0171807b39 100644 --- a/lib/rubygems/ext/rake_builder.rb +++ b/lib/rubygems/ext/rake_builder.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require_relative "../shellwords" + #-- # Copyright 2006 by Chad Fowler, Rich Kilmer, Jim Weirich and others. # All rights reserved. @@ -15,8 +17,7 @@ class Gem::Ext::RakeBuilder < Gem::Ext::Builder rake = ENV["rake"] if rake - require "shellwords" - rake = rake.shellsplit + rake = Shellwords.split(rake) else begin rake = ruby << "-rrubygems" << Gem.bin_path("rake", "rake") diff --git a/lib/rubygems/gemcutter_utilities.rb b/lib/rubygems/gemcutter_utilities.rb index 15e61440e3..a8361b7ff1 100644 --- a/lib/rubygems/gemcutter_utilities.rb +++ b/lib/rubygems/gemcutter_utilities.rb @@ -2,14 +2,16 @@ require_relative "remote_fetcher" require_relative "text" -require_relative "webauthn_listener" +require_relative "gemcutter_utilities/webauthn_listener" +require_relative "gemcutter_utilities/webauthn_poller" ## # Utility methods for using the RubyGems API. module Gem::GemcutterUtilities ERROR_CODE = 1 - API_SCOPES = [:index_rubygems, :push_rubygem, :yank_rubygem, :add_owner, :remove_owner, :access_webhooks, :show_dashboard].freeze + API_SCOPES = [:index_rubygems, :push_rubygem, :yank_rubygem, :add_owner, :remove_owner, :access_webhooks].freeze + EXCLUSIVELY_API_SCOPES = [:show_dashboard].freeze include Gem::Text @@ -83,7 +85,7 @@ module Gem::GemcutterUtilities # If +allowed_push_host+ metadata is present, then it will only allow that host. def rubygems_api_request(method, path, host = nil, allowed_push_host = nil, scope: nil, credentials: {}, &block) - require "net/http" + require_relative "vendored_net_http" self.host = host if host unless self.host @@ -92,8 +94,8 @@ module Gem::GemcutterUtilities end if allowed_push_host - allowed_host_uri = URI.parse(allowed_push_host) - host_uri = URI.parse(self.host) + allowed_host_uri = Gem::URI.parse(allowed_push_host) + host_uri = Gem::URI.parse(self.host) unless (host_uri.scheme == allowed_host_uri.scheme) && (host_uri.host == allowed_host_uri.host) alert_error "#{self.host.inspect} is not allowed by the gemspec, which only allows #{allowed_push_host.inspect}" @@ -101,7 +103,7 @@ module Gem::GemcutterUtilities end end - uri = URI.parse "#{self.host}/#{path}" + uri = Gem::URI.parse "#{self.host}/#{path}" response = request_with_otp(method, uri, &block) if mfa_unauthorized?(response) @@ -118,7 +120,7 @@ module Gem::GemcutterUtilities end def mfa_unauthorized?(response) - response.is_a?(Net::HTTPUnauthorized) && response.body.start_with?("You have enabled multifactor authentication") + response.is_a?(Gem::Net::HTTPUnauthorized) && response.body.start_with?("You have enabled multifactor authentication") end def update_scope(scope) @@ -128,14 +130,14 @@ module Gem::GemcutterUtilities say "The existing key doesn't have access of #{scope} on #{pretty_host}. Please sign in to update access." - email = ask " Email: " - password = ask_for_password "Password: " + identifier = ask "Username/email: " + password = ask_for_password " Password: " response = rubygems_api_request(:put, "api/v1/api_key", sign_in_host, scope: scope) do |request| - request.basic_auth email, password + request.basic_auth identifier, password request["OTP"] = otp if otp - request.body = URI.encode_www_form({ :api_key => api_key }.merge(update_scope_params)) + request.body = Gem::URI.encode_www_form({ api_key: api_key }.merge(update_scope_params)) end with_response response do |_resp| @@ -157,25 +159,25 @@ module Gem::GemcutterUtilities say "Don't have an account yet? " \ "Create one at #{sign_in_host}/sign_up" - email = ask " Email: " - password = ask_for_password "Password: " + identifier = ask "Username/email: " + password = ask_for_password " Password: " say "\n" key_name = get_key_name(scope) scope_params = get_scope_params(scope) - profile = get_user_profile(email, password) + profile = get_user_profile(identifier, password) mfa_params = get_mfa_params(profile) all_params = scope_params.merge(mfa_params) warning = profile["warning"] - credentials = { email: email, password: password } + credentials = { identifier: identifier, password: password } say "#{warning}\n" if warning response = rubygems_api_request(:post, "api/v1/api_key", sign_in_host, credentials: credentials, scope: scope) do |request| - request.basic_auth email, password + request.basic_auth identifier, password request["OTP"] = otp if otp - request.body = URI.encode_www_form({ name: key_name }.merge(all_params)) + request.body = Gem::URI.encode_www_form({ name: key_name }.merge(all_params)) end with_response response do |resp| @@ -207,13 +209,13 @@ module Gem::GemcutterUtilities def with_response(response, error_prefix = nil) case response - when Net::HTTPSuccess then + when Gem::Net::HTTPSuccess then if block_given? yield response else say clean_text(response.body) end - when Net::HTTPPermanentRedirect, Net::HTTPRedirection then + when Gem::Net::HTTPPermanentRedirect, Gem::Net::HTTPRedirection then message = "The request has redirected permanently to #{response["location"]}. Please check your defined push host URL." message = "#{error_prefix}: #{message}" if error_prefix @@ -243,7 +245,7 @@ module Gem::GemcutterUtilities private def request_with_otp(method, uri, &block) - request_method = Net::HTTP.const_get method.to_s.capitalize + request_method = Gem::Net::HTTP.const_get method.to_s.capitalize Gem::RemoteFetcher.fetcher.request(uri, request_method) do |req| req["OTP"] = otp if otp @@ -253,36 +255,39 @@ module Gem::GemcutterUtilities def fetch_otp(credentials) options[:otp] = if webauthn_url = webauthn_verification_url(credentials) - wait_for_otp(webauthn_url) + server = TCPServer.new 0 + port = server.addr[1].to_s + + url_with_port = "#{webauthn_url}?port=#{port}" + say "You have enabled multi-factor authentication. Please visit #{url_with_port} to authenticate via security device. If you can't verify using WebAuthn but have OTP enabled, you can re-run the gem signin command with the `--otp [your_code]` option." + + threads = [WebauthnListener.listener_thread(host, server), WebauthnPoller.poll_thread(options, host, webauthn_url, credentials)] + otp_thread = wait_for_otp_thread(*threads) + + threads.each(&:join) + + if error = otp_thread[:error] + alert_error error.message + terminate_interaction(1) + end + + say "You are verified with a security device. You may close the browser window." + otp_thread[:otp] else say "You have enabled multi-factor authentication. Please enter OTP code." ask "Code: " end end - def wait_for_otp(webauthn_url) - server = TCPServer.new 0 - port = server.addr[1].to_s - - thread = Thread.new do - Thread.current[:otp] = Gem::WebauthnListener.wait_for_otp_code(host, server) - rescue Gem::WebauthnVerificationError => e - Thread.current[:error] = e - end - thread.abort_on_exception = true - thread.report_on_exception = false - - url_with_port = "#{webauthn_url}?port=#{port}" - say "You have enabled multi-factor authentication. Please visit #{url_with_port} to authenticate via security device. If you can't verify using WebAuthn but have OTP enabled, you can re-run the gem signin command with the `--otp [your_code]` option." - - thread.join - if error = thread[:error] - alert_error error.message - terminate_interaction(1) + def wait_for_otp_thread(*threads) + loop do + threads.each do |otp_thread| + return otp_thread unless otp_thread.alive? + end + sleep 0.1 end - - say "You are verified with a security device. You may close the browser window." - thread[:otp] + ensure + threads.each(&:exit) end def webauthn_verification_url(credentials) @@ -290,10 +295,10 @@ module Gem::GemcutterUtilities if credentials.empty? request.add_field "Authorization", api_key else - request.basic_auth credentials[:email], credentials[:password] + request.basic_auth credentials[:identifier], credentials[:password] end end - response.is_a?(Net::HTTPSuccess) ? response.body : nil + response.is_a?(Gem::Net::HTTPSuccess) ? response.body : nil end def pretty_host(host) @@ -305,15 +310,31 @@ module Gem::GemcutterUtilities end def get_scope_params(scope) - scope_params = {} + scope_params = { index_rubygems: true } if scope scope_params = { scope => true } else - say "Please select scopes you want to enable for the API key (y/n)" - API_SCOPES.each do |s| - selected = ask_yes_no(s.to_s, false) - scope_params[s] = true if selected + say "The default access scope is:" + scope_params.each do |k, _v| + say " #{k}: y" + end + say "\n" + customise = ask_yes_no("Do you want to customise scopes?", false) + if customise + EXCLUSIVELY_API_SCOPES.each do |excl_scope| + selected = ask_yes_no("#{excl_scope} (exclusive scope, answering yes will not prompt for other scopes)", false) + next unless selected + + return { excl_scope => true } + end + + scope_params = {} + + API_SCOPES.each do |s| + selected = ask_yes_no(s.to_s, false) + scope_params[s] = true if selected + end end say "\n" end @@ -325,11 +346,11 @@ module Gem::GemcutterUtilities host == Gem::DEFAULT_HOST end - def get_user_profile(email, password) + def get_user_profile(identifier, password) return {} unless default_host? response = rubygems_api_request(:get, "api/v1/profile/me.yaml") do |request| - request.basic_auth email, password + request.basic_auth identifier, password end with_response response do |resp| @@ -362,6 +383,6 @@ module Gem::GemcutterUtilities end def api_key_forbidden?(response) - response.is_a?(Net::HTTPForbidden) && response.body.start_with?("The API key doesn't have access") + response.is_a?(Gem::Net::HTTPForbidden) && response.body.start_with?("The API key doesn't have access") end end diff --git a/lib/rubygems/gemcutter_utilities/webauthn_listener.rb b/lib/rubygems/gemcutter_utilities/webauthn_listener.rb new file mode 100644 index 0000000000..abf65efe37 --- /dev/null +++ b/lib/rubygems/gemcutter_utilities/webauthn_listener.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +require_relative "webauthn_listener/response" + +## +# The WebauthnListener class retrieves an OTP after a user successfully WebAuthns with the Gem host. +# An instance opens a socket using the TCPServer instance given and listens for a request from the Gem host. +# The request should be a GET request to the root path and contains the OTP code in the form +# of a query parameter `code`. The listener will return the code which will be used as the OTP for +# API requests. +# +# Types of responses sent by the listener after receiving a request: +# - 200 OK: OTP code was successfully retrieved +# - 204 No Content: If the request was an OPTIONS request +# - 400 Bad Request: If the request did not contain a query parameter `code` +# - 404 Not Found: The request was not to the root path +# - 405 Method Not Allowed: OTP code was not retrieved because the request was not a GET/OPTIONS request +# +# Example usage: +# +# thread = Gem::WebauthnListener.listener_thread("https://rubygems.example", server) +# thread.join +# otp = thread[:otp] +# error = thread[:error] +# + +module Gem::GemcutterUtilities + class WebauthnListener + attr_reader :host + + def initialize(host) + @host = host + end + + def self.listener_thread(host, server) + Thread.new do + thread = Thread.current + thread.abort_on_exception = true + thread.report_on_exception = false + thread[:otp] = new(host).wait_for_otp_code(server) + rescue Gem::WebauthnVerificationError => e + thread[:error] = e + ensure + server.close + end + end + + def wait_for_otp_code(server) + loop do + socket = server.accept + request_line = socket.gets + + method, req_uri, _protocol = request_line.split(" ") + req_uri = Gem::URI.parse(req_uri) + + responder = SocketResponder.new(socket) + + unless root_path?(req_uri) + responder.send(NotFoundResponse.for(host)) + raise Gem::WebauthnVerificationError, "Page at #{req_uri.path} not found." + end + + case method.upcase + when "OPTIONS" + responder.send(NoContentResponse.for(host)) + next # will be GET + when "GET" + if otp = parse_otp_from_uri(req_uri) + responder.send(OkResponse.for(host)) + return otp + end + responder.send(BadRequestResponse.for(host)) + raise Gem::WebauthnVerificationError, "Did not receive OTP from #{host}." + else + responder.send(MethodNotAllowedResponse.for(host)) + raise Gem::WebauthnVerificationError, "Invalid HTTP method #{method.upcase} received." + end + end + end + + private + + def root_path?(uri) + uri.path == "/" + end + + def parse_otp_from_uri(uri) + require "cgi" + + return if uri.query.nil? + CGI.parse(uri.query).dig("code", 0) + end + + class SocketResponder + def initialize(socket) + @socket = socket + end + + def send(response) + @socket.print response.to_s + @socket.close + end + end + end +end diff --git a/lib/rubygems/gemcutter_utilities/webauthn_listener/response.rb b/lib/rubygems/gemcutter_utilities/webauthn_listener/response.rb new file mode 100644 index 0000000000..17baa64fff --- /dev/null +++ b/lib/rubygems/gemcutter_utilities/webauthn_listener/response.rb @@ -0,0 +1,163 @@ +# frozen_string_literal: true + +## +# The WebauthnListener Response class is used by the WebauthnListener to create +# responses to be sent to the Gem host. It creates a Gem::Net::HTTPResponse instance +# when initialized and can be converted to the appropriate format to be sent by a socket using `to_s`. +# Gem::Net::HTTPResponse instances cannot be directly sent over a socket. +# +# Types of response classes: +# - OkResponse +# - NoContentResponse +# - BadRequestResponse +# - NotFoundResponse +# - MethodNotAllowedResponse +# +# Example usage: +# +# server = TCPServer.new(0) +# socket = server.accept +# +# response = OkResponse.for("https://rubygems.example") +# socket.print response.to_s +# socket.close +# + +module Gem::GemcutterUtilities + class WebauthnListener + class Response + attr_reader :http_response + + def self.for(host) + new(host) + end + + def initialize(host) + @host = host + + build_http_response + end + + def to_s + status_line = "HTTP/#{@http_response.http_version} #{@http_response.code} #{@http_response.message}\r\n" + headers = @http_response.to_hash.map {|header, value| "#{header}: #{value.join(", ")}\r\n" }.join + "\r\n" + body = @http_response.body ? "#{@http_response.body}\n" : "" + + status_line + headers + body + end + + private + + # Must be implemented in subclasses + def code + raise NotImplementedError + end + + def reason_phrase + raise NotImplementedError + end + + def body; end + + def build_http_response + response_class = Gem::Net::HTTPResponse::CODE_TO_OBJ[code.to_s] + @http_response = response_class.new("1.1", code, reason_phrase) + @http_response.instance_variable_set(:@read, true) + + add_connection_header + add_access_control_headers + add_body + end + + def add_connection_header + @http_response["connection"] = "close" + end + + def add_access_control_headers + @http_response["access-control-allow-origin"] = @host + @http_response["access-control-allow-methods"] = "POST" + @http_response["access-control-allow-headers"] = %w[Content-Type Authorization x-csrf-token] + end + + def add_body + return unless body + @http_response["content-type"] = "text/plain; charset=utf-8" + @http_response["content-length"] = body.bytesize + @http_response.instance_variable_set(:@body, body) + end + end + + class OkResponse < Response + private + + def code + 200 + end + + def reason_phrase + "OK" + end + + def body + "success" + end + end + + class NoContentResponse < Response + private + + def code + 204 + end + + def reason_phrase + "No Content" + end + end + + class BadRequestResponse < Response + private + + def code + 400 + end + + def reason_phrase + "Bad Request" + end + + def body + "missing code parameter" + end + end + + class NotFoundResponse < Response + private + + def code + 404 + end + + def reason_phrase + "Not Found" + end + end + + class MethodNotAllowedResponse < Response + private + + def code + 405 + end + + def reason_phrase + "Method Not Allowed" + end + + def add_access_control_headers + super + @http_response["allow"] = %w[GET OPTIONS] + end + end + end +end diff --git a/lib/rubygems/gemcutter_utilities/webauthn_poller.rb b/lib/rubygems/gemcutter_utilities/webauthn_poller.rb new file mode 100644 index 0000000000..fe3f163a88 --- /dev/null +++ b/lib/rubygems/gemcutter_utilities/webauthn_poller.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +## +# The WebauthnPoller class retrieves an OTP after a user successfully WebAuthns. An instance +# polls the Gem host for the OTP code. The polling request (api/v1/webauthn_verification/<webauthn_token>/status.json) +# is sent to the Gem host every 5 seconds and will timeout after 5 minutes. If the status field in the json response +# is "success", the code field will contain the OTP code. +# +# Example usage: +# +# thread = Gem::WebauthnPoller.poll_thread( +# {}, +# "RubyGems.org", +# "https://rubygems.org/api/v1/webauthn_verification/odow34b93t6aPCdY", +# { email: "email@example.com", password: "password" } +# ) +# thread.join +# otp = thread[:otp] +# error = thread[:error] +# + +module Gem::GemcutterUtilities + class WebauthnPoller + include Gem::GemcutterUtilities + TIMEOUT_IN_SECONDS = 300 + + attr_reader :options, :host + + def initialize(options, host) + @options = options + @host = host + end + + def self.poll_thread(options, host, webauthn_url, credentials) + Thread.new do + thread = Thread.current + thread.abort_on_exception = true + thread.report_on_exception = false + thread[:otp] = new(options, host).poll_for_otp(webauthn_url, credentials) + rescue Gem::WebauthnVerificationError, Gem::Timeout::Error => e + thread[:error] = e + end + end + + def poll_for_otp(webauthn_url, credentials) + Gem::Timeout.timeout(TIMEOUT_IN_SECONDS) do + loop do + response = webauthn_verification_poll_response(webauthn_url, credentials) + raise Gem::WebauthnVerificationError, response.message unless response.is_a?(Gem::Net::HTTPSuccess) + + require "json" + parsed_response = JSON.parse(response.body) + case parsed_response["status"] + when "pending" + sleep 5 + when "success" + return parsed_response["code"] + else + raise Gem::WebauthnVerificationError, parsed_response.fetch("message", "Invalid response from server") + end + end + end + end + + private + + def webauthn_verification_poll_response(webauthn_url, credentials) + webauthn_token = %r{(?<=\/)[^\/]+(?=$)}.match(webauthn_url)[0] + rubygems_api_request(:get, "api/v1/webauthn_verification/#{webauthn_token}/status.json") do |request| + if credentials.empty? + request.add_field "Authorization", api_key + elsif credentials[:identifier] && credentials[:password] + request.basic_auth credentials[:identifier], credentials[:password] + else + raise Gem::WebauthnVerificationError, "Provided missing credentials" + end + end + end + end +end diff --git a/lib/rubygems/gemspec_helpers.rb b/lib/rubygems/gemspec_helpers.rb new file mode 100644 index 0000000000..2b20fcafa1 --- /dev/null +++ b/lib/rubygems/gemspec_helpers.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require_relative "../rubygems" + +## +# Mixin methods for commands that work with gemspecs. + +module Gem::GemspecHelpers + def find_gemspec(glob = "*.gemspec") + gemspecs = Dir.glob(glob).sort + + if gemspecs.size > 1 + alert_error "Multiple gemspecs found: #{gemspecs}, please specify one" + terminate_interaction(1) + end + + gemspecs.first + end +end diff --git a/lib/rubygems/indexer.rb b/lib/rubygems/indexer.rb deleted file mode 100644 index c6691517b3..0000000000 --- a/lib/rubygems/indexer.rb +++ /dev/null @@ -1,428 +0,0 @@ -# frozen_string_literal: true - -require_relative "../rubygems" -require_relative "package" -require "tmpdir" - -## -# Top level class for building the gem repository index. - -class Gem::Indexer - include Gem::UserInteraction - - ## - # Build indexes for RubyGems 1.2.0 and newer when true - - attr_accessor :build_modern - - ## - # Index install location - - attr_reader :dest_directory - - ## - # Specs index install location - - attr_reader :dest_specs_index - - ## - # Latest specs index install location - - attr_reader :dest_latest_specs_index - - ## - # Prerelease specs index install location - - attr_reader :dest_prerelease_specs_index - - ## - # Index build directory - - attr_reader :directory - - ## - # Create an indexer that will index the gems in +directory+. - - def initialize(directory, options = {}) - require "fileutils" - require "tmpdir" - require "zlib" - - options = { :build_modern => true }.merge options - - @build_modern = options[:build_modern] - - @dest_directory = directory - @directory = Dir.mktmpdir "gem_generate_index" - - marshal_name = "Marshal.#{Gem.marshal_version}" - - @master_index = File.join @directory, "yaml" - @marshal_index = File.join @directory, marshal_name - - @quick_dir = File.join @directory, "quick" - @quick_marshal_dir = File.join @quick_dir, marshal_name - @quick_marshal_dir_base = File.join "quick", marshal_name # FIX: UGH - - @quick_index = File.join @quick_dir, "index" - @latest_index = File.join @quick_dir, "latest_index" - - @specs_index = File.join @directory, "specs.#{Gem.marshal_version}" - @latest_specs_index = - File.join(@directory, "latest_specs.#{Gem.marshal_version}") - @prerelease_specs_index = - File.join(@directory, "prerelease_specs.#{Gem.marshal_version}") - @dest_specs_index = - File.join(@dest_directory, "specs.#{Gem.marshal_version}") - @dest_latest_specs_index = - File.join(@dest_directory, "latest_specs.#{Gem.marshal_version}") - @dest_prerelease_specs_index = - File.join(@dest_directory, "prerelease_specs.#{Gem.marshal_version}") - - @files = [] - end - - ## - # Build various indices - - def build_indices - specs = map_gems_to_specs gem_file_list - Gem::Specification._resort! specs - build_marshal_gemspecs specs - build_modern_indices specs if @build_modern - - compress_indices - end - - ## - # Builds Marshal quick index gemspecs. - - def build_marshal_gemspecs(specs) - count = specs.count - progress = ui.progress_reporter count, - "Generating Marshal quick index gemspecs for #{count} gems", - "Complete" - - files = [] - - Gem.time "Generated Marshal quick index gemspecs" do - specs.each do |spec| - next if spec.default_gem? - spec_file_name = "#{spec.original_name}.gemspec.rz" - marshal_name = File.join @quick_marshal_dir, spec_file_name - - marshal_zipped = Gem.deflate Marshal.dump(spec) - - File.open marshal_name, "wb" do |io| - io.write marshal_zipped - end - - files << marshal_name - - progress.updated spec.original_name - end - - progress.done - end - - @files << @quick_marshal_dir - - files - end - - ## - # Build a single index for RubyGems 1.2 and newer - - def build_modern_index(index, file, name) - say "Generating #{name} index" - - Gem.time "Generated #{name} index" do - File.open(file, "wb") do |io| - specs = index.map do |*spec| - # We have to splat here because latest_specs is an array, while the - # others are hashes. - spec = spec.flatten.last - platform = spec.original_platform - - # win32-api-1.0.4-x86-mswin32-60 - unless String === platform - alert_warning "Skipping invalid platform in gem: #{spec.full_name}" - next - end - - platform = Gem::Platform::RUBY if platform.nil? || platform.empty? - [spec.name, spec.version, platform] - end - - specs = compact_specs(specs) - Marshal.dump(specs, io) - end - end - end - - ## - # Builds indices for RubyGems 1.2 and newer. Handles full, latest, prerelease - - def build_modern_indices(specs) - prerelease, released = specs.partition do |s| - s.version.prerelease? - end - latest_specs = - Gem::Specification._latest_specs specs - - build_modern_index(released.sort, @specs_index, "specs") - build_modern_index(latest_specs.sort, @latest_specs_index, "latest specs") - build_modern_index(prerelease.sort, @prerelease_specs_index, - "prerelease specs") - - @files += [@specs_index, - "#{@specs_index}.gz", - @latest_specs_index, - "#{@latest_specs_index}.gz", - @prerelease_specs_index, - "#{@prerelease_specs_index}.gz"] - end - - def map_gems_to_specs(gems) - gems.map do |gemfile| - if File.size(gemfile) == 0 - alert_warning "Skipping zero-length gem: #{gemfile}" - next - end - - begin - spec = Gem::Package.new(gemfile).spec - spec.loaded_from = gemfile - - spec.abbreviate - spec.sanitize - - spec - rescue SignalException - alert_error "Received signal, exiting" - raise - rescue StandardError => e - msg = ["Unable to process #{gemfile}", - "#{e.message} (#{e.class})", - "\t#{e.backtrace.join "\n\t"}"].join("\n") - alert_error msg - end - end.compact - end - - ## - # Compresses indices on disk - #-- - # All future files should be compressed using gzip, not deflate - - def compress_indices - say "Compressing indices" - - Gem.time "Compressed indices" do - if @build_modern - gzip @specs_index - gzip @latest_specs_index - gzip @prerelease_specs_index - end - end - end - - ## - # Compacts Marshal output for the specs index data source by using identical - # objects as much as possible. - - def compact_specs(specs) - names = {} - versions = {} - platforms = {} - - specs.map do |(name, version, platform)| - names[name] = name unless names.include? name - versions[version] = version unless versions.include? version - platforms[platform] = platform unless platforms.include? platform - - [names[name], versions[version], platforms[platform]] - end - end - - ## - # Compress +filename+ with +extension+. - - def compress(filename, extension) - data = Gem.read_binary filename - - zipped = Gem.deflate data - - File.open "#{filename}.#{extension}", "wb" do |io| - io.write zipped - end - end - - ## - # List of gem file names to index. - - def gem_file_list - Gem::Util.glob_files_in_dir("*.gem", File.join(@dest_directory, "gems")) - end - - ## - # Builds and installs indices. - - def generate_index - make_temp_directories - build_indices - install_indices - rescue SignalException - ensure - FileUtils.rm_rf @directory - end - - ## - # Zlib::GzipWriter wrapper that gzips +filename+ on disk. - - def gzip(filename) - Zlib::GzipWriter.open "#{filename}.gz" do |io| - io.write Gem.read_binary(filename) - end - end - - ## - # Install generated indices into the destination directory. - - def install_indices - verbose = Gem.configuration.really_verbose - - say "Moving index into production dir #{@dest_directory}" if verbose - - files = @files - files.delete @quick_marshal_dir if files.include? @quick_dir - - if files.include?(@quick_marshal_dir) && !files.include?(@quick_dir) - files.delete @quick_marshal_dir - - dst_name = File.join(@dest_directory, @quick_marshal_dir_base) - - FileUtils.mkdir_p File.dirname(dst_name), :verbose => verbose - FileUtils.rm_rf dst_name, :verbose => verbose - FileUtils.mv(@quick_marshal_dir, dst_name, - :verbose => verbose, :force => true) - end - - files = files.map do |path| - path.sub(%r{^#{Regexp.escape @directory}/?}, "") # HACK? - end - - files.each do |file| - src_name = File.join @directory, file - dst_name = File.join @dest_directory, file - - FileUtils.rm_rf dst_name, :verbose => verbose - FileUtils.mv(src_name, @dest_directory, - :verbose => verbose, :force => true) - end - end - - ## - # Make directories for index generation - - def make_temp_directories - FileUtils.rm_rf @directory - FileUtils.mkdir_p @directory, :mode => 0o700 - FileUtils.mkdir_p @quick_marshal_dir - end - - ## - # Ensure +path+ and path with +extension+ are identical. - - def paranoid(path, extension) - data = Gem.read_binary path - compressed_data = Gem.read_binary "#{path}.#{extension}" - - unless data == Gem::Util.inflate(compressed_data) - raise "Compressed file #{compressed_path} does not match uncompressed file #{path}" - end - end - - ## - # Perform an in-place update of the repository from newly added gems. - - def update_index - make_temp_directories - - specs_mtime = File.stat(@dest_specs_index).mtime - newest_mtime = Time.at 0 - - updated_gems = gem_file_list.select do |gem| - gem_mtime = File.stat(gem).mtime - newest_mtime = gem_mtime if gem_mtime > newest_mtime - gem_mtime >= specs_mtime - end - - if updated_gems.empty? - say "No new gems" - terminate_interaction 0 - end - - specs = map_gems_to_specs updated_gems - prerelease, released = specs.partition {|s| s.version.prerelease? } - - files = build_marshal_gemspecs specs - - Gem.time "Updated indexes" do - update_specs_index released, @dest_specs_index, @specs_index - update_specs_index released, @dest_latest_specs_index, @latest_specs_index - update_specs_index(prerelease, - @dest_prerelease_specs_index, - @prerelease_specs_index) - end - - compress_indices - - verbose = Gem.configuration.really_verbose - - say "Updating production dir #{@dest_directory}" if verbose - - files << @specs_index - files << "#{@specs_index}.gz" - files << @latest_specs_index - files << "#{@latest_specs_index}.gz" - files << @prerelease_specs_index - files << "#{@prerelease_specs_index}.gz" - - files = files.map do |path| - path.sub(%r{^#{Regexp.escape @directory}/?}, "") # HACK? - end - - files.each do |file| - src_name = File.join @directory, file - dst_name = File.join @dest_directory, file # REFACTOR: duped above - - FileUtils.mv src_name, dst_name, :verbose => verbose, - :force => true - - File.utime newest_mtime, newest_mtime, dst_name - end - ensure - FileUtils.rm_rf @directory - end - - ## - # Combines specs in +index+ and +source+ then writes out a new copy to - # +dest+. For a latest index, does not ensure the new file is minimal. - - def update_specs_index(index, source, dest) - specs_index = Marshal.load Gem.read_binary(source) - - index.each do |spec| - platform = spec.original_platform - platform = Gem::Platform::RUBY if platform.nil? || platform.empty? - specs_index << [spec.name, spec.version, platform] - end - - specs_index = compact_specs specs_index.uniq.sort - - File.open dest, "wb" do |io| - Marshal.dump specs_index, io - end - end -end diff --git a/lib/rubygems/install_update_options.rb b/lib/rubygems/install_update_options.rb index 5c7289426e..aad207a718 100644 --- a/lib/rubygems/install_update_options.rb +++ b/lib/rubygems/install_update_options.rb @@ -186,7 +186,7 @@ module Gem::InstallUpdateOptions def install_update_options { - :document => %w[ri], + document: %w[ri], } end diff --git a/lib/rubygems/installer.rb b/lib/rubygems/installer.rb index 162a50fdb7..844f292ba2 100644 --- a/lib/rubygems/installer.rb +++ b/lib/rubygems/installer.rb @@ -189,10 +189,12 @@ class Gem::Installer @package.prog_mode = options[:prog_mode] @package.data_mode = options[:data_mode] - if options[:user_install] - @gem_home = Gem.user_dir - @bin_dir = Gem.bindir gem_home unless options[:bin_dir] - @plugins_dir = Gem.plugindir(gem_home) + if @gem_home == Gem.user_dir + # If we get here, then one of the following likely happened: + # - `--user-install` was specified + # - `Gem::PathSupport#home` fell back to `Gem.user_dir` + # - GEM_HOME was manually set to `Gem.user_dir` + check_that_user_bin_dir_is_in_path end end @@ -222,10 +224,11 @@ class Gem::Installer File.open generated_bin, "rb" do |io| line = io.gets - shebang = /^#!.*ruby/ + shebang = /^#!.*ruby/o - if load_relative_enabled? - until line.nil? || line =~ shebang do + # TruffleRuby uses a bash prelude in default launchers + if load_relative_enabled? || RUBY_ENGINE == "truffleruby" + until line.nil? || shebang.match?(line) do line = io.gets end end @@ -236,7 +239,7 @@ class Gem::Installer # TODO: detect a specially formatted comment instead of trying # to find a string inside Ruby code. - next unless io.gets.to_s.include?("This file was generated by RubyGems") + next unless io.gets&.include?("This file was generated by RubyGems") ruby_executable = true existing = io.read.slice(/ @@ -316,7 +319,7 @@ class Gem::Installer FileUtils.rm_rf spec.extension_dir dir_mode = options[:dir_mode] - FileUtils.mkdir_p gem_dir, :mode => dir_mode && 0o755 + FileUtils.mkdir_p gem_dir, mode: dir_mode && 0o755 if @options[:install_as_default] extract_bin @@ -341,18 +344,16 @@ class Gem::Installer say spec.post_install_message if options[:post_install_message] && !spec.post_install_message.nil? - Gem::Specification.add_spec(spec) + Gem::Specification.add_spec(spec) unless @install_dir load_plugin run_post_install_hooks spec - - # TODO: This rescue is in the wrong place. What is raising this exception? - # move this rescue to around the code that actually might raise it. - rescue Zlib::GzipFile::Error - raise Gem::InstallError, "gzip error installing #{gem}" + rescue Errno::EACCES => e + # Permission denied - /path/to/foo + raise Gem::FilePermissionError, e.message.split(" - ").last end def run_pre_install_hooks # :nodoc: @@ -393,7 +394,7 @@ class Gem::Installer specs = [] Gem::Util.glob_files_in_dir("*.gemspec", File.join(gem_home, "specifications")).each do |path| - spec = Gem::Specification.load path.tap(&Gem::UNTAINT) + spec = Gem::Specification.load path specs << spec if spec end @@ -492,7 +493,6 @@ class Gem::Installer ensure_writable_dir @bin_dir spec.executables.each do |filename| - filename.tap(&Gem::UNTAINT) bin_path = File.join gem_dir, spec.bindir, filename next unless File.exist? bin_path @@ -569,7 +569,7 @@ class Gem::Installer File.unlink dst end - FileUtils.symlink src, dst, :verbose => Gem.configuration.really_verbose + FileUtils.symlink src, dst, verbose: Gem.configuration.really_verbose rescue NotImplementedError, SystemCallError alert_warning "Unable to use symlinks, installing wrapper" generate_bin_script filename, bindir @@ -634,7 +634,6 @@ class Gem::Installer def ensure_loadable_spec ruby = spec.to_ruby_for_cache - ruby.tap(&Gem::UNTAINT) begin eval ruby @@ -655,32 +654,37 @@ class Gem::Installer def process_options # :nodoc: @options = { - :bin_dir => nil, - :env_shebang => false, - :force => false, - :only_install_dir => false, - :post_install_message => true, + bin_dir: nil, + env_shebang: false, + force: false, + only_install_dir: false, + post_install_message: true, }.merge options @env_shebang = options[:env_shebang] @force = options[:force] @install_dir = options[:install_dir] - @gem_home = options[:install_dir] || Gem.dir - @plugins_dir = Gem.plugindir(@gem_home) + @user_install = options[:user_install] @ignore_dependencies = options[:ignore_dependencies] @format_executable = options[:format_executable] @wrappers = options[:wrappers] @only_install_dir = options[:only_install_dir] - # If the user has asked for the gem to be installed in a directory that is - # the system gem directory, then use the system bin directory, else create - # (or use) a new bin dir under the gem_home. - @bin_dir = options[:bin_dir] || Gem.bindir(gem_home) + @bin_dir = options[:bin_dir] @development = options[:development] @build_root = options[:build_root] @build_args = options[:build_args] + @gem_home = @install_dir || user_install_dir || Gem.dir + + # If the user has asked for the gem to be installed in a directory that is + # the system gem directory, then use the system bin directory, else create + # (or use) a new bin dir under the gem_home. + @bin_dir ||= Gem.bindir(@gem_home) + + @plugins_dir = Gem.plugindir(@gem_home) + unless @build_root.nil? @bin_dir = File.join(@build_root, @bin_dir.gsub(/^[a-zA-Z]:/, "")) @gem_home = File.join(@build_root, @gem_home.gsub(/^[a-zA-Z]:/, "")) @@ -714,8 +718,7 @@ class Gem::Installer end def verify_gem_home # :nodoc: - FileUtils.mkdir_p gem_home, :mode => options[:dir_mode] && 0o755 - raise Gem::FilePermissionError, gem_home unless File.writable?(gem_home) + FileUtils.mkdir_p gem_home, mode: options[:dir_mode] && 0o755 end def verify_spec @@ -936,7 +939,7 @@ TEXT build_info_dir = File.join gem_home, "build_info" dir_mode = options[:dir_mode] - FileUtils.mkdir_p build_info_dir, :mode => dir_mode && 0o755 + FileUtils.mkdir_p build_info_dir, mode: dir_mode && 0o755 build_info_file = File.join build_info_dir, "#{spec.full_name}.info" @@ -969,6 +972,19 @@ TEXT private + def user_install_dir + # never install to user home in --build-root mode + return unless @build_root.nil? + + # Please note that @user_install might have three states: + # * `true`: `--user-install` + # * `false`: `--no-user-install` and + # * `nil`: option was not specified + if @user_install || (@user_install.nil? && Gem.default_user_install) + Gem.user_dir + end + end + def build_args @build_args ||= begin require_relative "command" diff --git a/lib/rubygems/local_remote_options.rb b/lib/rubygems/local_remote_options.rb index e2a008fada..51a61213a5 100644 --- a/lib/rubygems/local_remote_options.rb +++ b/lib/rubygems/local_remote_options.rb @@ -6,7 +6,7 @@ # See LICENSE.txt for permissions. #++ -require "uri" +require_relative "vendor/uri/lib/uri" require_relative "../rubygems" ## @@ -17,10 +17,10 @@ module Gem::LocalRemoteOptions # Allows Gem::OptionParser to handle HTTP URIs. def accept_uri_http - Gem::OptionParser.accept URI::HTTP do |value| + Gem::OptionParser.accept Gem::URI::HTTP do |value| begin - uri = URI.parse value - rescue URI::InvalidURIError + uri = Gem::URI.parse value + rescue Gem::URI::InvalidURIError raise Gem::OptionParser::InvalidArgument, value end @@ -88,7 +88,7 @@ module Gem::LocalRemoteOptions def add_proxy_option accept_uri_http - add_option(:"Local/Remote", "-p", "--[no-]http-proxy [URL]", URI::HTTP, + add_option(:"Local/Remote", "-p", "--[no-]http-proxy [URL]", Gem::URI::HTTP, "Use HTTP proxy for remote operations") do |value, options| options[:http_proxy] = value == false ? :no_proxy : value Gem.configuration[:http_proxy] = options[:http_proxy] @@ -101,7 +101,7 @@ module Gem::LocalRemoteOptions def add_source_option accept_uri_http - add_option(:"Local/Remote", "-s", "--source URL", URI::HTTP, + add_option(:"Local/Remote", "-s", "--source URL", Gem::URI::HTTP, "Append URL to list of remote gem sources") do |source, options| source << "/" unless source.end_with?("/") diff --git a/lib/rubygems/name_tuple.rb b/lib/rubygems/name_tuple.rb index 9963c8bc5c..3f4a6fcf3d 100644 --- a/lib/rubygems/name_tuple.rb +++ b/lib/rubygems/name_tuple.rb @@ -6,14 +6,12 @@ # wrap the data returned from the indexes. class Gem::NameTuple - def initialize(name, version, platform="ruby") + def initialize(name, version, platform=Gem::Platform::RUBY) @name = name @version = version - unless platform.is_a? Gem::Platform - platform = "ruby" if !platform || platform.empty? - end - + platform &&= platform.to_s + platform = Gem::Platform::RUBY if !platform || platform.empty? @platform = platform end @@ -49,11 +47,11 @@ class Gem::NameTuple def full_name case @platform - when nil, "ruby", "" + when nil, "", Gem::Platform::RUBY "#{@name}-#{@version}" else "#{@name}-#{@version}-#{@platform}" - end.dup.tap(&Gem::UNTAINT) + end end ## diff --git a/lib/rubygems/optparse.rb b/lib/rubygems/optparse.rb deleted file mode 100644 index 6ed718423c..0000000000 --- a/lib/rubygems/optparse.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -require_relative "optparse/lib/optparse" diff --git a/lib/rubygems/optparse/lib/optparse/uri.rb b/lib/rubygems/optparse/lib/optparse/uri.rb deleted file mode 100644 index 664d7f2af4..0000000000 --- a/lib/rubygems/optparse/lib/optparse/uri.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: false -# -*- ruby -*- - -require_relative '../optparse' -require 'uri' - -Gem::OptionParser.accept(URI) {|s,| URI.parse(s) if s} diff --git a/lib/rubygems/package.rb b/lib/rubygems/package.rb index 61cae75357..c855423ed7 100644 --- a/lib/rubygems/package.rb +++ b/lib/rubygems/package.rb @@ -7,7 +7,6 @@ # rubocop:enable Style/AsciiComments -require_relative "../rubygems" require_relative "security" require_relative "user_interaction" @@ -59,7 +58,7 @@ class Gem::Package def initialize(message, source = nil) if source - @path = source.path + @path = source.is_a?(String) ? source : source.path message += " in #{path}" if path end @@ -155,7 +154,7 @@ class Gem::Package Gem::Package::FileSource.new gem end - return super unless Gem::Package == self + return super unless self == Gem::Package return super unless gem.present? return super unless gem.start @@ -268,7 +267,7 @@ class Gem::Package tar.add_file_simple file, stat.mode, stat.size do |dst_io| File.open file, "rb" do |src_io| - dst_io.write src_io.read 16_384 until src_io.eof? + copy_stream(src_io, dst_io) end end end @@ -295,7 +294,6 @@ class Gem::Package Gem.load_yaml - @spec.mark_version @spec.validate true, strict_validation unless skip_validation setup_signer( @@ -347,6 +345,8 @@ EOM return @contents end end + rescue Zlib::GzipFile::Error, EOFError, Gem::Package::TarInvalidError => e + raise Gem::Package::FormatError.new e.message, @gem end ## @@ -355,18 +355,21 @@ EOM def digest(entry) # :nodoc: algorithms = if @checksums - @checksums.keys - else - [Gem::Security::DIGEST_NAME].compact + @checksums.to_h {|algorithm, _| [algorithm, Gem::Security.create_digest(algorithm)] } + elsif Gem::Security::DIGEST_NAME + { Gem::Security::DIGEST_NAME => Gem::Security.create_digest(Gem::Security::DIGEST_NAME) } end - algorithms.each do |algorithm| - digester = Gem::Security.create_digest(algorithm) + return @digests if algorithms.nil? || algorithms.empty? - digester << entry.read(16_384) until entry.eof? - - entry.rewind + buf = String.new(capacity: 16_384, encoding: Encoding::BINARY) + until entry.eof? + entry.readpartial(16_384, buf) + algorithms.each_value {|digester| digester << buf } + end + entry.rewind + algorithms.each do |algorithm, digester| @digests[algorithm][entry.full_name] = digester end @@ -382,7 +385,7 @@ EOM def extract_files(destination_dir, pattern = "*") verify unless @spec - FileUtils.mkdir_p destination_dir, :mode => dir_mode && 0o755 + FileUtils.mkdir_p destination_dir, mode: dir_mode && 0o755 @gem.with_read_io do |io| reader = Gem::Package::TarReader.new io @@ -395,6 +398,8 @@ EOM break # ignore further entries end end + rescue Zlib::GzipFile::Error, EOFError, Gem::Package::TarInvalidError => e + raise Gem::Package::FormatError.new e.message, @gem end ## @@ -409,6 +414,8 @@ EOM # extracted. def extract_tar_gz(io, destination_dir, pattern = "*") # :nodoc: + destination_dir = File.realpath(destination_dir) + directories = [] symlinks = [] @@ -431,8 +438,6 @@ EOM FileUtils.rm_rf destination - mkdir_options = {} - mkdir_options[:mode] = dir_mode ? 0o755 : (entry.header.mode if entry.directory?) mkdir = if entry.directory? destination @@ -441,13 +446,13 @@ EOM end unless directories.include?(mkdir) - FileUtils.mkdir_p mkdir, **mkdir_options + FileUtils.mkdir_p mkdir, mode: dir_mode ? 0o755 : (entry.header.mode if entry.directory?) directories << mkdir end if entry.file? - File.open(destination, "wb") {|out| out.write entry.read } - FileUtils.chmod file_mode(entry.header.mode), destination + File.open(destination, "wb") {|out| copy_stream(entry, out) } + FileUtils.chmod file_mode(entry.header.mode) & ~File.umask, destination end verbose destination @@ -506,7 +511,6 @@ EOM raise Gem::Package::PathError.new(destination, destination_dir) unless normalize_path(destination).start_with? normalize_path(destination_dir + "/") - destination.tap(&Gem::UNTAINT) destination end @@ -522,12 +526,13 @@ EOM # Loads a Gem::Specification from the TarEntry +entry+ def load_spec(entry) # :nodoc: + limit = 10 * 1024 * 1024 case entry.full_name when "metadata" then - @spec = Gem::Specification.from_yaml entry.read + @spec = Gem::Specification.from_yaml limit_read(entry, "metadata", limit) when "metadata.gz" then Zlib::GzipReader.wrap(entry, external_encoding: Encoding::UTF_8) do |gzio| - @spec = Gem::Specification.from_yaml gzio.read + @spec = Gem::Specification.from_yaml limit_read(gzio, "metadata.gz", limit) end end end @@ -551,7 +556,7 @@ EOM @checksums = gem.seek "checksums.yaml.gz" do |entry| Zlib::GzipReader.wrap entry do |gz_io| - Gem::SafeYAML.safe_load gz_io.read + Gem::SafeYAML.safe_load limit_read(gz_io, "checksums.yaml.gz", 10 * 1024 * 1024) end end end @@ -626,7 +631,7 @@ EOM raise rescue Errno::ENOENT => e raise Gem::Package::FormatError.new e.message - rescue Gem::Package::TarInvalidError => e + rescue Zlib::GzipFile::Error, EOFError, Gem::Package::TarInvalidError => e raise Gem::Package::FormatError.new e.message, @gem end @@ -658,7 +663,7 @@ EOM case file_name when /\.sig$/ then - @signatures[$`] = entry.read if @security_policy + @signatures[$`] = limit_read(entry, file_name, 1024 * 1024) if @security_policy return else digest entry @@ -702,11 +707,28 @@ EOM def verify_gz(entry) # :nodoc: Zlib::GzipReader.wrap entry do |gzio| + # TODO: read into a buffer once zlib supports it gzio.read 16_384 until gzio.eof? # gzip checksum verification end rescue Zlib::GzipFile::Error => e raise Gem::Package::FormatError.new(e.message, entry.full_name) end + + if RUBY_ENGINE == "truffleruby" + def copy_stream(src, dst) # :nodoc: + dst.write src.read + end + else + def copy_stream(src, dst) # :nodoc: + IO.copy_stream(src, dst) + end + end + + def limit_read(io, name, limit) + bytes = io.read(limit + 1) + raise Gem::Package::FormatError, "#{name} is too big (over #{limit} bytes)" if bytes.size > limit + bytes + end end require_relative "package/digest_io" diff --git a/lib/rubygems/package/old.rb b/lib/rubygems/package/old.rb index 41f6986b8f..1a13ac3e29 100644 --- a/lib/rubygems/package/old.rb +++ b/lib/rubygems/package/old.rb @@ -70,7 +70,7 @@ class Gem::Package::Old < Gem::Package file_data << line end - file_data = file_data.strip.unpack("m")[0] + file_data = file_data.strip.unpack1("m") file_data = Zlib::Inflate.inflate file_data raise Gem::Package::FormatError, "#{full_name} in #{@gem} is corrupt" if @@ -78,7 +78,7 @@ class Gem::Package::Old < Gem::Package FileUtils.rm_rf destination - FileUtils.mkdir_p File.dirname(destination), :mode => dir_mode && 0o755 + FileUtils.mkdir_p File.dirname(destination), mode: dir_mode && 0o755 File.open destination, "wb", file_mode(entry["mode"]) do |out| out.write file_data diff --git a/lib/rubygems/package/tar_header.rb b/lib/rubygems/package/tar_header.rb index 062a943942..dd5e835a1e 100644 --- a/lib/rubygems/package/tar_header.rb +++ b/lib/rubygems/package/tar_header.rb @@ -95,39 +95,40 @@ class Gem::Package::TarHeader attr_reader(*FIELDS) - EMPTY_HEADER = ("\0" * 512).freeze # :nodoc: + EMPTY_HEADER = ("\0" * 512).b.freeze # :nodoc: ## # Creates a tar header from IO +stream+ def self.from(stream) header = stream.read 512 - empty = (EMPTY_HEADER == header) + return EMPTY if header == EMPTY_HEADER fields = header.unpack UNPACK_FORMAT - new :name => fields.shift, - :mode => strict_oct(fields.shift), - :uid => oct_or_256based(fields.shift), - :gid => oct_or_256based(fields.shift), - :size => strict_oct(fields.shift), - :mtime => strict_oct(fields.shift), - :checksum => strict_oct(fields.shift), - :typeflag => fields.shift, - :linkname => fields.shift, - :magic => fields.shift, - :version => strict_oct(fields.shift), - :uname => fields.shift, - :gname => fields.shift, - :devmajor => strict_oct(fields.shift), - :devminor => strict_oct(fields.shift), - :prefix => fields.shift, - - :empty => empty + new name: fields.shift, + mode: strict_oct(fields.shift), + uid: oct_or_256based(fields.shift), + gid: oct_or_256based(fields.shift), + size: strict_oct(fields.shift), + mtime: strict_oct(fields.shift), + checksum: strict_oct(fields.shift), + typeflag: fields.shift, + linkname: fields.shift, + magic: fields.shift, + version: strict_oct(fields.shift), + uname: fields.shift, + gname: fields.shift, + devmajor: strict_oct(fields.shift), + devminor: strict_oct(fields.shift), + prefix: fields.shift, + + empty: false end def self.strict_oct(str) - return str.strip.oct if /\A[0-7]*\z/.match?(str.strip) + str.strip! + return str.oct if /\A[0-7]*\z/.match?(str) raise ArgumentError, "#{str.inspect} is not an octal string" end @@ -137,7 +138,8 @@ class Gem::Package::TarHeader # \ff flags a negative 256-based number # In case we have a match, parse it as a signed binary value # in big-endian order, except that the high-order bit is ignored. - return str.unpack("N2").last if /\A[\x80\xff]/n.match?(str) + + return str.unpack1("@4N") if /\A[\x80\xff]/n.match?(str) strict_oct(str) end @@ -149,25 +151,43 @@ class Gem::Package::TarHeader raise ArgumentError, ":name, :size, :prefix and :mode required" end - vals[:uid] ||= 0 - vals[:gid] ||= 0 - vals[:mtime] ||= 0 - vals[:checksum] ||= "" - vals[:typeflag] = "0" if vals[:typeflag].nil? || vals[:typeflag].empty? - vals[:magic] ||= "ustar" - vals[:version] ||= "00" - vals[:uname] ||= "wheel" - vals[:gname] ||= "wheel" - vals[:devmajor] ||= 0 - vals[:devminor] ||= 0 - - FIELDS.each do |name| - instance_variable_set "@#{name}", vals[name] - end + @checksum = vals[:checksum] || "" + @devmajor = vals[:devmajor] || 0 + @devminor = vals[:devminor] || 0 + @gid = vals[:gid] || 0 + @gname = vals[:gname] || "wheel" + @linkname = vals[:linkname] + @magic = vals[:magic] || "ustar" + @mode = vals[:mode] + @mtime = vals[:mtime] || 0 + @name = vals[:name] + @prefix = vals[:prefix] + @size = vals[:size] + @typeflag = vals[:typeflag] + @typeflag = "0" if @typeflag.nil? || @typeflag.empty? + @uid = vals[:uid] || 0 + @uname = vals[:uname] || "wheel" + @version = vals[:version] || "00" @empty = vals[:empty] end + EMPTY = new({ # :nodoc: + checksum: 0, + gname: "", + linkname: "", + magic: "", + mode: 0, + name: "", + prefix: "", + size: 0, + uname: "", + version: 0, + + empty: true, + }).freeze + private_constant :EMPTY + ## # Is the tar entry empty? @@ -237,7 +257,7 @@ class Gem::Package::TarHeader header = header.pack PACK_FORMAT - header << ("\0" * ((512 - header.size) % 512)) + header.ljust 512, "\0" end def oct(num, len) diff --git a/lib/rubygems/package/tar_reader.rb b/lib/rubygems/package/tar_reader.rb index b12e83a703..25f9b2f945 100644 --- a/lib/rubygems/package/tar_reader.rb +++ b/lib/rubygems/package/tar_reader.rb @@ -14,11 +14,6 @@ class Gem::Package::TarReader include Enumerable ## - # Raised if the tar IO is not seekable - - class UnexpectedEOF < StandardError; end - - ## # Creates a new TarReader on +io+ and yields it to the block, if given. def self.new(io) @@ -57,7 +52,14 @@ class Gem::Package::TarReader return enum_for __method__ unless block_given? until @io.eof? do - header = Gem::Package::TarHeader.from @io + begin + header = Gem::Package::TarHeader.from @io + rescue ArgumentError => e + # Specialize only exceptions from Gem::Package::TarHeader.strict_oct + raise e unless e.message.match?(/ is not an octal string$/) + raise Gem::Package::TarInvalidError, e.message + end + return if header.empty? entry = Gem::Package::TarReader::Entry.new header, @io yield entry diff --git a/lib/rubygems/package/tar_reader/entry.rb b/lib/rubygems/package/tar_reader/entry.rb index e22efa95b3..5e9d9af5c6 100644 --- a/lib/rubygems/package/tar_reader/entry.rb +++ b/lib/rubygems/package/tar_reader/entry.rb @@ -102,9 +102,7 @@ class Gem::Package::TarReader::Entry # Read one byte from the tar entry def getc - check_closed - - return nil if @read >= @header.size + return nil if eof? ret = @io.getc @read += 1 if ret @@ -156,30 +154,28 @@ class Gem::Package::TarReader::Entry alias_method :length, :size ## - # Reads +len+ bytes from the tar file entry, or the rest of the entry if - # nil - - def read(len = nil) - check_closed - - len ||= @header.size - @read + # Reads +maxlen+ bytes from the tar file entry, or the rest of the entry if nil - return nil if len > 0 && @read >= @header.size + def read(maxlen = nil) + if eof? + return maxlen.to_i.zero? ? "" : nil + end - max_read = [len, @header.size - @read].min + max_read = [maxlen, @header.size - @read].compact.min ret = @io.read max_read + if ret.nil? + return maxlen ? nil : "" # IO.read returns nil on EOF with len argument + end @read += ret.size ret end - def readpartial(maxlen = nil, outbuf = "".b) - check_closed - - maxlen ||= @header.size - @read - - raise EOFError if maxlen > 0 && @read >= @header.size + def readpartial(maxlen, outbuf = "".b) + if eof? && maxlen > 0 + raise EOFError, "end of file reached" + end max_read = [maxlen, @header.size - @read].min @@ -213,6 +209,8 @@ class Gem::Package::TarReader::Entry pending = new_pos - @io.pos + return 0 if pending == 0 + if @io.respond_to?(:seek) begin # avoid reading if the @io supports seeking @@ -230,8 +228,8 @@ class Gem::Package::TarReader::Entry end while pending > 0 do - size_read = @io.read([pending, 4096].min).size - raise UnexpectedEOF if @io.eof? + size_read = @io.read([pending, 4096].min)&.size + raise(EOFError, "end of file reached") if size_read.nil? pending -= size_read end diff --git a/lib/rubygems/package/tar_writer.rb b/lib/rubygems/package/tar_writer.rb index e06ec25961..b24bdb63e7 100644 --- a/lib/rubygems/package/tar_writer.rb +++ b/lib/rubygems/package/tar_writer.rb @@ -116,9 +116,9 @@ class Gem::Package::TarWriter final_pos = @io.pos @io.pos = init_pos - header = Gem::Package::TarHeader.new :name => name, :mode => mode, - :size => size, :prefix => prefix, - :mtime => Gem.source_date_epoch + header = Gem::Package::TarHeader.new name: name, mode: mode, + size: size, prefix: prefix, + mtime: Gem.source_date_epoch @io.write header @io.pos = final_pos @@ -209,9 +209,9 @@ class Gem::Package::TarWriter name, prefix = split_name name - header = Gem::Package::TarHeader.new(:name => name, :mode => mode, - :size => size, :prefix => prefix, - :mtime => Gem.source_date_epoch).to_s + header = Gem::Package::TarHeader.new(name: name, mode: mode, + size: size, prefix: prefix, + mtime: Gem.source_date_epoch).to_s @io.write header os = BoundedStream.new @io, size @@ -235,11 +235,11 @@ class Gem::Package::TarWriter name, prefix = split_name name - header = Gem::Package::TarHeader.new(:name => name, :mode => mode, - :size => 0, :typeflag => "2", - :linkname => target, - :prefix => prefix, - :mtime => Gem.source_date_epoch).to_s + header = Gem::Package::TarHeader.new(name: name, mode: mode, + size: 0, typeflag: "2", + linkname: target, + prefix: prefix, + mtime: Gem.source_date_epoch).to_s @io.write header @@ -289,10 +289,10 @@ class Gem::Package::TarWriter name, prefix = split_name(name) - header = Gem::Package::TarHeader.new :name => name, :mode => mode, - :typeflag => "5", :size => 0, - :prefix => prefix, - :mtime => Gem.source_date_epoch + header = Gem::Package::TarHeader.new name: name, mode: mode, + typeflag: "5", size: 0, + prefix: prefix, + mtime: Gem.source_date_epoch @io.write header diff --git a/lib/rubygems/package_task.rb b/lib/rubygems/package_task.rb index a67d8cb916..d26411684d 100644 --- a/lib/rubygems/package_task.rb +++ b/lib/rubygems/package_task.rb @@ -97,13 +97,13 @@ class Gem::PackageTask < Rake::PackageTask gem_path = File.join package_dir, gem_file gem_dir = File.join package_dir, gem_spec.full_name - task :package => [:gem] + task package: [:gem] directory package_dir directory gem_dir desc "Build the gem file #{gem_file}" - task :gem => [gem_path] + task gem: [gem_path] trace = Rake.application.options.trace Gem.configuration.verbose = trace diff --git a/lib/rubygems/path_support.rb b/lib/rubygems/path_support.rb index b596b4f6b8..13091e29ba 100644 --- a/lib/rubygems/path_support.rb +++ b/lib/rubygems/path_support.rb @@ -24,23 +24,22 @@ class Gem::PathSupport # hashtable, or defaults to ENV, the system environment. # def initialize(env) - @home = env["GEM_HOME"] || Gem.default_dir - - if File::ALT_SEPARATOR - @home = @home.gsub(File::ALT_SEPARATOR, File::SEPARATOR) - end - - @home = expand(@home) - + @home = normalize_home_dir(env["GEM_HOME"] || Gem.default_dir) @path = split_gem_path env["GEM_PATH"], @home @spec_cache_dir = env["GEM_SPEC_CACHE"] || Gem.default_spec_cache_dir - - @spec_cache_dir = @spec_cache_dir.dup.tap(&Gem::UNTAINT) end private + def normalize_home_dir(home) + if File::ALT_SEPARATOR + home = home.gsub(File::ALT_SEPARATOR, File::SEPARATOR) + end + + expand(home) + end + ## # Split the Gem search path (as reported by Gem.path). diff --git a/lib/rubygems/platform.rb b/lib/rubygems/platform.rb index cccfa06ae9..d54ad12880 100644 --- a/lib/rubygems/platform.rb +++ b/lib/rubygems/platform.rb @@ -13,15 +13,22 @@ class Gem::Platform attr_accessor :cpu, :os, :version def self.local - arch = RbConfig::CONFIG["arch"] - arch = "#{arch}_60" if /mswin(?:32|64)$/.match?(arch) - @local ||= new(arch) + @local ||= begin + arch = RbConfig::CONFIG["arch"] + arch = "#{arch}_60" if /mswin(?:32|64)$/.match?(arch) + new(arch) + end end def self.match(platform) match_platforms?(platform, Gem.platforms) end + class << self + extend Gem::Deprecate + rubygems_deprecate :match, "Gem::Platform.match_spec? or match_gem?" + end + def self.match_platforms?(platform, platforms) platform = Gem::Platform.new(platform) unless platform.is_a?(Gem::Platform) platforms.any? do |local_platform| @@ -36,10 +43,20 @@ class Gem::Platform match_gem?(spec.platform, spec.name) end - def self.match_gem?(platform, gem_name) - # NOTE: this method might be redefined by Ruby implementations to - # customize behavior per RUBY_ENGINE, gem_name or other criteria. - match_platforms?(platform, Gem.platforms) + if RUBY_ENGINE == "truffleruby" + def self.match_gem?(platform, gem_name) + raise "Not a string: #{gem_name.inspect}" unless String === gem_name + + if REUSE_AS_BINARY_ON_TRUFFLERUBY.include?(gem_name) + match_platforms?(platform, [Gem::Platform::RUBY, Gem::Platform.local]) + else + match_platforms?(platform, Gem.platforms) + end + end + else + def self.match_gem?(platform, gem_name) + match_platforms?(platform, Gem.platforms) + end end def self.sort_priority(platform) @@ -72,7 +89,7 @@ class Gem::Platform when String then arch = arch.split "-" - if arch.length > 2 && arch.last !~ /\d+(\.\d+)?$/ # reassemble x86-linux-{libc} + if arch.length > 2 && !arch.last.match?(/\d+(\.\d+)?$/) # reassemble x86-linux-{libc} extra = arch.pop arch.last << "-#{extra}" end @@ -84,7 +101,7 @@ class Gem::Platform else cpu end - if arch.length == 2 && arch.last =~ /^\d+(\.\d+)?$/ # for command-line + if arch.length == 2 && arch.last.match?(/^\d+(\.\d+)?$/) # for command-line @os, @version = arch return end @@ -117,6 +134,7 @@ class Gem::Platform when /netbsdelf/ then ["netbsdelf", nil] when /openbsd(\d+\.\d+)?/ then ["openbsd", $1] when /solaris(\d+\.\d+)?/ then ["solaris", $1] + when /wasi/ then ["wasi", nil] # test when /^(\w+_platform)(\d+)?/ then [$1, $2] else ["unknown", nil] diff --git a/lib/rubygems/psych_tree.rb b/lib/rubygems/psych_tree.rb index 2d478c94d9..24857adb9d 100644 --- a/lib/rubygems/psych_tree.rb +++ b/lib/rubygems/psych_tree.rb @@ -14,6 +14,10 @@ module Gem @emitter.scalar str, nil, nil, false, true, quote end + def visit_Hash(o) + super(o.compact) + end + # Noop this out so there are no anchors def register(target, obj) end diff --git a/lib/rubygems/query_utils.rb b/lib/rubygems/query_utils.rb index 9a6a736461..a95a759401 100644 --- a/lib/rubygems/query_utils.rb +++ b/lib/rubygems/query_utils.rb @@ -311,8 +311,8 @@ module Gem::QueryUtils label = "Installed at" specs.each do |s| version = s.version.to_s - version << ", default" if s.default_gem? - entry << "\n" << " #{label} (#{version}): #{s.base_dir}" + default = ", default" if s.default_gem? + entry << "\n" << " #{label} (#{version}#{default}): #{s.base_dir}" label = " " * label.length end end diff --git a/lib/rubygems/remote_fetcher.rb b/lib/rubygems/remote_fetcher.rb index f9f8c45817..c3a41592f6 100644 --- a/lib/rubygems/remote_fetcher.rb +++ b/lib/rubygems/remote_fetcher.rb @@ -74,9 +74,9 @@ class Gem::RemoteFetcher def initialize(proxy=nil, dns=nil, headers={}) require_relative "core_ext/tcpsocket_init" if Gem.configuration.ipv4_fallback_enabled - require "net/http" + require_relative "vendored_net_http" require "stringio" - require "uri" + require_relative "vendor/uri/lib/uri" Socket.do_not_reverse_lookup = true @@ -135,7 +135,7 @@ class Gem::RemoteFetcher scheme = source_uri.scheme - # URI.parse gets confused by MS Windows paths with forward slashes. + # Gem::URI.parse gets confused by MS Windows paths with forward slashes. scheme = nil if /^[a-z]$/i.match?(scheme) # REFACTOR: split this up and dispatch on scheme (eg download_http) @@ -210,17 +210,17 @@ class Gem::RemoteFetcher # HTTP Fetcher. Dispatched by +fetch_path+. Use it instead. def fetch_http(uri, last_modified = nil, head = false, depth = 0) - fetch_type = head ? Net::HTTP::Head : Net::HTTP::Get + fetch_type = head ? Gem::Net::HTTP::Head : Gem::Net::HTTP::Get response = request uri, fetch_type, last_modified do |req| headers.each {|k,v| req.add_field(k,v) } end case response - when Net::HTTPOK, Net::HTTPNotModified then + when Gem::Net::HTTPOK, Gem::Net::HTTPNotModified then response.uri = uri head ? response : response.body - when Net::HTTPMovedPermanently, Net::HTTPFound, Net::HTTPSeeOther, - Net::HTTPTemporaryRedirect then + when Gem::Net::HTTPMovedPermanently, Gem::Net::HTTPFound, Gem::Net::HTTPSeeOther, + Gem::Net::HTTPTemporaryRedirect then raise FetchError.new("too many redirects", uri) if depth > 10 unless location = response["Location"] @@ -261,7 +261,7 @@ class Gem::RemoteFetcher end data - rescue Timeout::Error, IOError, SocketError, SystemCallError, + rescue Gem::Timeout::Error, IOError, SocketError, SystemCallError, *(OpenSSL::SSL::SSLError if Gem::HAVE_OPENSSL) => e raise FetchError.new("#{e.class}: #{e}", uri) end @@ -305,8 +305,8 @@ class Gem::RemoteFetcher end ## - # Performs a Net::HTTP request of type +request_class+ on +uri+ returning - # a Net::HTTP response object. request maintains a table of persistent + # Performs a Gem::Net::HTTP request of type +request_class+ on +uri+ returning + # a Gem::Net::HTTP response object. request maintains a table of persistent # connections to reduce connect overhead. def request(uri, request_class, last_modified = nil) diff --git a/lib/rubygems/request.rb b/lib/rubygems/request.rb index 619a5934b1..9116785231 100644 --- a/lib/rubygems/request.rb +++ b/lib/rubygems/request.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require "net/http" +require_relative "vendored_net_http" require_relative "user_interaction" class Gem::Request @@ -18,11 +18,11 @@ class Gem::Request end def self.proxy_uri(proxy) # :nodoc: - require "uri" + require_relative "vendor/uri/lib/uri" case proxy when :no_proxy then nil - when URI::HTTP then proxy - else URI.parse(proxy) + when Gem::URI::HTTP then proxy + else Gem::URI.parse(proxy) end end @@ -30,7 +30,7 @@ class Gem::Request @uri = uri @request_class = request_class @last_modified = last_modified - @requests = Hash.new 0 + @requests = Hash.new(0).compare_by_identity @user_agent = user_agent @connection_pool = pool @@ -176,7 +176,7 @@ class Gem::Request end require "uri" - uri = URI(Gem::UriFormatter.new(env_proxy).normalize) + uri = Gem::URI(Gem::UriFormatter.new(env_proxy).normalize) if uri && uri.user.nil? && uri.password.nil? user = ENV["#{downcase_scheme}_proxy_user"] || ENV["#{upcase_scheme}_PROXY_USER"] @@ -196,7 +196,7 @@ class Gem::Request bad_response = false begin - @requests[connection.object_id] += 1 + @requests[connection] += 1 verbose "#{request.method} #{Gem::Uri.redact(@uri)}" @@ -205,7 +205,7 @@ class Gem::Request if request.response_body_permitted? && file_name =~ /\.gem$/ reporter = ui.download_reporter response = connection.request(request) do |incomplete_response| - if Net::HTTPOK === incomplete_response + if Gem::Net::HTTPOK === incomplete_response reporter.fetch(file_name, incomplete_response.content_length) downloaded = 0 data = String.new @@ -228,7 +228,7 @@ class Gem::Request end verbose "#{response.code} #{response.message}" - rescue Net::HTTPBadResponse + rescue Gem::Net::HTTPBadResponse verbose "bad response" reset connection @@ -237,17 +237,17 @@ class Gem::Request bad_response = true retry - rescue Net::HTTPFatalError + rescue Gem::Net::HTTPFatalError verbose "fatal error" raise Gem::RemoteFetcher::FetchError.new("fatal error", @uri) - # HACK: work around EOFError bug in Net::HTTP + # HACK: work around EOFError bug in Gem::Net::HTTP # NOTE Errno::ECONNABORTED raised a lot on Windows, and make impossible # to install gems. - rescue EOFError, Timeout::Error, + rescue EOFError, Gem::Timeout::Error, Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE - requests = @requests[connection.object_id] + requests = @requests[connection] verbose "connection reset after #{requests} requests, retrying" raise Gem::RemoteFetcher::FetchError.new("too many connection resets", @uri) if retried @@ -267,7 +267,7 @@ class Gem::Request # Resets HTTP connection +connection+. def reset(connection) - @requests.delete connection.object_id + @requests.delete connection connection.finish connection.start diff --git a/lib/rubygems/request/connection_pools.rb b/lib/rubygems/request/connection_pools.rb index b7ad92f5c6..6c1b04ab65 100644 --- a/lib/rubygems/request/connection_pools.rb +++ b/lib/rubygems/request/connection_pools.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class Gem::Request::ConnectionPools # :nodoc: - @client = Net::HTTP + @client = Gem::Net::HTTP class << self attr_accessor :client diff --git a/lib/rubygems/request_set.rb b/lib/rubygems/request_set.rb index 46e9cc346c..875df7e019 100644 --- a/lib/rubygems/request_set.rb +++ b/lib/rubygems/request_set.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative "tsort" +require_relative "vendored_tsort" ## # A RequestSet groups a request to activate a set of dependencies. @@ -324,7 +324,7 @@ class Gem::RequestSet @git_set.root_dir = @install_dir - lock_file = "#{File.expand_path(path)}.lock".dup.tap(&Gem::UNTAINT) + lock_file = "#{File.expand_path(path)}.lock" begin tokenizer = Gem::RequestSet::Lockfile::Tokenizer.from_file lock_file parser = tokenizer.make_parser self, [] diff --git a/lib/rubygems/request_set/gem_dependency_api.rb b/lib/rubygems/request_set/gem_dependency_api.rb index a7b3c78f6d..4347d22ccb 100644 --- a/lib/rubygems/request_set/gem_dependency_api.rb +++ b/lib/rubygems/request_set/gem_dependency_api.rb @@ -33,22 +33,22 @@ class Gem::RequestSet::GemDependencyAPI ENGINE_MAP = { # :nodoc: - :jruby => %w[jruby], - :jruby_18 => %w[jruby], - :jruby_19 => %w[jruby], - :maglev => %w[maglev], - :mri => %w[ruby], - :mri_18 => %w[ruby], - :mri_19 => %w[ruby], - :mri_20 => %w[ruby], - :mri_21 => %w[ruby], - :rbx => %w[rbx], - :truffleruby => %w[truffleruby], - :ruby => %w[ruby rbx maglev truffleruby], - :ruby_18 => %w[ruby rbx maglev truffleruby], - :ruby_19 => %w[ruby rbx maglev truffleruby], - :ruby_20 => %w[ruby rbx maglev truffleruby], - :ruby_21 => %w[ruby rbx maglev truffleruby], + jruby: %w[jruby], + jruby_18: %w[jruby], + jruby_19: %w[jruby], + maglev: %w[maglev], + mri: %w[ruby], + mri_18: %w[ruby], + mri_19: %w[ruby], + mri_20: %w[ruby], + mri_21: %w[ruby], + rbx: %w[rbx], + truffleruby: %w[truffleruby], + ruby: %w[ruby rbx maglev truffleruby], + ruby_18: %w[ruby rbx maglev truffleruby], + ruby_19: %w[ruby rbx maglev truffleruby], + ruby_20: %w[ruby rbx maglev truffleruby], + ruby_21: %w[ruby rbx maglev truffleruby], }.freeze mswin = Gem::Platform.new "x86-mswin32" @@ -57,39 +57,39 @@ class Gem::RequestSet::GemDependencyAPI x64_mingw = Gem::Platform.new "x64-mingw32" PLATFORM_MAP = { # :nodoc: - :jruby => Gem::Platform::RUBY, - :jruby_18 => Gem::Platform::RUBY, - :jruby_19 => Gem::Platform::RUBY, - :maglev => Gem::Platform::RUBY, - :mingw => x86_mingw, - :mingw_18 => x86_mingw, - :mingw_19 => x86_mingw, - :mingw_20 => x86_mingw, - :mingw_21 => x86_mingw, - :mri => Gem::Platform::RUBY, - :mri_18 => Gem::Platform::RUBY, - :mri_19 => Gem::Platform::RUBY, - :mri_20 => Gem::Platform::RUBY, - :mri_21 => Gem::Platform::RUBY, - :mswin => mswin, - :mswin_18 => mswin, - :mswin_19 => mswin, - :mswin_20 => mswin, - :mswin_21 => mswin, - :mswin64 => mswin64, - :mswin64_19 => mswin64, - :mswin64_20 => mswin64, - :mswin64_21 => mswin64, - :rbx => Gem::Platform::RUBY, - :ruby => Gem::Platform::RUBY, - :ruby_18 => Gem::Platform::RUBY, - :ruby_19 => Gem::Platform::RUBY, - :ruby_20 => Gem::Platform::RUBY, - :ruby_21 => Gem::Platform::RUBY, - :truffleruby => Gem::Platform::RUBY, - :x64_mingw => x64_mingw, - :x64_mingw_20 => x64_mingw, - :x64_mingw_21 => x64_mingw, + jruby: Gem::Platform::RUBY, + jruby_18: Gem::Platform::RUBY, + jruby_19: Gem::Platform::RUBY, + maglev: Gem::Platform::RUBY, + mingw: x86_mingw, + mingw_18: x86_mingw, + mingw_19: x86_mingw, + mingw_20: x86_mingw, + mingw_21: x86_mingw, + mri: Gem::Platform::RUBY, + mri_18: Gem::Platform::RUBY, + mri_19: Gem::Platform::RUBY, + mri_20: Gem::Platform::RUBY, + mri_21: Gem::Platform::RUBY, + mswin: mswin, + mswin_18: mswin, + mswin_19: mswin, + mswin_20: mswin, + mswin_21: mswin, + mswin64: mswin64, + mswin64_19: mswin64, + mswin64_20: mswin64, + mswin64_21: mswin64, + rbx: Gem::Platform::RUBY, + ruby: Gem::Platform::RUBY, + ruby_18: Gem::Platform::RUBY, + ruby_19: Gem::Platform::RUBY, + ruby_20: Gem::Platform::RUBY, + ruby_21: Gem::Platform::RUBY, + truffleruby: Gem::Platform::RUBY, + x64_mingw: x64_mingw, + x64_mingw_20: x64_mingw, + x64_mingw_21: x64_mingw, }.freeze gt_eq_0 = Gem::Requirement.new ">= 0" @@ -99,70 +99,70 @@ class Gem::RequestSet::GemDependencyAPI tilde_gt_2_1_0 = Gem::Requirement.new "~> 2.1.0" VERSION_MAP = { # :nodoc: - :jruby => gt_eq_0, - :jruby_18 => tilde_gt_1_8_0, - :jruby_19 => tilde_gt_1_9_0, - :maglev => gt_eq_0, - :mingw => gt_eq_0, - :mingw_18 => tilde_gt_1_8_0, - :mingw_19 => tilde_gt_1_9_0, - :mingw_20 => tilde_gt_2_0_0, - :mingw_21 => tilde_gt_2_1_0, - :mri => gt_eq_0, - :mri_18 => tilde_gt_1_8_0, - :mri_19 => tilde_gt_1_9_0, - :mri_20 => tilde_gt_2_0_0, - :mri_21 => tilde_gt_2_1_0, - :mswin => gt_eq_0, - :mswin_18 => tilde_gt_1_8_0, - :mswin_19 => tilde_gt_1_9_0, - :mswin_20 => tilde_gt_2_0_0, - :mswin_21 => tilde_gt_2_1_0, - :mswin64 => gt_eq_0, - :mswin64_19 => tilde_gt_1_9_0, - :mswin64_20 => tilde_gt_2_0_0, - :mswin64_21 => tilde_gt_2_1_0, - :rbx => gt_eq_0, - :ruby => gt_eq_0, - :ruby_18 => tilde_gt_1_8_0, - :ruby_19 => tilde_gt_1_9_0, - :ruby_20 => tilde_gt_2_0_0, - :ruby_21 => tilde_gt_2_1_0, - :truffleruby => gt_eq_0, - :x64_mingw => gt_eq_0, - :x64_mingw_20 => tilde_gt_2_0_0, - :x64_mingw_21 => tilde_gt_2_1_0, + jruby: gt_eq_0, + jruby_18: tilde_gt_1_8_0, + jruby_19: tilde_gt_1_9_0, + maglev: gt_eq_0, + mingw: gt_eq_0, + mingw_18: tilde_gt_1_8_0, + mingw_19: tilde_gt_1_9_0, + mingw_20: tilde_gt_2_0_0, + mingw_21: tilde_gt_2_1_0, + mri: gt_eq_0, + mri_18: tilde_gt_1_8_0, + mri_19: tilde_gt_1_9_0, + mri_20: tilde_gt_2_0_0, + mri_21: tilde_gt_2_1_0, + mswin: gt_eq_0, + mswin_18: tilde_gt_1_8_0, + mswin_19: tilde_gt_1_9_0, + mswin_20: tilde_gt_2_0_0, + mswin_21: tilde_gt_2_1_0, + mswin64: gt_eq_0, + mswin64_19: tilde_gt_1_9_0, + mswin64_20: tilde_gt_2_0_0, + mswin64_21: tilde_gt_2_1_0, + rbx: gt_eq_0, + ruby: gt_eq_0, + ruby_18: tilde_gt_1_8_0, + ruby_19: tilde_gt_1_9_0, + ruby_20: tilde_gt_2_0_0, + ruby_21: tilde_gt_2_1_0, + truffleruby: gt_eq_0, + x64_mingw: gt_eq_0, + x64_mingw_20: tilde_gt_2_0_0, + x64_mingw_21: tilde_gt_2_1_0, }.freeze WINDOWS = { # :nodoc: - :mingw => :only, - :mingw_18 => :only, - :mingw_19 => :only, - :mingw_20 => :only, - :mingw_21 => :only, - :mri => :never, - :mri_18 => :never, - :mri_19 => :never, - :mri_20 => :never, - :mri_21 => :never, - :mswin => :only, - :mswin_18 => :only, - :mswin_19 => :only, - :mswin_20 => :only, - :mswin_21 => :only, - :mswin64 => :only, - :mswin64_19 => :only, - :mswin64_20 => :only, - :mswin64_21 => :only, - :rbx => :never, - :ruby => :never, - :ruby_18 => :never, - :ruby_19 => :never, - :ruby_20 => :never, - :ruby_21 => :never, - :x64_mingw => :only, - :x64_mingw_20 => :only, - :x64_mingw_21 => :only, + mingw: :only, + mingw_18: :only, + mingw_19: :only, + mingw_20: :only, + mingw_21: :only, + mri: :never, + mri_18: :never, + mri_19: :never, + mri_20: :never, + mri_21: :never, + mswin: :only, + mswin_18: :only, + mswin_19: :only, + mswin_20: :only, + mswin_21: :only, + mswin64: :only, + mswin64_19: :only, + mswin64_20: :only, + mswin64_21: :only, + rbx: :never, + ruby: :never, + ruby_18: :never, + ruby_19: :never, + ruby_20: :never, + ruby_21: :never, + x64_mingw: :only, + x64_mingw_20: :only, + x64_mingw_21: :only, }.freeze ## @@ -280,7 +280,7 @@ class Gem::RequestSet::GemDependencyAPI # Loads the gem dependency file and returns self. def load - instance_eval File.read(@path).tap(&Gem::UNTAINT), @path, 1 + instance_eval File.read(@path), @path, 1 self end @@ -790,7 +790,7 @@ Gem dependencies file #{@path} includes git reference for both ref/branch and ta return true if @installing - unless RUBY_VERSION == version + unless version == RUBY_VERSION message = "Your Ruby version is #{RUBY_VERSION}, " \ "but your #{gem_deps_file} requires #{version}" diff --git a/lib/rubygems/request_set/lockfile.rb b/lib/rubygems/request_set/lockfile.rb index d9ddc43734..c446b3ae51 100644 --- a/lib/rubygems/request_set/lockfile.rb +++ b/lib/rubygems/request_set/lockfile.rb @@ -76,11 +76,6 @@ class Gem::RequestSet::Lockfile @dependencies = dependencies @gem_deps_file = File.expand_path(gem_deps_file) @gem_deps_dir = File.dirname(@gem_deps_file) - - if RUBY_VERSION < "2.7" - @gem_deps_file.untaint unless gem_deps_file.tainted? - end - @platforms = [] end @@ -109,7 +104,7 @@ class Gem::RequestSet::Lockfile requests.sort_by(&:name).each do |request| next if request.spec.name == "bundler" platform = "-#{request.spec.platform}" unless - Gem::Platform::RUBY == request.spec.platform + request.spec.platform == Gem::Platform::RUBY out << " #{request.name} (#{request.version}#{platform})" diff --git a/lib/rubygems/requirement.rb b/lib/rubygems/requirement.rb index 09497f1a2e..02543cb14a 100644 --- a/lib/rubygems/requirement.rb +++ b/lib/rubygems/requirement.rb @@ -23,12 +23,12 @@ class Gem::Requirement SOURCE_SET_REQUIREMENT = Struct.new(:for_lockfile).new "!" # :nodoc: quoted = OPS.keys.map {|k| Regexp.quote k }.join "|" - PATTERN_RAW = "\\s*(#{quoted})?\\s*(#{Gem::Version::VERSION_PATTERN})\\s*" # :nodoc: + PATTERN_RAW = "\\s*(#{quoted})?\\s*(#{Gem::Version::VERSION_PATTERN})\\s*".freeze # :nodoc: ## # A regular expression that matches a requirement - PATTERN = /\A#{PATTERN_RAW}\z/.freeze + PATTERN = /\A#{PATTERN_RAW}\z/ ## # The default requirement matches any non-prerelease version @@ -284,6 +284,11 @@ class Gem::Requirement def _tilde_requirements @_tilde_requirements ||= _sorted_requirements.select {|r| r.first == "~>" } end + + def initialize_copy(other) # :nodoc: + @requirements = other.requirements.dup + super + end end class Gem::Version diff --git a/lib/rubygems/resolver.rb b/lib/rubygems/resolver.rb index 8b9d41b40f..115c716b6b 100644 --- a/lib/rubygems/resolver.rb +++ b/lib/rubygems/resolver.rb @@ -11,7 +11,7 @@ require_relative "util/list" # all the requirements. class Gem::Resolver - require_relative "resolver/molinillo" + require_relative "vendored_molinillo" ## # If the DEBUG_RESOLVER environment variable is set then debugging mode is @@ -38,8 +38,6 @@ class Gem::Resolver ## # List of dependencies that could not be found in the configured sources. - attr_reader :missing - attr_reader :stats ## @@ -49,8 +47,7 @@ class Gem::Resolver attr_accessor :skip_gems ## - # When a missing dependency, don't stop. Just go on and record what was - # missing. + # attr_accessor :soft_missing @@ -106,7 +103,6 @@ class Gem::Resolver @development = false @development_shallow = false @ignore_dependencies = false - @missing = [] @skip_gems = {} @soft_missing = false @stats = Gem::Resolver::Stats.new @@ -171,7 +167,7 @@ class Gem::Resolver reqs end - include Molinillo::UI + include Gem::Molinillo::UI def output @output ||= debug? ? $stdout : File.open(IO::NULL, "w") @@ -181,15 +177,14 @@ class Gem::Resolver DEBUG_RESOLVER end - include Molinillo::SpecificationProvider + include Gem::Molinillo::SpecificationProvider ## # Proceed with resolution! Returns an array of ActivationRequest objects. def resolve - locking_dg = Molinillo::DependencyGraph.new - Molinillo::Resolver.new(self, self).resolve(@needed.map {|d| DependencyRequest.new d, nil }, locking_dg).tsort.map(&:payload).compact - rescue Molinillo::VersionConflict => e + Gem::Molinillo::Resolver.new(self, self).resolve(@needed.map {|d| DependencyRequest.new d, nil }).tsort.map(&:payload).compact + rescue Gem::Molinillo::VersionConflict => e conflict = e.conflicts.values.first raise Gem::DependencyResolutionError, Conflict.new(conflict.requirement_trees.first.first, conflict.existing, conflict.requirement) ensure @@ -228,7 +223,6 @@ class Gem::Resolver def search_for(dependency) possibles, all = find_possible(dependency) if !@soft_missing && possibles.empty? - @missing << dependency exc = Gem::UnsatisfiableDependencyError.new dependency, all exc.errors = @set.errors raise exc @@ -275,7 +269,6 @@ class Gem::Resolver end def allow_missing?(dependency) - @missing << dependency @soft_missing end diff --git a/lib/rubygems/resolver/api_set.rb b/lib/rubygems/resolver/api_set.rb index e8e3747361..3e4dadc40f 100644 --- a/lib/rubygems/resolver/api_set.rb +++ b/lib/rubygems/resolver/api_set.rb @@ -30,7 +30,7 @@ class Gem::Resolver::APISet < Gem::Resolver::Set def initialize(dep_uri = "https://index.rubygems.org/info/") super() - dep_uri = URI dep_uri unless URI === dep_uri + dep_uri = Gem::URI dep_uri unless Gem::URI === dep_uri @dep_uri = dep_uri @uri = dep_uri + ".." diff --git a/lib/rubygems/resolver/api_set/gem_parser.rb b/lib/rubygems/resolver/api_set/gem_parser.rb index 685c39558d..643b857107 100644 --- a/lib/rubygems/resolver/api_set/gem_parser.rb +++ b/lib/rubygems/resolver/api_set/gem_parser.rb @@ -1,12 +1,15 @@ # frozen_string_literal: true class Gem::Resolver::APISet::GemParser + EMPTY_ARRAY = [].freeze + private_constant :EMPTY_ARRAY + def parse(line) version_and_platform, rest = line.split(" ", 2) version, platform = version_and_platform.split("-", 2) - dependencies, requirements = rest.split("|", 2).map {|s| s.split(",") } if rest - dependencies = dependencies ? dependencies.map {|d| parse_dependency(d) } : [] - requirements = requirements ? requirements.map {|d| parse_dependency(d) } : [] + dependencies, requirements = rest.split("|", 2).map! {|s| s.split(",") } if rest + dependencies = dependencies ? dependencies.map! {|d| parse_dependency(d) } : EMPTY_ARRAY + requirements = requirements ? requirements.map! {|d| parse_dependency(d) } : EMPTY_ARRAY [version, platform, dependencies, requirements] end @@ -15,6 +18,7 @@ class Gem::Resolver::APISet::GemParser def parse_dependency(string) dependency = string.split(":") dependency[-1] = dependency[-1].split("&") if dependency.size > 1 + dependency[0] = -dependency[0] dependency end end diff --git a/lib/rubygems/resolver/api_specification.rb b/lib/rubygems/resolver/api_specification.rb index f26f82757e..a14bcbfeb1 100644 --- a/lib/rubygems/resolver/api_specification.rb +++ b/lib/rubygems/resolver/api_specification.rb @@ -22,7 +22,7 @@ class Gem::Resolver::APISpecification < Gem::Resolver::Specification # Creates an APISpecification for the given +set+ from the rubygems.org # +api_data+. # - # See https://guides.rubygems.org/rubygems-org-api/#misc_methods for the + # See https://guides.rubygems.org/rubygems-org-api/#misc-methods for the # format of the +api_data+. def initialize(set, api_data) diff --git a/lib/rubygems/resolver/best_set.rb b/lib/rubygems/resolver/best_set.rb index c2e8982047..a983f8c6b6 100644 --- a/lib/rubygems/resolver/best_set.rb +++ b/lib/rubygems/resolver/best_set.rb @@ -60,7 +60,7 @@ class Gem::Resolver::BestSet < Gem::Resolver::ComposedSet def replace_failed_api_set(error) # :nodoc: uri = error.original_uri - uri = URI uri unless URI === uri + uri = Gem::URI uri unless Gem::URI === uri uri += "." raise error unless api_set = @sets.find do |set| diff --git a/lib/rubygems/resolver/index_specification.rb b/lib/rubygems/resolver/index_specification.rb index b23807682b..7b95608071 100644 --- a/lib/rubygems/resolver/index_specification.rb +++ b/lib/rubygems/resolver/index_specification.rb @@ -76,7 +76,7 @@ class Gem::Resolver::IndexSpecification < Gem::Resolver::Specification q.breakable q.text full_name - unless Gem::Platform::RUBY == @platform + unless @platform == Gem::Platform::RUBY q.breakable q.text @platform.to_s end diff --git a/lib/rubygems/resolver/installer_set.rb b/lib/rubygems/resolver/installer_set.rb index b0e3241a1a..d9fe36c589 100644 --- a/lib/rubygems/resolver/installer_set.rb +++ b/lib/rubygems/resolver/installer_set.rb @@ -148,6 +148,8 @@ class Gem::Resolver::InstallerSet < Gem::Resolver::Set res << Gem::Resolver::InstalledSpecification.new(self, gemspec) end unless @ignore_installed + matching_local = [] + if consider_local? matching_local = @local.values.select do |spec, _| req.match? spec @@ -169,7 +171,7 @@ class Gem::Resolver::InstallerSet < Gem::Resolver::Set end end - res.concat @remote_set.find_all req if consider_remote? + res.concat @remote_set.find_all req if consider_remote? && matching_local.empty? res end diff --git a/lib/rubygems/resolver/molinillo.rb b/lib/rubygems/resolver/molinillo.rb deleted file mode 100644 index d703505410..0000000000 --- a/lib/rubygems/resolver/molinillo.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -require_relative "molinillo/lib/molinillo" diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/delegates/resolution_state.rb b/lib/rubygems/resolver/molinillo/lib/molinillo/delegates/resolution_state.rb deleted file mode 100644 index d540d3baff..0000000000 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/delegates/resolution_state.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -module Gem::Resolver::Molinillo - # @!visibility private - module Delegates - # Delegates all {Gem::Resolver::Molinillo::ResolutionState} methods to a `#state` property. - module ResolutionState - # (see Gem::Resolver::Molinillo::ResolutionState#name) - def name - current_state = state || Gem::Resolver::Molinillo::ResolutionState.empty - current_state.name - end - - # (see Gem::Resolver::Molinillo::ResolutionState#requirements) - def requirements - current_state = state || Gem::Resolver::Molinillo::ResolutionState.empty - current_state.requirements - end - - # (see Gem::Resolver::Molinillo::ResolutionState#activated) - def activated - current_state = state || Gem::Resolver::Molinillo::ResolutionState.empty - current_state.activated - end - - # (see Gem::Resolver::Molinillo::ResolutionState#requirement) - def requirement - current_state = state || Gem::Resolver::Molinillo::ResolutionState.empty - current_state.requirement - end - - # (see Gem::Resolver::Molinillo::ResolutionState#possibilities) - def possibilities - current_state = state || Gem::Resolver::Molinillo::ResolutionState.empty - current_state.possibilities - end - - # (see Gem::Resolver::Molinillo::ResolutionState#depth) - def depth - current_state = state || Gem::Resolver::Molinillo::ResolutionState.empty - current_state.depth - end - - # (see Gem::Resolver::Molinillo::ResolutionState#conflicts) - def conflicts - current_state = state || Gem::Resolver::Molinillo::ResolutionState.empty - current_state.conflicts - end - - # (see Gem::Resolver::Molinillo::ResolutionState#unused_unwind_options) - def unused_unwind_options - current_state = state || Gem::Resolver::Molinillo::ResolutionState.empty - current_state.unused_unwind_options - end - end - end -end diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/gem_metadata.rb b/lib/rubygems/resolver/molinillo/lib/molinillo/gem_metadata.rb deleted file mode 100644 index 86c249c404..0000000000 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/gem_metadata.rb +++ /dev/null @@ -1,6 +0,0 @@ -# frozen_string_literal: true - -module Gem::Resolver::Molinillo - # The version of Gem::Resolver::Molinillo. - VERSION = '0.8.0'.freeze -end diff --git a/lib/rubygems/resolver/spec_specification.rb b/lib/rubygems/resolver/spec_specification.rb index 79a34d8063..00ef9fdba0 100644 --- a/lib/rubygems/resolver/spec_specification.rb +++ b/lib/rubygems/resolver/spec_specification.rb @@ -66,4 +66,11 @@ class Gem::Resolver::SpecSpecification < Gem::Resolver::Specification def version spec.version end + + ## + # The hash value for this specification. + + def hash + spec.hash + end end diff --git a/lib/rubygems/s3_uri_signer.rb b/lib/rubygems/s3_uri_signer.rb index 20e95345d1..7c95a9d4f5 100644 --- a/lib/rubygems/s3_uri_signer.rb +++ b/lib/rubygems/s3_uri_signer.rb @@ -49,7 +49,7 @@ class Gem::S3URISigner string_to_sign = generate_string_to_sign(date_time, credential_info, canonical_request) signature = generate_signature(s3_config, date, string_to_sign) - URI.parse("https://#{canonical_host}#{uri.path}?#{query_params}&X-Amz-Signature=#{signature}") + Gem::URI.parse("https://#{canonical_host}#{uri.path}?#{query_params}&X-Amz-Signature=#{signature}") end private @@ -140,7 +140,7 @@ class Gem::S3URISigner end def ec2_metadata_credentials_json - require "net/http" + require_relative "vendored_net_http" require_relative "request" require_relative "request/connection_pools" require "json" @@ -152,13 +152,13 @@ class Gem::S3URISigner end def ec2_metadata_request(url) - uri = URI(url) + uri = Gem::URI(url) @request_pool ||= create_request_pool(uri) - request = Gem::Request.new(uri, Net::HTTP::Get, nil, @request_pool) + request = Gem::Request.new(uri, Gem::Net::HTTP::Get, nil, @request_pool) response = request.fetch case response - when Net::HTTPOK then + when Gem::Net::HTTPOK then JSON.parse(response.body) else raise InstanceProfileError.new("Unable to fetch AWS metadata from #{uri}: #{response.message} #{response.code}") diff --git a/lib/rubygems/safe_marshal.rb b/lib/rubygems/safe_marshal.rb new file mode 100644 index 0000000000..b81d1a0a47 --- /dev/null +++ b/lib/rubygems/safe_marshal.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require "stringio" + +require_relative "safe_marshal/reader" +require_relative "safe_marshal/visitors/to_ruby" + +module Gem + ### + # This module is used for safely loading Marshal specs from a gem. The + # `safe_load` method defined on this module is specifically designed for + # loading Gem specifications. + + module SafeMarshal + PERMITTED_CLASSES = %w[ + Date + Time + Rational + + Gem::Dependency + Gem::NameTuple + Gem::Platform + Gem::Requirement + Gem::Specification + Gem::Version + Gem::Version::Requirement + + YAML::Syck::DefaultKey + YAML::PrivateType + ].freeze + private_constant :PERMITTED_CLASSES + + PERMITTED_SYMBOLS = %w[ + development + runtime + + name + number + platform + dependencies + ].freeze + private_constant :PERMITTED_SYMBOLS + + PERMITTED_IVARS = { + "String" => %w[E encoding @taguri @debug_created_info], + "Time" => %w[ + offset zone nano_num nano_den submicro + @_zone @marshal_with_utc_coercion + ], + "Gem::Dependency" => %w[ + @name @requirement @prerelease @version_requirement @version_requirements @type + @force_ruby_platform + ], + "Gem::NameTuple" => %w[@name @version @platform], + "Gem::Platform" => %w[@os @cpu @version], + "Psych::PrivateType" => %w[@value @type_id], + }.freeze + private_constant :PERMITTED_IVARS + + def self.safe_load(input) + load(input, permitted_classes: PERMITTED_CLASSES, permitted_symbols: PERMITTED_SYMBOLS, permitted_ivars: PERMITTED_IVARS) + end + + def self.load(input, permitted_classes: [::Symbol], permitted_symbols: [], permitted_ivars: {}) + root = Reader.new(StringIO.new(input, "r").binmode).read! + + Visitors::ToRuby.new( + permitted_classes: permitted_classes, + permitted_symbols: permitted_symbols, + permitted_ivars: permitted_ivars, + ).visit(root) + end + end +end diff --git a/lib/rubygems/safe_marshal/elements.rb b/lib/rubygems/safe_marshal/elements.rb new file mode 100644 index 0000000000..f8874b1b2f --- /dev/null +++ b/lib/rubygems/safe_marshal/elements.rb @@ -0,0 +1,146 @@ +# frozen_string_literal: true + +module Gem + module SafeMarshal + module Elements + class Element + end + + class Symbol < Element + def initialize(name) + @name = name + end + attr_reader :name + end + + class UserDefined < Element + def initialize(name, binary_string) + @name = name + @binary_string = binary_string + end + + attr_reader :name, :binary_string + end + + class UserMarshal < Element + def initialize(name, data) + @name = name + @data = data + end + + attr_reader :name, :data + end + + class String < Element + def initialize(str) + @str = str + end + + attr_reader :str + end + + class Hash < Element + def initialize(pairs) + @pairs = pairs + end + + attr_reader :pairs + end + + class HashWithDefaultValue < Hash + def initialize(pairs, default) + super(pairs) + @default = default + end + + attr_reader :default + end + + class Array < Element + def initialize(elements) + @elements = elements + end + + attr_reader :elements + end + + class Integer < Element + def initialize(int) + @int = int + end + + attr_reader :int + end + + class True < Element + def initialize + end + TRUE = new.freeze + end + + class False < Element + def initialize + end + + FALSE = new.freeze + end + + class WithIvars < Element + def initialize(object, ivars) + @object = object + @ivars = ivars + end + + attr_reader :object, :ivars + end + + class Object < Element + def initialize(name) + @name = name + end + attr_reader :name + end + + class Nil < Element + NIL = new.freeze + end + + class ObjectLink < Element + def initialize(offset) + @offset = offset + end + attr_reader :offset + end + + class SymbolLink < Element + def initialize(offset) + @offset = offset + end + attr_reader :offset + end + + class Float < Element + def initialize(string) + @string = string + end + attr_reader :string + end + + class Bignum < Element # rubocop:disable Lint/UnifiedInteger + def initialize(sign, data) + @sign = sign + @data = data + end + attr_reader :sign, :data + end + + class UserClass < Element + def initialize(name, wrapped_object) + @name = name + @wrapped_object = wrapped_object + end + attr_reader :name, :wrapped_object + end + end + end +end diff --git a/lib/rubygems/safe_marshal/reader.rb b/lib/rubygems/safe_marshal/reader.rb new file mode 100644 index 0000000000..740be113e5 --- /dev/null +++ b/lib/rubygems/safe_marshal/reader.rb @@ -0,0 +1,308 @@ +# frozen_string_literal: true + +require_relative "elements" + +module Gem + module SafeMarshal + class Reader + class Error < StandardError + end + + class UnsupportedVersionError < Error + end + + class UnconsumedBytesError < Error + end + + class NotImplementedError < Error + end + + class EOFError < Error + end + + def initialize(io) + @io = io + end + + def read! + read_header + root = read_element + raise UnconsumedBytesError unless @io.eof? + root + end + + private + + MARSHAL_VERSION = [Marshal::MAJOR_VERSION, Marshal::MINOR_VERSION].map(&:chr).join.freeze + private_constant :MARSHAL_VERSION + + def read_header + v = @io.read(2) + raise UnsupportedVersionError, "Unsupported marshal version #{v.bytes.map(&:ord).join(".")}, expected #{Marshal::MAJOR_VERSION}.#{Marshal::MINOR_VERSION}" unless v == MARSHAL_VERSION + end + + def read_byte + @io.getbyte + end + + def read_integer + b = read_byte + + case b + when 0x00 + 0 + when 0x01 + read_byte + when 0x02 + read_byte | (read_byte << 8) + when 0x03 + read_byte | (read_byte << 8) | (read_byte << 16) + when 0x04 + read_byte | (read_byte << 8) | (read_byte << 16) | (read_byte << 24) + when 0xFC + read_byte | (read_byte << 8) | (read_byte << 16) | (read_byte << 24) | -0x100000000 + when 0xFD + read_byte | (read_byte << 8) | (read_byte << 16) | -0x1000000 + when 0xFE + read_byte | (read_byte << 8) | -0x10000 + when 0xFF + read_byte | -0x100 + when nil + raise EOFError, "Unexpected EOF" + else + signed = (b ^ 128) - 128 + if b >= 128 + signed + 5 + else + signed - 5 + end + end + end + + def read_element + type = read_byte + case type + when 34 then read_string # ?" + when 48 then read_nil # ?0 + when 58 then read_symbol # ?: + when 59 then read_symbol_link # ?; + when 64 then read_object_link # ?@ + when 70 then read_false # ?F + when 73 then read_object_with_ivars # ?I + when 84 then read_true # ?T + when 85 then read_user_marshal # ?U + when 91 then read_array # ?[ + when 102 then read_float # ?f + when 105 then Elements::Integer.new(read_integer) # ?i + when 108 then read_bignum # ?l + when 111 then read_object # ?o + when 117 then read_user_defined # ?u + when 123 then read_hash # ?{ + when 125 then read_hash_with_default_value # ?} + when 101 then read_extended_object # ?e + when 99 then read_class # ?c + when 109 then read_module # ?m + when 77 then read_class_or_module # ?M + when 100 then read_data # ?d + when 47 then read_regexp # ?/ + when 83 then read_struct # ?S + when 67 then read_user_class # ?C + when nil + raise EOFError, "Unexpected EOF" + else + raise Error, "Unknown marshal type discriminator #{type.chr.inspect} (#{type})" + end + end + + STRING_E_SYMBOL = Elements::Symbol.new("E").freeze + private_constant :STRING_E_SYMBOL + + def read_symbol + len = read_integer + if len == 1 + byte = read_byte + if byte == 69 # ?E + STRING_E_SYMBOL + else + Elements::Symbol.new(byte.chr) + end + else + name = -@io.read(len) + Elements::Symbol.new(name) + end + end + + EMPTY_STRING = Elements::String.new("".b.freeze).freeze + private_constant :EMPTY_STRING + + def read_string + length = read_integer + return EMPTY_STRING if length == 0 + str = @io.read(length) + Elements::String.new(str) + end + + def read_true + Elements::True::TRUE + end + + def read_false + Elements::False::FALSE + end + + def read_user_defined + name = read_element + binary_string = @io.read(read_integer) + Elements::UserDefined.new(name, binary_string) + end + + EMPTY_ARRAY = Elements::Array.new([].freeze).freeze + private_constant :EMPTY_ARRAY + + def read_array + length = read_integer + return EMPTY_ARRAY if length == 0 + elements = Array.new(length) do + read_element + end + Elements::Array.new(elements) + end + + def read_object_with_ivars + object = read_element + ivars = Array.new(read_integer) do + [read_element, read_element] + end + Elements::WithIvars.new(object, ivars) + end + + def read_symbol_link + offset = read_integer + Elements::SymbolLink.new(offset) + end + + def read_user_marshal + name = read_element + data = read_element + Elements::UserMarshal.new(name, data) + end + + # profiling bundle install --full-index shows that + # offset 6 is by far the most common object link, + # so we special case it to avoid allocating a new + # object a third of the time. + # the following are all the object links that + # appear more than 10000 times in my profiling + + OBJECT_LINKS = { + 6 => Elements::ObjectLink.new(6).freeze, + 30 => Elements::ObjectLink.new(30).freeze, + 81 => Elements::ObjectLink.new(81).freeze, + 34 => Elements::ObjectLink.new(34).freeze, + 38 => Elements::ObjectLink.new(38).freeze, + 50 => Elements::ObjectLink.new(50).freeze, + 91 => Elements::ObjectLink.new(91).freeze, + 42 => Elements::ObjectLink.new(42).freeze, + 46 => Elements::ObjectLink.new(46).freeze, + 150 => Elements::ObjectLink.new(150).freeze, + 100 => Elements::ObjectLink.new(100).freeze, + 104 => Elements::ObjectLink.new(104).freeze, + 108 => Elements::ObjectLink.new(108).freeze, + 242 => Elements::ObjectLink.new(242).freeze, + 246 => Elements::ObjectLink.new(246).freeze, + 139 => Elements::ObjectLink.new(139).freeze, + 143 => Elements::ObjectLink.new(143).freeze, + 114 => Elements::ObjectLink.new(114).freeze, + 308 => Elements::ObjectLink.new(308).freeze, + 200 => Elements::ObjectLink.new(200).freeze, + 54 => Elements::ObjectLink.new(54).freeze, + 62 => Elements::ObjectLink.new(62).freeze, + 1_286_245 => Elements::ObjectLink.new(1_286_245).freeze, + }.freeze + private_constant :OBJECT_LINKS + + def read_object_link + offset = read_integer + OBJECT_LINKS[offset] || Elements::ObjectLink.new(offset) + end + + EMPTY_HASH = Elements::Hash.new([].freeze).freeze + private_constant :EMPTY_HASH + + def read_hash + length = read_integer + return EMPTY_HASH if length == 0 + pairs = Array.new(length) do + [read_element, read_element] + end + Elements::Hash.new(pairs) + end + + def read_hash_with_default_value + pairs = Array.new(read_integer) do + [read_element, read_element] + end + default = read_element + Elements::HashWithDefaultValue.new(pairs, default) + end + + def read_object + name = read_element + object = Elements::Object.new(name) + ivars = Array.new(read_integer) do + [read_element, read_element] + end + Elements::WithIvars.new(object, ivars) + end + + def read_nil + Elements::Nil::NIL + end + + def read_float + string = @io.read(read_integer) + Elements::Float.new(string) + end + + def read_bignum + sign = read_byte + data = @io.read(read_integer * 2) + Elements::Bignum.new(sign, data) + end + + def read_extended_object + raise NotImplementedError, "Reading Marshal objects of type extended_object is not implemented" + end + + def read_class + raise NotImplementedError, "Reading Marshal objects of type class is not implemented" + end + + def read_module + raise NotImplementedError, "Reading Marshal objects of type module is not implemented" + end + + def read_class_or_module + raise NotImplementedError, "Reading Marshal objects of type class_or_module is not implemented" + end + + def read_data + raise NotImplementedError, "Reading Marshal objects of type data is not implemented" + end + + def read_regexp + raise NotImplementedError, "Reading Marshal objects of type regexp is not implemented" + end + + def read_struct + raise NotImplementedError, "Reading Marshal objects of type struct is not implemented" + end + + def read_user_class + name = read_element + wrapped_object = read_element + Elements::UserClass.new(name, wrapped_object) + end + end + end +end diff --git a/lib/rubygems/safe_marshal/visitors/stream_printer.rb b/lib/rubygems/safe_marshal/visitors/stream_printer.rb new file mode 100644 index 0000000000..162b36ad05 --- /dev/null +++ b/lib/rubygems/safe_marshal/visitors/stream_printer.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require_relative "visitor" + +module Gem::SafeMarshal + module Visitors + class StreamPrinter < Visitor + def initialize(io, indent: "") + @io = io + @indent = indent + @level = 0 + end + + def visit(target) + @io.write("#{@indent * @level}#{target.class}") + target.instance_variables.each do |ivar| + value = target.instance_variable_get(ivar) + next if Elements::Element === value || Array === value + @io.write(" #{ivar}=#{value.inspect}") + end + @io.write("\n") + begin + @level += 1 + super + ensure + @level -= 1 + end + end + end + end +end diff --git a/lib/rubygems/safe_marshal/visitors/to_ruby.rb b/lib/rubygems/safe_marshal/visitors/to_ruby.rb new file mode 100644 index 0000000000..a9f1d048d4 --- /dev/null +++ b/lib/rubygems/safe_marshal/visitors/to_ruby.rb @@ -0,0 +1,415 @@ +# frozen_string_literal: true + +require_relative "visitor" + +module Gem::SafeMarshal + module Visitors + class ToRuby < Visitor + def initialize(permitted_classes:, permitted_symbols:, permitted_ivars:) + @permitted_classes = permitted_classes + @permitted_symbols = ["E"].concat(permitted_symbols).concat(permitted_classes) + @permitted_ivars = permitted_ivars + + @objects = [] + @symbols = [] + @class_cache = {} + + @stack = ["root"] + @stack_idx = 1 + end + + def inspect # :nodoc: + format("#<%s permitted_classes: %p permitted_symbols: %p permitted_ivars: %p>", + self.class, @permitted_classes, @permitted_symbols, @permitted_ivars) + end + + def visit(target) + stack_idx = @stack_idx + super + ensure + @stack_idx = stack_idx - 1 + end + + private + + def push_stack(element) + @stack[@stack_idx] = element + @stack_idx += 1 + end + + def visit_Gem_SafeMarshal_Elements_Array(a) + array = register_object([]) + + elements = a.elements + size = elements.size + idx = 0 + # not idiomatic, but there's a huge number of IMEMOs allocated here, so we avoid the block + # because this is such a hot path when doing a bundle install with the full index + until idx == size + push_stack idx + array << visit(elements[idx]) + idx += 1 + end + + array + end + + def visit_Gem_SafeMarshal_Elements_Symbol(s) + name = s.name + raise UnpermittedSymbolError.new(symbol: name, stack: formatted_stack) unless @permitted_symbols.include?(name) + visit_symbol_type(s) + end + + def map_ivars(klass, ivars) + stack_idx = @stack_idx + ivars.map.with_index do |(k, v), i| + @stack_idx = stack_idx + + push_stack "ivar_" + push_stack i + k = resolve_ivar(klass, k) + + @stack_idx = stack_idx + push_stack k + + next k, visit(v) + end + end + + def visit_Gem_SafeMarshal_Elements_WithIvars(e) + object_offset = @objects.size + push_stack "object" + object = visit(e.object) + ivars = map_ivars(object.class, e.ivars) + + case e.object + when Elements::UserDefined + if object.class == ::Time + internal = [] + + ivars.reject! do |k, v| + case k + when :offset, :zone, :nano_num, :nano_den, :submicro + internal << [k, v] + true + else + false + end + end + + s = e.object.binary_string + + marshal_string = "\x04\bIu:\tTime".b + marshal_string.concat(s.size + 5) + marshal_string << s + marshal_string.concat(internal.size + 5) + + internal.each do |k, v| + marshal_string.concat(":") + marshal_string.concat(k.size + 5) + marshal_string.concat(k.to_s) + dumped = Marshal.dump(v) + dumped[0, 2] = "" + marshal_string.concat(dumped) + end + + object = @objects[object_offset] = Marshal.load(marshal_string) + end + when Elements::String + enc = nil + + ivars.reject! do |k, v| + case k + when :E + case v + when TrueClass + enc = "UTF-8" + when FalseClass + enc = "US-ASCII" + else + raise FormatError, "Unexpected value for String :E #{v.inspect}" + end + when :encoding + enc = v + else + next false + end + true + end + + object.force_encoding(enc) if enc + end + + ivars.each do |k, v| + object.instance_variable_set k, v + end + object + end + + def visit_Gem_SafeMarshal_Elements_Hash(o) + hash = register_object({}) + + o.pairs.each_with_index do |(k, v), i| + push_stack i + k = visit(k) + push_stack k + hash[k] = visit(v) + end + + hash + end + + def visit_Gem_SafeMarshal_Elements_HashWithDefaultValue(o) + hash = visit_Gem_SafeMarshal_Elements_Hash(o) + push_stack :default + hash.default = visit(o.default) + hash + end + + def visit_Gem_SafeMarshal_Elements_Object(o) + register_object(resolve_class(o.name).allocate) + end + + def visit_Gem_SafeMarshal_Elements_ObjectLink(o) + @objects[o.offset] + end + + def visit_Gem_SafeMarshal_Elements_SymbolLink(o) + @symbols[o.offset] + end + + def visit_Gem_SafeMarshal_Elements_UserDefined(o) + register_object(call_method(resolve_class(o.name), :_load, o.binary_string)) + end + + def visit_Gem_SafeMarshal_Elements_UserMarshal(o) + klass = resolve_class(o.name) + compat = COMPAT_CLASSES.fetch(klass, nil) + idx = @objects.size + object = register_object(call_method(compat || klass, :allocate)) + + push_stack :data + ret = call_method(object, :marshal_load, visit(o.data)) + + if compat + object = @objects[idx] = ret + end + + object + end + + def visit_Gem_SafeMarshal_Elements_Integer(i) + i.int + end + + def visit_Gem_SafeMarshal_Elements_Nil(_) + nil + end + + def visit_Gem_SafeMarshal_Elements_True(_) + true + end + + def visit_Gem_SafeMarshal_Elements_False(_) + false + end + + def visit_Gem_SafeMarshal_Elements_String(s) + register_object(+s.str) + end + + def visit_Gem_SafeMarshal_Elements_Float(f) + case f.string + when "inf" + ::Float::INFINITY + when "-inf" + -::Float::INFINITY + when "nan" + ::Float::NAN + else + f.string.to_f + end + end + + def visit_Gem_SafeMarshal_Elements_Bignum(b) + result = 0 + b.data.each_byte.with_index do |byte, exp| + result += (byte * 2**(exp * 8)) + end + + case b.sign + when 43 # ?+ + result + when 45 # ?- + -result + else + raise FormatError, "Unexpected sign for Bignum #{b.sign.chr.inspect} (#{b.sign})" + end + end + + def visit_Gem_SafeMarshal_Elements_UserClass(r) + if resolve_class(r.name) == ::Hash && r.wrapped_object.is_a?(Elements::Hash) + + hash = register_object({}.compare_by_identity) + + o = r.wrapped_object + o.pairs.each_with_index do |(k, v), i| + push_stack i + k = visit(k) + push_stack k + hash[k] = visit(v) + end + + if o.is_a?(Elements::HashWithDefaultValue) + push_stack :default + hash.default = visit(o.default) + end + + hash + else + raise UnsupportedError.new("Unsupported user class #{resolve_class(r.name)} in marshal stream", stack: formatted_stack) + end + end + + def resolve_class(n) + @class_cache[n] ||= begin + to_s = resolve_symbol_name(n) + raise UnpermittedClassError.new(name: to_s, stack: formatted_stack) unless @permitted_classes.include?(to_s) + visit_symbol_type(n) + begin + ::Object.const_get(to_s) + rescue NameError + raise ArgumentError, "Undefined class #{to_s.inspect}" + end + end + end + + class RationalCompat + def marshal_load(s) + num, den = s + raise ArgumentError, "Expected 2 ints" unless s.size == 2 && num.is_a?(Integer) && den.is_a?(Integer) + Rational(num, den) + end + end + private_constant :RationalCompat + + COMPAT_CLASSES = {}.tap do |h| + h[Rational] = RationalCompat + end.compare_by_identity.freeze + private_constant :COMPAT_CLASSES + + def resolve_ivar(klass, name) + to_s = resolve_symbol_name(name) + + raise UnpermittedIvarError.new(symbol: to_s, klass: klass, stack: formatted_stack) unless @permitted_ivars.fetch(klass.name, [].freeze).include?(to_s) + + visit_symbol_type(name) + end + + def visit_symbol_type(element) + case element + when Elements::Symbol + sym = element.name.to_sym + @symbols << sym + sym + when Elements::SymbolLink + visit_Gem_SafeMarshal_Elements_SymbolLink(element) + end + end + + # This is a hot method, so avoid respond_to? checks on every invocation + if :read.respond_to?(:name) + def resolve_symbol_name(element) + case element + when Elements::Symbol + element.name + when Elements::SymbolLink + visit_Gem_SafeMarshal_Elements_SymbolLink(element).name + else + raise FormatError, "Expected symbol or symbol link, got #{element.inspect} @ #{formatted_stack.join(".")}" + end + end + else + def resolve_symbol_name(element) + case element + when Elements::Symbol + element.name + when Elements::SymbolLink + visit_Gem_SafeMarshal_Elements_SymbolLink(element).to_s + else + raise FormatError, "Expected symbol or symbol link, got #{element.inspect} @ #{formatted_stack.join(".")}" + end + end + end + + def register_object(o) + @objects << o + o + end + + def call_method(receiver, method, *args) + receiver.__send__(method, *args) + rescue NoMethodError => e + raise unless e.receiver == receiver + + raise MethodCallError, "Unable to call #{method.inspect} on #{receiver.inspect}, perhaps it is a class using marshal compat, which is not visible in ruby? #{e}" + end + + def formatted_stack + formatted = [] + @stack[0, @stack_idx].each do |e| + if e.is_a?(Integer) + if formatted.last == "ivar_" + formatted[-1] = "ivar_#{e}" + else + formatted << "[#{e}]" + end + else + formatted << e + end + end + formatted + end + + class Error < StandardError + end + + class UnpermittedSymbolError < Error + def initialize(symbol:, stack:) + @symbol = symbol + @stack = stack + super "Attempting to load unpermitted symbol #{symbol.inspect} @ #{stack.join "."}" + end + end + + class UnpermittedIvarError < Error + def initialize(symbol:, klass:, stack:) + @symbol = symbol + @klass = klass + @stack = stack + super "Attempting to set unpermitted ivar #{symbol.inspect} on object of class #{klass} @ #{stack.join "."}" + end + end + + class UnpermittedClassError < Error + def initialize(name:, stack:) + @name = name + @stack = stack + super "Attempting to load unpermitted class #{name.inspect} @ #{stack.join "."}" + end + end + + class UnsupportedError < Error + def initialize(message, stack:) + super "#{message} @ #{stack.join "."}" + end + end + + class FormatError < Error + end + + class MethodCallError < Error + end + end + end +end diff --git a/lib/rubygems/safe_marshal/visitors/visitor.rb b/lib/rubygems/safe_marshal/visitors/visitor.rb new file mode 100644 index 0000000000..c9a079dc0e --- /dev/null +++ b/lib/rubygems/safe_marshal/visitors/visitor.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gem::SafeMarshal::Visitors + class Visitor + def visit(target) + send DISPATCH.fetch(target.class), target + end + + private + + DISPATCH = Gem::SafeMarshal::Elements.constants.each_with_object({}) do |c, h| + next if c == :Element + + klass = Gem::SafeMarshal::Elements.const_get(c) + h[klass] = :"visit_#{klass.name.gsub("::", "_")}" + h.default = :visit_unknown_element + end.compare_by_identity.freeze + private_constant :DISPATCH + + def visit_unknown_element(e) + raise ArgumentError, "Attempting to visit unknown element #{e.inspect}" + end + + def visit_Gem_SafeMarshal_Elements_Array(target) + target.elements.each {|e| visit(e) } + end + + def visit_Gem_SafeMarshal_Elements_Bignum(target); end + def visit_Gem_SafeMarshal_Elements_False(target); end + def visit_Gem_SafeMarshal_Elements_Float(target); end + + def visit_Gem_SafeMarshal_Elements_Hash(target) + target.pairs.each do |k, v| + visit(k) + visit(v) + end + end + + def visit_Gem_SafeMarshal_Elements_HashWithDefaultValue(target) + visit_Gem_SafeMarshal_Elements_Hash(target) + visit(target.default) + end + + def visit_Gem_SafeMarshal_Elements_Integer(target); end + def visit_Gem_SafeMarshal_Elements_Nil(target); end + + def visit_Gem_SafeMarshal_Elements_Object(target) + visit(target.name) + end + + def visit_Gem_SafeMarshal_Elements_ObjectLink(target); end + def visit_Gem_SafeMarshal_Elements_String(target); end + def visit_Gem_SafeMarshal_Elements_Symbol(target); end + def visit_Gem_SafeMarshal_Elements_SymbolLink(target); end + def visit_Gem_SafeMarshal_Elements_True(target); end + + def visit_Gem_SafeMarshal_Elements_UserDefined(target) + visit(target.name) + end + + def visit_Gem_SafeMarshal_Elements_UserMarshal(target) + visit(target.name) + visit(target.data) + end + + def visit_Gem_SafeMarshal_Elements_WithIvars(target) + visit(target.object) + target.ivars.each do |k, v| + visit(k) + visit(v) + end + end + end +end diff --git a/lib/rubygems/safe_yaml.rb b/lib/rubygems/safe_yaml.rb index dba3cfb16d..6a02a48230 100644 --- a/lib/rubygems/safe_yaml.rb +++ b/lib/rubygems/safe_yaml.rb @@ -25,8 +25,17 @@ module Gem runtime ].freeze + @aliases_enabled = true + def self.aliases_enabled=(value) # :nodoc: + @aliases_enabled = !!value + end + + def self.aliases_enabled? # :nodoc: + @aliases_enabled + end + def self.safe_load(input) - ::Psych.safe_load(input, permitted_classes: PERMITTED_CLASSES, permitted_symbols: PERMITTED_SYMBOLS, aliases: true) + ::Psych.safe_load(input, permitted_classes: PERMITTED_CLASSES, permitted_symbols: PERMITTED_SYMBOLS, aliases: @aliases_enabled) end def self.load(input) diff --git a/lib/rubygems/security.rb b/lib/rubygems/security.rb index 1e8e57b5a9..69ba87b07f 100644 --- a/lib/rubygems/security.rb +++ b/lib/rubygems/security.rb @@ -323,7 +323,7 @@ require_relative "openssl" # == Original author # # Paul Duncan <pabs@pablotron.org> -# http://pablotron.org/ +# https://pablotron.org/ module Gem::Security ## diff --git a/lib/rubygems/security/policies.rb b/lib/rubygems/security/policies.rb index 4180832d85..41f66043ad 100644 --- a/lib/rubygems/security/policies.rb +++ b/lib/rubygems/security/policies.rb @@ -6,12 +6,12 @@ module Gem::Security NoSecurity = Policy.new( "No Security", - :verify_data => false, - :verify_signer => false, - :verify_chain => false, - :verify_root => false, - :only_trusted => false, - :only_signed => false + verify_data: false, + verify_signer: false, + verify_chain: false, + verify_root: false, + only_trusted: false, + only_signed: false ) ## @@ -24,12 +24,12 @@ module Gem::Security AlmostNoSecurity = Policy.new( "Almost No Security", - :verify_data => true, - :verify_signer => false, - :verify_chain => false, - :verify_root => false, - :only_trusted => false, - :only_signed => false + verify_data: true, + verify_signer: false, + verify_chain: false, + verify_root: false, + only_trusted: false, + only_signed: false ) ## @@ -41,12 +41,12 @@ module Gem::Security LowSecurity = Policy.new( "Low Security", - :verify_data => true, - :verify_signer => true, - :verify_chain => false, - :verify_root => false, - :only_trusted => false, - :only_signed => false + verify_data: true, + verify_signer: true, + verify_chain: false, + verify_root: false, + only_trusted: false, + only_signed: false ) ## @@ -60,12 +60,12 @@ module Gem::Security MediumSecurity = Policy.new( "Medium Security", - :verify_data => true, - :verify_signer => true, - :verify_chain => true, - :verify_root => true, - :only_trusted => true, - :only_signed => false + verify_data: true, + verify_signer: true, + verify_chain: true, + verify_root: true, + only_trusted: true, + only_signed: false ) ## @@ -79,12 +79,12 @@ module Gem::Security HighSecurity = Policy.new( "High Security", - :verify_data => true, - :verify_signer => true, - :verify_chain => true, - :verify_root => true, - :only_trusted => true, - :only_signed => true + verify_data: true, + verify_signer: true, + verify_chain: true, + verify_root: true, + only_trusted: true, + only_signed: true ) ## @@ -92,12 +92,12 @@ module Gem::Security SigningPolicy = Policy.new( "Signing Policy", - :verify_data => false, - :verify_signer => true, - :verify_chain => true, - :verify_root => true, - :only_trusted => false, - :only_signed => false + verify_data: false, + verify_signer: true, + verify_chain: true, + verify_root: true, + only_trusted: false, + only_signed: false ) ## diff --git a/lib/rubygems/security/trust_dir.rb b/lib/rubygems/security/trust_dir.rb index ea011e6d65..d23d161cfe 100644 --- a/lib/rubygems/security/trust_dir.rb +++ b/lib/rubygems/security/trust_dir.rb @@ -9,8 +9,8 @@ class Gem::Security::TrustDir # Default permissions for the trust directory and its contents DEFAULT_PERMISSIONS = { - :trust_dir => 0o700, - :trusted_cert => 0o600, + trust_dir: 0o700, + trusted_cert: 0o600, }.freeze ## @@ -111,7 +111,7 @@ class Gem::Security::TrustDir FileUtils.chmod 0o700, @dir else - FileUtils.mkdir_p @dir, :mode => @permissions[:trust_dir] + FileUtils.mkdir_p @dir, mode: @permissions[:trust_dir] end end end diff --git a/lib/rubygems/shellwords.rb b/lib/rubygems/shellwords.rb new file mode 100644 index 0000000000..741dccb363 --- /dev/null +++ b/lib/rubygems/shellwords.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +autoload :Shellwords, "shellwords" diff --git a/lib/rubygems/source.rb b/lib/rubygems/source.rb index 8b3a8828d1..d90e311b65 100644 --- a/lib/rubygems/source.rb +++ b/lib/rubygems/source.rb @@ -12,9 +12,9 @@ class Gem::Source include Gem::Text FILES = { # :nodoc: - :released => "specs", - :latest => "latest_specs", - :prerelease => "prerelease_specs", + released: "specs", + latest: "latest_specs", + prerelease: "prerelease_specs", }.freeze ## @@ -101,7 +101,6 @@ class Gem::Source def cache_dir(uri) # Correct for windows paths escaped_path = uri.path.sub(%r{^/([a-z]):/}i, '/\\1-/') - escaped_path.tap(&Gem::UNTAINT) File.join Gem.spec_cache_dir, "#{uri.host}%#{uri.port}", File.dirname(escaped_path) end @@ -135,8 +134,9 @@ class Gem::Source if File.exist? local_spec spec = Gem.read_binary local_spec + Gem.load_safe_marshal spec = begin - Marshal.load(spec) + Gem::SafeMarshal.safe_load(spec) rescue StandardError nil end @@ -157,8 +157,9 @@ class Gem::Source end end + Gem.load_safe_marshal # TODO: Investigate setting Gem::Specification#loaded_from to a URI - Marshal.load spec + Gem::SafeMarshal.safe_load spec end ## @@ -188,8 +189,9 @@ class Gem::Source spec_dump = fetcher.cache_update_path spec_path, local_file, update_cache? + Gem.load_safe_marshal begin - Gem::NameTuple.from_list Marshal.load(spec_dump) + Gem::NameTuple.from_list Gem::SafeMarshal.safe_load(spec_dump) rescue ArgumentError if update_cache? && !retried FileUtils.rm local_file diff --git a/lib/rubygems/source/git.rb b/lib/rubygems/source/git.rb index 70e8d1b06e..bda63c6844 100644 --- a/lib/rubygems/source/git.rb +++ b/lib/rubygems/source/git.rb @@ -221,14 +221,14 @@ class Gem::Source::Git < Gem::Source end ## - # A hash for the git gem based on the git repository URI. + # A hash for the git gem based on the git repository Gem::URI. def uri_hash # :nodoc: require_relative "../openssl" normalized = - if @repository =~ %r{^\w+://(\w+@)?} - uri = URI(@repository).normalize.to_s.sub %r{/$},"" + if @repository.match?(%r{^\w+://(\w+@)?}) + uri = Gem::URI(@repository).normalize.to_s.sub %r{/$},"" uri.sub(/\A(\w+)/) { $1.downcase } else @repository diff --git a/lib/rubygems/source/local.rb b/lib/rubygems/source/local.rb index 533b0a4d06..d81d8343a8 100644 --- a/lib/rubygems/source/local.rb +++ b/lib/rubygems/source/local.rb @@ -40,10 +40,11 @@ class Gem::Source::Local < Gem::Source Dir["*.gem"].each do |file| pkg = Gem::Package.new(file) + spec = pkg.spec rescue SystemCallError, Gem::Package::FormatError # ignore else - tup = pkg.spec.name_tuple + tup = spec.name_tuple @specs[tup] = [File.expand_path(file), pkg] case type diff --git a/lib/rubygems/source_list.rb b/lib/rubygems/source_list.rb index 9e8a9e16ef..33db64fbc1 100644 --- a/lib/rubygems/source_list.rb +++ b/lib/rubygems/source_list.rb @@ -44,7 +44,7 @@ class Gem::SourceList end ## - # Appends +obj+ to the source list which may be a Gem::Source, URI or URI + # Appends +obj+ to the source list which may be a Gem::Source, Gem::URI or URI # String. def <<(obj) diff --git a/lib/rubygems/spec_fetcher.rb b/lib/rubygems/spec_fetcher.rb index 34e6d7638c..610edf25c9 100644 --- a/lib/rubygems/spec_fetcher.rb +++ b/lib/rubygems/spec_fetcher.rb @@ -69,9 +69,9 @@ class Gem::SpecFetcher @prerelease_specs = {} @caches = { - :latest => @latest_specs, - :prerelease => @prerelease_specs, - :released => @specs, + latest: @latest_specs, + prerelease: @prerelease_specs, + released: @specs, } @fetcher = Gem::RemoteFetcher.fetcher diff --git a/lib/rubygems/specification.rb b/lib/rubygems/specification.rb index abdc1dbf4b..57f9b45cf7 100644 --- a/lib/rubygems/specification.rb +++ b/lib/rubygems/specification.rb @@ -11,6 +11,7 @@ require_relative "deprecate" require_relative "basic_specification" require_relative "stub_specification" require_relative "platform" +require_relative "specification_record" require_relative "util/list" require "rbconfig" @@ -108,7 +109,7 @@ class Gem::Specification < Gem::BasicSpecification @load_cache = {} # :nodoc: @load_cache_mutex = Thread::Mutex.new - VALID_NAME_PATTERN = /\A[a-zA-Z0-9\.\-\_]+\z/.freeze # :nodoc: + VALID_NAME_PATTERN = /\A[a-zA-Z0-9\.\-\_]+\z/ # :nodoc: # :startdoc: @@ -127,35 +128,35 @@ class Gem::Specification < Gem::BasicSpecification # Map of attribute names to default values. @@default_value = { - :authors => [], - :autorequire => nil, - :bindir => "bin", - :cert_chain => [], - :date => nil, - :dependencies => [], - :description => nil, - :email => nil, - :executables => [], - :extensions => [], - :extra_rdoc_files => [], - :files => [], - :homepage => nil, - :licenses => [], - :metadata => {}, - :name => nil, - :platform => Gem::Platform::RUBY, - :post_install_message => nil, - :rdoc_options => [], - :require_paths => ["lib"], - :required_ruby_version => Gem::Requirement.default, - :required_rubygems_version => Gem::Requirement.default, - :requirements => [], - :rubygems_version => Gem::VERSION, - :signing_key => nil, - :specification_version => CURRENT_SPECIFICATION_VERSION, - :summary => nil, - :test_files => [], - :version => nil, + authors: [], + autorequire: nil, + bindir: "bin", + cert_chain: [], + date: nil, + dependencies: [], + description: nil, + email: nil, + executables: [], + extensions: [], + extra_rdoc_files: [], + files: [], + homepage: nil, + licenses: [], + metadata: {}, + name: nil, + platform: Gem::Platform::RUBY, + post_install_message: nil, + rdoc_options: [], + require_paths: ["lib"], + required_ruby_version: Gem::Requirement.default, + required_rubygems_version: Gem::Requirement.default, + requirements: [], + rubygems_version: Gem::VERSION, + signing_key: nil, + specification_version: CURRENT_SPECIFICATION_VERSION, + summary: nil, + test_files: [], + version: nil, }.freeze # rubocop:disable Style/MutableConstant @@ -179,22 +180,12 @@ class Gem::Specification < Gem::BasicSpecification @@default_value[k].nil? end - def self.clear_specs # :nodoc: - @@all = nil - @@stubs = nil - @@stubs_by_name = {} - @@spec_with_requirable_file = {} - @@active_stub_with_requirable_file = {} - end - private_class_method :clear_specs - - clear_specs - # Sentinel object to represent "not found" stubs NOT_FOUND = Struct.new(:to_spec, :this).new # :nodoc: + deprecate_constant :NOT_FOUND # Tracking removed method calls to warn users during build time. - REMOVED_METHODS = [:rubyforge_project=].freeze # :nodoc: + REMOVED_METHODS = [:rubyforge_project=, :mark_version].freeze # :nodoc: def removed_method_calls @removed_method_calls ||= [] end @@ -301,7 +292,7 @@ class Gem::Specification < Gem::BasicSpecification # # Usage: # - # spec.description = <<-EOF + # spec.description = <<~EOF # Rake is a Make-like program implemented in Ruby. Tasks and # dependencies are specified in standard Ruby syntax. # EOF @@ -341,7 +332,7 @@ class Gem::Specification < Gem::BasicSpecification # https://opensource.org/licenses/ approved. # # The most commonly used OSI-approved licenses are MIT and Apache-2.0. - # GitHub also provides a license picker at http://choosealicense.com/. + # GitHub also provides a license picker at https://choosealicense.com/. # # You can also use a custom license file along with your gemspec and specify # a LicenseRef-<idstring>, where idstring is the name of the file containing @@ -426,11 +417,11 @@ class Gem::Specification < Gem::BasicSpecification end ## - # The path in the gem for executable scripts. Usually 'bin' + # The path in the gem for executable scripts. Usually 'exe' # # Usage: # - # spec.bindir = 'bin' + # spec.bindir = 'exe' attr_accessor :bindir @@ -531,13 +522,6 @@ class Gem::Specification < Gem::BasicSpecification attr_reader :required_rubygems_version ## - # The version of RubyGems used to create this gem. - # - # Do not set this, it is set automatically when the gem is packaged. - - attr_accessor :rubygems_version - - ## # The key used to sign this gem. See Gem::Security for details. attr_accessor :signing_key @@ -725,6 +709,21 @@ class Gem::Specification < Gem::BasicSpecification end ###################################################################### + # :section: Read-only attributes + + ## + # The version of RubyGems used to create this gem. + + attr_accessor :rubygems_version + + ## + # The path where this gem installs its extensions. + + def extensions_dir + @extensions_dir ||= super + end + + ###################################################################### # :section: Specification internals ## @@ -762,7 +761,7 @@ class Gem::Specification < Gem::BasicSpecification attr_accessor :specification_version def self._all # :nodoc: - @@all ||= Gem.loaded_specs.values | stubs.map(&:to_spec) + specification_record.all end def self.clear_load_cache # :nodoc: @@ -775,31 +774,14 @@ class Gem::Specification < Gem::BasicSpecification def self.each_gemspec(dirs) # :nodoc: dirs.each do |dir| Gem::Util.glob_files_in_dir("*.gemspec", dir).each do |path| - yield path.tap(&Gem::UNTAINT) + yield path end end end - def self.gemspec_stubs_in(dir, pattern) + def self.gemspec_stubs_in(dir, pattern) # :nodoc: Gem::Util.glob_files_in_dir(pattern, dir).map {|path| yield path }.select(&:valid?) end - private_class_method :gemspec_stubs_in - - def self.installed_stubs(dirs, pattern) - map_stubs(dirs, pattern) do |path, base_dir, gems_dir| - Gem::StubSpecification.gemspec_stub(path, base_dir, gems_dir) - end - end - private_class_method :installed_stubs - - def self.map_stubs(dirs, pattern) # :nodoc: - dirs.flat_map do |dir| - base_dir = File.dirname dir - gems_dir = File.join base_dir, "gems" - gemspec_stubs_in(dir, pattern) {|path| yield path, base_dir, gems_dir } - end - end - private_class_method :map_stubs def self.each_spec(dirs) # :nodoc: each_gemspec(dirs) do |path| @@ -812,13 +794,7 @@ class Gem::Specification < Gem::BasicSpecification # Returns a Gem::StubSpecification for every installed gem def self.stubs - @@stubs ||= begin - pattern = "*.gemspec" - stubs = stubs_for_pattern(pattern, false) - - @@stubs_by_name = stubs.select {|s| Gem::Platform.match_spec? s }.group_by(&:name) - stubs - end + specification_record.stubs end ## @@ -837,13 +813,7 @@ class Gem::Specification < Gem::BasicSpecification # only returns stubs that match Gem.platforms def self.stubs_for(name) - if @@stubs - @@stubs_by_name[name] || [] - else - @@stubs_by_name[name] ||= stubs_for_pattern("#{name}-*.gemspec").select do |s| - s.name == name - end - end + specification_record.stubs_for(name) end ## @@ -851,12 +821,7 @@ class Gem::Specification < Gem::BasicSpecification # optionally filtering out specs not matching the current platform # def self.stubs_for_pattern(pattern, match_platform = true) # :nodoc: - installed_stubs = installed_stubs(Gem::Specification.dirs, pattern) - installed_stubs.select! {|s| Gem::Platform.match_spec? s } if match_platform - stubs = installed_stubs + default_stubs(pattern) - stubs = stubs.uniq(&:full_name) - _resort!(stubs) - stubs + specification_record.stubs_for_pattern(pattern, match_platform) end def self._resort!(specs) # :nodoc: @@ -885,23 +850,14 @@ class Gem::Specification < Gem::BasicSpecification # properly sorted. def self.add_spec(spec) - return if _all.include? spec - - _all << spec - stubs << spec - (@@stubs_by_name[spec.name] ||= []) << spec - - _resort!(@@stubs_by_name[spec.name]) - _resort!(stubs) + specification_record.add_spec(spec) end ## # Removes +spec+ from the known specs. def self.remove_spec(spec) - _all.delete spec.to_spec - stubs.delete spec - (@@stubs_by_name[spec.name] || []).delete spec + specification_record.remove_spec(spec) end ## @@ -915,27 +871,17 @@ class Gem::Specification < Gem::BasicSpecification end ## - # Sets the known specs to +specs+. Not guaranteed to work for you in - # the future. Use at your own risk. Caveat emptor. Doomy doom doom. - # Etc etc. - # - #-- - # Makes +specs+ the known specs - # Listen, time is a river - # Winter comes, code breaks - # - # -- wilsonb + # Sets the known specs to +specs+. def self.all=(specs) - @@stubs_by_name = specs.group_by(&:name) - @@all = @@stubs = specs + specification_record.all = specs end ## # Return full names of all specs in sorted order. def self.all_names - _all.map(&:full_name) + specification_record.all_names end ## @@ -960,9 +906,7 @@ class Gem::Specification < Gem::BasicSpecification # Return the directories that Specification uses to find specs. def self.dirs - @@dirs ||= Gem.path.collect do |dir| - File.join dir.dup.tap(&Gem::UNTAINT), "specifications" - end + @@dirs ||= Gem::SpecificationRecord.dirs_from(Gem.path) end ## @@ -972,7 +916,7 @@ class Gem::Specification < Gem::BasicSpecification def self.dirs=(dirs) reset - @@dirs = Array(dirs).map {|dir| File.join dir, "specifications" } + @@dirs = Gem::SpecificationRecord.dirs_from(Array(dirs)) end extend Enumerable @@ -981,23 +925,15 @@ class Gem::Specification < Gem::BasicSpecification # Enumerate every known spec. See ::dirs= and ::add_spec to set the list of # specs. - def self.each - return enum_for(:each) unless block_given? - - _all.each do |x| - yield x - end + def self.each(&block) + specification_record.each(&block) end ## # Returns every spec that matches +name+ and optional +requirements+. def self.find_all_by_name(name, *requirements) - requirements = Gem::Requirement.default if requirements.empty? - - # TODO: maybe try: find_all { |s| spec === dep } - - Gem::Dependency.new(name, *requirements).matching_specs + specification_record.find_all_by_name(name, *requirements) end ## @@ -1014,8 +950,6 @@ class Gem::Specification < Gem::BasicSpecification def self.find_by_name(name, *requirements) requirements = Gem::Requirement.default if requirements.empty? - # TODO: maybe try: find { |s| spec === dep } - Gem::Dependency.new(name, *requirements).to_spec end @@ -1029,12 +963,7 @@ class Gem::Specification < Gem::BasicSpecification # Return the best specification that contains the file matching +path+. def self.find_by_path(path) - path = path.dup.freeze - spec = @@spec_with_requirable_file[path] ||= stubs.find do |s| - s.contains_requirable_file? path - end || NOT_FOUND - - spec.to_spec + specification_record.find_by_path(path) end ## @@ -1042,19 +971,15 @@ class Gem::Specification < Gem::BasicSpecification # amongst the specs that are not activated. def self.find_inactive_by_path(path) - stub = stubs.find do |s| - next if s.activated? - s.contains_requirable_file? path - end - stub&.to_spec + specification_record.find_inactive_by_path(path) end - def self.find_active_stub_by_path(path) - stub = @@active_stub_with_requirable_file[path] ||= stubs.find do |s| - s.activated? && s.contains_requirable_file?(path) - end || NOT_FOUND + ## + # Return the best specification that contains the file matching +path+, among + # those already activated. - stub.this + def self.find_active_stub_by_path(path) + specification_record.find_active_stub_by_path(path) end ## @@ -1121,14 +1046,14 @@ class Gem::Specification < Gem::BasicSpecification # +prerelease+ is true. def self.latest_specs(prerelease = false) - _latest_specs Gem::Specification.stubs, prerelease + specification_record.latest_specs(prerelease) end ## # Return the latest installed spec for gem +name+. def self.latest_spec_for(name) - latest_specs(true).find {|installed_spec| installed_spec.name == name } + specification_record.latest_spec_for(name) end def self._latest_specs(specs, prerelease = false) # :nodoc: @@ -1154,13 +1079,10 @@ class Gem::Specification < Gem::BasicSpecification spec = @load_cache_mutex.synchronize { @load_cache[file] } return spec if spec - file = file.dup.tap(&Gem::UNTAINT) return unless File.file?(file) code = Gem.open_file(file, "r:UTF-8:-", &:read) - code.tap(&Gem::UNTAINT) - begin spec = eval code, binding, file @@ -1269,7 +1191,7 @@ class Gem::Specification < Gem::BasicSpecification def self.reset @@dirs = nil Gem.pre_reset_hooks.each(&:call) - clear_specs + @specification_record = nil clear_load_cache unresolved = unresolved_deps unless unresolved.empty? @@ -1290,6 +1212,13 @@ class Gem::Specification < Gem::BasicSpecification Gem.post_reset_hooks.each(&:call) end + ## + # Keeps track of all currently known specifications + + def self.specification_record + @specification_record ||= Gem::SpecificationRecord.new(dirs) + end + # DOC: This method needs documented or nodoc'd def self.unresolved_deps @unresolved_deps ||= Hash.new {|h, n| h[n] = Gem::Dependency.new n } @@ -1300,10 +1229,23 @@ class Gem::Specification < Gem::BasicSpecification def self._load(str) Gem.load_yaml + Gem.load_safe_marshal + + yaml_set = false + retry_count = 0 array = begin - Marshal.load str + Gem::SafeMarshal.safe_load str rescue ArgumentError => e + # Avoid an infinite retry loop when the argument error has nothing to do + # with the classes not being defined. + # 1 retry each allowed in case all 3 of + # - YAML + # - YAML::Syck::DefaultKey + # - YAML::PrivateType + # need to be defined + raise if retry_count >= 3 + # # Some very old marshaled specs included references to `YAML::PrivateType` # and `YAML::Syck::DefaultKey` constants due to bugs in the old emitter @@ -1313,17 +1255,23 @@ class Gem::Specification < Gem::BasicSpecification message = e.message raise unless message.include?("YAML::") - Object.const_set "YAML", Psych unless Object.const_defined?(:YAML) + unless Object.const_defined?(:YAML) + Object.const_set "YAML", Psych + yaml_set = true + end if message.include?("YAML::Syck::") YAML.const_set "Syck", YAML unless YAML.const_defined?(:Syck) - YAML::Syck.const_set "DefaultKey", Class.new if message.include?("YAML::Syck::DefaultKey") - elsif message.include?("YAML::PrivateType") + YAML::Syck.const_set "DefaultKey", Class.new if message.include?("YAML::Syck::DefaultKey") && !YAML::Syck.const_defined?(:DefaultKey) + elsif message.include?("YAML::PrivateType") && !YAML.const_defined?(:PrivateType) YAML.const_set "PrivateType", Class.new end + retry_count += 1 retry + ensure + Object.__send__(:remove_const, "YAML") if yaml_set end spec = Gem::Specification.new @@ -1731,7 +1679,7 @@ class Gem::Specification < Gem::BasicSpecification /\A (\d{4})-(\d{2})-(\d{2}) (\s+ \d{2}:\d{2}:\d{2}\.\d+ \s* (Z | [-+]\d\d:\d\d) )? - \Z/x.freeze + \Z/x ## # The date this gem was created @@ -1854,8 +1802,6 @@ class Gem::Specification < Gem::BasicSpecification end def encode_with(coder) # :nodoc: - mark_version - coder.add "name", @name coder.add "version", @version platform = case @original_platform @@ -1870,7 +1816,8 @@ class Gem::Specification < Gem::BasicSpecification attributes = @@attributes.map(&:to_s) - %w[name version platform] attributes.each do |name| - coder.add name, instance_variable_get("@#{name}") + value = instance_variable_get("@#{name}") + coder.add name, value unless value.nil? end end @@ -2055,7 +2002,8 @@ class Gem::Specification < Gem::BasicSpecification end ## - # Duplicates array_attributes from +other_spec+ so state isn't shared. + # Duplicates Array and Gem::Requirement attributes from +other_spec+ so state isn't shared. + # def initialize_copy(other_spec) self.class.array_attributes.each do |name| @@ -2077,6 +2025,9 @@ class Gem::Specification < Gem::BasicSpecification raise e end end + + @required_ruby_version = other_spec.required_ruby_version.dup + @required_rubygems_version = other_spec.required_rubygems_version.dup end def base_dir @@ -2146,13 +2097,6 @@ class Gem::Specification < Gem::BasicSpecification end ## - # Sets the rubygems_version to the current RubyGems version. - - def mark_version - @rubygems_version = Gem::VERSION - end - - ## # Track removed method calls to warn about during build time. # Warn about unknown attributes while loading a spec. @@ -2344,12 +2288,12 @@ class Gem::Specification < Gem::BasicSpecification when Hash then seg = obj.keys.sort.map {|k| "#{k.to_s.dump} => #{obj[k].to_s.dump}" } "{ #{seg.join(", ")} }" - when Gem::Version then obj.to_s.dump + when Gem::Version then ruby_code(obj.to_s) when DateLike then obj.strftime("%Y-%m-%d").dump when Time then obj.strftime("%Y-%m-%d").dump when Numeric then obj.inspect when true, false, nil then obj.inspect - when Gem::Platform then "Gem::Platform.new(#{obj.to_a.inspect})" + when Gem::Platform then "Gem::Platform.new(#{ruby_code obj.to_a})" when Gem::Requirement then list = obj.as_list "Gem::Requirement.new(#{ruby_code(list.size == 1 ? obj.to_s : list)})" @@ -2469,7 +2413,6 @@ class Gem::Specification < Gem::BasicSpecification # still have their default values are omitted. def to_ruby - mark_version result = [] result << "# -*- encoding: utf-8 -*-" result << "#{Gem::StubSpecification::PREFIX}#{name} #{version} #{platform} #{raw_require_paths.join("\0")}" @@ -2514,12 +2457,12 @@ class Gem::Specification < Gem::BasicSpecification end if String === signing_key - result << " s.signing_key = #{signing_key.dump}.freeze" + result << " s.signing_key = #{ruby_code signing_key}" end if @installed_by_version result << nil - result << " s.installed_by_version = \"#{Gem::VERSION}\" if s.respond_to? :installed_by_version" + result << " s.installed_by_version = #{ruby_code Gem::VERSION} if s.respond_to? :installed_by_version" end unless dependencies.empty? @@ -2528,9 +2471,8 @@ class Gem::Specification < Gem::BasicSpecification result << nil dependencies.each do |dep| - req = dep.requirements_list.inspect dep.instance_variable_set :@type, :runtime if dep.type.nil? # HACK - result << " s.add_#{dep.type}_dependency(%q<#{dep.name}>.freeze, #{req})" + result << " s.add_#{dep.type}_dependency(%q<#{dep.name}>.freeze, #{ruby_code dep.requirements_list})" end end @@ -2651,19 +2593,12 @@ class Gem::Specification < Gem::BasicSpecification rubygems_deprecate :validate_permissions ## - # Set the version to +version+, potentially also setting - # required_rubygems_version if +version+ indicates it is a - # prerelease. + # Set the version to +version+. def version=(version) @version = Gem::Version.create(version) return if @version.nil? - # skip to set required_ruby_version when pre-released rubygems. - # It caused to raise CircularDependencyError - if @version.prerelease? && (@name.nil? || @name.strip != "rubygems") - self.required_rubygems_version = "> 1.3.1" - end invalidate_memoized_attributes end @@ -2676,9 +2611,9 @@ class Gem::Specification < Gem::BasicSpecification case ivar when "date" # Force Date to go through the extra coerce logic in date= - self.date = val.tap(&Gem::UNTAINT) + self.date = val else - instance_variable_set "@#{ivar}", val.tap(&Gem::UNTAINT) + instance_variable_set "@#{ivar}", val end end diff --git a/lib/rubygems/specification_policy.rb b/lib/rubygems/specification_policy.rb index da3beee628..812b0f889e 100644 --- a/lib/rubygems/specification_policy.rb +++ b/lib/rubygems/specification_policy.rb @@ -5,20 +5,21 @@ require_relative "user_interaction" class Gem::SpecificationPolicy include Gem::UserInteraction - VALID_NAME_PATTERN = /\A[a-zA-Z0-9\.\-\_]+\z/.freeze # :nodoc: + VALID_NAME_PATTERN = /\A[a-zA-Z0-9\.\-\_]+\z/ # :nodoc: - SPECIAL_CHARACTERS = /\A[#{Regexp.escape('.-_')}]+/.freeze # :nodoc: + SPECIAL_CHARACTERS = /\A[#{Regexp.escape(".-_")}]+/ # :nodoc: - VALID_URI_PATTERN = %r{\Ahttps?:\/\/([^\s:@]+:[^\s:@]*@)?[A-Za-z\d\-]+(\.[A-Za-z\d\-]+)+\.?(:\d{1,5})?([\/?]\S*)?\z}.freeze # :nodoc: + VALID_URI_PATTERN = %r{\Ahttps?:\/\/([^\s:@]+:[^\s:@]*@)?[A-Za-z\d\-]+(\.[A-Za-z\d\-]+)+\.?(:\d{1,5})?([\/?]\S*)?\z} # :nodoc: METADATA_LINK_KEYS = %w[ - bug_tracker_uri - changelog_uri - documentation_uri homepage_uri - mailing_list_uri + changelog_uri source_code_uri + documentation_uri wiki_uri + mailing_list_uri + bug_tracker_uri + download_uri funding_uri ].freeze # :nodoc: @@ -102,10 +103,14 @@ class Gem::SpecificationPolicy validate_dependencies + validate_required_ruby_version + validate_extensions validate_removed_attributes + validate_unique_links + if @warnings > 0 if strict error "specification has warnings" @@ -224,6 +229,12 @@ duplicate dependency on #{dep}, (#{prev.requirement}) use: end end + def validate_required_ruby_version + if @specification.required_ruby_version.requirements == [Gem::Requirement::DefaultRequirement] + warning "make sure you specify the oldest ruby version constraint (like \">= 3.0\") that you want your gem to support by setting the `required_ruby_version` gemspec attribute" + end + end + ## # Issues a warning for each file to be packaged which is world-readable. # @@ -263,7 +274,9 @@ duplicate dependency on #{dep}, (#{prev.requirement}) use: return if rubygems_version == Gem::VERSION - error "expected RubyGems version #{Gem::VERSION}, was #{rubygems_version}" + warning "expected RubyGems version #{Gem::VERSION}, was #{rubygems_version}" + + @specification.rubygems_version = Gem::VERSION end def validate_required_attributes @@ -343,7 +356,7 @@ duplicate dependency on #{dep}, (#{prev.requirement}) use: String end - unless Array === val && val.all? {|x| x.is_a?(klass) } + unless Array === val && val.all? {|x| x.is_a?(klass) || (field == :licenses && x.nil?) } error "#{field} must be an Array of #{klass}" end end @@ -358,6 +371,8 @@ duplicate dependency on #{dep}, (#{prev.requirement}) use: licenses = @specification.licenses licenses.each do |license| + next if license.nil? + if license.length > 64 error "each license must be 64 characters or less" end @@ -368,25 +383,38 @@ duplicate dependency on #{dep}, (#{prev.requirement}) use: licenses = @specification.licenses licenses.each do |license| - next if Gem::Licenses.match?(license) + next if Gem::Licenses.match?(license) || license.nil? + license_id_deprecated = Gem::Licenses.deprecated_license_id?(license) + exception_id_deprecated = Gem::Licenses.deprecated_exception_id?(license) suggestions = Gem::Licenses.suggestions(license) + + if license_id_deprecated + main_message = "License identifier '#{license}' is deprecated" + elsif exception_id_deprecated + main_message = "Exception identifier at '#{license}' is deprecated" + else + main_message = "License identifier '#{license}' is invalid" + end + message = <<-WARNING -license value '#{license}' is invalid. Use a license identifier from -http://spdx.org/licenses or '#{Gem::Licenses::NONSTANDARD}' for a nonstandard license. +#{main_message}. Use an identifier from +https://spdx.org/licenses or '#{Gem::Licenses::NONSTANDARD}' for a nonstandard license, +or set it to nil if you don't want to specify a license. WARNING message += "Did you mean #{suggestions.map {|s| "'#{s}'" }.join(", ")}?\n" unless suggestions.nil? warning(message) end warning <<-WARNING if licenses.empty? -licenses is empty, but is recommended. Use a license identifier from -http://spdx.org/licenses or '#{Gem::Licenses::NONSTANDARD}' for a nonstandard license. +licenses is empty, but is recommended. Use an license identifier from +https://spdx.org/licenses or '#{Gem::Licenses::NONSTANDARD}' for a nonstandard license, +or set it to nil if you don't want to specify a license. WARNING end LAZY = '"FIxxxXME" or "TOxxxDO"'.gsub(/xxx/, "") - LAZY_PATTERN = /\AFI XME|\ATO DO/x.freeze - HOMEPAGE_URI_PATTERN = /\A[a-z][a-z\d+.-]*:/i.freeze + LAZY_PATTERN = /\AFI XME|\ATO DO/x + HOMEPAGE_URI_PATTERN = /\A[a-z][a-z\d+.-]*:/i def validate_lazy_metadata unless @specification.authors.grep(LAZY_PATTERN).empty? @@ -409,13 +437,13 @@ http://spdx.org/licenses or '#{Gem::Licenses::NONSTANDARD}' for a nonstandard li # Make sure a homepage is valid HTTP/HTTPS URI if homepage && !homepage.empty? - require "uri" + require_relative "vendor/uri/lib/uri" begin - homepage_uri = URI.parse(homepage) - unless [URI::HTTP, URI::HTTPS].member? homepage_uri.class + homepage_uri = Gem::URI.parse(homepage) + unless [Gem::URI::HTTP, Gem::URI::HTTPS].member? homepage_uri.class error "\"#{homepage}\" is not a valid HTTP URI" end - rescue URI::InvalidURIError + rescue Gem::URI::InvalidURIError error "\"#{homepage}\" is not a valid HTTP URI" end end @@ -479,13 +507,29 @@ You have specified rust based extension, but Cargo.lock is not part of the gem f def validate_rake_extensions(builder) # :nodoc: rake_extension = @specification.extensions.any? {|s| builder.builder_for(s) == Gem::Ext::RakeBuilder } - rake_dependency = @specification.dependencies.any? {|d| d.name == "rake" } + rake_dependency = @specification.dependencies.any? {|d| d.name == "rake" && d.type == :runtime } warning <<-WARNING if rake_extension && !rake_dependency -You have specified rake based extension, but rake is not added as dependency. It is recommended to add rake as a dependency in gemspec since there's no guarantee rake will be already installed. +You have specified rake based extension, but rake is not added as runtime dependency. It is recommended to add rake as a runtime dependency in gemspec since there's no guarantee rake will be already installed. WARNING end + def validate_unique_links + links = @specification.metadata.slice(*METADATA_LINK_KEYS) + grouped = links.group_by {|_key, uri| uri } + grouped.each do |uri, copies| + next unless copies.length > 1 + keys = copies.map(&:first).join("\n ") + warning <<~WARNING + You have specified the uri: + #{uri} + for all of the following keys: + #{keys} + Only the first one will be shown on rubygems.org + WARNING + end + end + def warning(statement) # :nodoc: @warnings += 1 diff --git a/lib/rubygems/specification_record.rb b/lib/rubygems/specification_record.rb new file mode 100644 index 0000000000..dd6aa7eafa --- /dev/null +++ b/lib/rubygems/specification_record.rb @@ -0,0 +1,213 @@ +# frozen_string_literal: true + +module Gem + class SpecificationRecord + def self.dirs_from(paths) + paths.map do |path| + File.join(path, "specifications") + end + end + + def self.from_path(path) + new(dirs_from([path])) + end + + def initialize(dirs) + @all = nil + @stubs = nil + @stubs_by_name = {} + @spec_with_requirable_file = {} + @active_stub_with_requirable_file = {} + + @dirs = dirs + end + + # Sentinel object to represent "not found" stubs + NOT_FOUND = Struct.new(:to_spec, :this).new + private_constant :NOT_FOUND + + ## + # Returns the list of all specifications in the record + + def all + @all ||= Gem.loaded_specs.values | stubs.map(&:to_spec) + end + + ## + # Returns a Gem::StubSpecification for every specification in the record + + def stubs + @stubs ||= begin + pattern = "*.gemspec" + stubs = stubs_for_pattern(pattern, false) + + @stubs_by_name = stubs.select {|s| Gem::Platform.match_spec? s }.group_by(&:name) + stubs + end + end + + ## + # Returns a Gem::StubSpecification for every specification in the record + # named +name+ only returns stubs that match Gem.platforms + + def stubs_for(name) + if @stubs + @stubs_by_name[name] || [] + else + @stubs_by_name[name] ||= stubs_for_pattern("#{name}-*.gemspec").select do |s| + s.name == name + end + end + end + + ## + # Finds stub specifications matching a pattern in the record, optionally + # filtering out specs not matching the current platform + + def stubs_for_pattern(pattern, match_platform = true) + installed_stubs = installed_stubs(pattern) + installed_stubs.select! {|s| Gem::Platform.match_spec? s } if match_platform + stubs = installed_stubs + Gem::Specification.default_stubs(pattern) + stubs = stubs.uniq(&:full_name) + Gem::Specification._resort!(stubs) + stubs + end + + ## + # Adds +spec+ to the the record, keeping the collection properly sorted. + + def add_spec(spec) + return if all.include? spec + + all << spec + stubs << spec + (@stubs_by_name[spec.name] ||= []) << spec + + Gem::Specification._resort!(@stubs_by_name[spec.name]) + Gem::Specification._resort!(stubs) + end + + ## + # Removes +spec+ from the record. + + def remove_spec(spec) + all.delete spec.to_spec + stubs.delete spec + (@stubs_by_name[spec.name] || []).delete spec + end + + ## + # Sets the specs known by the record to +specs+. + + def all=(specs) + @stubs_by_name = specs.group_by(&:name) + @all = @stubs = specs + end + + ## + # Return full names of all specs in the record in sorted order. + + def all_names + all.map(&:full_name) + end + + include Enumerable + + ## + # Enumerate every known spec. + + def each + return enum_for(:each) unless block_given? + + all.each do |x| + yield x + end + end + + ## + # Returns every spec in the record that matches +name+ and optional +requirements+. + + def find_all_by_name(name, *requirements) + req = Gem::Requirement.create(*requirements) + env_req = Gem.env_requirement(name) + + matches = stubs_for(name).find_all do |spec| + req.satisfied_by?(spec.version) && env_req.satisfied_by?(spec.version) + end.map(&:to_spec) + + if name == "bundler" && !req.specific? + require_relative "bundler_version_finder" + Gem::BundlerVersionFinder.prioritize!(matches) + end + + matches + end + + ## + # Return the best specification in the record that contains the file matching +path+. + + def find_by_path(path) + path = path.dup.freeze + spec = @spec_with_requirable_file[path] ||= stubs.find do |s| + s.contains_requirable_file? path + end || NOT_FOUND + + spec.to_spec + end + + ## + # Return the best specification in the record that contains the file + # matching +path+ amongst the specs that are not activated. + + def find_inactive_by_path(path) + stub = stubs.find do |s| + next if s.activated? + s.contains_requirable_file? path + end + stub&.to_spec + end + + ## + # Return the best specification in the record that contains the file + # matching +path+, among those already activated. + + def find_active_stub_by_path(path) + stub = @active_stub_with_requirable_file[path] ||= stubs.find do |s| + s.activated? && s.contains_requirable_file?(path) + end || NOT_FOUND + + stub.this + end + + ## + # Return the latest specs in the record, optionally including prerelease + # specs if +prerelease+ is true. + + def latest_specs(prerelease) + Gem::Specification._latest_specs stubs, prerelease + end + + ## + # Return the latest installed spec in the record for gem +name+. + + def latest_spec_for(name) + latest_specs(true).find {|installed_spec| installed_spec.name == name } + end + + private + + def installed_stubs(pattern) + map_stubs(pattern) do |path, base_dir, gems_dir| + Gem::StubSpecification.gemspec_stub(path, base_dir, gems_dir) + end + end + + def map_stubs(pattern) + @dirs.flat_map do |dir| + base_dir = File.dirname dir + gems_dir = File.join base_dir, "gems" + Gem::Specification.gemspec_stubs_in(dir, pattern) {|path| yield path, base_dir, gems_dir } + end + end + end +end diff --git a/lib/rubygems/stub_specification.rb b/lib/rubygems/stub_specification.rb index a3793eaa20..58748df5d6 100644 --- a/lib/rubygems/stub_specification.rb +++ b/lib/rubygems/stub_specification.rb @@ -35,7 +35,7 @@ class Gem::StubSpecification < Gem::BasicSpecification def initialize(data, extensions) parts = data[PREFIX.length..-1].split(" ", 4) - @name = parts[0].freeze + @name = -parts[0] @version = if Gem::Version.correct?(parts[1]) Gem::Version.new(parts[1]) else @@ -69,7 +69,6 @@ class Gem::StubSpecification < Gem::BasicSpecification def initialize(filename, base_dir, gems_dir, default_gem) super() - filename.tap(&Gem::UNTAINT) self.loaded_from = filename @data = nil @@ -113,14 +112,19 @@ class Gem::StubSpecification < Gem::BasicSpecification Gem.open_file loaded_from, OPEN_MODE do |file| file.readline # discard encoding line - stubline = file.readline.chomp + stubline = file.readline if stubline.start_with?(PREFIX) - extensions = if /\A#{PREFIX}/ =~ file.readline.chomp - $'.split "\0" - else - StubLine::NO_EXTENSIONS - end + extline = file.readline + extensions = + if extline.delete_prefix!(PREFIX) + extline.chomp! + extline.split "\0" + else + StubLine::NO_EXTENSIONS + end + + stubline.chomp! # readline(chomp: true) allocates 3x as much as .readline.chomp! @data = StubLine.new stubline, extensions end rescue EOFError diff --git a/lib/rubygems/tsort.rb b/lib/rubygems/tsort.rb deleted file mode 100644 index 60ebe22e81..0000000000 --- a/lib/rubygems/tsort.rb +++ /dev/null @@ -1,3 +0,0 @@ -# frozen_string_literal: true - -require_relative "tsort/lib/tsort" diff --git a/lib/rubygems/uninstaller.rb b/lib/rubygems/uninstaller.rb index ac5fbfc8cf..4d72f6fd0a 100644 --- a/lib/rubygems/uninstaller.rb +++ b/lib/rubygems/uninstaller.rb @@ -32,7 +32,7 @@ class Gem::Uninstaller attr_reader :bin_dir ## - # The gem repository the gem will be installed into + # The gem repository the gem will be uninstalled from attr_reader :gem_home @@ -49,8 +49,9 @@ class Gem::Uninstaller # TODO: document the valid options @gem = gem @version = options[:version] || Gem::Requirement.default - @gem_home = File.realpath(options[:install_dir] || Gem.dir) - @plugins_dir = Gem.plugindir(@gem_home) + @install_dir = options[:install_dir] + @gem_home = File.realpath(@install_dir || Gem.dir) + @user_dir = File.exist?(Gem.user_dir) ? File.realpath(Gem.user_dir) : Gem.user_dir @force_executables = options[:executables] @force_all = options[:all] @force_ignore = options[:ignore] @@ -70,7 +71,7 @@ class Gem::Uninstaller # only add user directory if install_dir is not set @user_install = false - @user_install = options[:user_install] unless options[:install_dir] + @user_install = options[:user_install] unless @install_dir # Optimization: populated during #uninstall @default_specs_matching_uninstall_params = [] @@ -105,7 +106,7 @@ class Gem::Uninstaller list, other_repo_specs = list.partition do |spec| @gem_home == spec.base_dir || - (@user_install && spec.base_dir == Gem.user_dir) + (@user_install && spec.base_dir == @user_dir) end list.sort! @@ -133,7 +134,7 @@ class Gem::Uninstaller if index == list.size remove_all list - elsif index >= 0 && index < list.size + elsif index && index >= 0 && index < list.size uninstall_gem list[index] else say "Error: must enter a number [1-#{list.size + 1}]" @@ -239,7 +240,7 @@ class Gem::Uninstaller def remove(spec) unless path_ok?(@gem_home, spec) || - (@user_install && path_ok?(Gem.user_dir, spec)) + (@user_install && path_ok?(@user_dir, spec)) e = Gem::GemNotInHomeException.new \ "Gem '#{spec.full_name}' is not installed in directory #{@gem_home}" e.spec = spec @@ -284,17 +285,18 @@ class Gem::Uninstaller def remove_plugins(spec) # :nodoc: return if spec.plugins.empty? - remove_plugins_for(spec, @plugins_dir) + remove_plugins_for(spec, plugin_dir_for(spec)) end ## # Regenerates plugin wrappers after removal. def regenerate_plugins - latest = Gem::Specification.latest_spec_for(@spec.name) + specification_record = @install_dir ? Gem::SpecificationRecord.from_path(@install_dir) : Gem::Specification.specification_record + latest = specification_record.latest_spec_for(@spec.name) return if latest.nil? - regenerate_plugins_for(latest, @plugins_dir) + regenerate_plugins_for(latest, plugin_dir_for(@spec)) end ## @@ -406,4 +408,8 @@ class Gem::Uninstaller say "Gem #{spec.full_name} cannot be uninstalled because it is a default gem" end end + + def plugin_dir_for(spec) + Gem.plugindir(spec.base_dir) + end end diff --git a/lib/rubygems/update_suggestion.rb b/lib/rubygems/update_suggestion.rb index 4cf085fd4c..6f3ec5f493 100644 --- a/lib/rubygems/update_suggestion.rb +++ b/lib/rubygems/update_suggestion.rb @@ -4,15 +4,6 @@ # Mixin methods for Gem::Command to promote available RubyGems update module Gem::UpdateSuggestion - # list taken from https://github.com/watson/ci-info/blob/7a3c30d/index.js#L56-L66 - CI_ENV_VARS = [ - "CI", # Travis CI, CircleCI, Cirrus CI, Gitlab CI, Appveyor, CodeShip, dsari - "CONTINUOUS_INTEGRATION", # Travis CI, Cirrus CI - "BUILD_NUMBER", # Jenkins, TeamCity - "CI_APP_ID", "CI_BUILD_ID", "CI_BUILD_NUMBER", # Applfow - "RUN_ID" # TaskCluster, dsari - ].freeze - ONE_WEEK = 7 * 24 * 60 * 60 ## @@ -28,9 +19,9 @@ Run `gem update --system #{Gem.latest_rubygems_version}` to update your installa end ## - # Determines if current environment is eglible for update suggestion. + # Determines if current environment is eligible for update suggestion. - def eglible_for_update? + def eligible_for_update? # explicit opt-out return false if Gem.configuration[:prevent_update_suggestion] return false if ENV["RUBYGEMS_PREVENT_UPDATE_SUGGESTION"] @@ -39,7 +30,7 @@ Run `gem update --system #{Gem.latest_rubygems_version}` to update your installa return false unless Gem.ui.tty? return false if Gem.rubygems_version.prerelease? return false if Gem.disable_system_update_message - return false if ci? + return false if Gem::CIDetector.ci? # check makes sense only when we can store timestamp of last try # otherwise we will not be able to prevent "annoying" update message @@ -53,17 +44,13 @@ Run `gem update --system #{Gem.latest_rubygems_version}` to update your installa # compare current and latest version, this is the part where # latest rubygems spec is fetched from remote - (Gem.rubygems_version < Gem.latest_rubygems_version).tap do |eglible| + (Gem.rubygems_version < Gem.latest_rubygems_version).tap do |eligible| # store the time of last successful check into state file Gem.configuration.last_update_check = check_time - return eglible + return eligible end rescue StandardError # don't block install command on any problem false end - - def ci? - CI_ENV_VARS.any? {|var| ENV.include?(var) } - end end diff --git a/lib/rubygems/uri.rb b/lib/rubygems/uri.rb index 4b5d035aa0..a44aaceba5 100644 --- a/lib/rubygems/uri.rb +++ b/lib/rubygems/uri.rb @@ -16,9 +16,9 @@ class Gem::Uri # Parses uri, raising if it's invalid def self.parse!(uri) - require "uri" + require_relative "vendor/uri/lib/uri" - raise URI::InvalidURIError unless uri + raise Gem::URI::InvalidURIError unless uri return uri unless uri.is_a?(String) @@ -28,9 +28,9 @@ class Gem::Uri # as "%7BDESede%7D". If this is escaped again the percentage # symbols will be escaped. begin - URI.parse(uri) - rescue URI::InvalidURIError - URI.parse(URI::DEFAULT_PARSER.escape(uri)) + Gem::URI.parse(uri) + rescue Gem::URI::InvalidURIError + Gem::URI.parse(Gem::URI::DEFAULT_PARSER.escape(uri)) end end @@ -39,7 +39,7 @@ class Gem::Uri def self.parse(uri) parse!(uri) - rescue URI::InvalidURIError + rescue Gem::URI::InvalidURIError uri end diff --git a/lib/rubygems/user_interaction.rb b/lib/rubygems/user_interaction.rb index 9f0538a6b0..0172c4ee89 100644 --- a/lib/rubygems/user_interaction.rb +++ b/lib/rubygems/user_interaction.rb @@ -193,7 +193,7 @@ class Gem::StreamUI # then special operations (like asking for passwords) will use the TTY # commands to disable character echo. - def initialize(in_stream, out_stream, err_stream=STDERR, usetty=true) + def initialize(in_stream, out_stream, err_stream=$stderr, usetty=true) @ins = in_stream @outs = out_stream @errs = err_stream @@ -237,6 +237,7 @@ class Gem::StreamUI return nil, nil unless result result = result.strip.to_i - 1 + return nil, nil unless (0...list.size) === result [list[result], result] end @@ -590,8 +591,8 @@ class Gem::StreamUI end ## -# Subclass of StreamUI that instantiates the user interaction using STDIN, -# STDOUT, and STDERR. +# Subclass of StreamUI that instantiates the user interaction using $stdin, +# $stdout, and $stderr. class Gem::ConsoleUI < Gem::StreamUI ## @@ -599,7 +600,7 @@ class Gem::ConsoleUI < Gem::StreamUI # stdin, output to stdout and warnings or errors to stderr. def initialize - super STDIN, STDOUT, STDERR, true + super $stdin, $stdout, $stderr, true end end diff --git a/lib/rubygems/util.rb b/lib/rubygems/util.rb index 5c65dcc424..51f9c2029f 100644 --- a/lib/rubygems/util.rb +++ b/lib/rubygems/util.rb @@ -60,7 +60,7 @@ module Gem::Util # Invokes system, but silences all output. def self.silent_system(*command) - opt = { :out => IO::NULL, :err => [:child, :out] } + opt = { out: IO::NULL, err: [:child, :out] } if Hash === command.last opt.update(command.last) cmds = command[0...-1] @@ -105,11 +105,11 @@ module Gem::Util end ## - # Corrects +path+ (usually returned by `URI.parse().path` on Windows), that + # Corrects +path+ (usually returned by `Gem::URI.parse().path` on Windows), that # comes with a leading slash. def self.correct_for_windows_path(path) - if path[0].chr == "/" && path[1].chr =~ /[a-z]/i && path[2].chr == ":" + if path[0].chr == "/" && path[1].chr.match?(/[a-z]/i) && path[2].chr == ":" path[1..-1] else path diff --git a/lib/rubygems/util/licenses.rb b/lib/rubygems/util/licenses.rb index ab5485d020..192ae30b9b 100644 --- a/lib/rubygems/util/licenses.rb +++ b/lib/rubygems/util/licenses.rb @@ -15,6 +15,7 @@ class Gem::Licenses # license identifiers LICENSE_IDENTIFIERS = %w[ 0BSD + 3D-Slicer-1.0 AAL ADSL AFL-1.1 @@ -22,14 +23,14 @@ class Gem::Licenses AFL-2.0 AFL-2.1 AFL-3.0 - AGPL-1.0 AGPL-1.0-only AGPL-1.0-or-later - AGPL-3.0 AGPL-3.0-only AGPL-3.0-or-later + AMD-newlib AMDPLPA AML + AML-glslang AMPAS ANTLR-PD ANTLR-PD-fallback @@ -39,10 +40,14 @@ class Gem::Licenses APSL-1.1 APSL-1.2 APSL-2.0 + ASWF-Digital-Assets-1.0 + ASWF-Digital-Assets-1.1 Abstyles AdaCore-doc Adobe-2006 + Adobe-Display-PostScript Adobe-Glyph + Adobe-Utopia Afmparse Aladdin Apache-1.0 @@ -56,13 +61,14 @@ class Gem::Licenses Artistic-2.0 BSD-1-Clause BSD-2-Clause - BSD-2-Clause-FreeBSD - BSD-2-Clause-NetBSD + BSD-2-Clause-Darwin BSD-2-Clause-Patent BSD-2-Clause-Views + BSD-2-Clause-first-lines BSD-3-Clause BSD-3-Clause-Attribution BSD-3-Clause-Clear + BSD-3-Clause-HP BSD-3-Clause-LBNL BSD-3-Clause-Modification BSD-3-Clause-No-Military-License @@ -70,6 +76,9 @@ class Gem::Licenses BSD-3-Clause-No-Nuclear-License-2014 BSD-3-Clause-No-Nuclear-Warranty BSD-3-Clause-Open-MPI + BSD-3-Clause-Sun + BSD-3-Clause-acpica + BSD-3-Clause-flex BSD-4-Clause BSD-4-Clause-Shortened BSD-4-Clause-UC @@ -77,8 +86,12 @@ class Gem::Licenses BSD-4.3TAHOE BSD-Advertising-Acknowledgement BSD-Attribution-HPND-disclaimer + BSD-Inferno-Nettverk BSD-Protection BSD-Source-Code + BSD-Source-beginning-file + BSD-Systemics + BSD-Systemics-W3Works BSL-1.0 BUSL-1.1 Baekmuk @@ -90,7 +103,9 @@ class Gem::Licenses Bitstream-Charter Bitstream-Vera BlueOak-1.0.0 + Boehm-GC Borceux + Brian-Gladman-2-Clause Brian-Gladman-3-Clause C-UDA-1.0 CAL-1.0 @@ -102,6 +117,7 @@ class Gem::Licenses CC-BY-2.5-AU CC-BY-3.0 CC-BY-3.0-AT + CC-BY-3.0-AU CC-BY-3.0-DE CC-BY-3.0-IGO CC-BY-3.0-NL @@ -144,6 +160,7 @@ class Gem::Licenses CC-BY-SA-3.0 CC-BY-SA-3.0-AT CC-BY-SA-3.0-DE + CC-BY-SA-3.0-IGO CC-BY-SA-4.0 CC-PDDC CC0-1.0 @@ -166,6 +183,7 @@ class Gem::Licenses CERN-OHL-W-2.0 CFITSIO CMU-Mach + CMU-Mach-nodoc CNRI-Jython CNRI-Python CNRI-Python-GPL-Compatible @@ -175,18 +193,24 @@ class Gem::Licenses CPOL-1.02 CUA-OPL-1.0 Caldera + Caldera-no-preamble + Catharon ClArtistic Clips Community-Spec-1.0 Condor-1.1 Cornell-Lossless-JPEG + Cronyx Crossword CrystalStacker Cube D-FSL-1.0 + DEC-3-Clause DL-DE-BY-2.0 + DL-DE-ZERO-2.0 DOC DRL-1.0 + DRL-1.1 DSDP Dotseqn ECL-1.0 @@ -204,32 +228,34 @@ class Gem::Licenses Entessa ErlPL-1.1 Eurosym + FBM FDK-AAC FSFAP + FSFAP-no-warranty-disclaimer FSFUL FSFULLR FSFULLRWD FTL Fair + Ferguson-Twofish Frameworx-1.0 FreeBSD-DOC FreeImage + Furuseth + GCR-docs GD - GFDL-1.1 GFDL-1.1-invariants-only GFDL-1.1-invariants-or-later GFDL-1.1-no-invariants-only GFDL-1.1-no-invariants-or-later GFDL-1.1-only GFDL-1.1-or-later - GFDL-1.2 GFDL-1.2-invariants-only GFDL-1.2-invariants-or-later GFDL-1.2-no-invariants-only GFDL-1.2-no-invariants-or-later GFDL-1.2-only GFDL-1.2-or-later - GFDL-1.3 GFDL-1.3-invariants-only GFDL-1.3-invariants-or-later GFDL-1.3-no-invariants-only @@ -238,35 +264,42 @@ class Gem::Licenses GFDL-1.3-or-later GL2PS GLWTPL - GPL-1.0 - GPL-1.0+ GPL-1.0-only GPL-1.0-or-later - GPL-2.0 - GPL-2.0+ GPL-2.0-only GPL-2.0-or-later - GPL-2.0-with-GCC-exception - GPL-2.0-with-autoconf-exception - GPL-2.0-with-bison-exception - GPL-2.0-with-classpath-exception - GPL-2.0-with-font-exception - GPL-3.0 - GPL-3.0+ GPL-3.0-only GPL-3.0-or-later - GPL-3.0-with-GCC-exception - GPL-3.0-with-autoconf-exception Giftware Glide Glulxe Graphics-Gems + Gutmann HP-1986 + HP-1989 HPND + HPND-DEC + HPND-Fenneberg-Livingston + HPND-INRIA-IMAG + HPND-Intel + HPND-Kevlin-Henney + HPND-MIT-disclaimer HPND-Markus-Kuhn + HPND-Pbmplus + HPND-UC + HPND-UC-export-US + HPND-doc + HPND-doc-sell HPND-export-US + HPND-export-US-acknowledgement + HPND-export-US-modify + HPND-export2-US + HPND-merchantability-variant + HPND-sell-MIT-disclaimer-xserver + HPND-sell-regexpr HPND-sell-variant HPND-sell-variant-MIT-disclaimer + HPND-sell-variant-MIT-disclaimer-rev HTMLTIDY HaskellReport Hippocratic-2.1 @@ -278,9 +311,11 @@ class Gem::Licenses IPA IPL-1.0 ISC + ISC-Veillard ImageMagick Imlib2 Info-ZIP + Inner-Net-2.0 Intel Intel-ACPI Interbase-1.0 @@ -289,24 +324,20 @@ class Gem::Licenses JSON Jam JasPer-2.0 + Kastrup Kazlib Knuth-CTAN LAL-1.2 LAL-1.3 - LGPL-2.0 - LGPL-2.0+ LGPL-2.0-only LGPL-2.0-or-later - LGPL-2.1 - LGPL-2.1+ LGPL-2.1-only LGPL-2.1-or-later - LGPL-3.0 - LGPL-3.0+ LGPL-3.0-only LGPL-3.0-or-later LGPLLR LOOP + LPD-document LPL-1.0 LPL-1.02 LPPL-1.0 @@ -317,23 +348,33 @@ class Gem::Licenses LZMA-SDK-9.11-to-9.20 LZMA-SDK-9.22 Latex2e + Latex2e-translated-notice Leptonica LiLiQ-P-1.1 LiLiQ-R-1.1 LiLiQ-Rplus-1.1 Libpng Linux-OpenIB + Linux-man-pages-1-para Linux-man-pages-copyleft + Linux-man-pages-copyleft-2-para + Linux-man-pages-copyleft-var + Lucida-Bitmap-Fonts MIT MIT-0 MIT-CMU + MIT-Festival + MIT-Khronos-old MIT-Modern-Variant MIT-Wu MIT-advertising MIT-enna MIT-feh MIT-open-group + MIT-testregex MITNFA + MMIXware + MPEG-SSG MPL-1.0 MPL-1.1 MPL-2.0 @@ -342,8 +383,11 @@ class Gem::Licenses MS-PL MS-RL MTLL + Mackerras-3-Clause + Mackerras-3-Clause-acknowledgment MakeIndex Martin-Birgmeier + McPhee-slideshow Minpack MirOS Motosoto @@ -354,12 +398,15 @@ class Gem::Licenses NAIST-2003 NASA-1.3 NBPL-1.0 + NCBI-PD NCGL-UK-2.0 + NCL NCSA NGPL NICTA-1.0 NIST-PD NIST-PD-fallback + NIST-Software NLOD-1.0 NLOD-2.0 NLPL @@ -376,8 +423,8 @@ class Gem::Licenses Newsletr Nokia Noweb - Nunit O-UDA-1.0 + OAR OCCT-PL OCLC-2.0 ODC-By-1.0 @@ -412,8 +459,10 @@ class Gem::Licenses OLDAP-2.6 OLDAP-2.7 OLDAP-2.8 + OLFL-1.3 OML OPL-1.0 + OPL-UK-3.0 OPUBL-1.0 OSET-PL-2.1 OSL-1.0 @@ -423,12 +472,17 @@ class Gem::Licenses OSL-3.0 OpenPBS-2.3 OpenSSL + OpenSSL-standalone + OpenVision + PADL PDDL-1.0 PHP-3.0 PHP-3.01 + PPL PSF-2.0 Parity-6.0.0 Parity-7.0.0 + Pixar Plexus PolyForm-Noncommercial-1.0.0 PolyForm-Small-Business-1.0.0 @@ -447,20 +501,25 @@ class Gem::Licenses Rdisc Ruby SAX-PD + SAX-PD-2.0 SCEA SGI-B-1.0 SGI-B-1.1 SGI-B-2.0 + SGI-OpenGL + SGP4 SHL-0.5 SHL-0.51 SISSL SISSL-1.2 + SL SMLNJ SMPPL SNIA SPL-1.0 SSH-OpenSSH SSH-short + SSLeay-standalone SSPL-1.0 SWL Saxpath @@ -469,30 +528,39 @@ class Gem::Licenses Sendmail-8.23 SimPL-2.0 Sleepycat + Soundex Spencer-86 Spencer-94 Spencer-99 - StandardML-NJ SugarCRM-1.1.3 + Sun-PPP + Sun-PPP-2000 SunPro Symlinks TAPR-OHL-1.0 TCL TCP-wrappers + TGPPL-1.0 TMate TORQUE-1.1 TOSL TPDL TPL-1.0 TTWL + TTYP0 TU-Berlin-1.0 TU-Berlin-2.0 + TermReadKey UCAR UCL-1.0 + UMich-Merit UPL-1.0 + URT-RLE + Unicode-3.0 Unicode-DFS-2015 Unicode-DFS-2016 Unicode-TOU + UnixCrypt Unlicense VOSTROM VSL-1.0 @@ -502,12 +570,15 @@ class Gem::Licenses W3C-20150513 WTFPL Watcom-1.0 + Widget-Workshop Wsuipa X11 X11-distribute-modifications-variant XFree86-1.1 XSkat + Xdebug-1.03 Xerox + Xfig Xnet YPL-1.0 YPL-1.1 @@ -515,49 +586,109 @@ class Gem::Licenses ZPL-2.0 ZPL-2.1 Zed + Zeeff Zend-2.0 Zimbra-1.3 Zimbra-1.4 Zlib + any-OSI + bcrypt-Solar-Designer blessing - bzip2-1.0.5 bzip2-1.0.6 + check-cvs checkmk copyleft-next-0.3.0 copyleft-next-0.3.1 curl + cve-tou diffmark + dtoa dvipdfm - eCos-2.0 eGenix etalab-2.0 + fwlw gSOAP-1.3b gnuplot + gtkbook + hdparm iMatix libpng-2.0 libselinux-1.0 libtiff libutil-David-Nugent + lsof + magaz + mailprio + metamail mpi-permissive mpich2 mplus + pkgconf + pnmstitch psfrag psutils + python-ldap + radvd snprintf + softSurfer + ssh-keyscan + swrule + threeparttable + ulem w3m - wxWindows xinetd + xkeyboard-config-Zinoviev xlock xpp + xzoom zlib-acknowledgement ].freeze + DEPRECATED_LICENSE_IDENTIFIERS = %w[ + AGPL-1.0 + AGPL-3.0 + BSD-2-Clause-FreeBSD + BSD-2-Clause-NetBSD + GFDL-1.1 + GFDL-1.2 + GFDL-1.3 + GPL-1.0 + GPL-1.0+ + GPL-2.0 + GPL-2.0+ + GPL-2.0-with-GCC-exception + GPL-2.0-with-autoconf-exception + GPL-2.0-with-bison-exception + GPL-2.0-with-classpath-exception + GPL-2.0-with-font-exception + GPL-3.0 + GPL-3.0+ + GPL-3.0-with-GCC-exception + GPL-3.0-with-autoconf-exception + LGPL-2.0 + LGPL-2.0+ + LGPL-2.1 + LGPL-2.1+ + LGPL-3.0 + LGPL-3.0+ + Nunit + StandardML-NJ + bzip2-1.0.5 + eCos-2.0 + wxWindows + ].freeze + # exception identifiers EXCEPTION_IDENTIFIERS = %w[ 389-exception + Asterisk-exception + Asterisk-linking-protocols-exception Autoconf-exception-2.0 Autoconf-exception-3.0 Autoconf-exception-generic + Autoconf-exception-generic-3.0 + Autoconf-exception-macro + Bison-exception-1.24 Bison-exception-2.2 Bootloader-exception CLISP-exception-2.0 @@ -567,45 +698,64 @@ class Gem::Licenses Fawkes-Runtime-exception Font-exception-2.0 GCC-exception-2.0 + GCC-exception-2.0-note GCC-exception-3.1 GNAT-exception + GNOME-examples-exception + GNU-compiler-exception + GPL-3.0-interface-exception GPL-3.0-linking-exception GPL-3.0-linking-source-exception GPL-CC-1.0 GStreamer-exception-2005 GStreamer-exception-2008 + Gmsh-exception KiCad-libraries-exception LGPL-3.0-linking-exception + LLGPL LLVM-exception LZMA-exception Libtool-exception Linux-syscall-note - Nokia-Qt-exception-1.1 OCCT-exception-1.0 OCaml-LGPL-linking-exception OpenJDK-assembly-exception-1.0 + PCRE2-exception PS-or-PDF-font-exception-20170817 QPL-1.0-INRIA-2004-exception Qt-GPL-exception-1.0 Qt-LGPL-exception-1.1 Qwt-exception-1.0 + RRDtool-FLOSS-exception-2.0 + SANE-exception SHL-2.0 SHL-2.1 SWI-exception Swift-exception + Texinfo-exception + UBDL-exception Universal-FOSS-exception-1.0 WxWindows-exception-3.1 + cryptsetup-OpenSSL-exception eCos-exception-2.0 + fmt-exception freertos-exception-2.0 gnu-javamail-exception i2p-gpl-java-exception + libpri-OpenH323-exception mif-exception openvpn-openssl-exception + stunnel-exception u-boot-exception-2.0 + vsftpd-openssl-exception x11vnc-openssl-exception ].freeze - REGEXP = / + DEPRECATED_EXCEPTION_IDENTIFIERS = %w[ + Nokia-Qt-exception-1.1 + ].freeze + + VALID_REGEXP = / \A (?: #{Regexp.union(LICENSE_IDENTIFIERS)} @@ -615,10 +765,34 @@ class Gem::Licenses | #{LICENSE_REF} ) \Z - /ox.freeze + /ox + + DEPRECATED_LICENSE_REGEXP = / + \A + #{Regexp.union(DEPRECATED_LICENSE_IDENTIFIERS)} + \+? + (?:\s WITH \s .+?)? + \Z + /ox + + DEPRECATED_EXCEPTION_REGEXP = / + \A + .+? + \+? + (?:\s WITH \s #{Regexp.union(DEPRECATED_EXCEPTION_IDENTIFIERS)}) + \Z + /ox def self.match?(license) - REGEXP.match?(license) + VALID_REGEXP.match?(license) + end + + def self.deprecated_license_id?(license) + DEPRECATED_LICENSE_REGEXP.match?(license) + end + + def self.deprecated_exception_id?(license) + DEPRECATED_EXCEPTION_REGEXP.match?(license) end def self.suggestions(license) diff --git a/lib/rubygems/vendor/molinillo/.document b/lib/rubygems/vendor/molinillo/.document new file mode 100644 index 0000000000..0c43bbd6b3 --- /dev/null +++ b/lib/rubygems/vendor/molinillo/.document @@ -0,0 +1 @@ +# Vendored files do not need to be documented diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo.rb b/lib/rubygems/vendor/molinillo/lib/molinillo.rb index f67badbde7..dd5600c9e3 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo.rb @@ -6,6 +6,6 @@ require_relative 'molinillo/resolver' require_relative 'molinillo/modules/ui' require_relative 'molinillo/modules/specification_provider' -# Gem::Resolver::Molinillo is a generic dependency resolution algorithm. -module Gem::Resolver::Molinillo +# Gem::Molinillo is a generic dependency resolution algorithm. +module Gem::Molinillo end diff --git a/lib/rubygems/vendor/molinillo/lib/molinillo/delegates/resolution_state.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/delegates/resolution_state.rb new file mode 100644 index 0000000000..34842d46d5 --- /dev/null +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/delegates/resolution_state.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gem::Molinillo + # @!visibility private + module Delegates + # Delegates all {Gem::Molinillo::ResolutionState} methods to a `#state` property. + module ResolutionState + # (see Gem::Molinillo::ResolutionState#name) + def name + current_state = state || Gem::Molinillo::ResolutionState.empty + current_state.name + end + + # (see Gem::Molinillo::ResolutionState#requirements) + def requirements + current_state = state || Gem::Molinillo::ResolutionState.empty + current_state.requirements + end + + # (see Gem::Molinillo::ResolutionState#activated) + def activated + current_state = state || Gem::Molinillo::ResolutionState.empty + current_state.activated + end + + # (see Gem::Molinillo::ResolutionState#requirement) + def requirement + current_state = state || Gem::Molinillo::ResolutionState.empty + current_state.requirement + end + + # (see Gem::Molinillo::ResolutionState#possibilities) + def possibilities + current_state = state || Gem::Molinillo::ResolutionState.empty + current_state.possibilities + end + + # (see Gem::Molinillo::ResolutionState#depth) + def depth + current_state = state || Gem::Molinillo::ResolutionState.empty + current_state.depth + end + + # (see Gem::Molinillo::ResolutionState#conflicts) + def conflicts + current_state = state || Gem::Molinillo::ResolutionState.empty + current_state.conflicts + end + + # (see Gem::Molinillo::ResolutionState#unused_unwind_options) + def unused_unwind_options + current_state = state || Gem::Molinillo::ResolutionState.empty + current_state.unused_unwind_options + end + end + end +end diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/delegates/specification_provider.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/delegates/specification_provider.rb index b765226fb0..8417721537 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/delegates/specification_provider.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/delegates/specification_provider.rb @@ -1,67 +1,67 @@ # frozen_string_literal: true -module Gem::Resolver::Molinillo +module Gem::Molinillo module Delegates - # Delegates all {Gem::Resolver::Molinillo::SpecificationProvider} methods to a + # Delegates all {Gem::Molinillo::SpecificationProvider} methods to a # `#specification_provider` property. module SpecificationProvider - # (see Gem::Resolver::Molinillo::SpecificationProvider#search_for) + # (see Gem::Molinillo::SpecificationProvider#search_for) def search_for(dependency) with_no_such_dependency_error_handling do specification_provider.search_for(dependency) end end - # (see Gem::Resolver::Molinillo::SpecificationProvider#dependencies_for) + # (see Gem::Molinillo::SpecificationProvider#dependencies_for) def dependencies_for(specification) with_no_such_dependency_error_handling do specification_provider.dependencies_for(specification) end end - # (see Gem::Resolver::Molinillo::SpecificationProvider#requirement_satisfied_by?) + # (see Gem::Molinillo::SpecificationProvider#requirement_satisfied_by?) def requirement_satisfied_by?(requirement, activated, spec) with_no_such_dependency_error_handling do specification_provider.requirement_satisfied_by?(requirement, activated, spec) end end - # (see Gem::Resolver::Molinillo::SpecificationProvider#dependencies_equal?) + # (see Gem::Molinillo::SpecificationProvider#dependencies_equal?) def dependencies_equal?(dependencies, other_dependencies) with_no_such_dependency_error_handling do specification_provider.dependencies_equal?(dependencies, other_dependencies) end end - # (see Gem::Resolver::Molinillo::SpecificationProvider#name_for) + # (see Gem::Molinillo::SpecificationProvider#name_for) def name_for(dependency) with_no_such_dependency_error_handling do specification_provider.name_for(dependency) end end - # (see Gem::Resolver::Molinillo::SpecificationProvider#name_for_explicit_dependency_source) + # (see Gem::Molinillo::SpecificationProvider#name_for_explicit_dependency_source) def name_for_explicit_dependency_source with_no_such_dependency_error_handling do specification_provider.name_for_explicit_dependency_source end end - # (see Gem::Resolver::Molinillo::SpecificationProvider#name_for_locking_dependency_source) + # (see Gem::Molinillo::SpecificationProvider#name_for_locking_dependency_source) def name_for_locking_dependency_source with_no_such_dependency_error_handling do specification_provider.name_for_locking_dependency_source end end - # (see Gem::Resolver::Molinillo::SpecificationProvider#sort_dependencies) + # (see Gem::Molinillo::SpecificationProvider#sort_dependencies) def sort_dependencies(dependencies, activated, conflicts) with_no_such_dependency_error_handling do specification_provider.sort_dependencies(dependencies, activated, conflicts) end end - # (see Gem::Resolver::Molinillo::SpecificationProvider#allow_missing?) + # (see Gem::Molinillo::SpecificationProvider#allow_missing?) def allow_missing?(dependency) with_no_such_dependency_error_handling do specification_provider.allow_missing?(dependency) diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph.rb index 731a9e3e90..2dbbc589dc 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -require_relative '../../../../tsort' +require_relative '../../../../vendored_tsort' require_relative 'dependency_graph/log' require_relative 'dependency_graph/vertex' -module Gem::Resolver::Molinillo +module Gem::Molinillo # A directed acyclic graph that is tuned to hold named dependencies class DependencyGraph include Enumerable diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/action.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/action.rb index cc140031b3..8707ec451d 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/action.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/action.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gem::Resolver::Molinillo +module Gem::Molinillo class DependencyGraph # An action that modifies a {DependencyGraph} that is reversible. # @abstract diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/add_edge_no_circular.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/add_edge_no_circular.rb index 5570483253..aa9815c5ae 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/add_edge_no_circular.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/add_edge_no_circular.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require_relative 'action' -module Gem::Resolver::Molinillo +module Gem::Molinillo class DependencyGraph # @!visibility private # (see DependencyGraph#add_edge_no_circular) diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/add_vertex.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/add_vertex.rb index f1411d5efa..9c7066a669 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/add_vertex.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/add_vertex.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require_relative 'action' -module Gem::Resolver::Molinillo +module Gem::Molinillo class DependencyGraph # @!visibility private # (see DependencyGraph#add_vertex) diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/delete_edge.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/delete_edge.rb index 3b48d77a50..1e62c0a0b6 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/delete_edge.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/delete_edge.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require_relative 'action' -module Gem::Resolver::Molinillo +module Gem::Molinillo class DependencyGraph # @!visibility private # (see DependencyGraph#delete_edge) diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/detach_vertex_named.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/detach_vertex_named.rb index 92f60d5be8..6132f969b9 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/detach_vertex_named.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/detach_vertex_named.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require_relative 'action' -module Gem::Resolver::Molinillo +module Gem::Molinillo class DependencyGraph # @!visibility private # @see DependencyGraph#detach_vertex_named diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/log.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/log.rb index 7aeb8847ec..6954c4b1f8 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/log.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/log.rb @@ -7,7 +7,7 @@ require_relative 'detach_vertex_named' require_relative 'set_payload' require_relative 'tag' -module Gem::Resolver::Molinillo +module Gem::Molinillo class DependencyGraph # A log for dependency graph actions class Log diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/set_payload.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/set_payload.rb index 726292a2c3..9bcaaae0f9 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/set_payload.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/set_payload.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require_relative 'action' -module Gem::Resolver::Molinillo +module Gem::Molinillo class DependencyGraph # @!visibility private # @see DependencyGraph#set_payload diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/tag.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/tag.rb index bfe6fd31f8..62f243a2af 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/tag.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/tag.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require_relative 'action' -module Gem::Resolver::Molinillo +module Gem::Molinillo class DependencyGraph # @!visibility private # @see DependencyGraph#tag diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/vertex.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/vertex.rb index 77114951b2..074de369be 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/dependency_graph/vertex.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/dependency_graph/vertex.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gem::Resolver::Molinillo +module Gem::Molinillo class DependencyGraph # A vertex in a {DependencyGraph} that encapsulates a {#name} and a # {#payload} diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/errors.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/errors.rb index 4289902828..07ea5fdf37 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/errors.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/errors.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gem::Resolver::Molinillo +module Gem::Molinillo # An error that occurred during the resolution process class ResolverError < StandardError; end diff --git a/lib/rubygems/vendor/molinillo/lib/molinillo/gem_metadata.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/gem_metadata.rb new file mode 100644 index 0000000000..8ed3a920a2 --- /dev/null +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/gem_metadata.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true + +module Gem::Molinillo + # The version of Gem::Molinillo. + VERSION = '0.8.0'.freeze +end diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/modules/specification_provider.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/modules/specification_provider.rb index 1067bf7439..85860902fc 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/modules/specification_provider.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/modules/specification_provider.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -module Gem::Resolver::Molinillo +module Gem::Molinillo # Provides information about specifications and dependencies to the resolver, # allowing the {Resolver} class to remain generic while still providing power # and flexibility. # - # This module contains the methods that users of Gem::Resolver::Molinillo must to implement, + # This module contains the methods that users of Gem::Molinillo must to implement, # using knowledge of their own model classes. module SpecificationProvider # Search for the specifications that match the given dependency. diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/modules/ui.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/modules/ui.rb index a810fd519c..464722902e 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/modules/ui.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/modules/ui.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gem::Resolver::Molinillo +module Gem::Molinillo # Conveys information about the resolution process to a user. module UI # The {IO} object that should be used to print output. `STDOUT`, by default. diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/resolution.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/resolution.rb index 8b40e59e42..84ec6cb095 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/resolution.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/resolution.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gem::Resolver::Molinillo +module Gem::Molinillo class Resolver # A specific resolution from a given {Resolver} class Resolution @@ -103,7 +103,7 @@ module Gem::Resolver::Molinillo # @return [Boolean] where the requirement of the state we're unwinding # to directly caused the conflict. Note: in this case, it is - # impossible for the state we're unwinding to to be a parent of + # impossible for the state we're unwinding to be a parent of # any of the other conflicting requirements (or we would have # circularity) def unwinding_to_primary_requirement? @@ -244,8 +244,8 @@ module Gem::Resolver::Molinillo require_relative 'delegates/resolution_state' require_relative 'delegates/specification_provider' - include Gem::Resolver::Molinillo::Delegates::ResolutionState - include Gem::Resolver::Molinillo::Delegates::SpecificationProvider + include Gem::Molinillo::Delegates::ResolutionState + include Gem::Molinillo::Delegates::SpecificationProvider # Processes the topmost available {RequirementState} on the stack # @return [void] diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/resolver.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/resolver.rb index d43121f8ca..86229c3fa1 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/resolver.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/resolver.rb @@ -2,7 +2,7 @@ require_relative 'dependency_graph' -module Gem::Resolver::Molinillo +module Gem::Molinillo # This class encapsulates a dependency resolver. # The resolver is responsible for determining which set of dependencies to # activate, with feedback from the {#specification_provider} diff --git a/lib/rubygems/resolver/molinillo/lib/molinillo/state.rb b/lib/rubygems/vendor/molinillo/lib/molinillo/state.rb index 6e7c715fce..c48ec6af9c 100644 --- a/lib/rubygems/resolver/molinillo/lib/molinillo/state.rb +++ b/lib/rubygems/vendor/molinillo/lib/molinillo/state.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -module Gem::Resolver::Molinillo +module Gem::Molinillo # A state that a {Resolution} can be in # @attr [String] name the name of the current requirement # @attr [Array<Object>] requirements currently unsatisfied requirements diff --git a/lib/rubygems/vendor/net-http/.document b/lib/rubygems/vendor/net-http/.document new file mode 100644 index 0000000000..0c43bbd6b3 --- /dev/null +++ b/lib/rubygems/vendor/net-http/.document @@ -0,0 +1 @@ +# Vendored files do not need to be documented diff --git a/lib/rubygems/vendor/net-http/lib/net/http.rb b/lib/rubygems/vendor/net-http/lib/net/http.rb new file mode 100644 index 0000000000..7b15c3cf54 --- /dev/null +++ b/lib/rubygems/vendor/net-http/lib/net/http.rb @@ -0,0 +1,2496 @@ +# frozen_string_literal: true +# +# = net/http.rb +# +# Copyright (c) 1999-2007 Yukihiro Matsumoto +# Copyright (c) 1999-2007 Minero Aoki +# Copyright (c) 2001 GOTOU Yuuzou +# +# Written and maintained by Minero Aoki <aamine@loveruby.net>. +# HTTPS support added by GOTOU Yuuzou <gotoyuzo@notwork.org>. +# +# This file is derived from "http-access.rb". +# +# Documented by Minero Aoki; converted to RDoc by William Webber. +# +# This program is free software. You can re-distribute and/or +# modify this program under the same terms of ruby itself --- +# Ruby Distribution License or GNU General Public License. +# +# See Gem::Net::HTTP for an overview and examples. +# + +require_relative '../../../net-protocol/lib/net/protocol' +require_relative '../../../uri/lib/uri' +require_relative '../../../resolv/lib/resolv' +autoload :OpenSSL, 'openssl' + +module Gem::Net #:nodoc: + + # :stopdoc: + class HTTPBadResponse < StandardError; end + class HTTPHeaderSyntaxError < StandardError; end + # :startdoc: + + # \Class \Gem::Net::HTTP provides a rich library that implements the client + # in a client-server model that uses the \HTTP request-response protocol. + # For information about \HTTP, see: + # + # - {Hypertext Transfer Protocol}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol]. + # - {Technical overview}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Technical_overview]. + # + # == About the Examples + # + # :include: doc/net-http/examples.rdoc + # + # == Strategies + # + # - If you will make only a few GET requests, + # consider using {OpenURI}[rdoc-ref:OpenURI]. + # - If you will make only a few requests of all kinds, + # consider using the various singleton convenience methods in this class. + # Each of the following methods automatically starts and finishes + # a {session}[rdoc-ref:Gem::Net::HTTP@Sessions] that sends a single request: + # + # # Return string response body. + # Gem::Net::HTTP.get(hostname, path) + # Gem::Net::HTTP.get(uri) + # + # # Write string response body to $stdout. + # Gem::Net::HTTP.get_print(hostname, path) + # Gem::Net::HTTP.get_print(uri) + # + # # Return response as Gem::Net::HTTPResponse object. + # Gem::Net::HTTP.get_response(hostname, path) + # Gem::Net::HTTP.get_response(uri) + # data = '{"title": "foo", "body": "bar", "userId": 1}' + # Gem::Net::HTTP.post(uri, data) + # params = {title: 'foo', body: 'bar', userId: 1} + # Gem::Net::HTTP.post_form(uri, params) + # + # - If performance is important, consider using sessions, which lower request overhead. + # This {session}[rdoc-ref:Gem::Net::HTTP@Sessions] has multiple requests for + # {HTTP methods}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Request_methods] + # and {WebDAV methods}[https://en.wikipedia.org/wiki/WebDAV#Implementation]: + # + # Gem::Net::HTTP.start(hostname) do |http| + # # Session started automatically before block execution. + # http.get(path) + # http.head(path) + # body = 'Some text' + # http.post(path, body) # Can also have a block. + # http.put(path, body) + # http.delete(path) + # http.options(path) + # http.trace(path) + # http.patch(path, body) # Can also have a block. + # http.copy(path) + # http.lock(path, body) + # http.mkcol(path, body) + # http.move(path) + # http.propfind(path, body) + # http.proppatch(path, body) + # http.unlock(path, body) + # # Session finished automatically at block exit. + # end + # + # The methods cited above are convenience methods that, via their few arguments, + # allow minimal control over the requests. + # For greater control, consider using {request objects}[rdoc-ref:Gem::Net::HTTPRequest]. + # + # == URIs + # + # On the internet, a URI + # ({Universal Resource Identifier}[https://en.wikipedia.org/wiki/Uniform_Resource_Identifier]) + # is a string that identifies a particular resource. + # It consists of some or all of: scheme, hostname, path, query, and fragment; + # see {URI syntax}[https://en.wikipedia.org/wiki/Uniform_Resource_Identifier#Syntax]. + # + # A Ruby {Gem::URI::Generic}[rdoc-ref:Gem::URI::Generic] object + # represents an internet URI. + # It provides, among others, methods + # +scheme+, +hostname+, +path+, +query+, and +fragment+. + # + # === Schemes + # + # An internet \Gem::URI has + # a {scheme}[https://en.wikipedia.org/wiki/List_of_URI_schemes]. + # + # The two schemes supported in \Gem::Net::HTTP are <tt>'https'</tt> and <tt>'http'</tt>: + # + # uri.scheme # => "https" + # Gem::URI('http://example.com').scheme # => "http" + # + # === Hostnames + # + # A hostname identifies a server (host) to which requests may be sent: + # + # hostname = uri.hostname # => "jsonplaceholder.typicode.com" + # Gem::Net::HTTP.start(hostname) do |http| + # # Some HTTP stuff. + # end + # + # === Paths + # + # A host-specific path identifies a resource on the host: + # + # _uri = uri.dup + # _uri.path = '/todos/1' + # hostname = _uri.hostname + # path = _uri.path + # Gem::Net::HTTP.get(hostname, path) + # + # === Queries + # + # A host-specific query adds name/value pairs to the URI: + # + # _uri = uri.dup + # params = {userId: 1, completed: false} + # _uri.query = Gem::URI.encode_www_form(params) + # _uri # => #<Gem::URI::HTTPS https://jsonplaceholder.typicode.com?userId=1&completed=false> + # Gem::Net::HTTP.get(_uri) + # + # === Fragments + # + # A {URI fragment}[https://en.wikipedia.org/wiki/URI_fragment] has no effect + # in \Gem::Net::HTTP; + # the same data is returned, regardless of whether a fragment is included. + # + # == Request Headers + # + # Request headers may be used to pass additional information to the host, + # similar to arguments passed in a method call; + # each header is a name/value pair. + # + # Each of the \Gem::Net::HTTP methods that sends a request to the host + # has optional argument +headers+, + # where the headers are expressed as a hash of field-name/value pairs: + # + # headers = {Accept: 'application/json', Connection: 'Keep-Alive'} + # Gem::Net::HTTP.get(uri, headers) + # + # See lists of both standard request fields and common request fields at + # {Request Fields}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Request_fields]. + # A host may also accept other custom fields. + # + # == \HTTP Sessions + # + # A _session_ is a connection between a server (host) and a client that: + # + # - Is begun by instance method Gem::Net::HTTP#start. + # - May contain any number of requests. + # - Is ended by instance method Gem::Net::HTTP#finish. + # + # See example sessions at {Strategies}[rdoc-ref:Gem::Net::HTTP@Strategies]. + # + # === Session Using \Gem::Net::HTTP.start + # + # If you have many requests to make to a single host (and port), + # consider using singleton method Gem::Net::HTTP.start with a block; + # the method handles the session automatically by: + # + # - Calling #start before block execution. + # - Executing the block. + # - Calling #finish after block execution. + # + # In the block, you can use these instance methods, + # each of which that sends a single request: + # + # - {HTTP methods}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Request_methods]: + # + # - #get, #request_get: GET. + # - #head, #request_head: HEAD. + # - #post, #request_post: POST. + # - #delete: DELETE. + # - #options: OPTIONS. + # - #trace: TRACE. + # - #patch: PATCH. + # + # - {WebDAV methods}[https://en.wikipedia.org/wiki/WebDAV#Implementation]: + # + # - #copy: COPY. + # - #lock: LOCK. + # - #mkcol: MKCOL. + # - #move: MOVE. + # - #propfind: PROPFIND. + # - #proppatch: PROPPATCH. + # - #unlock: UNLOCK. + # + # === Session Using \Gem::Net::HTTP.start and \Gem::Net::HTTP.finish + # + # You can manage a session manually using methods #start and #finish: + # + # http = Gem::Net::HTTP.new(hostname) + # http.start + # http.get('/todos/1') + # http.get('/todos/2') + # http.delete('/posts/1') + # http.finish # Needed to free resources. + # + # === Single-Request Session + # + # Certain convenience methods automatically handle a session by: + # + # - Creating an \HTTP object + # - Starting a session. + # - Sending a single request. + # - Finishing the session. + # - Destroying the object. + # + # Such methods that send GET requests: + # + # - ::get: Returns the string response body. + # - ::get_print: Writes the string response body to $stdout. + # - ::get_response: Returns a Gem::Net::HTTPResponse object. + # + # Such methods that send POST requests: + # + # - ::post: Posts data to the host. + # - ::post_form: Posts form data to the host. + # + # == \HTTP Requests and Responses + # + # Many of the methods above are convenience methods, + # each of which sends a request and returns a string + # without directly using \Gem::Net::HTTPRequest and \Gem::Net::HTTPResponse objects. + # + # You can, however, directly create a request object, send the request, + # and retrieve the response object; see: + # + # - Gem::Net::HTTPRequest. + # - Gem::Net::HTTPResponse. + # + # == Following Redirection + # + # Each returned response is an instance of a subclass of Gem::Net::HTTPResponse. + # See the {response class hierarchy}[rdoc-ref:Gem::Net::HTTPResponse@Response+Subclasses]. + # + # In particular, class Gem::Net::HTTPRedirection is the parent + # of all redirection classes. + # This allows you to craft a case statement to handle redirections properly: + # + # def fetch(uri, limit = 10) + # # You should choose a better exception. + # raise ArgumentError, 'Too many HTTP redirects' if limit == 0 + # + # res = Gem::Net::HTTP.get_response(Gem::URI(uri)) + # case res + # when Gem::Net::HTTPSuccess # Any success class. + # res + # when Gem::Net::HTTPRedirection # Any redirection class. + # location = res['Location'] + # warn "Redirected to #{location}" + # fetch(location, limit - 1) + # else # Any other class. + # res.value + # end + # end + # + # fetch(uri) + # + # == Basic Authentication + # + # Basic authentication is performed according to + # {RFC2617}[http://www.ietf.org/rfc/rfc2617.txt]: + # + # req = Gem::Net::HTTP::Get.new(uri) + # req.basic_auth('user', 'pass') + # res = Gem::Net::HTTP.start(hostname) do |http| + # http.request(req) + # end + # + # == Streaming Response Bodies + # + # By default \Gem::Net::HTTP reads an entire response into memory. If you are + # handling large files or wish to implement a progress bar you can instead + # stream the body directly to an IO. + # + # Gem::Net::HTTP.start(hostname) do |http| + # req = Gem::Net::HTTP::Get.new(uri) + # http.request(req) do |res| + # open('t.tmp', 'w') do |f| + # res.read_body do |chunk| + # f.write chunk + # end + # end + # end + # end + # + # == HTTPS + # + # HTTPS is enabled for an \HTTP connection by Gem::Net::HTTP#use_ssl=: + # + # Gem::Net::HTTP.start(hostname, :use_ssl => true) do |http| + # req = Gem::Net::HTTP::Get.new(uri) + # res = http.request(req) + # end + # + # Or if you simply want to make a GET request, you may pass in a URI + # object that has an \HTTPS URL. \Gem::Net::HTTP automatically turns on TLS + # verification if the URI object has a 'https' :URI scheme: + # + # uri # => #<Gem::URI::HTTPS https://jsonplaceholder.typicode.com/> + # Gem::Net::HTTP.get(uri) + # + # == Proxy Server + # + # An \HTTP object can have + # a {proxy server}[https://en.wikipedia.org/wiki/Proxy_server]. + # + # You can create an \HTTP object with a proxy server + # using method Gem::Net::HTTP.new or method Gem::Net::HTTP.start. + # + # The proxy may be defined either by argument +p_addr+ + # or by environment variable <tt>'http_proxy'</tt>. + # + # === Proxy Using Argument +p_addr+ as a \String + # + # When argument +p_addr+ is a string hostname, + # the returned +http+ has the given host as its proxy: + # + # http = Gem::Net::HTTP.new(hostname, nil, 'proxy.example') + # http.proxy? # => true + # http.proxy_from_env? # => false + # http.proxy_address # => "proxy.example" + # # These use default values. + # http.proxy_port # => 80 + # http.proxy_user # => nil + # http.proxy_pass # => nil + # + # The port, username, and password for the proxy may also be given: + # + # http = Gem::Net::HTTP.new(hostname, nil, 'proxy.example', 8000, 'pname', 'ppass') + # # => #<Gem::Net::HTTP jsonplaceholder.typicode.com:80 open=false> + # http.proxy? # => true + # http.proxy_from_env? # => false + # http.proxy_address # => "proxy.example" + # http.proxy_port # => 8000 + # http.proxy_user # => "pname" + # http.proxy_pass # => "ppass" + # + # === Proxy Using '<tt>ENV['http_proxy']</tt>' + # + # When environment variable <tt>'http_proxy'</tt> + # is set to a \Gem::URI string, + # the returned +http+ will have the server at that URI as its proxy; + # note that the \Gem::URI string must have a protocol + # such as <tt>'http'</tt> or <tt>'https'</tt>: + # + # ENV['http_proxy'] = 'http://example.com' + # http = Gem::Net::HTTP.new(hostname) + # http.proxy? # => true + # http.proxy_from_env? # => true + # http.proxy_address # => "example.com" + # # These use default values. + # http.proxy_port # => 80 + # http.proxy_user # => nil + # http.proxy_pass # => nil + # + # The \Gem::URI string may include proxy username, password, and port number: + # + # ENV['http_proxy'] = 'http://pname:ppass@example.com:8000' + # http = Gem::Net::HTTP.new(hostname) + # http.proxy? # => true + # http.proxy_from_env? # => true + # http.proxy_address # => "example.com" + # http.proxy_port # => 8000 + # http.proxy_user # => "pname" + # http.proxy_pass # => "ppass" + # + # === Filtering Proxies + # + # With method Gem::Net::HTTP.new (but not Gem::Net::HTTP.start), + # you can use argument +p_no_proxy+ to filter proxies: + # + # - Reject a certain address: + # + # http = Gem::Net::HTTP.new('example.com', nil, 'proxy.example', 8000, 'pname', 'ppass', 'proxy.example') + # http.proxy_address # => nil + # + # - Reject certain domains or subdomains: + # + # http = Gem::Net::HTTP.new('example.com', nil, 'my.proxy.example', 8000, 'pname', 'ppass', 'proxy.example') + # http.proxy_address # => nil + # + # - Reject certain addresses and port combinations: + # + # http = Gem::Net::HTTP.new('example.com', nil, 'proxy.example', 8000, 'pname', 'ppass', 'proxy.example:1234') + # http.proxy_address # => "proxy.example" + # + # http = Gem::Net::HTTP.new('example.com', nil, 'proxy.example', 8000, 'pname', 'ppass', 'proxy.example:8000') + # http.proxy_address # => nil + # + # - Reject a list of the types above delimited using a comma: + # + # http = Gem::Net::HTTP.new('example.com', nil, 'proxy.example', 8000, 'pname', 'ppass', 'my.proxy,proxy.example:8000') + # http.proxy_address # => nil + # + # http = Gem::Net::HTTP.new('example.com', nil, 'my.proxy', 8000, 'pname', 'ppass', 'my.proxy,proxy.example:8000') + # http.proxy_address # => nil + # + # == Compression and Decompression + # + # \Gem::Net::HTTP does not compress the body of a request before sending. + # + # By default, \Gem::Net::HTTP adds header <tt>'Accept-Encoding'</tt> + # to a new {request object}[rdoc-ref:Gem::Net::HTTPRequest]: + # + # Gem::Net::HTTP::Get.new(uri)['Accept-Encoding'] + # # => "gzip;q=1.0,deflate;q=0.6,identity;q=0.3" + # + # This requests the server to zip-encode the response body if there is one; + # the server is not required to do so. + # + # \Gem::Net::HTTP does not automatically decompress a response body + # if the response has header <tt>'Content-Range'</tt>. + # + # Otherwise decompression (or not) depends on the value of header + # {Content-Encoding}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#content-encoding-response-header]: + # + # - <tt>'deflate'</tt>, <tt>'gzip'</tt>, or <tt>'x-gzip'</tt>: + # decompresses the body and deletes the header. + # - <tt>'none'</tt> or <tt>'identity'</tt>: + # does not decompress the body, but deletes the header. + # - Any other value: + # leaves the body and header unchanged. + # + # == What's Here + # + # This is a categorized summary of methods and attributes. + # + # === \Gem::Net::HTTP Objects + # + # - {::new}[rdoc-ref:Gem::Net::HTTP.new]: + # Creates a new instance. + # - {#inspect}[rdoc-ref:Gem::Net::HTTP#inspect]: + # Returns a string representation of +self+. + # + # === Sessions + # + # - {::start}[rdoc-ref:Gem::Net::HTTP.start]: + # Begins a new session in a new \Gem::Net::HTTP object. + # - {#started?}[rdoc-ref:Gem::Net::HTTP#started?] + # (aliased as {#active?}[rdoc-ref:Gem::Net::HTTP#active?]): + # Returns whether in a session. + # - {#finish}[rdoc-ref:Gem::Net::HTTP#finish]: + # Ends an active session. + # - {#start}[rdoc-ref:Gem::Net::HTTP#start]: + # Begins a new session in an existing \Gem::Net::HTTP object (+self+). + # + # === Connections + # + # - {:continue_timeout}[rdoc-ref:Gem::Net::HTTP#continue_timeout]: + # Returns the continue timeout. + # - {#continue_timeout=}[rdoc-ref:Gem::Net::HTTP#continue_timeout=]: + # Sets the continue timeout seconds. + # - {:keep_alive_timeout}[rdoc-ref:Gem::Net::HTTP#keep_alive_timeout]: + # Returns the keep-alive timeout. + # - {:keep_alive_timeout=}[rdoc-ref:Gem::Net::HTTP#keep_alive_timeout=]: + # Sets the keep-alive timeout. + # - {:max_retries}[rdoc-ref:Gem::Net::HTTP#max_retries]: + # Returns the maximum retries. + # - {#max_retries=}[rdoc-ref:Gem::Net::HTTP#max_retries=]: + # Sets the maximum retries. + # - {:open_timeout}[rdoc-ref:Gem::Net::HTTP#open_timeout]: + # Returns the open timeout. + # - {:open_timeout=}[rdoc-ref:Gem::Net::HTTP#open_timeout=]: + # Sets the open timeout. + # - {:read_timeout}[rdoc-ref:Gem::Net::HTTP#read_timeout]: + # Returns the open timeout. + # - {:read_timeout=}[rdoc-ref:Gem::Net::HTTP#read_timeout=]: + # Sets the read timeout. + # - {:ssl_timeout}[rdoc-ref:Gem::Net::HTTP#ssl_timeout]: + # Returns the ssl timeout. + # - {:ssl_timeout=}[rdoc-ref:Gem::Net::HTTP#ssl_timeout=]: + # Sets the ssl timeout. + # - {:write_timeout}[rdoc-ref:Gem::Net::HTTP#write_timeout]: + # Returns the write timeout. + # - {write_timeout=}[rdoc-ref:Gem::Net::HTTP#write_timeout=]: + # Sets the write timeout. + # + # === Requests + # + # - {::get}[rdoc-ref:Gem::Net::HTTP.get]: + # Sends a GET request and returns the string response body. + # - {::get_print}[rdoc-ref:Gem::Net::HTTP.get_print]: + # Sends a GET request and write the string response body to $stdout. + # - {::get_response}[rdoc-ref:Gem::Net::HTTP.get_response]: + # Sends a GET request and returns a response object. + # - {::post_form}[rdoc-ref:Gem::Net::HTTP.post_form]: + # Sends a POST request with form data and returns a response object. + # - {::post}[rdoc-ref:Gem::Net::HTTP.post]: + # Sends a POST request with data and returns a response object. + # - {#copy}[rdoc-ref:Gem::Net::HTTP#copy]: + # Sends a COPY request and returns a response object. + # - {#delete}[rdoc-ref:Gem::Net::HTTP#delete]: + # Sends a DELETE request and returns a response object. + # - {#get}[rdoc-ref:Gem::Net::HTTP#get]: + # Sends a GET request and returns a response object. + # - {#head}[rdoc-ref:Gem::Net::HTTP#head]: + # Sends a HEAD request and returns a response object. + # - {#lock}[rdoc-ref:Gem::Net::HTTP#lock]: + # Sends a LOCK request and returns a response object. + # - {#mkcol}[rdoc-ref:Gem::Net::HTTP#mkcol]: + # Sends a MKCOL request and returns a response object. + # - {#move}[rdoc-ref:Gem::Net::HTTP#move]: + # Sends a MOVE request and returns a response object. + # - {#options}[rdoc-ref:Gem::Net::HTTP#options]: + # Sends a OPTIONS request and returns a response object. + # - {#patch}[rdoc-ref:Gem::Net::HTTP#patch]: + # Sends a PATCH request and returns a response object. + # - {#post}[rdoc-ref:Gem::Net::HTTP#post]: + # Sends a POST request and returns a response object. + # - {#propfind}[rdoc-ref:Gem::Net::HTTP#propfind]: + # Sends a PROPFIND request and returns a response object. + # - {#proppatch}[rdoc-ref:Gem::Net::HTTP#proppatch]: + # Sends a PROPPATCH request and returns a response object. + # - {#put}[rdoc-ref:Gem::Net::HTTP#put]: + # Sends a PUT request and returns a response object. + # - {#request}[rdoc-ref:Gem::Net::HTTP#request]: + # Sends a request and returns a response object. + # - {#request_get}[rdoc-ref:Gem::Net::HTTP#request_get] + # (aliased as {#get2}[rdoc-ref:Gem::Net::HTTP#get2]): + # Sends a GET request and forms a response object; + # if a block given, calls the block with the object, + # otherwise returns the object. + # - {#request_head}[rdoc-ref:Gem::Net::HTTP#request_head] + # (aliased as {#head2}[rdoc-ref:Gem::Net::HTTP#head2]): + # Sends a HEAD request and forms a response object; + # if a block given, calls the block with the object, + # otherwise returns the object. + # - {#request_post}[rdoc-ref:Gem::Net::HTTP#request_post] + # (aliased as {#post2}[rdoc-ref:Gem::Net::HTTP#post2]): + # Sends a POST request and forms a response object; + # if a block given, calls the block with the object, + # otherwise returns the object. + # - {#send_request}[rdoc-ref:Gem::Net::HTTP#send_request]: + # Sends a request and returns a response object. + # - {#trace}[rdoc-ref:Gem::Net::HTTP#trace]: + # Sends a TRACE request and returns a response object. + # - {#unlock}[rdoc-ref:Gem::Net::HTTP#unlock]: + # Sends an UNLOCK request and returns a response object. + # + # === Responses + # + # - {:close_on_empty_response}[rdoc-ref:Gem::Net::HTTP#close_on_empty_response]: + # Returns whether to close connection on empty response. + # - {:close_on_empty_response=}[rdoc-ref:Gem::Net::HTTP#close_on_empty_response=]: + # Sets whether to close connection on empty response. + # - {:ignore_eof}[rdoc-ref:Gem::Net::HTTP#ignore_eof]: + # Returns whether to ignore end-of-file when reading a response body + # with <tt>Content-Length</tt> headers. + # - {:ignore_eof=}[rdoc-ref:Gem::Net::HTTP#ignore_eof=]: + # Sets whether to ignore end-of-file when reading a response body + # with <tt>Content-Length</tt> headers. + # - {:response_body_encoding}[rdoc-ref:Gem::Net::HTTP#response_body_encoding]: + # Returns the encoding to use for the response body. + # - {#response_body_encoding=}[rdoc-ref:Gem::Net::HTTP#response_body_encoding=]: + # Sets the response body encoding. + # + # === Proxies + # + # - {:proxy_address}[rdoc-ref:Gem::Net::HTTP#proxy_address]: + # Returns the proxy address. + # - {:proxy_address=}[rdoc-ref:Gem::Net::HTTP#proxy_address=]: + # Sets the proxy address. + # - {::proxy_class?}[rdoc-ref:Gem::Net::HTTP.proxy_class?]: + # Returns whether +self+ is a proxy class. + # - {#proxy?}[rdoc-ref:Gem::Net::HTTP#proxy?]: + # Returns whether +self+ has a proxy. + # - {#proxy_address}[rdoc-ref:Gem::Net::HTTP#proxy_address] + # (aliased as {#proxyaddr}[rdoc-ref:Gem::Net::HTTP#proxyaddr]): + # Returns the proxy address. + # - {#proxy_from_env?}[rdoc-ref:Gem::Net::HTTP#proxy_from_env?]: + # Returns whether the proxy is taken from an environment variable. + # - {:proxy_from_env=}[rdoc-ref:Gem::Net::HTTP#proxy_from_env=]: + # Sets whether the proxy is to be taken from an environment variable. + # - {:proxy_pass}[rdoc-ref:Gem::Net::HTTP#proxy_pass]: + # Returns the proxy password. + # - {:proxy_pass=}[rdoc-ref:Gem::Net::HTTP#proxy_pass=]: + # Sets the proxy password. + # - {:proxy_port}[rdoc-ref:Gem::Net::HTTP#proxy_port]: + # Returns the proxy port. + # - {:proxy_port=}[rdoc-ref:Gem::Net::HTTP#proxy_port=]: + # Sets the proxy port. + # - {#proxy_user}[rdoc-ref:Gem::Net::HTTP#proxy_user]: + # Returns the proxy user name. + # - {:proxy_user=}[rdoc-ref:Gem::Net::HTTP#proxy_user=]: + # Sets the proxy user. + # + # === Security + # + # - {:ca_file}[rdoc-ref:Gem::Net::HTTP#ca_file]: + # Returns the path to a CA certification file. + # - {:ca_file=}[rdoc-ref:Gem::Net::HTTP#ca_file=]: + # Sets the path to a CA certification file. + # - {:ca_path}[rdoc-ref:Gem::Net::HTTP#ca_path]: + # Returns the path of to CA directory containing certification files. + # - {:ca_path=}[rdoc-ref:Gem::Net::HTTP#ca_path=]: + # Sets the path of to CA directory containing certification files. + # - {:cert}[rdoc-ref:Gem::Net::HTTP#cert]: + # Returns the OpenSSL::X509::Certificate object to be used for client certification. + # - {:cert=}[rdoc-ref:Gem::Net::HTTP#cert=]: + # Sets the OpenSSL::X509::Certificate object to be used for client certification. + # - {:cert_store}[rdoc-ref:Gem::Net::HTTP#cert_store]: + # Returns the X509::Store to be used for verifying peer certificate. + # - {:cert_store=}[rdoc-ref:Gem::Net::HTTP#cert_store=]: + # Sets the X509::Store to be used for verifying peer certificate. + # - {:ciphers}[rdoc-ref:Gem::Net::HTTP#ciphers]: + # Returns the available SSL ciphers. + # - {:ciphers=}[rdoc-ref:Gem::Net::HTTP#ciphers=]: + # Sets the available SSL ciphers. + # - {:extra_chain_cert}[rdoc-ref:Gem::Net::HTTP#extra_chain_cert]: + # Returns the extra X509 certificates to be added to the certificate chain. + # - {:extra_chain_cert=}[rdoc-ref:Gem::Net::HTTP#extra_chain_cert=]: + # Sets the extra X509 certificates to be added to the certificate chain. + # - {:key}[rdoc-ref:Gem::Net::HTTP#key]: + # Returns the OpenSSL::PKey::RSA or OpenSSL::PKey::DSA object. + # - {:key=}[rdoc-ref:Gem::Net::HTTP#key=]: + # Sets the OpenSSL::PKey::RSA or OpenSSL::PKey::DSA object. + # - {:max_version}[rdoc-ref:Gem::Net::HTTP#max_version]: + # Returns the maximum SSL version. + # - {:max_version=}[rdoc-ref:Gem::Net::HTTP#max_version=]: + # Sets the maximum SSL version. + # - {:min_version}[rdoc-ref:Gem::Net::HTTP#min_version]: + # Returns the minimum SSL version. + # - {:min_version=}[rdoc-ref:Gem::Net::HTTP#min_version=]: + # Sets the minimum SSL version. + # - {#peer_cert}[rdoc-ref:Gem::Net::HTTP#peer_cert]: + # Returns the X509 certificate chain for the session's socket peer. + # - {:ssl_version}[rdoc-ref:Gem::Net::HTTP#ssl_version]: + # Returns the SSL version. + # - {:ssl_version=}[rdoc-ref:Gem::Net::HTTP#ssl_version=]: + # Sets the SSL version. + # - {#use_ssl=}[rdoc-ref:Gem::Net::HTTP#use_ssl=]: + # Sets whether a new session is to use Transport Layer Security. + # - {#use_ssl?}[rdoc-ref:Gem::Net::HTTP#use_ssl?]: + # Returns whether +self+ uses SSL. + # - {:verify_callback}[rdoc-ref:Gem::Net::HTTP#verify_callback]: + # Returns the callback for the server certification verification. + # - {:verify_callback=}[rdoc-ref:Gem::Net::HTTP#verify_callback=]: + # Sets the callback for the server certification verification. + # - {:verify_depth}[rdoc-ref:Gem::Net::HTTP#verify_depth]: + # Returns the maximum depth for the certificate chain verification. + # - {:verify_depth=}[rdoc-ref:Gem::Net::HTTP#verify_depth=]: + # Sets the maximum depth for the certificate chain verification. + # - {:verify_hostname}[rdoc-ref:Gem::Net::HTTP#verify_hostname]: + # Returns the flags for server the certification verification at the beginning of the SSL/TLS session. + # - {:verify_hostname=}[rdoc-ref:Gem::Net::HTTP#verify_hostname=]: + # Sets he flags for server the certification verification at the beginning of the SSL/TLS session. + # - {:verify_mode}[rdoc-ref:Gem::Net::HTTP#verify_mode]: + # Returns the flags for server the certification verification at the beginning of the SSL/TLS session. + # - {:verify_mode=}[rdoc-ref:Gem::Net::HTTP#verify_mode=]: + # Sets the flags for server the certification verification at the beginning of the SSL/TLS session. + # + # === Addresses and Ports + # + # - {:address}[rdoc-ref:Gem::Net::HTTP#address]: + # Returns the string host name or host IP. + # - {::default_port}[rdoc-ref:Gem::Net::HTTP.default_port]: + # Returns integer 80, the default port to use for HTTP requests. + # - {::http_default_port}[rdoc-ref:Gem::Net::HTTP.http_default_port]: + # Returns integer 80, the default port to use for HTTP requests. + # - {::https_default_port}[rdoc-ref:Gem::Net::HTTP.https_default_port]: + # Returns integer 443, the default port to use for HTTPS requests. + # - {#ipaddr}[rdoc-ref:Gem::Net::HTTP#ipaddr]: + # Returns the IP address for the connection. + # - {#ipaddr=}[rdoc-ref:Gem::Net::HTTP#ipaddr=]: + # Sets the IP address for the connection. + # - {:local_host}[rdoc-ref:Gem::Net::HTTP#local_host]: + # Returns the string local host used to establish the connection. + # - {:local_host=}[rdoc-ref:Gem::Net::HTTP#local_host=]: + # Sets the string local host used to establish the connection. + # - {:local_port}[rdoc-ref:Gem::Net::HTTP#local_port]: + # Returns the integer local port used to establish the connection. + # - {:local_port=}[rdoc-ref:Gem::Net::HTTP#local_port=]: + # Sets the integer local port used to establish the connection. + # - {:port}[rdoc-ref:Gem::Net::HTTP#port]: + # Returns the integer port number. + # + # === \HTTP Version + # + # - {::version_1_2?}[rdoc-ref:Gem::Net::HTTP.version_1_2?] + # (aliased as {::is_version_1_2?}[rdoc-ref:Gem::Net::HTTP.is_version_1_2?] + # and {::version_1_2}[rdoc-ref:Gem::Net::HTTP.version_1_2]): + # Returns true; retained for compatibility. + # + # === Debugging + # + # - {#set_debug_output}[rdoc-ref:Gem::Net::HTTP#set_debug_output]: + # Sets the output stream for debugging. + # + class HTTP < Protocol + + # :stopdoc: + VERSION = "0.4.0" + HTTPVersion = '1.1' + begin + require 'zlib' + HAVE_ZLIB=true + rescue LoadError + HAVE_ZLIB=false + end + # :startdoc: + + # Returns +true+; retained for compatibility. + def HTTP.version_1_2 + true + end + + # Returns +true+; retained for compatibility. + def HTTP.version_1_2? + true + end + + # Returns +false+; retained for compatibility. + def HTTP.version_1_1? #:nodoc: + false + end + + class << HTTP + alias is_version_1_1? version_1_1? #:nodoc: + alias is_version_1_2? version_1_2? #:nodoc: + end + + # :call-seq: + # Gem::Net::HTTP.get_print(hostname, path, port = 80) -> nil + # Gem::Net::HTTP:get_print(uri, headers = {}, port = uri.port) -> nil + # + # Like Gem::Net::HTTP.get, but writes the returned body to $stdout; + # returns +nil+. + def HTTP.get_print(uri_or_host, path_or_headers = nil, port = nil) + get_response(uri_or_host, path_or_headers, port) {|res| + res.read_body do |chunk| + $stdout.print chunk + end + } + nil + end + + # :call-seq: + # Gem::Net::HTTP.get(hostname, path, port = 80) -> body + # Gem::Net::HTTP:get(uri, headers = {}, port = uri.port) -> body + # + # Sends a GET request and returns the \HTTP response body as a string. + # + # With string arguments +hostname+ and +path+: + # + # hostname = 'jsonplaceholder.typicode.com' + # path = '/todos/1' + # puts Gem::Net::HTTP.get(hostname, path) + # + # Output: + # + # { + # "userId": 1, + # "id": 1, + # "title": "delectus aut autem", + # "completed": false + # } + # + # With URI object +uri+ and optional hash argument +headers+: + # + # uri = Gem::URI('https://jsonplaceholder.typicode.com/todos/1') + # headers = {'Content-type' => 'application/json; charset=UTF-8'} + # Gem::Net::HTTP.get(uri, headers) + # + # Related: + # + # - Gem::Net::HTTP::Get: request class for \HTTP method +GET+. + # - Gem::Net::HTTP#get: convenience method for \HTTP method +GET+. + # + def HTTP.get(uri_or_host, path_or_headers = nil, port = nil) + get_response(uri_or_host, path_or_headers, port).body + end + + # :call-seq: + # Gem::Net::HTTP.get_response(hostname, path, port = 80) -> http_response + # Gem::Net::HTTP:get_response(uri, headers = {}, port = uri.port) -> http_response + # + # Like Gem::Net::HTTP.get, but returns a Gem::Net::HTTPResponse object + # instead of the body string. + def HTTP.get_response(uri_or_host, path_or_headers = nil, port = nil, &block) + if path_or_headers && !path_or_headers.is_a?(Hash) + host = uri_or_host + path = path_or_headers + new(host, port || HTTP.default_port).start {|http| + return http.request_get(path, &block) + } + else + uri = uri_or_host + headers = path_or_headers + start(uri.hostname, uri.port, + :use_ssl => uri.scheme == 'https') {|http| + return http.request_get(uri, headers, &block) + } + end + end + + # Posts data to a host; returns a Gem::Net::HTTPResponse object. + # + # Argument +url+ must be a URL; + # argument +data+ must be a string: + # + # _uri = uri.dup + # _uri.path = '/posts' + # data = '{"title": "foo", "body": "bar", "userId": 1}' + # headers = {'content-type': 'application/json'} + # res = Gem::Net::HTTP.post(_uri, data, headers) # => #<Gem::Net::HTTPCreated 201 Created readbody=true> + # puts res.body + # + # Output: + # + # { + # "title": "foo", + # "body": "bar", + # "userId": 1, + # "id": 101 + # } + # + # Related: + # + # - Gem::Net::HTTP::Post: request class for \HTTP method +POST+. + # - Gem::Net::HTTP#post: convenience method for \HTTP method +POST+. + # + def HTTP.post(url, data, header = nil) + start(url.hostname, url.port, + :use_ssl => url.scheme == 'https' ) {|http| + http.post(url, data, header) + } + end + + # Posts data to a host; returns a Gem::Net::HTTPResponse object. + # + # Argument +url+ must be a URI; + # argument +data+ must be a hash: + # + # _uri = uri.dup + # _uri.path = '/posts' + # data = {title: 'foo', body: 'bar', userId: 1} + # res = Gem::Net::HTTP.post_form(_uri, data) # => #<Gem::Net::HTTPCreated 201 Created readbody=true> + # puts res.body + # + # Output: + # + # { + # "title": "foo", + # "body": "bar", + # "userId": "1", + # "id": 101 + # } + # + def HTTP.post_form(url, params) + req = Post.new(url) + req.form_data = params + req.basic_auth url.user, url.password if url.user + start(url.hostname, url.port, + :use_ssl => url.scheme == 'https' ) {|http| + http.request(req) + } + end + + # + # \HTTP session management + # + + # Returns integer +80+, the default port to use for \HTTP requests: + # + # Gem::Net::HTTP.default_port # => 80 + # + def HTTP.default_port + http_default_port() + end + + # Returns integer +80+, the default port to use for \HTTP requests: + # + # Gem::Net::HTTP.http_default_port # => 80 + # + def HTTP.http_default_port + 80 + end + + # Returns integer +443+, the default port to use for HTTPS requests: + # + # Gem::Net::HTTP.https_default_port # => 443 + # + def HTTP.https_default_port + 443 + end + + def HTTP.socket_type #:nodoc: obsolete + BufferedIO + end + + # :call-seq: + # HTTP.start(address, port = nil, p_addr = :ENV, p_port = nil, p_user = nil, p_pass = nil, opts) -> http + # HTTP.start(address, port = nil, p_addr = :ENV, p_port = nil, p_user = nil, p_pass = nil, opts) {|http| ... } -> object + # + # Creates a new \Gem::Net::HTTP object, +http+, via \Gem::Net::HTTP.new: + # + # - For arguments +address+ and +port+, see Gem::Net::HTTP.new. + # - For proxy-defining arguments +p_addr+ through +p_pass+, + # see {Proxy Server}[rdoc-ref:Gem::Net::HTTP@Proxy+Server]. + # - For argument +opts+, see below. + # + # With no block given: + # + # - Calls <tt>http.start</tt> with no block (see #start), + # which opens a TCP connection and \HTTP session. + # - Returns +http+. + # - The caller should call #finish to close the session: + # + # http = Gem::Net::HTTP.start(hostname) + # http.started? # => true + # http.finish + # http.started? # => false + # + # With a block given: + # + # - Calls <tt>http.start</tt> with the block (see #start), which: + # + # - Opens a TCP connection and \HTTP session. + # - Calls the block, + # which may make any number of requests to the host. + # - Closes the \HTTP session and TCP connection on block exit. + # - Returns the block's value +object+. + # + # - Returns +object+. + # + # Example: + # + # hostname = 'jsonplaceholder.typicode.com' + # Gem::Net::HTTP.start(hostname) do |http| + # puts http.get('/todos/1').body + # puts http.get('/todos/2').body + # end + # + # Output: + # + # { + # "userId": 1, + # "id": 1, + # "title": "delectus aut autem", + # "completed": false + # } + # { + # "userId": 1, + # "id": 2, + # "title": "quis ut nam facilis et officia qui", + # "completed": false + # } + # + # If the last argument given is a hash, it is the +opts+ hash, + # where each key is a method or accessor to be called, + # and its value is the value to be set. + # + # The keys may include: + # + # - #ca_file + # - #ca_path + # - #cert + # - #cert_store + # - #ciphers + # - #close_on_empty_response + # - +ipaddr+ (calls #ipaddr=) + # - #keep_alive_timeout + # - #key + # - #open_timeout + # - #read_timeout + # - #ssl_timeout + # - #ssl_version + # - +use_ssl+ (calls #use_ssl=) + # - #verify_callback + # - #verify_depth + # - #verify_mode + # - #write_timeout + # + # Note: If +port+ is +nil+ and <tt>opts[:use_ssl]</tt> is a truthy value, + # the value passed to +new+ is Gem::Net::HTTP.https_default_port, not +port+. + # + def HTTP.start(address, *arg, &block) # :yield: +http+ + arg.pop if opt = Hash.try_convert(arg[-1]) + port, p_addr, p_port, p_user, p_pass = *arg + p_addr = :ENV if arg.size < 2 + port = https_default_port if !port && opt && opt[:use_ssl] + http = new(address, port, p_addr, p_port, p_user, p_pass) + http.ipaddr = opt[:ipaddr] if opt && opt[:ipaddr] + + if opt + if opt[:use_ssl] + opt = {verify_mode: OpenSSL::SSL::VERIFY_PEER}.update(opt) + end + http.methods.grep(/\A(\w+)=\z/) do |meth| + key = $1.to_sym + opt.key?(key) or next + http.__send__(meth, opt[key]) + end + end + + http.start(&block) + end + + class << HTTP + alias newobj new # :nodoc: + end + + # Returns a new \Gem::Net::HTTP object +http+ + # (but does not open a TCP connection or \HTTP session). + # + # With only string argument +address+ given + # (and <tt>ENV['http_proxy']</tt> undefined or +nil+), + # the returned +http+: + # + # - Has the given address. + # - Has the default port number, Gem::Net::HTTP.default_port (80). + # - Has no proxy. + # + # Example: + # + # http = Gem::Net::HTTP.new(hostname) + # # => #<Gem::Net::HTTP jsonplaceholder.typicode.com:80 open=false> + # http.address # => "jsonplaceholder.typicode.com" + # http.port # => 80 + # http.proxy? # => false + # + # With integer argument +port+ also given, + # the returned +http+ has the given port: + # + # http = Gem::Net::HTTP.new(hostname, 8000) + # # => #<Gem::Net::HTTP jsonplaceholder.typicode.com:8000 open=false> + # http.port # => 8000 + # + # For proxy-defining arguments +p_addr+ through +p_no_proxy+, + # see {Proxy Server}[rdoc-ref:Gem::Net::HTTP@Proxy+Server]. + # + def HTTP.new(address, port = nil, p_addr = :ENV, p_port = nil, p_user = nil, p_pass = nil, p_no_proxy = nil) + http = super address, port + + if proxy_class? then # from Gem::Net::HTTP::Proxy() + http.proxy_from_env = @proxy_from_env + http.proxy_address = @proxy_address + http.proxy_port = @proxy_port + http.proxy_user = @proxy_user + http.proxy_pass = @proxy_pass + elsif p_addr == :ENV then + http.proxy_from_env = true + else + if p_addr && p_no_proxy && !Gem::URI::Generic.use_proxy?(address, address, port, p_no_proxy) + p_addr = nil + p_port = nil + end + http.proxy_address = p_addr + http.proxy_port = p_port || default_port + http.proxy_user = p_user + http.proxy_pass = p_pass + end + + http + end + + # Creates a new \Gem::Net::HTTP object for the specified server address, + # without opening the TCP connection or initializing the \HTTP session. + # The +address+ should be a DNS hostname or IP address. + def initialize(address, port = nil) # :nodoc: + @address = address + @port = (port || HTTP.default_port) + @ipaddr = nil + @local_host = nil + @local_port = nil + @curr_http_version = HTTPVersion + @keep_alive_timeout = 2 + @last_communicated = nil + @close_on_empty_response = false + @socket = nil + @started = false + @open_timeout = 60 + @read_timeout = 60 + @write_timeout = 60 + @continue_timeout = nil + @max_retries = 1 + @debug_output = nil + @response_body_encoding = false + @ignore_eof = true + + @proxy_from_env = false + @proxy_uri = nil + @proxy_address = nil + @proxy_port = nil + @proxy_user = nil + @proxy_pass = nil + + @use_ssl = false + @ssl_context = nil + @ssl_session = nil + @sspi_enabled = false + SSL_IVNAMES.each do |ivname| + instance_variable_set ivname, nil + end + end + + # Returns a string representation of +self+: + # + # Gem::Net::HTTP.new(hostname).inspect + # # => "#<Gem::Net::HTTP jsonplaceholder.typicode.com:80 open=false>" + # + def inspect + "#<#{self.class} #{@address}:#{@port} open=#{started?}>" + end + + # *WARNING* This method opens a serious security hole. + # Never use this method in production code. + # + # Sets the output stream for debugging: + # + # http = Gem::Net::HTTP.new(hostname) + # File.open('t.tmp', 'w') do |file| + # http.set_debug_output(file) + # http.start + # http.get('/nosuch/1') + # http.finish + # end + # puts File.read('t.tmp') + # + # Output: + # + # opening connection to jsonplaceholder.typicode.com:80... + # opened + # <- "GET /nosuch/1 HTTP/1.1\r\nAccept-Encoding: gzip;q=1.0,deflate;q=0.6,identity;q=0.3\r\nAccept: */*\r\nUser-Agent: Ruby\r\nHost: jsonplaceholder.typicode.com\r\n\r\n" + # -> "HTTP/1.1 404 Not Found\r\n" + # -> "Date: Mon, 12 Dec 2022 21:14:11 GMT\r\n" + # -> "Content-Type: application/json; charset=utf-8\r\n" + # -> "Content-Length: 2\r\n" + # -> "Connection: keep-alive\r\n" + # -> "X-Powered-By: Express\r\n" + # -> "X-Ratelimit-Limit: 1000\r\n" + # -> "X-Ratelimit-Remaining: 999\r\n" + # -> "X-Ratelimit-Reset: 1670879660\r\n" + # -> "Vary: Origin, Accept-Encoding\r\n" + # -> "Access-Control-Allow-Credentials: true\r\n" + # -> "Cache-Control: max-age=43200\r\n" + # -> "Pragma: no-cache\r\n" + # -> "Expires: -1\r\n" + # -> "X-Content-Type-Options: nosniff\r\n" + # -> "Etag: W/\"2-vyGp6PvFo4RvsFtPoIWeCReyIC8\"\r\n" + # -> "Via: 1.1 vegur\r\n" + # -> "CF-Cache-Status: MISS\r\n" + # -> "Server-Timing: cf-q-config;dur=1.3000000762986e-05\r\n" + # -> "Report-To: {\"endpoints\":[{\"url\":\"https:\\/\\/a.nel.cloudflare.com\\/report\\/v3?s=yOr40jo%2BwS1KHzhTlVpl54beJ5Wx2FcG4gGV0XVrh3X9OlR5q4drUn2dkt5DGO4GDcE%2BVXT7CNgJvGs%2BZleIyMu8CLieFiDIvOviOY3EhHg94m0ZNZgrEdpKD0S85S507l1vsEwEHkoTm%2Ff19SiO\"}],\"group\":\"cf-nel\",\"max_age\":604800}\r\n" + # -> "NEL: {\"success_fraction\":0,\"report_to\":\"cf-nel\",\"max_age\":604800}\r\n" + # -> "Server: cloudflare\r\n" + # -> "CF-RAY: 778977dc484ce591-DFW\r\n" + # -> "alt-svc: h3=\":443\"; ma=86400, h3-29=\":443\"; ma=86400\r\n" + # -> "\r\n" + # reading 2 bytes... + # -> "{}" + # read 2 bytes + # Conn keep-alive + # + def set_debug_output(output) + warn 'Gem::Net::HTTP#set_debug_output called after HTTP started', uplevel: 1 if started? + @debug_output = output + end + + # Returns the string host name or host IP given as argument +address+ in ::new. + attr_reader :address + + # Returns the integer port number given as argument +port+ in ::new. + attr_reader :port + + # Sets or returns the string local host used to establish the connection; + # initially +nil+. + attr_accessor :local_host + + # Sets or returns the integer local port used to establish the connection; + # initially +nil+. + attr_accessor :local_port + + # Returns the encoding to use for the response body; + # see #response_body_encoding=. + attr_reader :response_body_encoding + + # Sets the encoding to be used for the response body; + # returns the encoding. + # + # The given +value+ may be: + # + # - An Encoding object. + # - The name of an encoding. + # - An alias for an encoding name. + # + # See {Encoding}[rdoc-ref:Encoding]. + # + # Examples: + # + # http = Gem::Net::HTTP.new(hostname) + # http.response_body_encoding = Encoding::US_ASCII # => #<Encoding:US-ASCII> + # http.response_body_encoding = 'US-ASCII' # => "US-ASCII" + # http.response_body_encoding = 'ASCII' # => "ASCII" + # + def response_body_encoding=(value) + value = Encoding.find(value) if value.is_a?(String) + @response_body_encoding = value + end + + # Sets whether to determine the proxy from environment variable + # '<tt>ENV['http_proxy']</tt>'; + # see {Proxy Using ENV['http_proxy']}[rdoc-ref:Gem::Net::HTTP@Proxy+Using+-27ENV-5B-27http_proxy-27-5D-27]. + attr_writer :proxy_from_env + + # Sets the proxy address; + # see {Proxy Server}[rdoc-ref:Gem::Net::HTTP@Proxy+Server]. + attr_writer :proxy_address + + # Sets the proxy port; + # see {Proxy Server}[rdoc-ref:Gem::Net::HTTP@Proxy+Server]. + attr_writer :proxy_port + + # Sets the proxy user; + # see {Proxy Server}[rdoc-ref:Gem::Net::HTTP@Proxy+Server]. + attr_writer :proxy_user + + # Sets the proxy password; + # see {Proxy Server}[rdoc-ref:Gem::Net::HTTP@Proxy+Server]. + attr_writer :proxy_pass + + # Returns the IP address for the connection. + # + # If the session has not been started, + # returns the value set by #ipaddr=, + # or +nil+ if it has not been set: + # + # http = Gem::Net::HTTP.new(hostname) + # http.ipaddr # => nil + # http.ipaddr = '172.67.155.76' + # http.ipaddr # => "172.67.155.76" + # + # If the session has been started, + # returns the IP address from the socket: + # + # http = Gem::Net::HTTP.new(hostname) + # http.start + # http.ipaddr # => "172.67.155.76" + # http.finish + # + def ipaddr + started? ? @socket.io.peeraddr[3] : @ipaddr + end + + # Sets the IP address for the connection: + # + # http = Gem::Net::HTTP.new(hostname) + # http.ipaddr # => nil + # http.ipaddr = '172.67.155.76' + # http.ipaddr # => "172.67.155.76" + # + # The IP address may not be set if the session has been started. + def ipaddr=(addr) + raise IOError, "ipaddr value changed, but session already started" if started? + @ipaddr = addr + end + + # Sets or returns the numeric (\Integer or \Float) number of seconds + # to wait for a connection to open; + # initially 60. + # If the connection is not made in the given interval, + # an exception is raised. + attr_accessor :open_timeout + + # Returns the numeric (\Integer or \Float) number of seconds + # to wait for one block to be read (via one read(2) call); + # see #read_timeout=. + attr_reader :read_timeout + + # Returns the numeric (\Integer or \Float) number of seconds + # to wait for one block to be written (via one write(2) call); + # see #write_timeout=. + attr_reader :write_timeout + + # Sets the maximum number of times to retry an idempotent request in case of + # \Gem::Net::ReadTimeout, IOError, EOFError, Errno::ECONNRESET, + # Errno::ECONNABORTED, Errno::EPIPE, OpenSSL::SSL::SSLError, + # Gem::Timeout::Error. + # The initial value is 1. + # + # Argument +retries+ must be a non-negative numeric value: + # + # http = Gem::Net::HTTP.new(hostname) + # http.max_retries = 2 # => 2 + # http.max_retries # => 2 + # + def max_retries=(retries) + retries = retries.to_int + if retries < 0 + raise ArgumentError, 'max_retries should be non-negative integer number' + end + @max_retries = retries + end + + # Returns the maximum number of times to retry an idempotent request; + # see #max_retries=. + attr_reader :max_retries + + # Sets the read timeout, in seconds, for +self+ to integer +sec+; + # the initial value is 60. + # + # Argument +sec+ must be a non-negative numeric value: + # + # http = Gem::Net::HTTP.new(hostname) + # http.read_timeout # => 60 + # http.get('/todos/1') # => #<Gem::Net::HTTPOK 200 OK readbody=true> + # http.read_timeout = 0 + # http.get('/todos/1') # Raises Gem::Net::ReadTimeout. + # + def read_timeout=(sec) + @socket.read_timeout = sec if @socket + @read_timeout = sec + end + + # Sets the write timeout, in seconds, for +self+ to integer +sec+; + # the initial value is 60. + # + # Argument +sec+ must be a non-negative numeric value: + # + # _uri = uri.dup + # _uri.path = '/posts' + # body = 'bar' * 200000 + # data = <<EOF + # {"title": "foo", "body": "#{body}", "userId": "1"} + # EOF + # headers = {'content-type': 'application/json'} + # http = Gem::Net::HTTP.new(hostname) + # http.write_timeout # => 60 + # http.post(_uri.path, data, headers) + # # => #<Gem::Net::HTTPCreated 201 Created readbody=true> + # http.write_timeout = 0 + # http.post(_uri.path, data, headers) # Raises Gem::Net::WriteTimeout. + # + def write_timeout=(sec) + @socket.write_timeout = sec if @socket + @write_timeout = sec + end + + # Returns the continue timeout value; + # see continue_timeout=. + attr_reader :continue_timeout + + # Sets the continue timeout value, + # which is the number of seconds to wait for an expected 100 Continue response. + # If the \HTTP object does not receive a response in this many seconds + # it sends the request body. + def continue_timeout=(sec) + @socket.continue_timeout = sec if @socket + @continue_timeout = sec + end + + # Sets or returns the numeric (\Integer or \Float) number of seconds + # to keep the connection open after a request is sent; + # initially 2. + # If a new request is made during the given interval, + # the still-open connection is used; + # otherwise the connection will have been closed + # and a new connection is opened. + attr_accessor :keep_alive_timeout + + # Sets or returns whether to ignore end-of-file when reading a response body + # with <tt>Content-Length</tt> headers; + # initially +true+. + attr_accessor :ignore_eof + + # Returns +true+ if the \HTTP session has been started: + # + # http = Gem::Net::HTTP.new(hostname) + # http.started? # => false + # http.start + # http.started? # => true + # http.finish # => nil + # http.started? # => false + # + # Gem::Net::HTTP.start(hostname) do |http| + # http.started? + # end # => true + # http.started? # => false + # + def started? + @started + end + + alias active? started? #:nodoc: obsolete + + # Sets or returns whether to close the connection when the response is empty; + # initially +false+. + attr_accessor :close_on_empty_response + + # Returns +true+ if +self+ uses SSL, +false+ otherwise. + # See Gem::Net::HTTP#use_ssl=. + def use_ssl? + @use_ssl + end + + # Sets whether a new session is to use + # {Transport Layer Security}[https://en.wikipedia.org/wiki/Transport_Layer_Security]: + # + # Raises IOError if attempting to change during a session. + # + # Raises OpenSSL::SSL::SSLError if the port is not an HTTPS port. + def use_ssl=(flag) + flag = flag ? true : false + if started? and @use_ssl != flag + raise IOError, "use_ssl value changed, but session already started" + end + @use_ssl = flag + end + + SSL_IVNAMES = [ + :@ca_file, + :@ca_path, + :@cert, + :@cert_store, + :@ciphers, + :@extra_chain_cert, + :@key, + :@ssl_timeout, + :@ssl_version, + :@min_version, + :@max_version, + :@verify_callback, + :@verify_depth, + :@verify_mode, + :@verify_hostname, + ] # :nodoc: + SSL_ATTRIBUTES = [ + :ca_file, + :ca_path, + :cert, + :cert_store, + :ciphers, + :extra_chain_cert, + :key, + :ssl_timeout, + :ssl_version, + :min_version, + :max_version, + :verify_callback, + :verify_depth, + :verify_mode, + :verify_hostname, + ] # :nodoc: + + # Sets or returns the path to a CA certification file in PEM format. + attr_accessor :ca_file + + # Sets or returns the path of to CA directory + # containing certification files in PEM format. + attr_accessor :ca_path + + # Sets or returns the OpenSSL::X509::Certificate object + # to be used for client certification. + attr_accessor :cert + + # Sets or returns the X509::Store to be used for verifying peer certificate. + attr_accessor :cert_store + + # Sets or returns the available SSL ciphers. + # See {OpenSSL::SSL::SSLContext#ciphers=}[rdoc-ref:OpenSSL::SSL::SSLContext#ciphers-3D]. + attr_accessor :ciphers + + # Sets or returns the extra X509 certificates to be added to the certificate chain. + # See {OpenSSL::SSL::SSLContext#add_certificate}[rdoc-ref:OpenSSL::SSL::SSLContext#add_certificate]. + attr_accessor :extra_chain_cert + + # Sets or returns the OpenSSL::PKey::RSA or OpenSSL::PKey::DSA object. + attr_accessor :key + + # Sets or returns the SSL timeout seconds. + attr_accessor :ssl_timeout + + # Sets or returns the SSL version. + # See {OpenSSL::SSL::SSLContext#ssl_version=}[rdoc-ref:OpenSSL::SSL::SSLContext#ssl_version-3D]. + attr_accessor :ssl_version + + # Sets or returns the minimum SSL version. + # See {OpenSSL::SSL::SSLContext#min_version=}[rdoc-ref:OpenSSL::SSL::SSLContext#min_version-3D]. + attr_accessor :min_version + + # Sets or returns the maximum SSL version. + # See {OpenSSL::SSL::SSLContext#max_version=}[rdoc-ref:OpenSSL::SSL::SSLContext#max_version-3D]. + attr_accessor :max_version + + # Sets or returns the callback for the server certification verification. + attr_accessor :verify_callback + + # Sets or returns the maximum depth for the certificate chain verification. + attr_accessor :verify_depth + + # Sets or returns the flags for server the certification verification + # at the beginning of the SSL/TLS session. + # OpenSSL::SSL::VERIFY_NONE or OpenSSL::SSL::VERIFY_PEER are acceptable. + attr_accessor :verify_mode + + # Sets or returns whether to verify that the server certificate is valid + # for the hostname. + # See {OpenSSL::SSL::SSLContext#verify_hostname=}[rdoc-ref:OpenSSL::SSL::SSLContext#attribute-i-verify_mode]. + attr_accessor :verify_hostname + + # Returns the X509 certificate chain (an array of strings) + # for the session's socket peer, + # or +nil+ if none. + def peer_cert + if not use_ssl? or not @socket + return nil + end + @socket.io.peer_cert + end + + # Starts an \HTTP session. + # + # Without a block, returns +self+: + # + # http = Gem::Net::HTTP.new(hostname) + # # => #<Gem::Net::HTTP jsonplaceholder.typicode.com:80 open=false> + # http.start + # # => #<Gem::Net::HTTP jsonplaceholder.typicode.com:80 open=true> + # http.started? # => true + # http.finish + # + # With a block, calls the block with +self+, + # finishes the session when the block exits, + # and returns the block's value: + # + # http.start do |http| + # http + # end + # # => #<Gem::Net::HTTP jsonplaceholder.typicode.com:80 open=false> + # http.started? # => false + # + def start # :yield: http + raise IOError, 'HTTP session already opened' if @started + if block_given? + begin + do_start + return yield(self) + ensure + do_finish + end + end + do_start + self + end + + def do_start + connect + @started = true + end + private :do_start + + def connect + if use_ssl? + # reference early to load OpenSSL before connecting, + # as OpenSSL may take time to load. + @ssl_context = OpenSSL::SSL::SSLContext.new + end + + if proxy? then + conn_addr = proxy_address + conn_port = proxy_port + else + conn_addr = conn_address + conn_port = port + end + + debug "opening connection to #{conn_addr}:#{conn_port}..." + s = Gem::Timeout.timeout(@open_timeout, Gem::Net::OpenTimeout) { + begin + TCPSocket.open(conn_addr, conn_port, @local_host, @local_port) + rescue => e + raise e, "Failed to open TCP connection to " + + "#{conn_addr}:#{conn_port} (#{e.message})" + end + } + s.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) + debug "opened" + if use_ssl? + if proxy? + plain_sock = BufferedIO.new(s, read_timeout: @read_timeout, + write_timeout: @write_timeout, + continue_timeout: @continue_timeout, + debug_output: @debug_output) + buf = +"CONNECT #{conn_address}:#{@port} HTTP/#{HTTPVersion}\r\n" \ + "Host: #{@address}:#{@port}\r\n" + if proxy_user + credential = ["#{proxy_user}:#{proxy_pass}"].pack('m0') + buf << "Proxy-Authorization: Basic #{credential}\r\n" + end + buf << "\r\n" + plain_sock.write(buf) + HTTPResponse.read_new(plain_sock).value + # assuming nothing left in buffers after successful CONNECT response + end + + ssl_parameters = Hash.new + iv_list = instance_variables + SSL_IVNAMES.each_with_index do |ivname, i| + if iv_list.include?(ivname) + value = instance_variable_get(ivname) + unless value.nil? + ssl_parameters[SSL_ATTRIBUTES[i]] = value + end + end + end + @ssl_context.set_params(ssl_parameters) + unless @ssl_context.session_cache_mode.nil? # a dummy method on JRuby + @ssl_context.session_cache_mode = + OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT | + OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE + end + if @ssl_context.respond_to?(:session_new_cb) # not implemented under JRuby + @ssl_context.session_new_cb = proc {|sock, sess| @ssl_session = sess } + end + + # Still do the post_connection_check below even if connecting + # to IP address + verify_hostname = @ssl_context.verify_hostname + + # Server Name Indication (SNI) RFC 3546/6066 + case @address + when Gem::Resolv::IPv4::Regex, Gem::Resolv::IPv6::Regex + # don't set SNI, as IP addresses in SNI is not valid + # per RFC 6066, section 3. + + # Avoid openssl warning + @ssl_context.verify_hostname = false + else + ssl_host_address = @address + end + + debug "starting SSL for #{conn_addr}:#{conn_port}..." + s = OpenSSL::SSL::SSLSocket.new(s, @ssl_context) + s.sync_close = true + s.hostname = ssl_host_address if s.respond_to?(:hostname=) && ssl_host_address + + if @ssl_session and + Process.clock_gettime(Process::CLOCK_REALTIME) < @ssl_session.time.to_f + @ssl_session.timeout + s.session = @ssl_session + end + ssl_socket_connect(s, @open_timeout) + if (@ssl_context.verify_mode != OpenSSL::SSL::VERIFY_NONE) && verify_hostname + s.post_connection_check(@address) + end + debug "SSL established, protocol: #{s.ssl_version}, cipher: #{s.cipher[0]}" + end + @socket = BufferedIO.new(s, read_timeout: @read_timeout, + write_timeout: @write_timeout, + continue_timeout: @continue_timeout, + debug_output: @debug_output) + @last_communicated = nil + on_connect + rescue => exception + if s + debug "Conn close because of connect error #{exception}" + s.close + end + raise + end + private :connect + + def on_connect + end + private :on_connect + + # Finishes the \HTTP session: + # + # http = Gem::Net::HTTP.new(hostname) + # http.start + # http.started? # => true + # http.finish # => nil + # http.started? # => false + # + # Raises IOError if not in a session. + def finish + raise IOError, 'HTTP session not yet started' unless started? + do_finish + end + + def do_finish + @started = false + @socket.close if @socket + @socket = nil + end + private :do_finish + + # + # proxy + # + + public + + # no proxy + @is_proxy_class = false + @proxy_from_env = false + @proxy_addr = nil + @proxy_port = nil + @proxy_user = nil + @proxy_pass = nil + + # Creates an \HTTP proxy class which behaves like \Gem::Net::HTTP, but + # performs all access via the specified proxy. + # + # This class is obsolete. You may pass these same parameters directly to + # \Gem::Net::HTTP.new. See Gem::Net::HTTP.new for details of the arguments. + def HTTP.Proxy(p_addr = :ENV, p_port = nil, p_user = nil, p_pass = nil) #:nodoc: + return self unless p_addr + + Class.new(self) { + @is_proxy_class = true + + if p_addr == :ENV then + @proxy_from_env = true + @proxy_address = nil + @proxy_port = nil + else + @proxy_from_env = false + @proxy_address = p_addr + @proxy_port = p_port || default_port + end + + @proxy_user = p_user + @proxy_pass = p_pass + } + end + + class << HTTP + # Returns true if self is a class which was created by HTTP::Proxy. + def proxy_class? + defined?(@is_proxy_class) ? @is_proxy_class : false + end + + # Returns the address of the proxy host, or +nil+ if none; + # see Gem::Net::HTTP@Proxy+Server. + attr_reader :proxy_address + + # Returns the port number of the proxy host, or +nil+ if none; + # see Gem::Net::HTTP@Proxy+Server. + attr_reader :proxy_port + + # Returns the user name for accessing the proxy, or +nil+ if none; + # see Gem::Net::HTTP@Proxy+Server. + attr_reader :proxy_user + + # Returns the password for accessing the proxy, or +nil+ if none; + # see Gem::Net::HTTP@Proxy+Server. + attr_reader :proxy_pass + end + + # Returns +true+ if a proxy server is defined, +false+ otherwise; + # see {Proxy Server}[rdoc-ref:Gem::Net::HTTP@Proxy+Server]. + def proxy? + !!(@proxy_from_env ? proxy_uri : @proxy_address) + end + + # Returns +true+ if the proxy server is defined in the environment, + # +false+ otherwise; + # see {Proxy Server}[rdoc-ref:Gem::Net::HTTP@Proxy+Server]. + def proxy_from_env? + @proxy_from_env + end + + # The proxy URI determined from the environment for this connection. + def proxy_uri # :nodoc: + return if @proxy_uri == false + @proxy_uri ||= Gem::URI::HTTP.new( + "http", nil, address, port, nil, nil, nil, nil, nil + ).find_proxy || false + @proxy_uri || nil + end + + # Returns the address of the proxy server, if defined, +nil+ otherwise; + # see {Proxy Server}[rdoc-ref:Gem::Net::HTTP@Proxy+Server]. + def proxy_address + if @proxy_from_env then + proxy_uri&.hostname + else + @proxy_address + end + end + + # Returns the port number of the proxy server, if defined, +nil+ otherwise; + # see {Proxy Server}[rdoc-ref:Gem::Net::HTTP@Proxy+Server]. + def proxy_port + if @proxy_from_env then + proxy_uri&.port + else + @proxy_port + end + end + + # Returns the user name of the proxy server, if defined, +nil+ otherwise; + # see {Proxy Server}[rdoc-ref:Gem::Net::HTTP@Proxy+Server]. + def proxy_user + if @proxy_from_env + user = proxy_uri&.user + unescape(user) if user + else + @proxy_user + end + end + + # Returns the password of the proxy server, if defined, +nil+ otherwise; + # see {Proxy Server}[rdoc-ref:Gem::Net::HTTP@Proxy+Server]. + def proxy_pass + if @proxy_from_env + pass = proxy_uri&.password + unescape(pass) if pass + else + @proxy_pass + end + end + + alias proxyaddr proxy_address #:nodoc: obsolete + alias proxyport proxy_port #:nodoc: obsolete + + private + + def unescape(value) + require 'cgi/util' + CGI.unescape(value) + end + + # without proxy, obsolete + + def conn_address # :nodoc: + @ipaddr || address() + end + + def conn_port # :nodoc: + port() + end + + def edit_path(path) + if proxy? + if path.start_with?("ftp://") || use_ssl? + path + else + "http://#{addr_port}#{path}" + end + else + path + end + end + + # + # HTTP operations + # + + public + + # :call-seq: + # get(path, initheader = nil) {|res| ... } + # + # Sends a GET request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Get object + # created from string +path+ and initial headers hash +initheader+. + # + # With a block given, calls the block with the response body: + # + # http = Gem::Net::HTTP.new(hostname) + # http.get('/todos/1') do |res| + # p res + # end # => #<Gem::Net::HTTPOK 200 OK readbody=true> + # + # Output: + # + # "{\n \"userId\": 1,\n \"id\": 1,\n \"title\": \"delectus aut autem\",\n \"completed\": false\n}" + # + # With no block given, simply returns the response object: + # + # http.get('/') # => #<Gem::Net::HTTPOK 200 OK readbody=true> + # + # Related: + # + # - Gem::Net::HTTP::Get: request class for \HTTP method GET. + # - Gem::Net::HTTP.get: sends GET request, returns response body. + # + def get(path, initheader = nil, dest = nil, &block) # :yield: +body_segment+ + res = nil + + request(Get.new(path, initheader)) {|r| + r.read_body dest, &block + res = r + } + res + end + + # Sends a HEAD request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Head object + # created from string +path+ and initial headers hash +initheader+: + # + # res = http.head('/todos/1') # => #<Gem::Net::HTTPOK 200 OK readbody=true> + # res.body # => nil + # res.to_hash.take(3) + # # => + # [["date", ["Wed, 15 Feb 2023 15:25:42 GMT"]], + # ["content-type", ["application/json; charset=utf-8"]], + # ["connection", ["close"]]] + # + def head(path, initheader = nil) + request(Head.new(path, initheader)) + end + + # :call-seq: + # post(path, data, initheader = nil) {|res| ... } + # + # Sends a POST request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Post object + # created from string +path+, string +data+, and initial headers hash +initheader+. + # + # With a block given, calls the block with the response body: + # + # data = '{"userId": 1, "id": 1, "title": "delectus aut autem", "completed": false}' + # http = Gem::Net::HTTP.new(hostname) + # http.post('/todos', data) do |res| + # p res + # end # => #<Gem::Net::HTTPCreated 201 Created readbody=true> + # + # Output: + # + # "{\n \"{\\\"userId\\\": 1, \\\"id\\\": 1, \\\"title\\\": \\\"delectus aut autem\\\", \\\"completed\\\": false}\": \"\",\n \"id\": 201\n}" + # + # With no block given, simply returns the response object: + # + # http.post('/todos', data) # => #<Gem::Net::HTTPCreated 201 Created readbody=true> + # + # Related: + # + # - Gem::Net::HTTP::Post: request class for \HTTP method POST. + # - Gem::Net::HTTP.post: sends POST request, returns response body. + # + def post(path, data, initheader = nil, dest = nil, &block) # :yield: +body_segment+ + send_entity(path, data, initheader, dest, Post, &block) + end + + # :call-seq: + # patch(path, data, initheader = nil) {|res| ... } + # + # Sends a PATCH request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Patch object + # created from string +path+, string +data+, and initial headers hash +initheader+. + # + # With a block given, calls the block with the response body: + # + # data = '{"userId": 1, "id": 1, "title": "delectus aut autem", "completed": false}' + # http = Gem::Net::HTTP.new(hostname) + # http.patch('/todos/1', data) do |res| + # p res + # end # => #<Gem::Net::HTTPOK 200 OK readbody=true> + # + # Output: + # + # "{\n \"userId\": 1,\n \"id\": 1,\n \"title\": \"delectus aut autem\",\n \"completed\": false,\n \"{\\\"userId\\\": 1, \\\"id\\\": 1, \\\"title\\\": \\\"delectus aut autem\\\", \\\"completed\\\": false}\": \"\"\n}" + # + # With no block given, simply returns the response object: + # + # http.patch('/todos/1', data) # => #<Gem::Net::HTTPCreated 201 Created readbody=true> + # + def patch(path, data, initheader = nil, dest = nil, &block) # :yield: +body_segment+ + send_entity(path, data, initheader, dest, Patch, &block) + end + + # Sends a PUT request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Put object + # created from string +path+, string +data+, and initial headers hash +initheader+. + # + # data = '{"userId": 1, "id": 1, "title": "delectus aut autem", "completed": false}' + # http = Gem::Net::HTTP.new(hostname) + # http.put('/todos/1', data) # => #<Gem::Net::HTTPOK 200 OK readbody=true> + # + def put(path, data, initheader = nil) + request(Put.new(path, initheader), data) + end + + # Sends a PROPPATCH request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Proppatch object + # created from string +path+, string +body+, and initial headers hash +initheader+. + # + # data = '{"userId": 1, "id": 1, "title": "delectus aut autem", "completed": false}' + # http = Gem::Net::HTTP.new(hostname) + # http.proppatch('/todos/1', data) + # + def proppatch(path, body, initheader = nil) + request(Proppatch.new(path, initheader), body) + end + + # Sends a LOCK request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Lock object + # created from string +path+, string +body+, and initial headers hash +initheader+. + # + # data = '{"userId": 1, "id": 1, "title": "delectus aut autem", "completed": false}' + # http = Gem::Net::HTTP.new(hostname) + # http.lock('/todos/1', data) + # + def lock(path, body, initheader = nil) + request(Lock.new(path, initheader), body) + end + + # Sends an UNLOCK request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Unlock object + # created from string +path+, string +body+, and initial headers hash +initheader+. + # + # data = '{"userId": 1, "id": 1, "title": "delectus aut autem", "completed": false}' + # http = Gem::Net::HTTP.new(hostname) + # http.unlock('/todos/1', data) + # + def unlock(path, body, initheader = nil) + request(Unlock.new(path, initheader), body) + end + + # Sends an Options request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Options object + # created from string +path+ and initial headers hash +initheader+. + # + # http = Gem::Net::HTTP.new(hostname) + # http.options('/') + # + def options(path, initheader = nil) + request(Options.new(path, initheader)) + end + + # Sends a PROPFIND request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Propfind object + # created from string +path+, string +body+, and initial headers hash +initheader+. + # + # data = '{"userId": 1, "id": 1, "title": "delectus aut autem", "completed": false}' + # http = Gem::Net::HTTP.new(hostname) + # http.propfind('/todos/1', data) + # + def propfind(path, body = nil, initheader = {'Depth' => '0'}) + request(Propfind.new(path, initheader), body) + end + + # Sends a DELETE request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Delete object + # created from string +path+ and initial headers hash +initheader+. + # + # http = Gem::Net::HTTP.new(hostname) + # http.delete('/todos/1') + # + def delete(path, initheader = {'Depth' => 'Infinity'}) + request(Delete.new(path, initheader)) + end + + # Sends a MOVE request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Move object + # created from string +path+ and initial headers hash +initheader+. + # + # http = Gem::Net::HTTP.new(hostname) + # http.move('/todos/1') + # + def move(path, initheader = nil) + request(Move.new(path, initheader)) + end + + # Sends a COPY request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Copy object + # created from string +path+ and initial headers hash +initheader+. + # + # http = Gem::Net::HTTP.new(hostname) + # http.copy('/todos/1') + # + def copy(path, initheader = nil) + request(Copy.new(path, initheader)) + end + + # Sends a MKCOL request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Mkcol object + # created from string +path+, string +body+, and initial headers hash +initheader+. + # + # data = '{"userId": 1, "id": 1, "title": "delectus aut autem", "completed": false}' + # http.mkcol('/todos/1', data) + # http = Gem::Net::HTTP.new(hostname) + # + def mkcol(path, body = nil, initheader = nil) + request(Mkcol.new(path, initheader), body) + end + + # Sends a TRACE request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Trace object + # created from string +path+ and initial headers hash +initheader+. + # + # http = Gem::Net::HTTP.new(hostname) + # http.trace('/todos/1') + # + def trace(path, initheader = nil) + request(Trace.new(path, initheader)) + end + + # Sends a GET request to the server; + # forms the response into a Gem::Net::HTTPResponse object. + # + # The request is based on the Gem::Net::HTTP::Get object + # created from string +path+ and initial headers hash +initheader+. + # + # With no block given, returns the response object: + # + # http = Gem::Net::HTTP.new(hostname) + # http.request_get('/todos') # => #<Gem::Net::HTTPOK 200 OK readbody=true> + # + # With a block given, calls the block with the response object + # and returns the response object: + # + # http.request_get('/todos') do |res| + # p res + # end # => #<Gem::Net::HTTPOK 200 OK readbody=true> + # + # Output: + # + # #<Gem::Net::HTTPOK 200 OK readbody=false> + # + def request_get(path, initheader = nil, &block) # :yield: +response+ + request(Get.new(path, initheader), &block) + end + + # Sends a HEAD request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Head object + # created from string +path+ and initial headers hash +initheader+. + # + # http = Gem::Net::HTTP.new(hostname) + # http.head('/todos/1') # => #<Gem::Net::HTTPOK 200 OK readbody=true> + # + def request_head(path, initheader = nil, &block) + request(Head.new(path, initheader), &block) + end + + # Sends a POST request to the server; + # forms the response into a Gem::Net::HTTPResponse object. + # + # The request is based on the Gem::Net::HTTP::Post object + # created from string +path+, string +data+, and initial headers hash +initheader+. + # + # With no block given, returns the response object: + # + # http = Gem::Net::HTTP.new(hostname) + # http.post('/todos', 'xyzzy') + # # => #<Gem::Net::HTTPCreated 201 Created readbody=true> + # + # With a block given, calls the block with the response body + # and returns the response object: + # + # http.post('/todos', 'xyzzy') do |res| + # p res + # end # => #<Gem::Net::HTTPCreated 201 Created readbody=true> + # + # Output: + # + # "{\n \"xyzzy\": \"\",\n \"id\": 201\n}" + # + def request_post(path, data, initheader = nil, &block) # :yield: +response+ + request Post.new(path, initheader), data, &block + end + + # Sends a PUT request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTP::Put object + # created from string +path+, string +data+, and initial headers hash +initheader+. + # + # http = Gem::Net::HTTP.new(hostname) + # http.put('/todos/1', 'xyzzy') + # # => #<Gem::Net::HTTPOK 200 OK readbody=true> + # + def request_put(path, data, initheader = nil, &block) #:nodoc: + request Put.new(path, initheader), data, &block + end + + alias get2 request_get #:nodoc: obsolete + alias head2 request_head #:nodoc: obsolete + alias post2 request_post #:nodoc: obsolete + alias put2 request_put #:nodoc: obsolete + + # Sends an \HTTP request to the server; + # returns an instance of a subclass of Gem::Net::HTTPResponse. + # + # The request is based on the Gem::Net::HTTPRequest object + # created from string +path+, string +data+, and initial headers hash +header+. + # That object is an instance of the + # {subclass of Gem::Net::HTTPRequest}[rdoc-ref:Gem::Net::HTTPRequest@Request+Subclasses], + # that corresponds to the given uppercase string +name+, + # which must be + # an {HTTP request method}[https://en.wikipedia.org/wiki/HTTP#Request_methods] + # or a {WebDAV request method}[https://en.wikipedia.org/wiki/WebDAV#Implementation]. + # + # Examples: + # + # http = Gem::Net::HTTP.new(hostname) + # http.send_request('GET', '/todos/1') + # # => #<Gem::Net::HTTPOK 200 OK readbody=true> + # http.send_request('POST', '/todos', 'xyzzy') + # # => #<Gem::Net::HTTPCreated 201 Created readbody=true> + # + def send_request(name, path, data = nil, header = nil) + has_response_body = name != 'HEAD' + r = HTTPGenericRequest.new(name,(data ? true : false),has_response_body,path,header) + request r, data + end + + # Sends the given request +req+ to the server; + # forms the response into a Gem::Net::HTTPResponse object. + # + # The given +req+ must be an instance of a + # {subclass of Gem::Net::HTTPRequest}[rdoc-ref:Gem::Net::HTTPRequest@Request+Subclasses]. + # Argument +body+ should be given only if needed for the request. + # + # With no block given, returns the response object: + # + # http = Gem::Net::HTTP.new(hostname) + # + # req = Gem::Net::HTTP::Get.new('/todos/1') + # http.request(req) + # # => #<Gem::Net::HTTPOK 200 OK readbody=true> + # + # req = Gem::Net::HTTP::Post.new('/todos') + # http.request(req, 'xyzzy') + # # => #<Gem::Net::HTTPCreated 201 Created readbody=true> + # + # With a block given, calls the block with the response and returns the response: + # + # req = Gem::Net::HTTP::Get.new('/todos/1') + # http.request(req) do |res| + # p res + # end # => #<Gem::Net::HTTPOK 200 OK readbody=true> + # + # Output: + # + # #<Gem::Net::HTTPOK 200 OK readbody=false> + # + def request(req, body = nil, &block) # :yield: +response+ + unless started? + start { + req['connection'] ||= 'close' + return request(req, body, &block) + } + end + if proxy_user() + req.proxy_basic_auth proxy_user(), proxy_pass() unless use_ssl? + end + req.set_body_internal body + res = transport_request(req, &block) + if sspi_auth?(res) + sspi_auth(req) + res = transport_request(req, &block) + end + res + end + + private + + # Executes a request which uses a representation + # and returns its body. + def send_entity(path, data, initheader, dest, type, &block) + res = nil + request(type.new(path, initheader), data) {|r| + r.read_body dest, &block + res = r + } + res + end + + IDEMPOTENT_METHODS_ = %w/GET HEAD PUT DELETE OPTIONS TRACE/ # :nodoc: + + def transport_request(req) + count = 0 + begin + begin_transport req + res = catch(:response) { + begin + req.exec @socket, @curr_http_version, edit_path(req.path) + rescue Errno::EPIPE + # Failure when writing full request, but we can probably + # still read the received response. + end + + begin + res = HTTPResponse.read_new(@socket) + res.decode_content = req.decode_content + res.body_encoding = @response_body_encoding + res.ignore_eof = @ignore_eof + end while res.kind_of?(HTTPInformation) + + res.uri = req.uri + + res + } + res.reading_body(@socket, req.response_body_permitted?) { + yield res if block_given? + } + rescue Gem::Net::OpenTimeout + raise + rescue Gem::Net::ReadTimeout, IOError, EOFError, + Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE, Errno::ETIMEDOUT, + # avoid a dependency on OpenSSL + defined?(OpenSSL::SSL) ? OpenSSL::SSL::SSLError : IOError, + Gem::Timeout::Error => exception + if count < max_retries && IDEMPOTENT_METHODS_.include?(req.method) + count += 1 + @socket.close if @socket + debug "Conn close because of error #{exception}, and retry" + retry + end + debug "Conn close because of error #{exception}" + @socket.close if @socket + raise + end + + end_transport req, res + res + rescue => exception + debug "Conn close because of error #{exception}" + @socket.close if @socket + raise exception + end + + def begin_transport(req) + if @socket.closed? + connect + elsif @last_communicated + if @last_communicated + @keep_alive_timeout < Process.clock_gettime(Process::CLOCK_MONOTONIC) + debug 'Conn close because of keep_alive_timeout' + @socket.close + connect + elsif @socket.io.to_io.wait_readable(0) && @socket.eof? + debug "Conn close because of EOF" + @socket.close + connect + end + end + + if not req.response_body_permitted? and @close_on_empty_response + req['connection'] ||= 'close' + end + + req.update_uri address, port, use_ssl? + req['host'] ||= addr_port() + end + + def end_transport(req, res) + @curr_http_version = res.http_version + @last_communicated = nil + if @socket.closed? + debug 'Conn socket closed' + elsif not res.body and @close_on_empty_response + debug 'Conn close' + @socket.close + elsif keep_alive?(req, res) + debug 'Conn keep-alive' + @last_communicated = Process.clock_gettime(Process::CLOCK_MONOTONIC) + else + debug 'Conn close' + @socket.close + end + end + + def keep_alive?(req, res) + return false if req.connection_close? + if @curr_http_version <= '1.0' + res.connection_keep_alive? + else # HTTP/1.1 or later + not res.connection_close? + end + end + + def sspi_auth?(res) + return false unless @sspi_enabled + if res.kind_of?(HTTPProxyAuthenticationRequired) and + proxy? and res["Proxy-Authenticate"].include?("Negotiate") + begin + require 'win32/sspi' + true + rescue LoadError + false + end + else + false + end + end + + def sspi_auth(req) + n = Win32::SSPI::NegotiateAuth.new + req["Proxy-Authorization"] = "Negotiate #{n.get_initial_token}" + # Some versions of ISA will close the connection if this isn't present. + req["Connection"] = "Keep-Alive" + req["Proxy-Connection"] = "Keep-Alive" + res = transport_request(req) + authphrase = res["Proxy-Authenticate"] or return res + req["Proxy-Authorization"] = "Negotiate #{n.complete_authentication(authphrase)}" + rescue => err + raise HTTPAuthenticationError.new('HTTP authentication failed', err) + end + + # + # utils + # + + private + + def addr_port + addr = address + addr = "[#{addr}]" if addr.include?(":") + default_port = use_ssl? ? HTTP.https_default_port : HTTP.http_default_port + default_port == port ? addr : "#{addr}:#{port}" + end + + # Adds a message to debugging output + def debug(msg) + return unless @debug_output + @debug_output << msg + @debug_output << "\n" + end + + alias_method :D, :debug + end + +end + +require_relative 'http/exceptions' + +require_relative 'http/header' + +require_relative 'http/generic_request' +require_relative 'http/request' +require_relative 'http/requests' + +require_relative 'http/response' +require_relative 'http/responses' + +require_relative 'http/proxy_delta' + +require_relative 'http/backward' diff --git a/lib/rubygems/vendor/net-http/lib/net/http/backward.rb b/lib/rubygems/vendor/net-http/lib/net/http/backward.rb new file mode 100644 index 0000000000..10dbc16224 --- /dev/null +++ b/lib/rubygems/vendor/net-http/lib/net/http/backward.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true +# for backward compatibility + +# :enddoc: + +class Gem::Net::HTTP + ProxyMod = ProxyDelta + deprecate_constant :ProxyMod +end + +module Gem::Net::NetPrivate + HTTPRequest = ::Gem::Net::HTTPRequest + deprecate_constant :HTTPRequest +end + +module Gem::Net + HTTPSession = HTTP + + HTTPInformationCode = HTTPInformation + HTTPSuccessCode = HTTPSuccess + HTTPRedirectionCode = HTTPRedirection + HTTPRetriableCode = HTTPRedirection + HTTPClientErrorCode = HTTPClientError + HTTPFatalErrorCode = HTTPClientError + HTTPServerErrorCode = HTTPServerError + HTTPResponseReceiver = HTTPResponse + + HTTPResponceReceiver = HTTPResponse # Typo since 2001 + + deprecate_constant :HTTPSession, + :HTTPInformationCode, + :HTTPSuccessCode, + :HTTPRedirectionCode, + :HTTPRetriableCode, + :HTTPClientErrorCode, + :HTTPFatalErrorCode, + :HTTPServerErrorCode, + :HTTPResponseReceiver, + :HTTPResponceReceiver +end diff --git a/lib/rubygems/vendor/net-http/lib/net/http/exceptions.rb b/lib/rubygems/vendor/net-http/lib/net/http/exceptions.rb new file mode 100644 index 0000000000..c629c0113b --- /dev/null +++ b/lib/rubygems/vendor/net-http/lib/net/http/exceptions.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true +module Gem::Net + # Gem::Net::HTTP exception class. + # You cannot use Gem::Net::HTTPExceptions directly; instead, you must use + # its subclasses. + module HTTPExceptions + def initialize(msg, res) #:nodoc: + super msg + @response = res + end + attr_reader :response + alias data response #:nodoc: obsolete + end + + class HTTPError < ProtocolError + include HTTPExceptions + end + + class HTTPRetriableError < ProtoRetriableError + include HTTPExceptions + end + + class HTTPClientException < ProtoServerError + include HTTPExceptions + end + + class HTTPFatalError < ProtoFatalError + include HTTPExceptions + end + + # We cannot use the name "HTTPServerError", it is the name of the response. + HTTPServerException = HTTPClientException # :nodoc: + deprecate_constant(:HTTPServerException) +end diff --git a/lib/rubygems/vendor/net-http/lib/net/http/generic_request.rb b/lib/rubygems/vendor/net-http/lib/net/http/generic_request.rb new file mode 100644 index 0000000000..5cfe75a7cd --- /dev/null +++ b/lib/rubygems/vendor/net-http/lib/net/http/generic_request.rb @@ -0,0 +1,414 @@ +# frozen_string_literal: true +# +# \HTTPGenericRequest is the parent of the Gem::Net::HTTPRequest class. +# +# Do not use this directly; instead, use a subclass of Gem::Net::HTTPRequest. +# +# == About the Examples +# +# :include: doc/net-http/examples.rdoc +# +class Gem::Net::HTTPGenericRequest + + include Gem::Net::HTTPHeader + + def initialize(m, reqbody, resbody, uri_or_path, initheader = nil) # :nodoc: + @method = m + @request_has_body = reqbody + @response_has_body = resbody + + if Gem::URI === uri_or_path then + raise ArgumentError, "not an HTTP Gem::URI" unless Gem::URI::HTTP === uri_or_path + hostname = uri_or_path.hostname + raise ArgumentError, "no host component for Gem::URI" unless (hostname && hostname.length > 0) + @uri = uri_or_path.dup + host = @uri.hostname.dup + host << ":" << @uri.port.to_s if @uri.port != @uri.default_port + @path = uri_or_path.request_uri + raise ArgumentError, "no HTTP request path given" unless @path + else + @uri = nil + host = nil + raise ArgumentError, "no HTTP request path given" unless uri_or_path + raise ArgumentError, "HTTP request path is empty" if uri_or_path.empty? + @path = uri_or_path.dup + end + + @decode_content = false + + if Gem::Net::HTTP::HAVE_ZLIB then + if !initheader || + !initheader.keys.any? { |k| + %w[accept-encoding range].include? k.downcase + } then + @decode_content = true if @response_has_body + initheader = initheader ? initheader.dup : {} + initheader["accept-encoding"] = + "gzip;q=1.0,deflate;q=0.6,identity;q=0.3" + end + end + + initialize_http_header initheader + self['Accept'] ||= '*/*' + self['User-Agent'] ||= 'Ruby' + self['Host'] ||= host if host + @body = nil + @body_stream = nil + @body_data = nil + end + + # Returns the string method name for the request: + # + # Gem::Net::HTTP::Get.new(uri).method # => "GET" + # Gem::Net::HTTP::Post.new(uri).method # => "POST" + # + attr_reader :method + + # Returns the string path for the request: + # + # Gem::Net::HTTP::Get.new(uri).path # => "/" + # Gem::Net::HTTP::Post.new('example.com').path # => "example.com" + # + attr_reader :path + + # Returns the Gem::URI object for the request, or +nil+ if none: + # + # Gem::Net::HTTP::Get.new(uri).uri + # # => #<Gem::URI::HTTPS https://jsonplaceholder.typicode.com/> + # Gem::Net::HTTP::Get.new('example.com').uri # => nil + # + attr_reader :uri + + # Returns +false+ if the request's header <tt>'Accept-Encoding'</tt> + # has been set manually or deleted + # (indicating that the user intends to handle encoding in the response), + # +true+ otherwise: + # + # req = Gem::Net::HTTP::Get.new(uri) # => #<Gem::Net::HTTP::Get GET> + # req['Accept-Encoding'] # => "gzip;q=1.0,deflate;q=0.6,identity;q=0.3" + # req.decode_content # => true + # req['Accept-Encoding'] = 'foo' + # req.decode_content # => false + # req.delete('Accept-Encoding') + # req.decode_content # => false + # + attr_reader :decode_content + + # Returns a string representation of the request: + # + # Gem::Net::HTTP::Post.new(uri).inspect # => "#<Gem::Net::HTTP::Post POST>" + # + def inspect + "\#<#{self.class} #{@method}>" + end + + ## + # Don't automatically decode response content-encoding if the user indicates + # they want to handle it. + + def []=(key, val) # :nodoc: + @decode_content = false if key.downcase == 'accept-encoding' + + super key, val + end + + # Returns whether the request may have a body: + # + # Gem::Net::HTTP::Post.new(uri).request_body_permitted? # => true + # Gem::Net::HTTP::Get.new(uri).request_body_permitted? # => false + # + def request_body_permitted? + @request_has_body + end + + # Returns whether the response may have a body: + # + # Gem::Net::HTTP::Post.new(uri).response_body_permitted? # => true + # Gem::Net::HTTP::Head.new(uri).response_body_permitted? # => false + # + def response_body_permitted? + @response_has_body + end + + def body_exist? # :nodoc: + warn "Gem::Net::HTTPRequest#body_exist? is obsolete; use response_body_permitted?", uplevel: 1 if $VERBOSE + response_body_permitted? + end + + # Returns the string body for the request, or +nil+ if there is none: + # + # req = Gem::Net::HTTP::Post.new(uri) + # req.body # => nil + # req.body = '{"title": "foo","body": "bar","userId": 1}' + # req.body # => "{\"title\": \"foo\",\"body\": \"bar\",\"userId\": 1}" + # + attr_reader :body + + # Sets the body for the request: + # + # req = Gem::Net::HTTP::Post.new(uri) + # req.body # => nil + # req.body = '{"title": "foo","body": "bar","userId": 1}' + # req.body # => "{\"title\": \"foo\",\"body\": \"bar\",\"userId\": 1}" + # + def body=(str) + @body = str + @body_stream = nil + @body_data = nil + str + end + + # Returns the body stream object for the request, or +nil+ if there is none: + # + # req = Gem::Net::HTTP::Post.new(uri) # => #<Gem::Net::HTTP::Post POST> + # req.body_stream # => nil + # require 'stringio' + # req.body_stream = StringIO.new('xyzzy') # => #<StringIO:0x0000027d1e5affa8> + # req.body_stream # => #<StringIO:0x0000027d1e5affa8> + # + attr_reader :body_stream + + # Sets the body stream for the request: + # + # req = Gem::Net::HTTP::Post.new(uri) # => #<Gem::Net::HTTP::Post POST> + # req.body_stream # => nil + # require 'stringio' + # req.body_stream = StringIO.new('xyzzy') # => #<StringIO:0x0000027d1e5affa8> + # req.body_stream # => #<StringIO:0x0000027d1e5affa8> + # + def body_stream=(input) + @body = nil + @body_stream = input + @body_data = nil + input + end + + def set_body_internal(str) #:nodoc: internal use only + raise ArgumentError, "both of body argument and HTTPRequest#body set" if str and (@body or @body_stream) + self.body = str if str + if @body.nil? && @body_stream.nil? && @body_data.nil? && request_body_permitted? + self.body = '' + end + end + + # + # write + # + + def exec(sock, ver, path) #:nodoc: internal use only + if @body + send_request_with_body sock, ver, path, @body + elsif @body_stream + send_request_with_body_stream sock, ver, path, @body_stream + elsif @body_data + send_request_with_body_data sock, ver, path, @body_data + else + write_header sock, ver, path + end + end + + def update_uri(addr, port, ssl) # :nodoc: internal use only + # reflect the connection and @path to @uri + return unless @uri + + if ssl + scheme = 'https' + klass = Gem::URI::HTTPS + else + scheme = 'http' + klass = Gem::URI::HTTP + end + + if host = self['host'] + host.sub!(/:.*/m, '') + elsif host = @uri.host + else + host = addr + end + # convert the class of the Gem::URI + if @uri.is_a?(klass) + @uri.host = host + @uri.port = port + else + @uri = klass.new( + scheme, @uri.userinfo, + host, port, nil, + @uri.path, nil, @uri.query, nil) + end + end + + private + + class Chunker #:nodoc: + def initialize(sock) + @sock = sock + @prev = nil + end + + def write(buf) + # avoid memcpy() of buf, buf can huge and eat memory bandwidth + rv = buf.bytesize + @sock.write("#{rv.to_s(16)}\r\n", buf, "\r\n") + rv + end + + def finish + @sock.write("0\r\n\r\n") + end + end + + def send_request_with_body(sock, ver, path, body) + self.content_length = body.bytesize + delete 'Transfer-Encoding' + supply_default_content_type + write_header sock, ver, path + wait_for_continue sock, ver if sock.continue_timeout + sock.write body + end + + def send_request_with_body_stream(sock, ver, path, f) + unless content_length() or chunked? + raise ArgumentError, + "Content-Length not given and Transfer-Encoding is not `chunked'" + end + supply_default_content_type + write_header sock, ver, path + wait_for_continue sock, ver if sock.continue_timeout + if chunked? + chunker = Chunker.new(sock) + IO.copy_stream(f, chunker) + chunker.finish + else + IO.copy_stream(f, sock) + end + end + + def send_request_with_body_data(sock, ver, path, params) + if /\Amultipart\/form-data\z/i !~ self.content_type + self.content_type = 'application/x-www-form-urlencoded' + return send_request_with_body(sock, ver, path, Gem::URI.encode_www_form(params)) + end + + opt = @form_option.dup + require 'securerandom' unless defined?(SecureRandom) + opt[:boundary] ||= SecureRandom.urlsafe_base64(40) + self.set_content_type(self.content_type, boundary: opt[:boundary]) + if chunked? + write_header sock, ver, path + encode_multipart_form_data(sock, params, opt) + else + require 'tempfile' + file = Tempfile.new('multipart') + file.binmode + encode_multipart_form_data(file, params, opt) + file.rewind + self.content_length = file.size + write_header sock, ver, path + IO.copy_stream(file, sock) + file.close(true) + end + end + + def encode_multipart_form_data(out, params, opt) + charset = opt[:charset] + boundary = opt[:boundary] + require 'securerandom' unless defined?(SecureRandom) + boundary ||= SecureRandom.urlsafe_base64(40) + chunked_p = chunked? + + buf = +'' + params.each do |key, value, h={}| + key = quote_string(key, charset) + filename = + h.key?(:filename) ? h[:filename] : + value.respond_to?(:to_path) ? File.basename(value.to_path) : + nil + + buf << "--#{boundary}\r\n" + if filename + filename = quote_string(filename, charset) + type = h[:content_type] || 'application/octet-stream' + buf << "Content-Disposition: form-data; " \ + "name=\"#{key}\"; filename=\"#{filename}\"\r\n" \ + "Content-Type: #{type}\r\n\r\n" + if !out.respond_to?(:write) || !value.respond_to?(:read) + # if +out+ is not an IO or +value+ is not an IO + buf << (value.respond_to?(:read) ? value.read : value) + elsif value.respond_to?(:size) && chunked_p + # if +out+ is an IO and +value+ is a File, use IO.copy_stream + flush_buffer(out, buf, chunked_p) + out << "%x\r\n" % value.size if chunked_p + IO.copy_stream(value, out) + out << "\r\n" if chunked_p + else + # +out+ is an IO, and +value+ is not a File but an IO + flush_buffer(out, buf, chunked_p) + 1 while flush_buffer(out, value.read(4096), chunked_p) + end + else + # non-file field: + # HTML5 says, "The parts of the generated multipart/form-data + # resource that correspond to non-file fields must not have a + # Content-Type header specified." + buf << "Content-Disposition: form-data; name=\"#{key}\"\r\n\r\n" + buf << (value.respond_to?(:read) ? value.read : value) + end + buf << "\r\n" + end + buf << "--#{boundary}--\r\n" + flush_buffer(out, buf, chunked_p) + out << "0\r\n\r\n" if chunked_p + end + + def quote_string(str, charset) + str = str.encode(charset, fallback:->(c){'&#%d;'%c.encode("UTF-8").ord}) if charset + str.gsub(/[\\"]/, '\\\\\&') + end + + def flush_buffer(out, buf, chunked_p) + return unless buf + out << "%x\r\n"%buf.bytesize if chunked_p + out << buf + out << "\r\n" if chunked_p + buf.clear + end + + def supply_default_content_type + return if content_type() + warn 'net/http: Content-Type did not set; using application/x-www-form-urlencoded', uplevel: 1 if $VERBOSE + set_content_type 'application/x-www-form-urlencoded' + end + + ## + # Waits up to the continue timeout for a response from the server provided + # we're speaking HTTP 1.1 and are expecting a 100-continue response. + + def wait_for_continue(sock, ver) + if ver >= '1.1' and @header['expect'] and + @header['expect'].include?('100-continue') + if sock.io.to_io.wait_readable(sock.continue_timeout) + res = Gem::Net::HTTPResponse.read_new(sock) + unless res.kind_of?(Gem::Net::HTTPContinue) + res.decode_content = @decode_content + throw :response, res + end + end + end + end + + def write_header(sock, ver, path) + reqline = "#{@method} #{path} HTTP/#{ver}" + if /[\r\n]/ =~ reqline + raise ArgumentError, "A Request-Line must not contain CR or LF" + end + buf = +'' + buf << reqline << "\r\n" + each_capitalized do |k,v| + buf << "#{k}: #{v}\r\n" + end + buf << "\r\n" + sock.write buf + end + +end + diff --git a/lib/rubygems/vendor/net-http/lib/net/http/header.rb b/lib/rubygems/vendor/net-http/lib/net/http/header.rb new file mode 100644 index 0000000000..1488e60068 --- /dev/null +++ b/lib/rubygems/vendor/net-http/lib/net/http/header.rb @@ -0,0 +1,981 @@ +# frozen_string_literal: true +# +# The \HTTPHeader module provides access to \HTTP headers. +# +# The module is included in: +# +# - Gem::Net::HTTPGenericRequest (and therefore Gem::Net::HTTPRequest). +# - Gem::Net::HTTPResponse. +# +# The headers are a hash-like collection of key/value pairs called _fields_. +# +# == Request and Response Fields +# +# Headers may be included in: +# +# - A Gem::Net::HTTPRequest object: +# the object's headers will be sent with the request. +# Any fields may be defined in the request; +# see {Setters}[rdoc-ref:Gem::Net::HTTPHeader@Setters]. +# - A Gem::Net::HTTPResponse object: +# the objects headers are usually those returned from the host. +# Fields may be retrieved from the object; +# see {Getters}[rdoc-ref:Gem::Net::HTTPHeader@Getters] +# and {Iterators}[rdoc-ref:Gem::Net::HTTPHeader@Iterators]. +# +# Exactly which fields should be sent or expected depends on the host; +# see: +# +# - {Request fields}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Request_fields]. +# - {Response fields}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Response_fields]. +# +# == About the Examples +# +# :include: doc/net-http/examples.rdoc +# +# == Fields +# +# A header field is a key/value pair. +# +# === Field Keys +# +# A field key may be: +# +# - A string: Key <tt>'Accept'</tt> is treated as if it were +# <tt>'Accept'.downcase</tt>; i.e., <tt>'accept'</tt>. +# - A symbol: Key <tt>:Accept</tt> is treated as if it were +# <tt>:Accept.to_s.downcase</tt>; i.e., <tt>'accept'</tt>. +# +# Examples: +# +# req = Gem::Net::HTTP::Get.new(uri) +# req[:accept] # => "*/*" +# req['Accept'] # => "*/*" +# req['ACCEPT'] # => "*/*" +# +# req['accept'] = 'text/html' +# req[:accept] = 'text/html' +# req['ACCEPT'] = 'text/html' +# +# === Field Values +# +# A field value may be returned as an array of strings or as a string: +# +# - These methods return field values as arrays: +# +# - #get_fields: Returns the array value for the given key, +# or +nil+ if it does not exist. +# - #to_hash: Returns a hash of all header fields: +# each key is a field name; its value is the array value for the field. +# +# - These methods return field values as string; +# the string value for a field is equivalent to +# <tt>self[key.downcase.to_s].join(', '))</tt>: +# +# - #[]: Returns the string value for the given key, +# or +nil+ if it does not exist. +# - #fetch: Like #[], but accepts a default value +# to be returned if the key does not exist. +# +# The field value may be set: +# +# - #[]=: Sets the value for the given key; +# the given value may be a string, a symbol, an array, or a hash. +# - #add_field: Adds a given value to a value for the given key +# (not overwriting the existing value). +# - #delete: Deletes the field for the given key. +# +# Example field values: +# +# - \String: +# +# req['Accept'] = 'text/html' # => "text/html" +# req['Accept'] # => "text/html" +# req.get_fields('Accept') # => ["text/html"] +# +# - \Symbol: +# +# req['Accept'] = :text # => :text +# req['Accept'] # => "text" +# req.get_fields('Accept') # => ["text"] +# +# - Simple array: +# +# req[:foo] = %w[bar baz bat] +# req[:foo] # => "bar, baz, bat" +# req.get_fields(:foo) # => ["bar", "baz", "bat"] +# +# - Simple hash: +# +# req[:foo] = {bar: 0, baz: 1, bat: 2} +# req[:foo] # => "bar, 0, baz, 1, bat, 2" +# req.get_fields(:foo) # => ["bar", "0", "baz", "1", "bat", "2"] +# +# - Nested: +# +# req[:foo] = [%w[bar baz], {bat: 0, bam: 1}] +# req[:foo] # => "bar, baz, bat, 0, bam, 1" +# req.get_fields(:foo) # => ["bar", "baz", "bat", "0", "bam", "1"] +# +# req[:foo] = {bar: %w[baz bat], bam: {bah: 0, bad: 1}} +# req[:foo] # => "bar, baz, bat, bam, bah, 0, bad, 1" +# req.get_fields(:foo) # => ["bar", "baz", "bat", "bam", "bah", "0", "bad", "1"] +# +# == Convenience Methods +# +# Various convenience methods retrieve values, set values, query values, +# set form values, or iterate over fields. +# +# === Setters +# +# \Method #[]= can set any field, but does little to validate the new value; +# some of the other setter methods provide some validation: +# +# - #[]=: Sets the string or array value for the given key. +# - #add_field: Creates or adds to the array value for the given key. +# - #basic_auth: Sets the string authorization header for <tt>'Authorization'</tt>. +# - #content_length=: Sets the integer length for field <tt>'Content-Length</tt>. +# - #content_type=: Sets the string value for field <tt>'Content-Type'</tt>. +# - #proxy_basic_auth: Sets the string authorization header for <tt>'Proxy-Authorization'</tt>. +# - #set_range: Sets the value for field <tt>'Range'</tt>. +# +# === Form Setters +# +# - #set_form: Sets an HTML form data set. +# - #set_form_data: Sets header fields and a body from HTML form data. +# +# === Getters +# +# \Method #[] can retrieve the value of any field that exists, +# but always as a string; +# some of the other getter methods return something different +# from the simple string value: +# +# - #[]: Returns the string field value for the given key. +# - #content_length: Returns the integer value of field <tt>'Content-Length'</tt>. +# - #content_range: Returns the Range value of field <tt>'Content-Range'</tt>. +# - #content_type: Returns the string value of field <tt>'Content-Type'</tt>. +# - #fetch: Returns the string field value for the given key. +# - #get_fields: Returns the array field value for the given +key+. +# - #main_type: Returns first part of the string value of field <tt>'Content-Type'</tt>. +# - #sub_type: Returns second part of the string value of field <tt>'Content-Type'</tt>. +# - #range: Returns an array of Range objects of field <tt>'Range'</tt>, or +nil+. +# - #range_length: Returns the integer length of the range given in field <tt>'Content-Range'</tt>. +# - #type_params: Returns the string parameters for <tt>'Content-Type'</tt>. +# +# === Queries +# +# - #chunked?: Returns whether field <tt>'Transfer-Encoding'</tt> is set to <tt>'chunked'</tt>. +# - #connection_close?: Returns whether field <tt>'Connection'</tt> is set to <tt>'close'</tt>. +# - #connection_keep_alive?: Returns whether field <tt>'Connection'</tt> is set to <tt>'keep-alive'</tt>. +# - #key?: Returns whether a given key exists. +# +# === Iterators +# +# - #each_capitalized: Passes each field capitalized-name/value pair to the block. +# - #each_capitalized_name: Passes each capitalized field name to the block. +# - #each_header: Passes each field name/value pair to the block. +# - #each_name: Passes each field name to the block. +# - #each_value: Passes each string field value to the block. +# +module Gem::Net::HTTPHeader + MAX_KEY_LENGTH = 1024 + MAX_FIELD_LENGTH = 65536 + + def initialize_http_header(initheader) #:nodoc: + @header = {} + return unless initheader + initheader.each do |key, value| + warn "net/http: duplicated HTTP header: #{key}", uplevel: 3 if key?(key) and $VERBOSE + if value.nil? + warn "net/http: nil HTTP header: #{key}", uplevel: 3 if $VERBOSE + else + value = value.strip # raise error for invalid byte sequences + if key.to_s.bytesize > MAX_KEY_LENGTH + raise ArgumentError, "too long (#{key.bytesize} bytes) header: #{key[0, 30].inspect}..." + end + if value.to_s.bytesize > MAX_FIELD_LENGTH + raise ArgumentError, "header #{key} has too long field value: #{value.bytesize}" + end + if value.count("\r\n") > 0 + raise ArgumentError, "header #{key} has field value #{value.inspect}, this cannot include CR/LF" + end + @header[key.downcase.to_s] = [value] + end + end + end + + def size #:nodoc: obsolete + @header.size + end + + alias length size #:nodoc: obsolete + + # Returns the string field value for the case-insensitive field +key+, + # or +nil+ if there is no such key; + # see {Fields}[rdoc-ref:Gem::Net::HTTPHeader@Fields]: + # + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # res['Connection'] # => "keep-alive" + # res['Nosuch'] # => nil + # + # Note that some field values may be retrieved via convenience methods; + # see {Getters}[rdoc-ref:Gem::Net::HTTPHeader@Getters]. + def [](key) + a = @header[key.downcase.to_s] or return nil + a.join(', ') + end + + # Sets the value for the case-insensitive +key+ to +val+, + # overwriting the previous value if the field exists; + # see {Fields}[rdoc-ref:Gem::Net::HTTPHeader@Fields]: + # + # req = Gem::Net::HTTP::Get.new(uri) + # req['Accept'] # => "*/*" + # req['Accept'] = 'text/html' + # req['Accept'] # => "text/html" + # + # Note that some field values may be set via convenience methods; + # see {Setters}[rdoc-ref:Gem::Net::HTTPHeader@Setters]. + def []=(key, val) + unless val + @header.delete key.downcase.to_s + return val + end + set_field(key, val) + end + + # Adds value +val+ to the value array for field +key+ if the field exists; + # creates the field with the given +key+ and +val+ if it does not exist. + # see {Fields}[rdoc-ref:Gem::Net::HTTPHeader@Fields]: + # + # req = Gem::Net::HTTP::Get.new(uri) + # req.add_field('Foo', 'bar') + # req['Foo'] # => "bar" + # req.add_field('Foo', 'baz') + # req['Foo'] # => "bar, baz" + # req.add_field('Foo', %w[baz bam]) + # req['Foo'] # => "bar, baz, baz, bam" + # req.get_fields('Foo') # => ["bar", "baz", "baz", "bam"] + # + def add_field(key, val) + stringified_downcased_key = key.downcase.to_s + if @header.key?(stringified_downcased_key) + append_field_value(@header[stringified_downcased_key], val) + else + set_field(key, val) + end + end + + private def set_field(key, val) + case val + when Enumerable + ary = [] + append_field_value(ary, val) + @header[key.downcase.to_s] = ary + else + val = val.to_s # for compatibility use to_s instead of to_str + if val.b.count("\r\n") > 0 + raise ArgumentError, 'header field value cannot include CR/LF' + end + @header[key.downcase.to_s] = [val] + end + end + + private def append_field_value(ary, val) + case val + when Enumerable + val.each{|x| append_field_value(ary, x)} + else + val = val.to_s + if /[\r\n]/n.match?(val.b) + raise ArgumentError, 'header field value cannot include CR/LF' + end + ary.push val + end + end + + # Returns the array field value for the given +key+, + # or +nil+ if there is no such field; + # see {Fields}[rdoc-ref:Gem::Net::HTTPHeader@Fields]: + # + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # res.get_fields('Connection') # => ["keep-alive"] + # res.get_fields('Nosuch') # => nil + # + def get_fields(key) + stringified_downcased_key = key.downcase.to_s + return nil unless @header[stringified_downcased_key] + @header[stringified_downcased_key].dup + end + + # call-seq: + # fetch(key, default_val = nil) {|key| ... } -> object + # fetch(key, default_val = nil) -> value or default_val + # + # With a block, returns the string value for +key+ if it exists; + # otherwise returns the value of the block; + # ignores the +default_val+; + # see {Fields}[rdoc-ref:Gem::Net::HTTPHeader@Fields]: + # + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # + # # Field exists; block not called. + # res.fetch('Connection') do |value| + # fail 'Cannot happen' + # end # => "keep-alive" + # + # # Field does not exist; block called. + # res.fetch('Nosuch') do |value| + # value.downcase + # end # => "nosuch" + # + # With no block, returns the string value for +key+ if it exists; + # otherwise, returns +default_val+ if it was given; + # otherwise raises an exception: + # + # res.fetch('Connection', 'Foo') # => "keep-alive" + # res.fetch('Nosuch', 'Foo') # => "Foo" + # res.fetch('Nosuch') # Raises KeyError. + # + def fetch(key, *args, &block) #:yield: +key+ + a = @header.fetch(key.downcase.to_s, *args, &block) + a.kind_of?(Array) ? a.join(', ') : a + end + + # Calls the block with each key/value pair: + # + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # res.each_header do |key, value| + # p [key, value] if key.start_with?('c') + # end + # + # Output: + # + # ["content-type", "application/json; charset=utf-8"] + # ["connection", "keep-alive"] + # ["cache-control", "max-age=43200"] + # ["cf-cache-status", "HIT"] + # ["cf-ray", "771d17e9bc542cf5-ORD"] + # + # Returns an enumerator if no block is given. + # + # Gem::Net::HTTPHeader#each is an alias for Gem::Net::HTTPHeader#each_header. + def each_header #:yield: +key+, +value+ + block_given? or return enum_for(__method__) { @header.size } + @header.each do |k,va| + yield k, va.join(', ') + end + end + + alias each each_header + + # Calls the block with each field key: + # + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # res.each_key do |key| + # p key if key.start_with?('c') + # end + # + # Output: + # + # "content-type" + # "connection" + # "cache-control" + # "cf-cache-status" + # "cf-ray" + # + # Returns an enumerator if no block is given. + # + # Gem::Net::HTTPHeader#each_name is an alias for Gem::Net::HTTPHeader#each_key. + def each_name(&block) #:yield: +key+ + block_given? or return enum_for(__method__) { @header.size } + @header.each_key(&block) + end + + alias each_key each_name + + # Calls the block with each capitalized field name: + # + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # res.each_capitalized_name do |key| + # p key if key.start_with?('C') + # end + # + # Output: + # + # "Content-Type" + # "Connection" + # "Cache-Control" + # "Cf-Cache-Status" + # "Cf-Ray" + # + # The capitalization is system-dependent; + # see {Case Mapping}[https://docs.ruby-lang.org/en/master/case_mapping_rdoc.html]. + # + # Returns an enumerator if no block is given. + def each_capitalized_name #:yield: +key+ + block_given? or return enum_for(__method__) { @header.size } + @header.each_key do |k| + yield capitalize(k) + end + end + + # Calls the block with each string field value: + # + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # res.each_value do |value| + # p value if value.start_with?('c') + # end + # + # Output: + # + # "chunked" + # "cf-q-config;dur=6.0000002122251e-06" + # "cloudflare" + # + # Returns an enumerator if no block is given. + def each_value #:yield: +value+ + block_given? or return enum_for(__method__) { @header.size } + @header.each_value do |va| + yield va.join(', ') + end + end + + # Removes the header for the given case-insensitive +key+ + # (see {Fields}[rdoc-ref:Gem::Net::HTTPHeader@Fields]); + # returns the deleted value, or +nil+ if no such field exists: + # + # req = Gem::Net::HTTP::Get.new(uri) + # req.delete('Accept') # => ["*/*"] + # req.delete('Nosuch') # => nil + # + def delete(key) + @header.delete(key.downcase.to_s) + end + + # Returns +true+ if the field for the case-insensitive +key+ exists, +false+ otherwise: + # + # req = Gem::Net::HTTP::Get.new(uri) + # req.key?('Accept') # => true + # req.key?('Nosuch') # => false + # + def key?(key) + @header.key?(key.downcase.to_s) + end + + # Returns a hash of the key/value pairs: + # + # req = Gem::Net::HTTP::Get.new(uri) + # req.to_hash + # # => + # {"accept-encoding"=>["gzip;q=1.0,deflate;q=0.6,identity;q=0.3"], + # "accept"=>["*/*"], + # "user-agent"=>["Ruby"], + # "host"=>["jsonplaceholder.typicode.com"]} + # + def to_hash + @header.dup + end + + # Like #each_header, but the keys are returned in capitalized form. + # + # Gem::Net::HTTPHeader#canonical_each is an alias for Gem::Net::HTTPHeader#each_capitalized. + def each_capitalized + block_given? or return enum_for(__method__) { @header.size } + @header.each do |k,v| + yield capitalize(k), v.join(', ') + end + end + + alias canonical_each each_capitalized + + def capitalize(name) + name.to_s.split(/-/).map {|s| s.capitalize }.join('-') + end + private :capitalize + + # Returns an array of Range objects that represent + # the value of field <tt>'Range'</tt>, + # or +nil+ if there is no such field; + # see {Range request header}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#range-request-header]: + # + # req = Gem::Net::HTTP::Get.new(uri) + # req['Range'] = 'bytes=0-99,200-299,400-499' + # req.range # => [0..99, 200..299, 400..499] + # req.delete('Range') + # req.range # # => nil + # + def range + return nil unless @header['range'] + + value = self['Range'] + # byte-range-set = *( "," OWS ) ( byte-range-spec / suffix-byte-range-spec ) + # *( OWS "," [ OWS ( byte-range-spec / suffix-byte-range-spec ) ] ) + # corrected collected ABNF + # http://tools.ietf.org/html/draft-ietf-httpbis-p5-range-19#section-5.4.1 + # http://tools.ietf.org/html/draft-ietf-httpbis-p5-range-19#appendix-C + # http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-19#section-3.2.5 + unless /\Abytes=((?:,[ \t]*)*(?:\d+-\d*|-\d+)(?:[ \t]*,(?:[ \t]*\d+-\d*|-\d+)?)*)\z/ =~ value + raise Gem::Net::HTTPHeaderSyntaxError, "invalid syntax for byte-ranges-specifier: '#{value}'" + end + + byte_range_set = $1 + result = byte_range_set.split(/,/).map {|spec| + m = /(\d+)?\s*-\s*(\d+)?/i.match(spec) or + raise Gem::Net::HTTPHeaderSyntaxError, "invalid byte-range-spec: '#{spec}'" + d1 = m[1].to_i + d2 = m[2].to_i + if m[1] and m[2] + if d1 > d2 + raise Gem::Net::HTTPHeaderSyntaxError, "last-byte-pos MUST greater than or equal to first-byte-pos but '#{spec}'" + end + d1..d2 + elsif m[1] + d1..-1 + elsif m[2] + -d2..-1 + else + raise Gem::Net::HTTPHeaderSyntaxError, 'range is not specified' + end + } + # if result.empty? + # byte-range-set must include at least one byte-range-spec or suffix-byte-range-spec + # but above regexp already denies it. + if result.size == 1 && result[0].begin == 0 && result[0].end == -1 + raise Gem::Net::HTTPHeaderSyntaxError, 'only one suffix-byte-range-spec with zero suffix-length' + end + result + end + + # call-seq: + # set_range(length) -> length + # set_range(offset, length) -> range + # set_range(begin..length) -> range + # + # Sets the value for field <tt>'Range'</tt>; + # see {Range request header}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#range-request-header]: + # + # With argument +length+: + # + # req = Gem::Net::HTTP::Get.new(uri) + # req.set_range(100) # => 100 + # req['Range'] # => "bytes=0-99" + # + # With arguments +offset+ and +length+: + # + # req.set_range(100, 100) # => 100...200 + # req['Range'] # => "bytes=100-199" + # + # With argument +range+: + # + # req.set_range(100..199) # => 100..199 + # req['Range'] # => "bytes=100-199" + # + # Gem::Net::HTTPHeader#range= is an alias for Gem::Net::HTTPHeader#set_range. + def set_range(r, e = nil) + unless r + @header.delete 'range' + return r + end + r = (r...r+e) if e + case r + when Numeric + n = r.to_i + rangestr = (n > 0 ? "0-#{n-1}" : "-#{-n}") + when Range + first = r.first + last = r.end + last -= 1 if r.exclude_end? + if last == -1 + rangestr = (first > 0 ? "#{first}-" : "-#{-first}") + else + raise Gem::Net::HTTPHeaderSyntaxError, 'range.first is negative' if first < 0 + raise Gem::Net::HTTPHeaderSyntaxError, 'range.last is negative' if last < 0 + raise Gem::Net::HTTPHeaderSyntaxError, 'must be .first < .last' if first > last + rangestr = "#{first}-#{last}" + end + else + raise TypeError, 'Range/Integer is required' + end + @header['range'] = ["bytes=#{rangestr}"] + r + end + + alias range= set_range + + # Returns the value of field <tt>'Content-Length'</tt> as an integer, + # or +nil+ if there is no such field; + # see {Content-Length request header}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#content-length-request-header]: + # + # res = Gem::Net::HTTP.get_response(hostname, '/nosuch/1') + # res.content_length # => 2 + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # res.content_length # => nil + # + def content_length + return nil unless key?('Content-Length') + len = self['Content-Length'].slice(/\d+/) or + raise Gem::Net::HTTPHeaderSyntaxError, 'wrong Content-Length format' + len.to_i + end + + # Sets the value of field <tt>'Content-Length'</tt> to the given numeric; + # see {Content-Length response header}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#content-length-response-header]: + # + # _uri = uri.dup + # hostname = _uri.hostname # => "jsonplaceholder.typicode.com" + # _uri.path = '/posts' # => "/posts" + # req = Gem::Net::HTTP::Post.new(_uri) # => #<Gem::Net::HTTP::Post POST> + # req.body = '{"title": "foo","body": "bar","userId": 1}' + # req.content_length = req.body.size # => 42 + # req.content_type = 'application/json' + # res = Gem::Net::HTTP.start(hostname) do |http| + # http.request(req) + # end # => #<Gem::Net::HTTPCreated 201 Created readbody=true> + # + def content_length=(len) + unless len + @header.delete 'content-length' + return nil + end + @header['content-length'] = [len.to_i.to_s] + end + + # Returns +true+ if field <tt>'Transfer-Encoding'</tt> + # exists and has value <tt>'chunked'</tt>, + # +false+ otherwise; + # see {Transfer-Encoding response header}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#transfer-encoding-response-header]: + # + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # res['Transfer-Encoding'] # => "chunked" + # res.chunked? # => true + # + def chunked? + return false unless @header['transfer-encoding'] + field = self['Transfer-Encoding'] + (/(?:\A|[^\-\w])chunked(?![\-\w])/i =~ field) ? true : false + end + + # Returns a Range object representing the value of field + # <tt>'Content-Range'</tt>, or +nil+ if no such field exists; + # see {Content-Range response header}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#content-range-response-header]: + # + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # res['Content-Range'] # => nil + # res['Content-Range'] = 'bytes 0-499/1000' + # res['Content-Range'] # => "bytes 0-499/1000" + # res.content_range # => 0..499 + # + def content_range + return nil unless @header['content-range'] + m = %r<\A\s*(\w+)\s+(\d+)-(\d+)/(\d+|\*)>.match(self['Content-Range']) or + raise Gem::Net::HTTPHeaderSyntaxError, 'wrong Content-Range format' + return unless m[1] == 'bytes' + m[2].to_i .. m[3].to_i + end + + # Returns the integer representing length of the value of field + # <tt>'Content-Range'</tt>, or +nil+ if no such field exists; + # see {Content-Range response header}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#content-range-response-header]: + # + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # res['Content-Range'] # => nil + # res['Content-Range'] = 'bytes 0-499/1000' + # res.range_length # => 500 + # + def range_length + r = content_range() or return nil + r.end - r.begin + 1 + end + + # Returns the {media type}[https://en.wikipedia.org/wiki/Media_type] + # from the value of field <tt>'Content-Type'</tt>, + # or +nil+ if no such field exists; + # see {Content-Type response header}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#content-type-response-header]: + # + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # res['content-type'] # => "application/json; charset=utf-8" + # res.content_type # => "application/json" + # + def content_type + main = main_type() + return nil unless main + + sub = sub_type() + if sub + "#{main}/#{sub}" + else + main + end + end + + # Returns the leading ('type') part of the + # {media type}[https://en.wikipedia.org/wiki/Media_type] + # from the value of field <tt>'Content-Type'</tt>, + # or +nil+ if no such field exists; + # see {Content-Type response header}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#content-type-response-header]: + # + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # res['content-type'] # => "application/json; charset=utf-8" + # res.main_type # => "application" + # + def main_type + return nil unless @header['content-type'] + self['Content-Type'].split(';').first.to_s.split('/')[0].to_s.strip + end + + # Returns the trailing ('subtype') part of the + # {media type}[https://en.wikipedia.org/wiki/Media_type] + # from the value of field <tt>'Content-Type'</tt>, + # or +nil+ if no such field exists; + # see {Content-Type response header}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#content-type-response-header]: + # + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # res['content-type'] # => "application/json; charset=utf-8" + # res.sub_type # => "json" + # + def sub_type + return nil unless @header['content-type'] + _, sub = *self['Content-Type'].split(';').first.to_s.split('/') + return nil unless sub + sub.strip + end + + # Returns the trailing ('parameters') part of the value of field <tt>'Content-Type'</tt>, + # or +nil+ if no such field exists; + # see {Content-Type response header}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#content-type-response-header]: + # + # res = Gem::Net::HTTP.get_response(hostname, '/todos/1') + # res['content-type'] # => "application/json; charset=utf-8" + # res.type_params # => {"charset"=>"utf-8"} + # + def type_params + result = {} + list = self['Content-Type'].to_s.split(';') + list.shift + list.each do |param| + k, v = *param.split('=', 2) + result[k.strip] = v.strip + end + result + end + + # Sets the value of field <tt>'Content-Type'</tt>; + # returns the new value; + # see {Content-Type request header}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#content-type-request-header]: + # + # req = Gem::Net::HTTP::Get.new(uri) + # req.set_content_type('application/json') # => ["application/json"] + # + # Gem::Net::HTTPHeader#content_type= is an alias for Gem::Net::HTTPHeader#set_content_type. + def set_content_type(type, params = {}) + @header['content-type'] = [type + params.map{|k,v|"; #{k}=#{v}"}.join('')] + end + + alias content_type= set_content_type + + # Sets the request body to a URL-encoded string derived from argument +params+, + # and sets request header field <tt>'Content-Type'</tt> + # to <tt>'application/x-www-form-urlencoded'</tt>. + # + # The resulting request is suitable for HTTP request +POST+ or +PUT+. + # + # Argument +params+ must be suitable for use as argument +enum+ to + # {Gem::URI.encode_www_form}[https://docs.ruby-lang.org/en/master/Gem::URI.html#method-c-encode_www_form]. + # + # With only argument +params+ given, + # sets the body to a URL-encoded string with the default separator <tt>'&'</tt>: + # + # req = Gem::Net::HTTP::Post.new('example.com') + # + # req.set_form_data(q: 'ruby', lang: 'en') + # req.body # => "q=ruby&lang=en" + # req['Content-Type'] # => "application/x-www-form-urlencoded" + # + # req.set_form_data([['q', 'ruby'], ['lang', 'en']]) + # req.body # => "q=ruby&lang=en" + # + # req.set_form_data(q: ['ruby', 'perl'], lang: 'en') + # req.body # => "q=ruby&q=perl&lang=en" + # + # req.set_form_data([['q', 'ruby'], ['q', 'perl'], ['lang', 'en']]) + # req.body # => "q=ruby&q=perl&lang=en" + # + # With string argument +sep+ also given, + # uses that string as the separator: + # + # req.set_form_data({q: 'ruby', lang: 'en'}, '|') + # req.body # => "q=ruby|lang=en" + # + # Gem::Net::HTTPHeader#form_data= is an alias for Gem::Net::HTTPHeader#set_form_data. + def set_form_data(params, sep = '&') + query = Gem::URI.encode_www_form(params) + query.gsub!(/&/, sep) if sep != '&' + self.body = query + self.content_type = 'application/x-www-form-urlencoded' + end + + alias form_data= set_form_data + + # Stores form data to be used in a +POST+ or +PUT+ request. + # + # The form data given in +params+ consists of zero or more fields; + # each field is: + # + # - A scalar value. + # - A name/value pair. + # - An IO stream opened for reading. + # + # Argument +params+ should be an + # {Enumerable}[https://docs.ruby-lang.org/en/master/Enumerable.html#module-Enumerable-label-Enumerable+in+Ruby+Classes] + # (method <tt>params.map</tt> will be called), + # and is often an array or hash. + # + # First, we set up a request: + # + # _uri = uri.dup + # _uri.path ='/posts' + # req = Gem::Net::HTTP::Post.new(_uri) + # + # <b>Argument +params+ As an Array</b> + # + # When +params+ is an array, + # each of its elements is a subarray that defines a field; + # the subarray may contain: + # + # - One string: + # + # req.set_form([['foo'], ['bar'], ['baz']]) + # + # - Two strings: + # + # req.set_form([%w[foo 0], %w[bar 1], %w[baz 2]]) + # + # - When argument +enctype+ (see below) is given as + # <tt>'multipart/form-data'</tt>: + # + # - A string name and an IO stream opened for reading: + # + # require 'stringio' + # req.set_form([['file', StringIO.new('Ruby is cool.')]]) + # + # - A string name, an IO stream opened for reading, + # and an options hash, which may contain these entries: + # + # - +:filename+: The name of the file to use. + # - +:content_type+: The content type of the uploaded file. + # + # Example: + # + # req.set_form([['file', file, {filename: "other-filename.foo"}]] + # + # The various forms may be mixed: + # + # req.set_form(['foo', %w[bar 1], ['file', file]]) + # + # <b>Argument +params+ As a Hash</b> + # + # When +params+ is a hash, + # each of its entries is a name/value pair that defines a field: + # + # - The name is a string. + # - The value may be: + # + # - +nil+. + # - Another string. + # - An IO stream opened for reading + # (only when argument +enctype+ -- see below -- is given as + # <tt>'multipart/form-data'</tt>). + # + # Examples: + # + # # Nil-valued fields. + # req.set_form({'foo' => nil, 'bar' => nil, 'baz' => nil}) + # + # # String-valued fields. + # req.set_form({'foo' => 0, 'bar' => 1, 'baz' => 2}) + # + # # IO-valued field. + # require 'stringio' + # req.set_form({'file' => StringIO.new('Ruby is cool.')}) + # + # # Mixture of fields. + # req.set_form({'foo' => nil, 'bar' => 1, 'file' => file}) + # + # Optional argument +enctype+ specifies the value to be given + # to field <tt>'Content-Type'</tt>, and must be one of: + # + # - <tt>'application/x-www-form-urlencoded'</tt> (the default). + # - <tt>'multipart/form-data'</tt>; + # see {RFC 7578}[https://www.rfc-editor.org/rfc/rfc7578]. + # + # Optional argument +formopt+ is a hash of options + # (applicable only when argument +enctype+ + # is <tt>'multipart/form-data'</tt>) + # that may include the following entries: + # + # - +:boundary+: The value is the boundary string for the multipart message. + # If not given, the boundary is a random string. + # See {Boundary}[https://www.rfc-editor.org/rfc/rfc7578#section-4.1]. + # - +:charset+: Value is the character set for the form submission. + # Field names and values of non-file fields should be encoded with this charset. + # + def set_form(params, enctype='application/x-www-form-urlencoded', formopt={}) + @body_data = params + @body = nil + @body_stream = nil + @form_option = formopt + case enctype + when /\Aapplication\/x-www-form-urlencoded\z/i, + /\Amultipart\/form-data\z/i + self.content_type = enctype + else + raise ArgumentError, "invalid enctype: #{enctype}" + end + end + + # Sets header <tt>'Authorization'</tt> using the given + # +account+ and +password+ strings: + # + # req.basic_auth('my_account', 'my_password') + # req['Authorization'] + # # => "Basic bXlfYWNjb3VudDpteV9wYXNzd29yZA==" + # + def basic_auth(account, password) + @header['authorization'] = [basic_encode(account, password)] + end + + # Sets header <tt>'Proxy-Authorization'</tt> using the given + # +account+ and +password+ strings: + # + # req.proxy_basic_auth('my_account', 'my_password') + # req['Proxy-Authorization'] + # # => "Basic bXlfYWNjb3VudDpteV9wYXNzd29yZA==" + # + def proxy_basic_auth(account, password) + @header['proxy-authorization'] = [basic_encode(account, password)] + end + + def basic_encode(account, password) + 'Basic ' + ["#{account}:#{password}"].pack('m0') + end + private :basic_encode + +# Returns whether the HTTP session is to be closed. + def connection_close? + token = /(?:\A|,)\s*close\s*(?:\z|,)/i + @header['connection']&.grep(token) {return true} + @header['proxy-connection']&.grep(token) {return true} + false + end + +# Returns whether the HTTP session is to be kept alive. + def connection_keep_alive? + token = /(?:\A|,)\s*keep-alive\s*(?:\z|,)/i + @header['connection']&.grep(token) {return true} + @header['proxy-connection']&.grep(token) {return true} + false + end + +end diff --git a/lib/rubygems/vendor/net-http/lib/net/http/proxy_delta.rb b/lib/rubygems/vendor/net-http/lib/net/http/proxy_delta.rb new file mode 100644 index 0000000000..137295a883 --- /dev/null +++ b/lib/rubygems/vendor/net-http/lib/net/http/proxy_delta.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true +module Gem::Net::HTTP::ProxyDelta #:nodoc: internal use only + private + + def conn_address + proxy_address() + end + + def conn_port + proxy_port() + end + + def edit_path(path) + use_ssl? ? path : "http://#{addr_port()}#{path}" + end +end + diff --git a/lib/rubygems/vendor/net-http/lib/net/http/request.rb b/lib/rubygems/vendor/net-http/lib/net/http/request.rb new file mode 100644 index 0000000000..495ec9be54 --- /dev/null +++ b/lib/rubygems/vendor/net-http/lib/net/http/request.rb @@ -0,0 +1,88 @@ +# frozen_string_literal: true + +# This class is the base class for \Gem::Net::HTTP request classes. +# The class should not be used directly; +# instead you should use its subclasses, listed below. +# +# == Creating a Request +# +# An request object may be created with either a Gem::URI or a string hostname: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('https://jsonplaceholder.typicode.com/') +# req = Gem::Net::HTTP::Get.new(uri) # => #<Gem::Net::HTTP::Get GET> +# req = Gem::Net::HTTP::Get.new(uri.hostname) # => #<Gem::Net::HTTP::Get GET> +# +# And with any of the subclasses: +# +# req = Gem::Net::HTTP::Head.new(uri) # => #<Gem::Net::HTTP::Head HEAD> +# req = Gem::Net::HTTP::Post.new(uri) # => #<Gem::Net::HTTP::Post POST> +# req = Gem::Net::HTTP::Put.new(uri) # => #<Gem::Net::HTTP::Put PUT> +# # ... +# +# The new instance is suitable for use as the argument to Gem::Net::HTTP#request. +# +# == Request Headers +# +# A new request object has these header fields by default: +# +# req.to_hash +# # => +# {"accept-encoding"=>["gzip;q=1.0,deflate;q=0.6,identity;q=0.3"], +# "accept"=>["*/*"], +# "user-agent"=>["Ruby"], +# "host"=>["jsonplaceholder.typicode.com"]} +# +# See: +# +# - {Request header Accept-Encoding}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#Accept-Encoding] +# and {Compression and Decompression}[rdoc-ref:Gem::Net::HTTP@Compression+and+Decompression]. +# - {Request header Accept}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#accept-request-header]. +# - {Request header User-Agent}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#user-agent-request-header]. +# - {Request header Host}[https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#host-request-header]. +# +# You can add headers or override default headers: +# +# # res = Gem::Net::HTTP::Get.new(uri, {'foo' => '0', 'bar' => '1'}) +# +# This class (and therefore its subclasses) also includes (indirectly) +# module Gem::Net::HTTPHeader, which gives access to its +# {methods for setting headers}[rdoc-ref:Gem::Net::HTTPHeader@Setters]. +# +# == Request Subclasses +# +# Subclasses for HTTP requests: +# +# - Gem::Net::HTTP::Get +# - Gem::Net::HTTP::Head +# - Gem::Net::HTTP::Post +# - Gem::Net::HTTP::Put +# - Gem::Net::HTTP::Delete +# - Gem::Net::HTTP::Options +# - Gem::Net::HTTP::Trace +# - Gem::Net::HTTP::Patch +# +# Subclasses for WebDAV requests: +# +# - Gem::Net::HTTP::Propfind +# - Gem::Net::HTTP::Proppatch +# - Gem::Net::HTTP::Mkcol +# - Gem::Net::HTTP::Copy +# - Gem::Net::HTTP::Move +# - Gem::Net::HTTP::Lock +# - Gem::Net::HTTP::Unlock +# +class Gem::Net::HTTPRequest < Gem::Net::HTTPGenericRequest + # Creates an HTTP request object for +path+. + # + # +initheader+ are the default headers to use. Gem::Net::HTTP adds + # Accept-Encoding to enable compression of the response body unless + # Accept-Encoding or Range are supplied in +initheader+. + + def initialize(path, initheader = nil) + super self.class::METHOD, + self.class::REQUEST_HAS_BODY, + self.class::RESPONSE_HAS_BODY, + path, initheader + end +end diff --git a/lib/rubygems/vendor/net-http/lib/net/http/requests.rb b/lib/rubygems/vendor/net-http/lib/net/http/requests.rb new file mode 100644 index 0000000000..1a57ddc7c2 --- /dev/null +++ b/lib/rubygems/vendor/net-http/lib/net/http/requests.rb @@ -0,0 +1,425 @@ +# frozen_string_literal: true + +# HTTP/1.1 methods --- RFC2616 + +# \Class for representing +# {HTTP method GET}[https://en.wikipedia.org/w/index.php?title=Hypertext_Transfer_Protocol#GET_method]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# req = Gem::Net::HTTP::Get.new(uri) # => #<Gem::Net::HTTP::Get GET> +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Properties: +# +# - Request body: optional. +# - Response body: yes. +# - {Safe}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Safe_methods]: yes. +# - {Idempotent}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Idempotent_methods]: yes. +# - {Cacheable}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Cacheable_methods]: yes. +# +# Related: +# +# - Gem::Net::HTTP.get: sends +GET+ request, returns response body. +# - Gem::Net::HTTP#get: sends +GET+ request, returns response object. +# +class Gem::Net::HTTP::Get < Gem::Net::HTTPRequest + METHOD = 'GET' + REQUEST_HAS_BODY = false + RESPONSE_HAS_BODY = true +end + +# \Class for representing +# {HTTP method HEAD}[https://en.wikipedia.org/w/index.php?title=Hypertext_Transfer_Protocol#HEAD_method]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# req = Gem::Net::HTTP::Head.new(uri) # => #<Gem::Net::HTTP::Head HEAD> +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Properties: +# +# - Request body: optional. +# - Response body: no. +# - {Safe}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Safe_methods]: yes. +# - {Idempotent}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Idempotent_methods]: yes. +# - {Cacheable}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Cacheable_methods]: yes. +# +# Related: +# +# - Gem::Net::HTTP#head: sends +HEAD+ request, returns response object. +# +class Gem::Net::HTTP::Head < Gem::Net::HTTPRequest + METHOD = 'HEAD' + REQUEST_HAS_BODY = false + RESPONSE_HAS_BODY = false +end + +# \Class for representing +# {HTTP method POST}[https://en.wikipedia.org/w/index.php?title=Hypertext_Transfer_Protocol#POST_method]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# uri.path = '/posts' +# req = Gem::Net::HTTP::Post.new(uri) # => #<Gem::Net::HTTP::Post POST> +# req.body = '{"title": "foo","body": "bar","userId": 1}' +# req.content_type = 'application/json' +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Properties: +# +# - Request body: yes. +# - Response body: yes. +# - {Safe}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Safe_methods]: no. +# - {Idempotent}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Idempotent_methods]: no. +# - {Cacheable}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Cacheable_methods]: yes. +# +# Related: +# +# - Gem::Net::HTTP.post: sends +POST+ request, returns response object. +# - Gem::Net::HTTP#post: sends +POST+ request, returns response object. +# +class Gem::Net::HTTP::Post < Gem::Net::HTTPRequest + METHOD = 'POST' + REQUEST_HAS_BODY = true + RESPONSE_HAS_BODY = true +end + +# \Class for representing +# {HTTP method PUT}[https://en.wikipedia.org/w/index.php?title=Hypertext_Transfer_Protocol#PUT_method]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# uri.path = '/posts' +# req = Gem::Net::HTTP::Put.new(uri) # => #<Gem::Net::HTTP::Put PUT> +# req.body = '{"title": "foo","body": "bar","userId": 1}' +# req.content_type = 'application/json' +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Properties: +# +# - Request body: yes. +# - Response body: yes. +# - {Safe}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Safe_methods]: no. +# - {Idempotent}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Idempotent_methods]: yes. +# - {Cacheable}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Cacheable_methods]: no. +# +class Gem::Net::HTTP::Put < Gem::Net::HTTPRequest + METHOD = 'PUT' + REQUEST_HAS_BODY = true + RESPONSE_HAS_BODY = true +end + +# \Class for representing +# {HTTP method DELETE}[https://en.wikipedia.org/w/index.php?title=Hypertext_Transfer_Protocol#DELETE_method]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# uri.path = '/posts/1' +# req = Gem::Net::HTTP::Delete.new(uri) # => #<Gem::Net::HTTP::Delete DELETE> +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Properties: +# +# - Request body: optional. +# - Response body: yes. +# - {Safe}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Safe_methods]: no. +# - {Idempotent}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Idempotent_methods]: yes. +# - {Cacheable}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Cacheable_methods]: no. +# +# Related: +# +# - Gem::Net::HTTP#delete: sends +DELETE+ request, returns response object. +# +class Gem::Net::HTTP::Delete < Gem::Net::HTTPRequest + METHOD = 'DELETE' + REQUEST_HAS_BODY = false + RESPONSE_HAS_BODY = true +end + +# \Class for representing +# {HTTP method OPTIONS}[https://en.wikipedia.org/w/index.php?title=Hypertext_Transfer_Protocol#OPTIONS_method]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# req = Gem::Net::HTTP::Options.new(uri) # => #<Gem::Net::HTTP::Options OPTIONS> +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Properties: +# +# - Request body: optional. +# - Response body: yes. +# - {Safe}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Safe_methods]: yes. +# - {Idempotent}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Idempotent_methods]: yes. +# - {Cacheable}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Cacheable_methods]: no. +# +# Related: +# +# - Gem::Net::HTTP#options: sends +OPTIONS+ request, returns response object. +# +class Gem::Net::HTTP::Options < Gem::Net::HTTPRequest + METHOD = 'OPTIONS' + REQUEST_HAS_BODY = false + RESPONSE_HAS_BODY = true +end + +# \Class for representing +# {HTTP method TRACE}[https://en.wikipedia.org/w/index.php?title=Hypertext_Transfer_Protocol#TRACE_method]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# req = Gem::Net::HTTP::Trace.new(uri) # => #<Gem::Net::HTTP::Trace TRACE> +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Properties: +# +# - Request body: no. +# - Response body: yes. +# - {Safe}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Safe_methods]: yes. +# - {Idempotent}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Idempotent_methods]: yes. +# - {Cacheable}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Cacheable_methods]: no. +# +# Related: +# +# - Gem::Net::HTTP#trace: sends +TRACE+ request, returns response object. +# +class Gem::Net::HTTP::Trace < Gem::Net::HTTPRequest + METHOD = 'TRACE' + REQUEST_HAS_BODY = false + RESPONSE_HAS_BODY = true +end + +# \Class for representing +# {HTTP method PATCH}[https://en.wikipedia.org/w/index.php?title=Hypertext_Transfer_Protocol#PATCH_method]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# uri.path = '/posts' +# req = Gem::Net::HTTP::Patch.new(uri) # => #<Gem::Net::HTTP::Patch PATCH> +# req.body = '{"title": "foo","body": "bar","userId": 1}' +# req.content_type = 'application/json' +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Properties: +# +# - Request body: yes. +# - Response body: yes. +# - {Safe}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Safe_methods]: no. +# - {Idempotent}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Idempotent_methods]: no. +# - {Cacheable}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Cacheable_methods]: no. +# +# Related: +# +# - Gem::Net::HTTP#patch: sends +PATCH+ request, returns response object. +# +class Gem::Net::HTTP::Patch < Gem::Net::HTTPRequest + METHOD = 'PATCH' + REQUEST_HAS_BODY = true + RESPONSE_HAS_BODY = true +end + +# +# WebDAV methods --- RFC2518 +# + +# \Class for representing +# {WebDAV method PROPFIND}[http://www.webdav.org/specs/rfc4918.html#METHOD_PROPFIND]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# req = Gem::Net::HTTP::Propfind.new(uri) # => #<Gem::Net::HTTP::Propfind PROPFIND> +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Related: +# +# - Gem::Net::HTTP#propfind: sends +PROPFIND+ request, returns response object. +# +class Gem::Net::HTTP::Propfind < Gem::Net::HTTPRequest + METHOD = 'PROPFIND' + REQUEST_HAS_BODY = true + RESPONSE_HAS_BODY = true +end + +# \Class for representing +# {WebDAV method PROPPATCH}[http://www.webdav.org/specs/rfc4918.html#METHOD_PROPPATCH]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# req = Gem::Net::HTTP::Proppatch.new(uri) # => #<Gem::Net::HTTP::Proppatch PROPPATCH> +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Related: +# +# - Gem::Net::HTTP#proppatch: sends +PROPPATCH+ request, returns response object. +# +class Gem::Net::HTTP::Proppatch < Gem::Net::HTTPRequest + METHOD = 'PROPPATCH' + REQUEST_HAS_BODY = true + RESPONSE_HAS_BODY = true +end + +# \Class for representing +# {WebDAV method MKCOL}[http://www.webdav.org/specs/rfc4918.html#METHOD_MKCOL]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# req = Gem::Net::HTTP::Mkcol.new(uri) # => #<Gem::Net::HTTP::Mkcol MKCOL> +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Related: +# +# - Gem::Net::HTTP#mkcol: sends +MKCOL+ request, returns response object. +# +class Gem::Net::HTTP::Mkcol < Gem::Net::HTTPRequest + METHOD = 'MKCOL' + REQUEST_HAS_BODY = true + RESPONSE_HAS_BODY = true +end + +# \Class for representing +# {WebDAV method COPY}[http://www.webdav.org/specs/rfc4918.html#METHOD_COPY]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# req = Gem::Net::HTTP::Copy.new(uri) # => #<Gem::Net::HTTP::Copy COPY> +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Related: +# +# - Gem::Net::HTTP#copy: sends +COPY+ request, returns response object. +# +class Gem::Net::HTTP::Copy < Gem::Net::HTTPRequest + METHOD = 'COPY' + REQUEST_HAS_BODY = false + RESPONSE_HAS_BODY = true +end + +# \Class for representing +# {WebDAV method MOVE}[http://www.webdav.org/specs/rfc4918.html#METHOD_MOVE]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# req = Gem::Net::HTTP::Move.new(uri) # => #<Gem::Net::HTTP::Move MOVE> +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Related: +# +# - Gem::Net::HTTP#move: sends +MOVE+ request, returns response object. +# +class Gem::Net::HTTP::Move < Gem::Net::HTTPRequest + METHOD = 'MOVE' + REQUEST_HAS_BODY = false + RESPONSE_HAS_BODY = true +end + +# \Class for representing +# {WebDAV method LOCK}[http://www.webdav.org/specs/rfc4918.html#METHOD_LOCK]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# req = Gem::Net::HTTP::Lock.new(uri) # => #<Gem::Net::HTTP::Lock LOCK> +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Related: +# +# - Gem::Net::HTTP#lock: sends +LOCK+ request, returns response object. +# +class Gem::Net::HTTP::Lock < Gem::Net::HTTPRequest + METHOD = 'LOCK' + REQUEST_HAS_BODY = true + RESPONSE_HAS_BODY = true +end + +# \Class for representing +# {WebDAV method UNLOCK}[http://www.webdav.org/specs/rfc4918.html#METHOD_UNLOCK]: +# +# require 'rubygems/vendor/net-http/lib/net/http' +# uri = Gem::URI('http://example.com') +# hostname = uri.hostname # => "example.com" +# req = Gem::Net::HTTP::Unlock.new(uri) # => #<Gem::Net::HTTP::Unlock UNLOCK> +# res = Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end +# +# See {Request Headers}[rdoc-ref:Gem::Net::HTTPRequest@Request+Headers]. +# +# Related: +# +# - Gem::Net::HTTP#unlock: sends +UNLOCK+ request, returns response object. +# +class Gem::Net::HTTP::Unlock < Gem::Net::HTTPRequest + METHOD = 'UNLOCK' + REQUEST_HAS_BODY = true + RESPONSE_HAS_BODY = true +end + diff --git a/lib/rubygems/vendor/net-http/lib/net/http/response.rb b/lib/rubygems/vendor/net-http/lib/net/http/response.rb new file mode 100644 index 0000000000..cbbd191d87 --- /dev/null +++ b/lib/rubygems/vendor/net-http/lib/net/http/response.rb @@ -0,0 +1,738 @@ +# frozen_string_literal: true + +# This class is the base class for \Gem::Net::HTTP response classes. +# +# == About the Examples +# +# :include: doc/net-http/examples.rdoc +# +# == Returned Responses +# +# \Method Gem::Net::HTTP.get_response returns +# an instance of one of the subclasses of \Gem::Net::HTTPResponse: +# +# Gem::Net::HTTP.get_response(uri) +# # => #<Gem::Net::HTTPOK 200 OK readbody=true> +# Gem::Net::HTTP.get_response(hostname, '/nosuch') +# # => #<Gem::Net::HTTPNotFound 404 Not Found readbody=true> +# +# As does method Gem::Net::HTTP#request: +# +# req = Gem::Net::HTTP::Get.new(uri) +# Gem::Net::HTTP.start(hostname) do |http| +# http.request(req) +# end # => #<Gem::Net::HTTPOK 200 OK readbody=true> +# +# \Class \Gem::Net::HTTPResponse includes module Gem::Net::HTTPHeader, +# which provides access to response header values via (among others): +# +# - \Hash-like method <tt>[]</tt>. +# - Specific reader methods, such as +content_type+. +# +# Examples: +# +# res = Gem::Net::HTTP.get_response(uri) # => #<Gem::Net::HTTPOK 200 OK readbody=true> +# res['Content-Type'] # => "text/html; charset=UTF-8" +# res.content_type # => "text/html" +# +# == Response Subclasses +# +# \Class \Gem::Net::HTTPResponse has a subclass for each +# {HTTP status code}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes]. +# You can look up the response class for a given code: +# +# Gem::Net::HTTPResponse::CODE_TO_OBJ['200'] # => Gem::Net::HTTPOK +# Gem::Net::HTTPResponse::CODE_TO_OBJ['400'] # => Gem::Net::HTTPBadRequest +# Gem::Net::HTTPResponse::CODE_TO_OBJ['404'] # => Gem::Net::HTTPNotFound +# +# And you can retrieve the status code for a response object: +# +# Gem::Net::HTTP.get_response(uri).code # => "200" +# Gem::Net::HTTP.get_response(hostname, '/nosuch').code # => "404" +# +# The response subclasses (indentation shows class hierarchy): +# +# - Gem::Net::HTTPUnknownResponse (for unhandled \HTTP extensions). +# +# - Gem::Net::HTTPInformation: +# +# - Gem::Net::HTTPContinue (100) +# - Gem::Net::HTTPSwitchProtocol (101) +# - Gem::Net::HTTPProcessing (102) +# - Gem::Net::HTTPEarlyHints (103) +# +# - Gem::Net::HTTPSuccess: +# +# - Gem::Net::HTTPOK (200) +# - Gem::Net::HTTPCreated (201) +# - Gem::Net::HTTPAccepted (202) +# - Gem::Net::HTTPNonAuthoritativeInformation (203) +# - Gem::Net::HTTPNoContent (204) +# - Gem::Net::HTTPResetContent (205) +# - Gem::Net::HTTPPartialContent (206) +# - Gem::Net::HTTPMultiStatus (207) +# - Gem::Net::HTTPAlreadyReported (208) +# - Gem::Net::HTTPIMUsed (226) +# +# - Gem::Net::HTTPRedirection: +# +# - Gem::Net::HTTPMultipleChoices (300) +# - Gem::Net::HTTPMovedPermanently (301) +# - Gem::Net::HTTPFound (302) +# - Gem::Net::HTTPSeeOther (303) +# - Gem::Net::HTTPNotModified (304) +# - Gem::Net::HTTPUseProxy (305) +# - Gem::Net::HTTPTemporaryRedirect (307) +# - Gem::Net::HTTPPermanentRedirect (308) +# +# - Gem::Net::HTTPClientError: +# +# - Gem::Net::HTTPBadRequest (400) +# - Gem::Net::HTTPUnauthorized (401) +# - Gem::Net::HTTPPaymentRequired (402) +# - Gem::Net::HTTPForbidden (403) +# - Gem::Net::HTTPNotFound (404) +# - Gem::Net::HTTPMethodNotAllowed (405) +# - Gem::Net::HTTPNotAcceptable (406) +# - Gem::Net::HTTPProxyAuthenticationRequired (407) +# - Gem::Net::HTTPRequestTimeOut (408) +# - Gem::Net::HTTPConflict (409) +# - Gem::Net::HTTPGone (410) +# - Gem::Net::HTTPLengthRequired (411) +# - Gem::Net::HTTPPreconditionFailed (412) +# - Gem::Net::HTTPRequestEntityTooLarge (413) +# - Gem::Net::HTTPRequestURITooLong (414) +# - Gem::Net::HTTPUnsupportedMediaType (415) +# - Gem::Net::HTTPRequestedRangeNotSatisfiable (416) +# - Gem::Net::HTTPExpectationFailed (417) +# - Gem::Net::HTTPMisdirectedRequest (421) +# - Gem::Net::HTTPUnprocessableEntity (422) +# - Gem::Net::HTTPLocked (423) +# - Gem::Net::HTTPFailedDependency (424) +# - Gem::Net::HTTPUpgradeRequired (426) +# - Gem::Net::HTTPPreconditionRequired (428) +# - Gem::Net::HTTPTooManyRequests (429) +# - Gem::Net::HTTPRequestHeaderFieldsTooLarge (431) +# - Gem::Net::HTTPUnavailableForLegalReasons (451) +# +# - Gem::Net::HTTPServerError: +# +# - Gem::Net::HTTPInternalServerError (500) +# - Gem::Net::HTTPNotImplemented (501) +# - Gem::Net::HTTPBadGateway (502) +# - Gem::Net::HTTPServiceUnavailable (503) +# - Gem::Net::HTTPGatewayTimeOut (504) +# - Gem::Net::HTTPVersionNotSupported (505) +# - Gem::Net::HTTPVariantAlsoNegotiates (506) +# - Gem::Net::HTTPInsufficientStorage (507) +# - Gem::Net::HTTPLoopDetected (508) +# - Gem::Net::HTTPNotExtended (510) +# - Gem::Net::HTTPNetworkAuthenticationRequired (511) +# +# There is also the Gem::Net::HTTPBadResponse exception which is raised when +# there is a protocol error. +# +class Gem::Net::HTTPResponse + class << self + # true if the response has a body. + def body_permitted? + self::HAS_BODY + end + + def exception_type # :nodoc: internal use only + self::EXCEPTION_TYPE + end + + def read_new(sock) #:nodoc: internal use only + httpv, code, msg = read_status_line(sock) + res = response_class(code).new(httpv, code, msg) + each_response_header(sock) do |k,v| + res.add_field k, v + end + res + end + + private + + def read_status_line(sock) + str = sock.readline + m = /\AHTTP(?:\/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?\z/in.match(str) or + raise Gem::Net::HTTPBadResponse, "wrong status line: #{str.dump}" + m.captures + end + + def response_class(code) + CODE_TO_OBJ[code] or + CODE_CLASS_TO_OBJ[code[0,1]] or + Gem::Net::HTTPUnknownResponse + end + + def each_response_header(sock) + key = value = nil + while true + line = sock.readuntil("\n", true).sub(/\s+\z/, '') + break if line.empty? + if line[0] == ?\s or line[0] == ?\t and value + value << ' ' unless value.empty? + value << line.strip + else + yield key, value if key + key, value = line.strip.split(/\s*:\s*/, 2) + raise Gem::Net::HTTPBadResponse, 'wrong header line format' if value.nil? + end + end + yield key, value if key + end + end + + # next is to fix bug in RDoc, where the private inside class << self + # spills out. + public + + include Gem::Net::HTTPHeader + + def initialize(httpv, code, msg) #:nodoc: internal use only + @http_version = httpv + @code = code + @message = msg + initialize_http_header nil + @body = nil + @read = false + @uri = nil + @decode_content = false + @body_encoding = false + @ignore_eof = true + end + + # The HTTP version supported by the server. + attr_reader :http_version + + # The HTTP result code string. For example, '302'. You can also + # determine the response type by examining which response subclass + # the response object is an instance of. + attr_reader :code + + # The HTTP result message sent by the server. For example, 'Not Found'. + attr_reader :message + alias msg message # :nodoc: obsolete + + # The Gem::URI used to fetch this response. The response Gem::URI is only available + # if a Gem::URI was used to create the request. + attr_reader :uri + + # Set to true automatically when the request did not contain an + # Accept-Encoding header from the user. + attr_accessor :decode_content + + # Returns the value set by body_encoding=, or +false+ if none; + # see #body_encoding=. + attr_reader :body_encoding + + # Sets the encoding that should be used when reading the body: + # + # - If the given value is an Encoding object, that encoding will be used. + # - Otherwise if the value is a string, the value of + # {Encoding#find(value)}[https://docs.ruby-lang.org/en/master/Encoding.html#method-c-find] + # will be used. + # - Otherwise an encoding will be deduced from the body itself. + # + # Examples: + # + # http = Gem::Net::HTTP.new(hostname) + # req = Gem::Net::HTTP::Get.new('/') + # + # http.request(req) do |res| + # p res.body.encoding # => #<Encoding:ASCII-8BIT> + # end + # + # http.request(req) do |res| + # res.body_encoding = "UTF-8" + # p res.body.encoding # => #<Encoding:UTF-8> + # end + # + def body_encoding=(value) + value = Encoding.find(value) if value.is_a?(String) + @body_encoding = value + end + + # Whether to ignore EOF when reading bodies with a specified Content-Length + # header. + attr_accessor :ignore_eof + + def inspect + "#<#{self.class} #{@code} #{@message} readbody=#{@read}>" + end + + # + # response <-> exception relationship + # + + def code_type #:nodoc: + self.class + end + + def error! #:nodoc: + message = @code + message = "#{message} #{@message.dump}" if @message + raise error_type().new(message, self) + end + + def error_type #:nodoc: + self.class::EXCEPTION_TYPE + end + + # Raises an HTTP error if the response is not 2xx (success). + def value + error! unless self.kind_of?(Gem::Net::HTTPSuccess) + end + + def uri= uri # :nodoc: + @uri = uri.dup if uri + end + + # + # header (for backward compatibility only; DO NOT USE) + # + + def response #:nodoc: + warn "Gem::Net::HTTPResponse#response is obsolete", uplevel: 1 if $VERBOSE + self + end + + def header #:nodoc: + warn "Gem::Net::HTTPResponse#header is obsolete", uplevel: 1 if $VERBOSE + self + end + + def read_header #:nodoc: + warn "Gem::Net::HTTPResponse#read_header is obsolete", uplevel: 1 if $VERBOSE + self + end + + # + # body + # + + def reading_body(sock, reqmethodallowbody) #:nodoc: internal use only + @socket = sock + @body_exist = reqmethodallowbody && self.class.body_permitted? + begin + yield + self.body # ensure to read body + ensure + @socket = nil + end + end + + # Gets the entity body returned by the remote HTTP server. + # + # If a block is given, the body is passed to the block, and + # the body is provided in fragments, as it is read in from the socket. + # + # If +dest+ argument is given, response is read into that variable, + # with <code>dest#<<</code> method (it could be String or IO, or any + # other object responding to <code><<</code>). + # + # Calling this method a second or subsequent time for the same + # HTTPResponse object will return the value already read. + # + # http.request_get('/index.html') {|res| + # puts res.read_body + # } + # + # http.request_get('/index.html') {|res| + # p res.read_body.object_id # 538149362 + # p res.read_body.object_id # 538149362 + # } + # + # # using iterator + # http.request_get('/index.html') {|res| + # res.read_body do |segment| + # print segment + # end + # } + # + def read_body(dest = nil, &block) + if @read + raise IOError, "#{self.class}\#read_body called twice" if dest or block + return @body + end + to = procdest(dest, block) + stream_check + if @body_exist + read_body_0 to + @body = to + else + @body = nil + end + @read = true + return if @body.nil? + + case enc = @body_encoding + when Encoding, false, nil + # Encoding: force given encoding + # false/nil: do not force encoding + else + # other value: detect encoding from body + enc = detect_encoding(@body) + end + + @body.force_encoding(enc) if enc + + @body + end + + # Returns the string response body; + # note that repeated calls for the unmodified body return a cached string: + # + # path = '/todos/1' + # Gem::Net::HTTP.start(hostname) do |http| + # res = http.get(path) + # p res.body + # p http.head(path).body # No body. + # end + # + # Output: + # + # "{\n \"userId\": 1,\n \"id\": 1,\n \"title\": \"delectus aut autem\",\n \"completed\": false\n}" + # nil + # + def body + read_body() + end + + # Sets the body of the response to the given value. + def body=(value) + @body = value + end + + alias entity body #:nodoc: obsolete + + private + + # :nodoc: + def detect_encoding(str, encoding=nil) + if encoding + elsif encoding = type_params['charset'] + elsif encoding = check_bom(str) + else + encoding = case content_type&.downcase + when %r{text/x(?:ht)?ml|application/(?:[^+]+\+)?xml} + /\A<xml[ \t\r\n]+ + version[ \t\r\n]*=[ \t\r\n]*(?:"[0-9.]+"|'[0-9.]*')[ \t\r\n]+ + encoding[ \t\r\n]*=[ \t\r\n]* + (?:"([A-Za-z][\-A-Za-z0-9._]*)"|'([A-Za-z][\-A-Za-z0-9._]*)')/x =~ str + encoding = $1 || $2 || Encoding::UTF_8 + when %r{text/html.*} + sniff_encoding(str) + end + end + return encoding + end + + # :nodoc: + def sniff_encoding(str, encoding=nil) + # the encoding sniffing algorithm + # http://www.w3.org/TR/html5/parsing.html#determining-the-character-encoding + if enc = scanning_meta(str) + enc + # 6. last visited page or something + # 7. frequency + elsif str.ascii_only? + Encoding::US_ASCII + elsif str.dup.force_encoding(Encoding::UTF_8).valid_encoding? + Encoding::UTF_8 + end + # 8. implementation-defined or user-specified + end + + # :nodoc: + def check_bom(str) + case str.byteslice(0, 2) + when "\xFE\xFF" + return Encoding::UTF_16BE + when "\xFF\xFE" + return Encoding::UTF_16LE + end + if "\xEF\xBB\xBF" == str.byteslice(0, 3) + return Encoding::UTF_8 + end + nil + end + + # :nodoc: + def scanning_meta(str) + require 'strscan' + ss = StringScanner.new(str) + if ss.scan_until(/<meta[\t\n\f\r ]*/) + attrs = {} # attribute_list + got_pragma = false + need_pragma = nil + charset = nil + + # step: Attributes + while attr = get_attribute(ss) + name, value = *attr + next if attrs[name] + attrs[name] = true + case name + when 'http-equiv' + got_pragma = true if value == 'content-type' + when 'content' + encoding = extracting_encodings_from_meta_elements(value) + unless charset + charset = encoding + end + need_pragma = true + when 'charset' + need_pragma = false + charset = value + end + end + + # step: Processing + return if need_pragma.nil? + return if need_pragma && !got_pragma + + charset = Encoding.find(charset) rescue nil + return unless charset + charset = Encoding::UTF_8 if charset == Encoding::UTF_16 + return charset # tentative + end + nil + end + + def get_attribute(ss) + ss.scan(/[\t\n\f\r \/]*/) + if ss.peek(1) == '>' + ss.getch + return nil + end + name = ss.scan(/[^=\t\n\f\r \/>]*/) + name.downcase! + raise if name.empty? + ss.skip(/[\t\n\f\r ]*/) + if ss.getch != '=' + value = '' + return [name, value] + end + ss.skip(/[\t\n\f\r ]*/) + case ss.peek(1) + when '"' + ss.getch + value = ss.scan(/[^"]+/) + value.downcase! + ss.getch + when "'" + ss.getch + value = ss.scan(/[^']+/) + value.downcase! + ss.getch + when '>' + value = '' + else + value = ss.scan(/[^\t\n\f\r >]+/) + value.downcase! + end + [name, value] + end + + def extracting_encodings_from_meta_elements(value) + # http://dev.w3.org/html5/spec/fetching-resources.html#algorithm-for-extracting-an-encoding-from-a-meta-element + if /charset[\t\n\f\r ]*=(?:"([^"]*)"|'([^']*)'|["']|\z|([^\t\n\f\r ;]+))/i =~ value + return $1 || $2 || $3 + end + return nil + end + + ## + # Checks for a supported Content-Encoding header and yields an Inflate + # wrapper for this response's socket when zlib is present. If the + # Content-Encoding is not supported or zlib is missing, the plain socket is + # yielded. + # + # If a Content-Range header is present, a plain socket is yielded as the + # bytes in the range may not be a complete deflate block. + + def inflater # :nodoc: + return yield @socket unless Gem::Net::HTTP::HAVE_ZLIB + return yield @socket unless @decode_content + return yield @socket if self['content-range'] + + v = self['content-encoding'] + case v&.downcase + when 'deflate', 'gzip', 'x-gzip' then + self.delete 'content-encoding' + + inflate_body_io = Inflater.new(@socket) + + begin + yield inflate_body_io + success = true + ensure + begin + inflate_body_io.finish + if self['content-length'] + self['content-length'] = inflate_body_io.bytes_inflated.to_s + end + rescue => err + # Ignore #finish's error if there is an exception from yield + raise err if success + end + end + when 'none', 'identity' then + self.delete 'content-encoding' + + yield @socket + else + yield @socket + end + end + + def read_body_0(dest) + inflater do |inflate_body_io| + if chunked? + read_chunked dest, inflate_body_io + return + end + + @socket = inflate_body_io + + clen = content_length() + if clen + @socket.read clen, dest, @ignore_eof + return + end + clen = range_length() + if clen + @socket.read clen, dest + return + end + @socket.read_all dest + end + end + + ## + # read_chunked reads from +@socket+ for chunk-size, chunk-extension, CRLF, + # etc. and +chunk_data_io+ for chunk-data which may be deflate or gzip + # encoded. + # + # See RFC 2616 section 3.6.1 for definitions + + def read_chunked(dest, chunk_data_io) # :nodoc: + total = 0 + while true + line = @socket.readline + hexlen = line.slice(/[0-9a-fA-F]+/) or + raise Gem::Net::HTTPBadResponse, "wrong chunk size line: #{line}" + len = hexlen.hex + break if len == 0 + begin + chunk_data_io.read len, dest + ensure + total += len + @socket.read 2 # \r\n + end + end + until @socket.readline.empty? + # none + end + end + + def stream_check + raise IOError, 'attempt to read body out of block' if @socket.nil? || @socket.closed? + end + + def procdest(dest, block) + raise ArgumentError, 'both arg and block given for HTTP method' if + dest and block + if block + Gem::Net::ReadAdapter.new(block) + else + dest || +'' + end + end + + ## + # Inflater is a wrapper around Gem::Net::BufferedIO that transparently inflates + # zlib and gzip streams. + + class Inflater # :nodoc: + + ## + # Creates a new Inflater wrapping +socket+ + + def initialize socket + @socket = socket + # zlib with automatic gzip detection + @inflate = Zlib::Inflate.new(32 + Zlib::MAX_WBITS) + end + + ## + # Finishes the inflate stream. + + def finish + return if @inflate.total_in == 0 + @inflate.finish + end + + ## + # The number of bytes inflated, used to update the Content-Length of + # the response. + + def bytes_inflated + @inflate.total_out + end + + ## + # Returns a Gem::Net::ReadAdapter that inflates each read chunk into +dest+. + # + # This allows a large response body to be inflated without storing the + # entire body in memory. + + def inflate_adapter(dest) + if dest.respond_to?(:set_encoding) + dest.set_encoding(Encoding::ASCII_8BIT) + elsif dest.respond_to?(:force_encoding) + dest.force_encoding(Encoding::ASCII_8BIT) + end + block = proc do |compressed_chunk| + @inflate.inflate(compressed_chunk) do |chunk| + compressed_chunk.clear + dest << chunk + end + end + + Gem::Net::ReadAdapter.new(block) + end + + ## + # Reads +clen+ bytes from the socket, inflates them, then writes them to + # +dest+. +ignore_eof+ is passed down to Gem::Net::BufferedIO#read + # + # Unlike Gem::Net::BufferedIO#read, this method returns more than +clen+ bytes. + # At this time there is no way for a user of Gem::Net::HTTPResponse to read a + # specific number of bytes from the HTTP response body, so this internal + # API does not return the same number of bytes as were requested. + # + # See https://bugs.ruby-lang.org/issues/6492 for further discussion. + + def read clen, dest, ignore_eof = false + temp_dest = inflate_adapter(dest) + + @socket.read clen, temp_dest, ignore_eof + end + + ## + # Reads the rest of the socket, inflates it, then writes it to +dest+. + + def read_all dest + temp_dest = inflate_adapter(dest) + + @socket.read_all temp_dest + end + + end + +end + diff --git a/lib/rubygems/vendor/net-http/lib/net/http/responses.rb b/lib/rubygems/vendor/net-http/lib/net/http/responses.rb new file mode 100644 index 0000000000..0f26ae6c26 --- /dev/null +++ b/lib/rubygems/vendor/net-http/lib/net/http/responses.rb @@ -0,0 +1,1174 @@ +# frozen_string_literal: true +#-- +# https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml + +module Gem::Net + + class HTTPUnknownResponse < HTTPResponse + HAS_BODY = true + EXCEPTION_TYPE = HTTPError # + end + + # Parent class for informational (1xx) HTTP response classes. + # + # An informational response indicates that the request was received and understood. + # + # References: + # + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#status.1xx]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#1xx_informational_response]. + # + class HTTPInformation < HTTPResponse + HAS_BODY = false + EXCEPTION_TYPE = HTTPError # + end + + # Parent class for success (2xx) HTTP response classes. + # + # A success response indicates the action requested by the client + # was received, understood, and accepted. + # + # References: + # + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#status.2xx]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#2xx_success]. + # + class HTTPSuccess < HTTPResponse + HAS_BODY = true + EXCEPTION_TYPE = HTTPError # + end + + # Parent class for redirection (3xx) HTTP response classes. + # + # A redirection response indicates the client must take additional action + # to complete the request. + # + # References: + # + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#status.3xx]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#3xx_redirection]. + # + class HTTPRedirection < HTTPResponse + HAS_BODY = true + EXCEPTION_TYPE = HTTPRetriableError # + end + + # Parent class for client error (4xx) HTTP response classes. + # + # A client error response indicates that the client may have caused an error. + # + # References: + # + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#status.4xx]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#4xx_client_errors]. + # + class HTTPClientError < HTTPResponse + HAS_BODY = true + EXCEPTION_TYPE = HTTPClientException # + end + + # Parent class for server error (5xx) HTTP response classes. + # + # A server error response indicates that the server failed to fulfill a request. + # + # References: + # + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#status.5xx]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#5xx_server_errors]. + # + class HTTPServerError < HTTPResponse + HAS_BODY = true + EXCEPTION_TYPE = HTTPFatalError # + end + + # Response class for +Continue+ responses (status code 100). + # + # A +Continue+ response indicates that the server has received the request headers. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/100]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-100-continue]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#100]. + # + class HTTPContinue < HTTPInformation + HAS_BODY = false + end + + # Response class for <tt>Switching Protocol</tt> responses (status code 101). + # + # The <tt>Switching Protocol<tt> response indicates that the server has received + # a request to switch protocols, and has agreed to do so. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/101]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-101-switching-protocols]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#101]. + # + class HTTPSwitchProtocol < HTTPInformation + HAS_BODY = false + end + + # Response class for +Processing+ responses (status code 102). + # + # The +Processing+ response indicates that the server has received + # and is processing the request, but no response is available yet. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {RFC 2518}[https://www.rfc-editor.org/rfc/rfc2518#section-10.1]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#102]. + # + class HTTPProcessing < HTTPInformation + HAS_BODY = false + end + + # Response class for <tt>Early Hints</tt> responses (status code 103). + # + # The <tt>Early Hints</tt> indicates that the server has received + # and is processing the request, and contains certain headers; + # the final response is not available yet. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/103]. + # - {RFC 8297}[https://www.rfc-editor.org/rfc/rfc8297.html#section-2]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#103]. + # + class HTTPEarlyHints < HTTPInformation + HAS_BODY = false + end + + # Response class for +OK+ responses (status code 200). + # + # The +OK+ response indicates that the server has received + # a request and has responded successfully. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/200]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-200-ok]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#200]. + # + class HTTPOK < HTTPSuccess + HAS_BODY = true + end + + # Response class for +Created+ responses (status code 201). + # + # The +Created+ response indicates that the server has received + # and has fulfilled a request to create a new resource. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/201]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-201-created]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#201]. + # + class HTTPCreated < HTTPSuccess + HAS_BODY = true + end + + # Response class for +Accepted+ responses (status code 202). + # + # The +Accepted+ response indicates that the server has received + # and is processing a request, but the processing has not yet been completed. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/202]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-202-accepted]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#202]. + # + class HTTPAccepted < HTTPSuccess + HAS_BODY = true + end + + # Response class for <tt>Non-Authoritative Information</tt> responses (status code 203). + # + # The <tt>Non-Authoritative Information</tt> response indicates that the server + # is a transforming proxy (such as a Web accelerator) + # that received a 200 OK response from its origin, + # and is returning a modified version of the origin's response. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/203]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-203-non-authoritative-infor]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#203]. + # + class HTTPNonAuthoritativeInformation < HTTPSuccess + HAS_BODY = true + end + + # Response class for <tt>No Content</tt> responses (status code 204). + # + # The <tt>No Content</tt> response indicates that the server + # successfully processed the request, and is not returning any content. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/204]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-204-no-content]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#204]. + # + class HTTPNoContent < HTTPSuccess + HAS_BODY = false + end + + # Response class for <tt>Reset Content</tt> responses (status code 205). + # + # The <tt>Reset Content</tt> response indicates that the server + # successfully processed the request, + # asks that the client reset its document view, and is not returning any content. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/205]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-205-reset-content]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#205]. + # + class HTTPResetContent < HTTPSuccess + HAS_BODY = false + end + + # Response class for <tt>Partial Content</tt> responses (status code 206). + # + # The <tt>Partial Content</tt> response indicates that the server is delivering + # only part of the resource (byte serving) + # due to a Range header in the request. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/206]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-206-partial-content]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#206]. + # + class HTTPPartialContent < HTTPSuccess + HAS_BODY = true + end + + # Response class for <tt>Multi-Status (WebDAV)</tt> responses (status code 207). + # + # The <tt>Multi-Status (WebDAV)</tt> response indicates that the server + # has received the request, + # and that the message body can contain a number of separate response codes. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {RFC 4818}[https://www.rfc-editor.org/rfc/rfc4918#section-11.1]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#207]. + # + class HTTPMultiStatus < HTTPSuccess + HAS_BODY = true + end + + # Response class for <tt>Already Reported (WebDAV)</tt> responses (status code 208). + # + # The <tt>Already Reported (WebDAV)</tt> response indicates that the server + # has received the request, + # and that the members of a DAV binding have already been enumerated + # in a preceding part of the (multi-status) response, + # and are not being included again. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {RFC 5842}[https://www.rfc-editor.org/rfc/rfc5842.html#section-7.1]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#208]. + # + class HTTPAlreadyReported < HTTPSuccess + HAS_BODY = true + end + + # Response class for <tt>IM Used</tt> responses (status code 226). + # + # The <tt>IM Used</tt> response indicates that the server has fulfilled a request + # for the resource, and the response is a representation of the result + # of one or more instance-manipulations applied to the current instance. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {RFC 3229}[https://www.rfc-editor.org/rfc/rfc3229.html#section-10.4.1]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#226]. + # + class HTTPIMUsed < HTTPSuccess + HAS_BODY = true + end + + # Response class for <tt>Multiple Choices</tt> responses (status code 300). + # + # The <tt>Multiple Choices</tt> response indicates that the server + # offers multiple options for the resource from which the client may choose. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/300]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-300-multiple-choices]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#300]. + # + class HTTPMultipleChoices < HTTPRedirection + HAS_BODY = true + end + HTTPMultipleChoice = HTTPMultipleChoices + + # Response class for <tt>Moved Permanently</tt> responses (status code 301). + # + # The <tt>Moved Permanently</tt> response indicates that links or records + # returning this response should be updated to use the given URL. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/301]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-301-moved-permanently]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#301]. + # + class HTTPMovedPermanently < HTTPRedirection + HAS_BODY = true + end + + # Response class for <tt>Found</tt> responses (status code 302). + # + # The <tt>Found</tt> response indicates that the client + # should look at (browse to) another URL. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/302]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-302-found]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#302]. + # + class HTTPFound < HTTPRedirection + HAS_BODY = true + end + HTTPMovedTemporarily = HTTPFound + + # Response class for <tt>See Other</tt> responses (status code 303). + # + # The response to the request can be found under another Gem::URI using the GET method. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/303]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-303-see-other]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#303]. + # + class HTTPSeeOther < HTTPRedirection + HAS_BODY = true + end + + # Response class for <tt>Not Modified</tt> responses (status code 304). + # + # Indicates that the resource has not been modified since the version + # specified by the request headers. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/304]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-304-not-modified]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#304]. + # + class HTTPNotModified < HTTPRedirection + HAS_BODY = false + end + + # Response class for <tt>Use Proxy</tt> responses (status code 305). + # + # The requested resource is available only through a proxy, + # whose address is provided in the response. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-305-use-proxy]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#305]. + # + class HTTPUseProxy < HTTPRedirection + HAS_BODY = false + end + + # Response class for <tt>Temporary Redirect</tt> responses (status code 307). + # + # The request should be repeated with another Gem::URI; + # however, future requests should still use the original Gem::URI. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/307]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-307-temporary-redirect]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#307]. + # + class HTTPTemporaryRedirect < HTTPRedirection + HAS_BODY = true + end + + # Response class for <tt>Permanent Redirect</tt> responses (status code 308). + # + # This and all future requests should be directed to the given Gem::URI. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/308]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-308-permanent-redirect]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#308]. + # + class HTTPPermanentRedirect < HTTPRedirection + HAS_BODY = true + end + + # Response class for <tt>Bad Request</tt> responses (status code 400). + # + # The server cannot or will not process the request due to an apparent client error. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/400]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-400-bad-request]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#400]. + # + class HTTPBadRequest < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Unauthorized</tt> responses (status code 401). + # + # Authentication is required, but either was not provided or failed. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/401]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-401-unauthorized]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#401]. + # + class HTTPUnauthorized < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Payment Required</tt> responses (status code 402). + # + # Reserved for future use. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/402]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-402-payment-required]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#402]. + # + class HTTPPaymentRequired < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Forbidden</tt> responses (status code 403). + # + # The request contained valid data and was understood by the server, + # but the server is refusing action. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/403]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-403-forbidden]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#403]. + # + class HTTPForbidden < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Not Found</tt> responses (status code 404). + # + # The requested resource could not be found but may be available in the future. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/404]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-404-not-found]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#404]. + # + class HTTPNotFound < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Method Not Allowed</tt> responses (status code 405). + # + # The request method is not supported for the requested resource. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/405]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-405-method-not-allowed]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#405]. + # + class HTTPMethodNotAllowed < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Not Acceptable</tt> responses (status code 406). + # + # The requested resource is capable of generating only content + # that not acceptable according to the Accept headers sent in the request. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/406]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-406-not-acceptable]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#406]. + # + class HTTPNotAcceptable < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Proxy Authentication Required</tt> responses (status code 407). + # + # The client must first authenticate itself with the proxy. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/407]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-407-proxy-authentication-re]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#407]. + # + class HTTPProxyAuthenticationRequired < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Request Gem::Timeout</tt> responses (status code 408). + # + # The server timed out waiting for the request. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/408]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-408-request-timeout]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#408]. + # + class HTTPRequestTimeout < HTTPClientError + HAS_BODY = true + end + HTTPRequestTimeOut = HTTPRequestTimeout + + # Response class for <tt>Conflict</tt> responses (status code 409). + # + # The request could not be processed because of conflict in the current state of the resource. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/409]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-409-conflict]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#409]. + # + class HTTPConflict < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Gone</tt> responses (status code 410). + # + # The resource requested was previously in use but is no longer available + # and will not be available again. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/410]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-410-gone]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#410]. + # + class HTTPGone < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Length Required</tt> responses (status code 411). + # + # The request did not specify the length of its content, + # which is required by the requested resource. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/411]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-411-length-required]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#411]. + # + class HTTPLengthRequired < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Precondition Failed</tt> responses (status code 412). + # + # The server does not meet one of the preconditions + # specified in the request headers. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/412]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-412-precondition-failed]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#412]. + # + class HTTPPreconditionFailed < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Payload Too Large</tt> responses (status code 413). + # + # The request is larger than the server is willing or able to process. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/413]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-413-content-too-large]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#413]. + # + class HTTPPayloadTooLarge < HTTPClientError + HAS_BODY = true + end + HTTPRequestEntityTooLarge = HTTPPayloadTooLarge + + # Response class for <tt>Gem::URI Too Long</tt> responses (status code 414). + # + # The Gem::URI provided was too long for the server to process. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/414]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-414-uri-too-long]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#414]. + # + class HTTPURITooLong < HTTPClientError + HAS_BODY = true + end + HTTPRequestURITooLong = HTTPURITooLong + HTTPRequestURITooLarge = HTTPRequestURITooLong + + # Response class for <tt>Unsupported Media Type</tt> responses (status code 415). + # + # The request entity has a media type which the server or resource does not support. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/415]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-415-unsupported-media-type]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#415]. + # + class HTTPUnsupportedMediaType < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Range Not Satisfiable</tt> responses (status code 416). + # + # The request entity has a media type which the server or resource does not support. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/416]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-416-range-not-satisfiable]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#416]. + # + class HTTPRangeNotSatisfiable < HTTPClientError + HAS_BODY = true + end + HTTPRequestedRangeNotSatisfiable = HTTPRangeNotSatisfiable + + # Response class for <tt>Expectation Failed</tt> responses (status code 417). + # + # The server cannot meet the requirements of the Expect request-header field. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/417]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-417-expectation-failed]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#417]. + # + class HTTPExpectationFailed < HTTPClientError + HAS_BODY = true + end + + # 418 I'm a teapot - RFC 2324; a joke RFC + # See https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#418. + + # 420 Enhance Your Calm - Twitter + + # Response class for <tt>Misdirected Request</tt> responses (status code 421). + # + # The request was directed at a server that is not able to produce a response. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-421-misdirected-request]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#421]. + # + class HTTPMisdirectedRequest < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Unprocessable Entity</tt> responses (status code 422). + # + # The request was well-formed but had semantic errors. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/422]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-422-unprocessable-content]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#422]. + # + class HTTPUnprocessableEntity < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Locked (WebDAV)</tt> responses (status code 423). + # + # The requested resource is locked. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {RFC 4918}[https://www.rfc-editor.org/rfc/rfc4918#section-11.3]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#423]. + # + class HTTPLocked < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Failed Dependency (WebDAV)</tt> responses (status code 424). + # + # The request failed because it depended on another request and that request failed. + # See {424 Failed Dependency (WebDAV)}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#424]. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {RFC 4918}[https://www.rfc-editor.org/rfc/rfc4918#section-11.4]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#424]. + # + class HTTPFailedDependency < HTTPClientError + HAS_BODY = true + end + + # 425 Too Early + # https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#425. + + # Response class for <tt>Upgrade Required</tt> responses (status code 426). + # + # The client should switch to the protocol given in the Upgrade header field. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/426]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-426-upgrade-required]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#426]. + # + class HTTPUpgradeRequired < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Precondition Required</tt> responses (status code 428). + # + # The origin server requires the request to be conditional. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/428]. + # - {RFC 6585}[https://www.rfc-editor.org/rfc/rfc6585#section-3]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#428]. + # + class HTTPPreconditionRequired < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Too Many Requests</tt> responses (status code 429). + # + # The user has sent too many requests in a given amount of time. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/429]. + # - {RFC 6585}[https://www.rfc-editor.org/rfc/rfc6585#section-4]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#429]. + # + class HTTPTooManyRequests < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Request Header Fields Too Large</tt> responses (status code 431). + # + # An individual header field is too large, + # or all the header fields collectively, are too large. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/431]. + # - {RFC 6585}[https://www.rfc-editor.org/rfc/rfc6585#section-5]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#431]. + # + class HTTPRequestHeaderFieldsTooLarge < HTTPClientError + HAS_BODY = true + end + + # Response class for <tt>Unavailable For Legal Reasons</tt> responses (status code 451). + # + # A server operator has received a legal demand to deny access to a resource or to a set of resources + # that includes the requested resource. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/451]. + # - {RFC 7725}[https://www.rfc-editor.org/rfc/rfc7725.html#section-3]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#451]. + # + class HTTPUnavailableForLegalReasons < HTTPClientError + HAS_BODY = true + end + # 444 No Response - Nginx + # 449 Retry With - Microsoft + # 450 Blocked by Windows Parental Controls - Microsoft + # 499 Client Closed Request - Nginx + + # Response class for <tt>Internal Server Error</tt> responses (status code 500). + # + # An unexpected condition was encountered and no more specific message is suitable. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/500]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-500-internal-server-error]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#500]. + # + class HTTPInternalServerError < HTTPServerError + HAS_BODY = true + end + + # Response class for <tt>Not Implemented</tt> responses (status code 501). + # + # The server either does not recognize the request method, + # or it lacks the ability to fulfil the request. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/501]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-501-not-implemented]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#501]. + # + class HTTPNotImplemented < HTTPServerError + HAS_BODY = true + end + + # Response class for <tt>Bad Gateway</tt> responses (status code 502). + # + # The server was acting as a gateway or proxy + # and received an invalid response from the upstream server. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/502]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-502-bad-gateway]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#502]. + # + class HTTPBadGateway < HTTPServerError + HAS_BODY = true + end + + # Response class for <tt>Service Unavailable</tt> responses (status code 503). + # + # The server cannot handle the request + # (because it is overloaded or down for maintenance). + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/503]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-503-service-unavailable]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#503]. + # + class HTTPServiceUnavailable < HTTPServerError + HAS_BODY = true + end + + # Response class for <tt>Gateway Gem::Timeout</tt> responses (status code 504). + # + # The server was acting as a gateway or proxy + # and did not receive a timely response from the upstream server. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/504]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-504-gateway-timeout]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#504]. + # + class HTTPGatewayTimeout < HTTPServerError + HAS_BODY = true + end + HTTPGatewayTimeOut = HTTPGatewayTimeout + + # Response class for <tt>HTTP Version Not Supported</tt> responses (status code 505). + # + # The server does not support the HTTP version used in the request. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/505]. + # - {RFC 9110}[https://www.rfc-editor.org/rfc/rfc9110.html#name-505-http-version-not-suppor]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#505]. + # + class HTTPVersionNotSupported < HTTPServerError + HAS_BODY = true + end + + # Response class for <tt>Variant Also Negotiates</tt> responses (status code 506). + # + # Transparent content negotiation for the request results in a circular reference. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/506]. + # - {RFC 2295}[https://www.rfc-editor.org/rfc/rfc2295#section-8.1]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#506]. + # + class HTTPVariantAlsoNegotiates < HTTPServerError + HAS_BODY = true + end + + # Response class for <tt>Insufficient Storage (WebDAV)</tt> responses (status code 507). + # + # The server is unable to store the representation needed to complete the request. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/507]. + # - {RFC 4918}[https://www.rfc-editor.org/rfc/rfc4918#section-11.5]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#507]. + # + class HTTPInsufficientStorage < HTTPServerError + HAS_BODY = true + end + + # Response class for <tt>Loop Detected (WebDAV)</tt> responses (status code 508). + # + # The server detected an infinite loop while processing the request. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/508]. + # - {RFC 5942}[https://www.rfc-editor.org/rfc/rfc5842.html#section-7.2]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#508]. + # + class HTTPLoopDetected < HTTPServerError + HAS_BODY = true + end + # 509 Bandwidth Limit Exceeded - Apache bw/limited extension + + # Response class for <tt>Not Extended</tt> responses (status code 510). + # + # Further extensions to the request are required for the server to fulfill it. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/510]. + # - {RFC 2774}[https://www.rfc-editor.org/rfc/rfc2774.html#section-7]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#510]. + # + class HTTPNotExtended < HTTPServerError + HAS_BODY = true + end + + # Response class for <tt>Network Authentication Required</tt> responses (status code 511). + # + # The client needs to authenticate to gain network access. + # + # :include: doc/net-http/included_getters.rdoc + # + # References: + # + # - {Mozilla}[https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/511]. + # - {RFC 6585}[https://www.rfc-editor.org/rfc/rfc6585#section-6]. + # - {Wikipedia}[https://en.wikipedia.org/wiki/List_of_HTTP_status_codes#511]. + # + class HTTPNetworkAuthenticationRequired < HTTPServerError + HAS_BODY = true + end + +end + +class Gem::Net::HTTPResponse + CODE_CLASS_TO_OBJ = { + '1' => Gem::Net::HTTPInformation, + '2' => Gem::Net::HTTPSuccess, + '3' => Gem::Net::HTTPRedirection, + '4' => Gem::Net::HTTPClientError, + '5' => Gem::Net::HTTPServerError + } + CODE_TO_OBJ = { + '100' => Gem::Net::HTTPContinue, + '101' => Gem::Net::HTTPSwitchProtocol, + '102' => Gem::Net::HTTPProcessing, + '103' => Gem::Net::HTTPEarlyHints, + + '200' => Gem::Net::HTTPOK, + '201' => Gem::Net::HTTPCreated, + '202' => Gem::Net::HTTPAccepted, + '203' => Gem::Net::HTTPNonAuthoritativeInformation, + '204' => Gem::Net::HTTPNoContent, + '205' => Gem::Net::HTTPResetContent, + '206' => Gem::Net::HTTPPartialContent, + '207' => Gem::Net::HTTPMultiStatus, + '208' => Gem::Net::HTTPAlreadyReported, + '226' => Gem::Net::HTTPIMUsed, + + '300' => Gem::Net::HTTPMultipleChoices, + '301' => Gem::Net::HTTPMovedPermanently, + '302' => Gem::Net::HTTPFound, + '303' => Gem::Net::HTTPSeeOther, + '304' => Gem::Net::HTTPNotModified, + '305' => Gem::Net::HTTPUseProxy, + '307' => Gem::Net::HTTPTemporaryRedirect, + '308' => Gem::Net::HTTPPermanentRedirect, + + '400' => Gem::Net::HTTPBadRequest, + '401' => Gem::Net::HTTPUnauthorized, + '402' => Gem::Net::HTTPPaymentRequired, + '403' => Gem::Net::HTTPForbidden, + '404' => Gem::Net::HTTPNotFound, + '405' => Gem::Net::HTTPMethodNotAllowed, + '406' => Gem::Net::HTTPNotAcceptable, + '407' => Gem::Net::HTTPProxyAuthenticationRequired, + '408' => Gem::Net::HTTPRequestTimeout, + '409' => Gem::Net::HTTPConflict, + '410' => Gem::Net::HTTPGone, + '411' => Gem::Net::HTTPLengthRequired, + '412' => Gem::Net::HTTPPreconditionFailed, + '413' => Gem::Net::HTTPPayloadTooLarge, + '414' => Gem::Net::HTTPURITooLong, + '415' => Gem::Net::HTTPUnsupportedMediaType, + '416' => Gem::Net::HTTPRangeNotSatisfiable, + '417' => Gem::Net::HTTPExpectationFailed, + '421' => Gem::Net::HTTPMisdirectedRequest, + '422' => Gem::Net::HTTPUnprocessableEntity, + '423' => Gem::Net::HTTPLocked, + '424' => Gem::Net::HTTPFailedDependency, + '426' => Gem::Net::HTTPUpgradeRequired, + '428' => Gem::Net::HTTPPreconditionRequired, + '429' => Gem::Net::HTTPTooManyRequests, + '431' => Gem::Net::HTTPRequestHeaderFieldsTooLarge, + '451' => Gem::Net::HTTPUnavailableForLegalReasons, + + '500' => Gem::Net::HTTPInternalServerError, + '501' => Gem::Net::HTTPNotImplemented, + '502' => Gem::Net::HTTPBadGateway, + '503' => Gem::Net::HTTPServiceUnavailable, + '504' => Gem::Net::HTTPGatewayTimeout, + '505' => Gem::Net::HTTPVersionNotSupported, + '506' => Gem::Net::HTTPVariantAlsoNegotiates, + '507' => Gem::Net::HTTPInsufficientStorage, + '508' => Gem::Net::HTTPLoopDetected, + '510' => Gem::Net::HTTPNotExtended, + '511' => Gem::Net::HTTPNetworkAuthenticationRequired, + } +end diff --git a/lib/rubygems/vendor/net-http/lib/net/http/status.rb b/lib/rubygems/vendor/net-http/lib/net/http/status.rb new file mode 100644 index 0000000000..9110b108b8 --- /dev/null +++ b/lib/rubygems/vendor/net-http/lib/net/http/status.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require_relative '../http' + +if $0 == __FILE__ + require 'open-uri' + File.foreach(__FILE__) do |line| + puts line + break if line.start_with?('end') + end + puts + puts "Gem::Net::HTTP::STATUS_CODES = {" + url = "https://www.iana.org/assignments/http-status-codes/http-status-codes-1.csv" + Gem::URI(url).read.each_line do |line| + code, mes, = line.split(',') + next if ['(Unused)', 'Unassigned', 'Description'].include?(mes) + puts " #{code} => '#{mes}'," + end + puts "} # :nodoc:" +end + +Gem::Net::HTTP::STATUS_CODES = { + 100 => 'Continue', + 101 => 'Switching Protocols', + 102 => 'Processing', + 103 => 'Early Hints', + 200 => 'OK', + 201 => 'Created', + 202 => 'Accepted', + 203 => 'Non-Authoritative Information', + 204 => 'No Content', + 205 => 'Reset Content', + 206 => 'Partial Content', + 207 => 'Multi-Status', + 208 => 'Already Reported', + 226 => 'IM Used', + 300 => 'Multiple Choices', + 301 => 'Moved Permanently', + 302 => 'Found', + 303 => 'See Other', + 304 => 'Not Modified', + 305 => 'Use Proxy', + 307 => 'Temporary Redirect', + 308 => 'Permanent Redirect', + 400 => 'Bad Request', + 401 => 'Unauthorized', + 402 => 'Payment Required', + 403 => 'Forbidden', + 404 => 'Not Found', + 405 => 'Method Not Allowed', + 406 => 'Not Acceptable', + 407 => 'Proxy Authentication Required', + 408 => 'Request Timeout', + 409 => 'Conflict', + 410 => 'Gone', + 411 => 'Length Required', + 412 => 'Precondition Failed', + 413 => 'Content Too Large', + 414 => 'URI Too Long', + 415 => 'Unsupported Media Type', + 416 => 'Range Not Satisfiable', + 417 => 'Expectation Failed', + 421 => 'Misdirected Request', + 422 => 'Unprocessable Content', + 423 => 'Locked', + 424 => 'Failed Dependency', + 425 => 'Too Early', + 426 => 'Upgrade Required', + 428 => 'Precondition Required', + 429 => 'Too Many Requests', + 431 => 'Request Header Fields Too Large', + 451 => 'Unavailable For Legal Reasons', + 500 => 'Internal Server Error', + 501 => 'Not Implemented', + 502 => 'Bad Gateway', + 503 => 'Service Unavailable', + 504 => 'Gateway Timeout', + 505 => 'HTTP Version Not Supported', + 506 => 'Variant Also Negotiates', + 507 => 'Insufficient Storage', + 508 => 'Loop Detected', + 510 => 'Not Extended (OBSOLETED)', + 511 => 'Network Authentication Required', +} # :nodoc: diff --git a/lib/rubygems/vendor/net-http/lib/net/https.rb b/lib/rubygems/vendor/net-http/lib/net/https.rb new file mode 100644 index 0000000000..d2784f0be0 --- /dev/null +++ b/lib/rubygems/vendor/net-http/lib/net/https.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +=begin + += net/https -- SSL/TLS enhancement for Gem::Net::HTTP. + + This file has been merged with net/http. There is no longer any need to + require 'rubygems/vendor/net-http/lib/net/https' to use HTTPS. + + See Gem::Net::HTTP for details on how to make HTTPS connections. + +== Info + 'OpenSSL for Ruby 2' project + Copyright (C) 2001 GOTOU Yuuzou <gotoyuzo@notwork.org> + All rights reserved. + +== Licence + This program is licensed under the same licence as Ruby. + (See the file 'LICENCE'.) + +=end + +require_relative 'http' +require 'openssl' diff --git a/lib/rubygems/vendor/net-protocol/.document b/lib/rubygems/vendor/net-protocol/.document new file mode 100644 index 0000000000..0c43bbd6b3 --- /dev/null +++ b/lib/rubygems/vendor/net-protocol/.document @@ -0,0 +1 @@ +# Vendored files do not need to be documented diff --git a/lib/rubygems/vendor/net-protocol/lib/net/protocol.rb b/lib/rubygems/vendor/net-protocol/lib/net/protocol.rb new file mode 100644 index 0000000000..53d34d8d98 --- /dev/null +++ b/lib/rubygems/vendor/net-protocol/lib/net/protocol.rb @@ -0,0 +1,544 @@ +# frozen_string_literal: true +# +# = net/protocol.rb +# +#-- +# Copyright (c) 1999-2004 Yukihiro Matsumoto +# Copyright (c) 1999-2004 Minero Aoki +# +# written and maintained by Minero Aoki <aamine@loveruby.net> +# +# This program is free software. You can re-distribute and/or +# modify this program under the same terms as Ruby itself, +# Ruby Distribute License or GNU General Public License. +# +# $Id$ +#++ +# +# WARNING: This file is going to remove. +# Do not rely on the implementation written in this file. +# + +require 'socket' +require_relative '../../../timeout/lib/timeout' +require 'io/wait' + +module Gem::Net # :nodoc: + + class Protocol #:nodoc: internal use only + VERSION = "0.2.2" + + private + def Protocol.protocol_param(name, val) + module_eval(<<-End, __FILE__, __LINE__ + 1) + def #{name} + #{val} + end + End + end + + def ssl_socket_connect(s, timeout) + if timeout + while true + raise Gem::Net::OpenTimeout if timeout <= 0 + start = Process.clock_gettime Process::CLOCK_MONOTONIC + # to_io is required because SSLSocket doesn't have wait_readable yet + case s.connect_nonblock(exception: false) + when :wait_readable; s.to_io.wait_readable(timeout) + when :wait_writable; s.to_io.wait_writable(timeout) + else; break + end + timeout -= Process.clock_gettime(Process::CLOCK_MONOTONIC) - start + end + else + s.connect + end + end + end + + + class ProtocolError < StandardError; end + class ProtoSyntaxError < ProtocolError; end + class ProtoFatalError < ProtocolError; end + class ProtoUnknownError < ProtocolError; end + class ProtoServerError < ProtocolError; end + class ProtoAuthError < ProtocolError; end + class ProtoCommandError < ProtocolError; end + class ProtoRetriableError < ProtocolError; end + ProtocRetryError = ProtoRetriableError + + ## + # OpenTimeout, a subclass of Gem::Timeout::Error, is raised if a connection cannot + # be created within the open_timeout. + + class OpenTimeout < Gem::Timeout::Error; end + + ## + # ReadTimeout, a subclass of Gem::Timeout::Error, is raised if a chunk of the + # response cannot be read within the read_timeout. + + class ReadTimeout < Gem::Timeout::Error + def initialize(io = nil) + @io = io + end + attr_reader :io + + def message + msg = super + if @io + msg = "#{msg} with #{@io.inspect}" + end + msg + end + end + + ## + # WriteTimeout, a subclass of Gem::Timeout::Error, is raised if a chunk of the + # response cannot be written within the write_timeout. Not raised on Windows. + + class WriteTimeout < Gem::Timeout::Error + def initialize(io = nil) + @io = io + end + attr_reader :io + + def message + msg = super + if @io + msg = "#{msg} with #{@io.inspect}" + end + msg + end + end + + + class BufferedIO #:nodoc: internal use only + def initialize(io, read_timeout: 60, write_timeout: 60, continue_timeout: nil, debug_output: nil) + @io = io + @read_timeout = read_timeout + @write_timeout = write_timeout + @continue_timeout = continue_timeout + @debug_output = debug_output + @rbuf = ''.b + @rbuf_empty = true + @rbuf_offset = 0 + end + + attr_reader :io + attr_accessor :read_timeout + attr_accessor :write_timeout + attr_accessor :continue_timeout + attr_accessor :debug_output + + def inspect + "#<#{self.class} io=#{@io}>" + end + + def eof? + @io.eof? + end + + def closed? + @io.closed? + end + + def close + @io.close + end + + # + # Read + # + + public + + def read(len, dest = ''.b, ignore_eof = false) + LOG "reading #{len} bytes..." + read_bytes = 0 + begin + while read_bytes + rbuf_size < len + if s = rbuf_consume_all + read_bytes += s.bytesize + dest << s + end + rbuf_fill + end + s = rbuf_consume(len - read_bytes) + read_bytes += s.bytesize + dest << s + rescue EOFError + raise unless ignore_eof + end + LOG "read #{read_bytes} bytes" + dest + end + + def read_all(dest = ''.b) + LOG 'reading all...' + read_bytes = 0 + begin + while true + if s = rbuf_consume_all + read_bytes += s.bytesize + dest << s + end + rbuf_fill + end + rescue EOFError + ; + end + LOG "read #{read_bytes} bytes" + dest + end + + def readuntil(terminator, ignore_eof = false) + offset = @rbuf_offset + begin + until idx = @rbuf.index(terminator, offset) + offset = @rbuf.bytesize + rbuf_fill + end + return rbuf_consume(idx + terminator.bytesize - @rbuf_offset) + rescue EOFError + raise unless ignore_eof + return rbuf_consume + end + end + + def readline + readuntil("\n").chop + end + + private + + BUFSIZE = 1024 * 16 + + def rbuf_fill + tmp = @rbuf_empty ? @rbuf : nil + case rv = @io.read_nonblock(BUFSIZE, tmp, exception: false) + when String + @rbuf_empty = false + if rv.equal?(tmp) + @rbuf_offset = 0 + else + @rbuf << rv + rv.clear + end + return + when :wait_readable + (io = @io.to_io).wait_readable(@read_timeout) or raise Gem::Net::ReadTimeout.new(io) + # continue looping + when :wait_writable + # OpenSSL::Buffering#read_nonblock may fail with IO::WaitWritable. + # http://www.openssl.org/support/faq.html#PROG10 + (io = @io.to_io).wait_writable(@read_timeout) or raise Gem::Net::ReadTimeout.new(io) + # continue looping + when nil + raise EOFError, 'end of file reached' + end while true + end + + def rbuf_flush + if @rbuf_empty + @rbuf.clear + @rbuf_offset = 0 + end + nil + end + + def rbuf_size + @rbuf.bytesize - @rbuf_offset + end + + def rbuf_consume_all + rbuf_consume if rbuf_size > 0 + end + + def rbuf_consume(len = nil) + if @rbuf_offset == 0 && (len.nil? || len == @rbuf.bytesize) + s = @rbuf + @rbuf = ''.b + @rbuf_offset = 0 + @rbuf_empty = true + elsif len.nil? + s = @rbuf.byteslice(@rbuf_offset..-1) + @rbuf = ''.b + @rbuf_offset = 0 + @rbuf_empty = true + else + s = @rbuf.byteslice(@rbuf_offset, len) + @rbuf_offset += len + @rbuf_empty = @rbuf_offset == @rbuf.bytesize + rbuf_flush + end + + @debug_output << %Q[-> #{s.dump}\n] if @debug_output + s + end + + # + # Write + # + + public + + def write(*strs) + writing { + write0(*strs) + } + end + + alias << write + + def writeline(str) + writing { + write0 str + "\r\n" + } + end + + private + + def writing + @written_bytes = 0 + @debug_output << '<- ' if @debug_output + yield + @debug_output << "\n" if @debug_output + bytes = @written_bytes + @written_bytes = nil + bytes + end + + def write0(*strs) + @debug_output << strs.map(&:dump).join if @debug_output + orig_written_bytes = @written_bytes + strs.each_with_index do |str, i| + need_retry = true + case len = @io.write_nonblock(str, exception: false) + when Integer + @written_bytes += len + len -= str.bytesize + if len == 0 + if strs.size == i+1 + return @written_bytes - orig_written_bytes + else + need_retry = false + # next string + end + elsif len < 0 + str = str.byteslice(len, -len) + else # len > 0 + need_retry = false + # next string + end + # continue looping + when :wait_writable + (io = @io.to_io).wait_writable(@write_timeout) or raise Gem::Net::WriteTimeout.new(io) + # continue looping + end while need_retry + end + end + + # + # Logging + # + + private + + def LOG_off + @save_debug_out = @debug_output + @debug_output = nil + end + + def LOG_on + @debug_output = @save_debug_out + end + + def LOG(msg) + return unless @debug_output + @debug_output << msg + "\n" + end + end + + + class InternetMessageIO < BufferedIO #:nodoc: internal use only + def initialize(*, **) + super + @wbuf = nil + end + + # + # Read + # + + def each_message_chunk + LOG 'reading message...' + LOG_off() + read_bytes = 0 + while (line = readuntil("\r\n")) != ".\r\n" + read_bytes += line.size + yield line.delete_prefix('.') + end + LOG_on() + LOG "read message (#{read_bytes} bytes)" + end + + # *library private* (cannot handle 'break') + def each_list_item + while (str = readuntil("\r\n")) != ".\r\n" + yield str.chop + end + end + + def write_message_0(src) + prev = @written_bytes + each_crlf_line(src) do |line| + write0 dot_stuff(line) + end + @written_bytes - prev + end + + # + # Write + # + + def write_message(src) + LOG "writing message from #{src.class}" + LOG_off() + len = writing { + using_each_crlf_line { + write_message_0 src + } + } + LOG_on() + LOG "wrote #{len} bytes" + len + end + + def write_message_by_block(&block) + LOG 'writing message from block' + LOG_off() + len = writing { + using_each_crlf_line { + begin + block.call(WriteAdapter.new(self.method(:write_message_0))) + rescue LocalJumpError + # allow `break' from writer block + end + } + } + LOG_on() + LOG "wrote #{len} bytes" + len + end + + private + + def dot_stuff(s) + s.sub(/\A\./, '..') + end + + def using_each_crlf_line + @wbuf = ''.b + yield + if not @wbuf.empty? # unterminated last line + write0 dot_stuff(@wbuf.chomp) + "\r\n" + elsif @written_bytes == 0 # empty src + write0 "\r\n" + end + write0 ".\r\n" + @wbuf = nil + end + + def each_crlf_line(src) + buffer_filling(@wbuf, src) do + while line = @wbuf.slice!(/\A[^\r\n]*(?:\n|\r(?:\n|(?!\z)))/) + yield line.chomp("\n") + "\r\n" + end + end + end + + def buffer_filling(buf, src) + case src + when String # for speeding up. + 0.step(src.size - 1, 1024) do |i| + buf << src[i, 1024] + yield + end + when File # for speeding up. + while s = src.read(1024) + buf << s + yield + end + else # generic reader + src.each do |str| + buf << str + yield if buf.size > 1024 + end + yield unless buf.empty? + end + end + end + + + # + # The writer adapter class + # + class WriteAdapter + def initialize(writer) + @writer = writer + end + + def inspect + "#<#{self.class} writer=#{@writer.inspect}>" + end + + def write(str) + @writer.call(str) + end + + alias print write + + def <<(str) + write str + self + end + + def puts(str = '') + write str.chomp("\n") + "\n" + end + + def printf(*args) + write sprintf(*args) + end + end + + + class ReadAdapter #:nodoc: internal use only + def initialize(block) + @block = block + end + + def inspect + "#<#{self.class}>" + end + + def <<(str) + call_block(str, &@block) if @block + end + + private + + # This method is needed because @block must be called by yield, + # not Proc#call. You can see difference when using `break' in + # the block. + def call_block(str) + yield str + end + end + + + module NetPrivate #:nodoc: obsolete + Socket = ::Gem::Net::InternetMessageIO + end + +end # module Gem::Net diff --git a/lib/rubygems/vendor/optparse/.document b/lib/rubygems/vendor/optparse/.document new file mode 100644 index 0000000000..0c43bbd6b3 --- /dev/null +++ b/lib/rubygems/vendor/optparse/.document @@ -0,0 +1 @@ +# Vendored files do not need to be documented diff --git a/lib/rubygems/optparse/lib/optionparser.rb b/lib/rubygems/vendor/optparse/lib/optionparser.rb index 4b9b40d82a..4b9b40d82a 100644 --- a/lib/rubygems/optparse/lib/optionparser.rb +++ b/lib/rubygems/vendor/optparse/lib/optionparser.rb diff --git a/lib/rubygems/optparse/lib/optparse.rb b/lib/rubygems/vendor/optparse/lib/optparse.rb index 1e50bda769..5937431720 100644 --- a/lib/rubygems/optparse/lib/optparse.rb +++ b/lib/rubygems/vendor/optparse/lib/optparse.rb @@ -48,7 +48,7 @@ # # == Gem::OptionParser # -# === New to \Gem::OptionParser? +# === New to +Gem::OptionParser+? # # See the {Tutorial}[optparse/tutorial.rdoc]. # @@ -73,7 +73,7 @@ # # === Minimal example # -# require 'rubygems/optparse/lib/optparse' +# require 'rubygems/vendor/optparse/lib/optparse' # # options = {} # Gem::OptionParser.new do |parser| @@ -92,7 +92,7 @@ # Gem::OptionParser can be used to automatically generate help for the commands you # write: # -# require 'rubygems/optparse/lib/optparse' +# require 'rubygems/vendor/optparse/lib/optparse' # # Options = Struct.new(:name) # @@ -130,7 +130,7 @@ # option name in all caps. If an option is used without the required argument, # an exception will be raised. # -# require 'rubygems/optparse/lib/optparse' +# require 'rubygems/vendor/optparse/lib/optparse' # # options = {} # Gem::OptionParser.new do |parser| @@ -152,14 +152,14 @@ # Gem::OptionParser supports the ability to coerce command line arguments # into objects for us. # -# Gem::OptionParser comes with a few ready-to-use kinds of type +# Gem::OptionParser comes with a few ready-to-use kinds of type # coercion. They are: # -# - Date -- Anything accepted by +Date.parse+ -# - DateTime -- Anything accepted by +DateTime.parse+ -# - Time -- Anything accepted by +Time.httpdate+ or +Time.parse+ -# - URI -- Anything accepted by +URI.parse+ -# - Shellwords -- Anything accepted by +Shellwords.shellwords+ +# - Date -- Anything accepted by +Date.parse+ (need to require +optparse/date+) +# - DateTime -- Anything accepted by +DateTime.parse+ (need to require +optparse/date+) +# - Time -- Anything accepted by +Time.httpdate+ or +Time.parse+ (need to require +optparse/time+) +# - URI -- Anything accepted by +Gem::URI.parse+ (need to require +optparse/uri+) +# - Shellwords -- Anything accepted by +Shellwords.shellwords+ (need to require +optparse/shellwords+) # - String -- Any non-empty string # - Integer -- Any integer. Will convert octal. (e.g. 124, -3, 040) # - Float -- Any float. (e.g. 10, 3.14, -100E+13) @@ -183,8 +183,8 @@ # as a +Time+. If it succeeds, that time will be passed to the # handler block. Otherwise, an exception will be raised. # -# require 'rubygems/optparse/lib/optparse' -# require 'rubygems/optparse/lib/optparse/time' +# require 'rubygems/vendor/optparse/lib/optparse' +# require 'rubygems/vendor/optparse/lib/optparse/time' # Gem::OptionParser.new do |parser| # parser.on("-t", "--time [TIME]", Time, "Begin execution at given time") do |time| # p time @@ -206,7 +206,7 @@ # It specifies which conversion block to call whenever a class is specified. # The example below uses it to fetch a +User+ object before the +on+ handler receives it. # -# require 'rubygems/optparse/lib/optparse' +# require 'rubygems/vendor/optparse/lib/optparse' # # User = Struct.new(:id, :name) # @@ -242,7 +242,7 @@ # # The +into+ option of +order+, +parse+ and so on methods stores command line options into a Hash. # -# require 'rubygems/optparse/lib/optparse' +# require 'rubygems/vendor/optparse/lib/optparse' # # options = {} # Gem::OptionParser.new do |parser| @@ -268,8 +268,8 @@ # effect of specifying various options. This is probably the best way to learn # the features of +optparse+. # -# require 'rubygems/optparse/lib/optparse' -# require 'rubygems/optparse/lib/optparse/time' +# require 'rubygems/vendor/optparse/lib/optparse' +# require 'rubygems/vendor/optparse/lib/optparse/time' # require 'ostruct' # require 'pp' # @@ -425,7 +425,7 @@ # If you have any questions, file a ticket at http://bugs.ruby-lang.org. # class Gem::OptionParser - Gem::OptionParser::Version = "0.3.0" + Gem::OptionParser::Version = "0.4.0" # :stopdoc: NoArgument = [NO_ARGUMENT = :NONE, nil].freeze @@ -1084,7 +1084,7 @@ XXX Switch::OptionalArgument.new do |pkg| if pkg begin - require 'rubygems/optparse/lib/optparse/version' + require 'rubygems/vendor/optparse/lib/optparse/version' rescue LoadError else show_version(*pkg.split(/,/)) or @@ -1775,7 +1775,16 @@ XXX # # params["bar"] = "x" # --bar x # # params["zot"] = "z" # --zot Z # - def getopts(*args) + # Option +symbolize_names+ (boolean) specifies whether returned Hash keys should be Symbols; defaults to +false+ (use Strings). + # + # params = ARGV.getopts("ab:", "foo", "bar:", "zot:Z;zot option", symbolize_names: true) + # # params[:a] = true # -a + # # params[:b] = "1" # -b1 + # # params[:foo] = "1" # --foo + # # params[:bar] = "x" # --bar x + # # params[:zot] = "z" # --zot Z + # + def getopts(*args, symbolize_names: false) argv = Array === args.first ? args.shift : default_argv single_options, *long_options = *args @@ -1804,14 +1813,14 @@ XXX end parse_in_order(argv, result.method(:[]=)) - result + symbolize_names ? result.transform_keys(&:to_sym) : result end # # See #getopts. # - def self.getopts(*args) - new.getopts(*args) + def self.getopts(*args, symbolize_names: false) + new.getopts(*args, symbolize_names: symbolize_names) end # @@ -2084,10 +2093,23 @@ XXX f |= Regexp::IGNORECASE if /i/ =~ o f |= Regexp::MULTILINE if /m/ =~ o f |= Regexp::EXTENDED if /x/ =~ o - k = o.delete("imx") - k = nil if k.empty? + case o = o.delete("imx") + when "" + when "u" + s = s.encode(Encoding::UTF_8) + when "e" + s = s.encode(Encoding::EUC_JP) + when "s" + s = s.encode(Encoding::SJIS) + when "n" + f |= Regexp::NOENCODING + else + raise Gem::OptionParser::InvalidArgument, "unknown regexp option - #{o}" + end + else + s ||= all end - Regexp.new(s || all, f, k) + Regexp.new(s, f) end # @@ -2276,8 +2298,8 @@ XXX # rescue Gem::OptionParser::ParseError # end # - def getopts(*args) - options.getopts(self, *args) + def getopts(*args, symbolize_names: false) + options.getopts(self, *args, symbolize_names: symbolize_names) end # diff --git a/lib/rubygems/optparse/lib/optparse/ac.rb b/lib/rubygems/vendor/optparse/lib/optparse/ac.rb index e84d01bf91..e84d01bf91 100644 --- a/lib/rubygems/optparse/lib/optparse/ac.rb +++ b/lib/rubygems/vendor/optparse/lib/optparse/ac.rb diff --git a/lib/rubygems/optparse/lib/optparse/date.rb b/lib/rubygems/vendor/optparse/lib/optparse/date.rb index d9a9f4f48a..d9a9f4f48a 100644 --- a/lib/rubygems/optparse/lib/optparse/date.rb +++ b/lib/rubygems/vendor/optparse/lib/optparse/date.rb diff --git a/lib/rubygems/optparse/lib/optparse/kwargs.rb b/lib/rubygems/vendor/optparse/lib/optparse/kwargs.rb index 6987a5ed62..6987a5ed62 100644 --- a/lib/rubygems/optparse/lib/optparse/kwargs.rb +++ b/lib/rubygems/vendor/optparse/lib/optparse/kwargs.rb diff --git a/lib/rubygems/optparse/lib/optparse/shellwords.rb b/lib/rubygems/vendor/optparse/lib/optparse/shellwords.rb index d47ad60255..d47ad60255 100644 --- a/lib/rubygems/optparse/lib/optparse/shellwords.rb +++ b/lib/rubygems/vendor/optparse/lib/optparse/shellwords.rb diff --git a/lib/rubygems/optparse/lib/optparse/time.rb b/lib/rubygems/vendor/optparse/lib/optparse/time.rb index c59e1e4ced..c59e1e4ced 100644 --- a/lib/rubygems/optparse/lib/optparse/time.rb +++ b/lib/rubygems/vendor/optparse/lib/optparse/time.rb diff --git a/lib/rubygems/vendor/optparse/lib/optparse/uri.rb b/lib/rubygems/vendor/optparse/lib/optparse/uri.rb new file mode 100644 index 0000000000..398127479a --- /dev/null +++ b/lib/rubygems/vendor/optparse/lib/optparse/uri.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: false +# -*- ruby -*- + +require_relative '../optparse' +require_relative '../../../uri/lib/uri' + +Gem::OptionParser.accept(Gem::URI) {|s,| Gem::URI.parse(s) if s} diff --git a/lib/rubygems/optparse/lib/optparse/version.rb b/lib/rubygems/vendor/optparse/lib/optparse/version.rb index 5d79e9db44..5d79e9db44 100644 --- a/lib/rubygems/optparse/lib/optparse/version.rb +++ b/lib/rubygems/vendor/optparse/lib/optparse/version.rb diff --git a/lib/rubygems/vendor/resolv/.document b/lib/rubygems/vendor/resolv/.document new file mode 100644 index 0000000000..0c43bbd6b3 --- /dev/null +++ b/lib/rubygems/vendor/resolv/.document @@ -0,0 +1 @@ +# Vendored files do not need to be documented diff --git a/lib/rubygems/vendor/resolv/lib/resolv.rb b/lib/rubygems/vendor/resolv/lib/resolv.rb new file mode 100644 index 0000000000..ac0ba0b313 --- /dev/null +++ b/lib/rubygems/vendor/resolv/lib/resolv.rb @@ -0,0 +1,3442 @@ +# frozen_string_literal: true + +require 'socket' +require_relative '../../timeout/lib/timeout' +require 'io/wait' + +begin + require 'securerandom' +rescue LoadError +end + +# Gem::Resolv is a thread-aware DNS resolver library written in Ruby. Gem::Resolv can +# handle multiple DNS requests concurrently without blocking the entire Ruby +# interpreter. +# +# See also resolv-replace.rb to replace the libc resolver with Gem::Resolv. +# +# Gem::Resolv can look up various DNS resources using the DNS module directly. +# +# Examples: +# +# p Gem::Resolv.getaddress "www.ruby-lang.org" +# p Gem::Resolv.getname "210.251.121.214" +# +# Gem::Resolv::DNS.open do |dns| +# ress = dns.getresources "www.ruby-lang.org", Gem::Resolv::DNS::Resource::IN::A +# p ress.map(&:address) +# ress = dns.getresources "ruby-lang.org", Gem::Resolv::DNS::Resource::IN::MX +# p ress.map { |r| [r.exchange.to_s, r.preference] } +# end +# +# +# == Bugs +# +# * NIS is not supported. +# * /etc/nsswitch.conf is not supported. + +class Gem::Resolv + + VERSION = "0.4.0" + + ## + # Looks up the first IP address for +name+. + + def self.getaddress(name) + DefaultResolver.getaddress(name) + end + + ## + # Looks up all IP address for +name+. + + def self.getaddresses(name) + DefaultResolver.getaddresses(name) + end + + ## + # Iterates over all IP addresses for +name+. + + def self.each_address(name, &block) + DefaultResolver.each_address(name, &block) + end + + ## + # Looks up the hostname of +address+. + + def self.getname(address) + DefaultResolver.getname(address) + end + + ## + # Looks up all hostnames for +address+. + + def self.getnames(address) + DefaultResolver.getnames(address) + end + + ## + # Iterates over all hostnames for +address+. + + def self.each_name(address, &proc) + DefaultResolver.each_name(address, &proc) + end + + ## + # Creates a new Gem::Resolv using +resolvers+. + + def initialize(resolvers=nil, use_ipv6: nil) + @resolvers = resolvers || [Hosts.new, DNS.new(DNS::Config.default_config_hash.merge(use_ipv6: use_ipv6))] + end + + ## + # Looks up the first IP address for +name+. + + def getaddress(name) + each_address(name) {|address| return address} + raise ResolvError.new("no address for #{name}") + end + + ## + # Looks up all IP address for +name+. + + def getaddresses(name) + ret = [] + each_address(name) {|address| ret << address} + return ret + end + + ## + # Iterates over all IP addresses for +name+. + + def each_address(name) + if AddressRegex =~ name + yield name + return + end + yielded = false + @resolvers.each {|r| + r.each_address(name) {|address| + yield address.to_s + yielded = true + } + return if yielded + } + end + + ## + # Looks up the hostname of +address+. + + def getname(address) + each_name(address) {|name| return name} + raise ResolvError.new("no name for #{address}") + end + + ## + # Looks up all hostnames for +address+. + + def getnames(address) + ret = [] + each_name(address) {|name| ret << name} + return ret + end + + ## + # Iterates over all hostnames for +address+. + + def each_name(address) + yielded = false + @resolvers.each {|r| + r.each_name(address) {|name| + yield name.to_s + yielded = true + } + return if yielded + } + end + + ## + # Indicates a failure to resolve a name or address. + + class ResolvError < StandardError; end + + ## + # Indicates a timeout resolving a name or address. + + class ResolvTimeout < Gem::Timeout::Error; end + + ## + # Gem::Resolv::Hosts is a hostname resolver that uses the system hosts file. + + class Hosts + if /mswin|mingw|cygwin/ =~ RUBY_PLATFORM and + begin + require 'win32/resolv' + DefaultFileName = Win32::Resolv.get_hosts_path || IO::NULL + rescue LoadError + end + end + DefaultFileName ||= '/etc/hosts' + + ## + # Creates a new Gem::Resolv::Hosts, using +filename+ for its data source. + + def initialize(filename = DefaultFileName) + @filename = filename + @mutex = Thread::Mutex.new + @initialized = nil + end + + def lazy_initialize # :nodoc: + @mutex.synchronize { + unless @initialized + @name2addr = {} + @addr2name = {} + File.open(@filename, 'rb') {|f| + f.each {|line| + line.sub!(/#.*/, '') + addr, *hostnames = line.split(/\s+/) + next unless addr + (@addr2name[addr] ||= []).concat(hostnames) + hostnames.each {|hostname| (@name2addr[hostname] ||= []) << addr} + } + } + @name2addr.each {|name, arr| arr.reverse!} + @initialized = true + end + } + self + end + + ## + # Gets the IP address of +name+ from the hosts file. + + def getaddress(name) + each_address(name) {|address| return address} + raise ResolvError.new("#{@filename} has no name: #{name}") + end + + ## + # Gets all IP addresses for +name+ from the hosts file. + + def getaddresses(name) + ret = [] + each_address(name) {|address| ret << address} + return ret + end + + ## + # Iterates over all IP addresses for +name+ retrieved from the hosts file. + + def each_address(name, &proc) + lazy_initialize + @name2addr[name]&.each(&proc) + end + + ## + # Gets the hostname of +address+ from the hosts file. + + def getname(address) + each_name(address) {|name| return name} + raise ResolvError.new("#{@filename} has no address: #{address}") + end + + ## + # Gets all hostnames for +address+ from the hosts file. + + def getnames(address) + ret = [] + each_name(address) {|name| ret << name} + return ret + end + + ## + # Iterates over all hostnames for +address+ retrieved from the hosts file. + + def each_name(address, &proc) + lazy_initialize + @addr2name[address]&.each(&proc) + end + end + + ## + # Gem::Resolv::DNS is a DNS stub resolver. + # + # Information taken from the following places: + # + # * STD0013 + # * RFC 1035 + # * ftp://ftp.isi.edu/in-notes/iana/assignments/dns-parameters + # * etc. + + class DNS + + ## + # Default DNS Port + + Port = 53 + + ## + # Default DNS UDP packet size + + UDPSize = 512 + + ## + # Creates a new DNS resolver. See Gem::Resolv::DNS.new for argument details. + # + # Yields the created DNS resolver to the block, if given, otherwise + # returns it. + + def self.open(*args) + dns = new(*args) + return dns unless block_given? + begin + yield dns + ensure + dns.close + end + end + + ## + # Creates a new DNS resolver. + # + # +config_info+ can be: + # + # nil:: Uses /etc/resolv.conf. + # String:: Path to a file using /etc/resolv.conf's format. + # Hash:: Must contain :nameserver, :search and :ndots keys. + # :nameserver_port can be used to specify port number of nameserver address. + # :raise_timeout_errors can be used to raise timeout errors + # as exceptions instead of treating the same as an NXDOMAIN response. + # + # The value of :nameserver should be an address string or + # an array of address strings. + # - :nameserver => '8.8.8.8' + # - :nameserver => ['8.8.8.8', '8.8.4.4'] + # + # The value of :nameserver_port should be an array of + # pair of nameserver address and port number. + # - :nameserver_port => [['8.8.8.8', 53], ['8.8.4.4', 53]] + # + # Example: + # + # Gem::Resolv::DNS.new(:nameserver => ['210.251.121.21'], + # :search => ['ruby-lang.org'], + # :ndots => 1) + + def initialize(config_info=nil) + @mutex = Thread::Mutex.new + @config = Config.new(config_info) + @initialized = nil + end + + # Sets the resolver timeouts. This may be a single positive number + # or an array of positive numbers representing timeouts in seconds. + # If an array is specified, a DNS request will retry and wait for + # each successive interval in the array until a successful response + # is received. Specifying +nil+ reverts to the default timeouts: + # [ 5, second = 5 * 2 / nameserver_count, 2 * second, 4 * second ] + # + # Example: + # + # dns.timeouts = 3 + # + def timeouts=(values) + @config.timeouts = values + end + + def lazy_initialize # :nodoc: + @mutex.synchronize { + unless @initialized + @config.lazy_initialize + @initialized = true + end + } + self + end + + ## + # Closes the DNS resolver. + + def close + @mutex.synchronize { + if @initialized + @initialized = false + end + } + end + + ## + # Gets the IP address of +name+ from the DNS resolver. + # + # +name+ can be a Gem::Resolv::DNS::Name or a String. Retrieved address will + # be a Gem::Resolv::IPv4 or Gem::Resolv::IPv6 + + def getaddress(name) + each_address(name) {|address| return address} + raise ResolvError.new("DNS result has no information for #{name}") + end + + ## + # Gets all IP addresses for +name+ from the DNS resolver. + # + # +name+ can be a Gem::Resolv::DNS::Name or a String. Retrieved addresses will + # be a Gem::Resolv::IPv4 or Gem::Resolv::IPv6 + + def getaddresses(name) + ret = [] + each_address(name) {|address| ret << address} + return ret + end + + ## + # Iterates over all IP addresses for +name+ retrieved from the DNS + # resolver. + # + # +name+ can be a Gem::Resolv::DNS::Name or a String. Retrieved addresses will + # be a Gem::Resolv::IPv4 or Gem::Resolv::IPv6 + + def each_address(name) + each_resource(name, Resource::IN::A) {|resource| yield resource.address} + if use_ipv6? + each_resource(name, Resource::IN::AAAA) {|resource| yield resource.address} + end + end + + def use_ipv6? # :nodoc: + use_ipv6 = @config.use_ipv6? + unless use_ipv6.nil? + return use_ipv6 + end + + begin + list = Socket.ip_address_list + rescue NotImplementedError + return true + end + list.any? {|a| a.ipv6? && !a.ipv6_loopback? && !a.ipv6_linklocal? } + end + private :use_ipv6? + + ## + # Gets the hostname for +address+ from the DNS resolver. + # + # +address+ must be a Gem::Resolv::IPv4, Gem::Resolv::IPv6 or a String. Retrieved + # name will be a Gem::Resolv::DNS::Name. + + def getname(address) + each_name(address) {|name| return name} + raise ResolvError.new("DNS result has no information for #{address}") + end + + ## + # Gets all hostnames for +address+ from the DNS resolver. + # + # +address+ must be a Gem::Resolv::IPv4, Gem::Resolv::IPv6 or a String. Retrieved + # names will be Gem::Resolv::DNS::Name instances. + + def getnames(address) + ret = [] + each_name(address) {|name| ret << name} + return ret + end + + ## + # Iterates over all hostnames for +address+ retrieved from the DNS + # resolver. + # + # +address+ must be a Gem::Resolv::IPv4, Gem::Resolv::IPv6 or a String. Retrieved + # names will be Gem::Resolv::DNS::Name instances. + + def each_name(address) + case address + when Name + ptr = address + when IPv4, IPv6 + ptr = address.to_name + when IPv4::Regex + ptr = IPv4.create(address).to_name + when IPv6::Regex + ptr = IPv6.create(address).to_name + else + raise ResolvError.new("cannot interpret as address: #{address}") + end + each_resource(ptr, Resource::IN::PTR) {|resource| yield resource.name} + end + + ## + # Look up the +typeclass+ DNS resource of +name+. + # + # +name+ must be a Gem::Resolv::DNS::Name or a String. + # + # +typeclass+ should be one of the following: + # + # * Gem::Resolv::DNS::Resource::IN::A + # * Gem::Resolv::DNS::Resource::IN::AAAA + # * Gem::Resolv::DNS::Resource::IN::ANY + # * Gem::Resolv::DNS::Resource::IN::CNAME + # * Gem::Resolv::DNS::Resource::IN::HINFO + # * Gem::Resolv::DNS::Resource::IN::MINFO + # * Gem::Resolv::DNS::Resource::IN::MX + # * Gem::Resolv::DNS::Resource::IN::NS + # * Gem::Resolv::DNS::Resource::IN::PTR + # * Gem::Resolv::DNS::Resource::IN::SOA + # * Gem::Resolv::DNS::Resource::IN::TXT + # * Gem::Resolv::DNS::Resource::IN::WKS + # + # Returned resource is represented as a Gem::Resolv::DNS::Resource instance, + # i.e. Gem::Resolv::DNS::Resource::IN::A. + + def getresource(name, typeclass) + each_resource(name, typeclass) {|resource| return resource} + raise ResolvError.new("DNS result has no information for #{name}") + end + + ## + # Looks up all +typeclass+ DNS resources for +name+. See #getresource for + # argument details. + + def getresources(name, typeclass) + ret = [] + each_resource(name, typeclass) {|resource| ret << resource} + return ret + end + + ## + # Iterates over all +typeclass+ DNS resources for +name+. See + # #getresource for argument details. + + def each_resource(name, typeclass, &proc) + fetch_resource(name, typeclass) {|reply, reply_name| + extract_resources(reply, reply_name, typeclass, &proc) + } + end + + def fetch_resource(name, typeclass) + lazy_initialize + begin + requester = make_udp_requester + rescue Errno::EACCES + # fall back to TCP + end + senders = {} + begin + @config.resolv(name) {|candidate, tout, nameserver, port| + requester ||= make_tcp_requester(nameserver, port) + msg = Message.new + msg.rd = 1 + msg.add_question(candidate, typeclass) + unless sender = senders[[candidate, nameserver, port]] + sender = requester.sender(msg, candidate, nameserver, port) + next if !sender + senders[[candidate, nameserver, port]] = sender + end + reply, reply_name = requester.request(sender, tout) + case reply.rcode + when RCode::NoError + if reply.tc == 1 and not Requester::TCP === requester + requester.close + # Retry via TCP: + requester = make_tcp_requester(nameserver, port) + senders = {} + # This will use TCP for all remaining candidates (assuming the + # current candidate does not already respond successfully via + # TCP). This makes sense because we already know the full + # response will not fit in an untruncated UDP packet. + redo + else + yield(reply, reply_name) + end + return + when RCode::NXDomain + raise Config::NXDomain.new(reply_name.to_s) + else + raise Config::OtherResolvError.new(reply_name.to_s) + end + } + ensure + requester&.close + end + end + + def make_udp_requester # :nodoc: + nameserver_port = @config.nameserver_port + if nameserver_port.length == 1 + Requester::ConnectedUDP.new(*nameserver_port[0]) + else + Requester::UnconnectedUDP.new(*nameserver_port) + end + end + + def make_tcp_requester(host, port) # :nodoc: + return Requester::TCP.new(host, port) + end + + def extract_resources(msg, name, typeclass) # :nodoc: + if typeclass < Resource::ANY + n0 = Name.create(name) + msg.each_resource {|n, ttl, data| + yield data if n0 == n + } + end + yielded = false + n0 = Name.create(name) + msg.each_resource {|n, ttl, data| + if n0 == n + case data + when typeclass + yield data + yielded = true + when Resource::CNAME + n0 = data.name + end + end + } + return if yielded + msg.each_resource {|n, ttl, data| + if n0 == n + case data + when typeclass + yield data + end + end + } + end + + if defined? SecureRandom + def self.random(arg) # :nodoc: + begin + SecureRandom.random_number(arg) + rescue NotImplementedError + rand(arg) + end + end + else + def self.random(arg) # :nodoc: + rand(arg) + end + end + + RequestID = {} # :nodoc: + RequestIDMutex = Thread::Mutex.new # :nodoc: + + def self.allocate_request_id(host, port) # :nodoc: + id = nil + RequestIDMutex.synchronize { + h = (RequestID[[host, port]] ||= {}) + begin + id = random(0x0000..0xffff) + end while h[id] + h[id] = true + } + id + end + + def self.free_request_id(host, port, id) # :nodoc: + RequestIDMutex.synchronize { + key = [host, port] + if h = RequestID[key] + h.delete id + if h.empty? + RequestID.delete key + end + end + } + end + + def self.bind_random_port(udpsock, bind_host="0.0.0.0") # :nodoc: + begin + port = random(1024..65535) + udpsock.bind(bind_host, port) + rescue Errno::EADDRINUSE, # POSIX + Errno::EACCES, # SunOS: See PRIV_SYS_NFS in privileges(5) + Errno::EPERM # FreeBSD: security.mac.portacl.port_high is configurable. See mac_portacl(4). + retry + end + end + + class Requester # :nodoc: + def initialize + @senders = {} + @socks = nil + end + + def request(sender, tout) + start = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timelimit = start + tout + begin + sender.send + rescue Errno::EHOSTUNREACH, # multi-homed IPv6 may generate this + Errno::ENETUNREACH + raise ResolvTimeout + end + while true + before_select = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout = timelimit - before_select + if timeout <= 0 + raise ResolvTimeout + end + if @socks.size == 1 + select_result = @socks[0].wait_readable(timeout) ? [ @socks ] : nil + else + select_result = IO.select(@socks, nil, nil, timeout) + end + if !select_result + after_select = Process.clock_gettime(Process::CLOCK_MONOTONIC) + next if after_select < timelimit + raise ResolvTimeout + end + begin + reply, from = recv_reply(select_result[0]) + rescue Errno::ECONNREFUSED, # GNU/Linux, FreeBSD + Errno::ECONNRESET # Windows + # No name server running on the server? + # Don't wait anymore. + raise ResolvTimeout + end + begin + msg = Message.decode(reply) + rescue DecodeError + next # broken DNS message ignored + end + if sender == sender_for(from, msg) + break + else + # unexpected DNS message ignored + end + end + return msg, sender.data + end + + def sender_for(addr, msg) + @senders[[addr,msg.id]] + end + + def close + socks = @socks + @socks = nil + socks&.each(&:close) + end + + class Sender # :nodoc: + def initialize(msg, data, sock) + @msg = msg + @data = data + @sock = sock + end + end + + class UnconnectedUDP < Requester # :nodoc: + def initialize(*nameserver_port) + super() + @nameserver_port = nameserver_port + @initialized = false + @mutex = Thread::Mutex.new + end + + def lazy_initialize + @mutex.synchronize { + next if @initialized + @initialized = true + @socks_hash = {} + @socks = [] + @nameserver_port.each {|host, port| + if host.index(':') + bind_host = "::" + af = Socket::AF_INET6 + else + bind_host = "0.0.0.0" + af = Socket::AF_INET + end + next if @socks_hash[bind_host] + begin + sock = UDPSocket.new(af) + rescue Errno::EAFNOSUPPORT, Errno::EPROTONOSUPPORT + next # The kernel doesn't support the address family. + end + @socks << sock + @socks_hash[bind_host] = sock + sock.do_not_reverse_lookup = true + DNS.bind_random_port(sock, bind_host) + } + } + self + end + + def recv_reply(readable_socks) + lazy_initialize + reply, from = readable_socks[0].recvfrom(UDPSize) + return reply, [from[3],from[1]] + end + + def sender(msg, data, host, port=Port) + host = Addrinfo.ip(host).ip_address + lazy_initialize + sock = @socks_hash[host.index(':') ? "::" : "0.0.0.0"] + return nil if !sock + service = [host, port] + id = DNS.allocate_request_id(host, port) + request = msg.encode + request[0,2] = [id].pack('n') + return @senders[[service, id]] = + Sender.new(request, data, sock, host, port) + end + + def close + @mutex.synchronize { + if @initialized + super + @senders.each_key {|service, id| + DNS.free_request_id(service[0], service[1], id) + } + @initialized = false + end + } + end + + class Sender < Requester::Sender # :nodoc: + def initialize(msg, data, sock, host, port) + super(msg, data, sock) + @host = host + @port = port + end + attr_reader :data + + def send + raise "@sock is nil." if @sock.nil? + @sock.send(@msg, 0, @host, @port) + end + end + end + + class ConnectedUDP < Requester # :nodoc: + def initialize(host, port=Port) + super() + @host = host + @port = port + @mutex = Thread::Mutex.new + @initialized = false + end + + def lazy_initialize + @mutex.synchronize { + next if @initialized + @initialized = true + is_ipv6 = @host.index(':') + sock = UDPSocket.new(is_ipv6 ? Socket::AF_INET6 : Socket::AF_INET) + @socks = [sock] + sock.do_not_reverse_lookup = true + DNS.bind_random_port(sock, is_ipv6 ? "::" : "0.0.0.0") + sock.connect(@host, @port) + } + self + end + + def recv_reply(readable_socks) + lazy_initialize + reply = readable_socks[0].recv(UDPSize) + return reply, nil + end + + def sender(msg, data, host=@host, port=@port) + lazy_initialize + unless host == @host && port == @port + raise RequestError.new("host/port don't match: #{host}:#{port}") + end + id = DNS.allocate_request_id(@host, @port) + request = msg.encode + request[0,2] = [id].pack('n') + return @senders[[nil,id]] = Sender.new(request, data, @socks[0]) + end + + def close + @mutex.synchronize do + if @initialized + super + @senders.each_key {|from, id| + DNS.free_request_id(@host, @port, id) + } + @initialized = false + end + end + end + + class Sender < Requester::Sender # :nodoc: + def send + raise "@sock is nil." if @sock.nil? + @sock.send(@msg, 0) + end + attr_reader :data + end + end + + class MDNSOneShot < UnconnectedUDP # :nodoc: + def sender(msg, data, host, port=Port) + lazy_initialize + id = DNS.allocate_request_id(host, port) + request = msg.encode + request[0,2] = [id].pack('n') + sock = @socks_hash[host.index(':') ? "::" : "0.0.0.0"] + return @senders[id] = + UnconnectedUDP::Sender.new(request, data, sock, host, port) + end + + def sender_for(addr, msg) + lazy_initialize + @senders[msg.id] + end + end + + class TCP < Requester # :nodoc: + def initialize(host, port=Port) + super() + @host = host + @port = port + sock = TCPSocket.new(@host, @port) + @socks = [sock] + @senders = {} + end + + def recv_reply(readable_socks) + len = readable_socks[0].read(2).unpack('n')[0] + reply = @socks[0].read(len) + return reply, nil + end + + def sender(msg, data, host=@host, port=@port) + unless host == @host && port == @port + raise RequestError.new("host/port don't match: #{host}:#{port}") + end + id = DNS.allocate_request_id(@host, @port) + request = msg.encode + request[0,2] = [request.length, id].pack('nn') + return @senders[[nil,id]] = Sender.new(request, data, @socks[0]) + end + + class Sender < Requester::Sender # :nodoc: + def send + @sock.print(@msg) + @sock.flush + end + attr_reader :data + end + + def close + super + @senders.each_key {|from,id| + DNS.free_request_id(@host, @port, id) + } + end + end + + ## + # Indicates a problem with the DNS request. + + class RequestError < StandardError + end + end + + class Config # :nodoc: + def initialize(config_info=nil) + @mutex = Thread::Mutex.new + @config_info = config_info + @initialized = nil + @timeouts = nil + end + + def timeouts=(values) + if values + values = Array(values) + values.each do |t| + Numeric === t or raise ArgumentError, "#{t.inspect} is not numeric" + t > 0.0 or raise ArgumentError, "timeout=#{t} must be positive" + end + @timeouts = values + else + @timeouts = nil + end + end + + def Config.parse_resolv_conf(filename) + nameserver = [] + search = nil + ndots = 1 + File.open(filename, 'rb') {|f| + f.each {|line| + line.sub!(/[#;].*/, '') + keyword, *args = line.split(/\s+/) + next unless keyword + case keyword + when 'nameserver' + nameserver.concat(args) + when 'domain' + next if args.empty? + search = [args[0]] + when 'search' + next if args.empty? + search = args + when 'options' + args.each {|arg| + case arg + when /\Andots:(\d+)\z/ + ndots = $1.to_i + end + } + end + } + } + return { :nameserver => nameserver, :search => search, :ndots => ndots } + end + + def Config.default_config_hash(filename="/etc/resolv.conf") + if File.exist? filename + config_hash = Config.parse_resolv_conf(filename) + else + if /mswin|cygwin|mingw|bccwin/ =~ RUBY_PLATFORM + require 'win32/resolv' + search, nameserver = Win32::Resolv.get_resolv_info + config_hash = {} + config_hash[:nameserver] = nameserver if nameserver + config_hash[:search] = [search].flatten if search + end + end + config_hash || {} + end + + def lazy_initialize + @mutex.synchronize { + unless @initialized + @nameserver_port = [] + @use_ipv6 = nil + @search = nil + @ndots = 1 + case @config_info + when nil + config_hash = Config.default_config_hash + when String + config_hash = Config.parse_resolv_conf(@config_info) + when Hash + config_hash = @config_info.dup + if String === config_hash[:nameserver] + config_hash[:nameserver] = [config_hash[:nameserver]] + end + if String === config_hash[:search] + config_hash[:search] = [config_hash[:search]] + end + else + raise ArgumentError.new("invalid resolv configuration: #{@config_info.inspect}") + end + if config_hash.include? :nameserver + @nameserver_port = config_hash[:nameserver].map {|ns| [ns, Port] } + end + if config_hash.include? :nameserver_port + @nameserver_port = config_hash[:nameserver_port].map {|ns, port| [ns, (port || Port)] } + end + if config_hash.include? :use_ipv6 + @use_ipv6 = config_hash[:use_ipv6] + end + @search = config_hash[:search] if config_hash.include? :search + @ndots = config_hash[:ndots] if config_hash.include? :ndots + @raise_timeout_errors = config_hash[:raise_timeout_errors] + + if @nameserver_port.empty? + @nameserver_port << ['0.0.0.0', Port] + end + if @search + @search = @search.map {|arg| Label.split(arg) } + else + hostname = Socket.gethostname + if /\./ =~ hostname + @search = [Label.split($')] + else + @search = [[]] + end + end + + if !@nameserver_port.kind_of?(Array) || + @nameserver_port.any? {|ns_port| + !(Array === ns_port) || + ns_port.length != 2 + !(String === ns_port[0]) || + !(Integer === ns_port[1]) + } + raise ArgumentError.new("invalid nameserver config: #{@nameserver_port.inspect}") + end + + if !@search.kind_of?(Array) || + !@search.all? {|ls| ls.all? {|l| Label::Str === l } } + raise ArgumentError.new("invalid search config: #{@search.inspect}") + end + + if !@ndots.kind_of?(Integer) + raise ArgumentError.new("invalid ndots config: #{@ndots.inspect}") + end + + @initialized = true + end + } + self + end + + def single? + lazy_initialize + if @nameserver_port.length == 1 + return @nameserver_port[0] + else + return nil + end + end + + def nameserver_port + @nameserver_port + end + + def use_ipv6? + @use_ipv6 + end + + def generate_candidates(name) + candidates = nil + name = Name.create(name) + if name.absolute? + candidates = [name] + else + if @ndots <= name.length - 1 + candidates = [Name.new(name.to_a)] + else + candidates = [] + end + candidates.concat(@search.map {|domain| Name.new(name.to_a + domain)}) + fname = Name.create("#{name}.") + if !candidates.include?(fname) + candidates << fname + end + end + return candidates + end + + InitialTimeout = 5 + + def generate_timeouts + ts = [InitialTimeout] + ts << ts[-1] * 2 / @nameserver_port.length + ts << ts[-1] * 2 + ts << ts[-1] * 2 + return ts + end + + def resolv(name) + candidates = generate_candidates(name) + timeouts = @timeouts || generate_timeouts + timeout_error = false + begin + candidates.each {|candidate| + begin + timeouts.each {|tout| + @nameserver_port.each {|nameserver, port| + begin + yield candidate, tout, nameserver, port + rescue ResolvTimeout + end + } + } + timeout_error = true + raise ResolvError.new("DNS resolv timeout: #{name}") + rescue NXDomain + end + } + rescue ResolvError + raise if @raise_timeout_errors && timeout_error + end + end + + ## + # Indicates no such domain was found. + + class NXDomain < ResolvError + end + + ## + # Indicates some other unhandled resolver error was encountered. + + class OtherResolvError < ResolvError + end + end + + module OpCode # :nodoc: + Query = 0 + IQuery = 1 + Status = 2 + Notify = 4 + Update = 5 + end + + module RCode # :nodoc: + NoError = 0 + FormErr = 1 + ServFail = 2 + NXDomain = 3 + NotImp = 4 + Refused = 5 + YXDomain = 6 + YXRRSet = 7 + NXRRSet = 8 + NotAuth = 9 + NotZone = 10 + BADVERS = 16 + BADSIG = 16 + BADKEY = 17 + BADTIME = 18 + BADMODE = 19 + BADNAME = 20 + BADALG = 21 + end + + ## + # Indicates that the DNS response was unable to be decoded. + + class DecodeError < StandardError + end + + ## + # Indicates that the DNS request was unable to be encoded. + + class EncodeError < StandardError + end + + module Label # :nodoc: + def self.split(arg) + labels = [] + arg.scan(/[^\.]+/) {labels << Str.new($&)} + return labels + end + + class Str # :nodoc: + def initialize(string) + @string = string + # case insensivity of DNS labels doesn't apply non-ASCII characters. [RFC 4343] + # This assumes @string is given in ASCII compatible encoding. + @downcase = string.b.downcase + end + attr_reader :string, :downcase + + def to_s + return @string + end + + def inspect + return "#<#{self.class} #{self}>" + end + + def ==(other) + return self.class == other.class && @downcase == other.downcase + end + + def eql?(other) + return self == other + end + + def hash + return @downcase.hash + end + end + end + + ## + # A representation of a DNS name. + + class Name + + ## + # Creates a new DNS name from +arg+. +arg+ can be: + # + # Name:: returns +arg+. + # String:: Creates a new Name. + + def self.create(arg) + case arg + when Name + return arg + when String + return Name.new(Label.split(arg), /\.\z/ =~ arg ? true : false) + else + raise ArgumentError.new("cannot interpret as DNS name: #{arg.inspect}") + end + end + + def initialize(labels, absolute=true) # :nodoc: + labels = labels.map {|label| + case label + when String then Label::Str.new(label) + when Label::Str then label + else + raise ArgumentError, "unexpected label: #{label.inspect}" + end + } + @labels = labels + @absolute = absolute + end + + def inspect # :nodoc: + "#<#{self.class}: #{self}#{@absolute ? '.' : ''}>" + end + + ## + # True if this name is absolute. + + def absolute? + return @absolute + end + + def ==(other) # :nodoc: + return false unless Name === other + return false unless @absolute == other.absolute? + return @labels == other.to_a + end + + alias eql? == # :nodoc: + + ## + # Returns true if +other+ is a subdomain. + # + # Example: + # + # domain = Gem::Resolv::DNS::Name.create("y.z") + # p Gem::Resolv::DNS::Name.create("w.x.y.z").subdomain_of?(domain) #=> true + # p Gem::Resolv::DNS::Name.create("x.y.z").subdomain_of?(domain) #=> true + # p Gem::Resolv::DNS::Name.create("y.z").subdomain_of?(domain) #=> false + # p Gem::Resolv::DNS::Name.create("z").subdomain_of?(domain) #=> false + # p Gem::Resolv::DNS::Name.create("x.y.z.").subdomain_of?(domain) #=> false + # p Gem::Resolv::DNS::Name.create("w.z").subdomain_of?(domain) #=> false + # + + def subdomain_of?(other) + raise ArgumentError, "not a domain name: #{other.inspect}" unless Name === other + return false if @absolute != other.absolute? + other_len = other.length + return false if @labels.length <= other_len + return @labels[-other_len, other_len] == other.to_a + end + + def hash # :nodoc: + return @labels.hash ^ @absolute.hash + end + + def to_a # :nodoc: + return @labels + end + + def length # :nodoc: + return @labels.length + end + + def [](i) # :nodoc: + return @labels[i] + end + + ## + # returns the domain name as a string. + # + # The domain name doesn't have a trailing dot even if the name object is + # absolute. + # + # Example: + # + # p Gem::Resolv::DNS::Name.create("x.y.z.").to_s #=> "x.y.z" + # p Gem::Resolv::DNS::Name.create("x.y.z").to_s #=> "x.y.z" + + def to_s + return @labels.join('.') + end + end + + class Message # :nodoc: + @@identifier = -1 + + def initialize(id = (@@identifier += 1) & 0xffff) + @id = id + @qr = 0 + @opcode = 0 + @aa = 0 + @tc = 0 + @rd = 0 # recursion desired + @ra = 0 # recursion available + @rcode = 0 + @question = [] + @answer = [] + @authority = [] + @additional = [] + end + + attr_accessor :id, :qr, :opcode, :aa, :tc, :rd, :ra, :rcode + attr_reader :question, :answer, :authority, :additional + + def ==(other) + return @id == other.id && + @qr == other.qr && + @opcode == other.opcode && + @aa == other.aa && + @tc == other.tc && + @rd == other.rd && + @ra == other.ra && + @rcode == other.rcode && + @question == other.question && + @answer == other.answer && + @authority == other.authority && + @additional == other.additional + end + + def add_question(name, typeclass) + @question << [Name.create(name), typeclass] + end + + def each_question + @question.each {|name, typeclass| + yield name, typeclass + } + end + + def add_answer(name, ttl, data) + @answer << [Name.create(name), ttl, data] + end + + def each_answer + @answer.each {|name, ttl, data| + yield name, ttl, data + } + end + + def add_authority(name, ttl, data) + @authority << [Name.create(name), ttl, data] + end + + def each_authority + @authority.each {|name, ttl, data| + yield name, ttl, data + } + end + + def add_additional(name, ttl, data) + @additional << [Name.create(name), ttl, data] + end + + def each_additional + @additional.each {|name, ttl, data| + yield name, ttl, data + } + end + + def each_resource + each_answer {|name, ttl, data| yield name, ttl, data} + each_authority {|name, ttl, data| yield name, ttl, data} + each_additional {|name, ttl, data| yield name, ttl, data} + end + + def encode + return MessageEncoder.new {|msg| + msg.put_pack('nnnnnn', + @id, + (@qr & 1) << 15 | + (@opcode & 15) << 11 | + (@aa & 1) << 10 | + (@tc & 1) << 9 | + (@rd & 1) << 8 | + (@ra & 1) << 7 | + (@rcode & 15), + @question.length, + @answer.length, + @authority.length, + @additional.length) + @question.each {|q| + name, typeclass = q + msg.put_name(name) + msg.put_pack('nn', typeclass::TypeValue, typeclass::ClassValue) + } + [@answer, @authority, @additional].each {|rr| + rr.each {|r| + name, ttl, data = r + msg.put_name(name) + msg.put_pack('nnN', data.class::TypeValue, data.class::ClassValue, ttl) + msg.put_length16 {data.encode_rdata(msg)} + } + } + }.to_s + end + + class MessageEncoder # :nodoc: + def initialize + @data = ''.dup + @names = {} + yield self + end + + def to_s + return @data + end + + def put_bytes(d) + @data << d + end + + def put_pack(template, *d) + @data << d.pack(template) + end + + def put_length16 + length_index = @data.length + @data << "\0\0" + data_start = @data.length + yield + data_end = @data.length + @data[length_index, 2] = [data_end - data_start].pack("n") + end + + def put_string(d) + self.put_pack("C", d.length) + @data << d + end + + def put_string_list(ds) + ds.each {|d| + self.put_string(d) + } + end + + def put_name(d, compress: true) + put_labels(d.to_a, compress: compress) + end + + def put_labels(d, compress: true) + d.each_index {|i| + domain = d[i..-1] + if compress && idx = @names[domain] + self.put_pack("n", 0xc000 | idx) + return + else + if @data.length < 0x4000 + @names[domain] = @data.length + end + self.put_label(d[i]) + end + } + @data << "\0" + end + + def put_label(d) + self.put_string(d.to_s) + end + end + + def Message.decode(m) + o = Message.new(0) + MessageDecoder.new(m) {|msg| + id, flag, qdcount, ancount, nscount, arcount = + msg.get_unpack('nnnnnn') + o.id = id + o.tc = (flag >> 9) & 1 + o.rcode = flag & 15 + return o unless o.tc.zero? + + o.qr = (flag >> 15) & 1 + o.opcode = (flag >> 11) & 15 + o.aa = (flag >> 10) & 1 + o.rd = (flag >> 8) & 1 + o.ra = (flag >> 7) & 1 + (1..qdcount).each { + name, typeclass = msg.get_question + o.add_question(name, typeclass) + } + (1..ancount).each { + name, ttl, data = msg.get_rr + o.add_answer(name, ttl, data) + } + (1..nscount).each { + name, ttl, data = msg.get_rr + o.add_authority(name, ttl, data) + } + (1..arcount).each { + name, ttl, data = msg.get_rr + o.add_additional(name, ttl, data) + } + } + return o + end + + class MessageDecoder # :nodoc: + def initialize(data) + @data = data + @index = 0 + @limit = data.bytesize + yield self + end + + def inspect + "\#<#{self.class}: #{@data.byteslice(0, @index).inspect} #{@data.byteslice(@index..-1).inspect}>" + end + + def get_length16 + len, = self.get_unpack('n') + save_limit = @limit + @limit = @index + len + d = yield(len) + if @index < @limit + raise DecodeError.new("junk exists") + elsif @limit < @index + raise DecodeError.new("limit exceeded") + end + @limit = save_limit + return d + end + + def get_bytes(len = @limit - @index) + raise DecodeError.new("limit exceeded") if @limit < @index + len + d = @data.byteslice(@index, len) + @index += len + return d + end + + def get_unpack(template) + len = 0 + template.each_byte {|byte| + byte = "%c" % byte + case byte + when ?c, ?C + len += 1 + when ?n + len += 2 + when ?N + len += 4 + else + raise StandardError.new("unsupported template: '#{byte.chr}' in '#{template}'") + end + } + raise DecodeError.new("limit exceeded") if @limit < @index + len + arr = @data.unpack("@#{@index}#{template}") + @index += len + return arr + end + + def get_string + raise DecodeError.new("limit exceeded") if @limit <= @index + len = @data.getbyte(@index) + raise DecodeError.new("limit exceeded") if @limit < @index + 1 + len + d = @data.byteslice(@index + 1, len) + @index += 1 + len + return d + end + + def get_string_list + strings = [] + while @index < @limit + strings << self.get_string + end + strings + end + + def get_list + [].tap do |values| + while @index < @limit + values << yield + end + end + end + + def get_name + return Name.new(self.get_labels) + end + + def get_labels + prev_index = @index + save_index = nil + d = [] + while true + raise DecodeError.new("limit exceeded") if @limit <= @index + case @data.getbyte(@index) + when 0 + @index += 1 + if save_index + @index = save_index + end + return d + when 192..255 + idx = self.get_unpack('n')[0] & 0x3fff + if prev_index <= idx + raise DecodeError.new("non-backward name pointer") + end + prev_index = idx + if !save_index + save_index = @index + end + @index = idx + else + d << self.get_label + end + end + end + + def get_label + return Label::Str.new(self.get_string) + end + + def get_question + name = self.get_name + type, klass = self.get_unpack("nn") + return name, Resource.get_class(type, klass) + end + + def get_rr + name = self.get_name + type, klass, ttl = self.get_unpack('nnN') + typeclass = Resource.get_class(type, klass) + res = self.get_length16 do + begin + typeclass.decode_rdata self + rescue => e + raise DecodeError, e.message, e.backtrace + end + end + res.instance_variable_set :@ttl, ttl + return name, ttl, res + end + end + end + + ## + # SvcParams for service binding RRs. [RFC9460] + + class SvcParams + include Enumerable + + ## + # Create a list of SvcParams with the given initial content. + # + # +params+ has to be an enumerable of +SvcParam+s. + # If its content has +SvcParam+s with the duplicate key, + # the one appears last takes precedence. + + def initialize(params = []) + @params = {} + + params.each do |param| + add param + end + end + + ## + # Get SvcParam for the given +key+ in this list. + + def [](key) + @params[canonical_key(key)] + end + + ## + # Get the number of SvcParams in this list. + + def count + @params.count + end + + ## + # Get whether this list is empty. + + def empty? + @params.empty? + end + + ## + # Add the SvcParam +param+ to this list, overwriting the existing one with the same key. + + def add(param) + @params[param.class.key_number] = param + end + + ## + # Remove the +SvcParam+ with the given +key+ and return it. + + def delete(key) + @params.delete(canonical_key(key)) + end + + ## + # Enumerate the +SvcParam+s in this list. + + def each(&block) + return enum_for(:each) unless block + @params.each_value(&block) + end + + def encode(msg) # :nodoc: + @params.keys.sort.each do |key| + msg.put_pack('n', key) + msg.put_length16 do + @params.fetch(key).encode(msg) + end + end + end + + def self.decode(msg) # :nodoc: + params = msg.get_list do + key, = msg.get_unpack('n') + msg.get_length16 do + SvcParam::ClassHash[key].decode(msg) + end + end + + return self.new(params) + end + + private + + def canonical_key(key) # :nodoc: + case key + when Integer + key + when /\Akey(\d+)\z/ + Integer($1) + when Symbol + SvcParam::ClassHash[key].key_number + else + raise TypeError, 'key must be either String or Symbol' + end + end + end + + + ## + # Base class for SvcParam. [RFC9460] + + class SvcParam + + ## + # Get the presentation name of the SvcParamKey. + + def self.key_name + const_get(:KeyName) + end + + ## + # Get the registered number of the SvcParamKey. + + def self.key_number + const_get(:KeyNumber) + end + + ClassHash = Hash.new do |h, key| # :nodoc: + case key + when Integer + Generic.create(key) + when /\Akey(?<key>\d+)\z/ + Generic.create(key.to_int) + when Symbol + raise KeyError, "unknown key #{key}" + else + raise TypeError, 'key must be either String or Symbol' + end + end + + ## + # Generic SvcParam abstract class. + + class Generic < SvcParam + + ## + # SvcParamValue in wire-format byte string. + + attr_reader :value + + ## + # Create generic SvcParam + + def initialize(value) + @value = value + end + + def encode(msg) # :nodoc: + msg.put_bytes(@value) + end + + def self.decode(msg) # :nodoc: + return self.new(msg.get_bytes) + end + + def self.create(key_number) + c = Class.new(Generic) + key_name = :"key#{key_number}" + c.const_set(:KeyName, key_name) + c.const_set(:KeyNumber, key_number) + self.const_set(:"Key#{key_number}", c) + ClassHash[key_name] = ClassHash[key_number] = c + return c + end + end + + ## + # "mandatory" SvcParam -- Mandatory keys in service binding RR + + class Mandatory < SvcParam + KeyName = :mandatory + KeyNumber = 0 + ClassHash[KeyName] = ClassHash[KeyNumber] = self # :nodoc: + + ## + # Mandatory keys. + + attr_reader :keys + + ## + # Initialize "mandatory" ScvParam. + + def initialize(keys) + @keys = keys.map(&:to_int) + end + + def encode(msg) # :nodoc: + @keys.sort.each do |key| + msg.put_pack('n', key) + end + end + + def self.decode(msg) # :nodoc: + keys = msg.get_list { msg.get_unpack('n')[0] } + return self.new(keys) + end + end + + ## + # "alpn" SvcParam -- Additional supported protocols + + class ALPN < SvcParam + KeyName = :alpn + KeyNumber = 1 + ClassHash[KeyName] = ClassHash[KeyNumber] = self # :nodoc: + + ## + # Supported protocol IDs. + + attr_reader :protocol_ids + + ## + # Initialize "alpn" ScvParam. + + def initialize(protocol_ids) + @protocol_ids = protocol_ids.map(&:to_str) + end + + def encode(msg) # :nodoc: + msg.put_string_list(@protocol_ids) + end + + def self.decode(msg) # :nodoc: + return self.new(msg.get_string_list) + end + end + + ## + # "no-default-alpn" SvcParam -- No support for default protocol + + class NoDefaultALPN < SvcParam + KeyName = :'no-default-alpn' + KeyNumber = 2 + ClassHash[KeyName] = ClassHash[KeyNumber] = self # :nodoc: + + def encode(msg) # :nodoc: + # no payload + end + + def self.decode(msg) # :nodoc: + return self.new + end + end + + ## + # "port" SvcParam -- Port for alternative endpoint + + class Port < SvcParam + KeyName = :port + KeyNumber = 3 + ClassHash[KeyName] = ClassHash[KeyNumber] = self # :nodoc: + + ## + # Port number. + + attr_reader :port + + ## + # Initialize "port" ScvParam. + + def initialize(port) + @port = port.to_int + end + + def encode(msg) # :nodoc: + msg.put_pack('n', @port) + end + + def self.decode(msg) # :nodoc: + port, = msg.get_unpack('n') + return self.new(port) + end + end + + ## + # "ipv4hint" SvcParam -- IPv4 address hints + + class IPv4Hint < SvcParam + KeyName = :ipv4hint + KeyNumber = 4 + ClassHash[KeyName] = ClassHash[KeyNumber] = self # :nodoc: + + ## + # Set of IPv4 addresses. + + attr_reader :addresses + + ## + # Initialize "ipv4hint" ScvParam. + + def initialize(addresses) + @addresses = addresses.map {|address| IPv4.create(address) } + end + + def encode(msg) # :nodoc: + @addresses.each do |address| + msg.put_bytes(address.address) + end + end + + def self.decode(msg) # :nodoc: + addresses = msg.get_list { IPv4.new(msg.get_bytes(4)) } + return self.new(addresses) + end + end + + ## + # "ipv6hint" SvcParam -- IPv6 address hints + + class IPv6Hint < SvcParam + KeyName = :ipv6hint + KeyNumber = 6 + ClassHash[KeyName] = ClassHash[KeyNumber] = self # :nodoc: + + ## + # Set of IPv6 addresses. + + attr_reader :addresses + + ## + # Initialize "ipv6hint" ScvParam. + + def initialize(addresses) + @addresses = addresses.map {|address| IPv6.create(address) } + end + + def encode(msg) # :nodoc: + @addresses.each do |address| + msg.put_bytes(address.address) + end + end + + def self.decode(msg) # :nodoc: + addresses = msg.get_list { IPv6.new(msg.get_bytes(16)) } + return self.new(addresses) + end + end + + ## + # "dohpath" SvcParam -- DNS over HTTPS path template [RFC9461] + + class DoHPath < SvcParam + KeyName = :dohpath + KeyNumber = 7 + ClassHash[KeyName] = ClassHash[KeyNumber] = self # :nodoc: + + ## + # URI template for DoH queries. + + attr_reader :template + + ## + # Initialize "dohpath" ScvParam. + + def initialize(template) + @template = template.encode('utf-8') + end + + def encode(msg) # :nodoc: + msg.put_bytes(@template) + end + + def self.decode(msg) # :nodoc: + template = msg.get_bytes.force_encoding('utf-8') + return self.new(template) + end + end + end + + ## + # A DNS query abstract class. + + class Query + def encode_rdata(msg) # :nodoc: + raise EncodeError.new("#{self.class} is query.") + end + + def self.decode_rdata(msg) # :nodoc: + raise DecodeError.new("#{self.class} is query.") + end + end + + ## + # A DNS resource abstract class. + + class Resource < Query + + ## + # Remaining Time To Live for this Resource. + + attr_reader :ttl + + ClassHash = {} # :nodoc: + + def encode_rdata(msg) # :nodoc: + raise NotImplementedError.new + end + + def self.decode_rdata(msg) # :nodoc: + raise NotImplementedError.new + end + + def ==(other) # :nodoc: + return false unless self.class == other.class + s_ivars = self.instance_variables + s_ivars.sort! + s_ivars.delete :@ttl + o_ivars = other.instance_variables + o_ivars.sort! + o_ivars.delete :@ttl + return s_ivars == o_ivars && + s_ivars.collect {|name| self.instance_variable_get name} == + o_ivars.collect {|name| other.instance_variable_get name} + end + + def eql?(other) # :nodoc: + return self == other + end + + def hash # :nodoc: + h = 0 + vars = self.instance_variables + vars.delete :@ttl + vars.each {|name| + h ^= self.instance_variable_get(name).hash + } + return h + end + + def self.get_class(type_value, class_value) # :nodoc: + return ClassHash[[type_value, class_value]] || + Generic.create(type_value, class_value) + end + + ## + # A generic resource abstract class. + + class Generic < Resource + + ## + # Creates a new generic resource. + + def initialize(data) + @data = data + end + + ## + # Data for this generic resource. + + attr_reader :data + + def encode_rdata(msg) # :nodoc: + msg.put_bytes(data) + end + + def self.decode_rdata(msg) # :nodoc: + return self.new(msg.get_bytes) + end + + def self.create(type_value, class_value) # :nodoc: + c = Class.new(Generic) + c.const_set(:TypeValue, type_value) + c.const_set(:ClassValue, class_value) + Generic.const_set("Type#{type_value}_Class#{class_value}", c) + ClassHash[[type_value, class_value]] = c + return c + end + end + + ## + # Domain Name resource abstract class. + + class DomainName < Resource + + ## + # Creates a new DomainName from +name+. + + def initialize(name) + @name = name + end + + ## + # The name of this DomainName. + + attr_reader :name + + def encode_rdata(msg) # :nodoc: + msg.put_name(@name) + end + + def self.decode_rdata(msg) # :nodoc: + return self.new(msg.get_name) + end + end + + # Standard (class generic) RRs + + ClassValue = nil # :nodoc: + + ## + # An authoritative name server. + + class NS < DomainName + TypeValue = 2 # :nodoc: + end + + ## + # The canonical name for an alias. + + class CNAME < DomainName + TypeValue = 5 # :nodoc: + end + + ## + # Start Of Authority resource. + + class SOA < Resource + + TypeValue = 6 # :nodoc: + + ## + # Creates a new SOA record. See the attr documentation for the + # details of each argument. + + def initialize(mname, rname, serial, refresh, retry_, expire, minimum) + @mname = mname + @rname = rname + @serial = serial + @refresh = refresh + @retry = retry_ + @expire = expire + @minimum = minimum + end + + ## + # Name of the host where the master zone file for this zone resides. + + attr_reader :mname + + ## + # The person responsible for this domain name. + + attr_reader :rname + + ## + # The version number of the zone file. + + attr_reader :serial + + ## + # How often, in seconds, a secondary name server is to check for + # updates from the primary name server. + + attr_reader :refresh + + ## + # How often, in seconds, a secondary name server is to retry after a + # failure to check for a refresh. + + attr_reader :retry + + ## + # Time in seconds that a secondary name server is to use the data + # before refreshing from the primary name server. + + attr_reader :expire + + ## + # The minimum number of seconds to be used for TTL values in RRs. + + attr_reader :minimum + + def encode_rdata(msg) # :nodoc: + msg.put_name(@mname) + msg.put_name(@rname) + msg.put_pack('NNNNN', @serial, @refresh, @retry, @expire, @minimum) + end + + def self.decode_rdata(msg) # :nodoc: + mname = msg.get_name + rname = msg.get_name + serial, refresh, retry_, expire, minimum = msg.get_unpack('NNNNN') + return self.new( + mname, rname, serial, refresh, retry_, expire, minimum) + end + end + + ## + # A Pointer to another DNS name. + + class PTR < DomainName + TypeValue = 12 # :nodoc: + end + + ## + # Host Information resource. + + class HINFO < Resource + + TypeValue = 13 # :nodoc: + + ## + # Creates a new HINFO running +os+ on +cpu+. + + def initialize(cpu, os) + @cpu = cpu + @os = os + end + + ## + # CPU architecture for this resource. + + attr_reader :cpu + + ## + # Operating system for this resource. + + attr_reader :os + + def encode_rdata(msg) # :nodoc: + msg.put_string(@cpu) + msg.put_string(@os) + end + + def self.decode_rdata(msg) # :nodoc: + cpu = msg.get_string + os = msg.get_string + return self.new(cpu, os) + end + end + + ## + # Mailing list or mailbox information. + + class MINFO < Resource + + TypeValue = 14 # :nodoc: + + def initialize(rmailbx, emailbx) + @rmailbx = rmailbx + @emailbx = emailbx + end + + ## + # Domain name responsible for this mail list or mailbox. + + attr_reader :rmailbx + + ## + # Mailbox to use for error messages related to the mail list or mailbox. + + attr_reader :emailbx + + def encode_rdata(msg) # :nodoc: + msg.put_name(@rmailbx) + msg.put_name(@emailbx) + end + + def self.decode_rdata(msg) # :nodoc: + rmailbx = msg.get_string + emailbx = msg.get_string + return self.new(rmailbx, emailbx) + end + end + + ## + # Mail Exchanger resource. + + class MX < Resource + + TypeValue= 15 # :nodoc: + + ## + # Creates a new MX record with +preference+, accepting mail at + # +exchange+. + + def initialize(preference, exchange) + @preference = preference + @exchange = exchange + end + + ## + # The preference for this MX. + + attr_reader :preference + + ## + # The host of this MX. + + attr_reader :exchange + + def encode_rdata(msg) # :nodoc: + msg.put_pack('n', @preference) + msg.put_name(@exchange) + end + + def self.decode_rdata(msg) # :nodoc: + preference, = msg.get_unpack('n') + exchange = msg.get_name + return self.new(preference, exchange) + end + end + + ## + # Unstructured text resource. + + class TXT < Resource + + TypeValue = 16 # :nodoc: + + def initialize(first_string, *rest_strings) + @strings = [first_string, *rest_strings] + end + + ## + # Returns an Array of Strings for this TXT record. + + attr_reader :strings + + ## + # Returns the concatenated string from +strings+. + + def data + @strings.join("") + end + + def encode_rdata(msg) # :nodoc: + msg.put_string_list(@strings) + end + + def self.decode_rdata(msg) # :nodoc: + strings = msg.get_string_list + return self.new(*strings) + end + end + + ## + # Location resource + + class LOC < Resource + + TypeValue = 29 # :nodoc: + + def initialize(version, ssize, hprecision, vprecision, latitude, longitude, altitude) + @version = version + @ssize = Gem::Resolv::LOC::Size.create(ssize) + @hprecision = Gem::Resolv::LOC::Size.create(hprecision) + @vprecision = Gem::Resolv::LOC::Size.create(vprecision) + @latitude = Gem::Resolv::LOC::Coord.create(latitude) + @longitude = Gem::Resolv::LOC::Coord.create(longitude) + @altitude = Gem::Resolv::LOC::Alt.create(altitude) + end + + ## + # Returns the version value for this LOC record which should always be 00 + + attr_reader :version + + ## + # The spherical size of this LOC + # in meters using scientific notation as 2 integers of XeY + + attr_reader :ssize + + ## + # The horizontal precision using ssize type values + # in meters using scientific notation as 2 integers of XeY + # for precision use value/2 e.g. 2m = +/-1m + + attr_reader :hprecision + + ## + # The vertical precision using ssize type values + # in meters using scientific notation as 2 integers of XeY + # for precision use value/2 e.g. 2m = +/-1m + + attr_reader :vprecision + + ## + # The latitude for this LOC where 2**31 is the equator + # in thousandths of an arc second as an unsigned 32bit integer + + attr_reader :latitude + + ## + # The longitude for this LOC where 2**31 is the prime meridian + # in thousandths of an arc second as an unsigned 32bit integer + + attr_reader :longitude + + ## + # The altitude of the LOC above a reference sphere whose surface sits 100km below the WGS84 spheroid + # in centimeters as an unsigned 32bit integer + + attr_reader :altitude + + + def encode_rdata(msg) # :nodoc: + msg.put_bytes(@version) + msg.put_bytes(@ssize.scalar) + msg.put_bytes(@hprecision.scalar) + msg.put_bytes(@vprecision.scalar) + msg.put_bytes(@latitude.coordinates) + msg.put_bytes(@longitude.coordinates) + msg.put_bytes(@altitude.altitude) + end + + def self.decode_rdata(msg) # :nodoc: + version = msg.get_bytes(1) + ssize = msg.get_bytes(1) + hprecision = msg.get_bytes(1) + vprecision = msg.get_bytes(1) + latitude = msg.get_bytes(4) + longitude = msg.get_bytes(4) + altitude = msg.get_bytes(4) + return self.new( + version, + Gem::Resolv::LOC::Size.new(ssize), + Gem::Resolv::LOC::Size.new(hprecision), + Gem::Resolv::LOC::Size.new(vprecision), + Gem::Resolv::LOC::Coord.new(latitude,"lat"), + Gem::Resolv::LOC::Coord.new(longitude,"lon"), + Gem::Resolv::LOC::Alt.new(altitude) + ) + end + end + + ## + # A Query type requesting any RR. + + class ANY < Query + TypeValue = 255 # :nodoc: + end + + ## + # CAA resource record defined in RFC 8659 + # + # These records identify certificate authority allowed to issue + # certificates for the given domain. + + class CAA < Resource + TypeValue = 257 + + ## + # Creates a new CAA for +flags+, +tag+ and +value+. + + def initialize(flags, tag, value) + unless (0..255) === flags + raise ArgumentError.new('flags must be an Integer between 0 and 255') + end + unless (1..15) === tag.bytesize + raise ArgumentError.new('length of tag must be between 1 and 15') + end + + @flags = flags + @tag = tag + @value = value + end + + ## + # Flags for this proprty: + # - Bit 0 : 0 = not critical, 1 = critical + + attr_reader :flags + + ## + # Property tag ("issue", "issuewild", "iodef"...). + + attr_reader :tag + + ## + # Property value. + + attr_reader :value + + ## + # Whether the critical flag is set on this property. + + def critical? + flags & 0x80 != 0 + end + + def encode_rdata(msg) # :nodoc: + msg.put_pack('C', @flags) + msg.put_string(@tag) + msg.put_bytes(@value) + end + + def self.decode_rdata(msg) # :nodoc: + flags, = msg.get_unpack('C') + tag = msg.get_string + value = msg.get_bytes + self.new flags, tag, value + end + end + + ClassInsensitiveTypes = [ # :nodoc: + NS, CNAME, SOA, PTR, HINFO, MINFO, MX, TXT, LOC, ANY, CAA + ] + + ## + # module IN contains ARPA Internet specific RRs. + + module IN + + ClassValue = 1 # :nodoc: + + ClassInsensitiveTypes.each {|s| + c = Class.new(s) + c.const_set(:TypeValue, s::TypeValue) + c.const_set(:ClassValue, ClassValue) + ClassHash[[s::TypeValue, ClassValue]] = c + self.const_set(s.name.sub(/.*::/, ''), c) + } + + ## + # IPv4 Address resource + + class A < Resource + TypeValue = 1 + ClassValue = IN::ClassValue + ClassHash[[TypeValue, ClassValue]] = self # :nodoc: + + ## + # Creates a new A for +address+. + + def initialize(address) + @address = IPv4.create(address) + end + + ## + # The Gem::Resolv::IPv4 address for this A. + + attr_reader :address + + def encode_rdata(msg) # :nodoc: + msg.put_bytes(@address.address) + end + + def self.decode_rdata(msg) # :nodoc: + return self.new(IPv4.new(msg.get_bytes(4))) + end + end + + ## + # Well Known Service resource. + + class WKS < Resource + TypeValue = 11 + ClassValue = IN::ClassValue + ClassHash[[TypeValue, ClassValue]] = self # :nodoc: + + def initialize(address, protocol, bitmap) + @address = IPv4.create(address) + @protocol = protocol + @bitmap = bitmap + end + + ## + # The host these services run on. + + attr_reader :address + + ## + # IP protocol number for these services. + + attr_reader :protocol + + ## + # A bit map of enabled services on this host. + # + # If protocol is 6 (TCP) then the 26th bit corresponds to the SMTP + # service (port 25). If this bit is set, then an SMTP server should + # be listening on TCP port 25; if zero, SMTP service is not + # supported. + + attr_reader :bitmap + + def encode_rdata(msg) # :nodoc: + msg.put_bytes(@address.address) + msg.put_pack("n", @protocol) + msg.put_bytes(@bitmap) + end + + def self.decode_rdata(msg) # :nodoc: + address = IPv4.new(msg.get_bytes(4)) + protocol, = msg.get_unpack("n") + bitmap = msg.get_bytes + return self.new(address, protocol, bitmap) + end + end + + ## + # An IPv6 address record. + + class AAAA < Resource + TypeValue = 28 + ClassValue = IN::ClassValue + ClassHash[[TypeValue, ClassValue]] = self # :nodoc: + + ## + # Creates a new AAAA for +address+. + + def initialize(address) + @address = IPv6.create(address) + end + + ## + # The Gem::Resolv::IPv6 address for this AAAA. + + attr_reader :address + + def encode_rdata(msg) # :nodoc: + msg.put_bytes(@address.address) + end + + def self.decode_rdata(msg) # :nodoc: + return self.new(IPv6.new(msg.get_bytes(16))) + end + end + + ## + # SRV resource record defined in RFC 2782 + # + # These records identify the hostname and port that a service is + # available at. + + class SRV < Resource + TypeValue = 33 + ClassValue = IN::ClassValue + ClassHash[[TypeValue, ClassValue]] = self # :nodoc: + + # Create a SRV resource record. + # + # See the documentation for #priority, #weight, #port and #target + # for +priority+, +weight+, +port and +target+ respectively. + + def initialize(priority, weight, port, target) + @priority = priority.to_int + @weight = weight.to_int + @port = port.to_int + @target = Name.create(target) + end + + # The priority of this target host. + # + # A client MUST attempt to contact the target host with the + # lowest-numbered priority it can reach; target hosts with the same + # priority SHOULD be tried in an order defined by the weight field. + # The range is 0-65535. Note that it is not widely implemented and + # should be set to zero. + + attr_reader :priority + + # A server selection mechanism. + # + # The weight field specifies a relative weight for entries with the + # same priority. Larger weights SHOULD be given a proportionately + # higher probability of being selected. The range of this number is + # 0-65535. Domain administrators SHOULD use Weight 0 when there + # isn't any server selection to do, to make the RR easier to read + # for humans (less noisy). Note that it is not widely implemented + # and should be set to zero. + + attr_reader :weight + + # The port on this target host of this service. + # + # The range is 0-65535. + + attr_reader :port + + # The domain name of the target host. + # + # A target of "." means that the service is decidedly not available + # at this domain. + + attr_reader :target + + def encode_rdata(msg) # :nodoc: + msg.put_pack("n", @priority) + msg.put_pack("n", @weight) + msg.put_pack("n", @port) + msg.put_name(@target, compress: false) + end + + def self.decode_rdata(msg) # :nodoc: + priority, = msg.get_unpack("n") + weight, = msg.get_unpack("n") + port, = msg.get_unpack("n") + target = msg.get_name + return self.new(priority, weight, port, target) + end + end + + ## + # Common implementation for SVCB-compatible resource records. + + class ServiceBinding + + ## + # Create a service binding resource record. + + def initialize(priority, target, params = []) + @priority = priority.to_int + @target = Name.create(target) + @params = SvcParams.new(params) + end + + ## + # The priority of this target host. + # + # The range is 0-65535. + # If set to 0, this RR is in AliasMode. Otherwise, it is in ServiceMode. + + attr_reader :priority + + ## + # The domain name of the target host. + + attr_reader :target + + ## + # The service parameters for the target host. + + attr_reader :params + + ## + # Whether this RR is in AliasMode. + + def alias_mode? + self.priority == 0 + end + + ## + # Whether this RR is in ServiceMode. + + def service_mode? + !alias_mode? + end + + def encode_rdata(msg) # :nodoc: + msg.put_pack("n", @priority) + msg.put_name(@target, compress: false) + @params.encode(msg) + end + + def self.decode_rdata(msg) # :nodoc: + priority, = msg.get_unpack("n") + target = msg.get_name + params = SvcParams.decode(msg) + return self.new(priority, target, params) + end + end + + ## + # SVCB resource record [RFC9460] + + class SVCB < ServiceBinding + TypeValue = 64 + ClassValue = IN::ClassValue + ClassHash[[TypeValue, ClassValue]] = self # :nodoc: + end + + ## + # HTTPS resource record [RFC9460] + + class HTTPS < ServiceBinding + TypeValue = 65 + ClassValue = IN::ClassValue + ClassHash[[TypeValue, ClassValue]] = self # :nodoc: + end + end + end + end + + ## + # A Gem::Resolv::DNS IPv4 address. + + class IPv4 + + ## + # Regular expression IPv4 addresses must match. + + Regex256 = /0 + |1(?:[0-9][0-9]?)? + |2(?:[0-4][0-9]?|5[0-5]?|[6-9])? + |[3-9][0-9]?/x + Regex = /\A(#{Regex256})\.(#{Regex256})\.(#{Regex256})\.(#{Regex256})\z/ + + def self.create(arg) + case arg + when IPv4 + return arg + when Regex + if (0..255) === (a = $1.to_i) && + (0..255) === (b = $2.to_i) && + (0..255) === (c = $3.to_i) && + (0..255) === (d = $4.to_i) + return self.new([a, b, c, d].pack("CCCC")) + else + raise ArgumentError.new("IPv4 address with invalid value: " + arg) + end + else + raise ArgumentError.new("cannot interpret as IPv4 address: #{arg.inspect}") + end + end + + def initialize(address) # :nodoc: + unless address.kind_of?(String) + raise ArgumentError, 'IPv4 address must be a string' + end + unless address.length == 4 + raise ArgumentError, "IPv4 address expects 4 bytes but #{address.length} bytes" + end + @address = address + end + + ## + # A String representation of this IPv4 address. + + ## + # The raw IPv4 address as a String. + + attr_reader :address + + def to_s # :nodoc: + return sprintf("%d.%d.%d.%d", *@address.unpack("CCCC")) + end + + def inspect # :nodoc: + return "#<#{self.class} #{self}>" + end + + ## + # Turns this IPv4 address into a Gem::Resolv::DNS::Name. + + def to_name + return DNS::Name.create( + '%d.%d.%d.%d.in-addr.arpa.' % @address.unpack('CCCC').reverse) + end + + def ==(other) # :nodoc: + return @address == other.address + end + + def eql?(other) # :nodoc: + return self == other + end + + def hash # :nodoc: + return @address.hash + end + end + + ## + # A Gem::Resolv::DNS IPv6 address. + + class IPv6 + + ## + # IPv6 address format a:b:c:d:e:f:g:h + Regex_8Hex = /\A + (?:[0-9A-Fa-f]{1,4}:){7} + [0-9A-Fa-f]{1,4} + \z/x + + ## + # Compressed IPv6 address format a::b + + Regex_CompressedHex = /\A + ((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?) :: + ((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?) + \z/x + + ## + # IPv4 mapped IPv6 address format a:b:c:d:e:f:w.x.y.z + + Regex_6Hex4Dec = /\A + ((?:[0-9A-Fa-f]{1,4}:){6,6}) + (\d+)\.(\d+)\.(\d+)\.(\d+) + \z/x + + ## + # Compressed IPv4 mapped IPv6 address format a::b:w.x.y.z + + Regex_CompressedHex4Dec = /\A + ((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?) :: + ((?:[0-9A-Fa-f]{1,4}:)*) + (\d+)\.(\d+)\.(\d+)\.(\d+) + \z/x + + ## + # IPv6 link local address format fe80:b:c:d:e:f:g:h%em1 + Regex_8HexLinkLocal = /\A + [Ff][Ee]80 + (?::[0-9A-Fa-f]{1,4}){7} + %[-0-9A-Za-z._~]+ + \z/x + + ## + # Compressed IPv6 link local address format fe80::b%em1 + + Regex_CompressedHexLinkLocal = /\A + [Ff][Ee]80: + (?: + ((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?) :: + ((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?) + | + :((?:[0-9A-Fa-f]{1,4}(?::[0-9A-Fa-f]{1,4})*)?) + )? + :[0-9A-Fa-f]{1,4}%[-0-9A-Za-z._~]+ + \z/x + + ## + # A composite IPv6 address Regexp. + + Regex = / + (?:#{Regex_8Hex}) | + (?:#{Regex_CompressedHex}) | + (?:#{Regex_6Hex4Dec}) | + (?:#{Regex_CompressedHex4Dec}) | + (?:#{Regex_8HexLinkLocal}) | + (?:#{Regex_CompressedHexLinkLocal}) + /x + + ## + # Creates a new IPv6 address from +arg+ which may be: + # + # IPv6:: returns +arg+. + # String:: +arg+ must match one of the IPv6::Regex* constants + + def self.create(arg) + case arg + when IPv6 + return arg + when String + address = ''.b + if Regex_8Hex =~ arg + arg.scan(/[0-9A-Fa-f]+/) {|hex| address << [hex.hex].pack('n')} + elsif Regex_CompressedHex =~ arg + prefix = $1 + suffix = $2 + a1 = ''.b + a2 = ''.b + prefix.scan(/[0-9A-Fa-f]+/) {|hex| a1 << [hex.hex].pack('n')} + suffix.scan(/[0-9A-Fa-f]+/) {|hex| a2 << [hex.hex].pack('n')} + omitlen = 16 - a1.length - a2.length + address << a1 << "\0" * omitlen << a2 + elsif Regex_6Hex4Dec =~ arg + prefix, a, b, c, d = $1, $2.to_i, $3.to_i, $4.to_i, $5.to_i + if (0..255) === a && (0..255) === b && (0..255) === c && (0..255) === d + prefix.scan(/[0-9A-Fa-f]+/) {|hex| address << [hex.hex].pack('n')} + address << [a, b, c, d].pack('CCCC') + else + raise ArgumentError.new("not numeric IPv6 address: " + arg) + end + elsif Regex_CompressedHex4Dec =~ arg + prefix, suffix, a, b, c, d = $1, $2, $3.to_i, $4.to_i, $5.to_i, $6.to_i + if (0..255) === a && (0..255) === b && (0..255) === c && (0..255) === d + a1 = ''.b + a2 = ''.b + prefix.scan(/[0-9A-Fa-f]+/) {|hex| a1 << [hex.hex].pack('n')} + suffix.scan(/[0-9A-Fa-f]+/) {|hex| a2 << [hex.hex].pack('n')} + omitlen = 12 - a1.length - a2.length + address << a1 << "\0" * omitlen << a2 << [a, b, c, d].pack('CCCC') + else + raise ArgumentError.new("not numeric IPv6 address: " + arg) + end + else + raise ArgumentError.new("not numeric IPv6 address: " + arg) + end + return IPv6.new(address) + else + raise ArgumentError.new("cannot interpret as IPv6 address: #{arg.inspect}") + end + end + + def initialize(address) # :nodoc: + unless address.kind_of?(String) && address.length == 16 + raise ArgumentError.new('IPv6 address must be 16 bytes') + end + @address = address + end + + ## + # The raw IPv6 address as a String. + + attr_reader :address + + def to_s # :nodoc: + sprintf("%x:%x:%x:%x:%x:%x:%x:%x", *@address.unpack("nnnnnnnn")).sub(/(^|:)0(:0)+(:|$)/, '::') + end + + def inspect # :nodoc: + return "#<#{self.class} #{self}>" + end + + ## + # Turns this IPv6 address into a Gem::Resolv::DNS::Name. + #-- + # ip6.arpa should be searched too. [RFC3152] + + def to_name + return DNS::Name.new( + @address.unpack("H32")[0].split(//).reverse + ['ip6', 'arpa']) + end + + def ==(other) # :nodoc: + return @address == other.address + end + + def eql?(other) # :nodoc: + return self == other + end + + def hash # :nodoc: + return @address.hash + end + end + + ## + # Gem::Resolv::MDNS is a one-shot Multicast DNS (mDNS) resolver. It blindly + # makes queries to the mDNS addresses without understanding anything about + # multicast ports. + # + # Information taken form the following places: + # + # * RFC 6762 + + class MDNS < DNS + + ## + # Default mDNS Port + + Port = 5353 + + ## + # Default IPv4 mDNS address + + AddressV4 = '224.0.0.251' + + ## + # Default IPv6 mDNS address + + AddressV6 = 'ff02::fb' + + ## + # Default mDNS addresses + + Addresses = [ + [AddressV4, Port], + [AddressV6, Port], + ] + + ## + # Creates a new one-shot Multicast DNS (mDNS) resolver. + # + # +config_info+ can be: + # + # nil:: + # Uses the default mDNS addresses + # + # Hash:: + # Must contain :nameserver or :nameserver_port like + # Gem::Resolv::DNS#initialize. + + def initialize(config_info=nil) + if config_info then + super({ nameserver_port: Addresses }.merge(config_info)) + else + super(nameserver_port: Addresses) + end + end + + ## + # Iterates over all IP addresses for +name+ retrieved from the mDNS + # resolver, provided name ends with "local". If the name does not end in + # "local" no records will be returned. + # + # +name+ can be a Gem::Resolv::DNS::Name or a String. Retrieved addresses will + # be a Gem::Resolv::IPv4 or Gem::Resolv::IPv6 + + def each_address(name) + name = Gem::Resolv::DNS::Name.create(name) + + return unless name[-1].to_s == 'local' + + super(name) + end + + def make_udp_requester # :nodoc: + nameserver_port = @config.nameserver_port + Requester::MDNSOneShot.new(*nameserver_port) + end + + end + + module LOC + + ## + # A Gem::Resolv::LOC::Size + + class Size + + Regex = /^(\d+\.*\d*)[m]$/ + + ## + # Creates a new LOC::Size from +arg+ which may be: + # + # LOC::Size:: returns +arg+. + # String:: +arg+ must match the LOC::Size::Regex constant + + def self.create(arg) + case arg + when Size + return arg + when String + scalar = '' + if Regex =~ arg + scalar = [(($1.to_f*(1e2)).to_i.to_s[0].to_i*(2**4)+(($1.to_f*(1e2)).to_i.to_s.length-1))].pack("C") + else + raise ArgumentError.new("not a properly formed Size string: " + arg) + end + return Size.new(scalar) + else + raise ArgumentError.new("cannot interpret as Size: #{arg.inspect}") + end + end + + def initialize(scalar) + @scalar = scalar + end + + ## + # The raw size + + attr_reader :scalar + + def to_s # :nodoc: + s = @scalar.unpack("H2").join.to_s + return ((s[0].to_i)*(10**(s[1].to_i-2))).to_s << "m" + end + + def inspect # :nodoc: + return "#<#{self.class} #{self}>" + end + + def ==(other) # :nodoc: + return @scalar == other.scalar + end + + def eql?(other) # :nodoc: + return self == other + end + + def hash # :nodoc: + return @scalar.hash + end + + end + + ## + # A Gem::Resolv::LOC::Coord + + class Coord + + Regex = /^(\d+)\s(\d+)\s(\d+\.\d+)\s([NESW])$/ + + ## + # Creates a new LOC::Coord from +arg+ which may be: + # + # LOC::Coord:: returns +arg+. + # String:: +arg+ must match the LOC::Coord::Regex constant + + def self.create(arg) + case arg + when Coord + return arg + when String + coordinates = '' + if Regex =~ arg && $1.to_f < 180 + m = $~ + hemi = (m[4][/[NE]/]) || (m[4][/[SW]/]) ? 1 : -1 + coordinates = [ ((m[1].to_i*(36e5)) + (m[2].to_i*(6e4)) + + (m[3].to_f*(1e3))) * hemi+(2**31) ].pack("N") + orientation = m[4][/[NS]/] ? 'lat' : 'lon' + else + raise ArgumentError.new("not a properly formed Coord string: " + arg) + end + return Coord.new(coordinates,orientation) + else + raise ArgumentError.new("cannot interpret as Coord: #{arg.inspect}") + end + end + + def initialize(coordinates,orientation) + unless coordinates.kind_of?(String) + raise ArgumentError.new("Coord must be a 32bit unsigned integer in hex format: #{coordinates.inspect}") + end + unless orientation.kind_of?(String) && orientation[/^lon$|^lat$/] + raise ArgumentError.new('Coord expects orientation to be a String argument of "lat" or "lon"') + end + @coordinates = coordinates + @orientation = orientation + end + + ## + # The raw coordinates + + attr_reader :coordinates + + ## The orientation of the hemisphere as 'lat' or 'lon' + + attr_reader :orientation + + def to_s # :nodoc: + c = @coordinates.unpack("N").join.to_i + val = (c - (2**31)).abs + fracsecs = (val % 1e3).to_i.to_s + val = val / 1e3 + secs = (val % 60).to_i.to_s + val = val / 60 + mins = (val % 60).to_i.to_s + degs = (val / 60).to_i.to_s + posi = (c >= 2**31) + case posi + when true + hemi = @orientation[/^lat$/] ? "N" : "E" + else + hemi = @orientation[/^lon$/] ? "W" : "S" + end + return degs << " " << mins << " " << secs << "." << fracsecs << " " << hemi + end + + def inspect # :nodoc: + return "#<#{self.class} #{self}>" + end + + def ==(other) # :nodoc: + return @coordinates == other.coordinates + end + + def eql?(other) # :nodoc: + return self == other + end + + def hash # :nodoc: + return @coordinates.hash + end + + end + + ## + # A Gem::Resolv::LOC::Alt + + class Alt + + Regex = /^([+-]*\d+\.*\d*)[m]$/ + + ## + # Creates a new LOC::Alt from +arg+ which may be: + # + # LOC::Alt:: returns +arg+. + # String:: +arg+ must match the LOC::Alt::Regex constant + + def self.create(arg) + case arg + when Alt + return arg + when String + altitude = '' + if Regex =~ arg + altitude = [($1.to_f*(1e2))+(1e7)].pack("N") + else + raise ArgumentError.new("not a properly formed Alt string: " + arg) + end + return Alt.new(altitude) + else + raise ArgumentError.new("cannot interpret as Alt: #{arg.inspect}") + end + end + + def initialize(altitude) + @altitude = altitude + end + + ## + # The raw altitude + + attr_reader :altitude + + def to_s # :nodoc: + a = @altitude.unpack("N").join.to_i + return ((a.to_f/1e2)-1e5).to_s + "m" + end + + def inspect # :nodoc: + return "#<#{self.class} #{self}>" + end + + def ==(other) # :nodoc: + return @altitude == other.altitude + end + + def eql?(other) # :nodoc: + return self == other + end + + def hash # :nodoc: + return @altitude.hash + end + + end + + end + + ## + # Default resolver to use for Gem::Resolv class methods. + + DefaultResolver = self.new + + ## + # Replaces the resolvers in the default resolver with +new_resolvers+. This + # allows resolvers to be changed for resolv-replace. + + def DefaultResolver.replace_resolvers new_resolvers + @resolvers = new_resolvers + end + + ## + # Address Regexp to use for matching IP addresses. + + AddressRegex = /(?:#{IPv4::Regex})|(?:#{IPv6::Regex})/ + +end + diff --git a/lib/rubygems/vendor/timeout/.document b/lib/rubygems/vendor/timeout/.document new file mode 100644 index 0000000000..0c43bbd6b3 --- /dev/null +++ b/lib/rubygems/vendor/timeout/.document @@ -0,0 +1 @@ +# Vendored files do not need to be documented diff --git a/lib/rubygems/vendor/timeout/lib/timeout.rb b/lib/rubygems/vendor/timeout/lib/timeout.rb new file mode 100644 index 0000000000..df97d64ca0 --- /dev/null +++ b/lib/rubygems/vendor/timeout/lib/timeout.rb @@ -0,0 +1,199 @@ +# frozen_string_literal: true +# Timeout long-running blocks +# +# == Synopsis +# +# require 'rubygems/vendor/timeout/lib/timeout' +# status = Gem::Timeout::timeout(5) { +# # Something that should be interrupted if it takes more than 5 seconds... +# } +# +# == Description +# +# Gem::Timeout provides a way to auto-terminate a potentially long-running +# operation if it hasn't finished in a fixed amount of time. +# +# Previous versions didn't use a module for namespacing, however +# #timeout is provided for backwards compatibility. You +# should prefer Gem::Timeout.timeout instead. +# +# == Copyright +# +# Copyright:: (C) 2000 Network Applied Communication Laboratory, Inc. +# Copyright:: (C) 2000 Information-technology Promotion Agency, Japan + +module Gem::Timeout + VERSION = "0.4.1" + + # Internal error raised to when a timeout is triggered. + class ExitException < Exception + def exception(*) + self + end + end + + # Raised by Gem::Timeout.timeout when the block times out. + class Error < RuntimeError + def self.handle_timeout(message) + exc = ExitException.new(message) + + begin + yield exc + rescue ExitException => e + raise new(message) if exc.equal?(e) + raise + end + end + end + + # :stopdoc: + CONDVAR = ConditionVariable.new + QUEUE = Queue.new + QUEUE_MUTEX = Mutex.new + TIMEOUT_THREAD_MUTEX = Mutex.new + @timeout_thread = nil + private_constant :CONDVAR, :QUEUE, :QUEUE_MUTEX, :TIMEOUT_THREAD_MUTEX + + class Request + attr_reader :deadline + + def initialize(thread, timeout, exception_class, message) + @thread = thread + @deadline = GET_TIME.call(Process::CLOCK_MONOTONIC) + timeout + @exception_class = exception_class + @message = message + + @mutex = Mutex.new + @done = false # protected by @mutex + end + + def done? + @mutex.synchronize do + @done + end + end + + def expired?(now) + now >= @deadline + end + + def interrupt + @mutex.synchronize do + unless @done + @thread.raise @exception_class, @message + @done = true + end + end + end + + def finished + @mutex.synchronize do + @done = true + end + end + end + private_constant :Request + + def self.create_timeout_thread + watcher = Thread.new do + requests = [] + while true + until QUEUE.empty? and !requests.empty? # wait to have at least one request + req = QUEUE.pop + requests << req unless req.done? + end + closest_deadline = requests.min_by(&:deadline).deadline + + now = 0.0 + QUEUE_MUTEX.synchronize do + while (now = GET_TIME.call(Process::CLOCK_MONOTONIC)) < closest_deadline and QUEUE.empty? + CONDVAR.wait(QUEUE_MUTEX, closest_deadline - now) + end + end + + requests.each do |req| + req.interrupt if req.expired?(now) + end + requests.reject!(&:done?) + end + end + ThreadGroup::Default.add(watcher) unless watcher.group.enclosed? + watcher.name = "Gem::Timeout stdlib thread" + watcher.thread_variable_set(:"\0__detached_thread__", true) + watcher + end + private_class_method :create_timeout_thread + + def self.ensure_timeout_thread_created + unless @timeout_thread and @timeout_thread.alive? + TIMEOUT_THREAD_MUTEX.synchronize do + unless @timeout_thread and @timeout_thread.alive? + @timeout_thread = create_timeout_thread + end + end + end + end + + # We keep a private reference so that time mocking libraries won't break + # Gem::Timeout. + GET_TIME = Process.method(:clock_gettime) + private_constant :GET_TIME + + # :startdoc: + + # Perform an operation in a block, raising an error if it takes longer than + # +sec+ seconds to complete. + # + # +sec+:: Number of seconds to wait for the block to terminate. Any number + # may be used, including Floats to specify fractional seconds. A + # value of 0 or +nil+ will execute the block without any timeout. + # +klass+:: Exception Class to raise if the block fails to terminate + # in +sec+ seconds. Omitting will use the default, Gem::Timeout::Error + # +message+:: Error message to raise with Exception Class. + # Omitting will use the default, "execution expired" + # + # Returns the result of the block *if* the block completed before + # +sec+ seconds, otherwise throws an exception, based on the value of +klass+. + # + # The exception thrown to terminate the given block cannot be rescued inside + # the block unless +klass+ is given explicitly. However, the block can use + # ensure to prevent the handling of the exception. For that reason, this + # method cannot be relied on to enforce timeouts for untrusted blocks. + # + # If a scheduler is defined, it will be used to handle the timeout by invoking + # Scheduler#timeout_after. + # + # Note that this is both a method of module Gem::Timeout, so you can <tt>include + # Gem::Timeout</tt> into your classes so they have a #timeout method, as well as + # a module method, so you can call it directly as Gem::Timeout.timeout(). + def timeout(sec, klass = nil, message = nil, &block) #:yield: +sec+ + return yield(sec) if sec == nil or sec.zero? + + message ||= "execution expired" + + if Fiber.respond_to?(:current_scheduler) && (scheduler = Fiber.current_scheduler)&.respond_to?(:timeout_after) + return scheduler.timeout_after(sec, klass || Error, message, &block) + end + + Gem::Timeout.ensure_timeout_thread_created + perform = Proc.new do |exc| + request = Request.new(Thread.current, sec, exc, message) + QUEUE_MUTEX.synchronize do + QUEUE << request + CONDVAR.signal + end + begin + return yield(sec) + ensure + request.finished + end + end + + if klass + perform.call(klass) + else + Error.handle_timeout(message, &perform) + end + end + module_function :timeout +end diff --git a/lib/rubygems/vendor/tsort/.document b/lib/rubygems/vendor/tsort/.document new file mode 100644 index 0000000000..0c43bbd6b3 --- /dev/null +++ b/lib/rubygems/vendor/tsort/.document @@ -0,0 +1 @@ +# Vendored files do not need to be documented diff --git a/lib/rubygems/tsort/lib/tsort.rb b/lib/rubygems/vendor/tsort/lib/tsort.rb index f825f14257..9dd7c09521 100644 --- a/lib/rubygems/tsort/lib/tsort.rb +++ b/lib/rubygems/vendor/tsort/lib/tsort.rb @@ -32,7 +32,7 @@ # method, which fetches the array of child nodes and then iterates over that # array using the user-supplied block. # -# require 'rubygems/tsort/lib/tsort' +# require 'rubygems/vendor/tsort/lib/tsort' # # class Hash # include Gem::TSort @@ -52,7 +52,7 @@ # # A very simple `make' like tool can be implemented as follows: # -# require 'rubygems/tsort/lib/tsort' +# require 'rubygems/vendor/tsort/lib/tsort' # # class Make # def initialize @@ -122,6 +122,9 @@ # module Gem::TSort + + VERSION = "0.2.0" + class Cyclic < StandardError end diff --git a/lib/rubygems/vendor/uri/.document b/lib/rubygems/vendor/uri/.document new file mode 100644 index 0000000000..0c43bbd6b3 --- /dev/null +++ b/lib/rubygems/vendor/uri/.document @@ -0,0 +1 @@ +# Vendored files do not need to be documented diff --git a/lib/rubygems/vendor/uri/lib/uri.rb b/lib/rubygems/vendor/uri/lib/uri.rb new file mode 100644 index 0000000000..f1ccc167cc --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri.rb @@ -0,0 +1,104 @@ +# frozen_string_literal: false +# Gem::URI is a module providing classes to handle Uniform Resource Identifiers +# (RFC2396[http://tools.ietf.org/html/rfc2396]). +# +# == Features +# +# * Uniform way of handling URIs. +# * Flexibility to introduce custom Gem::URI schemes. +# * Flexibility to have an alternate Gem::URI::Parser (or just different patterns +# and regexp's). +# +# == Basic example +# +# require 'rubygems/vendor/uri/lib/uri' +# +# uri = Gem::URI("http://foo.com/posts?id=30&limit=5#time=1305298413") +# #=> #<Gem::URI::HTTP http://foo.com/posts?id=30&limit=5#time=1305298413> +# +# uri.scheme #=> "http" +# uri.host #=> "foo.com" +# uri.path #=> "/posts" +# uri.query #=> "id=30&limit=5" +# uri.fragment #=> "time=1305298413" +# +# uri.to_s #=> "http://foo.com/posts?id=30&limit=5#time=1305298413" +# +# == Adding custom URIs +# +# module Gem::URI +# class RSYNC < Generic +# DEFAULT_PORT = 873 +# end +# register_scheme 'RSYNC', RSYNC +# end +# #=> Gem::URI::RSYNC +# +# Gem::URI.scheme_list +# #=> {"FILE"=>Gem::URI::File, "FTP"=>Gem::URI::FTP, "HTTP"=>Gem::URI::HTTP, +# # "HTTPS"=>Gem::URI::HTTPS, "LDAP"=>Gem::URI::LDAP, "LDAPS"=>Gem::URI::LDAPS, +# # "MAILTO"=>Gem::URI::MailTo, "RSYNC"=>Gem::URI::RSYNC} +# +# uri = Gem::URI("rsync://rsync.foo.com") +# #=> #<Gem::URI::RSYNC rsync://rsync.foo.com> +# +# == RFC References +# +# A good place to view an RFC spec is http://www.ietf.org/rfc.html. +# +# Here is a list of all related RFC's: +# - RFC822[http://tools.ietf.org/html/rfc822] +# - RFC1738[http://tools.ietf.org/html/rfc1738] +# - RFC2255[http://tools.ietf.org/html/rfc2255] +# - RFC2368[http://tools.ietf.org/html/rfc2368] +# - RFC2373[http://tools.ietf.org/html/rfc2373] +# - RFC2396[http://tools.ietf.org/html/rfc2396] +# - RFC2732[http://tools.ietf.org/html/rfc2732] +# - RFC3986[http://tools.ietf.org/html/rfc3986] +# +# == Class tree +# +# - Gem::URI::Generic (in uri/generic.rb) +# - Gem::URI::File - (in uri/file.rb) +# - Gem::URI::FTP - (in uri/ftp.rb) +# - Gem::URI::HTTP - (in uri/http.rb) +# - Gem::URI::HTTPS - (in uri/https.rb) +# - Gem::URI::LDAP - (in uri/ldap.rb) +# - Gem::URI::LDAPS - (in uri/ldaps.rb) +# - Gem::URI::MailTo - (in uri/mailto.rb) +# - Gem::URI::Parser - (in uri/common.rb) +# - Gem::URI::REGEXP - (in uri/common.rb) +# - Gem::URI::REGEXP::PATTERN - (in uri/common.rb) +# - Gem::URI::Util - (in uri/common.rb) +# - Gem::URI::Error - (in uri/common.rb) +# - Gem::URI::InvalidURIError - (in uri/common.rb) +# - Gem::URI::InvalidComponentError - (in uri/common.rb) +# - Gem::URI::BadURIError - (in uri/common.rb) +# +# == Copyright Info +# +# Author:: Akira Yamada <akira@ruby-lang.org> +# Documentation:: +# Akira Yamada <akira@ruby-lang.org> +# Dmitry V. Sabanin <sdmitry@lrn.ru> +# Vincent Batts <vbatts@hashbangbash.com> +# License:: +# Copyright (c) 2001 akira yamada <akira@ruby-lang.org> +# You can redistribute it and/or modify it under the same term as Ruby. +# + +module Gem::URI +end + +require_relative 'uri/version' +require_relative 'uri/common' +require_relative 'uri/generic' +require_relative 'uri/file' +require_relative 'uri/ftp' +require_relative 'uri/http' +require_relative 'uri/https' +require_relative 'uri/ldap' +require_relative 'uri/ldaps' +require_relative 'uri/mailto' +require_relative 'uri/ws' +require_relative 'uri/wss' diff --git a/lib/rubygems/vendor/uri/lib/uri/common.rb b/lib/rubygems/vendor/uri/lib/uri/common.rb new file mode 100644 index 0000000000..921fb9dd28 --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri/common.rb @@ -0,0 +1,853 @@ +# frozen_string_literal: true +#-- +# = uri/common.rb +# +# Author:: Akira Yamada <akira@ruby-lang.org> +# License:: +# You can redistribute it and/or modify it under the same term as Ruby. +# +# See Gem::URI for general documentation +# + +require_relative "rfc2396_parser" +require_relative "rfc3986_parser" + +module Gem::URI + include RFC2396_REGEXP + + REGEXP = RFC2396_REGEXP + Parser = RFC2396_Parser + RFC3986_PARSER = RFC3986_Parser.new + Ractor.make_shareable(RFC3986_PARSER) if defined?(Ractor) + + # Gem::URI::Parser.new + DEFAULT_PARSER = Parser.new + DEFAULT_PARSER.pattern.each_pair do |sym, str| + unless REGEXP::PATTERN.const_defined?(sym) + REGEXP::PATTERN.const_set(sym, str) + end + end + DEFAULT_PARSER.regexp.each_pair do |sym, str| + const_set(sym, str) + end + Ractor.make_shareable(DEFAULT_PARSER) if defined?(Ractor) + + module Util # :nodoc: + def make_components_hash(klass, array_hash) + tmp = {} + if array_hash.kind_of?(Array) && + array_hash.size == klass.component.size - 1 + klass.component[1..-1].each_index do |i| + begin + tmp[klass.component[i + 1]] = array_hash[i].clone + rescue TypeError + tmp[klass.component[i + 1]] = array_hash[i] + end + end + + elsif array_hash.kind_of?(Hash) + array_hash.each do |key, value| + begin + tmp[key] = value.clone + rescue TypeError + tmp[key] = value + end + end + else + raise ArgumentError, + "expected Array of or Hash of components of #{klass} (#{klass.component[1..-1].join(', ')})" + end + tmp[:scheme] = klass.to_s.sub(/\A.*::/, '').downcase + + return tmp + end + module_function :make_components_hash + end + + module Schemes + end + private_constant :Schemes + + # Registers the given +klass+ as the class to be instantiated + # when parsing a \Gem::URI with the given +scheme+: + # + # Gem::URI.register_scheme('MS_SEARCH', Gem::URI::Generic) # => Gem::URI::Generic + # Gem::URI.scheme_list['MS_SEARCH'] # => Gem::URI::Generic + # + # Note that after calling String#upcase on +scheme+, it must be a valid + # constant name. + def self.register_scheme(scheme, klass) + Schemes.const_set(scheme.to_s.upcase, klass) + end + + # Returns a hash of the defined schemes: + # + # Gem::URI.scheme_list + # # => + # {"MAILTO"=>Gem::URI::MailTo, + # "LDAPS"=>Gem::URI::LDAPS, + # "WS"=>Gem::URI::WS, + # "HTTP"=>Gem::URI::HTTP, + # "HTTPS"=>Gem::URI::HTTPS, + # "LDAP"=>Gem::URI::LDAP, + # "FILE"=>Gem::URI::File, + # "FTP"=>Gem::URI::FTP} + # + # Related: Gem::URI.register_scheme. + def self.scheme_list + Schemes.constants.map { |name| + [name.to_s.upcase, Schemes.const_get(name)] + }.to_h + end + + INITIAL_SCHEMES = scheme_list + private_constant :INITIAL_SCHEMES + Ractor.make_shareable(INITIAL_SCHEMES) if defined?(Ractor) + + # Returns a new object constructed from the given +scheme+, +arguments+, + # and +default+: + # + # - The new object is an instance of <tt>Gem::URI.scheme_list[scheme.upcase]</tt>. + # - The object is initialized by calling the class initializer + # using +scheme+ and +arguments+. + # See Gem::URI::Generic.new. + # + # Examples: + # + # values = ['john.doe', 'www.example.com', '123', nil, '/forum/questions/', nil, 'tag=networking&order=newest', 'top'] + # Gem::URI.for('https', *values) + # # => #<Gem::URI::HTTPS https://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top> + # Gem::URI.for('foo', *values, default: Gem::URI::HTTP) + # # => #<Gem::URI::HTTP foo://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top> + # + def self.for(scheme, *arguments, default: Generic) + const_name = scheme.to_s.upcase + + uri_class = INITIAL_SCHEMES[const_name] + uri_class ||= if /\A[A-Z]\w*\z/.match?(const_name) && Schemes.const_defined?(const_name, false) + Schemes.const_get(const_name, false) + end + uri_class ||= default + + return uri_class.new(scheme, *arguments) + end + + # + # Base class for all Gem::URI exceptions. + # + class Error < StandardError; end + # + # Not a Gem::URI. + # + class InvalidURIError < Error; end + # + # Not a Gem::URI component. + # + class InvalidComponentError < Error; end + # + # Gem::URI is valid, bad usage is not. + # + class BadURIError < Error; end + + # Returns a 9-element array representing the parts of the \Gem::URI + # formed from the string +uri+; + # each array element is a string or +nil+: + # + # names = %w[scheme userinfo host port registry path opaque query fragment] + # values = Gem::URI.split('https://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top') + # names.zip(values) + # # => + # [["scheme", "https"], + # ["userinfo", "john.doe"], + # ["host", "www.example.com"], + # ["port", "123"], + # ["registry", nil], + # ["path", "/forum/questions/"], + # ["opaque", nil], + # ["query", "tag=networking&order=newest"], + # ["fragment", "top"]] + # + def self.split(uri) + RFC3986_PARSER.split(uri) + end + + # Returns a new \Gem::URI object constructed from the given string +uri+: + # + # Gem::URI.parse('https://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top') + # # => #<Gem::URI::HTTPS https://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top> + # Gem::URI.parse('http://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top') + # # => #<Gem::URI::HTTP http://john.doe@www.example.com:123/forum/questions/?tag=networking&order=newest#top> + # + # It's recommended to first ::escape string +uri+ + # if it may contain invalid Gem::URI characters. + # + def self.parse(uri) + RFC3986_PARSER.parse(uri) + end + + # Merges the given Gem::URI strings +str+ + # per {RFC 2396}[https://www.rfc-editor.org/rfc/rfc2396.html]. + # + # Each string in +str+ is converted to an + # {RFC3986 Gem::URI}[https://www.rfc-editor.org/rfc/rfc3986.html] before being merged. + # + # Examples: + # + # Gem::URI.join("http://example.com/","main.rbx") + # # => #<Gem::URI::HTTP http://example.com/main.rbx> + # + # Gem::URI.join('http://example.com', 'foo') + # # => #<Gem::URI::HTTP http://example.com/foo> + # + # Gem::URI.join('http://example.com', '/foo', '/bar') + # # => #<Gem::URI::HTTP http://example.com/bar> + # + # Gem::URI.join('http://example.com', '/foo', 'bar') + # # => #<Gem::URI::HTTP http://example.com/bar> + # + # Gem::URI.join('http://example.com', '/foo/', 'bar') + # # => #<Gem::URI::HTTP http://example.com/foo/bar> + # + def self.join(*str) + RFC3986_PARSER.join(*str) + end + + # + # == Synopsis + # + # Gem::URI::extract(str[, schemes][,&blk]) + # + # == Args + # + # +str+:: + # String to extract URIs from. + # +schemes+:: + # Limit Gem::URI matching to specific schemes. + # + # == Description + # + # Extracts URIs from a string. If block given, iterates through all matched URIs. + # Returns nil if block given or array with matches. + # + # == Usage + # + # require "rubygems/vendor/uri/lib/uri" + # + # Gem::URI.extract("text here http://foo.example.org/bla and here mailto:test@example.com and here also.") + # # => ["http://foo.example.com/bla", "mailto:test@example.com"] + # + def self.extract(str, schemes = nil, &block) # :nodoc: + warn "Gem::URI.extract is obsolete", uplevel: 1 if $VERBOSE + DEFAULT_PARSER.extract(str, schemes, &block) + end + + # + # == Synopsis + # + # Gem::URI::regexp([match_schemes]) + # + # == Args + # + # +match_schemes+:: + # Array of schemes. If given, resulting regexp matches to URIs + # whose scheme is one of the match_schemes. + # + # == Description + # + # Returns a Regexp object which matches to Gem::URI-like strings. + # The Regexp object returned by this method includes arbitrary + # number of capture group (parentheses). Never rely on its number. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # # extract first Gem::URI from html_string + # html_string.slice(Gem::URI.regexp) + # + # # remove ftp URIs + # html_string.sub(Gem::URI.regexp(['ftp']), '') + # + # # You should not rely on the number of parentheses + # html_string.scan(Gem::URI.regexp) do |*matches| + # p $& + # end + # + def self.regexp(schemes = nil)# :nodoc: + warn "Gem::URI.regexp is obsolete", uplevel: 1 if $VERBOSE + DEFAULT_PARSER.make_regexp(schemes) + end + + TBLENCWWWCOMP_ = {} # :nodoc: + 256.times do |i| + TBLENCWWWCOMP_[-i.chr] = -('%%%02X' % i) + end + TBLENCURICOMP_ = TBLENCWWWCOMP_.dup.freeze + TBLENCWWWCOMP_[' '] = '+' + TBLENCWWWCOMP_.freeze + TBLDECWWWCOMP_ = {} # :nodoc: + 256.times do |i| + h, l = i>>4, i&15 + TBLDECWWWCOMP_[-('%%%X%X' % [h, l])] = -i.chr + TBLDECWWWCOMP_[-('%%%x%X' % [h, l])] = -i.chr + TBLDECWWWCOMP_[-('%%%X%x' % [h, l])] = -i.chr + TBLDECWWWCOMP_[-('%%%x%x' % [h, l])] = -i.chr + end + TBLDECWWWCOMP_['+'] = ' ' + TBLDECWWWCOMP_.freeze + + # Returns a URL-encoded string derived from the given string +str+. + # + # The returned string: + # + # - Preserves: + # + # - Characters <tt>'*'</tt>, <tt>'.'</tt>, <tt>'-'</tt>, and <tt>'_'</tt>. + # - Character in ranges <tt>'a'..'z'</tt>, <tt>'A'..'Z'</tt>, + # and <tt>'0'..'9'</tt>. + # + # Example: + # + # Gem::URI.encode_www_form_component('*.-_azAZ09') + # # => "*.-_azAZ09" + # + # - Converts: + # + # - Character <tt>' '</tt> to character <tt>'+'</tt>. + # - Any other character to "percent notation"; + # the percent notation for character <i>c</i> is <tt>'%%%X' % c.ord</tt>. + # + # Example: + # + # Gem::URI.encode_www_form_component('Here are some punctuation characters: ,;?:') + # # => "Here+are+some+punctuation+characters%3A+%2C%3B%3F%3A" + # + # Encoding: + # + # - If +str+ has encoding Encoding::ASCII_8BIT, argument +enc+ is ignored. + # - Otherwise +str+ is converted first to Encoding::UTF_8 + # (with suitable character replacements), + # and then to encoding +enc+. + # + # In either case, the returned string has forced encoding Encoding::US_ASCII. + # + # Related: Gem::URI.encode_uri_component (encodes <tt>' '</tt> as <tt>'%20'</tt>). + def self.encode_www_form_component(str, enc=nil) + _encode_uri_component(/[^*\-.0-9A-Z_a-z]/, TBLENCWWWCOMP_, str, enc) + end + + # Returns a string decoded from the given \URL-encoded string +str+. + # + # The given string is first encoded as Encoding::ASCII-8BIT (using String#b), + # then decoded (as below), and finally force-encoded to the given encoding +enc+. + # + # The returned string: + # + # - Preserves: + # + # - Characters <tt>'*'</tt>, <tt>'.'</tt>, <tt>'-'</tt>, and <tt>'_'</tt>. + # - Character in ranges <tt>'a'..'z'</tt>, <tt>'A'..'Z'</tt>, + # and <tt>'0'..'9'</tt>. + # + # Example: + # + # Gem::URI.decode_www_form_component('*.-_azAZ09') + # # => "*.-_azAZ09" + # + # - Converts: + # + # - Character <tt>'+'</tt> to character <tt>' '</tt>. + # - Each "percent notation" to an ASCII character. + # + # Example: + # + # Gem::URI.decode_www_form_component('Here+are+some+punctuation+characters%3A+%2C%3B%3F%3A') + # # => "Here are some punctuation characters: ,;?:" + # + # Related: Gem::URI.decode_uri_component (preserves <tt>'+'</tt>). + def self.decode_www_form_component(str, enc=Encoding::UTF_8) + _decode_uri_component(/\+|%\h\h/, str, enc) + end + + # Like Gem::URI.encode_www_form_component, except that <tt>' '</tt> (space) + # is encoded as <tt>'%20'</tt> (instead of <tt>'+'</tt>). + def self.encode_uri_component(str, enc=nil) + _encode_uri_component(/[^*\-.0-9A-Z_a-z]/, TBLENCURICOMP_, str, enc) + end + + # Like Gem::URI.decode_www_form_component, except that <tt>'+'</tt> is preserved. + def self.decode_uri_component(str, enc=Encoding::UTF_8) + _decode_uri_component(/%\h\h/, str, enc) + end + + def self._encode_uri_component(regexp, table, str, enc) + str = str.to_s.dup + if str.encoding != Encoding::ASCII_8BIT + if enc && enc != Encoding::ASCII_8BIT + str.encode!(Encoding::UTF_8, invalid: :replace, undef: :replace) + str.encode!(enc, fallback: ->(x){"&##{x.ord};"}) + end + str.force_encoding(Encoding::ASCII_8BIT) + end + str.gsub!(regexp, table) + str.force_encoding(Encoding::US_ASCII) + end + private_class_method :_encode_uri_component + + def self._decode_uri_component(regexp, str, enc) + raise ArgumentError, "invalid %-encoding (#{str})" if /%(?!\h\h)/.match?(str) + str.b.gsub(regexp, TBLDECWWWCOMP_).force_encoding(enc) + end + private_class_method :_decode_uri_component + + # Returns a URL-encoded string derived from the given + # {Enumerable}[https://docs.ruby-lang.org/en/master/Enumerable.html#module-Enumerable-label-Enumerable+in+Ruby+Classes] + # +enum+. + # + # The result is suitable for use as form data + # for an \HTTP request whose <tt>Content-Type</tt> is + # <tt>'application/x-www-form-urlencoded'</tt>. + # + # The returned string consists of the elements of +enum+, + # each converted to one or more URL-encoded strings, + # and all joined with character <tt>'&'</tt>. + # + # Simple examples: + # + # Gem::URI.encode_www_form([['foo', 0], ['bar', 1], ['baz', 2]]) + # # => "foo=0&bar=1&baz=2" + # Gem::URI.encode_www_form({foo: 0, bar: 1, baz: 2}) + # # => "foo=0&bar=1&baz=2" + # + # The returned string is formed using method Gem::URI.encode_www_form_component, + # which converts certain characters: + # + # Gem::URI.encode_www_form('f#o': '/', 'b-r': '$', 'b z': '@') + # # => "f%23o=%2F&b-r=%24&b+z=%40" + # + # When +enum+ is Array-like, each element +ele+ is converted to a field: + # + # - If +ele+ is an array of two or more elements, + # the field is formed from its first two elements + # (and any additional elements are ignored): + # + # name = Gem::URI.encode_www_form_component(ele[0], enc) + # value = Gem::URI.encode_www_form_component(ele[1], enc) + # "#{name}=#{value}" + # + # Examples: + # + # Gem::URI.encode_www_form([%w[foo bar], %w[baz bat bah]]) + # # => "foo=bar&baz=bat" + # Gem::URI.encode_www_form([['foo', 0], ['bar', :baz, 'bat']]) + # # => "foo=0&bar=baz" + # + # - If +ele+ is an array of one element, + # the field is formed from <tt>ele[0]</tt>: + # + # Gem::URI.encode_www_form_component(ele[0]) + # + # Example: + # + # Gem::URI.encode_www_form([['foo'], [:bar], [0]]) + # # => "foo&bar&0" + # + # - Otherwise the field is formed from +ele+: + # + # Gem::URI.encode_www_form_component(ele) + # + # Example: + # + # Gem::URI.encode_www_form(['foo', :bar, 0]) + # # => "foo&bar&0" + # + # The elements of an Array-like +enum+ may be mixture: + # + # Gem::URI.encode_www_form([['foo', 0], ['bar', 1, 2], ['baz'], :bat]) + # # => "foo=0&bar=1&baz&bat" + # + # When +enum+ is Hash-like, + # each +key+/+value+ pair is converted to one or more fields: + # + # - If +value+ is + # {Array-convertible}[https://docs.ruby-lang.org/en/master/implicit_conversion_rdoc.html#label-Array-Convertible+Objects], + # each element +ele+ in +value+ is paired with +key+ to form a field: + # + # name = Gem::URI.encode_www_form_component(key, enc) + # value = Gem::URI.encode_www_form_component(ele, enc) + # "#{name}=#{value}" + # + # Example: + # + # Gem::URI.encode_www_form({foo: [:bar, 1], baz: [:bat, :bam, 2]}) + # # => "foo=bar&foo=1&baz=bat&baz=bam&baz=2" + # + # - Otherwise, +key+ and +value+ are paired to form a field: + # + # name = Gem::URI.encode_www_form_component(key, enc) + # value = Gem::URI.encode_www_form_component(value, enc) + # "#{name}=#{value}" + # + # Example: + # + # Gem::URI.encode_www_form({foo: 0, bar: 1, baz: 2}) + # # => "foo=0&bar=1&baz=2" + # + # The elements of a Hash-like +enum+ may be mixture: + # + # Gem::URI.encode_www_form({foo: [0, 1], bar: 2}) + # # => "foo=0&foo=1&bar=2" + # + def self.encode_www_form(enum, enc=nil) + enum.map do |k,v| + if v.nil? + encode_www_form_component(k, enc) + elsif v.respond_to?(:to_ary) + v.to_ary.map do |w| + str = encode_www_form_component(k, enc) + unless w.nil? + str << '=' + str << encode_www_form_component(w, enc) + end + end.join('&') + else + str = encode_www_form_component(k, enc) + str << '=' + str << encode_www_form_component(v, enc) + end + end.join('&') + end + + # Returns name/value pairs derived from the given string +str+, + # which must be an ASCII string. + # + # The method may be used to decode the body of Net::HTTPResponse object +res+ + # for which <tt>res['Content-Type']</tt> is <tt>'application/x-www-form-urlencoded'</tt>. + # + # The returned data is an array of 2-element subarrays; + # each subarray is a name/value pair (both are strings). + # Each returned string has encoding +enc+, + # and has had invalid characters removed via + # {String#scrub}[https://docs.ruby-lang.org/en/master/String.html#method-i-scrub]. + # + # A simple example: + # + # Gem::URI.decode_www_form('foo=0&bar=1&baz') + # # => [["foo", "0"], ["bar", "1"], ["baz", ""]] + # + # The returned strings have certain conversions, + # similar to those performed in Gem::URI.decode_www_form_component: + # + # Gem::URI.decode_www_form('f%23o=%2F&b-r=%24&b+z=%40') + # # => [["f#o", "/"], ["b-r", "$"], ["b z", "@"]] + # + # The given string may contain consecutive separators: + # + # Gem::URI.decode_www_form('foo=0&&bar=1&&baz=2') + # # => [["foo", "0"], ["", ""], ["bar", "1"], ["", ""], ["baz", "2"]] + # + # A different separator may be specified: + # + # Gem::URI.decode_www_form('foo=0--bar=1--baz', separator: '--') + # # => [["foo", "0"], ["bar", "1"], ["baz", ""]] + # + def self.decode_www_form(str, enc=Encoding::UTF_8, separator: '&', use__charset_: false, isindex: false) + raise ArgumentError, "the input of #{self.name}.#{__method__} must be ASCII only string" unless str.ascii_only? + ary = [] + return ary if str.empty? + enc = Encoding.find(enc) + str.b.each_line(separator) do |string| + string.chomp!(separator) + key, sep, val = string.partition('=') + if isindex + if sep.empty? + val = key + key = +'' + end + isindex = false + end + + if use__charset_ and key == '_charset_' and e = get_encoding(val) + enc = e + use__charset_ = false + end + + key.gsub!(/\+|%\h\h/, TBLDECWWWCOMP_) + if val + val.gsub!(/\+|%\h\h/, TBLDECWWWCOMP_) + else + val = +'' + end + + ary << [key, val] + end + ary.each do |k, v| + k.force_encoding(enc) + k.scrub! + v.force_encoding(enc) + v.scrub! + end + ary + end + + private +=begin command for WEB_ENCODINGS_ + curl https://encoding.spec.whatwg.org/encodings.json| + ruby -rjson -e 'H={} + h={ + "shift_jis"=>"Windows-31J", + "euc-jp"=>"cp51932", + "iso-2022-jp"=>"cp50221", + "x-mac-cyrillic"=>"macCyrillic", + } + JSON($<.read).map{|x|x["encodings"]}.flatten.each{|x| + Encoding.find(n=h.fetch(n=x["name"].downcase,n))rescue next + x["labels"].each{|y|H[y]=n} + } + puts "{" + H.each{|k,v|puts %[ #{k.dump}=>#{v.dump},]} + puts "}" +' +=end + WEB_ENCODINGS_ = { + "unicode-1-1-utf-8"=>"utf-8", + "utf-8"=>"utf-8", + "utf8"=>"utf-8", + "866"=>"ibm866", + "cp866"=>"ibm866", + "csibm866"=>"ibm866", + "ibm866"=>"ibm866", + "csisolatin2"=>"iso-8859-2", + "iso-8859-2"=>"iso-8859-2", + "iso-ir-101"=>"iso-8859-2", + "iso8859-2"=>"iso-8859-2", + "iso88592"=>"iso-8859-2", + "iso_8859-2"=>"iso-8859-2", + "iso_8859-2:1987"=>"iso-8859-2", + "l2"=>"iso-8859-2", + "latin2"=>"iso-8859-2", + "csisolatin3"=>"iso-8859-3", + "iso-8859-3"=>"iso-8859-3", + "iso-ir-109"=>"iso-8859-3", + "iso8859-3"=>"iso-8859-3", + "iso88593"=>"iso-8859-3", + "iso_8859-3"=>"iso-8859-3", + "iso_8859-3:1988"=>"iso-8859-3", + "l3"=>"iso-8859-3", + "latin3"=>"iso-8859-3", + "csisolatin4"=>"iso-8859-4", + "iso-8859-4"=>"iso-8859-4", + "iso-ir-110"=>"iso-8859-4", + "iso8859-4"=>"iso-8859-4", + "iso88594"=>"iso-8859-4", + "iso_8859-4"=>"iso-8859-4", + "iso_8859-4:1988"=>"iso-8859-4", + "l4"=>"iso-8859-4", + "latin4"=>"iso-8859-4", + "csisolatincyrillic"=>"iso-8859-5", + "cyrillic"=>"iso-8859-5", + "iso-8859-5"=>"iso-8859-5", + "iso-ir-144"=>"iso-8859-5", + "iso8859-5"=>"iso-8859-5", + "iso88595"=>"iso-8859-5", + "iso_8859-5"=>"iso-8859-5", + "iso_8859-5:1988"=>"iso-8859-5", + "arabic"=>"iso-8859-6", + "asmo-708"=>"iso-8859-6", + "csiso88596e"=>"iso-8859-6", + "csiso88596i"=>"iso-8859-6", + "csisolatinarabic"=>"iso-8859-6", + "ecma-114"=>"iso-8859-6", + "iso-8859-6"=>"iso-8859-6", + "iso-8859-6-e"=>"iso-8859-6", + "iso-8859-6-i"=>"iso-8859-6", + "iso-ir-127"=>"iso-8859-6", + "iso8859-6"=>"iso-8859-6", + "iso88596"=>"iso-8859-6", + "iso_8859-6"=>"iso-8859-6", + "iso_8859-6:1987"=>"iso-8859-6", + "csisolatingreek"=>"iso-8859-7", + "ecma-118"=>"iso-8859-7", + "elot_928"=>"iso-8859-7", + "greek"=>"iso-8859-7", + "greek8"=>"iso-8859-7", + "iso-8859-7"=>"iso-8859-7", + "iso-ir-126"=>"iso-8859-7", + "iso8859-7"=>"iso-8859-7", + "iso88597"=>"iso-8859-7", + "iso_8859-7"=>"iso-8859-7", + "iso_8859-7:1987"=>"iso-8859-7", + "sun_eu_greek"=>"iso-8859-7", + "csiso88598e"=>"iso-8859-8", + "csisolatinhebrew"=>"iso-8859-8", + "hebrew"=>"iso-8859-8", + "iso-8859-8"=>"iso-8859-8", + "iso-8859-8-e"=>"iso-8859-8", + "iso-ir-138"=>"iso-8859-8", + "iso8859-8"=>"iso-8859-8", + "iso88598"=>"iso-8859-8", + "iso_8859-8"=>"iso-8859-8", + "iso_8859-8:1988"=>"iso-8859-8", + "visual"=>"iso-8859-8", + "csisolatin6"=>"iso-8859-10", + "iso-8859-10"=>"iso-8859-10", + "iso-ir-157"=>"iso-8859-10", + "iso8859-10"=>"iso-8859-10", + "iso885910"=>"iso-8859-10", + "l6"=>"iso-8859-10", + "latin6"=>"iso-8859-10", + "iso-8859-13"=>"iso-8859-13", + "iso8859-13"=>"iso-8859-13", + "iso885913"=>"iso-8859-13", + "iso-8859-14"=>"iso-8859-14", + "iso8859-14"=>"iso-8859-14", + "iso885914"=>"iso-8859-14", + "csisolatin9"=>"iso-8859-15", + "iso-8859-15"=>"iso-8859-15", + "iso8859-15"=>"iso-8859-15", + "iso885915"=>"iso-8859-15", + "iso_8859-15"=>"iso-8859-15", + "l9"=>"iso-8859-15", + "iso-8859-16"=>"iso-8859-16", + "cskoi8r"=>"koi8-r", + "koi"=>"koi8-r", + "koi8"=>"koi8-r", + "koi8-r"=>"koi8-r", + "koi8_r"=>"koi8-r", + "koi8-ru"=>"koi8-u", + "koi8-u"=>"koi8-u", + "dos-874"=>"windows-874", + "iso-8859-11"=>"windows-874", + "iso8859-11"=>"windows-874", + "iso885911"=>"windows-874", + "tis-620"=>"windows-874", + "windows-874"=>"windows-874", + "cp1250"=>"windows-1250", + "windows-1250"=>"windows-1250", + "x-cp1250"=>"windows-1250", + "cp1251"=>"windows-1251", + "windows-1251"=>"windows-1251", + "x-cp1251"=>"windows-1251", + "ansi_x3.4-1968"=>"windows-1252", + "ascii"=>"windows-1252", + "cp1252"=>"windows-1252", + "cp819"=>"windows-1252", + "csisolatin1"=>"windows-1252", + "ibm819"=>"windows-1252", + "iso-8859-1"=>"windows-1252", + "iso-ir-100"=>"windows-1252", + "iso8859-1"=>"windows-1252", + "iso88591"=>"windows-1252", + "iso_8859-1"=>"windows-1252", + "iso_8859-1:1987"=>"windows-1252", + "l1"=>"windows-1252", + "latin1"=>"windows-1252", + "us-ascii"=>"windows-1252", + "windows-1252"=>"windows-1252", + "x-cp1252"=>"windows-1252", + "cp1253"=>"windows-1253", + "windows-1253"=>"windows-1253", + "x-cp1253"=>"windows-1253", + "cp1254"=>"windows-1254", + "csisolatin5"=>"windows-1254", + "iso-8859-9"=>"windows-1254", + "iso-ir-148"=>"windows-1254", + "iso8859-9"=>"windows-1254", + "iso88599"=>"windows-1254", + "iso_8859-9"=>"windows-1254", + "iso_8859-9:1989"=>"windows-1254", + "l5"=>"windows-1254", + "latin5"=>"windows-1254", + "windows-1254"=>"windows-1254", + "x-cp1254"=>"windows-1254", + "cp1255"=>"windows-1255", + "windows-1255"=>"windows-1255", + "x-cp1255"=>"windows-1255", + "cp1256"=>"windows-1256", + "windows-1256"=>"windows-1256", + "x-cp1256"=>"windows-1256", + "cp1257"=>"windows-1257", + "windows-1257"=>"windows-1257", + "x-cp1257"=>"windows-1257", + "cp1258"=>"windows-1258", + "windows-1258"=>"windows-1258", + "x-cp1258"=>"windows-1258", + "x-mac-cyrillic"=>"macCyrillic", + "x-mac-ukrainian"=>"macCyrillic", + "chinese"=>"gbk", + "csgb2312"=>"gbk", + "csiso58gb231280"=>"gbk", + "gb2312"=>"gbk", + "gb_2312"=>"gbk", + "gb_2312-80"=>"gbk", + "gbk"=>"gbk", + "iso-ir-58"=>"gbk", + "x-gbk"=>"gbk", + "gb18030"=>"gb18030", + "big5"=>"big5", + "big5-hkscs"=>"big5", + "cn-big5"=>"big5", + "csbig5"=>"big5", + "x-x-big5"=>"big5", + "cseucpkdfmtjapanese"=>"cp51932", + "euc-jp"=>"cp51932", + "x-euc-jp"=>"cp51932", + "csiso2022jp"=>"cp50221", + "iso-2022-jp"=>"cp50221", + "csshiftjis"=>"Windows-31J", + "ms932"=>"Windows-31J", + "ms_kanji"=>"Windows-31J", + "shift-jis"=>"Windows-31J", + "shift_jis"=>"Windows-31J", + "sjis"=>"Windows-31J", + "windows-31j"=>"Windows-31J", + "x-sjis"=>"Windows-31J", + "cseuckr"=>"euc-kr", + "csksc56011987"=>"euc-kr", + "euc-kr"=>"euc-kr", + "iso-ir-149"=>"euc-kr", + "korean"=>"euc-kr", + "ks_c_5601-1987"=>"euc-kr", + "ks_c_5601-1989"=>"euc-kr", + "ksc5601"=>"euc-kr", + "ksc_5601"=>"euc-kr", + "windows-949"=>"euc-kr", + "utf-16be"=>"utf-16be", + "utf-16"=>"utf-16le", + "utf-16le"=>"utf-16le", + } # :nodoc: + Ractor.make_shareable(WEB_ENCODINGS_) if defined?(Ractor) + + # :nodoc: + # return encoding or nil + # http://encoding.spec.whatwg.org/#concept-encoding-get + def self.get_encoding(label) + Encoding.find(WEB_ENCODINGS_[label.to_str.strip.downcase]) rescue nil + end +end # module Gem::URI + +module Gem + + # + # Returns a \Gem::URI object derived from the given +uri+, + # which may be a \Gem::URI string or an existing \Gem::URI object: + # + # # Returns a new Gem::URI. + # uri = Gem::URI('http://github.com/ruby/ruby') + # # => #<Gem::URI::HTTP http://github.com/ruby/ruby> + # # Returns the given Gem::URI. + # Gem::URI(uri) + # # => #<Gem::URI::HTTP http://github.com/ruby/ruby> + # + def URI(uri) + if uri.is_a?(Gem::URI::Generic) + uri + elsif uri = String.try_convert(uri) + Gem::URI.parse(uri) + else + raise ArgumentError, + "bad argument (expected Gem::URI object or Gem::URI string)" + end + end + module_function :URI +end diff --git a/lib/rubygems/vendor/uri/lib/uri/file.rb b/lib/rubygems/vendor/uri/lib/uri/file.rb new file mode 100644 index 0000000000..d419b26055 --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri/file.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require_relative 'generic' + +module Gem::URI + + # + # The "file" Gem::URI is defined by RFC8089. + # + class File < Generic + # A Default port of nil for Gem::URI::File. + DEFAULT_PORT = nil + + # + # An Array of the available components for Gem::URI::File. + # + COMPONENT = [ + :scheme, + :host, + :path + ].freeze + + # + # == Description + # + # Creates a new Gem::URI::File object from components, with syntax checking. + # + # The components accepted are +host+ and +path+. + # + # The components should be provided either as an Array, or as a Hash + # with keys formed by preceding the component names with a colon. + # + # If an Array is used, the components must be passed in the + # order <code>[host, path]</code>. + # + # A path from e.g. the File class should be escaped before + # being passed. + # + # Examples: + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri1 = Gem::URI::File.build(['host.example.com', '/path/file.zip']) + # uri1.to_s # => "file://host.example.com/path/file.zip" + # + # uri2 = Gem::URI::File.build({:host => 'host.example.com', + # :path => '/ruby/src'}) + # uri2.to_s # => "file://host.example.com/ruby/src" + # + # uri3 = Gem::URI::File.build({:path => Gem::URI::escape('/path/my file.txt')}) + # uri3.to_s # => "file:///path/my%20file.txt" + # + def self.build(args) + tmp = Util::make_components_hash(self, args) + super(tmp) + end + + # Protected setter for the host component +v+. + # + # See also Gem::URI::Generic.host=. + # + def set_host(v) + v = "" if v.nil? || v == "localhost" + @host = v + end + + # do nothing + def set_port(v) + end + + # raise InvalidURIError + def check_userinfo(user) + raise Gem::URI::InvalidURIError, "can not set userinfo for file Gem::URI" + end + + # raise InvalidURIError + def check_user(user) + raise Gem::URI::InvalidURIError, "can not set user for file Gem::URI" + end + + # raise InvalidURIError + def check_password(user) + raise Gem::URI::InvalidURIError, "can not set password for file Gem::URI" + end + + # do nothing + def set_userinfo(v) + end + + # do nothing + def set_user(v) + end + + # do nothing + def set_password(v) + end + end + + register_scheme 'FILE', File +end diff --git a/lib/rubygems/vendor/uri/lib/uri/ftp.rb b/lib/rubygems/vendor/uri/lib/uri/ftp.rb new file mode 100644 index 0000000000..100498ffb2 --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri/ftp.rb @@ -0,0 +1,267 @@ +# frozen_string_literal: false +# = uri/ftp.rb +# +# Author:: Akira Yamada <akira@ruby-lang.org> +# License:: You can redistribute it and/or modify it under the same term as Ruby. +# +# See Gem::URI for general documentation +# + +require_relative 'generic' + +module Gem::URI + + # + # FTP Gem::URI syntax is defined by RFC1738 section 3.2. + # + # This class will be redesigned because of difference of implementations; + # the structure of its path. draft-hoffman-ftp-uri-04 is a draft but it + # is a good summary about the de facto spec. + # http://tools.ietf.org/html/draft-hoffman-ftp-uri-04 + # + class FTP < Generic + # A Default port of 21 for Gem::URI::FTP. + DEFAULT_PORT = 21 + + # + # An Array of the available components for Gem::URI::FTP. + # + COMPONENT = [ + :scheme, + :userinfo, :host, :port, + :path, :typecode + ].freeze + + # + # Typecode is "a", "i", or "d". + # + # * "a" indicates a text file (the FTP command was ASCII) + # * "i" indicates a binary file (FTP command IMAGE) + # * "d" indicates the contents of a directory should be displayed + # + TYPECODE = ['a', 'i', 'd'].freeze + + # Typecode prefix ";type=". + TYPECODE_PREFIX = ';type='.freeze + + def self.new2(user, password, host, port, path, + typecode = nil, arg_check = true) # :nodoc: + # Do not use this method! Not tested. [Bug #7301] + # This methods remains just for compatibility, + # Keep it undocumented until the active maintainer is assigned. + typecode = nil if typecode.size == 0 + if typecode && !TYPECODE.include?(typecode) + raise ArgumentError, + "bad typecode is specified: #{typecode}" + end + + # do escape + + self.new('ftp', + [user, password], + host, port, nil, + typecode ? path + TYPECODE_PREFIX + typecode : path, + nil, nil, nil, arg_check) + end + + # + # == Description + # + # Creates a new Gem::URI::FTP object from components, with syntax checking. + # + # The components accepted are +userinfo+, +host+, +port+, +path+, and + # +typecode+. + # + # The components should be provided either as an Array, or as a Hash + # with keys formed by preceding the component names with a colon. + # + # If an Array is used, the components must be passed in the + # order <code>[userinfo, host, port, path, typecode]</code>. + # + # If the path supplied is absolute, it will be escaped in order to + # make it absolute in the Gem::URI. + # + # Examples: + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri1 = Gem::URI::FTP.build(['user:password', 'ftp.example.com', nil, + # '/path/file.zip', 'i']) + # uri1.to_s # => "ftp://user:password@ftp.example.com/%2Fpath/file.zip;type=i" + # + # uri2 = Gem::URI::FTP.build({:host => 'ftp.example.com', + # :path => 'ruby/src'}) + # uri2.to_s # => "ftp://ftp.example.com/ruby/src" + # + def self.build(args) + + # Fix the incoming path to be generic URL syntax + # FTP path -> URL path + # foo/bar /foo/bar + # /foo/bar /%2Ffoo/bar + # + if args.kind_of?(Array) + args[3] = '/' + args[3].sub(/^\//, '%2F') + else + args[:path] = '/' + args[:path].sub(/^\//, '%2F') + end + + tmp = Util::make_components_hash(self, args) + + if tmp[:typecode] + if tmp[:typecode].size == 1 + tmp[:typecode] = TYPECODE_PREFIX + tmp[:typecode] + end + tmp[:path] << tmp[:typecode] + end + + return super(tmp) + end + + # + # == Description + # + # Creates a new Gem::URI::FTP object from generic URL components with no + # syntax checking. + # + # Unlike build(), this method does not escape the path component as + # required by RFC1738; instead it is treated as per RFC2396. + # + # Arguments are +scheme+, +userinfo+, +host+, +port+, +registry+, +path+, + # +opaque+, +query+, and +fragment+, in that order. + # + def initialize(scheme, + userinfo, host, port, registry, + path, opaque, + query, + fragment, + parser = nil, + arg_check = false) + raise InvalidURIError unless path + path = path.sub(/^\//,'') + path.sub!(/^%2F/,'/') + super(scheme, userinfo, host, port, registry, path, opaque, + query, fragment, parser, arg_check) + @typecode = nil + if tmp = @path.index(TYPECODE_PREFIX) + typecode = @path[tmp + TYPECODE_PREFIX.size..-1] + @path = @path[0..tmp - 1] + + if arg_check + self.typecode = typecode + else + self.set_typecode(typecode) + end + end + end + + # typecode accessor. + # + # See Gem::URI::FTP::COMPONENT. + attr_reader :typecode + + # Validates typecode +v+, + # returns +true+ or +false+. + # + def check_typecode(v) + if TYPECODE.include?(v) + return true + else + raise InvalidComponentError, + "bad typecode(expected #{TYPECODE.join(', ')}): #{v}" + end + end + private :check_typecode + + # Private setter for the typecode +v+. + # + # See also Gem::URI::FTP.typecode=. + # + def set_typecode(v) + @typecode = v + end + protected :set_typecode + + # + # == Args + # + # +v+:: + # String + # + # == Description + # + # Public setter for the typecode +v+ + # (with validation). + # + # See also Gem::URI::FTP.check_typecode. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse("ftp://john@ftp.example.com/my_file.img") + # #=> #<Gem::URI::FTP ftp://john@ftp.example.com/my_file.img> + # uri.typecode = "i" + # uri + # #=> #<Gem::URI::FTP ftp://john@ftp.example.com/my_file.img;type=i> + # + def typecode=(typecode) + check_typecode(typecode) + set_typecode(typecode) + typecode + end + + def merge(oth) # :nodoc: + tmp = super(oth) + if self != tmp + tmp.set_typecode(oth.typecode) + end + + return tmp + end + + # Returns the path from an FTP Gem::URI. + # + # RFC 1738 specifically states that the path for an FTP Gem::URI does not + # include the / which separates the Gem::URI path from the Gem::URI host. Example: + # + # <code>ftp://ftp.example.com/pub/ruby</code> + # + # The above Gem::URI indicates that the client should connect to + # ftp.example.com then cd to pub/ruby from the initial login directory. + # + # If you want to cd to an absolute directory, you must include an + # escaped / (%2F) in the path. Example: + # + # <code>ftp://ftp.example.com/%2Fpub/ruby</code> + # + # This method will then return "/pub/ruby". + # + def path + return @path.sub(/^\//,'').sub(/^%2F/,'/') + end + + # Private setter for the path of the Gem::URI::FTP. + def set_path(v) + super("/" + v.sub(/^\//, "%2F")) + end + protected :set_path + + # Returns a String representation of the Gem::URI::FTP. + def to_s + save_path = nil + if @typecode + save_path = @path + @path = @path + TYPECODE_PREFIX + @typecode + end + str = super + if @typecode + @path = save_path + end + + return str + end + end + + register_scheme 'FTP', FTP +end diff --git a/lib/rubygems/vendor/uri/lib/uri/generic.rb b/lib/rubygems/vendor/uri/lib/uri/generic.rb new file mode 100644 index 0000000000..72c52aa8ee --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri/generic.rb @@ -0,0 +1,1588 @@ +# frozen_string_literal: true + +# = uri/generic.rb +# +# Author:: Akira Yamada <akira@ruby-lang.org> +# License:: You can redistribute it and/or modify it under the same term as Ruby. +# +# See Gem::URI for general documentation +# + +require_relative 'common' +autoload :IPSocket, 'socket' +autoload :IPAddr, 'ipaddr' + +module Gem::URI + + # + # Base class for all Gem::URI classes. + # Implements generic Gem::URI syntax as per RFC 2396. + # + class Generic + include Gem::URI + + # + # A Default port of nil for Gem::URI::Generic. + # + DEFAULT_PORT = nil + + # + # Returns default port. + # + def self.default_port + self::DEFAULT_PORT + end + + # + # Returns default port. + # + def default_port + self.class.default_port + end + + # + # An Array of the available components for Gem::URI::Generic. + # + COMPONENT = [ + :scheme, + :userinfo, :host, :port, :registry, + :path, :opaque, + :query, + :fragment + ].freeze + + # + # Components of the Gem::URI in the order. + # + def self.component + self::COMPONENT + end + + USE_REGISTRY = false # :nodoc: + + def self.use_registry # :nodoc: + self::USE_REGISTRY + end + + # + # == Synopsis + # + # See ::new. + # + # == Description + # + # At first, tries to create a new Gem::URI::Generic instance using + # Gem::URI::Generic::build. But, if exception Gem::URI::InvalidComponentError is raised, + # then it does Gem::URI::Escape.escape all Gem::URI components and tries again. + # + def self.build2(args) + begin + return self.build(args) + rescue InvalidComponentError + if args.kind_of?(Array) + return self.build(args.collect{|x| + if x.is_a?(String) + DEFAULT_PARSER.escape(x) + else + x + end + }) + elsif args.kind_of?(Hash) + tmp = {} + args.each do |key, value| + tmp[key] = if value + DEFAULT_PARSER.escape(value) + else + value + end + end + return self.build(tmp) + end + end + end + + # + # == Synopsis + # + # See ::new. + # + # == Description + # + # Creates a new Gem::URI::Generic instance from components of Gem::URI::Generic + # with check. Components are: scheme, userinfo, host, port, registry, path, + # opaque, query, and fragment. You can provide arguments either by an Array or a Hash. + # See ::new for hash keys to use or for order of array items. + # + def self.build(args) + if args.kind_of?(Array) && + args.size == ::Gem::URI::Generic::COMPONENT.size + tmp = args.dup + elsif args.kind_of?(Hash) + tmp = ::Gem::URI::Generic::COMPONENT.collect do |c| + if args.include?(c) + args[c] + else + nil + end + end + else + component = self.class.component rescue ::Gem::URI::Generic::COMPONENT + raise ArgumentError, + "expected Array of or Hash of components of #{self.class} (#{component.join(', ')})" + end + + tmp << nil + tmp << true + return self.new(*tmp) + end + + # + # == Args + # + # +scheme+:: + # Protocol scheme, i.e. 'http','ftp','mailto' and so on. + # +userinfo+:: + # User name and password, i.e. 'sdmitry:bla'. + # +host+:: + # Server host name. + # +port+:: + # Server port. + # +registry+:: + # Registry of naming authorities. + # +path+:: + # Path on server. + # +opaque+:: + # Opaque part. + # +query+:: + # Query data. + # +fragment+:: + # Part of the Gem::URI after '#' character. + # +parser+:: + # Parser for internal use [Gem::URI::DEFAULT_PARSER by default]. + # +arg_check+:: + # Check arguments [false by default]. + # + # == Description + # + # Creates a new Gem::URI::Generic instance from ``generic'' components without check. + # + def initialize(scheme, + userinfo, host, port, registry, + path, opaque, + query, + fragment, + parser = DEFAULT_PARSER, + arg_check = false) + @scheme = nil + @user = nil + @password = nil + @host = nil + @port = nil + @path = nil + @query = nil + @opaque = nil + @fragment = nil + @parser = parser == DEFAULT_PARSER ? nil : parser + + if arg_check + self.scheme = scheme + self.userinfo = userinfo + self.hostname = host + self.port = port + self.path = path + self.query = query + self.opaque = opaque + self.fragment = fragment + else + self.set_scheme(scheme) + self.set_userinfo(userinfo) + self.set_host(host) + self.set_port(port) + self.set_path(path) + self.query = query + self.set_opaque(opaque) + self.fragment=(fragment) + end + if registry + raise InvalidURIError, + "the scheme #{@scheme} does not accept registry part: #{registry} (or bad hostname?)" + end + + @scheme&.freeze + self.set_path('') if !@path && !@opaque # (see RFC2396 Section 5.2) + self.set_port(self.default_port) if self.default_port && !@port + end + + # + # Returns the scheme component of the Gem::URI. + # + # Gem::URI("http://foo/bar/baz").scheme #=> "http" + # + attr_reader :scheme + + # Returns the host component of the Gem::URI. + # + # Gem::URI("http://foo/bar/baz").host #=> "foo" + # + # It returns nil if no host component exists. + # + # Gem::URI("mailto:foo@example.org").host #=> nil + # + # The component does not contain the port number. + # + # Gem::URI("http://foo:8080/bar/baz").host #=> "foo" + # + # Since IPv6 addresses are wrapped with brackets in URIs, + # this method returns IPv6 addresses wrapped with brackets. + # This form is not appropriate to pass to socket methods such as TCPSocket.open. + # If unwrapped host names are required, use the #hostname method. + # + # Gem::URI("http://[::1]/bar/baz").host #=> "[::1]" + # Gem::URI("http://[::1]/bar/baz").hostname #=> "::1" + # + attr_reader :host + + # Returns the port component of the Gem::URI. + # + # Gem::URI("http://foo/bar/baz").port #=> 80 + # Gem::URI("http://foo:8080/bar/baz").port #=> 8080 + # + attr_reader :port + + def registry # :nodoc: + nil + end + + # Returns the path component of the Gem::URI. + # + # Gem::URI("http://foo/bar/baz").path #=> "/bar/baz" + # + attr_reader :path + + # Returns the query component of the Gem::URI. + # + # Gem::URI("http://foo/bar/baz?search=FooBar").query #=> "search=FooBar" + # + attr_reader :query + + # Returns the opaque part of the Gem::URI. + # + # Gem::URI("mailto:foo@example.org").opaque #=> "foo@example.org" + # Gem::URI("http://foo/bar/baz").opaque #=> nil + # + # The portion of the path that does not make use of the slash '/'. + # The path typically refers to an absolute path or an opaque part. + # (See RFC2396 Section 3 and 5.2.) + # + attr_reader :opaque + + # Returns the fragment component of the Gem::URI. + # + # Gem::URI("http://foo/bar/baz?search=FooBar#ponies").fragment #=> "ponies" + # + attr_reader :fragment + + # Returns the parser to be used. + # + # Unless a Gem::URI::Parser is defined, DEFAULT_PARSER is used. + # + def parser + if !defined?(@parser) || !@parser + DEFAULT_PARSER + else + @parser || DEFAULT_PARSER + end + end + + # Replaces self by other Gem::URI object. + # + def replace!(oth) + if self.class != oth.class + raise ArgumentError, "expected #{self.class} object" + end + + component.each do |c| + self.__send__("#{c}=", oth.__send__(c)) + end + end + private :replace! + + # + # Components of the Gem::URI in the order. + # + def component + self.class.component + end + + # + # Checks the scheme +v+ component against the Gem::URI::Parser Regexp for :SCHEME. + # + def check_scheme(v) + if v && parser.regexp[:SCHEME] !~ v + raise InvalidComponentError, + "bad component(expected scheme component): #{v}" + end + + return true + end + private :check_scheme + + # Protected setter for the scheme component +v+. + # + # See also Gem::URI::Generic.scheme=. + # + def set_scheme(v) + @scheme = v&.downcase + end + protected :set_scheme + + # + # == Args + # + # +v+:: + # String + # + # == Description + # + # Public setter for the scheme component +v+ + # (with validation). + # + # See also Gem::URI::Generic.check_scheme. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse("http://my.example.com") + # uri.scheme = "https" + # uri.to_s #=> "https://my.example.com" + # + def scheme=(v) + check_scheme(v) + set_scheme(v) + v + end + + # + # Checks the +user+ and +password+. + # + # If +password+ is not provided, then +user+ is + # split, using Gem::URI::Generic.split_userinfo, to + # pull +user+ and +password. + # + # See also Gem::URI::Generic.check_user, Gem::URI::Generic.check_password. + # + def check_userinfo(user, password = nil) + if !password + user, password = split_userinfo(user) + end + check_user(user) + check_password(password, user) + + return true + end + private :check_userinfo + + # + # Checks the user +v+ component for RFC2396 compliance + # and against the Gem::URI::Parser Regexp for :USERINFO. + # + # Can not have a registry or opaque component defined, + # with a user component defined. + # + def check_user(v) + if @opaque + raise InvalidURIError, + "can not set user with opaque" + end + + return v unless v + + if parser.regexp[:USERINFO] !~ v + raise InvalidComponentError, + "bad component(expected userinfo component or user component): #{v}" + end + + return true + end + private :check_user + + # + # Checks the password +v+ component for RFC2396 compliance + # and against the Gem::URI::Parser Regexp for :USERINFO. + # + # Can not have a registry or opaque component defined, + # with a user component defined. + # + def check_password(v, user = @user) + if @opaque + raise InvalidURIError, + "can not set password with opaque" + end + return v unless v + + if !user + raise InvalidURIError, + "password component depends user component" + end + + if parser.regexp[:USERINFO] !~ v + raise InvalidComponentError, + "bad password component" + end + + return true + end + private :check_password + + # + # Sets userinfo, argument is string like 'name:pass'. + # + def userinfo=(userinfo) + if userinfo.nil? + return nil + end + check_userinfo(*userinfo) + set_userinfo(*userinfo) + # returns userinfo + end + + # + # == Args + # + # +v+:: + # String + # + # == Description + # + # Public setter for the +user+ component + # (with validation). + # + # See also Gem::URI::Generic.check_user. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse("http://john:S3nsit1ve@my.example.com") + # uri.user = "sam" + # uri.to_s #=> "http://sam:V3ry_S3nsit1ve@my.example.com" + # + def user=(user) + check_user(user) + set_user(user) + # returns user + end + + # + # == Args + # + # +v+:: + # String + # + # == Description + # + # Public setter for the +password+ component + # (with validation). + # + # See also Gem::URI::Generic.check_password. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse("http://john:S3nsit1ve@my.example.com") + # uri.password = "V3ry_S3nsit1ve" + # uri.to_s #=> "http://john:V3ry_S3nsit1ve@my.example.com" + # + def password=(password) + check_password(password) + set_password(password) + # returns password + end + + # Protected setter for the +user+ component, and +password+ if available + # (with validation). + # + # See also Gem::URI::Generic.userinfo=. + # + def set_userinfo(user, password = nil) + unless password + user, password = split_userinfo(user) + end + @user = user + @password = password if password + + [@user, @password] + end + protected :set_userinfo + + # Protected setter for the user component +v+. + # + # See also Gem::URI::Generic.user=. + # + def set_user(v) + set_userinfo(v, @password) + v + end + protected :set_user + + # Protected setter for the password component +v+. + # + # See also Gem::URI::Generic.password=. + # + def set_password(v) + @password = v + # returns v + end + protected :set_password + + # Returns the userinfo +ui+ as <code>[user, password]</code> + # if properly formatted as 'user:password'. + def split_userinfo(ui) + return nil, nil unless ui + user, password = ui.split(':', 2) + + return user, password + end + private :split_userinfo + + # Escapes 'user:password' +v+ based on RFC 1738 section 3.1. + def escape_userpass(v) + parser.escape(v, /[@:\/]/o) # RFC 1738 section 3.1 #/ + end + private :escape_userpass + + # Returns the userinfo, either as 'user' or 'user:password'. + def userinfo + if @user.nil? + nil + elsif @password.nil? + @user + else + @user + ':' + @password + end + end + + # Returns the user component (without Gem::URI decoding). + def user + @user + end + + # Returns the password component (without Gem::URI decoding). + def password + @password + end + + # Returns the user component after Gem::URI decoding. + def decoded_user + Gem::URI.decode_uri_component(@user) if @user + end + + # Returns the password component after Gem::URI decoding. + def decoded_password + Gem::URI.decode_uri_component(@password) if @password + end + + # + # Checks the host +v+ component for RFC2396 compliance + # and against the Gem::URI::Parser Regexp for :HOST. + # + # Can not have a registry or opaque component defined, + # with a host component defined. + # + def check_host(v) + return v unless v + + if @opaque + raise InvalidURIError, + "can not set host with registry or opaque" + elsif parser.regexp[:HOST] !~ v + raise InvalidComponentError, + "bad component(expected host component): #{v}" + end + + return true + end + private :check_host + + # Protected setter for the host component +v+. + # + # See also Gem::URI::Generic.host=. + # + def set_host(v) + @host = v + end + protected :set_host + + # + # == Args + # + # +v+:: + # String + # + # == Description + # + # Public setter for the host component +v+ + # (with validation). + # + # See also Gem::URI::Generic.check_host. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse("http://my.example.com") + # uri.host = "foo.com" + # uri.to_s #=> "http://foo.com" + # + def host=(v) + check_host(v) + set_host(v) + v + end + + # Extract the host part of the Gem::URI and unwrap brackets for IPv6 addresses. + # + # This method is the same as Gem::URI::Generic#host except + # brackets for IPv6 (and future IP) addresses are removed. + # + # uri = Gem::URI("http://[::1]/bar") + # uri.hostname #=> "::1" + # uri.host #=> "[::1]" + # + def hostname + v = self.host + v&.start_with?('[') && v.end_with?(']') ? v[1..-2] : v + end + + # Sets the host part of the Gem::URI as the argument with brackets for IPv6 addresses. + # + # This method is the same as Gem::URI::Generic#host= except + # the argument can be a bare IPv6 address. + # + # uri = Gem::URI("http://foo/bar") + # uri.hostname = "::1" + # uri.to_s #=> "http://[::1]/bar" + # + # If the argument seems to be an IPv6 address, + # it is wrapped with brackets. + # + def hostname=(v) + v = "[#{v}]" if !(v&.start_with?('[') && v&.end_with?(']')) && v&.index(':') + self.host = v + end + + # + # Checks the port +v+ component for RFC2396 compliance + # and against the Gem::URI::Parser Regexp for :PORT. + # + # Can not have a registry or opaque component defined, + # with a port component defined. + # + def check_port(v) + return v unless v + + if @opaque + raise InvalidURIError, + "can not set port with registry or opaque" + elsif !v.kind_of?(Integer) && parser.regexp[:PORT] !~ v + raise InvalidComponentError, + "bad component(expected port component): #{v.inspect}" + end + + return true + end + private :check_port + + # Protected setter for the port component +v+. + # + # See also Gem::URI::Generic.port=. + # + def set_port(v) + v = v.empty? ? nil : v.to_i unless !v || v.kind_of?(Integer) + @port = v + end + protected :set_port + + # + # == Args + # + # +v+:: + # String + # + # == Description + # + # Public setter for the port component +v+ + # (with validation). + # + # See also Gem::URI::Generic.check_port. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse("http://my.example.com") + # uri.port = 8080 + # uri.to_s #=> "http://my.example.com:8080" + # + def port=(v) + check_port(v) + set_port(v) + port + end + + def check_registry(v) # :nodoc: + raise InvalidURIError, "can not set registry" + end + private :check_registry + + def set_registry(v) #:nodoc: + raise InvalidURIError, "can not set registry" + end + protected :set_registry + + def registry=(v) + raise InvalidURIError, "can not set registry" + end + + # + # Checks the path +v+ component for RFC2396 compliance + # and against the Gem::URI::Parser Regexp + # for :ABS_PATH and :REL_PATH. + # + # Can not have a opaque component defined, + # with a path component defined. + # + def check_path(v) + # raise if both hier and opaque are not nil, because: + # absoluteURI = scheme ":" ( hier_part | opaque_part ) + # hier_part = ( net_path | abs_path ) [ "?" query ] + if v && @opaque + raise InvalidURIError, + "path conflicts with opaque" + end + + # If scheme is ftp, path may be relative. + # See RFC 1738 section 3.2.2, and RFC 2396. + if @scheme && @scheme != "ftp" + if v && v != '' && parser.regexp[:ABS_PATH] !~ v + raise InvalidComponentError, + "bad component(expected absolute path component): #{v}" + end + else + if v && v != '' && parser.regexp[:ABS_PATH] !~ v && + parser.regexp[:REL_PATH] !~ v + raise InvalidComponentError, + "bad component(expected relative path component): #{v}" + end + end + + return true + end + private :check_path + + # Protected setter for the path component +v+. + # + # See also Gem::URI::Generic.path=. + # + def set_path(v) + @path = v + end + protected :set_path + + # + # == Args + # + # +v+:: + # String + # + # == Description + # + # Public setter for the path component +v+ + # (with validation). + # + # See also Gem::URI::Generic.check_path. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse("http://my.example.com/pub/files") + # uri.path = "/faq/" + # uri.to_s #=> "http://my.example.com/faq/" + # + def path=(v) + check_path(v) + set_path(v) + v + end + + # + # == Args + # + # +v+:: + # String + # + # == Description + # + # Public setter for the query component +v+. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse("http://my.example.com/?id=25") + # uri.query = "id=1" + # uri.to_s #=> "http://my.example.com/?id=1" + # + def query=(v) + return @query = nil unless v + raise InvalidURIError, "query conflicts with opaque" if @opaque + + x = v.to_str + v = x.dup if x.equal? v + v.encode!(Encoding::UTF_8) rescue nil + v.delete!("\t\r\n") + v.force_encoding(Encoding::ASCII_8BIT) + raise InvalidURIError, "invalid percent escape: #{$1}" if /(%\H\H)/n.match(v) + v.gsub!(/(?!%\h\h|[!$-&(-;=?-_a-~])./n.freeze){'%%%02X' % $&.ord} + v.force_encoding(Encoding::US_ASCII) + @query = v + end + + # + # Checks the opaque +v+ component for RFC2396 compliance and + # against the Gem::URI::Parser Regexp for :OPAQUE. + # + # Can not have a host, port, user, or path component defined, + # with an opaque component defined. + # + def check_opaque(v) + return v unless v + + # raise if both hier and opaque are not nil, because: + # absoluteURI = scheme ":" ( hier_part | opaque_part ) + # hier_part = ( net_path | abs_path ) [ "?" query ] + if @host || @port || @user || @path # userinfo = @user + ':' + @password + raise InvalidURIError, + "can not set opaque with host, port, userinfo or path" + elsif v && parser.regexp[:OPAQUE] !~ v + raise InvalidComponentError, + "bad component(expected opaque component): #{v}" + end + + return true + end + private :check_opaque + + # Protected setter for the opaque component +v+. + # + # See also Gem::URI::Generic.opaque=. + # + def set_opaque(v) + @opaque = v + end + protected :set_opaque + + # + # == Args + # + # +v+:: + # String + # + # == Description + # + # Public setter for the opaque component +v+ + # (with validation). + # + # See also Gem::URI::Generic.check_opaque. + # + def opaque=(v) + check_opaque(v) + set_opaque(v) + v + end + + # + # Checks the fragment +v+ component against the Gem::URI::Parser Regexp for :FRAGMENT. + # + # + # == Args + # + # +v+:: + # String + # + # == Description + # + # Public setter for the fragment component +v+ + # (with validation). + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse("http://my.example.com/?id=25#time=1305212049") + # uri.fragment = "time=1305212086" + # uri.to_s #=> "http://my.example.com/?id=25#time=1305212086" + # + def fragment=(v) + return @fragment = nil unless v + + x = v.to_str + v = x.dup if x.equal? v + v.encode!(Encoding::UTF_8) rescue nil + v.delete!("\t\r\n") + v.force_encoding(Encoding::ASCII_8BIT) + v.gsub!(/(?!%\h\h|[!-~])./n){'%%%02X' % $&.ord} + v.force_encoding(Encoding::US_ASCII) + @fragment = v + end + + # + # Returns true if Gem::URI is hierarchical. + # + # == Description + # + # Gem::URI has components listed in order of decreasing significance from left to right, + # see RFC3986 https://tools.ietf.org/html/rfc3986 1.2.3. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse("http://my.example.com/") + # uri.hierarchical? + # #=> true + # uri = Gem::URI.parse("mailto:joe@example.com") + # uri.hierarchical? + # #=> false + # + def hierarchical? + if @path + true + else + false + end + end + + # + # Returns true if Gem::URI has a scheme (e.g. http:// or https://) specified. + # + def absolute? + if @scheme + true + else + false + end + end + alias absolute absolute? + + # + # Returns true if Gem::URI does not have a scheme (e.g. http:// or https://) specified. + # + def relative? + !absolute? + end + + # + # Returns an Array of the path split on '/'. + # + def split_path(path) + path.split("/", -1) + end + private :split_path + + # + # Merges a base path +base+, with relative path +rel+, + # returns a modified base path. + # + def merge_path(base, rel) + + # RFC2396, Section 5.2, 5) + # RFC2396, Section 5.2, 6) + base_path = split_path(base) + rel_path = split_path(rel) + + # RFC2396, Section 5.2, 6), a) + base_path << '' if base_path.last == '..' + while i = base_path.index('..') + base_path.slice!(i - 1, 2) + end + + if (first = rel_path.first) and first.empty? + base_path.clear + rel_path.shift + end + + # RFC2396, Section 5.2, 6), c) + # RFC2396, Section 5.2, 6), d) + rel_path.push('') if rel_path.last == '.' || rel_path.last == '..' + rel_path.delete('.') + + # RFC2396, Section 5.2, 6), e) + tmp = [] + rel_path.each do |x| + if x == '..' && + !(tmp.empty? || tmp.last == '..') + tmp.pop + else + tmp << x + end + end + + add_trailer_slash = !tmp.empty? + if base_path.empty? + base_path = [''] # keep '/' for root directory + elsif add_trailer_slash + base_path.pop + end + while x = tmp.shift + if x == '..' + # RFC2396, Section 4 + # a .. or . in an absolute path has no special meaning + base_path.pop if base_path.size > 1 + else + # if x == '..' + # valid absolute (but abnormal) path "/../..." + # else + # valid absolute path + # end + base_path << x + tmp.each {|t| base_path << t} + add_trailer_slash = false + break + end + end + base_path.push('') if add_trailer_slash + + return base_path.join('/') + end + private :merge_path + + # + # == Args + # + # +oth+:: + # Gem::URI or String + # + # == Description + # + # Destructive form of #merge. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse("http://my.example.com") + # uri.merge!("/main.rbx?page=1") + # uri.to_s # => "http://my.example.com/main.rbx?page=1" + # + def merge!(oth) + t = merge(oth) + if self == t + nil + else + replace!(t) + self + end + end + + # + # == Args + # + # +oth+:: + # Gem::URI or String + # + # == Description + # + # Merges two URIs. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse("http://my.example.com") + # uri.merge("/main.rbx?page=1") + # # => "http://my.example.com/main.rbx?page=1" + # + def merge(oth) + rel = parser.__send__(:convert_to_uri, oth) + + if rel.absolute? + #raise BadURIError, "both Gem::URI are absolute" if absolute? + # hmm... should return oth for usability? + return rel + end + + unless self.absolute? + raise BadURIError, "both Gem::URI are relative" + end + + base = self.dup + + authority = rel.userinfo || rel.host || rel.port + + # RFC2396, Section 5.2, 2) + if (rel.path.nil? || rel.path.empty?) && !authority && !rel.query + base.fragment=(rel.fragment) if rel.fragment + return base + end + + base.query = nil + base.fragment=(nil) + + # RFC2396, Section 5.2, 4) + if !authority + base.set_path(merge_path(base.path, rel.path)) if base.path && rel.path + else + # RFC2396, Section 5.2, 4) + base.set_path(rel.path) if rel.path + end + + # RFC2396, Section 5.2, 7) + base.set_userinfo(rel.userinfo) if rel.userinfo + base.set_host(rel.host) if rel.host + base.set_port(rel.port) if rel.port + base.query = rel.query if rel.query + base.fragment=(rel.fragment) if rel.fragment + + return base + end # merge + alias + merge + + # :stopdoc: + def route_from_path(src, dst) + case dst + when src + # RFC2396, Section 4.2 + return '' + when %r{(?:\A|/)\.\.?(?:/|\z)} + # dst has abnormal absolute path, + # like "/./", "/../", "/x/../", ... + return dst.dup + end + + src_path = src.scan(%r{[^/]*/}) + dst_path = dst.scan(%r{[^/]*/?}) + + # discard same parts + while !dst_path.empty? && dst_path.first == src_path.first + src_path.shift + dst_path.shift + end + + tmp = dst_path.join + + # calculate + if src_path.empty? + if tmp.empty? + return './' + elsif dst_path.first.include?(':') # (see RFC2396 Section 5) + return './' + tmp + else + return tmp + end + end + + return '../' * src_path.size + tmp + end + private :route_from_path + # :startdoc: + + # :stopdoc: + def route_from0(oth) + oth = parser.__send__(:convert_to_uri, oth) + if self.relative? + raise BadURIError, + "relative Gem::URI: #{self}" + end + if oth.relative? + raise BadURIError, + "relative Gem::URI: #{oth}" + end + + if self.scheme != oth.scheme + return self, self.dup + end + rel = Gem::URI::Generic.new(nil, # it is relative Gem::URI + self.userinfo, self.host, self.port, + nil, self.path, self.opaque, + self.query, self.fragment, parser) + + if rel.userinfo != oth.userinfo || + rel.host.to_s.downcase != oth.host.to_s.downcase || + rel.port != oth.port + + if self.userinfo.nil? && self.host.nil? + return self, self.dup + end + + rel.set_port(nil) if rel.port == oth.default_port + return rel, rel + end + rel.set_userinfo(nil) + rel.set_host(nil) + rel.set_port(nil) + + if rel.path && rel.path == oth.path + rel.set_path('') + rel.query = nil if rel.query == oth.query + return rel, rel + elsif rel.opaque && rel.opaque == oth.opaque + rel.set_opaque('') + rel.query = nil if rel.query == oth.query + return rel, rel + end + + # you can modify `rel', but can not `oth'. + return oth, rel + end + private :route_from0 + # :startdoc: + + # + # == Args + # + # +oth+:: + # Gem::URI or String + # + # == Description + # + # Calculates relative path from oth to self. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse('http://my.example.com/main.rbx?page=1') + # uri.route_from('http://my.example.com') + # #=> #<Gem::URI::Generic /main.rbx?page=1> + # + def route_from(oth) + # you can modify `rel', but can not `oth'. + begin + oth, rel = route_from0(oth) + rescue + raise $!.class, $!.message + end + if oth == rel + return rel + end + + rel.set_path(route_from_path(oth.path, self.path)) + if rel.path == './' && self.query + # "./?foo" -> "?foo" + rel.set_path('') + end + + return rel + end + + alias - route_from + + # + # == Args + # + # +oth+:: + # Gem::URI or String + # + # == Description + # + # Calculates relative path to oth from self. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse('http://my.example.com') + # uri.route_to('http://my.example.com/main.rbx?page=1') + # #=> #<Gem::URI::Generic /main.rbx?page=1> + # + def route_to(oth) + parser.__send__(:convert_to_uri, oth).route_from(self) + end + + # + # Returns normalized Gem::URI. + # + # require 'rubygems/vendor/uri/lib/uri' + # + # Gem::URI("HTTP://my.EXAMPLE.com").normalize + # #=> #<Gem::URI::HTTP http://my.example.com/> + # + # Normalization here means: + # + # * scheme and host are converted to lowercase, + # * an empty path component is set to "/". + # + def normalize + uri = dup + uri.normalize! + uri + end + + # + # Destructive version of #normalize. + # + def normalize! + if path&.empty? + set_path('/') + end + if scheme && scheme != scheme.downcase + set_scheme(self.scheme.downcase) + end + if host && host != host.downcase + set_host(self.host.downcase) + end + end + + # + # Constructs String from Gem::URI. + # + def to_s + str = ''.dup + if @scheme + str << @scheme + str << ':' + end + + if @opaque + str << @opaque + else + if @host || %w[file postgres].include?(@scheme) + str << '//' + end + if self.userinfo + str << self.userinfo + str << '@' + end + if @host + str << @host + end + if @port && @port != self.default_port + str << ':' + str << @port.to_s + end + str << @path + if @query + str << '?' + str << @query + end + end + if @fragment + str << '#' + str << @fragment + end + str + end + alias to_str to_s + + # + # Compares two URIs. + # + def ==(oth) + if self.class == oth.class + self.normalize.component_ary == oth.normalize.component_ary + else + false + end + end + + def hash + self.component_ary.hash + end + + def eql?(oth) + self.class == oth.class && + parser == oth.parser && + self.component_ary.eql?(oth.component_ary) + end + +=begin + +--- Gem::URI::Generic#===(oth) + +=end +# def ===(oth) +# raise NotImplementedError +# end + +=begin +=end + + + # Returns an Array of the components defined from the COMPONENT Array. + def component_ary + component.collect do |x| + self.__send__(x) + end + end + protected :component_ary + + # == Args + # + # +components+:: + # Multiple Symbol arguments defined in Gem::URI::HTTP. + # + # == Description + # + # Selects specified components from Gem::URI. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse('http://myuser:mypass@my.example.com/test.rbx') + # uri.select(:userinfo, :host, :path) + # # => ["myuser:mypass", "my.example.com", "/test.rbx"] + # + def select(*components) + components.collect do |c| + if component.include?(c) + self.__send__(c) + else + raise ArgumentError, + "expected of components of #{self.class} (#{self.class.component.join(', ')})" + end + end + end + + def inspect + "#<#{self.class} #{self}>" + end + + # + # == Args + # + # +v+:: + # Gem::URI or String + # + # == Description + # + # Attempts to parse other Gem::URI +oth+, + # returns [parsed_oth, self]. + # + # == Usage + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse("http://my.example.com") + # uri.coerce("http://foo.com") + # #=> [#<Gem::URI::HTTP http://foo.com>, #<Gem::URI::HTTP http://my.example.com>] + # + def coerce(oth) + case oth + when String + oth = parser.parse(oth) + else + super + end + + return oth, self + end + + # Returns a proxy Gem::URI. + # The proxy Gem::URI is obtained from environment variables such as http_proxy, + # ftp_proxy, no_proxy, etc. + # If there is no proper proxy, nil is returned. + # + # If the optional parameter +env+ is specified, it is used instead of ENV. + # + # Note that capitalized variables (HTTP_PROXY, FTP_PROXY, NO_PROXY, etc.) + # are examined, too. + # + # But http_proxy and HTTP_PROXY is treated specially under CGI environment. + # It's because HTTP_PROXY may be set by Proxy: header. + # So HTTP_PROXY is not used. + # http_proxy is not used too if the variable is case insensitive. + # CGI_HTTP_PROXY can be used instead. + def find_proxy(env=ENV) + raise BadURIError, "relative Gem::URI: #{self}" if self.relative? + name = self.scheme.downcase + '_proxy' + proxy_uri = nil + if name == 'http_proxy' && env.include?('REQUEST_METHOD') # CGI? + # HTTP_PROXY conflicts with *_proxy for proxy settings and + # HTTP_* for header information in CGI. + # So it should be careful to use it. + pairs = env.reject {|k, v| /\Ahttp_proxy\z/i !~ k } + case pairs.length + when 0 # no proxy setting anyway. + proxy_uri = nil + when 1 + k, _ = pairs.shift + if k == 'http_proxy' && env[k.upcase] == nil + # http_proxy is safe to use because ENV is case sensitive. + proxy_uri = env[name] + else + proxy_uri = nil + end + else # http_proxy is safe to use because ENV is case sensitive. + proxy_uri = env.to_hash[name] + end + if !proxy_uri + # Use CGI_HTTP_PROXY. cf. libwww-perl. + proxy_uri = env["CGI_#{name.upcase}"] + end + elsif name == 'http_proxy' + if RUBY_ENGINE == 'jruby' && p_addr = ENV_JAVA['http.proxyHost'] + p_port = ENV_JAVA['http.proxyPort'] + if p_user = ENV_JAVA['http.proxyUser'] + p_pass = ENV_JAVA['http.proxyPass'] + proxy_uri = "http://#{p_user}:#{p_pass}@#{p_addr}:#{p_port}" + else + proxy_uri = "http://#{p_addr}:#{p_port}" + end + else + unless proxy_uri = env[name] + if proxy_uri = env[name.upcase] + warn 'The environment variable HTTP_PROXY is discouraged. Use http_proxy.', uplevel: 1 + end + end + end + else + proxy_uri = env[name] || env[name.upcase] + end + + if proxy_uri.nil? || proxy_uri.empty? + return nil + end + + if self.hostname + begin + addr = IPSocket.getaddress(self.hostname) + return nil if /\A127\.|\A::1\z/ =~ addr + rescue SocketError + end + end + + name = 'no_proxy' + if no_proxy = env[name] || env[name.upcase] + return nil unless Gem::URI::Generic.use_proxy?(self.hostname, addr, self.port, no_proxy) + end + Gem::URI.parse(proxy_uri) + end + + def self.use_proxy?(hostname, addr, port, no_proxy) # :nodoc: + hostname = hostname.downcase + dothostname = ".#{hostname}" + no_proxy.scan(/([^:,\s]+)(?::(\d+))?/) {|p_host, p_port| + if !p_port || port == p_port.to_i + if p_host.start_with?('.') + return false if hostname.end_with?(p_host.downcase) + else + return false if dothostname.end_with?(".#{p_host.downcase}") + end + if addr + begin + return false if IPAddr.new(p_host).include?(addr) + rescue IPAddr::InvalidAddressError + next + end + end + end + } + true + end + end +end diff --git a/lib/rubygems/vendor/uri/lib/uri/http.rb b/lib/rubygems/vendor/uri/lib/uri/http.rb new file mode 100644 index 0000000000..bef43490a3 --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri/http.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: false +# = uri/http.rb +# +# Author:: Akira Yamada <akira@ruby-lang.org> +# License:: You can redistribute it and/or modify it under the same term as Ruby. +# +# See Gem::URI for general documentation +# + +require_relative 'generic' + +module Gem::URI + + # + # The syntax of HTTP URIs is defined in RFC1738 section 3.3. + # + # Note that the Ruby Gem::URI library allows HTTP URLs containing usernames and + # passwords. This is not legal as per the RFC, but used to be + # supported in Internet Explorer 5 and 6, before the MS04-004 security + # update. See <URL:http://support.microsoft.com/kb/834489>. + # + class HTTP < Generic + # A Default port of 80 for Gem::URI::HTTP. + DEFAULT_PORT = 80 + + # An Array of the available components for Gem::URI::HTTP. + COMPONENT = %i[ + scheme + userinfo host port + path + query + fragment + ].freeze + + # + # == Description + # + # Creates a new Gem::URI::HTTP object from components, with syntax checking. + # + # The components accepted are userinfo, host, port, path, query, and + # fragment. + # + # The components should be provided either as an Array, or as a Hash + # with keys formed by preceding the component names with a colon. + # + # If an Array is used, the components must be passed in the + # order <code>[userinfo, host, port, path, query, fragment]</code>. + # + # Example: + # + # uri = Gem::URI::HTTP.build(host: 'www.example.com', path: '/foo/bar') + # + # uri = Gem::URI::HTTP.build([nil, "www.example.com", nil, "/path", + # "query", 'fragment']) + # + # Currently, if passed userinfo components this method generates + # invalid HTTP URIs as per RFC 1738. + # + def self.build(args) + tmp = Util.make_components_hash(self, args) + super(tmp) + end + + # + # == Description + # + # Returns the full path for an HTTP request, as required by Net::HTTP::Get. + # + # If the Gem::URI contains a query, the full path is Gem::URI#path + '?' + Gem::URI#query. + # Otherwise, the path is simply Gem::URI#path. + # + # Example: + # + # uri = Gem::URI::HTTP.build(path: '/foo/bar', query: 'test=true') + # uri.request_uri # => "/foo/bar?test=true" + # + def request_uri + return unless @path + + url = @query ? "#@path?#@query" : @path.dup + url.start_with?(?/.freeze) ? url : ?/ + url + end + + # + # == Description + # + # Returns the authority for an HTTP uri, as defined in + # https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2. + # + # + # Example: + # + # Gem::URI::HTTP.build(host: 'www.example.com', path: '/foo/bar').authority #=> "www.example.com" + # Gem::URI::HTTP.build(host: 'www.example.com', port: 8000, path: '/foo/bar').authority #=> "www.example.com:8000" + # Gem::URI::HTTP.build(host: 'www.example.com', port: 80, path: '/foo/bar').authority #=> "www.example.com" + # + def authority + if port == default_port + host + else + "#{host}:#{port}" + end + end + + # + # == Description + # + # Returns the origin for an HTTP uri, as defined in + # https://datatracker.ietf.org/doc/html/rfc6454. + # + # + # Example: + # + # Gem::URI::HTTP.build(host: 'www.example.com', path: '/foo/bar').origin #=> "http://www.example.com" + # Gem::URI::HTTP.build(host: 'www.example.com', port: 8000, path: '/foo/bar').origin #=> "http://www.example.com:8000" + # Gem::URI::HTTP.build(host: 'www.example.com', port: 80, path: '/foo/bar').origin #=> "http://www.example.com" + # Gem::URI::HTTPS.build(host: 'www.example.com', path: '/foo/bar').origin #=> "https://www.example.com" + # + def origin + "#{scheme}://#{authority}" + end + end + + register_scheme 'HTTP', HTTP +end diff --git a/lib/rubygems/vendor/uri/lib/uri/https.rb b/lib/rubygems/vendor/uri/lib/uri/https.rb new file mode 100644 index 0000000000..6e8e732e1d --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri/https.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: false +# = uri/https.rb +# +# Author:: Akira Yamada <akira@ruby-lang.org> +# License:: You can redistribute it and/or modify it under the same term as Ruby. +# +# See Gem::URI for general documentation +# + +require_relative 'http' + +module Gem::URI + + # The default port for HTTPS URIs is 443, and the scheme is 'https:' rather + # than 'http:'. Other than that, HTTPS URIs are identical to HTTP URIs; + # see Gem::URI::HTTP. + class HTTPS < HTTP + # A Default port of 443 for Gem::URI::HTTPS + DEFAULT_PORT = 443 + end + + register_scheme 'HTTPS', HTTPS +end diff --git a/lib/rubygems/vendor/uri/lib/uri/ldap.rb b/lib/rubygems/vendor/uri/lib/uri/ldap.rb new file mode 100644 index 0000000000..1a08b5ab7e --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri/ldap.rb @@ -0,0 +1,261 @@ +# frozen_string_literal: false +# = uri/ldap.rb +# +# Author:: +# Takaaki Tateishi <ttate@jaist.ac.jp> +# Akira Yamada <akira@ruby-lang.org> +# License:: +# Gem::URI::LDAP is copyrighted free software by Takaaki Tateishi and Akira Yamada. +# You can redistribute it and/or modify it under the same term as Ruby. +# +# See Gem::URI for general documentation +# + +require_relative 'generic' + +module Gem::URI + + # + # LDAP Gem::URI SCHEMA (described in RFC2255). + #-- + # ldap://<host>/<dn>[?<attrs>[?<scope>[?<filter>[?<extensions>]]]] + #++ + class LDAP < Generic + + # A Default port of 389 for Gem::URI::LDAP. + DEFAULT_PORT = 389 + + # An Array of the available components for Gem::URI::LDAP. + COMPONENT = [ + :scheme, + :host, :port, + :dn, + :attributes, + :scope, + :filter, + :extensions, + ].freeze + + # Scopes available for the starting point. + # + # * SCOPE_BASE - the Base DN + # * SCOPE_ONE - one level under the Base DN, not including the base DN and + # not including any entries under this + # * SCOPE_SUB - subtrees, all entries at all levels + # + SCOPE = [ + SCOPE_ONE = 'one', + SCOPE_SUB = 'sub', + SCOPE_BASE = 'base', + ].freeze + + # + # == Description + # + # Creates a new Gem::URI::LDAP object from components, with syntax checking. + # + # The components accepted are host, port, dn, attributes, + # scope, filter, and extensions. + # + # The components should be provided either as an Array, or as a Hash + # with keys formed by preceding the component names with a colon. + # + # If an Array is used, the components must be passed in the + # order <code>[host, port, dn, attributes, scope, filter, extensions]</code>. + # + # Example: + # + # uri = Gem::URI::LDAP.build({:host => 'ldap.example.com', + # :dn => '/dc=example'}) + # + # uri = Gem::URI::LDAP.build(["ldap.example.com", nil, + # "/dc=example;dc=com", "query", nil, nil, nil]) + # + def self.build(args) + tmp = Util::make_components_hash(self, args) + + if tmp[:dn] + tmp[:path] = tmp[:dn] + end + + query = [] + [:extensions, :filter, :scope, :attributes].collect do |x| + next if !tmp[x] && query.size == 0 + query.unshift(tmp[x]) + end + + tmp[:query] = query.join('?') + + return super(tmp) + end + + # + # == Description + # + # Creates a new Gem::URI::LDAP object from generic Gem::URI components as per + # RFC 2396. No LDAP-specific syntax checking is performed. + # + # Arguments are +scheme+, +userinfo+, +host+, +port+, +registry+, +path+, + # +opaque+, +query+, and +fragment+, in that order. + # + # Example: + # + # uri = Gem::URI::LDAP.new("ldap", nil, "ldap.example.com", nil, nil, + # "/dc=example;dc=com", nil, "query", nil) + # + # See also Gem::URI::Generic.new. + # + def initialize(*arg) + super(*arg) + + if @fragment + raise InvalidURIError, 'bad LDAP URL' + end + + parse_dn + parse_query + end + + # Private method to cleanup +dn+ from using the +path+ component attribute. + def parse_dn + raise InvalidURIError, 'bad LDAP URL' unless @path + @dn = @path[1..-1] + end + private :parse_dn + + # Private method to cleanup +attributes+, +scope+, +filter+, and +extensions+ + # from using the +query+ component attribute. + def parse_query + @attributes = nil + @scope = nil + @filter = nil + @extensions = nil + + if @query + attrs, scope, filter, extensions = @query.split('?') + + @attributes = attrs if attrs && attrs.size > 0 + @scope = scope if scope && scope.size > 0 + @filter = filter if filter && filter.size > 0 + @extensions = extensions if extensions && extensions.size > 0 + end + end + private :parse_query + + # Private method to assemble +query+ from +attributes+, +scope+, +filter+, and +extensions+. + def build_path_query + @path = '/' + @dn + + query = [] + [@extensions, @filter, @scope, @attributes].each do |x| + next if !x && query.size == 0 + query.unshift(x) + end + @query = query.join('?') + end + private :build_path_query + + # Returns dn. + def dn + @dn + end + + # Private setter for dn +val+. + def set_dn(val) + @dn = val + build_path_query + @dn + end + protected :set_dn + + # Setter for dn +val+. + def dn=(val) + set_dn(val) + val + end + + # Returns attributes. + def attributes + @attributes + end + + # Private setter for attributes +val+. + def set_attributes(val) + @attributes = val + build_path_query + @attributes + end + protected :set_attributes + + # Setter for attributes +val+. + def attributes=(val) + set_attributes(val) + val + end + + # Returns scope. + def scope + @scope + end + + # Private setter for scope +val+. + def set_scope(val) + @scope = val + build_path_query + @scope + end + protected :set_scope + + # Setter for scope +val+. + def scope=(val) + set_scope(val) + val + end + + # Returns filter. + def filter + @filter + end + + # Private setter for filter +val+. + def set_filter(val) + @filter = val + build_path_query + @filter + end + protected :set_filter + + # Setter for filter +val+. + def filter=(val) + set_filter(val) + val + end + + # Returns extensions. + def extensions + @extensions + end + + # Private setter for extensions +val+. + def set_extensions(val) + @extensions = val + build_path_query + @extensions + end + protected :set_extensions + + # Setter for extensions +val+. + def extensions=(val) + set_extensions(val) + val + end + + # Checks if Gem::URI has a path. + # For Gem::URI::LDAP this will return +false+. + def hierarchical? + false + end + end + + register_scheme 'LDAP', LDAP +end diff --git a/lib/rubygems/vendor/uri/lib/uri/ldaps.rb b/lib/rubygems/vendor/uri/lib/uri/ldaps.rb new file mode 100644 index 0000000000..b7a5b50e27 --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri/ldaps.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: false +# = uri/ldap.rb +# +# License:: You can redistribute it and/or modify it under the same term as Ruby. +# +# See Gem::URI for general documentation +# + +require_relative 'ldap' + +module Gem::URI + + # The default port for LDAPS URIs is 636, and the scheme is 'ldaps:' rather + # than 'ldap:'. Other than that, LDAPS URIs are identical to LDAP URIs; + # see Gem::URI::LDAP. + class LDAPS < LDAP + # A Default port of 636 for Gem::URI::LDAPS + DEFAULT_PORT = 636 + end + + register_scheme 'LDAPS', LDAPS +end diff --git a/lib/rubygems/vendor/uri/lib/uri/mailto.rb b/lib/rubygems/vendor/uri/lib/uri/mailto.rb new file mode 100644 index 0000000000..7ae544d194 --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri/mailto.rb @@ -0,0 +1,293 @@ +# frozen_string_literal: false +# = uri/mailto.rb +# +# Author:: Akira Yamada <akira@ruby-lang.org> +# License:: You can redistribute it and/or modify it under the same term as Ruby. +# +# See Gem::URI for general documentation +# + +require_relative 'generic' + +module Gem::URI + + # + # RFC6068, the mailto URL scheme. + # + class MailTo < Generic + include RFC2396_REGEXP + + # A Default port of nil for Gem::URI::MailTo. + DEFAULT_PORT = nil + + # An Array of the available components for Gem::URI::MailTo. + COMPONENT = [ :scheme, :to, :headers ].freeze + + # :stopdoc: + # "hname" and "hvalue" are encodings of an RFC 822 header name and + # value, respectively. As with "to", all URL reserved characters must + # be encoded. + # + # "#mailbox" is as specified in RFC 822 [RFC822]. This means that it + # consists of zero or more comma-separated mail addresses, possibly + # including "phrase" and "comment" components. Note that all URL + # reserved characters in "to" must be encoded: in particular, + # parentheses, commas, and the percent sign ("%"), which commonly occur + # in the "mailbox" syntax. + # + # Within mailto URLs, the characters "?", "=", "&" are reserved. + + # ; RFC 6068 + # hfields = "?" hfield *( "&" hfield ) + # hfield = hfname "=" hfvalue + # hfname = *qchar + # hfvalue = *qchar + # qchar = unreserved / pct-encoded / some-delims + # some-delims = "!" / "$" / "'" / "(" / ")" / "*" + # / "+" / "," / ";" / ":" / "@" + # + # ; RFC3986 + # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + # pct-encoded = "%" HEXDIG HEXDIG + HEADER_REGEXP = /\A(?<hfield>(?:%\h\h|[!$'-.0-;@-Z_a-z~])*=(?:%\h\h|[!$'-.0-;@-Z_a-z~])*)(?:&\g<hfield>)*\z/ + # practical regexp for email address + # https://html.spec.whatwg.org/multipage/input.html#valid-e-mail-address + EMAIL_REGEXP = /\A[a-zA-Z0-9.!\#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*\z/ + # :startdoc: + + # + # == Description + # + # Creates a new Gem::URI::MailTo object from components, with syntax checking. + # + # Components can be provided as an Array or Hash. If an Array is used, + # the components must be supplied as <code>[to, headers]</code>. + # + # If a Hash is used, the keys are the component names preceded by colons. + # + # The headers can be supplied as a pre-encoded string, such as + # <code>"subject=subscribe&cc=address"</code>, or as an Array of Arrays + # like <code>[['subject', 'subscribe'], ['cc', 'address']]</code>. + # + # Examples: + # + # require 'rubygems/vendor/uri/lib/uri' + # + # m1 = Gem::URI::MailTo.build(['joe@example.com', 'subject=Ruby']) + # m1.to_s # => "mailto:joe@example.com?subject=Ruby" + # + # m2 = Gem::URI::MailTo.build(['john@example.com', [['Subject', 'Ruby'], ['Cc', 'jack@example.com']]]) + # m2.to_s # => "mailto:john@example.com?Subject=Ruby&Cc=jack@example.com" + # + # m3 = Gem::URI::MailTo.build({:to => 'listman@example.com', :headers => [['subject', 'subscribe']]}) + # m3.to_s # => "mailto:listman@example.com?subject=subscribe" + # + def self.build(args) + tmp = Util.make_components_hash(self, args) + + case tmp[:to] + when Array + tmp[:opaque] = tmp[:to].join(',') + when String + tmp[:opaque] = tmp[:to].dup + else + tmp[:opaque] = '' + end + + if tmp[:headers] + query = + case tmp[:headers] + when Array + tmp[:headers].collect { |x| + if x.kind_of?(Array) + x[0] + '=' + x[1..-1].join + else + x.to_s + end + }.join('&') + when Hash + tmp[:headers].collect { |h,v| + h + '=' + v + }.join('&') + else + tmp[:headers].to_s + end + unless query.empty? + tmp[:opaque] << '?' << query + end + end + + super(tmp) + end + + # + # == Description + # + # Creates a new Gem::URI::MailTo object from generic URL components with + # no syntax checking. + # + # This method is usually called from Gem::URI::parse, which checks + # the validity of each component. + # + def initialize(*arg) + super(*arg) + + @to = nil + @headers = [] + + # The RFC3986 parser does not normally populate opaque + @opaque = "?#{@query}" if @query && !@opaque + + unless @opaque + raise InvalidComponentError, + "missing opaque part for mailto URL" + end + to, header = @opaque.split('?', 2) + # allow semicolon as a addr-spec separator + # http://support.microsoft.com/kb/820868 + unless /\A(?:[^@,;]+@[^@,;]+(?:\z|[,;]))*\z/ =~ to + raise InvalidComponentError, + "unrecognised opaque part for mailtoURL: #{@opaque}" + end + + if arg[10] # arg_check + self.to = to + self.headers = header + else + set_to(to) + set_headers(header) + end + end + + # The primary e-mail address of the URL, as a String. + attr_reader :to + + # E-mail headers set by the URL, as an Array of Arrays. + attr_reader :headers + + # Checks the to +v+ component. + def check_to(v) + return true unless v + return true if v.size == 0 + + v.split(/[,;]/).each do |addr| + # check url safety as path-rootless + if /\A(?:%\h\h|[!$&-.0-;=@-Z_a-z~])*\z/ !~ addr + raise InvalidComponentError, + "an address in 'to' is invalid as Gem::URI #{addr.dump}" + end + + # check addr-spec + # don't s/\+/ /g + addr.gsub!(/%\h\h/, Gem::URI::TBLDECWWWCOMP_) + if EMAIL_REGEXP !~ addr + raise InvalidComponentError, + "an address in 'to' is invalid as uri-escaped addr-spec #{addr.dump}" + end + end + + true + end + private :check_to + + # Private setter for to +v+. + def set_to(v) + @to = v + end + protected :set_to + + # Setter for to +v+. + def to=(v) + check_to(v) + set_to(v) + v + end + + # Checks the headers +v+ component against either + # * HEADER_REGEXP + def check_headers(v) + return true unless v + return true if v.size == 0 + if HEADER_REGEXP !~ v + raise InvalidComponentError, + "bad component(expected opaque component): #{v}" + end + + true + end + private :check_headers + + # Private setter for headers +v+. + def set_headers(v) + @headers = [] + if v + v.split('&').each do |x| + @headers << x.split(/=/, 2) + end + end + end + protected :set_headers + + # Setter for headers +v+. + def headers=(v) + check_headers(v) + set_headers(v) + v + end + + # Constructs String from Gem::URI. + def to_s + @scheme + ':' + + if @to + @to + else + '' + end + + if @headers.size > 0 + '?' + @headers.collect{|x| x.join('=')}.join('&') + else + '' + end + + if @fragment + '#' + @fragment + else + '' + end + end + + # Returns the RFC822 e-mail text equivalent of the URL, as a String. + # + # Example: + # + # require 'rubygems/vendor/uri/lib/uri' + # + # uri = Gem::URI.parse("mailto:ruby-list@ruby-lang.org?Subject=subscribe&cc=myaddr") + # uri.to_mailtext + # # => "To: ruby-list@ruby-lang.org\nSubject: subscribe\nCc: myaddr\n\n\n" + # + def to_mailtext + to = Gem::URI.decode_www_form_component(@to) + head = '' + body = '' + @headers.each do |x| + case x[0] + when 'body' + body = Gem::URI.decode_www_form_component(x[1]) + when 'to' + to << ', ' + Gem::URI.decode_www_form_component(x[1]) + else + head << Gem::URI.decode_www_form_component(x[0]).capitalize + ': ' + + Gem::URI.decode_www_form_component(x[1]) + "\n" + end + end + + "To: #{to} +#{head} +#{body} +" + end + alias to_rfc822text to_mailtext + end + + register_scheme 'MAILTO', MailTo +end diff --git a/lib/rubygems/vendor/uri/lib/uri/rfc2396_parser.rb b/lib/rubygems/vendor/uri/lib/uri/rfc2396_parser.rb new file mode 100644 index 0000000000..735a269f2c --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri/rfc2396_parser.rb @@ -0,0 +1,539 @@ +# frozen_string_literal: false +#-- +# = uri/common.rb +# +# Author:: Akira Yamada <akira@ruby-lang.org> +# License:: +# You can redistribute it and/or modify it under the same term as Ruby. +# +# See Gem::URI for general documentation +# + +module Gem::URI + # + # Includes Gem::URI::REGEXP::PATTERN + # + module RFC2396_REGEXP + # + # Patterns used to parse Gem::URI's + # + module PATTERN + # :stopdoc: + + # RFC 2396 (Gem::URI Generic Syntax) + # RFC 2732 (IPv6 Literal Addresses in URL's) + # RFC 2373 (IPv6 Addressing Architecture) + + # alpha = lowalpha | upalpha + ALPHA = "a-zA-Z" + # alphanum = alpha | digit + ALNUM = "#{ALPHA}\\d" + + # hex = digit | "A" | "B" | "C" | "D" | "E" | "F" | + # "a" | "b" | "c" | "d" | "e" | "f" + HEX = "a-fA-F\\d" + # escaped = "%" hex hex + ESCAPED = "%[#{HEX}]{2}" + # mark = "-" | "_" | "." | "!" | "~" | "*" | "'" | + # "(" | ")" + # unreserved = alphanum | mark + UNRESERVED = "\\-_.!~*'()#{ALNUM}" + # reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | + # "$" | "," + # reserved = ";" | "/" | "?" | ":" | "@" | "&" | "=" | "+" | + # "$" | "," | "[" | "]" (RFC 2732) + RESERVED = ";/?:@&=+$,\\[\\]" + + # domainlabel = alphanum | alphanum *( alphanum | "-" ) alphanum + DOMLABEL = "(?:[#{ALNUM}](?:[-#{ALNUM}]*[#{ALNUM}])?)" + # toplabel = alpha | alpha *( alphanum | "-" ) alphanum + TOPLABEL = "(?:[#{ALPHA}](?:[-#{ALNUM}]*[#{ALNUM}])?)" + # hostname = *( domainlabel "." ) toplabel [ "." ] + HOSTNAME = "(?:#{DOMLABEL}\\.)*#{TOPLABEL}\\.?" + + # :startdoc: + end # PATTERN + + # :startdoc: + end # REGEXP + + # Class that parses String's into Gem::URI's. + # + # It contains a Hash set of patterns and Regexp's that match and validate. + # + class RFC2396_Parser + include RFC2396_REGEXP + + # + # == Synopsis + # + # Gem::URI::Parser.new([opts]) + # + # == Args + # + # The constructor accepts a hash as options for parser. + # Keys of options are pattern names of Gem::URI components + # and values of options are pattern strings. + # The constructor generates set of regexps for parsing URIs. + # + # You can use the following keys: + # + # * :ESCAPED (Gem::URI::PATTERN::ESCAPED in default) + # * :UNRESERVED (Gem::URI::PATTERN::UNRESERVED in default) + # * :DOMLABEL (Gem::URI::PATTERN::DOMLABEL in default) + # * :TOPLABEL (Gem::URI::PATTERN::TOPLABEL in default) + # * :HOSTNAME (Gem::URI::PATTERN::HOSTNAME in default) + # + # == Examples + # + # p = Gem::URI::Parser.new(:ESCAPED => "(?:%[a-fA-F0-9]{2}|%u[a-fA-F0-9]{4})") + # u = p.parse("http://example.jp/%uABCD") #=> #<Gem::URI::HTTP http://example.jp/%uABCD> + # Gem::URI.parse(u.to_s) #=> raises Gem::URI::InvalidURIError + # + # s = "http://example.com/ABCD" + # u1 = p.parse(s) #=> #<Gem::URI::HTTP http://example.com/ABCD> + # u2 = Gem::URI.parse(s) #=> #<Gem::URI::HTTP http://example.com/ABCD> + # u1 == u2 #=> true + # u1.eql?(u2) #=> false + # + def initialize(opts = {}) + @pattern = initialize_pattern(opts) + @pattern.each_value(&:freeze) + @pattern.freeze + + @regexp = initialize_regexp(@pattern) + @regexp.each_value(&:freeze) + @regexp.freeze + end + + # The Hash of patterns. + # + # See also Gem::URI::Parser.initialize_pattern. + attr_reader :pattern + + # The Hash of Regexp. + # + # See also Gem::URI::Parser.initialize_regexp. + attr_reader :regexp + + # Returns a split Gem::URI against +regexp[:ABS_URI]+. + def split(uri) + case uri + when '' + # null uri + + when @regexp[:ABS_URI] + scheme, opaque, userinfo, host, port, + registry, path, query, fragment = $~[1..-1] + + # Gem::URI-reference = [ absoluteURI | relativeURI ] [ "#" fragment ] + + # absoluteURI = scheme ":" ( hier_part | opaque_part ) + # hier_part = ( net_path | abs_path ) [ "?" query ] + # opaque_part = uric_no_slash *uric + + # abs_path = "/" path_segments + # net_path = "//" authority [ abs_path ] + + # authority = server | reg_name + # server = [ [ userinfo "@" ] hostport ] + + if !scheme + raise InvalidURIError, + "bad Gem::URI(absolute but no scheme): #{uri}" + end + if !opaque && (!path && (!host && !registry)) + raise InvalidURIError, + "bad Gem::URI(absolute but no path): #{uri}" + end + + when @regexp[:REL_URI] + scheme = nil + opaque = nil + + userinfo, host, port, registry, + rel_segment, abs_path, query, fragment = $~[1..-1] + if rel_segment && abs_path + path = rel_segment + abs_path + elsif rel_segment + path = rel_segment + elsif abs_path + path = abs_path + end + + # Gem::URI-reference = [ absoluteURI | relativeURI ] [ "#" fragment ] + + # relativeURI = ( net_path | abs_path | rel_path ) [ "?" query ] + + # net_path = "//" authority [ abs_path ] + # abs_path = "/" path_segments + # rel_path = rel_segment [ abs_path ] + + # authority = server | reg_name + # server = [ [ userinfo "@" ] hostport ] + + else + raise InvalidURIError, "bad Gem::URI(is not Gem::URI?): #{uri}" + end + + path = '' if !path && !opaque # (see RFC2396 Section 5.2) + ret = [ + scheme, + userinfo, host, port, # X + registry, # X + path, # Y + opaque, # Y + query, + fragment + ] + return ret + end + + # + # == Args + # + # +uri+:: + # String + # + # == Description + # + # Parses +uri+ and constructs either matching Gem::URI scheme object + # (File, FTP, HTTP, HTTPS, LDAP, LDAPS, or MailTo) or Gem::URI::Generic. + # + # == Usage + # + # p = Gem::URI::Parser.new + # p.parse("ldap://ldap.example.com/dc=example?user=john") + # #=> #<Gem::URI::LDAP ldap://ldap.example.com/dc=example?user=john> + # + def parse(uri) + Gem::URI.for(*self.split(uri), self) + end + + # + # == Args + # + # +uris+:: + # an Array of Strings + # + # == Description + # + # Attempts to parse and merge a set of URIs. + # + def join(*uris) + uris[0] = convert_to_uri(uris[0]) + uris.inject :merge + end + + # + # :call-seq: + # extract( str ) + # extract( str, schemes ) + # extract( str, schemes ) {|item| block } + # + # == Args + # + # +str+:: + # String to search + # +schemes+:: + # Patterns to apply to +str+ + # + # == Description + # + # Attempts to parse and merge a set of URIs. + # If no +block+ given, then returns the result, + # else it calls +block+ for each element in result. + # + # See also Gem::URI::Parser.make_regexp. + # + def extract(str, schemes = nil) + if block_given? + str.scan(make_regexp(schemes)) { yield $& } + nil + else + result = [] + str.scan(make_regexp(schemes)) { result.push $& } + result + end + end + + # Returns Regexp that is default +self.regexp[:ABS_URI_REF]+, + # unless +schemes+ is provided. Then it is a Regexp.union with +self.pattern[:X_ABS_URI]+. + def make_regexp(schemes = nil) + unless schemes + @regexp[:ABS_URI_REF] + else + /(?=#{Regexp.union(*schemes)}:)#{@pattern[:X_ABS_URI]}/x + end + end + + # + # :call-seq: + # escape( str ) + # escape( str, unsafe ) + # + # == Args + # + # +str+:: + # String to make safe + # +unsafe+:: + # Regexp to apply. Defaults to +self.regexp[:UNSAFE]+ + # + # == Description + # + # Constructs a safe String from +str+, removing unsafe characters, + # replacing them with codes. + # + def escape(str, unsafe = @regexp[:UNSAFE]) + unless unsafe.kind_of?(Regexp) + # perhaps unsafe is String object + unsafe = Regexp.new("[#{Regexp.quote(unsafe)}]", false) + end + str.gsub(unsafe) do + us = $& + tmp = '' + us.each_byte do |uc| + tmp << sprintf('%%%02X', uc) + end + tmp + end.force_encoding(Encoding::US_ASCII) + end + + # + # :call-seq: + # unescape( str ) + # unescape( str, escaped ) + # + # == Args + # + # +str+:: + # String to remove escapes from + # +escaped+:: + # Regexp to apply. Defaults to +self.regexp[:ESCAPED]+ + # + # == Description + # + # Removes escapes from +str+. + # + def unescape(str, escaped = @regexp[:ESCAPED]) + enc = str.encoding + enc = Encoding::UTF_8 if enc == Encoding::US_ASCII + str.gsub(escaped) { [$&[1, 2]].pack('H2').force_encoding(enc) } + end + + @@to_s = Kernel.instance_method(:to_s) + if @@to_s.respond_to?(:bind_call) + def inspect + @@to_s.bind_call(self) + end + else + def inspect + @@to_s.bind(self).call + end + end + + private + + # Constructs the default Hash of patterns. + def initialize_pattern(opts = {}) + ret = {} + ret[:ESCAPED] = escaped = (opts.delete(:ESCAPED) || PATTERN::ESCAPED) + ret[:UNRESERVED] = unreserved = opts.delete(:UNRESERVED) || PATTERN::UNRESERVED + ret[:RESERVED] = reserved = opts.delete(:RESERVED) || PATTERN::RESERVED + ret[:DOMLABEL] = opts.delete(:DOMLABEL) || PATTERN::DOMLABEL + ret[:TOPLABEL] = opts.delete(:TOPLABEL) || PATTERN::TOPLABEL + ret[:HOSTNAME] = hostname = opts.delete(:HOSTNAME) + + # RFC 2396 (Gem::URI Generic Syntax) + # RFC 2732 (IPv6 Literal Addresses in URL's) + # RFC 2373 (IPv6 Addressing Architecture) + + # uric = reserved | unreserved | escaped + ret[:URIC] = uric = "(?:[#{unreserved}#{reserved}]|#{escaped})" + # uric_no_slash = unreserved | escaped | ";" | "?" | ":" | "@" | + # "&" | "=" | "+" | "$" | "," + ret[:URIC_NO_SLASH] = uric_no_slash = "(?:[#{unreserved};?:@&=+$,]|#{escaped})" + # query = *uric + ret[:QUERY] = query = "#{uric}*" + # fragment = *uric + ret[:FRAGMENT] = fragment = "#{uric}*" + + # hostname = *( domainlabel "." ) toplabel [ "." ] + # reg-name = *( unreserved / pct-encoded / sub-delims ) # RFC3986 + unless hostname + ret[:HOSTNAME] = hostname = "(?:[a-zA-Z0-9\\-.]|%\\h\\h)+" + end + + # RFC 2373, APPENDIX B: + # IPv6address = hexpart [ ":" IPv4address ] + # IPv4address = 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT + # hexpart = hexseq | hexseq "::" [ hexseq ] | "::" [ hexseq ] + # hexseq = hex4 *( ":" hex4) + # hex4 = 1*4HEXDIG + # + # XXX: This definition has a flaw. "::" + IPv4address must be + # allowed too. Here is a replacement. + # + # IPv4address = 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT "." 1*3DIGIT + ret[:IPV4ADDR] = ipv4addr = "\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}" + # hex4 = 1*4HEXDIG + hex4 = "[#{PATTERN::HEX}]{1,4}" + # lastpart = hex4 | IPv4address + lastpart = "(?:#{hex4}|#{ipv4addr})" + # hexseq1 = *( hex4 ":" ) hex4 + hexseq1 = "(?:#{hex4}:)*#{hex4}" + # hexseq2 = *( hex4 ":" ) lastpart + hexseq2 = "(?:#{hex4}:)*#{lastpart}" + # IPv6address = hexseq2 | [ hexseq1 ] "::" [ hexseq2 ] + ret[:IPV6ADDR] = ipv6addr = "(?:#{hexseq2}|(?:#{hexseq1})?::(?:#{hexseq2})?)" + + # IPv6prefix = ( hexseq1 | [ hexseq1 ] "::" [ hexseq1 ] ) "/" 1*2DIGIT + # unused + + # ipv6reference = "[" IPv6address "]" (RFC 2732) + ret[:IPV6REF] = ipv6ref = "\\[#{ipv6addr}\\]" + + # host = hostname | IPv4address + # host = hostname | IPv4address | IPv6reference (RFC 2732) + ret[:HOST] = host = "(?:#{hostname}|#{ipv4addr}|#{ipv6ref})" + # port = *digit + ret[:PORT] = port = '\d*' + # hostport = host [ ":" port ] + ret[:HOSTPORT] = hostport = "#{host}(?::#{port})?" + + # userinfo = *( unreserved | escaped | + # ";" | ":" | "&" | "=" | "+" | "$" | "," ) + ret[:USERINFO] = userinfo = "(?:[#{unreserved};:&=+$,]|#{escaped})*" + + # pchar = unreserved | escaped | + # ":" | "@" | "&" | "=" | "+" | "$" | "," + pchar = "(?:[#{unreserved}:@&=+$,]|#{escaped})" + # param = *pchar + param = "#{pchar}*" + # segment = *pchar *( ";" param ) + segment = "#{pchar}*(?:;#{param})*" + # path_segments = segment *( "/" segment ) + ret[:PATH_SEGMENTS] = path_segments = "#{segment}(?:/#{segment})*" + + # server = [ [ userinfo "@" ] hostport ] + server = "(?:#{userinfo}@)?#{hostport}" + # reg_name = 1*( unreserved | escaped | "$" | "," | + # ";" | ":" | "@" | "&" | "=" | "+" ) + ret[:REG_NAME] = reg_name = "(?:[#{unreserved}$,;:@&=+]|#{escaped})+" + # authority = server | reg_name + authority = "(?:#{server}|#{reg_name})" + + # rel_segment = 1*( unreserved | escaped | + # ";" | "@" | "&" | "=" | "+" | "$" | "," ) + ret[:REL_SEGMENT] = rel_segment = "(?:[#{unreserved};@&=+$,]|#{escaped})+" + + # scheme = alpha *( alpha | digit | "+" | "-" | "." ) + ret[:SCHEME] = scheme = "[#{PATTERN::ALPHA}][\\-+.#{PATTERN::ALPHA}\\d]*" + + # abs_path = "/" path_segments + ret[:ABS_PATH] = abs_path = "/#{path_segments}" + # rel_path = rel_segment [ abs_path ] + ret[:REL_PATH] = rel_path = "#{rel_segment}(?:#{abs_path})?" + # net_path = "//" authority [ abs_path ] + ret[:NET_PATH] = net_path = "//#{authority}(?:#{abs_path})?" + + # hier_part = ( net_path | abs_path ) [ "?" query ] + ret[:HIER_PART] = hier_part = "(?:#{net_path}|#{abs_path})(?:\\?(?:#{query}))?" + # opaque_part = uric_no_slash *uric + ret[:OPAQUE_PART] = opaque_part = "#{uric_no_slash}#{uric}*" + + # absoluteURI = scheme ":" ( hier_part | opaque_part ) + ret[:ABS_URI] = abs_uri = "#{scheme}:(?:#{hier_part}|#{opaque_part})" + # relativeURI = ( net_path | abs_path | rel_path ) [ "?" query ] + ret[:REL_URI] = rel_uri = "(?:#{net_path}|#{abs_path}|#{rel_path})(?:\\?#{query})?" + + # Gem::URI-reference = [ absoluteURI | relativeURI ] [ "#" fragment ] + ret[:URI_REF] = "(?:#{abs_uri}|#{rel_uri})?(?:##{fragment})?" + + ret[:X_ABS_URI] = " + (#{scheme}): (?# 1: scheme) + (?: + (#{opaque_part}) (?# 2: opaque) + | + (?:(?: + //(?: + (?:(?:(#{userinfo})@)? (?# 3: userinfo) + (?:(#{host})(?::(\\d*))?))? (?# 4: host, 5: port) + | + (#{reg_name}) (?# 6: registry) + ) + | + (?!//)) (?# XXX: '//' is the mark for hostport) + (#{abs_path})? (?# 7: path) + )(?:\\?(#{query}))? (?# 8: query) + ) + (?:\\#(#{fragment}))? (?# 9: fragment) + " + + ret[:X_REL_URI] = " + (?: + (?: + // + (?: + (?:(#{userinfo})@)? (?# 1: userinfo) + (#{host})?(?::(\\d*))? (?# 2: host, 3: port) + | + (#{reg_name}) (?# 4: registry) + ) + ) + | + (#{rel_segment}) (?# 5: rel_segment) + )? + (#{abs_path})? (?# 6: abs_path) + (?:\\?(#{query}))? (?# 7: query) + (?:\\#(#{fragment}))? (?# 8: fragment) + " + + ret + end + + # Constructs the default Hash of Regexp's. + def initialize_regexp(pattern) + ret = {} + + # for Gem::URI::split + ret[:ABS_URI] = Regexp.new('\A\s*+' + pattern[:X_ABS_URI] + '\s*\z', Regexp::EXTENDED) + ret[:REL_URI] = Regexp.new('\A\s*+' + pattern[:X_REL_URI] + '\s*\z', Regexp::EXTENDED) + + # for Gem::URI::extract + ret[:URI_REF] = Regexp.new(pattern[:URI_REF]) + ret[:ABS_URI_REF] = Regexp.new(pattern[:X_ABS_URI], Regexp::EXTENDED) + ret[:REL_URI_REF] = Regexp.new(pattern[:X_REL_URI], Regexp::EXTENDED) + + # for Gem::URI::escape/unescape + ret[:ESCAPED] = Regexp.new(pattern[:ESCAPED]) + ret[:UNSAFE] = Regexp.new("[^#{pattern[:UNRESERVED]}#{pattern[:RESERVED]}]") + + # for Generic#initialize + ret[:SCHEME] = Regexp.new("\\A#{pattern[:SCHEME]}\\z") + ret[:USERINFO] = Regexp.new("\\A#{pattern[:USERINFO]}\\z") + ret[:HOST] = Regexp.new("\\A#{pattern[:HOST]}\\z") + ret[:PORT] = Regexp.new("\\A#{pattern[:PORT]}\\z") + ret[:OPAQUE] = Regexp.new("\\A#{pattern[:OPAQUE_PART]}\\z") + ret[:REGISTRY] = Regexp.new("\\A#{pattern[:REG_NAME]}\\z") + ret[:ABS_PATH] = Regexp.new("\\A#{pattern[:ABS_PATH]}\\z") + ret[:REL_PATH] = Regexp.new("\\A#{pattern[:REL_PATH]}\\z") + ret[:QUERY] = Regexp.new("\\A#{pattern[:QUERY]}\\z") + ret[:FRAGMENT] = Regexp.new("\\A#{pattern[:FRAGMENT]}\\z") + + ret + end + + def convert_to_uri(uri) + if uri.is_a?(Gem::URI::Generic) + uri + elsif uri = String.try_convert(uri) + parse(uri) + else + raise ArgumentError, + "bad argument (expected Gem::URI object or Gem::URI string)" + end + end + + end # class Parser +end # module Gem::URI diff --git a/lib/rubygems/vendor/uri/lib/uri/rfc3986_parser.rb b/lib/rubygems/vendor/uri/lib/uri/rfc3986_parser.rb new file mode 100644 index 0000000000..728bb55674 --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri/rfc3986_parser.rb @@ -0,0 +1,183 @@ +# frozen_string_literal: true +module Gem::URI + class RFC3986_Parser # :nodoc: + # Gem::URI defined in RFC3986 + HOST = %r[ + (?<IP-literal>\[(?: + (?<IPv6address> + (?:\h{1,4}:){6} + (?<ls32>\h{1,4}:\h{1,4} + | (?<IPv4address>(?<dec-octet>[1-9]\d|1\d{2}|2[0-4]\d|25[0-5]|\d) + \.\g<dec-octet>\.\g<dec-octet>\.\g<dec-octet>) + ) + | ::(?:\h{1,4}:){5}\g<ls32> + | \h{1,4}?::(?:\h{1,4}:){4}\g<ls32> + | (?:(?:\h{1,4}:)?\h{1,4})?::(?:\h{1,4}:){3}\g<ls32> + | (?:(?:\h{1,4}:){,2}\h{1,4})?::(?:\h{1,4}:){2}\g<ls32> + | (?:(?:\h{1,4}:){,3}\h{1,4})?::\h{1,4}:\g<ls32> + | (?:(?:\h{1,4}:){,4}\h{1,4})?::\g<ls32> + | (?:(?:\h{1,4}:){,5}\h{1,4})?::\h{1,4} + | (?:(?:\h{1,4}:){,6}\h{1,4})?:: + ) + | (?<IPvFuture>v\h++\.[!$&-.0-9:;=A-Z_a-z~]++) + )\]) + | \g<IPv4address> + | (?<reg-name>(?:%\h\h|[!$&-.0-9;=A-Z_a-z~])*+) + ]x + + USERINFO = /(?:%\h\h|[!$&-.0-9:;=A-Z_a-z~])*+/ + + SCHEME = %r[[A-Za-z][+\-.0-9A-Za-z]*+].source + SEG = %r[(?:%\h\h|[!$&-.0-9:;=@A-Z_a-z~/])].source + SEG_NC = %r[(?:%\h\h|[!$&-.0-9;=@A-Z_a-z~])].source + FRAGMENT = %r[(?:%\h\h|[!$&-.0-9:;=@A-Z_a-z~/?])*+].source + + RFC3986_URI = %r[\A + (?<seg>#{SEG}){0} + (?<Gem::URI> + (?<scheme>#{SCHEME}): + (?<hier-part>// + (?<authority> + (?:(?<userinfo>#{USERINFO.source})@)? + (?<host>#{HOST.source.delete(" \n")}) + (?::(?<port>\d*+))? + ) + (?<path-abempty>(?:/\g<seg>*+)?) + | (?<path-absolute>/((?!/)\g<seg>++)?) + | (?<path-rootless>(?!/)\g<seg>++) + | (?<path-empty>) + ) + (?:\?(?<query>[^\#]*+))? + (?:\#(?<fragment>#{FRAGMENT}))? + )\z]x + + RFC3986_relative_ref = %r[\A + (?<seg>#{SEG}){0} + (?<relative-ref> + (?<relative-part>// + (?<authority> + (?:(?<userinfo>#{USERINFO.source})@)? + (?<host>#{HOST.source.delete(" \n")}(?<!/))? + (?::(?<port>\d*+))? + ) + (?<path-abempty>(?:/\g<seg>*+)?) + | (?<path-absolute>/\g<seg>*+) + | (?<path-noscheme>#{SEG_NC}++(?:/\g<seg>*+)?) + | (?<path-empty>) + ) + (?:\?(?<query>[^#]*+))? + (?:\#(?<fragment>#{FRAGMENT}))? + )\z]x + attr_reader :regexp + + def initialize + @regexp = default_regexp.each_value(&:freeze).freeze + end + + def split(uri) #:nodoc: + begin + uri = uri.to_str + rescue NoMethodError + raise InvalidURIError, "bad Gem::URI(is not Gem::URI?): #{uri.inspect}" + end + uri.ascii_only? or + raise InvalidURIError, "Gem::URI must be ascii only #{uri.dump}" + if m = RFC3986_URI.match(uri) + query = m["query"] + scheme = m["scheme"] + opaque = m["path-rootless"] + if opaque + opaque << "?#{query}" if query + [ scheme, + nil, # userinfo + nil, # host + nil, # port + nil, # registry + nil, # path + opaque, + nil, # query + m["fragment"] + ] + else # normal + [ scheme, + m["userinfo"], + m["host"], + m["port"], + nil, # registry + (m["path-abempty"] || + m["path-absolute"] || + m["path-empty"]), + nil, # opaque + query, + m["fragment"] + ] + end + elsif m = RFC3986_relative_ref.match(uri) + [ nil, # scheme + m["userinfo"], + m["host"], + m["port"], + nil, # registry, + (m["path-abempty"] || + m["path-absolute"] || + m["path-noscheme"] || + m["path-empty"]), + nil, # opaque + m["query"], + m["fragment"] + ] + else + raise InvalidURIError, "bad Gem::URI(is not Gem::URI?): #{uri.inspect}" + end + end + + def parse(uri) # :nodoc: + Gem::URI.for(*self.split(uri), self) + end + + + def join(*uris) # :nodoc: + uris[0] = convert_to_uri(uris[0]) + uris.inject :merge + end + + @@to_s = Kernel.instance_method(:to_s) + if @@to_s.respond_to?(:bind_call) + def inspect + @@to_s.bind_call(self) + end + else + def inspect + @@to_s.bind(self).call + end + end + + private + + def default_regexp # :nodoc: + { + SCHEME: %r[\A#{SCHEME}\z]o, + USERINFO: %r[\A#{USERINFO}\z]o, + HOST: %r[\A#{HOST}\z]o, + ABS_PATH: %r[\A/#{SEG}*+\z]o, + REL_PATH: %r[\A(?!/)#{SEG}++\z]o, + QUERY: %r[\A(?:%\h\h|[!$&-.0-9:;=@A-Z_a-z~/?])*+\z], + FRAGMENT: %r[\A#{FRAGMENT}\z]o, + OPAQUE: %r[\A(?:[^/].*)?\z], + PORT: /\A[\x09\x0a\x0c\x0d ]*+\d*[\x09\x0a\x0c\x0d ]*\z/, + } + end + + def convert_to_uri(uri) + if uri.is_a?(Gem::URI::Generic) + uri + elsif uri = String.try_convert(uri) + parse(uri) + else + raise ArgumentError, + "bad argument (expected Gem::URI object or Gem::URI string)" + end + end + + end # class Parser +end # module Gem::URI diff --git a/lib/rubygems/vendor/uri/lib/uri/version.rb b/lib/rubygems/vendor/uri/lib/uri/version.rb new file mode 100644 index 0000000000..3c80c334d4 --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri/version.rb @@ -0,0 +1,6 @@ +module Gem::URI + # :stopdoc: + VERSION_CODE = '001300'.freeze + VERSION = VERSION_CODE.scan(/../).collect{|n| n.to_i}.join('.').freeze + # :startdoc: +end diff --git a/lib/rubygems/vendor/uri/lib/uri/ws.rb b/lib/rubygems/vendor/uri/lib/uri/ws.rb new file mode 100644 index 0000000000..0dd2a7a1bb --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri/ws.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: false +# = uri/ws.rb +# +# Author:: Matt Muller <mamuller@amazon.com> +# License:: You can redistribute it and/or modify it under the same term as Ruby. +# +# See Gem::URI for general documentation +# + +require_relative 'generic' + +module Gem::URI + + # + # The syntax of WS URIs is defined in RFC6455 section 3. + # + # Note that the Ruby Gem::URI library allows WS URLs containing usernames and + # passwords. This is not legal as per the RFC, but used to be + # supported in Internet Explorer 5 and 6, before the MS04-004 security + # update. See <URL:http://support.microsoft.com/kb/834489>. + # + class WS < Generic + # A Default port of 80 for Gem::URI::WS. + DEFAULT_PORT = 80 + + # An Array of the available components for Gem::URI::WS. + COMPONENT = %i[ + scheme + userinfo host port + path + query + ].freeze + + # + # == Description + # + # Creates a new Gem::URI::WS object from components, with syntax checking. + # + # The components accepted are userinfo, host, port, path, and query. + # + # The components should be provided either as an Array, or as a Hash + # with keys formed by preceding the component names with a colon. + # + # If an Array is used, the components must be passed in the + # order <code>[userinfo, host, port, path, query]</code>. + # + # Example: + # + # uri = Gem::URI::WS.build(host: 'www.example.com', path: '/foo/bar') + # + # uri = Gem::URI::WS.build([nil, "www.example.com", nil, "/path", "query"]) + # + # Currently, if passed userinfo components this method generates + # invalid WS URIs as per RFC 1738. + # + def self.build(args) + tmp = Util.make_components_hash(self, args) + super(tmp) + end + + # + # == Description + # + # Returns the full path for a WS Gem::URI, as required by Net::HTTP::Get. + # + # If the Gem::URI contains a query, the full path is Gem::URI#path + '?' + Gem::URI#query. + # Otherwise, the path is simply Gem::URI#path. + # + # Example: + # + # uri = Gem::URI::WS.build(path: '/foo/bar', query: 'test=true') + # uri.request_uri # => "/foo/bar?test=true" + # + def request_uri + return unless @path + + url = @query ? "#@path?#@query" : @path.dup + url.start_with?(?/.freeze) ? url : ?/ + url + end + end + + register_scheme 'WS', WS +end diff --git a/lib/rubygems/vendor/uri/lib/uri/wss.rb b/lib/rubygems/vendor/uri/lib/uri/wss.rb new file mode 100644 index 0000000000..0b91d334bb --- /dev/null +++ b/lib/rubygems/vendor/uri/lib/uri/wss.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: false +# = uri/wss.rb +# +# Author:: Matt Muller <mamuller@amazon.com> +# License:: You can redistribute it and/or modify it under the same term as Ruby. +# +# See Gem::URI for general documentation +# + +require_relative 'ws' + +module Gem::URI + + # The default port for WSS URIs is 443, and the scheme is 'wss:' rather + # than 'ws:'. Other than that, WSS URIs are identical to WS URIs; + # see Gem::URI::WS. + class WSS < WS + # A Default port of 443 for Gem::URI::WSS + DEFAULT_PORT = 443 + end + + register_scheme 'WSS', WSS +end diff --git a/lib/rubygems/vendored_molinillo.rb b/lib/rubygems/vendored_molinillo.rb new file mode 100644 index 0000000000..45906c0e5c --- /dev/null +++ b/lib/rubygems/vendored_molinillo.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative "vendor/molinillo/lib/molinillo" diff --git a/lib/rubygems/vendored_net_http.rb b/lib/rubygems/vendored_net_http.rb new file mode 100644 index 0000000000..a84c52a947 --- /dev/null +++ b/lib/rubygems/vendored_net_http.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +# Ruby 3.3 and RubyGems 3.5 is already load Gem::Timeout from lib/rubygems/net/http.rb +# We should avoid to load it again +require_relative "vendor/net-http/lib/net/http" unless defined?(Gem::Net::HTTP) diff --git a/lib/rubygems/vendored_optparse.rb b/lib/rubygems/vendored_optparse.rb new file mode 100644 index 0000000000..a5611d32f0 --- /dev/null +++ b/lib/rubygems/vendored_optparse.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative "vendor/optparse/lib/optparse" diff --git a/lib/rubygems/vendored_timeout.rb b/lib/rubygems/vendored_timeout.rb new file mode 100644 index 0000000000..45541928e6 --- /dev/null +++ b/lib/rubygems/vendored_timeout.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +# Ruby 3.3 and RubyGems 3.5 is already load Gem::Timeout from lib/rubygems/timeout.rb +# We should avoid to load it again +require_relative "vendor/timeout/lib/timeout" unless defined?(Gem::Timeout) diff --git a/lib/rubygems/vendored_tsort.rb b/lib/rubygems/vendored_tsort.rb new file mode 100644 index 0000000000..c3d815650d --- /dev/null +++ b/lib/rubygems/vendored_tsort.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +require_relative "vendor/tsort/lib/tsort" diff --git a/lib/rubygems/version.rb b/lib/rubygems/version.rb index 3d8e11b515..e174d8ad95 100644 --- a/lib/rubygems/version.rb +++ b/lib/rubygems/version.rb @@ -131,7 +131,7 @@ require_relative "deprecate" # # == Preventing Version Catastrophe: # -# From: http://blog.zenspider.com/2008/10/rubygems-howto-preventing-cata.html +# From: https://www.zenspider.com/ruby/2008/10/rubygems-how-to-preventing-catastrophe.html # # Let's say you're depending on the fnord gem version 2.y.z. If you # specify your dependency as ">= 2.0.0" then, you're good, right? What @@ -156,13 +156,13 @@ class Gem::Version include Comparable VERSION_PATTERN = '[0-9]+(?>\.[0-9a-zA-Z]+)*(-[0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*)?' # :nodoc: - ANCHORED_VERSION_PATTERN = /\A\s*(#{VERSION_PATTERN})?\s*\z/.freeze # :nodoc: + ANCHORED_VERSION_PATTERN = /\A\s*(#{VERSION_PATTERN})?\s*\z/ # :nodoc: ## # A string representation of this Version. def version - @version.dup + @version end alias_method :to_s, :version @@ -173,7 +173,7 @@ class Gem::Version def self.correct?(version) nil_versions_are_discouraged! if version.nil? - !!(version.to_s =~ ANCHORED_VERSION_PATTERN) + ANCHORED_VERSION_PATTERN.match?(version.to_s) end ## @@ -201,7 +201,7 @@ class Gem::Version @@release = {} def self.new(version) # :nodoc: - return super unless Gem::Version == self + return super unless self == Gem::Version @@all[version] ||= super end @@ -224,9 +224,17 @@ class Gem::Version end # If version is an empty string convert it to 0 - version = 0 if version.is_a?(String) && version =~ /\A\s*\Z/ + version = 0 if version.is_a?(String) && /\A\s*\Z/.match?(version) + + @version = version.to_s - @version = version.to_s.strip.gsub("-",".pre.") + # optimization to avoid allocation when given an integer, since we know + # it's to_s won't have any spaces or dashes + unless version.is_a?(Integer) + @version = @version.strip + @version.gsub!("-",".pre.") + end + @version = -@version @segments = nil end @@ -252,7 +260,7 @@ class Gem::Version # same precision. Version "1.0" is not the same as version "1". def eql?(other) - self.class === other && @version == other._version + self.class === other && @version == other.version end def hash # :nodoc: @@ -284,7 +292,7 @@ class Gem::Version end def yaml_initialize(tag, map) # :nodoc: - @version = map["version"] + @version = -map["version"] @segments = nil @hash = nil end @@ -302,7 +310,7 @@ class Gem::Version def prerelease? unless instance_variable_defined? :@prerelease - @prerelease = !(@version =~ /[a-zA-Z]/).nil? + @prerelease = /[a-zA-Z]/.match?(version) end @prerelease end @@ -354,7 +362,7 @@ class Gem::Version return self <=> self.class.new(other) if (String === other) && self.class.correct?(other) return unless Gem::Version === other - return 0 if @version == other._version || canonical_segments == other.canonical_segments + return 0 if @version == other.version || canonical_segments == other.canonical_segments lhsegments = canonical_segments rhsegments = other.canonical_segments @@ -380,39 +388,37 @@ class Gem::Version 0 end + # remove trailing zeros segments before first letter or at the end of the version def canonical_segments - @canonical_segments ||= - _split_segments.map! do |segments| - segments.reverse_each.drop_while {|s| s == 0 }.reverse - end.reduce(&:concat) + @canonical_segments ||= begin + # remove trailing 0 segments, using dot or letter as anchor + # may leave a trailing dot which will be ignored by partition_segments + canonical_version = @version.sub(/(?<=[a-zA-Z.])[.0]+\z/, "") + # remove 0 segments before the first letter in a prerelease version + canonical_version.sub!(/(?<=\.|\A)[0.]+(?=[a-zA-Z])/, "") if prerelease? + partition_segments(canonical_version) + end end def freeze prerelease? + _segments canonical_segments super end protected - def _version - @version - end - def _segments # segments is lazy so it can pick up version values that come from # old marshaled versions, which don't go through marshal_load. # since this version object is cached in @@all, its @segments should be frozen - - @segments ||= @version.scan(/[0-9]+|[a-z]+/i).map do |s| - /^\d+$/.match?(s) ? s.to_i : s - end.freeze + @segments ||= partition_segments(@version) end - def _split_segments - string_start = _segments.index {|s| s.is_a?(String) } - string_segments = segments - numeric_segments = string_segments.slice!(0, string_start || string_segments.size) - [numeric_segments, string_segments] + def partition_segments(ver) + ver.scan(/\d+|[a-z]+/i).map! do |s| + /\A\d/.match?(s) ? s.to_i : -s + end.freeze end end diff --git a/lib/rubygems/webauthn_listener.rb b/lib/rubygems/webauthn_listener.rb deleted file mode 100644 index 22f7ea2011..0000000000 --- a/lib/rubygems/webauthn_listener.rb +++ /dev/null @@ -1,92 +0,0 @@ -# frozen_string_literal: true - -require_relative "webauthn_listener/response" - -## -# The WebauthnListener class retrieves an OTP after a user successfully WebAuthns with the Gem host. -# An instance opens a socket using the TCPServer instance given and listens for a request from the Gem host. -# The request should be a GET request to the root path and contains the OTP code in the form -# of a query parameter `code`. The listener will return the code which will be used as the OTP for -# API requests. -# -# Types of responses sent by the listener after receiving a request: -# - 200 OK: OTP code was successfully retrieved -# - 204 No Content: If the request was an OPTIONS request -# - 400 Bad Request: If the request did not contain a query parameter `code` -# - 404 Not Found: The request was not to the root path -# - 405 Method Not Allowed: OTP code was not retrieved because the request was not a GET/OPTIONS request -# -# Example usage: -# -# server = TCPServer.new(0) -# otp = Gem::WebauthnListener.wait_for_otp_code("https://rubygems.example", server) -# - -class Gem::WebauthnListener - attr_reader :host - - def initialize(host) - @host = host - end - - def self.wait_for_otp_code(host, server) - new(host).fetch_otp_from_connection(server) - end - - def fetch_otp_from_connection(server) - loop do - socket = server.accept - request_line = socket.gets - - method, req_uri, _protocol = request_line.split(" ") - req_uri = URI.parse(req_uri) - - responder = SocketResponder.new(socket) - - unless root_path?(req_uri) - responder.send(NotFoundResponse.for(host)) - raise Gem::WebauthnVerificationError, "Page at #{req_uri.path} not found." - end - - case method.upcase - when "OPTIONS" - responder.send(NoContentResponse.for(host)) - next # will be GET - when "GET" - if otp = parse_otp_from_uri(req_uri) - responder.send(OkResponse.for(host)) - return otp - end - responder.send(BadRequestResponse.for(host)) - raise Gem::WebauthnVerificationError, "Did not receive OTP from #{host}." - else - responder.send(MethodNotAllowedResponse.for(host)) - raise Gem::WebauthnVerificationError, "Invalid HTTP method #{method.upcase} received." - end - end - end - - private - - def root_path?(uri) - uri.path == "/" - end - - def parse_otp_from_uri(uri) - require "cgi" - - return if uri.query.nil? - CGI.parse(uri.query).dig("code", 0) - end - - class SocketResponder - def initialize(socket) - @socket = socket - end - - def send(response) - @socket.print response.to_s - @socket.close - end - end -end diff --git a/lib/rubygems/webauthn_listener/response.rb b/lib/rubygems/webauthn_listener/response.rb deleted file mode 100644 index baa769c4ae..0000000000 --- a/lib/rubygems/webauthn_listener/response.rb +++ /dev/null @@ -1,161 +0,0 @@ -# frozen_string_literal: true - -## -# The WebauthnListener Response class is used by the WebauthnListener to create -# responses to be sent to the Gem host. It creates a Net::HTTPResponse instance -# when initialized and can be converted to the appropriate format to be sent by a socket using `to_s`. -# Net::HTTPResponse instances cannot be directly sent over a socket. -# -# Types of response classes: -# - OkResponse -# - NoContentResponse -# - BadRequestResponse -# - NotFoundResponse -# - MethodNotAllowedResponse -# -# Example usage: -# -# server = TCPServer.new(0) -# socket = server.accept -# -# response = OkResponse.for("https://rubygems.example") -# socket.print response.to_s -# socket.close -# - -class Gem::WebauthnListener - class Response - attr_reader :http_response - - def self.for(host) - new(host) - end - - def initialize(host) - @host = host - - build_http_response - end - - def to_s - status_line = "HTTP/#{@http_response.http_version} #{@http_response.code} #{@http_response.message}\r\n" - headers = @http_response.to_hash.map {|header, value| "#{header}: #{value.join(", ")}\r\n" }.join + "\r\n" - body = @http_response.body ? "#{@http_response.body}\n" : "" - - status_line + headers + body - end - - private - - # Must be implemented in subclasses - def code - raise NotImplementedError - end - - def reason_phrase - raise NotImplementedError - end - - def body; end - - def build_http_response - response_class = Net::HTTPResponse::CODE_TO_OBJ[code.to_s] - @http_response = response_class.new("1.1", code, reason_phrase) - @http_response.instance_variable_set(:@read, true) - - add_connection_header - add_access_control_headers - add_body - end - - def add_connection_header - @http_response["connection"] = "close" - end - - def add_access_control_headers - @http_response["access-control-allow-origin"] = @host - @http_response["access-control-allow-methods"] = "POST" - @http_response["access-control-allow-headers"] = %w[Content-Type Authorization x-csrf-token] - end - - def add_body - return unless body - @http_response["content-type"] = "text/plain" - @http_response["content-length"] = body.bytesize - @http_response.instance_variable_set(:@body, body) - end - end - - class OkResponse < Response - private - - def code - 200 - end - - def reason_phrase - "OK" - end - - def body - "success" - end - end - - class NoContentResponse < Response - private - - def code - 204 - end - - def reason_phrase - "No Content" - end - end - - class BadRequestResponse < Response - private - - def code - 400 - end - - def reason_phrase - "Bad Request" - end - - def body - "missing code parameter" - end - end - - class NotFoundResponse < Response - private - - def code - 404 - end - - def reason_phrase - "Not Found" - end - end - - class MethodNotAllowedResponse < Response - private - - def code - 405 - end - - def reason_phrase - "Method Not Allowed" - end - - def add_access_control_headers - super - @http_response["allow"] = %w[GET OPTIONS] - end - end -end diff --git a/lib/rubygems/yaml_serializer.rb b/lib/rubygems/yaml_serializer.rb index a447709c74..128becc1ce 100644 --- a/lib/rubygems/yaml_serializer.rb +++ b/lib/rubygems/yaml_serializer.rb @@ -17,7 +17,11 @@ module Gem if v.is_a?(Hash) yaml << dump_hash(v).gsub(/^(?!$)/, " ") # indent all non-empty lines elsif v.is_a?(Array) # Expected to be array of strings - yaml << "\n- " << v.map {|s| s.to_s.gsub(/\s+/, " ").inspect }.join("\n- ") << "\n" + if v.empty? + yaml << " []\n" + else + yaml << "\n- " << v.map {|s| s.to_s.gsub(/\s+/, " ").inspect }.join("\n- ") << "\n" + end else yaml << " " << v.to_s.gsub(/\s+/, " ").inspect << "\n" end @@ -32,7 +36,7 @@ module Gem (.*) # value \1 # matching closing quote $ - /xo.freeze + /xo HASH_REGEX = / ^ @@ -44,18 +48,20 @@ module Gem (.*) # value \3 # matching closing quote $ - /xo.freeze + /xo def load(str) res = {} stack = [res] last_hash = nil last_empty_key = nil - str.split(/\r?\n/).each do |line| + str.split(/\r?\n/) do |line| if match = HASH_REGEX.match(line) indent, key, quote, val = match.captures - key = convert_to_backward_compatible_key(key) - depth = indent.scan(/ /).length + val = strip_comment(val) + + convert_to_backward_compatible_key!(key) + depth = indent.size / 2 if quote.empty? && val.empty? new_hash = {} stack[depth][key] = new_hash @@ -63,10 +69,13 @@ module Gem last_empty_key = key last_hash = stack[depth] else + val = [] if val == "[]" # empty array stack[depth][key] = val end elsif match = ARRAY_REGEX.match(line) _, val = match.captures + val = strip_comment(val) + last_hash[last_empty_key] = [] unless last_hash[last_empty_key].is_a?(Array) last_hash[last_empty_key].push(val) @@ -75,15 +84,22 @@ module Gem res end + def strip_comment(val) + if val.include?("#") && !val.start_with?("#") + val.split("#", 2).first.strip + else + val + end + end + # for settings' keys - def convert_to_backward_compatible_key(key) - key = "#{key}/" if key =~ /https?:/i && key !~ %r{/\Z} - key = key.gsub(".", "__") if key.include?(".") - key + def convert_to_backward_compatible_key!(key) + key << "/" if /https?:/i.match?(key) && !%r{/\Z}.match?(key) + key.gsub!(".", "__") end class << self - private :dump_hash, :convert_to_backward_compatible_key + private :dump_hash, :convert_to_backward_compatible_key! end end end diff --git a/lib/securerandom.gemspec b/lib/securerandom.gemspec index f42ddbcc97..0b023486a6 100644 --- a/lib/securerandom.gemspec +++ b/lib/securerandom.gemspec @@ -20,8 +20,13 @@ Gem::Specification.new do |spec| spec.metadata["homepage_uri"] = spec.homepage spec.metadata["source_code_uri"] = spec.homepage - spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do - `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } + # Specify which files should be added to the gem when it is released. + # The `git ls-files -z` loads the files in the RubyGem that have been added into git. + spec.files = Dir.chdir(__dir__) do + `git ls-files -z`.split("\x0").reject do |f| + (File.expand_path(f) == __FILE__) || + f.start_with?(*%w[bin/ test/ spec/ features/ docs/ rakelib/ .document .git Gemfile Rakefile]) + end end spec.bindir = "exe" spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } diff --git a/lib/securerandom.rb b/lib/securerandom.rb index 9faa09e4e0..81757f3100 100644 --- a/lib/securerandom.rb +++ b/lib/securerandom.rb @@ -40,19 +40,27 @@ require 'random/formatter' module SecureRandom - VERSION = "0.2.2" + # The version + VERSION = "0.3.1" class << self + # Returns a random binary string containing +size+ bytes. + # + # See Random.bytes def bytes(n) return gen_random(n) end private + # :stopdoc: + + # Implementation using OpenSSL def gen_random_openssl(n) return OpenSSL::Random.random_bytes(n) end + # Implementation using system random device def gen_random_urandom(n) ret = Random.urandom(n) unless ret @@ -78,6 +86,9 @@ module SecureRandom end end + # :startdoc: + + # Generate random data bytes for Random::Formatter public :gen_random end end diff --git a/lib/set.rb b/lib/set.rb index 5ac37dbf3c..a0954a31d1 100644 --- a/lib/set.rb +++ b/lib/set.rb @@ -3,7 +3,7 @@ # # set.rb - defines the Set class # -# Copyright (c) 2002-2020 Akinori MUSHA <knu@iDaemons.org> +# Copyright (c) 2002-2023 Akinori MUSHA <knu@iDaemons.org> # # Documentation by Akinori MUSHA and Gavin Sinclair. # @@ -216,7 +216,7 @@ # has been modified while an element in the set. # class Set - VERSION = "1.0.3" + VERSION = "1.1.0" include Enumerable @@ -286,18 +286,10 @@ class Set @hash = orig.instance_variable_get(:@hash).dup end - if Kernel.instance_method(:initialize_clone).arity != 1 - # Clone internal hash. - def initialize_clone(orig, **options) - super - @hash = orig.instance_variable_get(:@hash).clone(**options) - end - else - # Clone internal hash. - def initialize_clone(orig) - super - @hash = orig.instance_variable_get(:@hash).clone - end + # Clone internal hash. + def initialize_clone(orig, **options) + super + @hash = orig.instance_variable_get(:@hash).clone(**options) end def freeze # :nodoc: @@ -389,7 +381,7 @@ class Set # Equivalent to Set#flatten, but replaces the receiver with the # result in place. Returns nil if no modifications were made. def flatten! - replace(flatten()) if any? { |e| e.is_a?(Set) } + replace(flatten()) if any?(Set) end # Returns true if the set contains the given object. @@ -409,7 +401,7 @@ class Set when set.instance_of?(self.class) && @hash.respond_to?(:>=) @hash >= set.instance_variable_get(:@hash) when set.is_a?(Set) - size >= set.size && set.all? { |o| include?(o) } + size >= set.size && set.all?(self) else raise ArgumentError, "value must be a set" end @@ -422,7 +414,7 @@ class Set when set.instance_of?(self.class) && @hash.respond_to?(:>) @hash > set.instance_variable_get(:@hash) when set.is_a?(Set) - size > set.size && set.all? { |o| include?(o) } + size > set.size && set.all?(self) else raise ArgumentError, "value must be a set" end @@ -435,7 +427,7 @@ class Set when set.instance_of?(self.class) && @hash.respond_to?(:<=) @hash <= set.instance_variable_get(:@hash) when set.is_a?(Set) - size <= set.size && all? { |o| set.include?(o) } + size <= set.size && all?(set) else raise ArgumentError, "value must be a set" end @@ -448,7 +440,7 @@ class Set when set.instance_of?(self.class) && @hash.respond_to?(:<) @hash < set.instance_variable_get(:@hash) when set.is_a?(Set) - size < set.size && all? { |o| set.include?(o) } + size < set.size && all?(set) else raise ArgumentError, "value must be a set" end @@ -479,12 +471,12 @@ class Set case set when Set if size < set.size - any? { |o| set.include?(o) } + any?(set) else - set.any? { |o| include?(o) } + set.any?(self) end when Enumerable - set.any? { |o| include?(o) } + set.any?(self) else raise ArgumentError, "value must be enumerable" end diff --git a/lib/set/set.gemspec b/lib/set/set.gemspec index c415fac99c..2ebef6985d 100644 --- a/lib/set/set.gemspec +++ b/lib/set/set.gemspec @@ -15,7 +15,7 @@ Gem::Specification.new do |spec| spec.description = %q{Provides a class to deal with collections of unordered, unique values} spec.homepage = "https://github.com/ruby/set" spec.licenses = ["Ruby", "BSD-2-Clause"] - spec.required_ruby_version = Gem::Requirement.new(">= 2.7.0") + spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0") spec.metadata["homepage_uri"] = spec.homepage spec.metadata["source_code_uri"] = spec.homepage diff --git a/lib/shellwords.rb b/lib/shellwords.rb index 5866601b17..067432e374 100644 --- a/lib/shellwords.rb +++ b/lib/shellwords.rb @@ -68,8 +68,7 @@ # 1: {IEEE Std 1003.1-2008, 2016 Edition, the Shell & Utilities volume}[http://pubs.opengroup.org/onlinepubs/9699919799/utilities/contents.html] module Shellwords - - VERSION = "0.1.0" + VERSION = "0.2.0" # Splits a string into an array of tokens in the same way the UNIX # Bourne shell does. diff --git a/lib/singleton.rb b/lib/singleton.rb index 7ed17a769c..6da939124e 100644 --- a/lib/singleton.rb +++ b/lib/singleton.rb @@ -92,7 +92,7 @@ # p a.strip # => nil # module Singleton - VERSION = "0.1.1" + VERSION = "0.2.0" # Raises a TypeError to prevent cloning. def clone diff --git a/lib/syntax_suggest/api.rb b/lib/syntax_suggest/api.rb index 74e53c2563..65660ec5e5 100644 --- a/lib/syntax_suggest/api.rb +++ b/lib/syntax_suggest/api.rb @@ -5,9 +5,28 @@ require_relative "version" require "tmpdir" require "stringio" require "pathname" -require "ripper" require "timeout" +# We need Ripper loaded for `Prism.lex_compat` even if we're using Prism +# for lexing and parsing +require "ripper" + +# Prism is the new parser, replacing Ripper +# +# We need to "dual boot" both for now because syntax_suggest +# supports older rubies that do not ship with syntax suggest. +# +# We also need the ability to control loading of this library +# so we can test that both modes work correctly in CI. +if (value = ENV["SYNTAX_SUGGEST_DISABLE_PRISM"]) + warn "Skipping loading prism due to SYNTAX_SUGGEST_DISABLE_PRISM=#{value}" +else + begin + require "prism" + rescue LoadError + end +end + module SyntaxSuggest # Used to indicate a default value that cannot # be confused with another input. @@ -16,6 +35,14 @@ module SyntaxSuggest class Error < StandardError; end TIMEOUT_DEFAULT = ENV.fetch("SYNTAX_SUGGEST_TIMEOUT", 1).to_i + # SyntaxSuggest.use_prism_parser? [Private] + # + # Tells us if the prism parser is available for use + # or if we should fallback to `Ripper` + def self.use_prism_parser? + defined?(Prism) + end + # SyntaxSuggest.handle_error [Public] # # Takes a `SyntaxError` exception, uses the @@ -129,11 +156,20 @@ module SyntaxSuggest # SyntaxSuggest.invalid? [Private] # # Opposite of `SyntaxSuggest.valid?` - def self.invalid?(source) - source = source.join if source.is_a?(Array) - source = source.to_s + if defined?(Prism) + def self.invalid?(source) + source = source.join if source.is_a?(Array) + source = source.to_s - Ripper.new(source).tap(&:parse).error? + Prism.parse(source).failure? + end + else + def self.invalid?(source) + source = source.join if source.is_a?(Array) + source = source.to_s + + Ripper.new(source).tap(&:parse).error? + end end # SyntaxSuggest.valid? [Private] @@ -191,7 +227,6 @@ require_relative "lex_all" require_relative "code_line" require_relative "code_block" require_relative "block_expand" -require_relative "ripper_errors" require_relative "priority_queue" require_relative "unvisited_lines" require_relative "around_block_scan" diff --git a/lib/syntax_suggest/around_block_scan.rb b/lib/syntax_suggest/around_block_scan.rb index ce00431b3a..dd9af729c5 100644 --- a/lib/syntax_suggest/around_block_scan.rb +++ b/lib/syntax_suggest/around_block_scan.rb @@ -118,7 +118,7 @@ module SyntaxSuggest end # Scanning is intentionally conservative because - # we have no way of rolling back an agressive block (at this time) + # we have no way of rolling back an aggressive block (at this time) # # If a block was stopped for some trivial reason, (like an empty line) # but the next line would have caused it to be balanced then we @@ -224,7 +224,7 @@ module SyntaxSuggest @scanner.lines end - # Managable rspec errors + # Manageable rspec errors def inspect "#<#{self.class}:0x0000123843lol >" end diff --git a/lib/syntax_suggest/block_expand.rb b/lib/syntax_suggest/block_expand.rb index e9b486c720..2751ae2a64 100644 --- a/lib/syntax_suggest/block_expand.rb +++ b/lib/syntax_suggest/block_expand.rb @@ -157,7 +157,7 @@ module SyntaxSuggest end end - # Managable rspec errors + # Manageable rspec errors def inspect "#<SyntaxSuggest::CodeBlock:0x0000123843lol >" end diff --git a/lib/syntax_suggest/capture_code_context.rb b/lib/syntax_suggest/capture_code_context.rb index 6dc7047176..1f232cfae3 100644 --- a/lib/syntax_suggest/capture_code_context.rb +++ b/lib/syntax_suggest/capture_code_context.rb @@ -26,7 +26,7 @@ module SyntaxSuggest # they can't add extra data that's not present. # # In the case of known ambiguious cases, this class adds context - # back to the ambiguitiy so the programmer has full information. + # back to the ambiguity so the programmer has full information. # # Beyond handling these ambiguities, it also captures surrounding # code context information: diff --git a/lib/syntax_suggest/clean_document.rb b/lib/syntax_suggest/clean_document.rb index 2c26061bfc..2790ccae86 100644 --- a/lib/syntax_suggest/clean_document.rb +++ b/lib/syntax_suggest/clean_document.rb @@ -47,9 +47,9 @@ module SyntaxSuggest # ## Heredocs # # A heredoc is an way of defining a multi-line string. They can cause many - # problems. If left as a single line, Ripper would try to parse the contents + # problems. If left as a single line, the parser would try to parse the contents # as ruby code rather than as a string. Even without this problem, we still - # hit an issue with indentation + # hit an issue with indentation: # # 1 foo = <<~HEREDOC # 2 "Be yourself; everyone else is already taken."" @@ -224,7 +224,7 @@ module SyntaxSuggest # def join_consecutive! consecutive_groups = @document.select(&:ignore_newline_not_beg?).map do |code_line| - take_while_including(code_line.index..-1) do |line| + take_while_including(code_line.index..) do |line| line.ignore_newline_not_beg? end end @@ -245,7 +245,7 @@ module SyntaxSuggest # expect(lines[1].to_s).to eq("") def join_trailing_slash! trailing_groups = @document.select(&:trailing_slash?).map do |code_line| - take_while_including(code_line.index..-1) { |x| x.trailing_slash? } + take_while_including(code_line.index..) { |x| x.trailing_slash? } end join_groups(trailing_groups) self @@ -267,7 +267,7 @@ module SyntaxSuggest groups.each do |lines| line = lines.first - # Handle the case of multiple groups in a a row + # Handle the case of multiple groups in a row # if one is already replaced, move on next if @document[line.index].empty? @@ -279,7 +279,7 @@ module SyntaxSuggest ) # Hide the rest of the lines - lines[1..-1].each do |line| + lines[1..].each do |line| # The above lines already have newlines in them, if add more # then there will be double newline, use an empty line instead @document[line.index] = CodeLine.new(line: "", index: line.index, lex: []) @@ -293,7 +293,7 @@ module SyntaxSuggest # Like `take_while` except when it stops # iterating, it also returns the line # that caused it to stop - def take_while_including(range = 0..-1) + def take_while_including(range = 0..) take_next_and_stop = false @document[range].take_while do |line| next if take_next_and_stop diff --git a/lib/syntax_suggest/code_block.rb b/lib/syntax_suggest/code_block.rb index 61e7986da4..d842890300 100644 --- a/lib/syntax_suggest/code_block.rb +++ b/lib/syntax_suggest/code_block.rb @@ -81,7 +81,7 @@ module SyntaxSuggest # lines then the result cannot be invalid # # That means there's no reason to re-check all - # lines with ripper (which is expensive). + # lines with the parser (which is expensive). # Benchmark in commit message @valid = if lines.all? { |l| l.hidden? || l.empty? } true diff --git a/lib/syntax_suggest/code_frontier.rb b/lib/syntax_suggest/code_frontier.rb index 8e93b32514..0f870d0df0 100644 --- a/lib/syntax_suggest/code_frontier.rb +++ b/lib/syntax_suggest/code_frontier.rb @@ -117,7 +117,7 @@ module SyntaxSuggest if ENV["SYNTAX_SUGGEST_DEBUG"] puts "```" - puts @queue.peek.to_s + puts @queue.peek puts "```" puts " @frontier indent: #{frontier_indent}" puts " @unvisited indent: #{unvisited_indent}" diff --git a/lib/syntax_suggest/code_line.rb b/lib/syntax_suggest/code_line.rb index a20f34afa4..58197e95d0 100644 --- a/lib/syntax_suggest/code_line.rb +++ b/lib/syntax_suggest/code_line.rb @@ -180,12 +180,19 @@ module SyntaxSuggest # EOM # expect(lines.first.trailing_slash?).to eq(true) # - def trailing_slash? - last = @lex.last - return false unless last - return false unless last.type == :on_sp + if SyntaxSuggest.use_prism_parser? + def trailing_slash? + last = @lex.last + last&.type == :on_tstring_end + end + else + def trailing_slash? + last = @lex.last + return false unless last + return false unless last.type == :on_sp - last.token == TRAILING_SLASH + last.token == TRAILING_SLASH + end end # Endless method detection diff --git a/lib/syntax_suggest/code_search.rb b/lib/syntax_suggest/code_search.rb index 2a86dfea90..7628dcd131 100644 --- a/lib/syntax_suggest/code_search.rb +++ b/lib/syntax_suggest/code_search.rb @@ -43,7 +43,7 @@ module SyntaxSuggest def initialize(source, record_dir: DEFAULT_VALUE) record_dir = if record_dir == DEFAULT_VALUE - ENV["SYNTAX_SUGGEST_RECORD_DIR"] || ENV["SYNTAX_SUGGEST_DEBUG"] ? "tmp" : nil + (ENV["SYNTAX_SUGGEST_RECORD_DIR"] || ENV["SYNTAX_SUGGEST_DEBUG"]) ? "tmp" : nil else record_dir end @@ -73,7 +73,7 @@ module SyntaxSuggest if ENV["SYNTAX_SUGGEST_DEBUG"] puts "\n\n==== #{filename} ====" puts "\n```#{block.starts_at}..#{block.ends_at}" - puts block.to_s + puts block puts "```" puts " block indent: #{block.current_indent}" end diff --git a/lib/syntax_suggest/core_ext.rb b/lib/syntax_suggest/core_ext.rb index e0fd62b81c..c299627bb7 100644 --- a/lib/syntax_suggest/core_ext.rb +++ b/lib/syntax_suggest/core_ext.rb @@ -21,7 +21,7 @@ if SyntaxError.method_defined?(:detailed_message) attr_reader :string end - # SyntaxSuggest.record_dir [Private] + # SyntaxSuggest.module_for_detailed_message [Private] # # Used to monkeypatch SyntaxError via Module.prepend def self.module_for_detailed_message diff --git a/lib/syntax_suggest/display_invalid_blocks.rb b/lib/syntax_suggest/display_invalid_blocks.rb index 32ec0021a3..5e79b3a262 100644 --- a/lib/syntax_suggest/display_invalid_blocks.rb +++ b/lib/syntax_suggest/display_invalid_blocks.rb @@ -14,7 +14,7 @@ module SyntaxSuggest @filename = filename @code_lines = code_lines - @terminal = terminal == DEFAULT_VALUE ? io.isatty : terminal + @terminal = (terminal == DEFAULT_VALUE) ? io.isatty : terminal end def document_ok? diff --git a/lib/syntax_suggest/explain_syntax.rb b/lib/syntax_suggest/explain_syntax.rb index 142ed2e269..0d80c4d869 100644 --- a/lib/syntax_suggest/explain_syntax.rb +++ b/lib/syntax_suggest/explain_syntax.rb @@ -2,7 +2,21 @@ require_relative "left_right_lex_count" +if !SyntaxSuggest.use_prism_parser? + require_relative "ripper_errors" +end + module SyntaxSuggest + class GetParseErrors + def self.errors(source) + if SyntaxSuggest.use_prism_parser? + Prism.parse(source).errors.map(&:message) + else + RipperErrors.new(source).call.errors + end + end + end + # Explains syntax errors based on their source # # example: @@ -15,8 +29,8 @@ module SyntaxSuggest # # => "Unmatched keyword, missing `end' ?" # # When the error cannot be determined by lexical counting - # then ripper is run against the input and the raw ripper - # errors returned. + # then the parser is run against the input and the raw + # errors are returned. # # Example: # @@ -91,10 +105,10 @@ module SyntaxSuggest # Returns an array of syntax error messages # # If no missing pairs are found it falls back - # on the original ripper error messages + # on the original error messages def errors if missing.empty? - return RipperErrors.new(@code_lines.map(&:original).join).call.errors + return GetParseErrors.errors(@code_lines.map(&:original).join).uniq end missing.map { |miss| why(miss) } diff --git a/lib/syntax_suggest/lex_all.rb b/lib/syntax_suggest/lex_all.rb index 132cba9f5d..c16fbb52d3 100644 --- a/lib/syntax_suggest/lex_all.rb +++ b/lib/syntax_suggest/lex_all.rb @@ -3,34 +3,53 @@ module SyntaxSuggest # Ripper.lex is not guaranteed to lex the entire source document # - # lex = LexAll.new(source: source) - # lex.each do |value| - # puts value.line - # end + # This class guarantees the whole document is lex-ed by iteratively + # lexing the document where ripper stopped. + # + # Prism likely doesn't have the same problem. Once ripper support is removed + # we can likely reduce the complexity here if not remove the whole concept. + # + # Example usage: + # + # lex = LexAll.new(source: source) + # lex.each do |value| + # puts value.line + # end class LexAll include Enumerable def initialize(source:, source_lines: nil) - @lex = Ripper::Lexer.new(source, "-", 1).parse.sort_by(&:pos) - lineno = @lex.last.pos.first + 1 + @lex = self.class.lex(source, 1) + lineno = @lex.last[0][0] + 1 source_lines ||= source.lines last_lineno = source_lines.length until lineno >= last_lineno - lines = source_lines[lineno..-1] + lines = source_lines[lineno..] @lex.concat( - Ripper::Lexer.new(lines.join, "-", lineno + 1).parse.sort_by(&:pos) + self.class.lex(lines.join, lineno + 1) ) - lineno = @lex.last.pos.first + 1 + + lineno = @lex.last[0].first + 1 end last_lex = nil @lex.map! { |elem| - last_lex = LexValue.new(elem.pos.first, elem.event, elem.tok, elem.state, last_lex) + last_lex = LexValue.new(elem[0].first, elem[1], elem[2], elem[3], last_lex) } end + if SyntaxSuggest.use_prism_parser? + def self.lex(source, line_number) + Prism.lex_compat(source, line: line_number).value.sort_by { |values| values[0] } + end + else + def self.lex(source, line_number) + Ripper::Lexer.new(source, "-", line_number).parse.sort_by(&:pos) + end + end + def to_a @lex end diff --git a/lib/syntax_suggest/parse_blocks_from_indent_line.rb b/lib/syntax_suggest/parse_blocks_from_indent_line.rb index 241ed6acb4..39dfca55d2 100644 --- a/lib/syntax_suggest/parse_blocks_from_indent_line.rb +++ b/lib/syntax_suggest/parse_blocks_from_indent_line.rb @@ -8,7 +8,7 @@ module SyntaxSuggest # grabbing one that contains only an "end". In this example: # # def dog - # begonn # mispelled `begin` + # begonn # misspelled `begin` # puts "bark" # end # end diff --git a/lib/syntax_suggest/pathname_from_message.rb b/lib/syntax_suggest/pathname_from_message.rb index b6fe1617be..ab90227427 100644 --- a/lib/syntax_suggest/pathname_from_message.rb +++ b/lib/syntax_suggest/pathname_from_message.rb @@ -13,7 +13,7 @@ module SyntaxSuggest # # => "/tmp/scratch.rb" # class PathnameFromMessage - EVAL_RE = /^\(eval\):\d+/ + EVAL_RE = /^\(eval.*\):\d+/ STREAMING_RE = /^-:\d+/ attr_reader :name diff --git a/lib/syntax_suggest/ripper_errors.rb b/lib/syntax_suggest/ripper_errors.rb index 48eb206e48..4e2bc90948 100644 --- a/lib/syntax_suggest/ripper_errors.rb +++ b/lib/syntax_suggest/ripper_errors.rb @@ -1,7 +1,10 @@ # frozen_string_literal: true module SyntaxSuggest - # Capture parse errors from ripper + # Capture parse errors from Ripper + # + # Prism returns the errors with their messages, but Ripper + # does not. To get them we must make a custom subclass. # # Example: # diff --git a/lib/syntax_suggest/scan_history.rb b/lib/syntax_suggest/scan_history.rb index d15597c440..dc36e6ba2e 100644 --- a/lib/syntax_suggest/scan_history.rb +++ b/lib/syntax_suggest/scan_history.rb @@ -118,7 +118,7 @@ module SyntaxSuggest # Returns an array of all the CodeLines that exist after # the currently scanned block private def after_lines - @code_lines[@after_index.next..-1] || [] + @code_lines[@after_index.next..] || [] end private def current diff --git a/lib/syntax_suggest/syntax_suggest.gemspec b/lib/syntax_suggest/syntax_suggest.gemspec index 0e611c13d0..756a85bf63 100644 --- a/lib/syntax_suggest/syntax_suggest.gemspec +++ b/lib/syntax_suggest/syntax_suggest.gemspec @@ -16,7 +16,7 @@ Gem::Specification.new do |spec| spec.description = 'When you get an "unexpected end" in your syntax this gem helps you find it' spec.homepage = "https://github.com/ruby/syntax_suggest.git" spec.license = "MIT" - spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0") + spec.required_ruby_version = Gem::Requirement.new(">= 3.0.0") spec.metadata["homepage_uri"] = spec.homepage spec.metadata["source_code_uri"] = "https://github.com/ruby/syntax_suggest.git" diff --git a/lib/syntax_suggest/version.rb b/lib/syntax_suggest/version.rb index ac8c2f62e5..4320adb218 100644 --- a/lib/syntax_suggest/version.rb +++ b/lib/syntax_suggest/version.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true module SyntaxSuggest - VERSION = "1.1.0" + VERSION = "2.0.0" end diff --git a/lib/tempfile.gemspec b/lib/tempfile.gemspec index a3cb11296d..0b362b0a86 100644 --- a/lib/tempfile.gemspec +++ b/lib/tempfile.gemspec @@ -22,8 +22,12 @@ Gem::Specification.new do |spec| # Specify which files should be added to the gem when it is released. # The `git ls-files -z` loads the files in the RubyGem that have been added into git. - spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do - `git ls-files -z 2>#{IO::NULL}`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } + gemspec = File.basename(__FILE__) + spec.files = IO.popen(%w[git ls-files -z], chdir: __dir__, err: IO::NULL, exception: false) do |ls| + ls.readlines("\x0", chomp: true).reject do |f| + (f == gemspec) || + f.start_with?(*%w[bin/ test/ spec/ features/ .git Gemfile]) + end end spec.require_paths = ["lib"] end diff --git a/lib/tempfile.rb b/lib/tempfile.rb index d53d8c7196..1d7b80a74d 100644 --- a/lib/tempfile.rb +++ b/lib/tempfile.rb @@ -88,7 +88,8 @@ require 'tmpdir' # mutex. class Tempfile < DelegateClass(File) - VERSION = "0.1.3" + # The version + VERSION = "0.2.1" # Creates a file in the underlying file system; # returns a new \Tempfile object based on that file. @@ -152,26 +153,50 @@ class Tempfile < DelegateClass(File) @unlinked = false @mode = mode|File::RDWR|File::CREAT|File::EXCL + @finalizer_obj = Object.new + tmpfile = nil ::Dir::Tmpname.create(basename, tmpdir, **options) do |tmpname, n, opts| opts[:perm] = 0600 - @tmpfile = File.open(tmpname, @mode, **opts) + tmpfile = File.open(tmpname, @mode, **opts) @opts = opts.freeze end - ObjectSpace.define_finalizer(self, Remover.new(@tmpfile)) + ObjectSpace.define_finalizer(@finalizer_obj, Remover.new(tmpfile.path)) + ObjectSpace.define_finalizer(self, Closer.new(tmpfile)) - super(@tmpfile) + super(tmpfile) + end + + def initialize_dup(other) # :nodoc: + initialize_copy_iv(other) + super(other) + ObjectSpace.define_finalizer(self, Closer.new(__getobj__)) + end + + def initialize_clone(other) # :nodoc: + initialize_copy_iv(other) + super(other) + ObjectSpace.define_finalizer(self, Closer.new(__getobj__)) + end + + private def initialize_copy_iv(other) # :nodoc: + @unlinked = other.unlinked + @mode = other.mode + @opts = other.opts + @finalizer_obj = other.finalizer_obj end # Opens or reopens the file with mode "r+". def open _close + ObjectSpace.undefine_finalizer(self) mode = @mode & ~(File::CREAT|File::EXCL) - @tmpfile = File.open(@tmpfile.path, mode, **@opts) - __setobj__(@tmpfile) + __setobj__(File.open(__getobj__.path, mode, **@opts)) + ObjectSpace.define_finalizer(self, Closer.new(__getobj__)) + __getobj__ end def _close # :nodoc: - @tmpfile.close + __getobj__.close end protected :_close @@ -228,13 +253,13 @@ class Tempfile < DelegateClass(File) def unlink return if @unlinked begin - File.unlink(@tmpfile.path) + File.unlink(__getobj__.path) rescue Errno::ENOENT rescue Errno::EACCES # may not be able to unlink on Windows; just ignore return end - ObjectSpace.undefine_finalizer(self) + ObjectSpace.undefine_finalizer(@finalizer_obj) @unlinked = true end alias delete unlink @@ -242,43 +267,57 @@ class Tempfile < DelegateClass(File) # Returns the full path name of the temporary file. # This will be nil if #unlink has been called. def path - @unlinked ? nil : @tmpfile.path + @unlinked ? nil : __getobj__.path end # Returns the size of the temporary file. As a side effect, the IO # buffer is flushed before determining the size. def size - if !@tmpfile.closed? - @tmpfile.size # File#size calls rb_io_flush_raw() + if !__getobj__.closed? + __getobj__.size # File#size calls rb_io_flush_raw() else - File.size(@tmpfile.path) + File.size(__getobj__.path) end end alias length size # :stopdoc: def inspect - if @tmpfile.closed? + if __getobj__.closed? "#<#{self.class}:#{path} (closed)>" else "#<#{self.class}:#{path}>" end end + alias to_s inspect - class Remover # :nodoc: + protected + + attr_reader :unlinked, :mode, :opts, :finalizer_obj + + class Closer # :nodoc: def initialize(tmpfile) - @pid = Process.pid @tmpfile = tmpfile end def call(*args) + @tmpfile.close + end + end + + class Remover # :nodoc: + def initialize(path) + @pid = Process.pid + @path = path + end + + def call(*args) return if @pid != Process.pid - $stderr.puts "removing #{@tmpfile.path}..." if $DEBUG + $stderr.puts "removing #{@path}..." if $DEBUG - @tmpfile.close begin - File.unlink(@tmpfile.path) + File.unlink(@path) rescue Errno::ENOENT end diff --git a/lib/time.gemspec b/lib/time.gemspec index a9349a253f..4b9f9e1218 100644 --- a/lib/time.gemspec +++ b/lib/time.gemspec @@ -20,8 +20,12 @@ Gem::Specification.new do |spec| spec.metadata["homepage_uri"] = spec.homepage spec.metadata["source_code_uri"] = spec.homepage - spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do - `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } + srcdir, gemspec = File.split(__FILE__) + spec.files = Dir.chdir(srcdir) do + `git ls-files -z`.split("\x0").reject { |f| + f == gemspec or + f.start_with?(".git", "bin/", "test/", "rakelib/", "Gemfile", "Rakefile") + } end spec.bindir = "exe" spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } diff --git a/lib/time.rb b/lib/time.rb index c572e76d7d..b565130b50 100644 --- a/lib/time.rb +++ b/lib/time.rb @@ -26,7 +26,7 @@ require 'date' class Time - VERSION = "0.2.2" + VERSION = "0.3.0" class << Time @@ -391,6 +391,8 @@ class Time # heuristic to detect the format of the input string, you provide # a second argument that describes the format of the string. # + # Raises `ArgumentError` if the date or format is invalid. + # # If a block is given, the year described in +date+ is converted by the # block. For example: # @@ -455,7 +457,7 @@ class Time # def strptime(date, format, now=self.now) d = Date._strptime(date, format) - raise ArgumentError, "invalid date or strptime format - `#{date}' `#{format}'" unless d + raise ArgumentError, "invalid date or strptime format - '#{date}' '#{format}'" unless d if seconds = d[:seconds] if sec_fraction = d[:sec_fraction] usec = sec_fraction * 1000000 diff --git a/lib/timeout.gemspec b/lib/timeout.gemspec index 23c6e5fe95..5449494e07 100644 --- a/lib/timeout.gemspec +++ b/lib/timeout.gemspec @@ -20,6 +20,9 @@ Gem::Specification.new do |spec| spec.metadata["homepage_uri"] = spec.homepage spec.metadata["source_code_uri"] = spec.homepage + spec.metadata["changelog_uri"] = spec.homepage + "/releases" + + spec.required_ruby_version = '>= 2.6.0' spec.files = Dir.chdir(__dir__) do `git ls-files -z`.split("\x0").reject do |f| diff --git a/lib/timeout.rb b/lib/timeout.rb index e1107765ab..c67a748856 100644 --- a/lib/timeout.rb +++ b/lib/timeout.rb @@ -23,29 +23,27 @@ # Copyright:: (C) 2000 Information-technology Promotion Agency, Japan module Timeout - VERSION = "0.3.2" + # The version + VERSION = "0.4.1" + + # Internal error raised to when a timeout is triggered. + class ExitException < Exception + def exception(*) # :nodoc: + self + end + end # Raised by Timeout.timeout when the block times out. class Error < RuntimeError - attr_reader :thread + def self.handle_timeout(message) # :nodoc: + exc = ExitException.new(message) - def self.catch(*args) - exc = new(*args) - exc.instance_variable_set(:@thread, Thread.current) - exc.instance_variable_set(:@catch_value, exc) - ::Kernel.catch(exc) {yield exc} - end - - def exception(*) - # TODO: use Fiber.current to see if self can be thrown - if self.thread == Thread.current - bt = caller - begin - throw(@catch_value, bt) - rescue UncaughtThrowError - end + begin + yield exc + rescue ExitException => e + raise new(message) if exc.equal?(e) + raise end - super end end @@ -195,8 +193,7 @@ module Timeout if klass perform.call(klass) else - backtrace = Error.catch(&perform) - raise Error, message, backtrace + Error.handle_timeout(message, &perform) end end module_function :timeout diff --git a/lib/tmpdir.gemspec b/lib/tmpdir.gemspec index a480b9c018..4c96e5984b 100644 --- a/lib/tmpdir.gemspec +++ b/lib/tmpdir.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |spec| spec.name = "tmpdir" - spec.version = "0.1.3" + spec.version = "0.2.0" spec.authors = ["Yukihiro Matsumoto"] spec.email = ["matz@ruby-lang.org"] @@ -15,8 +15,12 @@ Gem::Specification.new do |spec| # Specify which files should be added to the gem when it is released. # The `git ls-files -z` loads the files in the RubyGem that have been added into git. - spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do - `git ls-files -z 2>#{IO::NULL}`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } + gemspec = File.basename(__FILE__) + spec.files = IO.popen(%w[git ls-files -z], chdir: __dir__, err: IO::NULL, exception: false) do |ls| + ls.readlines("\x0", chomp: true).reject do |f| + (f == gemspec) || + f.start_with?(*%w[bin/ test/ spec/ features/ .git Gemfile]) + end end spec.bindir = "exe" spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } diff --git a/lib/tmpdir.rb b/lib/tmpdir.rb index 55920a4a74..fe3e0e19d1 100644 --- a/lib/tmpdir.rb +++ b/lib/tmpdir.rb @@ -13,15 +13,23 @@ end class Dir - @@systmpdir ||= defined?(Etc.systmpdir) ? Etc.systmpdir : '/tmp' + # Class variables are inaccessible from non-main Ractor. + # And instance variables too, in Ruby 3.0. + + # System-wide temporary directory path + SYSTMPDIR = (defined?(Etc.systmpdir) ? Etc.systmpdir.freeze : '/tmp') + private_constant :SYSTMPDIR ## # Returns the operating system's temporary file path. + # + # require 'tmpdir' + # Dir.tmpdir # => "/tmp" def self.tmpdir - ['TMPDIR', 'TMP', 'TEMP', ['system temporary path', @@systmpdir], ['/tmp']*2, ['.']*2].find do |name, dir| + ['TMPDIR', 'TMP', 'TEMP', ['system temporary path', SYSTMPDIR], ['/tmp']*2, ['.']*2].find do |name, dir| unless dir - next if !(dir = ENV[name]) or dir.empty? + next if !(dir = ENV[name] rescue next) or dir.empty? end dir = File.expand_path(dir) stat = File.stat(dir) rescue next @@ -40,6 +48,11 @@ class Dir # Dir.mktmpdir creates a temporary directory. # + # require 'tmpdir' + # Dir.mktmpdir {|dir| + # # use the directory + # } + # # The directory is created with 0700 permission. # Application should not change the permission to make the temporary directory accessible from other users. # @@ -94,9 +107,10 @@ class Dir yield path.dup ensure unless base - stat = File.stat(File.dirname(path)) + base = File.dirname(path) + stat = File.stat(base) if stat.world_writable? and !stat.sticky? - raise ArgumentError, "parent directory is world writable but not sticky" + raise ArgumentError, "parent directory is world writable but not sticky: #{base}" end end FileUtils.remove_entry path @@ -118,16 +132,17 @@ class Dir UNUSABLE_CHARS = "^,-.0-9A-Z_a-z~" # Dedicated random number generator - RANDOM = Random.new + RANDOM = Object.new class << RANDOM # :nodoc: # Maximum random number MAX = 36**6 # < 0x100000000 # Returns new random string upto 6 bytes def next - rand(MAX).to_s(36) + (::Random.urandom(4).unpack1("L")%MAX).to_s(36) end end + RANDOM.freeze private_constant :RANDOM # Generates and yields random names to create a temporary name @@ -152,7 +167,7 @@ class Dir n ||= 0 n += 1 retry if !max_try or n < max_try - raise "cannot generate temporary name using `#{basename}' under `#{tmpdir}'" + raise "cannot generate temporary name using '#{basename}' under '#{tmpdir}'" end path end diff --git a/lib/tsort.rb b/lib/tsort.rb index 3a2a91a7b4..dbaed45415 100644 --- a/lib/tsort.rb +++ b/lib/tsort.rb @@ -123,7 +123,7 @@ module TSort - VERSION = "0.1.1" + VERSION = "0.2.0" class Cyclic < StandardError end @@ -30,7 +30,6 @@ require "fileutils" require "optparse" module FileUtils -# @fileutils_label = "" @fileutils_output = $stdout end @@ -413,7 +412,7 @@ end module UN # :nodoc: - VERSION = "0.2.1" + VERSION = "0.3.0" module_function def help(argv, output: $stdout) diff --git a/lib/uri.rb b/lib/uri.rb index 59a7c4ad28..dfdb052a79 100644 --- a/lib/uri.rb +++ b/lib/uri.rb @@ -1,6 +1,6 @@ # frozen_string_literal: false # URI is a module providing classes to handle Uniform Resource Identifiers -# (RFC2396[http://tools.ietf.org/html/rfc2396]). +# (RFC2396[https://www.rfc-editor.org/rfc/rfc2396]). # # == Features # @@ -47,14 +47,14 @@ # A good place to view an RFC spec is http://www.ietf.org/rfc.html. # # Here is a list of all related RFC's: -# - RFC822[http://tools.ietf.org/html/rfc822] -# - RFC1738[http://tools.ietf.org/html/rfc1738] -# - RFC2255[http://tools.ietf.org/html/rfc2255] -# - RFC2368[http://tools.ietf.org/html/rfc2368] -# - RFC2373[http://tools.ietf.org/html/rfc2373] -# - RFC2396[http://tools.ietf.org/html/rfc2396] -# - RFC2732[http://tools.ietf.org/html/rfc2732] -# - RFC3986[http://tools.ietf.org/html/rfc3986] +# - RFC822[https://www.rfc-editor.org/rfc/rfc822] +# - RFC1738[https://www.rfc-editor.org/rfc/rfc1738] +# - RFC2255[https://www.rfc-editor.org/rfc/rfc2255] +# - RFC2368[https://www.rfc-editor.org/rfc/rfc2368] +# - RFC2373[https://www.rfc-editor.org/rfc/rfc2373] +# - RFC2396[https://www.rfc-editor.org/rfc/rfc2396] +# - RFC2732[https://www.rfc-editor.org/rfc/rfc2732] +# - RFC3986[https://www.rfc-editor.org/rfc/rfc3986] # # == Class tree # diff --git a/lib/uri/ftp.rb b/lib/uri/ftp.rb index abad613c33..1c75e242ba 100644 --- a/lib/uri/ftp.rb +++ b/lib/uri/ftp.rb @@ -17,7 +17,7 @@ module URI # This class will be redesigned because of difference of implementations; # the structure of its path. draft-hoffman-ftp-uri-04 is a draft but it # is a good summary about the de facto spec. - # http://tools.ietf.org/html/draft-hoffman-ftp-uri-04 + # https://datatracker.ietf.org/doc/html/draft-hoffman-ftp-uri-04 # class FTP < Generic # A Default port of 21 for URI::FTP. diff --git a/lib/uri/generic.rb b/lib/uri/generic.rb index 69698c4e2d..bdd366661e 100644 --- a/lib/uri/generic.rb +++ b/lib/uri/generic.rb @@ -945,7 +945,7 @@ module URI # == Description # # URI has components listed in order of decreasing significance from left to right, - # see RFC3986 https://tools.ietf.org/html/rfc3986 1.2.3. + # see RFC3986 https://www.rfc-editor.org/rfc/rfc3986 1.2.3. # # == Usage # @@ -1364,6 +1364,9 @@ module URI str << ':' str << @port.to_s end + if (@host || @port) && !@path.empty? && !@path.start_with?('/') + str << '/' + end str << @path if @query str << '?' @@ -1376,6 +1379,7 @@ module URI end str end + alias to_str to_s # # Compares two URIs. @@ -1398,19 +1402,6 @@ module URI self.component_ary.eql?(oth.component_ary) end -=begin - ---- URI::Generic#===(oth) - -=end -# def ===(oth) -# raise NotImplementedError -# end - -=begin -=end - - # Returns an Array of the components defined from the COMPONENT Array. def component_ary component.collect do |x| diff --git a/lib/uri/http.rb b/lib/uri/http.rb index 306daf1965..900b132c8c 100644 --- a/lib/uri/http.rb +++ b/lib/uri/http.rb @@ -85,7 +85,7 @@ module URI # == Description # # Returns the authority for an HTTP uri, as defined in - # https://datatracker.ietf.org/doc/html/rfc3986/#section-3.2. + # https://www.rfc-editor.org/rfc/rfc3986#section-3.2. # # # Example: @@ -106,7 +106,7 @@ module URI # == Description # # Returns the origin for an HTTP uri, as defined in - # https://datatracker.ietf.org/doc/html/rfc6454. + # https://www.rfc-editor.org/rfc/rfc6454. # # # Example: diff --git a/lib/uri/rfc2396_parser.rb b/lib/uri/rfc2396_parser.rb index 76a8f99fd4..00c66cf042 100644 --- a/lib/uri/rfc2396_parser.rb +++ b/lib/uri/rfc2396_parser.rb @@ -497,8 +497,8 @@ module URI ret = {} # for URI::split - ret[:ABS_URI] = Regexp.new('\A\s*' + pattern[:X_ABS_URI] + '\s*\z', Regexp::EXTENDED) - ret[:REL_URI] = Regexp.new('\A\s*' + pattern[:X_REL_URI] + '\s*\z', Regexp::EXTENDED) + ret[:ABS_URI] = Regexp.new('\A\s*+' + pattern[:X_ABS_URI] + '\s*\z', Regexp::EXTENDED) + ret[:REL_URI] = Regexp.new('\A\s*+' + pattern[:X_REL_URI] + '\s*\z', Regexp::EXTENDED) # for URI::extract ret[:URI_REF] = Regexp.new(pattern[:URI_REF]) diff --git a/lib/uri/rfc3986_parser.rb b/lib/uri/rfc3986_parser.rb index dd24a409ea..092a1ac89d 100644 --- a/lib/uri/rfc3986_parser.rb +++ b/lib/uri/rfc3986_parser.rb @@ -1,9 +1,73 @@ -# frozen_string_literal: false +# frozen_string_literal: true module URI class RFC3986_Parser # :nodoc: # URI defined in RFC3986 - RFC3986_URI = /\A(?<URI>(?<scheme>[A-Za-z][+\-.0-9A-Za-z]*+):(?<hier-part>\/\/(?<authority>(?:(?<userinfo>(?:%\h\h|[!$&-.0-;=A-Z_a-z~])*+)@)?(?<host>(?<IP-literal>\[(?:(?<IPv6address>(?:\h{1,4}:){6}(?<ls32>\h{1,4}:\h{1,4}|(?<IPv4address>(?<dec-octet>[1-9]\d|1\d{2}|2[0-4]\d|25[0-5]|\d)\.\g<dec-octet>\.\g<dec-octet>\.\g<dec-octet>))|::(?:\h{1,4}:){5}\g<ls32>|\h{1,4}?::(?:\h{1,4}:){4}\g<ls32>|(?:(?:\h{1,4}:)?\h{1,4})?::(?:\h{1,4}:){3}\g<ls32>|(?:(?:\h{1,4}:){,2}\h{1,4})?::(?:\h{1,4}:){2}\g<ls32>|(?:(?:\h{1,4}:){,3}\h{1,4})?::\h{1,4}:\g<ls32>|(?:(?:\h{1,4}:){,4}\h{1,4})?::\g<ls32>|(?:(?:\h{1,4}:){,5}\h{1,4})?::\h{1,4}|(?:(?:\h{1,4}:){,6}\h{1,4})?::)|(?<IPvFuture>v\h++\.[!$&-.0-;=A-Z_a-z~]++))\])|\g<IPv4address>|(?<reg-name>(?:%\h\h|[!$&-.0-9;=A-Z_a-z~])*+))(?::(?<port>\d*+))?)(?<path-abempty>(?:\/(?<segment>(?:%\h\h|[!$&-.0-;=@-Z_a-z~])*+))*+)|(?<path-absolute>\/(?:(?<segment-nz>(?:%\h\h|[!$&-.0-;=@-Z_a-z~])++)(?:\/\g<segment>)*+)?)|(?<path-rootless>\g<segment-nz>(?:\/\g<segment>)*+)|(?<path-empty>))(?:\?(?<query>[^#]*+))?(?:\#(?<fragment>(?:%\h\h|[!$&-.0-;=@-Z_a-z~\/?])*+))?)\z/ - RFC3986_relative_ref = /\A(?<relative-ref>(?<relative-part>\/\/(?<authority>(?:(?<userinfo>(?:%\h\h|[!$&-.0-;=A-Z_a-z~])*+)@)?(?<host>(?<IP-literal>\[(?:(?<IPv6address>(?:\h{1,4}:){6}(?<ls32>\h{1,4}:\h{1,4}|(?<IPv4address>(?<dec-octet>[1-9]\d|1\d{2}|2[0-4]\d|25[0-5]|\d)\.\g<dec-octet>\.\g<dec-octet>\.\g<dec-octet>))|::(?:\h{1,4}:){5}\g<ls32>|\h{1,4}?::(?:\h{1,4}:){4}\g<ls32>|(?:(?:\h{1,4}:){,1}\h{1,4})?::(?:\h{1,4}:){3}\g<ls32>|(?:(?:\h{1,4}:){,2}\h{1,4})?::(?:\h{1,4}:){2}\g<ls32>|(?:(?:\h{1,4}:){,3}\h{1,4})?::\h{1,4}:\g<ls32>|(?:(?:\h{1,4}:){,4}\h{1,4})?::\g<ls32>|(?:(?:\h{1,4}:){,5}\h{1,4})?::\h{1,4}|(?:(?:\h{1,4}:){,6}\h{1,4})?::)|(?<IPvFuture>v\h++\.[!$&-.0-;=A-Z_a-z~]++))\])|\g<IPv4address>|(?<reg-name>(?:%\h\h|[!$&-.0-9;=A-Z_a-z~])++))?(?::(?<port>\d*+))?)(?<path-abempty>(?:\/(?<segment>(?:%\h\h|[!$&-.0-;=@-Z_a-z~])*+))*+)|(?<path-absolute>\/(?:(?<segment-nz>(?:%\h\h|[!$&-.0-;=@-Z_a-z~])++)(?:\/\g<segment>)*+)?)|(?<path-noscheme>(?<segment-nz-nc>(?:%\h\h|[!$&-.0-9;=@-Z_a-z~])++)(?:\/\g<segment>)*+)|(?<path-empty>))(?:\?(?<query>[^#]*+))?(?:\#(?<fragment>(?:%\h\h|[!$&-.0-;=@-Z_a-z~\/?])*+))?)\z/ + HOST = %r[ + (?<IP-literal>\[(?: + (?<IPv6address> + (?:\h{1,4}:){6} + (?<ls32>\h{1,4}:\h{1,4} + | (?<IPv4address>(?<dec-octet>[1-9]\d|1\d{2}|2[0-4]\d|25[0-5]|\d) + \.\g<dec-octet>\.\g<dec-octet>\.\g<dec-octet>) + ) + | ::(?:\h{1,4}:){5}\g<ls32> + | \h{1,4}?::(?:\h{1,4}:){4}\g<ls32> + | (?:(?:\h{1,4}:)?\h{1,4})?::(?:\h{1,4}:){3}\g<ls32> + | (?:(?:\h{1,4}:){,2}\h{1,4})?::(?:\h{1,4}:){2}\g<ls32> + | (?:(?:\h{1,4}:){,3}\h{1,4})?::\h{1,4}:\g<ls32> + | (?:(?:\h{1,4}:){,4}\h{1,4})?::\g<ls32> + | (?:(?:\h{1,4}:){,5}\h{1,4})?::\h{1,4} + | (?:(?:\h{1,4}:){,6}\h{1,4})?:: + ) + | (?<IPvFuture>v\h++\.[!$&-.0-9:;=A-Z_a-z~]++) + )\]) + | \g<IPv4address> + | (?<reg-name>(?:%\h\h|[!$&-.0-9;=A-Z_a-z~])*+) + ]x + + USERINFO = /(?:%\h\h|[!$&-.0-9:;=A-Z_a-z~])*+/ + + SCHEME = %r[[A-Za-z][+\-.0-9A-Za-z]*+].source + SEG = %r[(?:%\h\h|[!$&-.0-9:;=@A-Z_a-z~/])].source + SEG_NC = %r[(?:%\h\h|[!$&-.0-9;=@A-Z_a-z~])].source + FRAGMENT = %r[(?:%\h\h|[!$&-.0-9:;=@A-Z_a-z~/?])*+].source + + RFC3986_URI = %r[\A + (?<seg>#{SEG}){0} + (?<URI> + (?<scheme>#{SCHEME}): + (?<hier-part>// + (?<authority> + (?:(?<userinfo>#{USERINFO.source})@)? + (?<host>#{HOST.source.delete(" \n")}) + (?::(?<port>\d*+))? + ) + (?<path-abempty>(?:/\g<seg>*+)?) + | (?<path-absolute>/((?!/)\g<seg>++)?) + | (?<path-rootless>(?!/)\g<seg>++) + | (?<path-empty>) + ) + (?:\?(?<query>[^\#]*+))? + (?:\#(?<fragment>#{FRAGMENT}))? + )\z]x + + RFC3986_relative_ref = %r[\A + (?<seg>#{SEG}){0} + (?<relative-ref> + (?<relative-part>// + (?<authority> + (?:(?<userinfo>#{USERINFO.source})@)? + (?<host>#{HOST.source.delete(" \n")}(?<!/))? + (?::(?<port>\d*+))? + ) + (?<path-abempty>(?:/\g<seg>*+)?) + | (?<path-absolute>/\g<seg>*+) + | (?<path-noscheme>#{SEG_NC}++(?:/\g<seg>*+)?) + | (?<path-empty>) + ) + (?:\?(?<query>[^#]*+))? + (?:\#(?<fragment>#{FRAGMENT}))? + )\z]x attr_reader :regexp def initialize @@ -19,9 +83,9 @@ module URI uri.ascii_only? or raise InvalidURIError, "URI must be ascii only #{uri.dump}" if m = RFC3986_URI.match(uri) - query = m["query".freeze] - scheme = m["scheme".freeze] - opaque = m["path-rootless".freeze] + query = m["query"] + scheme = m["scheme"] + opaque = m["path-rootless"] if opaque opaque << "?#{query}" if query [ scheme, @@ -32,35 +96,35 @@ module URI nil, # path opaque, nil, # query - m["fragment".freeze] + m["fragment"] ] else # normal [ scheme, - m["userinfo".freeze], - m["host".freeze], - m["port".freeze], + m["userinfo"], + m["host"], + m["port"], nil, # registry - (m["path-abempty".freeze] || - m["path-absolute".freeze] || - m["path-empty".freeze]), + (m["path-abempty"] || + m["path-absolute"] || + m["path-empty"]), nil, # opaque query, - m["fragment".freeze] + m["fragment"] ] end elsif m = RFC3986_relative_ref.match(uri) [ nil, # scheme - m["userinfo".freeze], - m["host".freeze], - m["port".freeze], + m["userinfo"], + m["host"], + m["port"], nil, # registry, - (m["path-abempty".freeze] || - m["path-absolute".freeze] || - m["path-noscheme".freeze] || - m["path-empty".freeze]), + (m["path-abempty"] || + m["path-absolute"] || + m["path-noscheme"] || + m["path-empty"]), nil, # opaque - m["query".freeze], - m["fragment".freeze] + m["query"], + m["fragment"] ] else raise InvalidURIError, "bad URI(is not URI?): #{uri.inspect}" @@ -92,15 +156,15 @@ module URI def default_regexp # :nodoc: { - SCHEME: /\A[A-Za-z][A-Za-z0-9+\-.]*\z/, - USERINFO: /\A(?:%\h\h|[!$&-.0-;=A-Z_a-z~])*\z/, - HOST: /\A(?:(?<IP-literal>\[(?:(?<IPv6address>(?:\h{1,4}:){6}(?<ls32>\h{1,4}:\h{1,4}|(?<IPv4address>(?<dec-octet>[1-9]\d|1\d{2}|2[0-4]\d|25[0-5]|\d)\.\g<dec-octet>\.\g<dec-octet>\.\g<dec-octet>))|::(?:\h{1,4}:){5}\g<ls32>|\h{,4}::(?:\h{1,4}:){4}\g<ls32>|(?:(?:\h{1,4}:)?\h{1,4})?::(?:\h{1,4}:){3}\g<ls32>|(?:(?:\h{1,4}:){,2}\h{1,4})?::(?:\h{1,4}:){2}\g<ls32>|(?:(?:\h{1,4}:){,3}\h{1,4})?::\h{1,4}:\g<ls32>|(?:(?:\h{1,4}:){,4}\h{1,4})?::\g<ls32>|(?:(?:\h{1,4}:){,5}\h{1,4})?::\h{1,4}|(?:(?:\h{1,4}:){,6}\h{1,4})?::)|(?<IPvFuture>v\h+\.[!$&-.0-;=A-Z_a-z~]+))\])|\g<IPv4address>|(?<reg-name>(?:%\h\h|[!$&-.0-9;=A-Z_a-z~])*))\z/, - ABS_PATH: /\A\/(?:%\h\h|[!$&-.0-;=@-Z_a-z~])*(?:\/(?:%\h\h|[!$&-.0-;=@-Z_a-z~])*)*\z/, - REL_PATH: /\A(?:%\h\h|[!$&-.0-;=@-Z_a-z~])+(?:\/(?:%\h\h|[!$&-.0-;=@-Z_a-z~])*)*\z/, - QUERY: /\A(?:%\h\h|[!$&-.0-;=@-Z_a-z~\/?])*\z/, - FRAGMENT: /\A(?:%\h\h|[!$&-.0-;=@-Z_a-z~\/?])*\z/, - OPAQUE: /\A(?:[^\/].*)?\z/, - PORT: /\A[\x09\x0a\x0c\x0d ]*\d*[\x09\x0a\x0c\x0d ]*\z/, + SCHEME: %r[\A#{SCHEME}\z]o, + USERINFO: %r[\A#{USERINFO}\z]o, + HOST: %r[\A#{HOST}\z]o, + ABS_PATH: %r[\A/#{SEG}*+\z]o, + REL_PATH: %r[\A(?!/)#{SEG}++\z]o, + QUERY: %r[\A(?:%\h\h|[!$&-.0-9:;=@A-Z_a-z~/?])*+\z], + FRAGMENT: %r[\A#{FRAGMENT}\z]o, + OPAQUE: %r[\A(?:[^/].*)?\z], + PORT: /\A[\x09\x0a\x0c\x0d ]*+\d*[\x09\x0a\x0c\x0d ]*\z/, } end diff --git a/lib/uri/version.rb b/lib/uri/version.rb index 7497a7d31a..2dafa57d59 100644 --- a/lib/uri/version.rb +++ b/lib/uri/version.rb @@ -1,6 +1,6 @@ module URI # :stopdoc: - VERSION_CODE = '001201'.freeze + VERSION_CODE = '001300'.freeze VERSION = VERSION_CODE.scan(/../).collect{|n| n.to_i}.join('.').freeze # :startdoc: end diff --git a/lib/weakref.rb b/lib/weakref.rb index 2bbadf68f9..a8da39a26a 100644 --- a/lib/weakref.rb +++ b/lib/weakref.rb @@ -17,7 +17,7 @@ require "delegate" # class WeakRef < Delegator - VERSION = "0.1.2" + VERSION = "0.1.3" ## # RefError is raised when a referenced object has been recycled by the diff --git a/lib/yaml.rb b/lib/yaml.rb index 3b2a80f1df..b2669899dd 100644 --- a/lib/yaml.rb +++ b/lib/yaml.rb @@ -66,5 +66,5 @@ YAML = Psych # :nodoc: # # Syck can also be found on github: https://github.com/ruby/syck module YAML - LOADER_VERSION = "0.2.1" + LOADER_VERSION = "0.3.0" end diff --git a/lib/yaml/store.rb b/lib/yaml/store.rb index f8650b942f..f24e4ad332 100644 --- a/lib/yaml/store.rb +++ b/lib/yaml/store.rb @@ -3,7 +3,11 @@ # YAML::Store # require 'yaml' -require 'pstore' + +begin + require 'pstore' +rescue LoadError +end # YAML::Store provides the same functionality as PStore, except it uses YAML # to dump objects instead of Marshal. @@ -65,7 +69,7 @@ class YAML::Store < PStore end def load(content) - table = YAML.unsafe_load(content) + table = YAML.respond_to?(:unsafe_load) ? YAML.unsafe_load(content) : YAML.load(content) if table == false {} else @@ -83,4 +87,4 @@ class YAML::Store < PStore def empty_marshal_checksum CHECKSUM_ALGO.digest(empty_marshal_data) end -end +end if defined?(::PStore) |