diff options
Diffstat (limited to 'lib')
130 files changed, 2992 insertions, 1836 deletions
diff --git a/lib/bundled_gems.rb b/lib/bundled_gems.rb index ed5f940b57..932949576b 100644 --- a/lib/bundled_gems.rb +++ b/lib/bundled_gems.rb @@ -28,23 +28,14 @@ module Gem::BUNDLED_GEMS "syslog" => "3.4.0", "ostruct" => "3.5.0", "pstore" => "3.5.0", + "rdoc" => "3.5.0", + "win32ole" => "3.5.0", + "fiddle" => "3.5.0", + "logger" => "3.5.0", }.freeze EXACT = { - "abbrev" => true, - "base64" => true, - "bigdecimal" => true, - "csv" => true, - "drb" => true, - "getoptlong" => true, - "mutex_m" => true, - "nkf" => true, "kconv" => "nkf", - "observer" => true, - "resolv-replace" => true, - "rinda" => true, - "syslog" => true, - "ostruct" => true, - "pstore" => true, + "kconv" => "nkf", }.freeze PREFIXED = { @@ -95,7 +86,7 @@ module Gem::BUNDLED_GEMS else return end - EXACT[n] or PREFIXED[n = n[%r[\A[^/]+(?=/)]]] && n + (EXACT[n] || !!SINCE[n]) or PREFIXED[n = n[%r[\A[^/]+(?=/)]]] && n end def self.warning?(name, specs: nil) diff --git a/lib/bundler.rb b/lib/bundler.rb index 5033109db6..0081b9554f 100644 --- a/lib/bundler.rb +++ b/lib/bundler.rb @@ -42,6 +42,7 @@ module Bundler autoload :Checksum, File.expand_path("bundler/checksum", __dir__) autoload :CLI, File.expand_path("bundler/cli", __dir__) autoload :CIDetector, File.expand_path("bundler/ci_detector", __dir__) + autoload :CompactIndexClient, File.expand_path("bundler/compact_index_client", __dir__) autoload :Definition, File.expand_path("bundler/definition", __dir__) autoload :Dependency, File.expand_path("bundler/dependency", __dir__) autoload :Deprecate, File.expand_path("bundler/deprecate", __dir__) @@ -357,7 +358,7 @@ module Bundler def settings @settings ||= Settings.new(app_config_path) rescue GemfileNotFound - @settings = Settings.new(Pathname.new(".bundle").expand_path) + @settings = Settings.new end # @return [Hash] Environment present before Bundler was activated diff --git a/lib/bundler/cli.rb b/lib/bundler/cli.rb index 40f19c7fed..eb67668cd2 100644 --- a/lib/bundler/cli.rb +++ b/lib/bundler/cli.rb @@ -767,13 +767,10 @@ module Bundler return unless SharedHelpers.md5_available? - latest = Fetcher::CompactIndex. - new(nil, Source::Rubygems::Remote.new(Gem::URI("https://rubygems.org")), nil, nil). - send(:compact_index_client). - instance_variable_get(:@cache). - dependencies("bundler"). - map {|d| Gem::Version.new(d.first) }. - max + require_relative "vendored_uri" + remote = Source::Rubygems::Remote.new(Gem::URI("https://rubygems.org")) + cache_path = Bundler.user_cache.join("compact_index", remote.cache_slug) + latest = Bundler::CompactIndexClient.new(cache_path).latest_version("bundler") return unless latest current = Gem::Version.new(VERSION) diff --git a/lib/bundler/cli/install.rb b/lib/bundler/cli/install.rb index 6c102d537d..a233d5d2e5 100644 --- a/lib/bundler/cli/install.rb +++ b/lib/bundler/cli/install.rb @@ -14,7 +14,7 @@ module Bundler Bundler.self_manager.install_locked_bundler_and_restart_with_it_if_needed - Bundler::SharedHelpers.set_env "RB_USER_INSTALL", "1" if Bundler::FREEBSD + Bundler::SharedHelpers.set_env "RB_USER_INSTALL", "1" if Gem.freebsd_platform? # Disable color in deployment mode Bundler.ui.shell = Thor::Shell::Basic.new if options[:deployment] diff --git a/lib/bundler/compact_index_client.rb b/lib/bundler/compact_index_client.rb index 68e0d7e0d5..692d68e579 100644 --- a/lib/bundler/compact_index_client.rb +++ b/lib/bundler/compact_index_client.rb @@ -4,6 +4,29 @@ require "pathname" require "set" module Bundler + # The CompactIndexClient is responsible for fetching and parsing the compact index. + # + # The compact index is a set of caching optimized files that are used to fetch gem information. + # The files are: + # - names: a list of all gem names + # - versions: a list of all gem versions + # - info/[gem]: a list of all versions of a gem + # + # The client is instantiated with: + # - `directory`: the root directory where the cache files are stored. + # - `fetcher`: (optional) an object that responds to #call(uri_path, headers) and returns an http response. + # If the `fetcher` is not provided, the client will only read cached files from disk. + # + # The client is organized into: + # - `Updater`: updates the cached files on disk using the fetcher. + # - `Cache`: calls the updater, caches files, read and return them from disk + # - `Parser`: parses the compact index file data + # - `CacheFile`: a concurrency safe file reader/writer that verifies checksums + # + # The client is intended to optimize memory usage and performance. + # It is called 100s or 1000s of times, parsing files with hundreds of thousands of lines. + # It may be called concurrently without global interpreter lock in some Rubies. + # As a result, some methods may look more complex than necessary to save memory or time. class CompactIndexClient # NOTE: MD5 is here not because we expect a server to respond with it, but # because we use it to generate the etag on first request during the upgrade @@ -12,6 +35,13 @@ module Bundler SUPPORTED_DIGESTS = { "sha-256" => :SHA256, "md5" => :MD5 }.freeze DEBUG_MUTEX = Thread::Mutex.new + # info returns an Array of INFO Arrays. Each INFO Array has the following indices: + INFO_NAME = 0 + INFO_VERSION = 1 + INFO_PLATFORM = 2 + INFO_DEPS = 3 + INFO_REQS = 4 + def self.debug return unless ENV["DEBUG_COMPACT_INDEX"] DEBUG_MUTEX.synchronize { warn("[#{self}] #{yield}") } @@ -21,106 +51,47 @@ module Bundler require_relative "compact_index_client/cache" require_relative "compact_index_client/cache_file" + require_relative "compact_index_client/parser" require_relative "compact_index_client/updater" - attr_reader :directory - - def initialize(directory, fetcher) - @directory = Pathname.new(directory) - @updater = Updater.new(fetcher) - @cache = Cache.new(@directory) - @endpoints = Set.new - @info_checksums_by_name = {} - @parsed_checksums = false - @mutex = Thread::Mutex.new - end - - def execution_mode=(block) - Bundler::CompactIndexClient.debug { "execution_mode=" } - @endpoints = Set.new - - @execution_mode = block - end - - # @return [Lambda] A lambda that takes an array of inputs and a block, and - # maps the inputs with the block in parallel. - # - def execution_mode - @execution_mode || sequentially - end - - def sequential_execution_mode! - self.execution_mode = sequentially - end - - def sequentially - @sequentially ||= lambda do |inputs, &blk| - inputs.map(&blk) - end + def initialize(directory, fetcher = nil) + @cache = Cache.new(directory, fetcher) + @parser = Parser.new(@cache) end def names - Bundler::CompactIndexClient.debug { "/names" } - update("names", @cache.names_path, @cache.names_etag_path) - @cache.names + Bundler::CompactIndexClient.debug { "names" } + @parser.names end def versions - Bundler::CompactIndexClient.debug { "/versions" } - update("versions", @cache.versions_path, @cache.versions_etag_path) - versions, @info_checksums_by_name = @cache.versions - versions + Bundler::CompactIndexClient.debug { "versions" } + @parser.versions end def dependencies(names) Bundler::CompactIndexClient.debug { "dependencies(#{names})" } - execution_mode.call(names) do |name| - update_info(name) - @cache.dependencies(name).map {|d| d.unshift(name) } - end.flatten(1) + names.map {|name| info(name) } end - def update_and_parse_checksums! - Bundler::CompactIndexClient.debug { "update_and_parse_checksums!" } - return @info_checksums_by_name if @parsed_checksums - update("versions", @cache.versions_path, @cache.versions_etag_path) - @info_checksums_by_name = @cache.checksums - @parsed_checksums = true - end - - private - - def update(remote_path, local_path, local_etag_path) - Bundler::CompactIndexClient.debug { "update(#{local_path}, #{remote_path})" } - unless synchronize { @endpoints.add?(remote_path) } - Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" } - return - end - @updater.update(url(remote_path), local_path, local_etag_path) + def info(name) + Bundler::CompactIndexClient.debug { "info(#{names})" } + @parser.info(name) end - def update_info(name) - Bundler::CompactIndexClient.debug { "update_info(#{name})" } - path = @cache.info_path(name) - unless existing = @info_checksums_by_name[name] - Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since it is missing from versions" } - return - end - checksum = SharedHelpers.checksum_for_file(path, :MD5) - if checksum == existing - Bundler::CompactIndexClient.debug { "skipping updating info for #{name} since the versions checksum matches the local checksum" } - return - end - Bundler::CompactIndexClient.debug { "updating info for #{name} since the versions checksum #{existing} != the local checksum #{checksum}" } - update("info/#{name}", path, @cache.info_etag_path(name)) + def latest_version(name) + Bundler::CompactIndexClient.debug { "latest_version(#{name})" } + @parser.info(name).map {|d| Gem::Version.new(d[INFO_VERSION]) }.max end - def url(path) - path + def available? + Bundler::CompactIndexClient.debug { "available?" } + @parser.available? end - def synchronize - @mutex.synchronize { yield } + def reset! + Bundler::CompactIndexClient.debug { "reset!" } + @cache.reset! end end end diff --git a/lib/bundler/compact_index_client/cache.rb b/lib/bundler/compact_index_client/cache.rb index 55911fdecf..bedd7f8028 100644 --- a/lib/bundler/compact_index_client/cache.rb +++ b/lib/bundler/compact_index_client/cache.rb @@ -7,114 +7,89 @@ module Bundler class Cache attr_reader :directory - def initialize(directory) + def initialize(directory, fetcher = nil) @directory = Pathname.new(directory).expand_path - info_roots.each {|dir| mkdir(dir) } - mkdir(info_etag_root) + @updater = Updater.new(fetcher) if fetcher + @mutex = Thread::Mutex.new + @endpoints = Set.new + + @info_root = mkdir("info") + @special_characters_info_root = mkdir("info-special-characters") + @info_etag_root = mkdir("info-etags") end def names - lines(names_path) + fetch("names", names_path, names_etag_path) end - def names_path - directory.join("names") + def versions + fetch("versions", versions_path, versions_etag_path) end - def names_etag_path - directory.join("names.etag") - end + def info(name, remote_checksum = nil) + path = info_path(name) - def versions - versions_by_name = Hash.new {|hash, key| hash[key] = [] } - info_checksums_by_name = {} - - lines(versions_path).each do |line| - name, versions_string, info_checksum = line.split(" ", 3) - info_checksums_by_name[name] = info_checksum || "" - versions_string.split(",") do |version| - delete = version.delete_prefix!("-") - version = version.split("-", 2).unshift(name) - if delete - versions_by_name[name].delete(version) - else - versions_by_name[name] << version - end - end + if remote_checksum && remote_checksum != SharedHelpers.checksum_for_file(path, :MD5) + fetch("info/#{name}", path, info_etag_path(name)) + else + Bundler::CompactIndexClient.debug { "update skipped info/#{name} (#{remote_checksum ? "versions index checksum is nil" : "versions index checksum matches local"})" } + read(path) end - - [versions_by_name, info_checksums_by_name] - end - - def versions_path - directory.join("versions") end - def versions_etag_path - directory.join("versions.etag") + def reset! + @mutex.synchronize { @endpoints.clear } end - def checksums - checksums = {} - - lines(versions_path).each do |line| - name, _, checksum = line.split(" ", 3) - checksums[name] = checksum - end - - checksums - end + private - def dependencies(name) - lines(info_path(name)).map do |line| - parse_gem(line) - end - end + def names_path = directory.join("names") + def names_etag_path = directory.join("names.etag") + def versions_path = directory.join("versions") + def versions_etag_path = directory.join("versions.etag") def info_path(name) name = name.to_s + # TODO: converge this into the info_root by hashing all filenames like info_etag_path if /[^a-z0-9_-]/.match?(name) name += "-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}" - info_roots.last.join(name) + @special_characters_info_root.join(name) else - info_roots.first.join(name) + @info_root.join(name) end end def info_etag_path(name) name = name.to_s - info_etag_root.join("#{name}-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}") + @info_etag_root.join("#{name}-#{SharedHelpers.digest(:MD5).hexdigest(name).downcase}") end - private - - def mkdir(dir) - SharedHelpers.filesystem_access(dir) do - FileUtils.mkdir_p(dir) + def mkdir(name) + directory.join(name).tap do |dir| + SharedHelpers.filesystem_access(dir) do + FileUtils.mkdir_p(dir) + end end end - def lines(path) - return [] unless path.file? - lines = SharedHelpers.filesystem_access(path, :read, &:read).split("\n") - header = lines.index("---") - header ? lines[header + 1..-1] : lines - end + def fetch(remote_path, path, etag_path) + if already_fetched?(remote_path) + Bundler::CompactIndexClient.debug { "already fetched #{remote_path}" } + else + Bundler::CompactIndexClient.debug { "fetching #{remote_path}" } + @updater&.update(remote_path, path, etag_path) + end - def parse_gem(line) - @dependency_parser ||= GemParser.new - @dependency_parser.parse(line) + read(path) end - def info_roots - [ - directory.join("info"), - directory.join("info-special-characters"), - ] + def already_fetched?(remote_path) + @mutex.synchronize { !@endpoints.add?(remote_path) } end - def info_etag_root - directory.join("info-etags") + def read(path) + return unless path.file? + SharedHelpers.filesystem_access(path, :read, &:read) end end end diff --git a/lib/bundler/compact_index_client/cache_file.rb b/lib/bundler/compact_index_client/cache_file.rb index 5988bc91b3..299d683438 100644 --- a/lib/bundler/compact_index_client/cache_file.rb +++ b/lib/bundler/compact_index_client/cache_file.rb @@ -86,11 +86,6 @@ module Bundler end end - # remove this method when we stop generating md5 digests for legacy etags - def md5 - @digests && @digests["md5"] - end - def digests? @digests&.any? end diff --git a/lib/bundler/compact_index_client/parser.rb b/lib/bundler/compact_index_client/parser.rb new file mode 100644 index 0000000000..3a0dec4907 --- /dev/null +++ b/lib/bundler/compact_index_client/parser.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +module Bundler + class CompactIndexClient + class Parser + # `compact_index` - an object responding to #names, #versions, #info(name, checksum), + # returning the file contents as a string + def initialize(compact_index) + @compact_index = compact_index + @info_checksums = nil + @versions_by_name = nil + @available = nil + @gem_parser = nil + @versions_data = nil + end + + def names + lines(@compact_index.names) + end + + def versions + @versions_by_name ||= Hash.new {|hash, key| hash[key] = [] } + @info_checksums = {} + + lines(@compact_index.versions).each do |line| + name, versions_string, checksum = line.split(" ", 3) + @info_checksums[name] = checksum || "" + versions_string.split(",") do |version| + delete = version.delete_prefix!("-") + version = version.split("-", 2).unshift(name) + if delete + @versions_by_name[name].delete(version) + else + @versions_by_name[name] << version + end + end + end + + @versions_by_name + end + + def info(name) + data = @compact_index.info(name, info_checksum(name)) + lines(data).map {|line| gem_parser.parse(line).unshift(name) } + end + + # parse the last, most recently updated line of the versions file to determine availability + def available? + return @available unless @available.nil? + return @available = false unless versions_data&.size&.nonzero? + + line_end = versions_data.size - 1 + return @available = false if versions_data[line_end] != "\n" + + line_start = versions_data.rindex("\n", line_end - 1) + line_start ||= -1 # allow a single line versions file + + @available = !split_last_word(versions_data, line_start + 1, line_end).nil? + end + + private + + # Search for a line starting with gem name, then return last space-separated word (the checksum) + def info_checksum(name) + return unless versions_data + return unless (line_start = rindex_of_gem(name)) + return unless (line_end = versions_data.index("\n", line_start)) + split_last_word(versions_data, line_start, line_end) + end + + def gem_parser + @gem_parser ||= GemParser.new + end + + def versions_data + @versions_data ||= begin + data = @compact_index.versions + strip_header!(data) if data + data.freeze + end + end + + def rindex_of_gem(name) + if (pos = versions_data.rindex("\n#{name} ")) + pos + 1 + elsif versions_data.start_with?("#{name} ") + 0 + end + end + + # This is similar to `string.split(" ").last` but it avoids allocating extra objects. + def split_last_word(string, line_start, line_end) + return unless line_start < line_end && line_start >= 0 + word_start = string.rindex(" ", line_end).to_i + 1 + return if word_start < line_start + string[word_start, line_end - word_start] + end + + def lines(string) + return [] if string.nil? || string.empty? + strip_header!(string) + string.split("\n") + end + + def strip_header!(string) + header_end = string.index("---\n") + string.slice!(0, header_end + 4) if header_end + end + end + end +end diff --git a/lib/bundler/compact_index_client/updater.rb b/lib/bundler/compact_index_client/updater.rb index 36f6b81db8..88c7146900 100644 --- a/lib/bundler/compact_index_client/updater.rb +++ b/lib/bundler/compact_index_client/updater.rb @@ -28,7 +28,6 @@ module Bundler CacheFile.copy(local_path) do |file| etag = etag_path.read.tap(&:chomp!) if etag_path.file? - etag ||= generate_etag(etag_path, file) # Remove this after 2.5.0 has been out for a while. # Subtract a byte to ensure the range won't be empty. # Avoids 416 (Range Not Satisfiable) responses. @@ -67,16 +66,6 @@ module Bundler etag_path.read.tap(&:chomp!) if etag_path.file? end - # When first releasing this opaque etag feature, we want to generate the old MD5 etag - # based on the content of the file. After that it will always use the saved opaque etag. - # This transparently saves existing users with good caches from updating a bunch of files. - # Remove this behavior after 2.5.0 has been out for a while. - def generate_etag(etag_path, file) - etag = file.md5.hexdigest - CacheFile.write(etag_path, etag) - etag - end - def etag_from_response(response) return unless response["ETag"] etag = response["ETag"].delete_prefix("W/") diff --git a/lib/bundler/constants.rb b/lib/bundler/constants.rb index de9698b577..9564771e78 100644 --- a/lib/bundler/constants.rb +++ b/lib/bundler/constants.rb @@ -1,7 +1,14 @@ # frozen_string_literal: true +require "rbconfig" + module Bundler WINDOWS = RbConfig::CONFIG["host_os"] =~ /(msdos|mswin|djgpp|mingw)/ + deprecate_constant :WINDOWS + FREEBSD = RbConfig::CONFIG["host_os"].to_s.include?("bsd") - NULL = File::NULL + deprecate_constant :FREEBSD + + NULL = File::NULL + deprecate_constant :NULL end diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb index 22070b6b17..6cf1f9a255 100644 --- a/lib/bundler/definition.rb +++ b/lib/bundler/definition.rb @@ -69,7 +69,6 @@ module Bundler @sources = sources @unlock = unlock @optional_groups = optional_groups - @remote = false @prefer_local = false @specs = nil @ruby_version = ruby_version @@ -164,37 +163,24 @@ module Bundler end def resolve_only_locally! - @remote = false sources.local_only! resolve end def resolve_with_cache! + sources.local! sources.cached! resolve end def resolve_remotely! - @remote = true + sources.cached! sources.remote! resolve end - def resolution_mode=(options) - if options["local"] - @remote = false - else - @remote = true - @prefer_local = options["prefer-local"] - end - end - - def setup_sources_for_resolve - if @remote == false - sources.cached! - else - sources.remote! - end + def prefer_local! + @prefer_local = true end # For given dependency list returns a SpecSet with Gemspec of all the required @@ -310,7 +296,12 @@ module Bundler end end else - Bundler.ui.debug "Found changes from the lockfile, re-resolving dependencies because #{change_reason}" + if lockfile_exists? + Bundler.ui.debug "Found changes from the lockfile, re-resolving dependencies because #{change_reason}" + else + Bundler.ui.debug "Resolving dependencies because there's no lockfile" + end + start_resolution end end @@ -483,6 +474,8 @@ module Bundler private :sources def nothing_changed? + return false unless lockfile_exists? + !@source_changes && !@dependency_changes && !@new_platform && @@ -587,7 +580,7 @@ module Bundler if missing_specs.any? missing_specs.each do |s| locked_gem = @locked_specs[s.name].last - next if locked_gem.nil? || locked_gem.version != s.version || !@remote + next if locked_gem.nil? || locked_gem.version != s.version || sources.local_mode? raise GemNotFound, "Your bundle is locked to #{locked_gem} from #{locked_gem.source}, but that version can " \ "no longer be found in that source. That means the author of #{locked_gem} has removed it. " \ "You'll need to update your bundle to a version other than #{locked_gem} that hasn't been " \ @@ -606,7 +599,7 @@ module Bundler break if incomplete_specs.empty? Bundler.ui.debug("The lockfile does not have all gems needed for the current platform though, Bundler will still re-resolve dependencies") - setup_sources_for_resolve + sources.remote! resolution_packages.delete(incomplete_specs) @resolve = start_resolution specs = resolve.materialize(dependencies) @@ -967,7 +960,7 @@ module Bundler else { default: Source::RubygemsAggregate.new(sources, source_map) }.merge(source_map.direct_requirements) end - source_requirements.merge!(source_map.locked_requirements) unless @remote + source_requirements.merge!(source_map.locked_requirements) if nothing_changed? metadata_dependencies.each do |dep| source_requirements[dep.name] = sources.metadata_source end @@ -1038,8 +1031,6 @@ module Bundler def dup_for_full_unlock unlocked_definition = self.class.new(@lockfile, @dependencies, @sources, true, @ruby_version, @optional_groups, @gemfiles) - unlocked_definition.resolution_mode = { "local" => !@remote } - unlocked_definition.setup_sources_for_resolve unlocked_definition.gem_version_promoter.tap do |gvp| gvp.level = gem_version_promoter.level gvp.strict = gem_version_promoter.strict diff --git a/lib/bundler/errors.rb b/lib/bundler/errors.rb index b6a11cc721..c29b1bfed8 100644 --- a/lib/bundler/errors.rb +++ b/lib/bundler/errors.rb @@ -230,4 +230,18 @@ module Bundler status_code(38) end + + class CorruptBundlerInstallError < BundlerError + def initialize(loaded_spec) + @loaded_spec = loaded_spec + end + + def message + "The running version of Bundler (#{Bundler::VERSION}) does not match the version of the specification installed for it (#{@loaded_spec.version}). " \ + "This can be caused by reinstalling Ruby without removing previous installation, leaving around an upgraded default version of Bundler. " \ + "Reinstalling Ruby from scratch should fix the problem." + end + + status_code(39) + end end diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb index db914839b1..6e5656d26a 100644 --- a/lib/bundler/fetcher/compact_index.rb +++ b/lib/bundler/fetcher/compact_index.rb @@ -4,8 +4,6 @@ require_relative "base" require_relative "../worker" module Bundler - autoload :CompactIndexClient, File.expand_path("../compact_index_client", __dir__) - class Fetcher class CompactIndex < Base def self.compact_index_request(method_name) @@ -36,15 +34,8 @@ module Bundler until remaining_gems.empty? log_specs { "Looking up gems #{remaining_gems.inspect}" } - - deps = begin - parallel_compact_index_client.dependencies(remaining_gems) - rescue TooManyRequestsError - @bundle_worker&.stop - @bundle_worker = nil # reset it. Not sure if necessary - serial_compact_index_client.dependencies(remaining_gems) - end - next_gems = deps.flat_map {|d| d[3].flat_map(&:first) }.uniq + deps = fetch_gem_infos(remaining_gems).flatten(1) + next_gems = deps.flat_map {|d| d[CompactIndexClient::INFO_DEPS].flat_map(&:first) }.uniq deps.each {|dep| gem_info << dep } complete_gems.concat(deps.map(&:first)).uniq! remaining_gems = next_gems - complete_gems @@ -61,7 +52,7 @@ module Bundler return nil end # Read info file checksums out of /versions, so we can know if gems are up to date - compact_index_client.update_and_parse_checksums! + compact_index_client.available? rescue CompactIndexClient::Updater::MismatchedChecksumError => e Bundler.ui.debug(e.message) nil @@ -81,20 +72,20 @@ module Bundler end end - def parallel_compact_index_client - compact_index_client.execution_mode = lambda do |inputs, &blk| - func = lambda {|object, _index| blk.call(object) } - worker = bundle_worker(func) - inputs.each {|input| worker.enq(input) } - inputs.map { worker.deq } - end - - compact_index_client + def fetch_gem_infos(names) + in_parallel(names) {|name| compact_index_client.info(name) } + rescue TooManyRequestsError # rubygems.org is rate limiting us, slow down. + @bundle_worker&.stop + @bundle_worker = nil # reset it. Not sure if necessary + compact_index_client.reset! + names.map {|name| compact_index_client.info(name) } end - def serial_compact_index_client - compact_index_client.sequential_execution_mode! - compact_index_client + def in_parallel(inputs, &blk) + func = lambda {|object, _index| blk.call(object) } + worker = bundle_worker(func) + inputs.each {|input| worker.enq(input) } + inputs.map { worker.deq } end def bundle_worker(func = nil) diff --git a/lib/bundler/gem_helper.rb b/lib/bundler/gem_helper.rb index d535d54f9b..5ce0ef6280 100644 --- a/lib/bundler/gem_helper.rb +++ b/lib/bundler/gem_helper.rb @@ -47,7 +47,7 @@ module Bundler built_gem_path = build_gem end - desc "Generate SHA512 checksum if #{name}-#{version}.gem into the checksums directory." + desc "Generate SHA512 checksum of #{name}-#{version}.gem into the checksums directory." task "build:checksum" => "build" do build_checksum(built_gem_path) end diff --git a/lib/bundler/installer.rb b/lib/bundler/installer.rb index 72e5602cc3..256f0be348 100644 --- a/lib/bundler/installer.rb +++ b/lib/bundler/installer.rb @@ -235,15 +235,15 @@ module Bundler # returns whether or not a re-resolve was needed def resolve_if_needed(options) - @definition.resolution_mode = options - - if !@definition.unlocking? && !options["force"] && !Bundler.settings[:inline] && Bundler.default_lockfile.file? - return false if @definition.nothing_changed? && !@definition.missing_specs? + @definition.prefer_local! if options["prefer-local"] + + if options["local"] || (@definition.no_resolve_needed? && !@definition.missing_specs?) + @definition.resolve_with_cache! + false + else + @definition.resolve_remotely! + true end - - @definition.setup_sources_for_resolve - - true end def lock diff --git a/lib/bundler/installer/gem_installer.rb b/lib/bundler/installer/gem_installer.rb index d3bbcc90f5..a7770eb7e0 100644 --- a/lib/bundler/installer/gem_installer.rb +++ b/lib/bundler/installer/gem_installer.rb @@ -54,7 +54,6 @@ module Bundler spec.source.install( spec, force: force, - ensure_builtin_gems_cached: standalone, build_args: Array(spec_settings), previous_spec: previous_spec, ) diff --git a/lib/bundler/man/bundle-add.1 b/lib/bundler/man/bundle-add.1 index 0b09480f88..56a3b6f85c 100644 --- a/lib/bundler/man/bundle-add.1 +++ b/lib/bundler/man/bundle-add.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-ADD" "1" "April 2024" "" +.TH "BUNDLE\-ADD" "1" "May 2024" "" .SH "NAME" \fBbundle\-add\fR \- Add gem to the Gemfile and run bundle install .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-binstubs.1 b/lib/bundler/man/bundle-binstubs.1 index fdb86bfd29..4ec301951f 100644 --- a/lib/bundler/man/bundle-binstubs.1 +++ b/lib/bundler/man/bundle-binstubs.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-BINSTUBS" "1" "April 2024" "" +.TH "BUNDLE\-BINSTUBS" "1" "May 2024" "" .SH "NAME" \fBbundle\-binstubs\fR \- Install the binstubs of the listed gems .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-cache.1 b/lib/bundler/man/bundle-cache.1 index c5899ace1a..e2da1269e6 100644 --- a/lib/bundler/man/bundle-cache.1 +++ b/lib/bundler/man/bundle-cache.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CACHE" "1" "April 2024" "" +.TH "BUNDLE\-CACHE" "1" "May 2024" "" .SH "NAME" \fBbundle\-cache\fR \- Package your needed \fB\.gem\fR files into your application .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-check.1 b/lib/bundler/man/bundle-check.1 index e1a971edae..dee1af1326 100644 --- a/lib/bundler/man/bundle-check.1 +++ b/lib/bundler/man/bundle-check.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CHECK" "1" "April 2024" "" +.TH "BUNDLE\-CHECK" "1" "May 2024" "" .SH "NAME" \fBbundle\-check\fR \- Verifies if dependencies are satisfied by installed gems .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-clean.1 b/lib/bundler/man/bundle-clean.1 index ca2fb65f59..7c7f9b5c77 100644 --- a/lib/bundler/man/bundle-clean.1 +++ b/lib/bundler/man/bundle-clean.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CLEAN" "1" "April 2024" "" +.TH "BUNDLE\-CLEAN" "1" "May 2024" "" .SH "NAME" \fBbundle\-clean\fR \- Cleans up unused gems in your bundler directory .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-config.1 b/lib/bundler/man/bundle-config.1 index a78c6b0a18..2de52ee375 100644 --- a/lib/bundler/man/bundle-config.1 +++ b/lib/bundler/man/bundle-config.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CONFIG" "1" "April 2024" "" +.TH "BUNDLE\-CONFIG" "1" "May 2024" "" .SH "NAME" \fBbundle\-config\fR \- Set bundler configuration options .SH "SYNOPSIS" @@ -307,7 +307,7 @@ Any \fB\.\fR characters in a host name are mapped to a double underscore (\fB__\ .P This means that if you have a gem server named \fBmy\.gem\-host\.com\fR, you'll need to use the \fBBUNDLE_MY__GEM___HOST__COM\fR variable to configure credentials for it through ENV\. .SH "CONFIGURE BUNDLER DIRECTORIES" -Bundler's home, config, cache and plugin directories are able to be configured through environment variables\. The default location for Bundler's home directory is \fB~/\.bundle\fR, which all directories inherit from by default\. The following outlines the available environment variables and their default values +Bundler's home, cache and plugin directories and config file can be configured through environment variables\. The default location for Bundler's home directory is \fB~/\.bundle\fR, which all directories inherit from by default\. The following outlines the available environment variables and their default values .IP "" 4 .nf BUNDLE_USER_HOME : $HOME/\.bundle diff --git a/lib/bundler/man/bundle-config.1.ronn b/lib/bundler/man/bundle-config.1.ronn index 7e5f458fb2..1a0ec2a5dc 100644 --- a/lib/bundler/man/bundle-config.1.ronn +++ b/lib/bundler/man/bundle-config.1.ronn @@ -397,7 +397,7 @@ through ENV. ## CONFIGURE BUNDLER DIRECTORIES -Bundler's home, config, cache and plugin directories are able to be configured +Bundler's home, cache and plugin directories and config file can be configured through environment variables. The default location for Bundler's home directory is `~/.bundle`, which all directories inherit from by default. The following outlines the available environment variables and their default values diff --git a/lib/bundler/man/bundle-console.1 b/lib/bundler/man/bundle-console.1 index 86be5e4185..dca18ec43d 100644 --- a/lib/bundler/man/bundle-console.1 +++ b/lib/bundler/man/bundle-console.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-CONSOLE" "1" "April 2024" "" +.TH "BUNDLE\-CONSOLE" "1" "May 2024" "" .SH "NAME" \fBbundle\-console\fR \- Deprecated way to open an IRB session with the bundle pre\-loaded .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-doctor.1 b/lib/bundler/man/bundle-doctor.1 index 94cbff4cbd..6489cc07f7 100644 --- a/lib/bundler/man/bundle-doctor.1 +++ b/lib/bundler/man/bundle-doctor.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-DOCTOR" "1" "April 2024" "" +.TH "BUNDLE\-DOCTOR" "1" "May 2024" "" .SH "NAME" \fBbundle\-doctor\fR \- Checks the bundle for common problems .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-exec.1 b/lib/bundler/man/bundle-exec.1 index 7aa49e0096..1548d29670 100644 --- a/lib/bundler/man/bundle-exec.1 +++ b/lib/bundler/man/bundle-exec.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-EXEC" "1" "April 2024" "" +.TH "BUNDLE\-EXEC" "1" "May 2024" "" .SH "NAME" \fBbundle\-exec\fR \- Execute a command in the context of the bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-gem.1 b/lib/bundler/man/bundle-gem.1 index 89eb9f1980..5df7b0ef2f 100644 --- a/lib/bundler/man/bundle-gem.1 +++ b/lib/bundler/man/bundle-gem.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-GEM" "1" "April 2024" "" +.TH "BUNDLE\-GEM" "1" "May 2024" "" .SH "NAME" \fBbundle\-gem\fR \- Generate a project skeleton for creating a rubygem .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-help.1 b/lib/bundler/man/bundle-help.1 index 441ec12735..a3e7c7770d 100644 --- a/lib/bundler/man/bundle-help.1 +++ b/lib/bundler/man/bundle-help.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-HELP" "1" "April 2024" "" +.TH "BUNDLE\-HELP" "1" "May 2024" "" .SH "NAME" \fBbundle\-help\fR \- Displays detailed help for each subcommand .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-info.1 b/lib/bundler/man/bundle-info.1 index 287965c06a..a3d7ff0988 100644 --- a/lib/bundler/man/bundle-info.1 +++ b/lib/bundler/man/bundle-info.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INFO" "1" "April 2024" "" +.TH "BUNDLE\-INFO" "1" "May 2024" "" .SH "NAME" \fBbundle\-info\fR \- Show information for the given gem in your bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-init.1 b/lib/bundler/man/bundle-init.1 index d13f8ddc3b..a0edaaa18f 100644 --- a/lib/bundler/man/bundle-init.1 +++ b/lib/bundler/man/bundle-init.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INIT" "1" "April 2024" "" +.TH "BUNDLE\-INIT" "1" "May 2024" "" .SH "NAME" \fBbundle\-init\fR \- Generates a Gemfile into the current working directory .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-inject.1 b/lib/bundler/man/bundle-inject.1 index 086fc88156..7a1038206e 100644 --- a/lib/bundler/man/bundle-inject.1 +++ b/lib/bundler/man/bundle-inject.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INJECT" "1" "April 2024" "" +.TH "BUNDLE\-INJECT" "1" "May 2024" "" .SH "NAME" \fBbundle\-inject\fR \- Add named gem(s) with version requirements to Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-install.1 b/lib/bundler/man/bundle-install.1 index 762c99e7c0..cc46a03b7f 100644 --- a/lib/bundler/man/bundle-install.1 +++ b/lib/bundler/man/bundle-install.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-INSTALL" "1" "April 2024" "" +.TH "BUNDLE\-INSTALL" "1" "May 2024" "" .SH "NAME" \fBbundle\-install\fR \- Install the dependencies specified in your Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-list.1 b/lib/bundler/man/bundle-list.1 index 93db700091..21723608cc 100644 --- a/lib/bundler/man/bundle-list.1 +++ b/lib/bundler/man/bundle-list.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-LIST" "1" "April 2024" "" +.TH "BUNDLE\-LIST" "1" "May 2024" "" .SH "NAME" \fBbundle\-list\fR \- List all the gems in the bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-lock.1 b/lib/bundler/man/bundle-lock.1 index c0190f91de..8b81b7732f 100644 --- a/lib/bundler/man/bundle-lock.1 +++ b/lib/bundler/man/bundle-lock.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-LOCK" "1" "April 2024" "" +.TH "BUNDLE\-LOCK" "1" "May 2024" "" .SH "NAME" \fBbundle\-lock\fR \- Creates / Updates a lockfile without installing .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-open.1 b/lib/bundler/man/bundle-open.1 index 38bacb67dc..41a8804a09 100644 --- a/lib/bundler/man/bundle-open.1 +++ b/lib/bundler/man/bundle-open.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-OPEN" "1" "April 2024" "" +.TH "BUNDLE\-OPEN" "1" "May 2024" "" .SH "NAME" \fBbundle\-open\fR \- Opens the source directory for a gem in your bundle .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-outdated.1 b/lib/bundler/man/bundle-outdated.1 index cee9d63155..838fce45cd 100644 --- a/lib/bundler/man/bundle-outdated.1 +++ b/lib/bundler/man/bundle-outdated.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-OUTDATED" "1" "April 2024" "" +.TH "BUNDLE\-OUTDATED" "1" "May 2024" "" .SH "NAME" \fBbundle\-outdated\fR \- List installed gems with newer versions available .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-platform.1 b/lib/bundler/man/bundle-platform.1 index 069966f1b2..0dbce7a7a4 100644 --- a/lib/bundler/man/bundle-platform.1 +++ b/lib/bundler/man/bundle-platform.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PLATFORM" "1" "April 2024" "" +.TH "BUNDLE\-PLATFORM" "1" "May 2024" "" .SH "NAME" \fBbundle\-platform\fR \- Displays platform compatibility information .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-plugin.1 b/lib/bundler/man/bundle-plugin.1 index f4e043c363..1290abb3ba 100644 --- a/lib/bundler/man/bundle-plugin.1 +++ b/lib/bundler/man/bundle-plugin.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PLUGIN" "1" "April 2024" "" +.TH "BUNDLE\-PLUGIN" "1" "May 2024" "" .SH "NAME" \fBbundle\-plugin\fR \- Manage Bundler plugins .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-pristine.1 b/lib/bundler/man/bundle-pristine.1 index 76a0479bc6..bf4a7cd323 100644 --- a/lib/bundler/man/bundle-pristine.1 +++ b/lib/bundler/man/bundle-pristine.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-PRISTINE" "1" "April 2024" "" +.TH "BUNDLE\-PRISTINE" "1" "May 2024" "" .SH "NAME" \fBbundle\-pristine\fR \- Restores installed gems to their pristine condition .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-remove.1 b/lib/bundler/man/bundle-remove.1 index 90a664e331..c3c96b416d 100644 --- a/lib/bundler/man/bundle-remove.1 +++ b/lib/bundler/man/bundle-remove.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-REMOVE" "1" "April 2024" "" +.TH "BUNDLE\-REMOVE" "1" "May 2024" "" .SH "NAME" \fBbundle\-remove\fR \- Removes gems from the Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-show.1 b/lib/bundler/man/bundle-show.1 index f9f1fd1fa3..c054875efd 100644 --- a/lib/bundler/man/bundle-show.1 +++ b/lib/bundler/man/bundle-show.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-SHOW" "1" "April 2024" "" +.TH "BUNDLE\-SHOW" "1" "May 2024" "" .SH "NAME" \fBbundle\-show\fR \- Shows all the gems in your bundle, or the path to a gem .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-update.1 b/lib/bundler/man/bundle-update.1 index 340ebac687..acd80ec75c 100644 --- a/lib/bundler/man/bundle-update.1 +++ b/lib/bundler/man/bundle-update.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-UPDATE" "1" "April 2024" "" +.TH "BUNDLE\-UPDATE" "1" "May 2024" "" .SH "NAME" \fBbundle\-update\fR \- Update your gems to the latest available versions .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-version.1 b/lib/bundler/man/bundle-version.1 index 00ff0153cb..44eaf92224 100644 --- a/lib/bundler/man/bundle-version.1 +++ b/lib/bundler/man/bundle-version.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-VERSION" "1" "April 2024" "" +.TH "BUNDLE\-VERSION" "1" "May 2024" "" .SH "NAME" \fBbundle\-version\fR \- Prints Bundler version information .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle-viz.1 b/lib/bundler/man/bundle-viz.1 index eba3b7df25..77b902214c 100644 --- a/lib/bundler/man/bundle-viz.1 +++ b/lib/bundler/man/bundle-viz.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE\-VIZ" "1" "April 2024" "" +.TH "BUNDLE\-VIZ" "1" "May 2024" "" .SH "NAME" \fBbundle\-viz\fR \- Generates a visual dependency graph for your Gemfile .SH "SYNOPSIS" diff --git a/lib/bundler/man/bundle.1 b/lib/bundler/man/bundle.1 index 4bd2bcaf67..199d1ce9fd 100644 --- a/lib/bundler/man/bundle.1 +++ b/lib/bundler/man/bundle.1 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "BUNDLE" "1" "April 2024" "" +.TH "BUNDLE" "1" "May 2024" "" .SH "NAME" \fBbundle\fR \- Ruby Dependency Management .SH "SYNOPSIS" diff --git a/lib/bundler/man/gemfile.5 b/lib/bundler/man/gemfile.5 index c9478a953e..fa9e31f615 100644 --- a/lib/bundler/man/gemfile.5 +++ b/lib/bundler/man/gemfile.5 @@ -1,6 +1,6 @@ .\" generated with nRonn/v0.11.1 .\" https://github.com/n-ronn/nronn/tree/0.11.1 -.TH "GEMFILE" "5" "April 2024" "" +.TH "GEMFILE" "5" "May 2024" "" .SH "NAME" \fBGemfile\fR \- A format for describing gem dependencies for Ruby programs .SH "SYNOPSIS" diff --git a/lib/bundler/rubygems_ext.rb b/lib/bundler/rubygems_ext.rb index a7539f4adb..18180a81a1 100644 --- a/lib/bundler/rubygems_ext.rb +++ b/lib/bundler/rubygems_ext.rb @@ -1,11 +1,7 @@ # frozen_string_literal: true -require "pathname" - require "rubygems" unless defined?(Gem) -require "rubygems/specification" - # We can't let `Gem::Source` be autoloaded in the `Gem::Specification#source` # redefinition below, so we need to load it upfront. The reason is that if # Bundler monkeypatches are loaded before RubyGems activates an executable (for @@ -17,10 +13,6 @@ require "rubygems/specification" # `Gem::Source` from the redefined `Gem::Specification#source`. require "rubygems/source" -require_relative "match_metadata" -require_relative "force_platform" -require_relative "match_platform" - # Cherry-pick fixes to `Gem.ruby_version` to be useful for modern Bundler # versions and ignore patchlevels # (https://github.com/rubygems/rubygems/pull/5472, @@ -31,7 +23,19 @@ unless Gem.ruby_version.to_s == RUBY_VERSION || RUBY_PATCHLEVEL == -1 end module Gem + # Can be removed once RubyGems 3.5.11 support is dropped + unless Gem.respond_to?(:freebsd_platform?) + def self.freebsd_platform? + RbConfig::CONFIG["host_os"].to_s.include?("bsd") + end + end + + require "rubygems/specification" + class Specification + require_relative "match_metadata" + require_relative "match_platform" + include ::Bundler::MatchMetadata include ::Bundler::MatchPlatform @@ -48,7 +52,7 @@ module Gem def full_gem_path if source.respond_to?(:root) - Pathname.new(loaded_from).dirname.expand_path(source.root).to_s + File.expand_path(File.dirname(loaded_from), source.root) else rg_full_gem_path end @@ -148,17 +152,21 @@ module Gem module BetterPermissionError def data - Bundler::SharedHelpers.filesystem_access(loaded_from, :read) do - super - end + super + rescue Errno::EACCES + raise Bundler::PermissionError.new(loaded_from, :read) end end + require "rubygems/stub_specification" + class StubSpecification prepend BetterPermissionError end class Dependency + require_relative "force_platform" + include ::Bundler::ForcePlatform attr_accessor :source, :groups diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb index 494030eab2..b841462263 100644 --- a/lib/bundler/rubygems_integration.rb +++ b/lib/bundler/rubygems_integration.rb @@ -481,11 +481,25 @@ module Bundler end def all_specs + SharedHelpers.major_deprecation 2, "Bundler.rubygems.all_specs has been removed in favor of Bundler.rubygems.installed_specs" + Gem::Specification.stubs.map do |stub| StubSpecification.from_stub(stub) end end + def installed_specs + Gem::Specification.stubs.reject(&:default_gem?).map do |stub| + StubSpecification.from_stub(stub) + end + end + + def default_specs + Gem::Specification.default_stubs.map do |stub| + StubSpecification.from_stub(stub) + end + end + def find_bundler(version) find_name("bundler").find {|s| s.version.to_s == version } end diff --git a/lib/bundler/self_manager.rb b/lib/bundler/self_manager.rb index bfd000b1a0..5accda4bcb 100644 --- a/lib/bundler/self_manager.rb +++ b/lib/bundler/self_manager.rb @@ -113,7 +113,7 @@ module Bundler end def local_specs - @local_specs ||= Bundler::Source::Rubygems.new("allow_local" => true, "allow_cached" => true).specs.select {|spec| spec.name == "bundler" } + @local_specs ||= Bundler::Source::Rubygems.new("allow_local" => true).specs.select {|spec| spec.name == "bundler" } end def remote_specs diff --git a/lib/bundler/settings.rb b/lib/bundler/settings.rb index abbaec9783..878747a53b 100644 --- a/lib/bundler/settings.rb +++ b/lib/bundler/settings.rb @@ -103,6 +103,7 @@ module Bundler def initialize(root = nil) @root = root @local_config = load_config(local_config_file) + @local_root = root || Pathname.new(".bundle").expand_path @env_config = ENV.to_h @env_config.select! {|key, _value| key.start_with?("BUNDLE_") } @@ -142,7 +143,7 @@ module Bundler end def set_local(key, value) - local_config_file || raise(GemfileNotFound, "Could not locate Gemfile") + local_config_file = @local_root.join("config") set_key(key, value, @local_config, local_config_file) end @@ -491,6 +492,10 @@ module Bundler valid_file = file.exist? && !file.size.zero? return {} unless valid_file serializer_class.load(file.read).inject({}) do |config, (k, v)| + k = k.dup + k << "/" if /https?:/i.match?(k) && !k.end_with?("/", "__#{FALLBACK_TIMEOUT_URI_OPTION.upcase}") + k.gsub!(".", "__") + unless k.start_with?("#") if k.include?("-") Bundler.ui.warn "Your #{file} config includes `#{k}`, which contains the dash character (`-`).\n" \ @@ -518,26 +523,25 @@ module Bundler YAMLSerializer end - PER_URI_OPTIONS = %w[ - fallback_timeout - ].freeze + FALLBACK_TIMEOUT_URI_OPTION = "fallback_timeout" NORMALIZE_URI_OPTIONS_PATTERN = / \A (\w+\.)? # optional prefix key (https?.*?) # URI - (\.#{Regexp.union(PER_URI_OPTIONS)})? # optional suffix key + (\.#{FALLBACK_TIMEOUT_URI_OPTION})? # optional suffix key \z /ix def self.key_for(key) - key = normalize_uri(key).to_s if key.is_a?(String) && key.start_with?("http", "mirror.http") - key = key_to_s(key).gsub(".", "__") + key = key_to_s(key) + key = normalize_uri(key) if key.start_with?("http", "mirror.http") + key = key.gsub(".", "__") key.gsub!("-", "___") key.upcase! - key.prepend("BUNDLE_") + key.gsub(/\A([ #]*)/, '\1BUNDLE_') end # TODO: duplicates Rubygems#normalize_uri diff --git a/lib/bundler/shared_helpers.rb b/lib/bundler/shared_helpers.rb index 78760e6fa4..e55632b89f 100644 --- a/lib/bundler/shared_helpers.rb +++ b/lib/bundler/shared_helpers.rb @@ -1,14 +1,16 @@ # frozen_string_literal: true -require "pathname" -require "rbconfig" - require_relative "version" -require_relative "constants" require_relative "rubygems_integration" require_relative "current_ruby" +autoload :Pathname, "pathname" + module Bundler + autoload :WINDOWS, File.expand_path("constants", __dir__) + autoload :FREEBSD, File.expand_path("constants", __dir__) + autoload :NULL, File.expand_path("constants", __dir__) + module SharedHelpers def root gemfile = find_gemfile diff --git a/lib/bundler/source/git/git_proxy.rb b/lib/bundler/source/git/git_proxy.rb index 645851286c..2fc9c6535f 100644 --- a/lib/bundler/source/git/git_proxy.rb +++ b/lib/bundler/source/git/git_proxy.rb @@ -182,6 +182,14 @@ module Bundler if err.include?("Could not find remote branch") raise MissingGitRevisionError.new(command_with_no_credentials, nil, explicit_ref, credential_filtered_uri) else + idx = command.index("--depth") + if idx + command.delete_at(idx) + command.delete_at(idx) + command_with_no_credentials = check_allowed(command) + + err += "Retrying without --depth argument." + end raise GitCommandError.new(command_with_no_credentials, path, err) end end diff --git a/lib/bundler/source/metadata.rb b/lib/bundler/source/metadata.rb index 4d27761365..6b05e17727 100644 --- a/lib/bundler/source/metadata.rb +++ b/lib/bundler/source/metadata.rb @@ -11,6 +11,8 @@ module Bundler end if local_spec = Gem.loaded_specs["bundler"] + raise CorruptBundlerInstallError.new(local_spec) if local_spec.version.to_s != Bundler::VERSION + idx << local_spec else idx << Gem::Specification.new do |s| diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 2e76becb84..dafc674f9d 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -17,7 +17,7 @@ module Bundler @remotes = [] @dependency_names = [] @allow_remote = false - @allow_cached = options["allow_cached"] || false + @allow_cached = false @allow_local = options["allow_local"] || false @checksum_store = Checksum::Store.new @@ -50,10 +50,11 @@ module Bundler end def cached! + return unless File.exist?(cache_path) + return if @allow_cached @specs = nil - @allow_local = true @allow_cached = true end @@ -135,20 +136,17 @@ module Bundler index = @allow_remote ? remote_specs.dup : Index.new index.merge!(cached_specs) if @allow_cached index.merge!(installed_specs) if @allow_local + + # complete with default specs, only if not already available in the + # index through remote, cached, or installed specs + index.use(default_specs) if @allow_local + index end end def install(spec, options = {}) - force = options[:force] - ensure_builtin_gems_cached = options[:ensure_builtin_gems_cached] - - if ensure_builtin_gems_cached && spec.default_gem? && !cached_path(spec) - cached_built_in_gem(spec) unless spec.remote - force = true - end - - if installed?(spec) && !force + if (spec.default_gem? && !cached_built_in_gem(spec)) || (installed?(spec) && !options[:force]) print_using_message "Using #{version_message(spec, options[:previous_spec])}" return nil # no post-install message end @@ -361,7 +359,7 @@ module Bundler def installed_specs @installed_specs ||= Index.build do |idx| - Bundler.rubygems.all_specs.reverse_each do |spec| + Bundler.rubygems.installed_specs.reverse_each do |spec| spec.source = self if Bundler.rubygems.spec_missing_extensions?(spec, false) Bundler.ui.debug "Source #{self} is ignoring #{spec} because it is missing extensions" @@ -372,6 +370,15 @@ module Bundler end end + def default_specs + @default_specs ||= Index.build do |idx| + Bundler.rubygems.default_specs.each do |spec| + spec.source = self + idx << spec + end + end + end + def cached_specs @cached_specs ||= begin idx = Index.new diff --git a/lib/bundler/source_list.rb b/lib/bundler/source_list.rb index bbaac33a95..5f9dd68f17 100644 --- a/lib/bundler/source_list.rb +++ b/lib/bundler/source_list.rb @@ -9,7 +9,7 @@ module Bundler :metadata_source def global_rubygems_source - @global_rubygems_source ||= rubygems_aggregate_class.new("allow_local" => true, "allow_cached" => true) + @global_rubygems_source ||= rubygems_aggregate_class.new("allow_local" => true) end def initialize @@ -22,6 +22,7 @@ module Bundler @metadata_source = Source::Metadata.new @merged_gem_lockfile_sections = false + @local_mode = true end def merged_gem_lockfile_sections? @@ -73,6 +74,10 @@ module Bundler global_rubygems_source end + def local_mode? + @local_mode + end + def default_source global_path_source || global_rubygems_source end @@ -140,11 +145,17 @@ module Bundler all_sources.each(&:local_only!) end + def local! + all_sources.each(&:local!) + end + def cached! all_sources.each(&:cached!) end def remote! + @local_mode = false + all_sources.each(&:remote!) end @@ -178,7 +189,7 @@ module Bundler replacement_source = replace_rubygems_source(replacement_sources, global_rubygems_source) return global_rubygems_source unless replacement_source - replacement_source.cached! + replacement_source.local! replacement_source end diff --git a/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt b/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt index 175b821a62..67fe8cee79 100644 --- a/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt +++ b/lib/bundler/templates/newgem/CODE_OF_CONDUCT.md.tt @@ -2,83 +2,131 @@ ## Our Pledge -We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, religion, or sexual identity and orientation. +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. -We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. ## Our Standards -Examples of behavior that contributes to a positive environment for our community include: +Examples of behavior that contributes to a positive environment for our +community include: * Demonstrating empathy and kindness toward other people * Being respectful of differing opinions, viewpoints, and experiences * Giving and gracefully accepting constructive feedback -* Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience -* Focusing on what is best not just for us as individuals, but for the overall community +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community Examples of unacceptable behavior include: -* The use of sexualized language or imagery, and sexual attention or - advances of any kind +* The use of sexualized language or imagery, and sexual attention or advances of + any kind * Trolling, insulting or derogatory comments, and personal or political attacks * Public or private harassment -* Publishing others' private information, such as a physical or email - address, without their explicit permission +* Publishing others' private information, such as a physical or email address, + without their explicit permission * Other conduct which could reasonably be considered inappropriate in a professional setting ## Enforcement Responsibilities -Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful. +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. -Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, and will communicate reasons for moderation decisions when appropriate. +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. ## Scope -This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces. Examples of representing our community include using an official e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official email address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. ## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at <%= config[:email] %>. All complaints will be reviewed and investigated promptly and fairly. +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +[INSERT CONTACT METHOD]. +All complaints will be reviewed and investigated promptly and fairly. -All community leaders are obligated to respect the privacy and security of the reporter of any incident. +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. ## Enforcement Guidelines -Community leaders will follow these Community Impact Guidelines in determining the consequences for any action they deem in violation of this Code of Conduct: +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: ### 1. Correction -**Community Impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. -**Consequence**: A private, written warning from community leaders, providing clarity around the nature of the violation and an explanation of why the behavior was inappropriate. A public apology may be requested. +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. ### 2. Warning -**Community Impact**: A violation through a single incident or series of actions. +**Community Impact**: A violation through a single incident or series of +actions. -**Consequence**: A warning with consequences for continued behavior. No interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period of time. This includes avoiding interactions in community spaces as well as external channels like social media. Violating these terms may lead to a temporary or permanent ban. +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. ### 3. Temporary Ban -**Community Impact**: A serious violation of community standards, including sustained inappropriate behavior. +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. -**Consequence**: A temporary ban from any sort of interaction or public communication with the community for a specified period of time. No public or private interaction with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, is allowed during this period. Violating these terms may lead to a permanent ban. +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. ### 4. Permanent Ban -**Community Impact**: Demonstrating a pattern of violation of community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. -**Consequence**: A permanent ban from any sort of public interaction within the community. +**Consequence**: A permanent ban from any sort of public interaction within the +community. ## Attribution -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, -available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. -Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). - -[homepage]: https://www.contributor-covenant.org +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][Mozilla CoC]. For answers to common questions about this code of conduct, see the FAQ at -https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations. +[https://www.contributor-covenant.org/faq][FAQ]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[Mozilla CoC]: https://github.com/mozilla/diversity +[FAQ]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/lib/bundler/yaml_serializer.rb b/lib/bundler/yaml_serializer.rb index 42e6aaf89d..93d08a05aa 100644 --- a/lib/bundler/yaml_serializer.rb +++ b/lib/bundler/yaml_serializer.rb @@ -60,7 +60,6 @@ module Bundler indent, key, quote, val = match.captures val = strip_comment(val) - convert_to_backward_compatible_key!(key) depth = indent.size / 2 if quote.empty? && val.empty? new_hash = {} @@ -92,14 +91,8 @@ module Bundler end end - # for settings' keys - def convert_to_backward_compatible_key!(key) - key << "/" if /https?:/i.match?(key) && !%r{/\Z}.match?(key) - key.gsub!(".", "__") - end - class << self - private :dump_hash, :convert_to_backward_compatible_key! + private :dump_hash end end end diff --git a/lib/did_you_mean/did_you_mean.gemspec b/lib/did_you_mean/did_you_mean.gemspec index 8fe5723129..be4ac76b4b 100644 --- a/lib/did_you_mean/did_you_mean.gemspec +++ b/lib/did_you_mean/did_you_mean.gemspec @@ -22,6 +22,4 @@ Gem::Specification.new do |spec| spec.require_paths = ["lib"] spec.required_ruby_version = '>= 2.5.0' - - spec.add_development_dependency "rake" end diff --git a/lib/error_highlight/base.rb b/lib/error_highlight/base.rb index b9c68b8eb8..81b1b574de 100644 --- a/lib/error_highlight/base.rb +++ b/lib/error_highlight/base.rb @@ -60,14 +60,14 @@ module ErrorHighlight rescue RuntimeError => error # RubyVM::AbstractSyntaxTree.of raises an error with a message that # includes "prism" when the ISEQ was compiled with the prism compiler. - # In this case, we'll set the node to `nil`. In the future, we will - # reparse with the prism parser and pass the parsed node to Spotter. + # In this case, we'll try to parse again with prism instead. raise unless error.message.include?("prism") + prism_find(loc, **opts) end Spotter.new(node, **opts).spot - when RubyVM::AbstractSyntaxTree::Node + when RubyVM::AbstractSyntaxTree::Node, Prism::Node Spotter.new(obj, **opts).spot else @@ -81,6 +81,71 @@ module ErrorHighlight return nil end + # Accepts a Thread::Backtrace::Location object and returns a Prism::Node + # corresponding to the location in the source code. + def self.prism_find(loc, point_type: :name, name: nil) + require "prism" + return nil if Prism::VERSION < "0.29.0" + + path = loc.absolute_path + return unless path + + lineno = loc.lineno + column = RubyVM::AbstractSyntaxTree.node_id_for_backtrace_location(loc) + tunnel = Prism.parse_file(path).value.tunnel(lineno, column) + + # Prism provides the Prism::Node#tunnel API to find all of the nodes that + # correspond to the given line and column in the source code, with the first + # node in the list being the top-most node and the last node in the list + # being the bottom-most node. + tunnel.each_with_index.reverse_each.find do |part, index| + case part + when Prism::CallNode, Prism::CallOperatorWriteNode, Prism::IndexOperatorWriteNode, Prism::LocalVariableOperatorWriteNode + # If we find any of these nodes, we can stop searching as these are the + # nodes that triggered the exceptions. + break part + when Prism::ConstantReadNode, Prism::ConstantPathNode + if index != 0 && tunnel[index - 1].is_a?(Prism::ConstantPathOperatorWriteNode) + # If we're inside of a constant path operator write node, then this + # constant path may be highlighting a couple of different kinds of + # parts. + if part.name == name + # Explicitly turn off Foo::Bar += 1 where Foo and Bar are on + # different lines because error highlight expects this to not work. + break nil if part.delimiter_loc.end_line != part.name_loc.start_line + + # Otherwise, because we have matched the name we can return this + # part. + break part + end + + # If we haven't matched the name, it's the operator that we're looking + # for, and we can return the parent node here. + break tunnel[index - 1] + elsif part.name == name + # If we have matched the name of the constant, then we can return this + # inner node as the node that triggered the exception. + break part + else + # If we are at the beginning of the tunnel or we are at the beginning + # of a constant lookup chain, then we will return this node. + break part if index == 0 || !tunnel[index - 1].is_a?(Prism::ConstantPathNode) + end + when Prism::LocalVariableReadNode, Prism::ParenthesesNode + # If we find any of these nodes, we want to continue searching up the + # tree because these nodes cannot trigger the exceptions. + false + else + # If we find a different kind of node that we haven't already handled, + # we don't know how to handle it so we'll stop searching and assume this + # is not an exception we can decorate. + break nil + end + end + end + + private_class_method :prism_find + class Spotter class NonAscii < Exception; end private_constant :NonAscii @@ -205,6 +270,48 @@ module ErrorHighlight when :OP_CDECL spot_op_cdecl + + when :call_node + case @point_type + when :name + prism_spot_call_for_name + when :args + prism_spot_call_for_args + end + + when :local_variable_operator_write_node + case @point_type + when :name + prism_spot_local_variable_operator_write_for_name + when :args + prism_spot_local_variable_operator_write_for_args + end + + when :call_operator_write_node + case @point_type + when :name + prism_spot_call_operator_write_for_name + when :args + prism_spot_call_operator_write_for_args + end + + when :index_operator_write_node + case @point_type + when :name + prism_spot_index_operator_write_for_name + when :args + prism_spot_index_operator_write_for_args + end + + when :constant_read_node + prism_spot_constant_read + + when :constant_path_node + prism_spot_constant_path + + when :constant_path_operator_write_node + prism_spot_constant_path_operator_write + end if @snippet && @beg_column && @end_column && @beg_column < @end_column @@ -548,6 +655,200 @@ module ErrorHighlight @beg_lineno = @end_lineno = lineno @snippet = @fetch[lineno] end + + # Take a location from the prism parser and set the necessary instance + # variables. + def prism_location(location) + @beg_lineno = location.start_line + @beg_column = location.start_column + @end_lineno = location.end_line + @end_column = location.end_column + @snippet = @fetch[@beg_lineno, @end_lineno] + end + + # Example: + # x.foo + # ^^^^ + # x.foo(42) + # ^^^^ + # x&.foo + # ^^^^^ + # x[42] + # ^^^^ + # x.foo = 1 + # ^^^^^^ + # x[42] = 1 + # ^^^^^^ + # x + 1 + # ^ + # +x + # ^ + # foo(42) + # ^^^ + # foo 42 + # ^^^ + # foo + # ^^^ + def prism_spot_call_for_name + # Explicitly turn off foo.() syntax because error_highlight expects this + # to not work. + return nil if @node.name == :call && @node.message_loc.nil? + + location = @node.message_loc || @node.call_operator_loc || @node.location + location = @node.call_operator_loc.join(location) if @node.call_operator_loc&.start_line == location.start_line + + # If the method name ends with "=" but the message does not, then this is + # a method call using the "attribute assignment" syntax + # (e.g., foo.bar = 1). In this case we need to go retrieve the = sign and + # add it to the location. + if (name = @node.name).end_with?("=") && !@node.message.end_with?("=") + location = location.adjoin("=") + end + + prism_location(location) + + if !name.end_with?("=") && !name.match?(/[[:alpha:]_\[]/) + # If the method name is an operator, then error_highlight only + # highlights the first line. + fetch_line(location.start_line) + end + end + + # Example: + # x.foo(42) + # ^^ + # x[42] + # ^^ + # x.foo = 1 + # ^ + # x[42] = 1 + # ^^^^^^^ + # x[] = 1 + # ^^^^^ + # x + 1 + # ^ + # foo(42) + # ^^ + # foo 42 + # ^^ + def prism_spot_call_for_args + # Explicitly turn off foo.() syntax because error_highlight expects this + # to not work. + return nil if @node.name == :call && @node.message_loc.nil? + + if @node.name == :[]= && @node.opening == "[" && (@node.arguments&.arguments || []).length == 1 + prism_location(@node.opening_loc.copy(start_offset: @node.opening_loc.start_offset + 1).join(@node.arguments.location)) + else + prism_location(@node.arguments.location) + end + end + + # Example: + # x += 1 + # ^ + def prism_spot_local_variable_operator_write_for_name + prism_location(@node.binary_operator_loc.chop) + end + + # Example: + # x += 1 + # ^ + def prism_spot_local_variable_operator_write_for_args + prism_location(@node.value.location) + end + + # Example: + # x.foo += 42 + # ^^^ (for foo) + # x.foo += 42 + # ^ (for +) + # x.foo += 42 + # ^^^^^^^ (for foo=) + def prism_spot_call_operator_write_for_name + if !@name.start_with?(/[[:alpha:]_]/) + prism_location(@node.binary_operator_loc.chop) + else + location = @node.message_loc + if @node.call_operator_loc.start_line == location.start_line + location = @node.call_operator_loc.join(location) + end + + location = location.adjoin("=") if @name.end_with?("=") + prism_location(location) + end + end + + # Example: + # x.foo += 42 + # ^^ + def prism_spot_call_operator_write_for_args + prism_location(@node.value.location) + end + + # Example: + # x[1] += 42 + # ^^^ (for []) + # x[1] += 42 + # ^ (for +) + # x[1] += 42 + # ^^^^^^ (for []=) + def prism_spot_index_operator_write_for_name + case @name + when :[] + prism_location(@node.opening_loc.join(@node.closing_loc)) + when :[]= + prism_location(@node.opening_loc.join(@node.closing_loc).adjoin("=")) + else + # Explicitly turn off foo[] += 1 syntax when the operator is not on + # the same line because error_highlight expects this to not work. + return nil if @node.binary_operator_loc.start_line != @node.opening_loc.start_line + + prism_location(@node.binary_operator_loc.chop) + end + end + + # Example: + # x[1] += 42 + # ^^^^^^^^ + def prism_spot_index_operator_write_for_args + opening_loc = + if @node.arguments.nil? + @node.opening_loc.copy(start_offset: @node.opening_loc.start_offset + 1) + else + @node.arguments.location + end + + prism_location(opening_loc.join(@node.value.location)) + end + + # Example: + # Foo + # ^^^ + def prism_spot_constant_read + prism_location(@node.location) + end + + # Example: + # Foo::Bar + # ^^^^^ + def prism_spot_constant_path + if @node.parent && @node.parent.location.end_line == @node.location.end_line + fetch_line(@node.parent.location.end_line) + prism_location(@node.delimiter_loc.join(@node.name_loc)) + else + fetch_line(@node.location.end_line) + location = @node.name_loc + location = @node.delimiter_loc.join(location) if @node.delimiter_loc.end_line == location.start_line + prism_location(location) + end + end + + # Example: + # Foo::Bar += 1 + # ^^^^^^^^ + def prism_spot_constant_path_operator_write + prism_location(@node.binary_operator_loc.chop) + end end private_constant :Spotter diff --git a/lib/find.gemspec b/lib/find.gemspec index cb845e9409..aef24a5028 100644 --- a/lib/find.gemspec +++ b/lib/find.gemspec @@ -25,7 +25,5 @@ Gem::Specification.new do |spec| spec.files = Dir.chdir(File.expand_path('..', __FILE__)) do `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } end - spec.bindir = "exe" - spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ["lib"] end diff --git a/lib/irb.rb b/lib/irb.rb index 5a064cfced..b3435c257e 100644 --- a/lib/irb.rb +++ b/lib/irb.rb @@ -931,8 +931,11 @@ module IRB # The lexer used by this irb session attr_accessor :scanner + attr_reader :from_binding + # Creates a new irb session - def initialize(workspace = nil, input_method = nil) + def initialize(workspace = nil, input_method = nil, from_binding: false) + @from_binding = from_binding @context = Context.new(self, workspace, input_method) @context.workspace.load_helper_methods_to_main @signal_status = :IN_IRB @@ -1596,7 +1599,7 @@ class Binding else # If we're not in a debugger session, create a new IRB instance with the current # workspace - binding_irb = IRB::Irb.new(workspace) + binding_irb = IRB::Irb.new(workspace, from_binding: true) binding_irb.context.irb_path = irb_path binding_irb.run(IRB.conf) binding_irb.debug_break diff --git a/lib/irb/command/base.rb b/lib/irb/command/base.rb index b078b48237..1d406630a2 100644 --- a/lib/irb/command/base.rb +++ b/lib/irb/command/base.rb @@ -18,12 +18,12 @@ module IRB class << self def category(category = nil) @category = category if category - @category + @category || "No category" end def description(description = nil) @description = description if description - @description + @description || "No description provided." end def help_message(help_message = nil) diff --git a/lib/irb/command/debug.rb b/lib/irb/command/debug.rb index f9aca0a672..8a091a49ed 100644 --- a/lib/irb/command/debug.rb +++ b/lib/irb/command/debug.rb @@ -8,11 +8,6 @@ module IRB category "Debugging" description "Start the debugger of debug.gem." - BINDING_IRB_FRAME_REGEXPS = [ - '<internal:prelude>', - binding.method(:irb).source_location.first, - ].map { |file| /\A#{Regexp.escape(file)}:\d+:in (`|'Binding#)irb'\z/ } - def execute(_arg) execute_debug_command end @@ -36,7 +31,7 @@ module IRB # 3. Insert a debug breakpoint at `Irb#debug_break` with the intended command. # 4. Exit the current Irb#run call via `throw :IRB_EXIT`. # 5. `Irb#debug_break` will be called and trigger the breakpoint, which will run the intended command. - unless binding_irb? + unless irb_context.from_binding? puts "Debugging commands are only available when IRB is started with binding.irb" return end @@ -60,16 +55,6 @@ module IRB throw :IRB_EXIT end end - - private - - def binding_irb? - caller.any? do |frame| - BINDING_IRB_FRAME_REGEXPS.any? do |regexp| - frame.match?(regexp) - end - end - end end class DebugCommand < Debug diff --git a/lib/irb/command/help.rb b/lib/irb/command/help.rb index 1ed7a7707c..c2018f9b30 100644 --- a/lib/irb/command/help.rb +++ b/lib/irb/command/help.rb @@ -28,17 +28,9 @@ module IRB commands_grouped_by_categories = commands_info.group_by { |cmd| cmd[:category] } commands_grouped_by_categories["Helper methods"] = helper_methods_info - user_aliases = irb_context.instance_variable_get(:@user_aliases) - - commands_grouped_by_categories["Aliases"] = user_aliases.map do |alias_name, target| - { display_name: alias_name, description: "Alias for `#{target}`" } - end - if irb_context.with_debugger # Remove the original "Debugging" category commands_grouped_by_categories.delete("Debugging") - # Add an empty "Debugging (from debug.gem)" category at the end - commands_grouped_by_categories["Debugging (from debug.gem)"] = [] end longest_cmd_name_length = commands_info.map { |c| c[:display_name].length }.max @@ -46,15 +38,31 @@ module IRB output = StringIO.new help_cmds = commands_grouped_by_categories.delete("Help") + no_category_cmds = commands_grouped_by_categories.delete("No category") + aliases = irb_context.instance_variable_get(:@user_aliases).map do |alias_name, target| + { display_name: alias_name, description: "Alias for `#{target}`" } + end + # Display help commands first add_category_to_output("Help", help_cmds, output, longest_cmd_name_length) + # Display the rest of the commands grouped by categories commands_grouped_by_categories.each do |category, cmds| add_category_to_output(category, cmds, output, longest_cmd_name_length) end + # Display commands without a category + if no_category_cmds + add_category_to_output("No category", no_category_cmds, output, longest_cmd_name_length) + end + + # Display aliases + add_category_to_output("Aliases", aliases, output, longest_cmd_name_length) + # Append the debugger help at the end if irb_context.with_debugger + # Add "Debugging (from debug.gem)" category as title + add_category_to_output("Debugging (from debug.gem)", [], output, longest_cmd_name_length) output.puts DEBUGGER__.help end diff --git a/lib/irb/context.rb b/lib/irb/context.rb index 173f3dae5d..aafce7aade 100644 --- a/lib/irb/context.rb +++ b/lib/irb/context.rb @@ -73,11 +73,12 @@ module IRB self.prompt_mode = IRB.conf[:PROMPT_MODE] - if IRB.conf[:SINGLE_IRB] or !defined?(IRB::JobManager) - @irb_name = IRB.conf[:IRB_NAME] - else - @irb_name = IRB.conf[:IRB_NAME]+"#"+IRB.JobManager.n_jobs.to_s + @irb_name = IRB.conf[:IRB_NAME] + + unless IRB.conf[:SINGLE_IRB] or !defined?(IRB::JobManager) + @irb_name = @irb_name + "#" + IRB.JobManager.n_jobs.to_s end + self.irb_path = "(" + @irb_name + ")" case input_method @@ -607,6 +608,10 @@ module IRB nil end + def from_binding? + @irb.from_binding + end + def evaluate_expression(code, line_no) # :nodoc: result = nil if IRB.conf[:MEASURE] && IRB.conf[:MEASURE_CALLBACKS].empty? diff --git a/lib/irb/default_commands.rb b/lib/irb/default_commands.rb index 1bbc68efa7..91c6b2d041 100644 --- a/lib/irb/default_commands.rb +++ b/lib/irb/default_commands.rb @@ -2,32 +2,33 @@ require_relative "command" require_relative "command/internal_helpers" -require_relative "command/context" -require_relative "command/exit" -require_relative "command/force_exit" -require_relative "command/chws" -require_relative "command/pushws" -require_relative "command/subirb" -require_relative "command/load" -require_relative "command/debug" -require_relative "command/edit" +require_relative "command/backtrace" require_relative "command/break" require_relative "command/catch" -require_relative "command/next" -require_relative "command/delete" -require_relative "command/step" +require_relative "command/chws" +require_relative "command/context" require_relative "command/continue" +require_relative "command/debug" +require_relative "command/delete" +require_relative "command/disable_irb" +require_relative "command/edit" +require_relative "command/exit" require_relative "command/finish" -require_relative "command/backtrace" -require_relative "command/info" +require_relative "command/force_exit" require_relative "command/help" -require_relative "command/show_doc" +require_relative "command/history" +require_relative "command/info" require_relative "command/irb_info" +require_relative "command/load" require_relative "command/ls" require_relative "command/measure" +require_relative "command/next" +require_relative "command/pushws" +require_relative "command/show_doc" require_relative "command/show_source" +require_relative "command/step" +require_relative "command/subirb" require_relative "command/whereami" -require_relative "command/history" module IRB module Command @@ -235,6 +236,10 @@ module IRB [:history, NO_OVERRIDE], [:hist, NO_OVERRIDE] ) + + _register_with_aliases(:irb_disable_irb, Command::DisableIrb, + [:disable_irb, NO_OVERRIDE] + ) end ExtendCommand = Command diff --git a/lib/irb/helper_method/conf.rb b/lib/irb/helper_method/conf.rb index 460f5ab78a..718ed279c0 100644 --- a/lib/irb/helper_method/conf.rb +++ b/lib/irb/helper_method/conf.rb @@ -1,7 +1,7 @@ module IRB module HelperMethod class Conf < Base - description "Returns the current context." + description "Returns the current IRB context." def execute IRB.CurrentContext diff --git a/lib/irb/init.rb b/lib/irb/init.rb index 355047519c..7dc08912ef 100644 --- a/lib/irb/init.rb +++ b/lib/irb/init.rb @@ -52,6 +52,7 @@ module IRB # :nodoc: IRB.init_error IRB.parse_opts(argv: argv) IRB.run_config + IRB.validate_config IRB.load_modules unless @CONF[:PROMPT][@CONF[:PROMPT_MODE]] @@ -427,6 +428,40 @@ module IRB # :nodoc: @irbrc_files end + def IRB.validate_config + conf[:IRB_NAME] = conf[:IRB_NAME].to_s + + irb_rc = conf[:IRB_RC] + unless irb_rc.nil? || irb_rc.respond_to?(:call) + raise_validation_error "IRB.conf[:IRB_RC] should be a callable object. Got #{irb_rc.inspect}." + end + + back_trace_limit = conf[:BACK_TRACE_LIMIT] + unless back_trace_limit.is_a?(Integer) + raise_validation_error "IRB.conf[:BACK_TRACE_LIMIT] should be an integer. Got #{back_trace_limit.inspect}." + end + + prompt = conf[:PROMPT] + unless prompt.is_a?(Hash) + msg = "IRB.conf[:PROMPT] should be a Hash. Got #{prompt.inspect}." + + if prompt.is_a?(Symbol) + msg += " Did you mean to set `IRB.conf[:PROMPT_MODE]`?" + end + + raise_validation_error msg + end + + eval_history = conf[:EVAL_HISTORY] + unless eval_history.nil? || eval_history.is_a?(Integer) + raise_validation_error "IRB.conf[:EVAL_HISTORY] should be an integer. Got #{eval_history.inspect}." + end + end + + def IRB.raise_validation_error(msg) + raise TypeError, msg, @irbrc_files + end + # loading modules def IRB.load_modules for m in @CONF[:LOAD_MODULES] diff --git a/lib/irb/input-method.rb b/lib/irb/input-method.rb index 9a24c885ba..ced35a2c5a 100644 --- a/lib/irb/input-method.rb +++ b/lib/irb/input-method.rb @@ -67,7 +67,7 @@ module IRB # # See IO#gets for more information. def gets - puts + puts if @stdout.tty? # workaround for debug compatibility test print @prompt line = @stdin.gets @line[@line_no += 1] = line @@ -328,10 +328,11 @@ module IRB ->() { dialog.trap_key = nil alt_d = [ - [Reline::Key.new(nil, 0xE4, true)], # Normal Alt+d. [27, 100], # Normal Alt+d when convert-meta isn't used. - [195, 164], # The "ä" that appears when Alt+d is pressed on xterm. - [226, 136, 130] # The "∂" that appears when Alt+d in pressed on iTerm2. + # When option/alt is not configured as a meta key in terminal emulator, + # option/alt + d will send a unicode character depend on OS keyboard setting. + [195, 164], # "ä" in somewhere (FIXME: environment information is unknown). + [226, 136, 130] # "∂" Alt+d on Mac keyboard. ] if just_cursor_moving and completion_journey_data.nil? diff --git a/lib/irb/ruby-lex.rb b/lib/irb/ruby-lex.rb index cfe36be83f..f6ac7f0f5f 100644 --- a/lib/irb/ruby-lex.rb +++ b/lib/irb/ruby-lex.rb @@ -219,28 +219,7 @@ module IRB :unrecoverable_error rescue SyntaxError => e case e.message - when /unterminated (?:string|regexp) meets end of file/ - # "unterminated regexp meets end of file" - # - # example: - # / - # - # "unterminated string meets end of file" - # - # example: - # ' - return :recoverable_error - when /syntax error, unexpected end-of-input/ - # "syntax error, unexpected end-of-input, expecting keyword_end" - # - # example: - # if true - # hoge - # if false - # fuga - # end - return :recoverable_error - when /syntax error, unexpected keyword_end/ + when /unexpected keyword_end/ # "syntax error, unexpected keyword_end" # # example: @@ -250,7 +229,7 @@ module IRB # example: # end return :unrecoverable_error - when /syntax error, unexpected '\.'/ + when /unexpected '\.'/ # "syntax error, unexpected '.'" # # example: @@ -262,6 +241,27 @@ module IRB # example: # method / f / return :unrecoverable_error + when /unterminated (?:string|regexp) meets end of file/ + # "unterminated regexp meets end of file" + # + # example: + # / + # + # "unterminated string meets end of file" + # + # example: + # ' + return :recoverable_error + when /unexpected end-of-input/ + # "syntax error, unexpected end-of-input, expecting keyword_end" + # + # example: + # if true + # hoge + # if false + # fuga + # end + return :recoverable_error else return :other_error end diff --git a/lib/irb/version.rb b/lib/irb/version.rb index 146daf7258..c41917329c 100644 --- a/lib/irb/version.rb +++ b/lib/irb/version.rb @@ -5,7 +5,7 @@ # module IRB # :nodoc: - VERSION = "1.13.0" + VERSION = "1.13.1" @RELEASE_VERSION = VERSION - @LAST_UPDATE_DATE = "2024-05-01" + @LAST_UPDATE_DATE = "2024-05-05" end diff --git a/lib/logger/period.rb b/lib/logger/period.rb index 0a291dbbbe..a0359defe3 100644 --- a/lib/logger/period.rb +++ b/lib/logger/period.rb @@ -8,14 +8,14 @@ class Logger def next_rotate_time(now, shift_age) case shift_age - when 'daily' + when 'daily', :daily t = Time.mktime(now.year, now.month, now.mday) + SiD - when 'weekly' + when 'weekly', :weekly t = Time.mktime(now.year, now.month, now.mday) + SiD * (7 - now.wday) - when 'monthly' + when 'monthly', :monthly t = Time.mktime(now.year, now.month, 1) + SiD * 32 return Time.mktime(t.year, t.month, 1) - when 'now', 'everytime' + when 'now', 'everytime', :now, :everytime return now else raise ArgumentError, "invalid :shift_age #{shift_age.inspect}, should be daily, weekly, monthly, or everytime" @@ -30,13 +30,13 @@ class Logger def previous_period_end(now, shift_age) case shift_age - when 'daily' + when 'daily', :daily t = Time.mktime(now.year, now.month, now.mday) - SiD / 2 - when 'weekly' + when 'weekly', :weekly t = Time.mktime(now.year, now.month, now.mday) - (SiD * now.wday + SiD / 2) - when 'monthly' + when 'monthly', :monthly t = Time.mktime(now.year, now.month, 1) - SiD / 2 - when 'now', 'everytime' + when 'now', 'everytime', :now, :everytime return now else raise ArgumentError, "invalid :shift_age #{shift_age.inspect}, should be daily, weekly, monthly, or everytime" diff --git a/lib/net/http.rb b/lib/net/http.rb index 6b78c264af..958ff09f0e 100644 --- a/lib/net/http.rb +++ b/lib/net/http.rb @@ -67,6 +67,8 @@ module Net #:nodoc: # Net::HTTP.post(uri, data) # params = {title: 'foo', body: 'bar', userId: 1} # Net::HTTP.post_form(uri, params) + # data = '{"title": "foo", "body": "bar", "userId": 1}' + # Net::HTTP.put(uri, data) # # - If performance is important, consider using sessions, which lower request overhead. # This {session}[rdoc-ref:Net::HTTP@Sessions] has multiple requests for @@ -524,6 +526,8 @@ module Net #:nodoc: # Sends a POST request with form data and returns a response object. # - {::post}[rdoc-ref:Net::HTTP.post]: # Sends a POST request with data and returns a response object. + # - {::put}[rdoc-ref:Net::HTTP.put]: + # Sends a PUT request with data and returns a response object. # - {#copy}[rdoc-ref:Net::HTTP#copy]: # Sends a COPY request and returns a response object. # - {#delete}[rdoc-ref:Net::HTTP#delete]: @@ -893,6 +897,39 @@ module Net #:nodoc: } end + # Sends a PUT request to the server; returns a Net::HTTPResponse object. + # + # Argument +url+ must be a URL; + # argument +data+ must be a string: + # + # _uri = uri.dup + # _uri.path = '/posts' + # data = '{"title": "foo", "body": "bar", "userId": 1}' + # headers = {'content-type': 'application/json'} + # res = Net::HTTP.put(_uri, data, headers) # => #<Net::HTTPCreated 201 Created readbody=true> + # puts res.body + # + # Output: + # + # { + # "title": "foo", + # "body": "bar", + # "userId": 1, + # "id": 101 + # } + # + # Related: + # + # - Net::HTTP::Put: request class for \HTTP method +PUT+. + # - Net::HTTP#put: convenience method for \HTTP method +PUT+. + # + def HTTP.put(url, data, header = nil) + start(url.hostname, url.port, + :use_ssl => url.scheme == 'https' ) {|http| + http.put(url, data, header) + } + end + # # \HTTP session management # @@ -2016,6 +2053,11 @@ module Net #:nodoc: # http = Net::HTTP.new(hostname) # http.put('/todos/1', data) # => #<Net::HTTPOK 200 OK readbody=true> # + # Related: + # + # - Net::HTTP::Put: request class for \HTTP method PUT. + # - Net::HTTP.put: sends PUT request, returns response body. + # def put(path, data, initheader = nil) request(Put.new(path, initheader), data) end diff --git a/lib/net/http/header.rb b/lib/net/http/header.rb index 6660c8075a..f6c36f1b5e 100644 --- a/lib/net/http/header.rb +++ b/lib/net/http/header.rb @@ -491,7 +491,7 @@ module Net::HTTPHeader alias canonical_each each_capitalized def capitalize(name) - name.to_s.split(/-/).map {|s| s.capitalize }.join('-') + name.to_s.split('-'.freeze).map {|s| s.capitalize }.join('-'.freeze) end private :capitalize diff --git a/lib/net/http/requests.rb b/lib/net/http/requests.rb index 5724164205..e58057adf1 100644 --- a/lib/net/http/requests.rb +++ b/lib/net/http/requests.rb @@ -124,6 +124,11 @@ end # - {Idempotent}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Idempotent_methods]: yes. # - {Cacheable}[https://en.wikipedia.org/wiki/Hypertext_Transfer_Protocol#Cacheable_methods]: no. # +# Related: +# +# - Net::HTTP.put: sends +PUT+ request, returns response object. +# - Net::HTTP#put: sends +PUT+ request, returns response object. +# class Net::HTTP::Put < Net::HTTPRequest METHOD = 'PUT' REQUEST_HAS_BODY = true diff --git a/lib/prism.rb b/lib/prism.rb index c733baf8c9..66a64e7fd0 100644 --- a/lib/prism.rb +++ b/lib/prism.rb @@ -13,7 +13,6 @@ module Prism autoload :BasicVisitor, "prism/visitor" autoload :Compiler, "prism/compiler" - autoload :Debug, "prism/debug" autoload :DesugarCompiler, "prism/desugar_compiler" autoload :Dispatcher, "prism/dispatcher" autoload :DotVisitor, "prism/dot_visitor" @@ -32,7 +31,6 @@ module Prism # Some of these constants are not meant to be exposed, so marking them as # private here. - private_constant :Debug private_constant :LexCompat private_constant :LexRipper @@ -67,11 +65,10 @@ module Prism end end +require_relative "prism/polyfill/byteindex" require_relative "prism/node" require_relative "prism/node_ext" require_relative "prism/parse_result" -require_relative "prism/parse_result/comments" -require_relative "prism/parse_result/newlines" # This is a Ruby implementation of the prism parser. If we're running on CRuby # and we haven't explicitly set the PRISM_FFI_BACKEND environment variable, then diff --git a/lib/prism/debug.rb b/lib/prism/debug.rb deleted file mode 100644 index 74f824faa7..0000000000 --- a/lib/prism/debug.rb +++ /dev/null @@ -1,249 +0,0 @@ -# frozen_string_literal: true - -module Prism - # This module is used for testing and debugging and is not meant to be used by - # consumers of this library. - module Debug - # A wrapper around a RubyVM::InstructionSequence that provides a more - # convenient interface for accessing parts of the iseq. - class ISeq # :nodoc: - attr_reader :parts - - def initialize(parts) - @parts = parts - end - - def type - parts[0] - end - - def local_table - parts[10] - end - - def instructions - parts[13] - end - - def each_child - instructions.each do |instruction| - # Only look at arrays. Other instructions are line numbers or - # tracepoint events. - next unless instruction.is_a?(Array) - - instruction.each do |opnd| - # Only look at arrays. Other operands are literals. - next unless opnd.is_a?(Array) - - # Only look at instruction sequences. Other operands are literals. - next unless opnd[0] == "YARVInstructionSequence/SimpleDataFormat" - - yield ISeq.new(opnd) - end - end - end - end - - private_constant :ISeq - - # :call-seq: - # Debug::cruby_locals(source) -> Array - # - # For the given source, compiles with CRuby and returns a list of all of the - # sets of local variables that were encountered. - def self.cruby_locals(source) - verbose, $VERBOSE = $VERBOSE, nil - - begin - locals = [] #: Array[Array[Symbol | Integer]] - stack = [ISeq.new(RubyVM::InstructionSequence.compile(source).to_a)] - - while (iseq = stack.pop) - names = [*iseq.local_table] - names.map!.with_index do |name, index| - # When an anonymous local variable is present in the iseq's local - # table, it is represented as the stack offset from the top. - # However, when these are dumped to binary and read back in, they - # are replaced with the symbol :#arg_rest. To consistently handle - # this, we replace them here with their index. - if name == :"#arg_rest" - names.length - index + 1 - else - name - end - end - - locals << names - iseq.each_child { |child| stack << child } - end - - locals - ensure - $VERBOSE = verbose - end - end - - # Used to hold the place of a local that will be in the local table but - # cannot be accessed directly from the source code. For example, the - # iteration variable in a for loop or the positional parameter on a method - # definition that is destructured. - AnonymousLocal = Object.new - private_constant :AnonymousLocal - - # :call-seq: - # Debug::prism_locals(source) -> Array - # - # For the given source, parses with prism and returns a list of all of the - # sets of local variables that were encountered. - def self.prism_locals(source) - locals = [] #: Array[Array[Symbol | Integer]] - stack = [Prism.parse(source).value] #: Array[Prism::node] - - while (node = stack.pop) - case node - when BlockNode, DefNode, LambdaNode - names = node.locals - params = - if node.is_a?(DefNode) - node.parameters - elsif node.parameters.is_a?(NumberedParametersNode) - nil - else - node.parameters&.parameters - end - - # prism places parameters in the same order that they appear in the - # source. CRuby places them in the order that they need to appear - # according to their own internal calling convention. We mimic that - # order here so that we can compare properly. - if params - sorted = [ - *params.requireds.map do |required| - if required.is_a?(RequiredParameterNode) - required.name - else - AnonymousLocal - end - end, - *params.optionals.map(&:name), - *((params.rest.name || :*) if params.rest && !params.rest.is_a?(ImplicitRestNode)), - *params.posts.map do |post| - if post.is_a?(RequiredParameterNode) - post.name - else - AnonymousLocal - end - end, - *params.keywords.grep(RequiredKeywordParameterNode).map(&:name), - *params.keywords.grep(OptionalKeywordParameterNode).map(&:name), - ] - - sorted << AnonymousLocal if params.keywords.any? - - if params.keyword_rest.is_a?(ForwardingParameterNode) - sorted.push(:*, :**, :&, :"...") - elsif params.keyword_rest.is_a?(KeywordRestParameterNode) - sorted << (params.keyword_rest.name || :**) - end - - # Recurse down the parameter tree to find any destructured - # parameters and add them after the other parameters. - param_stack = params.requireds.concat(params.posts).grep(MultiTargetNode).reverse - while (param = param_stack.pop) - case param - when MultiTargetNode - param_stack.concat(param.rights.reverse) - param_stack << param.rest if param.rest&.expression && !sorted.include?(param.rest.expression.name) - param_stack.concat(param.lefts.reverse) - when RequiredParameterNode - sorted << param.name - when SplatNode - sorted << param.expression.name - end - end - - if params.block - sorted << (params.block.name || :&) - end - - names = sorted.concat(names - sorted) - end - - names.map!.with_index do |name, index| - if name == AnonymousLocal - names.length - index + 1 - else - name - end - end - - locals << names - when ClassNode, ModuleNode, ProgramNode, SingletonClassNode - locals << node.locals - when ForNode - locals << [2] - when PostExecutionNode - locals.push([], []) - when InterpolatedRegularExpressionNode - locals << [] if node.once? - end - - stack.concat(node.compact_child_nodes) - end - - locals - end - - # :call-seq: - # Debug::newlines(source) -> Array - # - # For the given source string, return the byte offsets of every newline in - # the source. - def self.newlines(source) - Prism.parse(source).source.offsets - end - - # A wrapping around prism's internal encoding data structures. This is used - # for reflection and debugging purposes. - class Encoding - # The name of the encoding, that can be passed to Encoding.find. - attr_reader :name - - # Initialize a new encoding with the given name and whether or not it is - # a multibyte encoding. - def initialize(name, multibyte) - @name = name - @multibyte = multibyte - end - - # Whether or not the encoding is a multibyte encoding. - def multibyte? - @multibyte - end - - # Returns the number of bytes of the first character in the source string, - # if it is valid for the encoding. Otherwise, returns 0. - def width(source) - Encoding._width(name, source) - end - - # Returns true if the first character in the source string is a valid - # alphanumeric character for the encoding. - def alnum?(source) - Encoding._alnum?(name, source) - end - - # Returns true if the first character in the source string is a valid - # alphabetic character for the encoding. - def alpha?(source) - Encoding._alpha?(name, source) - end - - # Returns true if the first character in the source string is a valid - # uppercase character for the encoding. - def upper?(source) - Encoding._upper?(name, source) - end - end - end -end diff --git a/lib/prism/desugar_compiler.rb b/lib/prism/desugar_compiler.rb index 9b62c00df3..de02445149 100644 --- a/lib/prism/desugar_compiler.rb +++ b/lib/prism/desugar_compiler.rb @@ -73,7 +73,7 @@ module Prism # Desugar `x += y` to `x = x + y` def compile - operator_loc = node.operator_loc.chop + binary_operator_loc = node.binary_operator_loc.chop write_class.new( source, @@ -84,15 +84,15 @@ module Prism 0, read_class.new(source, *arguments, node.name_loc), nil, - operator_loc.slice.to_sym, - operator_loc, + binary_operator_loc.slice.to_sym, + binary_operator_loc, nil, ArgumentsNode.new(source, 0, [node.value], node.value.location), nil, nil, node.location ), - node.operator_loc.copy(start_offset: node.operator_loc.end_offset - 1, length: 1), + node.binary_operator_loc.copy(start_offset: node.binary_operator_loc.end_offset - 1, length: 1), node.location ) end diff --git a/lib/prism/ffi.rb b/lib/prism/ffi.rb index 2014ccea31..6b48af43cc 100644 --- a/lib/prism/ffi.rb +++ b/lib/prism/ffi.rb @@ -200,8 +200,8 @@ module Prism class << self # Mirror the Prism.dump API by using the serialization API. - def dump(code, **options) - LibRubyParser::PrismString.with_string(code) { |string| dump_common(string, options) } + def dump(source, **options) + LibRubyParser::PrismString.with_string(source) { |string| dump_common(string, options) } end # Mirror the Prism.dump_file API by using the serialization API. @@ -302,6 +302,27 @@ module Prism !parse_file_success?(filepath, **options) end + # Mirror the Prism.profile API by using the serialization API. + def profile(source, **options) + LibRubyParser::PrismString.with_string(source) do |string| + LibRubyParser::PrismBuffer.with do |buffer| + LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options)) + nil + end + end + end + + # Mirror the Prism.profile_file API by using the serialization API. + def profile_file(filepath, **options) + LibRubyParser::PrismString.with_file(filepath) do |string| + LibRubyParser::PrismBuffer.with do |buffer| + options[:filepath] = filepath + LibRubyParser.pm_serialize_parse(buffer.pointer, string.pointer, string.length, dump_options(options)) + nil + end + end + end + private def dump_common(string, options) # :nodoc: @@ -317,7 +338,7 @@ module Prism buffer.read end - Serialize.load_tokens(Source.new(code), serialized) + Serialize.load_tokens(Source.for(code), serialized) end def parse_common(string, code, options) # :nodoc: @@ -329,7 +350,7 @@ module Prism LibRubyParser::PrismBuffer.with do |buffer| LibRubyParser.pm_serialize_parse_comments(buffer.pointer, string.pointer, string.length, dump_options(options)) - source = Source.new(code) + source = Source.for(code) loader = Serialize::Loader.new(source, buffer.read) loader.load_header @@ -343,7 +364,7 @@ module Prism LibRubyParser::PrismBuffer.with do |buffer| LibRubyParser.pm_serialize_parse_lex(buffer.pointer, string.pointer, string.length, dump_options(options)) - source = Source.new(code) + source = Source.for(code) loader = Serialize::Loader.new(source, buffer.read) tokens = loader.load_tokens @@ -394,7 +415,7 @@ module Prism template << "L" if (encoding = options[:encoding]) - name = encoding.name + name = encoding.is_a?(Encoding) ? encoding.name : encoding values.push(name.bytesize, name.b) template << "A*" else @@ -408,7 +429,7 @@ module Prism values << dump_options_command_line(options) template << "C" - values << { nil => 0, "3.3.0" => 1, "3.4.0" => 0, "latest" => 0 }.fetch(options[:version]) + values << { nil => 0, "3.3.0" => 1, "3.3.1" => 1, "3.4.0" => 0, "latest" => 0 }.fetch(options[:version]) template << "L" if (scopes = options[:scopes]) diff --git a/lib/prism/lex_compat.rb b/lib/prism/lex_compat.rb index f199af1883..4f8e443a3b 100644 --- a/lib/prism/lex_compat.rb +++ b/lib/prism/lex_compat.rb @@ -861,7 +861,7 @@ module Prism # We sort by location to compare against Ripper's output tokens.sort_by!(&:location) - Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, Source.new(source)) + Result.new(tokens, result.comments, result.magic_comments, result.data_loc, result.errors, result.warnings, Source.for(source)) end end diff --git a/lib/prism/node_ext.rb b/lib/prism/node_ext.rb index 10fc1decbf..aa6a18cf29 100644 --- a/lib/prism/node_ext.rb +++ b/lib/prism/node_ext.rb @@ -3,6 +3,20 @@ # Here we are reopening the prism module to provide methods on nodes that aren't # templated and are meant as convenience methods. module Prism + class Node + def deprecated(*replacements) # :nodoc: + location = caller_locations(1, 1) + location = location[0].label if location + suggest = replacements.map { |replacement| "#{self.class}##{replacement}" } + + warn(<<~MSG, category: :deprecated) + [deprecation]: #{self.class}##{location} is deprecated and will be \ + removed in the next major version. Use #{suggest.join("/")} instead. + #{(caller(1, 3) || []).join("\n")} + MSG + end + end + module RegularExpressionOptions # :nodoc: # Returns a numeric value that represents the flags that were used to create # the regular expression. @@ -92,7 +106,19 @@ module Prism class RationalNode < Node # Returns the value of the node as a Ruby Rational. def value - Rational(numeric.is_a?(IntegerNode) ? numeric.value : slice.chomp("r")) + Rational(numerator, denominator) + end + + # Returns the value of the node as an IntegerNode or a FloatNode. This + # method is deprecated in favor of #value or #numerator/#denominator. + def numeric + deprecated("value", "numerator", "denominator") + + if denominator == 1 + IntegerNode.new(source, flags, numerator, location.chop) + else + FloatNode.new(source, numerator.to_f / denominator, location.chop) + end end end @@ -168,13 +194,7 @@ module Prism # constant read or a missing node. To not cause a breaking change, we # continue to supply that API. def child - warn(<<~MSG, category: :deprecated) - DEPRECATED: ConstantPathNode#child is deprecated and will be removed \ - in the next major version. Use \ - ConstantPathNode#name/ConstantPathNode#name_loc instead. Called from \ - #{caller(1..1).first}. - MSG - + deprecated("name", "name_loc") name ? ConstantReadNode.new(source, name, name_loc) : MissingNode.new(source, location) end end @@ -210,13 +230,7 @@ module Prism # constant read or a missing node. To not cause a breaking change, we # continue to supply that API. def child - warn(<<~MSG, category: :deprecated) - DEPRECATED: ConstantPathTargetNode#child is deprecated and will be \ - removed in the next major version. Use \ - ConstantPathTargetNode#name/ConstantPathTargetNode#name_loc instead. \ - Called from #{caller(1..1).first}. - MSG - + deprecated("name", "name_loc") name ? ConstantReadNode.new(source, name, name_loc) : MissingNode.new(source, location) end end @@ -250,9 +264,10 @@ module Prism end posts.each do |param| - if param.is_a?(MultiTargetNode) + case param + when MultiTargetNode names << [:req] - elsif param.is_a?(NoKeywordsParameterNode) + when NoKeywordsParameterNode, KeywordRestParameterNode, ForwardingParameterNode # Invalid syntax, e.g. "def f(**nil, ...)" moves the NoKeywordsParameterNode to posts raise "Invalid syntax" else @@ -286,4 +301,147 @@ module Prism names end end + + class CallNode < Node + # When a call node has the attribute_write flag set, it means that the call + # is using the attribute write syntax. This is either a method call to []= + # or a method call to a method that ends with =. Either way, the = sign is + # present in the source. + # + # Prism returns the message_loc _without_ the = sign attached, because there + # can be any amount of space between the message and the = sign. However, + # sometimes you want the location of the full message including the inner + # space and the = sign. This method provides that. + def full_message_loc + attribute_write? ? message_loc&.adjoin("=") : message_loc + end + end + + class CallOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class ClassVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class ConstantOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class ConstantPathOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class GlobalVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class IndexOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class InstanceVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end + + class LocalVariableOperatorWriteNode < Node + # Returns the binary operator used to modify the receiver. This method is + # deprecated in favor of #binary_operator. + def operator + deprecated("binary_operator") + binary_operator + end + + # Returns the location of the binary operator used to modify the receiver. + # This method is deprecated in favor of #binary_operator_loc. + def operator_loc + deprecated("binary_operator_loc") + binary_operator_loc + end + end end diff --git a/lib/prism/parse_result.rb b/lib/prism/parse_result.rb index ff8b1dc8bf..798fde09e5 100644 --- a/lib/prism/parse_result.rb +++ b/lib/prism/parse_result.rb @@ -5,6 +5,14 @@ module Prism # conjunction with locations to allow them to resolve line numbers and source # ranges. class Source + # Create a new source object with the given source code. This method should + # be used instead of `new` and it will return either a `Source` or a + # specialized and more performant `ASCIISource` if no multibyte characters + # are present in the source code. + def self.for(source, start_line = 1, offsets = []) + source.ascii_only? ? ASCIISource.new(source, start_line, offsets): new(source, start_line, offsets) + end + # The source code that this source object represents. attr_reader :source @@ -111,6 +119,39 @@ module Prism end end + # Specialized version of Prism::Source for source code that includes ASCII + # characters only. This class is used to apply performance optimizations that + # cannot be applied to sources that include multibyte characters. Sources that + # include multibyte characters are represented by the Prism::Source class. + class ASCIISource < Source + # Return the character offset for the given byte offset. + def character_offset(byte_offset) + byte_offset + end + + # Return the column number in characters for the given byte offset. + def character_column(byte_offset) + byte_offset - line_start(byte_offset) + end + + # Returns the offset from the start of the file for the given byte offset + # counting in code units for the given encoding. + # + # This method is tested with UTF-8, UTF-16, and UTF-32. If there is the + # concept of code units that differs from the number of characters in other + # encodings, it is not captured here. + def code_units_offset(byte_offset, encoding) + byte_offset + end + + # Specialized version of `code_units_column` that does not depend on + # `code_units_offset`, which is a more expensive operation. This is + # essentialy the same as `Prism::Source#column`. + def code_units_column(byte_offset, encoding) + byte_offset - line_start(byte_offset) + end + end + # This represents a location in the source. class Location # A Source object that is used to determine more information from the given @@ -306,6 +347,18 @@ module Prism Location.new(source, start_offset, other.end_offset - start_offset) end + + # Join this location with the first occurrence of the string in the source + # that occurs after this location on the same line, and return the new + # location. This will raise an error if the string does not exist. + def adjoin(string) + line_suffix = source.slice(end_offset, source.line_end(end_offset) - end_offset) + + line_suffix_index = line_suffix.byteindex(string) + raise "Could not find #{string}" if line_suffix_index.nil? + + Location.new(source, start_offset, length + line_suffix_index + string.bytesize) + end end # This represents a comment that was encountered during parsing. It is the @@ -521,6 +574,12 @@ module Prism # This is a result specific to the `parse` and `parse_file` methods. class ParseResult < Result + autoload :Comments, "prism/parse_result/comments" + autoload :Newlines, "prism/parse_result/newlines" + + private_constant :Comments + private_constant :Newlines + # The syntax tree that was parsed from the source code. attr_reader :value @@ -534,6 +593,17 @@ module Prism def deconstruct_keys(keys) super.merge!(value: value) end + + # Attach the list of comments to their respective locations in the tree. + def attach_comments! + Comments.new(self).attach! # steep:ignore + end + + # Walk the tree and mark nodes that are on a new line, loosely emulating + # the behavior of CRuby's `:line` tracepoint event. + def mark_newlines! + value.accept(Newlines.new(source.offsets.size)) # steep:ignore + end end # This is a result specific to the `lex` and `lex_file` methods. diff --git a/lib/prism/parse_result/comments.rb b/lib/prism/parse_result/comments.rb index f8f74d2503..22c4148b2c 100644 --- a/lib/prism/parse_result/comments.rb +++ b/lib/prism/parse_result/comments.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Prism - class ParseResult + class ParseResult < Result # When we've parsed the source, we have both the syntax tree and the list of # comments that we found in the source. This class is responsible for # walking the tree and finding the nearest location to attach each comment. @@ -183,12 +183,5 @@ module Prism [preceding, NodeTarget.new(node), following] end end - - private_constant :Comments - - # Attach the list of comments to their respective locations in the tree. - def attach_comments! - Comments.new(self).attach! # steep:ignore - end end end diff --git a/lib/prism/parse_result/newlines.rb b/lib/prism/parse_result/newlines.rb index 927c17fe2f..808a129a6b 100644 --- a/lib/prism/parse_result/newlines.rb +++ b/lib/prism/parse_result/newlines.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module Prism - class ParseResult + class ParseResult < Result # The :line tracepoint event gets fired whenever the Ruby VM encounters an # expression on a new line. The types of expressions that can trigger this # event are: @@ -17,21 +17,27 @@ module Prism # Note that the logic in this file should be kept in sync with the Java # MarkNewlinesVisitor, since that visitor is responsible for marking the # newlines for JRuby/TruffleRuby. + # + # This file is autoloaded only when `mark_newlines!` is called, so the + # re-opening of the various nodes in this file will only be performed in + # that case. We do that to avoid storing the extra `@newline` instance + # variable on every node if we don't need it. class Newlines < Visitor # Create a new Newlines visitor with the given newline offsets. - def initialize(newline_marked) - @newline_marked = newline_marked + def initialize(lines) + # @type var lines: Integer + @lines = Array.new(1 + lines, false) end # Permit block/lambda nodes to mark newlines within themselves. def visit_block_node(node) - old_newline_marked = @newline_marked - @newline_marked = Array.new(old_newline_marked.size, false) + old_lines = @lines + @lines = Array.new(old_lines.size, false) begin super(node) ensure - @newline_marked = old_newline_marked + @lines = old_lines end end @@ -39,7 +45,7 @@ module Prism # Mark if/unless nodes as newlines. def visit_if_node(node) - node.set_newline_flag(@newline_marked) + node.newline!(@lines) super(node) end @@ -48,17 +54,101 @@ module Prism # Permit statements lists to mark newlines within themselves. def visit_statements_node(node) node.body.each do |child| - child.set_newline_flag(@newline_marked) + child.newline!(@lines) end super(node) end end + end + + class Node + def newline? # :nodoc: + @newline ? true : false + end + + def newline!(lines) # :nodoc: + line = location.start_line + unless lines[line] + lines[line] = true + @newline = true + end + end + end + + class BeginNode < Node + def newline!(lines) # :nodoc: + # Never mark BeginNode with a newline flag, mark children instead. + end + end + + class ParenthesesNode < Node + def newline!(lines) # :nodoc: + # Never mark ParenthesesNode with a newline flag, mark children instead. + end + end + + class IfNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class UnlessNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class UntilNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class WhileNode < Node + def newline!(lines) # :nodoc: + predicate.newline!(lines) + end + end + + class RescueModifierNode < Node + def newline!(lines) # :nodoc: + expression.newline!(lines) + end + end + + class InterpolatedMatchLastLineNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end + + class InterpolatedRegularExpressionNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end + + class InterpolatedStringNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end - private_constant :Newlines + class InterpolatedSymbolNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first + end + end - # Walk the tree and mark nodes that are on a new line. - def mark_newlines! - value.accept(Newlines.new(Array.new(1 + source.offsets.size, false))) # steep:ignore + class InterpolatedXStringNode < Node + def newline!(lines) # :nodoc: + first = parts.first + first.newline!(lines) if first end end end diff --git a/lib/prism/pattern.rb b/lib/prism/pattern.rb index 91b23afe3e..03fec26789 100644 --- a/lib/prism/pattern.rb +++ b/lib/prism/pattern.rb @@ -149,7 +149,10 @@ module Prism parent = node.parent if parent.is_a?(ConstantReadNode) && parent.slice == "Prism" - compile_constant_name(node, node.name) + name = node.name + raise CompilationError, node.inspect if name.nil? + + compile_constant_name(node, name) else compile_error(node) end diff --git a/lib/prism/polyfill/byteindex.rb b/lib/prism/polyfill/byteindex.rb new file mode 100644 index 0000000000..98c6089f14 --- /dev/null +++ b/lib/prism/polyfill/byteindex.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +# Polyfill for String#byteindex, which didn't exist until Ruby 3.2. +if !("".respond_to?(:byteindex)) + String.include( + Module.new { + def byteindex(needle, offset = 0) + charindex = index(needle, offset) + slice(0...charindex).bytesize if charindex + end + } + ) +end diff --git a/lib/prism/polyfill/string.rb b/lib/prism/polyfill/unpack1.rb index 3fa9b5a0c5..3fa9b5a0c5 100644 --- a/lib/prism/polyfill/string.rb +++ b/lib/prism/polyfill/unpack1.rb diff --git a/lib/prism/prism.gemspec b/lib/prism/prism.gemspec index 18d0c00a41..b39ba706dc 100644 --- a/lib/prism/prism.gemspec +++ b/lib/prism/prism.gemspec @@ -2,7 +2,7 @@ Gem::Specification.new do |spec| spec.name = "prism" - spec.version = "0.27.0" + spec.version = "0.30.0" spec.authors = ["Shopify"] spec.email = ["ruby@shopify.com"] @@ -65,12 +65,10 @@ Gem::Specification.new do |spec| "include/prism/util/pm_newline_list.h", "include/prism/util/pm_strncasecmp.h", "include/prism/util/pm_string.h", - "include/prism/util/pm_string_list.h", "include/prism/util/pm_strpbrk.h", "include/prism/version.h", "lib/prism.rb", "lib/prism/compiler.rb", - "lib/prism/debug.rb", "lib/prism/desugar_compiler.rb", "lib/prism/dispatcher.rb", "lib/prism/dot_visitor.rb", @@ -86,7 +84,8 @@ Gem::Specification.new do |spec| "lib/prism/parse_result/comments.rb", "lib/prism/parse_result/newlines.rb", "lib/prism/pattern.rb", - "lib/prism/polyfill/string.rb", + "lib/prism/polyfill/byteindex.rb", + "lib/prism/polyfill/unpack1.rb", "lib/prism/reflection.rb", "lib/prism/serialize.rb", "lib/prism/translation.rb", @@ -120,6 +119,7 @@ Gem::Specification.new do |spec| "sig/prism/dot_visitor.rbs", "sig/prism/dsl.rbs", "sig/prism/inspect_visitor.rbs", + "sig/prism/lex_compat.rbs", "sig/prism/mutation_compiler.rbs", "sig/prism/node_ext.rbs", "sig/prism/node.rbs", @@ -147,7 +147,6 @@ Gem::Specification.new do |spec| "src/util/pm_list.c", "src/util/pm_memchr.c", "src/util/pm_newline_list.c", - "src/util/pm_string_list.c", "src/util/pm_string.c", "src/util/pm_strncasecmp.c", "src/util/pm_strpbrk.c" diff --git a/lib/prism/translation/parser.rb b/lib/prism/translation/parser.rb index 2d769302c3..3748fc896e 100644 --- a/lib/prism/translation/parser.rb +++ b/lib/prism/translation/parser.rb @@ -1,6 +1,11 @@ # frozen_string_literal: true -require "parser" +begin + require "parser" +rescue LoadError + warn(%q{Error: Unable to load parser. Add `gem "parser"` to your Gemfile.}) + exit(1) +end module Prism module Translation @@ -284,7 +289,7 @@ module Prism def convert_for_prism(version) case version when 33 - "3.3.0" + "3.3.1" when 34 "3.4.0" else diff --git a/lib/prism/translation/parser/compiler.rb b/lib/prism/translation/parser/compiler.rb index c311613388..48f3d4db5c 100644 --- a/lib/prism/translation/parser/compiler.rb +++ b/lib/prism/translation/parser/compiler.rb @@ -90,7 +90,11 @@ module Prism end if node.constant - builder.const_pattern(visit(node.constant), token(node.opening_loc), builder.array_pattern(nil, visited, nil), token(node.closing_loc)) + if visited.empty? + builder.const_pattern(visit(node.constant), token(node.opening_loc), builder.array_pattern(token(node.opening_loc), visited, token(node.closing_loc)), token(node.closing_loc)) + else + builder.const_pattern(visit(node.constant), token(node.opening_loc), builder.array_pattern(nil, visited, nil), token(node.closing_loc)) + end else builder.array_pattern(token(node.opening_loc), visited, token(node.closing_loc)) end @@ -105,38 +109,46 @@ module Prism # { a: 1 } # ^^^^ def visit_assoc_node(node) + key = node.key + if in_pattern if node.value.is_a?(ImplicitNode) - if node.key.is_a?(SymbolNode) - builder.match_hash_var([node.key.unescaped, srange(node.key.location)]) + if key.is_a?(SymbolNode) + if key.opening.nil? + builder.match_hash_var([key.unescaped, srange(key.location)]) + else + builder.match_hash_var_from_str(token(key.opening_loc), [builder.string_internal([key.unescaped, srange(key.value_loc)])], token(key.closing_loc)) + end else - builder.match_hash_var_from_str(token(node.key.opening_loc), visit_all(node.key.parts), token(node.key.closing_loc)) + builder.match_hash_var_from_str(token(key.opening_loc), visit_all(key.parts), token(key.closing_loc)) end + elsif key.opening.nil? + builder.pair_keyword([key.unescaped, srange(key.location)], visit(node.value)) else - builder.pair_keyword([node.key.unescaped, srange(node.key.location)], visit(node.value)) + builder.pair_quoted(token(key.opening_loc), [builder.string_internal([key.unescaped, srange(key.value_loc)])], token(key.closing_loc), visit(node.value)) end elsif node.value.is_a?(ImplicitNode) if (value = node.value.value).is_a?(LocalVariableReadNode) builder.pair_keyword( - [node.key.unescaped, srange(node.key)], - builder.ident([value.name, srange(node.key.value_loc)]).updated(:lvar) + [key.unescaped, srange(key)], + builder.ident([value.name, srange(key.value_loc)]).updated(:lvar) ) else - builder.pair_label([node.key.unescaped, srange(node.key.location)]) + builder.pair_label([key.unescaped, srange(key.location)]) end elsif node.operator_loc - builder.pair(visit(node.key), token(node.operator_loc), visit(node.value)) - elsif node.key.is_a?(SymbolNode) && node.key.opening_loc.nil? - builder.pair_keyword([node.key.unescaped, srange(node.key.location)], visit(node.value)) + builder.pair(visit(key), token(node.operator_loc), visit(node.value)) + elsif key.is_a?(SymbolNode) && key.opening_loc.nil? + builder.pair_keyword([key.unescaped, srange(key.location)], visit(node.value)) else parts = - if node.key.is_a?(SymbolNode) - [builder.string_internal([node.key.unescaped, srange(node.key.value_loc)])] + if key.is_a?(SymbolNode) + [builder.string_internal([key.unescaped, srange(key.value_loc)])] else - visit_all(node.key.parts) + visit_all(key.parts) end - builder.pair_quoted(token(node.key.opening_loc), parts, token(node.key.closing_loc), visit(node.value)) + builder.pair_quoted(token(key.opening_loc), parts, token(key.closing_loc), visit(node.value)) end end @@ -146,7 +158,9 @@ module Prism # { **foo } # ^^^^^ def visit_assoc_splat_node(node) - if node.value.nil? && forwarding.include?(:**) + if in_pattern + builder.match_rest(token(node.operator_loc), token(node.value&.location)) + elsif node.value.nil? && forwarding.include?(:**) builder.forwarded_kwrestarg(token(node.operator_loc)) else builder.kwsplat(token(node.operator_loc), visit(node.value)) @@ -328,18 +342,48 @@ module Prism [], nil ), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # foo.bar &&= baz # ^^^^^^^^^^^^^^^ - alias visit_call_and_write_node visit_call_operator_write_node + def visit_call_and_write_node(node) + call_operator_loc = node.call_operator_loc + + builder.op_assign( + builder.call_method( + visit(node.receiver), + call_operator_loc.nil? ? nil : [{ "." => :dot, "&." => :anddot, "::" => "::" }.fetch(call_operator_loc.slice), srange(call_operator_loc)], + node.message_loc ? [node.read_name, srange(node.message_loc)] : nil, + nil, + [], + nil + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo.bar ||= baz # ^^^^^^^^^^^^^^^ - alias visit_call_or_write_node visit_call_operator_write_node + def visit_call_or_write_node(node) + call_operator_loc = node.call_operator_loc + + builder.op_assign( + builder.call_method( + visit(node.receiver), + call_operator_loc.nil? ? nil : [{ "." => :dot, "&." => :anddot, "::" => "::" }.fetch(call_operator_loc.slice), srange(call_operator_loc)], + node.message_loc ? [node.read_name, srange(node.message_loc)] : nil, + nil, + [], + nil + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo.bar, = 1 # ^^^^^^^ @@ -419,18 +463,30 @@ module Prism def visit_class_variable_operator_write_node(node) builder.op_assign( builder.assignable(builder.cvar(token(node.name_loc))), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # @@foo &&= bar # ^^^^^^^^^^^^^ - alias visit_class_variable_and_write_node visit_class_variable_operator_write_node + def visit_class_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.cvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # @@foo ||= bar # ^^^^^^^^^^^^^ - alias visit_class_variable_or_write_node visit_class_variable_operator_write_node + def visit_class_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.cvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # @@foo, = bar # ^^^^^ @@ -458,18 +514,30 @@ module Prism def visit_constant_operator_write_node(node) builder.op_assign( builder.assignable(builder.const([node.name, srange(node.name_loc)])), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # Foo &&= bar # ^^^^^^^^^^^^ - alias visit_constant_and_write_node visit_constant_operator_write_node + def visit_constant_and_write_node(node) + builder.op_assign( + builder.assignable(builder.const([node.name, srange(node.name_loc)])), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # Foo ||= bar # ^^^^^^^^^^^^ - alias visit_constant_or_write_node visit_constant_operator_write_node + def visit_constant_or_write_node(node) + builder.op_assign( + builder.assignable(builder.const([node.name, srange(node.name_loc)])), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # Foo, = bar # ^^^ @@ -512,18 +580,30 @@ module Prism def visit_constant_path_operator_write_node(node) builder.op_assign( builder.assignable(visit(node.target)), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # Foo::Bar &&= baz # ^^^^^^^^^^^^^^^^ - alias visit_constant_path_and_write_node visit_constant_path_operator_write_node + def visit_constant_path_and_write_node(node) + builder.op_assign( + builder.assignable(visit(node.target)), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # Foo::Bar ||= baz # ^^^^^^^^^^^^^^^^ - alias visit_constant_path_or_write_node visit_constant_path_operator_write_node + def visit_constant_path_or_write_node(node) + builder.op_assign( + builder.assignable(visit(node.target)), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # Foo::Bar, = baz # ^^^^^^^^ @@ -711,18 +791,30 @@ module Prism def visit_global_variable_operator_write_node(node) builder.op_assign( builder.assignable(builder.gvar(token(node.name_loc))), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # $foo &&= bar # ^^^^^^^^^^^^ - alias visit_global_variable_and_write_node visit_global_variable_operator_write_node + def visit_global_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.gvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # $foo ||= bar # ^^^^^^^^^^^^ - alias visit_global_variable_or_write_node visit_global_variable_operator_write_node + def visit_global_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.gvar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # $foo, = bar # ^^^^ @@ -803,7 +895,7 @@ module Prism # 1i # ^^ def visit_imaginary_node(node) - visit_numeric(node, builder.complex([imaginary_value(node), srange(node.location)])) + visit_numeric(node, builder.complex([Complex(0, node.numeric.value), srange(node.location)])) end # { foo: } @@ -857,18 +949,46 @@ module Prism visit_all(arguments), token(node.closing_loc) ), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # foo[bar] &&= baz # ^^^^^^^^^^^^^^^^ - alias visit_index_and_write_node visit_index_operator_write_node + def visit_index_and_write_node(node) + arguments = node.arguments&.arguments || [] + arguments << node.block if node.block + + builder.op_assign( + builder.index( + visit(node.receiver), + token(node.opening_loc), + visit_all(arguments), + token(node.closing_loc) + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo[bar] ||= baz # ^^^^^^^^^^^^^^^^ - alias visit_index_or_write_node visit_index_operator_write_node + def visit_index_or_write_node(node) + arguments = node.arguments&.arguments || [] + arguments << node.block if node.block + + builder.op_assign( + builder.index( + visit(node.receiver), + token(node.opening_loc), + visit_all(arguments), + token(node.closing_loc) + ), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo[bar], = 1 # ^^^^^^^^ @@ -902,18 +1022,30 @@ module Prism def visit_instance_variable_operator_write_node(node) builder.op_assign( builder.assignable(builder.ivar(token(node.name_loc))), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # @foo &&= bar # ^^^^^^^^^^^^ - alias visit_instance_variable_and_write_node visit_instance_variable_operator_write_node + def visit_instance_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.ivar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # @foo ||= bar # ^^^^^^^^^^^^ - alias visit_instance_variable_or_write_node visit_instance_variable_operator_write_node + def visit_instance_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.ivar(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # @foo, = bar # ^^^^ @@ -946,36 +1078,7 @@ module Prism # ^^^^^^^^^^^^ def visit_interpolated_string_node(node) if node.heredoc? - children, closing = visit_heredoc(node) - opening = token(node.opening_loc) - - start_offset = node.opening_loc.end_offset + 1 - end_offset = node.parts.first.location.start_offset - - # In the below case, the offsets should be the same: - # - # <<~HEREDOC - # a #{b} - # HEREDOC - # - # But in this case, the end_offset would be greater than the start_offset: - # - # <<~HEREDOC - # #{b} - # HEREDOC - # - # So we need to make sure the result node's heredoc range is correct, without updating the children - result = if start_offset < end_offset - # We need to add a padding string to ensure that the heredoc has correct range for its body - padding_string_node = builder.string_internal(["", srange_offsets(start_offset, end_offset)]) - node_with_correct_location = builder.string_compose(opening, [padding_string_node, *children], closing) - # But the padding string should not be included in the final AST, so we need to update the result's children - node_with_correct_location.updated(:dstr, children) - else - builder.string_compose(opening, children, closing) - end - - return result + return visit_heredoc(node) { |children, closing| builder.string_compose(token(node.opening_loc), children, closing) } end parts = if node.parts.one? { |part| part.type == :string_node } @@ -1019,8 +1122,7 @@ module Prism # ^^^^^^^^^^^^ def visit_interpolated_x_string_node(node) if node.heredoc? - children, closing = visit_heredoc(node) - builder.xstring_compose(token(node.opening_loc), children, closing) + visit_heredoc(node) { |children, closing| builder.xstring_compose(token(node.opening_loc), children, closing) } else builder.xstring_compose( token(node.opening_loc), @@ -1031,6 +1133,12 @@ module Prism end # -> { it } + # ^^ + def visit_it_local_variable_read_node(node) + builder.ident([:it, srange(node.location)]).updated(:lvar) + end + + # -> { it } # ^^^^^^^^^ def visit_it_parameters_node(node) builder.args(nil, [], nil, false) @@ -1083,14 +1191,7 @@ module Prism # foo # ^^^ def visit_local_variable_read_node(node) - name = node.name - - # This is just a guess. parser doesn't have support for the implicit - # `it` variable yet, so we'll probably have to visit this once it - # does. - name = :it if name == :"0it" - - builder.ident([name, srange(node.location)]).updated(:lvar) + builder.ident([node.name, srange(node.location)]).updated(:lvar) end # foo = 1 @@ -1108,18 +1209,30 @@ module Prism def visit_local_variable_operator_write_node(node) builder.op_assign( builder.assignable(builder.ident(token(node.name_loc))), - [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + [node.binary_operator_loc.slice.chomp("="), srange(node.binary_operator_loc)], visit(node.value) ) end # foo &&= bar # ^^^^^^^^^^^ - alias visit_local_variable_and_write_node visit_local_variable_operator_write_node + def visit_local_variable_and_write_node(node) + builder.op_assign( + builder.assignable(builder.ident(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo ||= bar # ^^^^^^^^^^^ - alias visit_local_variable_or_write_node visit_local_variable_operator_write_node + def visit_local_variable_or_write_node(node) + builder.op_assign( + builder.assignable(builder.ident(token(node.name_loc))), + [node.operator_loc.slice.chomp("="), srange(node.operator_loc)], + visit(node.value) + ) + end # foo, = bar # ^^^ @@ -1182,13 +1295,9 @@ module Prism # foo, bar = baz # ^^^^^^^^ def visit_multi_target_node(node) - elements = [*node.lefts] - elements << node.rest if !node.rest.nil? && !node.rest.is_a?(ImplicitRestNode) - elements.concat(node.rights) - builder.multi_lhs( token(node.lparen_loc), - visit_all(elements), + visit_all(multi_target_elements(node)), token(node.rparen_loc) ) end @@ -1196,9 +1305,11 @@ module Prism # foo, bar = baz # ^^^^^^^^^^^^^^ def visit_multi_write_node(node) - elements = [*node.lefts] - elements << node.rest if !node.rest.nil? && !node.rest.is_a?(ImplicitRestNode) - elements.concat(node.rights) + elements = multi_target_elements(node) + + if elements.length == 1 && elements.first.is_a?(MultiTargetNode) + elements = multi_target_elements(elements.first) + end builder.multi_assign( builder.multi_lhs( @@ -1279,12 +1390,12 @@ module Prism if node.requireds.any? node.requireds.each do |required| - if required.is_a?(RequiredParameterNode) - params << visit(required) - else - compiler = copy_compiler(in_destructure: true) - params << required.accept(compiler) - end + params << + if required.is_a?(RequiredParameterNode) + visit(required) + else + required.accept(copy_compiler(in_destructure: true)) + end end end @@ -1293,12 +1404,12 @@ module Prism if node.posts.any? node.posts.each do |post| - if post.is_a?(RequiredParameterNode) - params << visit(post) - else - compiler = copy_compiler(in_destructure: true) - params << post.accept(compiler) - end + params << + if post.is_a?(RequiredParameterNode) + visit(post) + else + post.accept(copy_compiler(in_destructure: true)) + end end end @@ -1384,7 +1495,7 @@ module Prism # 1r # ^^ def visit_rational_node(node) - visit_numeric(node, builder.rational([rational_value(node), srange(node.location)])) + visit_numeric(node, builder.rational([node.value, srange(node.location)])) end # redo @@ -1396,9 +1507,20 @@ module Prism # /foo/ # ^^^^^ def visit_regular_expression_node(node) + content = node.content + parts = + if content.include?("\n") + offset = node.content_loc.start_offset + content.lines.map do |line| + builder.string_internal([line, srange_offsets(offset, offset += line.bytesize)]) + end + else + [builder.string_internal(token(node.content_loc))] + end + builder.regexp_compose( token(node.opening_loc), - [builder.string_internal(token(node.content_loc))], + parts, [node.closing[0], srange_offsets(node.closing_loc.start_offset, node.closing_loc.start_offset + 1)], builder.regexp_options([node.closing[1..], srange_offsets(node.closing_loc.start_offset + 1, node.closing_loc.end_offset)]) ) @@ -1544,10 +1666,11 @@ module Prism # ^^^^^ def visit_string_node(node) if node.heredoc? - children, closing = visit_heredoc(node.to_interpolated) - builder.string_compose(token(node.opening_loc), children, closing) + visit_heredoc(node.to_interpolated) { |children, closing| builder.string_compose(token(node.opening_loc), children, closing) } elsif node.opening == "?" builder.character([node.unescaped, srange(node.location)]) + elsif node.opening&.start_with?("%") && node.unescaped.empty? + builder.string_compose(token(node.opening_loc), [], token(node.closing_loc)) else content_lines = node.content.lines unescaped_lines = node.unescaped.lines @@ -1747,8 +1870,7 @@ module Prism # ^^^^^ def visit_x_string_node(node) if node.heredoc? - children, closing = visit_heredoc(node.to_interpolated) - builder.xstring_compose(token(node.opening_loc), children, closing) + visit_heredoc(node.to_interpolated) { |children, closing| builder.xstring_compose(token(node.opening_loc), children, closing) } else parts = if node.unescaped.lines.one? [builder.string_internal([node.unescaped, srange(node.content_loc)])] @@ -1810,10 +1932,12 @@ module Prism forwarding end - # Because we have mutated the AST to allow for newlines in the middle of - # a rational, we need to manually handle the value here. - def imaginary_value(node) - Complex(0, node.numeric.is_a?(RationalNode) ? rational_value(node.numeric) : node.numeric.value) + # Returns the set of targets for a MultiTargetNode or a MultiWriteNode. + def multi_target_elements(node) + elements = [*node.lefts] + elements << node.rest if !node.rest.nil? && !node.rest.is_a?(ImplicitRestNode) + elements.concat(node.rights) + elements end # Negate the value of a numeric node. This is a special case where you @@ -1825,7 +1949,9 @@ module Prism case receiver.type when :integer_node, :float_node receiver.copy(value: -receiver.value, location: message_loc.join(receiver.location)) - when :rational_node, :imaginary_node + when :rational_node + receiver.copy(numerator: -receiver.numerator, location: message_loc.join(receiver.location)) + when :imaginary_node receiver.copy(numeric: numeric_negate(message_loc, receiver.numeric), location: message_loc.join(receiver.location)) end end @@ -1844,16 +1970,6 @@ module Prism parameters.block.nil? end - # Because we have mutated the AST to allow for newlines in the middle of - # a rational, we need to manually handle the value here. - def rational_value(node) - if node.numeric.is_a?(IntegerNode) - Rational(node.numeric.value) - else - Rational(node.slice.gsub(/\s/, "").chomp("r")) - end - end - # Locations in the parser gem AST are generated using this class. We # store a reference to its constant to make it slightly faster to look # up. @@ -1876,7 +1992,7 @@ module Prism # Note that end_offset is allowed to be nil, in which case this will # search until the end of the string. def srange_find(start_offset, end_offset, tokens) - if (match = source_buffer.source.byteslice(start_offset...end_offset).match(/(\s*)(#{tokens.join("|")})/)) + if (match = source_buffer.source.byteslice(start_offset...end_offset).match(/\A(\s*)(#{tokens.join("|")})/)) _, whitespace, token = *match token_offset = start_offset + whitespace.bytesize @@ -1907,7 +2023,8 @@ module Prism token(parameters.opening_loc), if procarg0?(parameters.parameters) parameter = parameters.parameters.requireds.first - [builder.procarg0(visit(parameter))].concat(visit_all(parameters.locals)) + visited = parameter.is_a?(RequiredParameterNode) ? visit(parameter) : parameter.accept(copy_compiler(in_destructure: true)) + [builder.procarg0(visited)].concat(visit_all(parameters.locals)) else visit(parameters) end, @@ -1923,29 +2040,55 @@ module Prism end end + # The parser gem automatically converts \r\n to \n, meaning our offsets + # need to be adjusted to always subtract 1 from the length. + def chomped_bytesize(line) + chomped = line.chomp + chomped.bytesize + (chomped == line ? 0 : 1) + end + # Visit a heredoc that can be either a string or an xstring. def visit_heredoc(node) children = Array.new + indented = false + + # If this is a dedenting heredoc, then we need to insert the opening + # content into the children as well. + if node.opening.start_with?("<<~") && node.parts.length > 0 && !node.parts.first.is_a?(StringNode) + location = node.parts.first.location + location = location.copy(start_offset: location.start_offset - location.start_line_slice.bytesize) + children << builder.string_internal(token(location)) + indented = true + end + node.parts.each do |part| pushing = if part.is_a?(StringNode) && part.unescaped.include?("\n") - unescaped = part.unescaped.lines(chomp: true) - escaped = part.content.lines(chomp: true) + unescaped = part.unescaped.lines + escaped = part.content.lines - escaped_lengths = - if node.opening.end_with?("'") - escaped.map { |line| line.bytesize + 1 } - else - escaped.chunk_while { |before, after| before.match?(/(?<!\\)\\$/) }.map { |line| line.join.bytesize + line.length } + escaped_lengths = [] + normalized_lengths = [] + + if node.opening.end_with?("'") + escaped.each do |line| + escaped_lengths << line.bytesize + normalized_lengths << chomped_bytesize(line) end + else + escaped + .chunk_while { |before, after| before.match?(/(?<!\\)\\\r?\n$/) } + .each do |lines| + escaped_lengths << lines.sum(&:bytesize) + normalized_lengths << lines.sum { |line| chomped_bytesize(line) } + end + end start_offset = part.location.start_offset - end_offset = nil - unescaped.zip(escaped_lengths).map do |unescaped_line, escaped_length| - end_offset = start_offset + (escaped_length || 0) - inner_part = builder.string_internal(["#{unescaped_line}\n", srange_offsets(start_offset, end_offset)]) - start_offset = end_offset + unescaped.map.with_index do |unescaped_line, index| + inner_part = builder.string_internal([unescaped_line, srange_offsets(start_offset, start_offset + normalized_lengths.fetch(index, 0))]) + start_offset += escaped_lengths.fetch(index, 0) inner_part end else @@ -1956,7 +2099,12 @@ module Prism if child.type == :str && child.children.last == "" # nothing elsif child.type == :str && children.last && children.last.type == :str && !children.last.children.first.end_with?("\n") - children.last.children.first << child.children.first + appendee = children[-1] + + location = appendee.loc + location = location.with_expression(location.expression.join(child.loc.expression)) + + children[-1] = appendee.updated(:str, [appendee.children.first << child.children.first], location: location) else children << child end @@ -1965,8 +2113,10 @@ module Prism closing = node.closing closing_t = [closing.chomp, srange_offsets(node.closing_loc.start_offset, node.closing_loc.end_offset - (closing[/\s+$/]&.length || 0))] + composed = yield children, closing_t - [children, closing_t] + composed = composed.updated(nil, children[1..-1]) if indented + composed end # Visit a numeric node and account for the optional sign. diff --git a/lib/prism/translation/ripper.rb b/lib/prism/translation/ripper.rb index d226bb58df..79ba0e7ab3 100644 --- a/lib/prism/translation/ripper.rb +++ b/lib/prism/translation/ripper.rb @@ -1181,8 +1181,8 @@ module Prism bounds(node.location) target = on_field(receiver, call_operator, message) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1339,8 +1339,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_cvar(node.name.to_s)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1409,8 +1409,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_const(node.name.to_s)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1510,8 +1510,8 @@ module Prism target = visit_constant_path_write_node_target(node.target) value = visit(node.value) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1802,8 +1802,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_gvar(node.name.to_s)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -1983,8 +1983,8 @@ module Prism bounds(node.location) target = on_aref_field(receiver, arguments) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -2059,8 +2059,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_ivar(node.name.to_s)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) @@ -2218,6 +2218,13 @@ module Prism end # -> { it } + # ^^ + def visit_it_local_variable_read_node(node) + bounds(node.location) + on_vcall(on_ident(node.slice)) + end + + # -> { it } # ^^^^^^^^^ def visit_it_parameters_node(node) end @@ -2312,12 +2319,7 @@ module Prism # ^^^ def visit_local_variable_read_node(node) bounds(node.location) - - if node.name == :"0it" - on_vcall(on_ident(node.slice)) - else - on_var_ref(on_ident(node.slice)) - end + on_var_ref(on_ident(node.slice)) end # foo = 1 @@ -2337,8 +2339,8 @@ module Prism bounds(node.name_loc) target = on_var_field(on_ident(node.name_loc.slice)) - bounds(node.operator_loc) - operator = on_op("#{node.operator}=") + bounds(node.binary_operator_loc) + operator = on_op("#{node.binary_operator}=") value = visit_write_value(node.value) bounds(node.location) diff --git a/lib/prism/translation/ruby_parser.rb b/lib/prism/translation/ruby_parser.rb index c6f741c8e5..38690c54b3 100644 --- a/lib/prism/translation/ruby_parser.rb +++ b/lib/prism/translation/ruby_parser.rb @@ -1,6 +1,11 @@ # frozen_string_literal: true -require "ruby_parser" +begin + require "ruby_parser" +rescue LoadError + warn(%q{Error: Unable to load ruby_parser. Add `gem "ruby_parser"` to your Gemfile.}) + exit(1) +end module Prism module Translation @@ -271,9 +276,9 @@ module Prism # ^^^^^^^^^^^^^^^ def visit_call_operator_write_node(node) if op_asgn?(node) - s(node, op_asgn_type(node, :op_asgn), visit(node.receiver), visit_write_value(node.value), node.read_name, node.operator) + s(node, op_asgn_type(node, :op_asgn), visit(node.receiver), visit_write_value(node.value), node.read_name, node.binary_operator) else - s(node, op_asgn_type(node, :op_asgn2), visit(node.receiver), node.write_name, node.operator, visit_write_value(node.value)) + s(node, op_asgn_type(node, :op_asgn2), visit(node.receiver), node.write_name, node.binary_operator, visit_write_value(node.value)) end end @@ -372,7 +377,7 @@ module Prism # @@foo += bar # ^^^^^^^^^^^^ def visit_class_variable_operator_write_node(node) - s(node, class_variable_write_type, node.name, s(node, :call, s(node, :cvar, node.name), node.operator, visit_write_value(node.value))) + s(node, class_variable_write_type, node.name, s(node, :call, s(node, :cvar, node.name), node.binary_operator, visit_write_value(node.value))) end # @@foo &&= bar @@ -417,7 +422,7 @@ module Prism # Foo += bar # ^^^^^^^^^^^ def visit_constant_operator_write_node(node) - s(node, :cdecl, node.name, s(node, :call, s(node, :const, node.name), node.operator, visit_write_value(node.value))) + s(node, :cdecl, node.name, s(node, :call, s(node, :const, node.name), node.binary_operator, visit_write_value(node.value))) end # Foo &&= bar @@ -460,7 +465,7 @@ module Prism # Foo::Bar += baz # ^^^^^^^^^^^^^^^ def visit_constant_path_operator_write_node(node) - s(node, :op_asgn, visit(node.target), node.operator, visit_write_value(node.value)) + s(node, :op_asgn, visit(node.target), node.binary_operator, visit_write_value(node.value)) end # Foo::Bar &&= baz @@ -480,9 +485,9 @@ module Prism def visit_constant_path_target_node(node) inner = if node.parent.nil? - s(node, :colon3, node.child.name) + s(node, :colon3, node.name) else - s(node, :colon2, visit(node.parent), node.child.name) + s(node, :colon2, visit(node.parent), node.name) end s(node, :const, inner) @@ -627,7 +632,7 @@ module Prism # $foo += bar # ^^^^^^^^^^^ def visit_global_variable_operator_write_node(node) - s(node, :gasgn, node.name, s(node, :call, s(node, :gvar, node.name), node.operator, visit(node.value))) + s(node, :gasgn, node.name, s(node, :call, s(node, :gvar, node.name), node.binary_operator, visit(node.value))) end # $foo &&= bar @@ -719,7 +724,7 @@ module Prism arglist << visit(node.block) if !node.block.nil? end - s(node, :op_asgn1, visit(node.receiver), arglist, node.operator, visit_write_value(node.value)) + s(node, :op_asgn1, visit(node.receiver), arglist, node.binary_operator, visit_write_value(node.value)) end # foo[bar] &&= baz @@ -775,7 +780,7 @@ module Prism # @foo += bar # ^^^^^^^^^^^ def visit_instance_variable_operator_write_node(node) - s(node, :iasgn, node.name, s(node, :call, s(node, :ivar, node.name), node.operator, visit_write_value(node.value))) + s(node, :iasgn, node.name, s(node, :call, s(node, :ivar, node.name), node.binary_operator, visit_write_value(node.value))) end # @foo &&= bar @@ -870,6 +875,8 @@ module Prism else visited << result end + elsif result[0] == :dstr + visited.concat(result[1..-1]) else visited << result end @@ -900,12 +907,23 @@ module Prism results << result state = :interpolated_content end - else - results << result + when :interpolated_content + if result.is_a?(Array) && result[0] == :str && results[-1][0] == :str && (results[-1].line_max == result.line) + results[-1][1] << result[1] + results[-1].line_max = result.line_max + else + results << result + end end end end + # -> { it } + # ^^ + def visit_it_local_variable_read_node(node) + s(node, :call, nil, :it) + end + # foo(bar: baz) # ^^^^^^^^ def visit_keyword_hash_node(node) @@ -960,7 +978,7 @@ module Prism # foo += bar # ^^^^^^^^^^ def visit_local_variable_operator_write_node(node) - s(node, :lasgn, node.name, s(node, :call, s(node, :lvar, node.name), node.operator, visit_write_value(node.value))) + s(node, :lasgn, node.name, s(node, :call, s(node, :lvar, node.name), node.binary_operator, visit_write_value(node.value))) end # foo &&= bar diff --git a/lib/rdoc/markdown.rb b/lib/rdoc/markdown.rb index 5c72a5f224..1416059763 100644 --- a/lib/rdoc/markdown.rb +++ b/lib/rdoc/markdown.rb @@ -6,7 +6,7 @@ # RDoc::Markdown as described by the [markdown syntax][syntax]. # # To choose Markdown as your only default format see -# RDoc::Options@Saved+Options for instructions on setting up a `.doc_options` +# RDoc::Options@Saved+Options for instructions on setting up a `.rdoc_options` # file to store your project default. # # ## Usage diff --git a/lib/rdoc/ri/driver.rb b/lib/rdoc/ri/driver.rb index 64783dc163..c6fddbac67 100644 --- a/lib/rdoc/ri/driver.rb +++ b/lib/rdoc/ri/driver.rb @@ -110,10 +110,6 @@ class RDoc::RI::Driver options = default_options opts = OptionParser.new do |opt| - opt.accept File do |file,| - File.readable?(file) and not File.directory?(file) and file - end - opt.program_name = File.basename $0 opt.version = RDoc::VERSION opt.release = nil @@ -345,9 +341,17 @@ or the PAGER environment variable. opt.separator nil - opt.on("--dump=CACHE", File, + opt.on("--dump=CACHE", "Dump data from an ri cache or data file.") do |value| - options[:dump_path] = value + unless File.readable?(value) + abort "#{value.inspect} is not readable" + end + + if File.directory?(value) + abort "#{value.inspect} is a directory" + end + + options[:dump_path] = File.new(value) end end diff --git a/lib/rdoc/version.rb b/lib/rdoc/version.rb index 87842d9847..427d4ae232 100644 --- a/lib/rdoc/version.rb +++ b/lib/rdoc/version.rb @@ -5,6 +5,6 @@ module RDoc ## # RDoc version you are using - VERSION = '6.6.3.1' + VERSION = '6.7.0' end diff --git a/lib/reline.rb b/lib/reline.rb index d5bf2e363b..720df286a1 100644 --- a/lib/reline.rb +++ b/lib/reline.rb @@ -7,6 +7,7 @@ require 'reline/key_stroke' require 'reline/line_editor' require 'reline/history' require 'reline/terminfo' +require 'reline/io' require 'reline/face' require 'rbconfig' @@ -18,20 +19,10 @@ module Reline class ConfigEncodingConversionError < StandardError; end Key = Struct.new(:char, :combined_char, :with_meta) do - def match?(other) - case other - when Reline::Key - (other.char.nil? or char.nil? or char == other.char) and - (other.combined_char.nil? or combined_char.nil? or combined_char == other.combined_char) and - (other.with_meta.nil? or with_meta.nil? or with_meta == other.with_meta) - when Integer, Symbol - (combined_char and combined_char == other) or - (combined_char.nil? and char and char == other) - else - false - end + # For dialog_proc `key.match?(dialog.name)` + def match?(sym) + combined_char.is_a?(Symbol) && combined_char == sym end - alias_method :==, :match? end CursorPos = Struct.new(:x, :y) DialogRenderInfo = Struct.new( @@ -263,7 +254,6 @@ module Reline raise ArgumentError.new('#readmultiline needs block to confirm multiline termination') end - Reline.update_iogate io_gate.with_raw_input do inner_readline(prompt, add_hist, true, &confirm_multiline_termination) end @@ -286,7 +276,6 @@ module Reline def readline(prompt = '', add_hist = false) @mutex.synchronize do - Reline.update_iogate io_gate.with_raw_input do inner_readline(prompt, add_hist, false) end @@ -312,6 +301,10 @@ module Reline $stderr.sync = true $stderr.puts "Reline is used by #{Process.pid}" end + unless config.test_mode or config.loaded? + config.read + io_gate.set_default_key_bindings(config) + end otio = io_gate.prep may_req_ambiguous_char_width @@ -332,17 +325,12 @@ module Reline line_editor.auto_indent_proc = auto_indent_proc line_editor.dig_perfect_match_proc = dig_perfect_match_proc pre_input_hook&.call - unless Reline::IOGate == Reline::GeneralIO + unless Reline::IOGate.dumb? @dialog_proc_list.each_pair do |name_sym, d| line_editor.add_dialog_proc(name_sym, d.dialog_proc, d.context) end end - unless config.test_mode or config.loaded? - config.read - io_gate.set_default_key_bindings(config) - end - line_editor.print_nomultiline_prompt(prompt) line_editor.update_dialogs line_editor.rerender @@ -352,7 +340,15 @@ module Reline loop do read_io(config.keyseq_timeout) { |inputs| line_editor.set_pasting_state(io_gate.in_pasting?) - inputs.each { |key| line_editor.update(key) } + inputs.each do |key| + if key.char == :bracketed_paste_start + text = io_gate.read_bracketed_paste + line_editor.insert_pasted_text(text) + line_editor.scroll_into_view + else + line_editor.update(key) + end + end } if line_editor.finished? line_editor.render_finished @@ -371,92 +367,39 @@ module Reline end end - # GNU Readline waits for "keyseq-timeout" milliseconds to see if the ESC - # is followed by a character, and times out and treats it as a standalone - # ESC if the second character does not arrive. If the second character - # comes before timed out, it is treated as a modifier key with the - # meta-property of meta-key, so that it can be distinguished from - # multibyte characters with the 8th bit turned on. - # - # GNU Readline will wait for the 2nd character with "keyseq-timeout" - # milli-seconds but wait forever after 3rd characters. + # GNU Readline watis for "keyseq-timeout" milliseconds when the input is + # ambiguous whether it is matching or matched. + # If the next character does not arrive within the specified timeout, input + # is considered as matched. + # `ESC` is ambiguous because it can be a standalone ESC (matched) or part of + # `ESC char` or part of CSI sequence (matching). private def read_io(keyseq_timeout, &block) buffer = [] + status = KeyStroke::MATCHING loop do - c = io_gate.getc(Float::INFINITY) - if c == -1 - result = :unmatched - else - buffer << c - result = key_stroke.match_status(buffer) - end - case result - when :matched - expanded = key_stroke.expand(buffer).map{ |expanded_c| - Reline::Key.new(expanded_c, expanded_c, false) - } - block.(expanded) - break - when :matching - if buffer.size == 1 - case read_2nd_character_of_key_sequence(keyseq_timeout, buffer, c, block) - when :break then break - when :next then next - end - end - when :unmatched - if buffer.size == 1 and c == "\e".ord - read_escaped_key(keyseq_timeout, c, block) + timeout = status == KeyStroke::MATCHING_MATCHED ? keyseq_timeout.fdiv(1000) : Float::INFINITY + c = io_gate.getc(timeout) + if c.nil? || c == -1 + if status == KeyStroke::MATCHING_MATCHED + status = KeyStroke::MATCHED + elsif buffer.empty? + # io_gate is closed and reached EOF + block.call([Key.new(nil, nil, false)]) + return else - expanded = buffer.map{ |expanded_c| - Reline::Key.new(expanded_c, expanded_c, false) - } - block.(expanded) + status = KeyStroke::UNMATCHED end - break + else + buffer << c + status = key_stroke.match_status(buffer) end - end - end - private def read_2nd_character_of_key_sequence(keyseq_timeout, buffer, c, block) - succ_c = io_gate.getc(keyseq_timeout.fdiv(1000)) - if succ_c - case key_stroke.match_status(buffer.dup.push(succ_c)) - when :unmatched - if c == "\e".ord - block.([Reline::Key.new(succ_c, succ_c | 0b10000000, true)]) - else - block.([Reline::Key.new(c, c, false), Reline::Key.new(succ_c, succ_c, false)]) - end - return :break - when :matching - io_gate.ungetc(succ_c) - return :next - when :matched - buffer << succ_c - expanded = key_stroke.expand(buffer).map{ |expanded_c| - Reline::Key.new(expanded_c, expanded_c, false) - } - block.(expanded) - return :break + if status == KeyStroke::MATCHED || status == KeyStroke::UNMATCHED + expanded, rest_bytes = key_stroke.expand(buffer) + rest_bytes.reverse_each { |c| io_gate.ungetc(c) } + block.call(expanded) + return end - else - block.([Reline::Key.new(c, c, false)]) - return :break - end - end - - private def read_escaped_key(keyseq_timeout, c, block) - escaped_c = io_gate.getc(keyseq_timeout.fdiv(1000)) - - if escaped_c.nil? - block.([Reline::Key.new(c, c, false)]) - elsif escaped_c >= 128 # maybe, first byte of multi byte - block.([Reline::Key.new(c, c, false), Reline::Key.new(escaped_c, escaped_c, false)]) - elsif escaped_c == "\e".ord # escape twice - block.([Reline::Key.new(c, c, false), Reline::Key.new(c, c, false)]) - else - block.([Reline::Key.new(escaped_c, escaped_c | 0b10000000, true)]) end end @@ -466,7 +409,7 @@ module Reline end private def may_req_ambiguous_char_width - @ambiguous_width = 2 if io_gate == Reline::GeneralIO or !STDOUT.tty? + @ambiguous_width = 2 if io_gate.dumb? || !STDIN.tty? || !STDOUT.tty? return if defined? @ambiguous_width io_gate.move_cursor_column(0) begin @@ -560,37 +503,13 @@ module Reline def self.line_editor core.line_editor end +end - def self.update_iogate - return if core.config.test_mode - # Need to change IOGate when `$stdout.tty?` change from false to true by `$stdout.reopen` - # Example: rails/spring boot the application in non-tty, then run console in tty. - if ENV['TERM'] != 'dumb' && core.io_gate == Reline::GeneralIO && $stdout.tty? - require 'reline/ansi' - remove_const(:IOGate) - const_set(:IOGate, Reline::ANSI) - end - end -end +Reline::IOGate = Reline::IO.decide_io_gate -require 'reline/general_io' -io = Reline::GeneralIO -unless ENV['TERM'] == 'dumb' - case RbConfig::CONFIG['host_os'] - when /mswin|msys|mingw|cygwin|bccwin|wince|emc/ - require 'reline/windows' - tty = (io = Reline::Windows).msys_tty? - else - tty = $stdout.tty? - end -end -Reline::IOGate = if tty - require 'reline/ansi' - Reline::ANSI -else - io -end +# Deprecated +Reline::GeneralIO = Reline::Dumb.new Reline::Face.load_initial_configs diff --git a/lib/reline/config.rb b/lib/reline/config.rb index 234758963f..774b06b6fd 100644 --- a/lib/reline/config.rb +++ b/lib/reline/config.rb @@ -8,31 +8,12 @@ class Reline::Config end VARIABLE_NAMES = %w{ - bind-tty-special-chars - blink-matching-paren - byte-oriented completion-ignore-case convert-meta disable-completion - enable-keypad - expand-tilde - history-preserve-point history-size - horizontal-scroll-mode - input-meta keyseq-timeout - mark-directories - mark-modified-lines - mark-symlinked-directories - match-hidden-files - meta-flag - output-meta - page-completions - prefer-visible-bell - print-completions-horizontally show-all-if-ambiguous - show-all-if-unmodified - visible-stats show-mode-in-prompt vi-cmd-mode-string vi-ins-mode-string @@ -48,18 +29,20 @@ class Reline::Config attr_accessor :autocompletion def initialize - @additional_key_bindings = {} # from inputrc - @additional_key_bindings[:emacs] = {} - @additional_key_bindings[:vi_insert] = {} - @additional_key_bindings[:vi_command] = {} - @oneshot_key_bindings = {} + @additional_key_bindings = { # from inputrc + emacs: Reline::KeyActor::Base.new, + vi_insert: Reline::KeyActor::Base.new, + vi_command: Reline::KeyActor::Base.new + } + @oneshot_key_bindings = Reline::KeyActor::Base.new @editing_mode_label = :emacs @keymap_label = :emacs @keymap_prefix = [] - @key_actors = {} - @key_actors[:emacs] = Reline::KeyActor::Emacs.new - @key_actors[:vi_insert] = Reline::KeyActor::ViInsert.new - @key_actors[:vi_command] = Reline::KeyActor::ViCommand.new + @default_key_bindings = { + emacs: Reline::KeyActor::Base.new(Reline::KeyActor::EMACS_MAPPING), + vi_insert: Reline::KeyActor::Base.new(Reline::KeyActor::VI_INSERT_MAPPING), + vi_command: Reline::KeyActor::Base.new(Reline::KeyActor::VI_COMMAND_MAPPING) + } @vi_cmd_mode_string = '(cmd)' @vi_ins_mode_string = '(ins)' @emacs_mode_string = '@' @@ -70,6 +53,7 @@ class Reline::Config @autocompletion = false @convert_meta = true if seven_bit_encoding?(Reline::IOGate.encoding) @loaded = false + @enable_bracketed_paste = true end def reset @@ -80,7 +64,7 @@ class Reline::Config end def editing_mode - @key_actors[@editing_mode_label] + @default_key_bindings[@editing_mode_label] end def editing_mode=(val) @@ -92,7 +76,7 @@ class Reline::Config end def keymap - @key_actors[@keymap_label] + @default_key_bindings[@keymap_label] end def loaded? @@ -151,14 +135,14 @@ class Reline::Config def key_bindings # The key bindings for each editing mode will be overwritten by the user-defined ones. - kb = @key_actors[@editing_mode_label].default_key_bindings.dup - kb.merge!(@additional_key_bindings[@editing_mode_label]) - kb.merge!(@oneshot_key_bindings) - kb + Reline::KeyActor::Composite.new([@oneshot_key_bindings, @additional_key_bindings[@editing_mode_label], @default_key_bindings[@editing_mode_label]]) end def add_oneshot_key_binding(keystroke, target) - @oneshot_key_bindings[keystroke] = target + # IRB sets invalid keystroke [Reline::Key]. We should ignore it. + return unless keystroke.all? { |c| c.is_a?(Integer) } + + @oneshot_key_bindings.add(keystroke, target) end def reset_oneshot_key_bindings @@ -166,11 +150,11 @@ class Reline::Config end def add_default_key_binding_by_keymap(keymap, keystroke, target) - @key_actors[keymap].default_key_bindings[keystroke] = target + @default_key_bindings[keymap].add(keystroke, target) end def add_default_key_binding(keystroke, target) - @key_actors[@keymap_label].default_key_bindings[keystroke] = target + add_default_key_binding_by_keymap(@keymap_label, keystroke, target) end def read_lines(lines, file = nil) @@ -200,16 +184,17 @@ class Reline::Config next if if_stack.any? { |_no, skip| skip } case line - when /^set +([^ ]+) +([^ ]+)/i - var, value = $1.downcase, $2 - bind_variable(var, value) + when /^set +([^ ]+) +(.+)/i + # value ignores everything after a space, raw_value does not. + var, value, raw_value = $1.downcase, $2.partition(' ').first, $2 + bind_variable(var, value, raw_value) next when /\s*("#{KEYSEQ_PATTERN}+")\s*:\s*(.*)\s*$/o key, func_name = $1, $2 func_name = func_name.split.first keystroke, func = bind_key(key, func_name) next unless keystroke - @additional_key_bindings[@keymap_label][@keymap_prefix + keystroke] = func + @additional_key_bindings[@keymap_label].add(@keymap_prefix + keystroke, func) end end unless if_stack.empty? @@ -252,7 +237,7 @@ class Reline::Config end end - def bind_variable(name, value) + def bind_variable(name, value, raw_value) case name when 'history-size' begin @@ -277,7 +262,7 @@ class Reline::Config when 'completion-query-items' @completion_query_items = value.to_i when 'isearch-terminators' - @isearch_terminators = retrieve_string(value) + @isearch_terminators = retrieve_string(raw_value) when 'editing-mode' case value when 'emacs' @@ -319,11 +304,11 @@ class Reline::Config @show_mode_in_prompt = false end when 'vi-cmd-mode-string' - @vi_cmd_mode_string = retrieve_string(value) + @vi_cmd_mode_string = retrieve_string(raw_value) when 'vi-ins-mode-string' - @vi_ins_mode_string = retrieve_string(value) + @vi_ins_mode_string = retrieve_string(raw_value) when 'emacs-mode-string' - @emacs_mode_string = retrieve_string(value) + @emacs_mode_string = retrieve_string(raw_value) when *VARIABLE_NAMES then variable_name = :"@#{name.tr(?-, ?_)}" instance_variable_set(variable_name, value.nil? || value == '1' || value == 'on') diff --git a/lib/reline/general_io.rb b/lib/reline/general_io.rb deleted file mode 100644 index d52151ad3c..0000000000 --- a/lib/reline/general_io.rb +++ /dev/null @@ -1,111 +0,0 @@ -require 'io/wait' - -class Reline::GeneralIO - RESET_COLOR = '' # Do not send color reset sequence - - def self.reset(encoding: nil) - @@pasting = false - if encoding - @@encoding = encoding - elsif defined?(@@encoding) - remove_class_variable(:@@encoding) - end - end - - def self.encoding - if defined?(@@encoding) - @@encoding - elsif RUBY_PLATFORM =~ /mswin|mingw/ - Encoding::UTF_8 - else - Encoding::default_external - end - end - - def self.win? - false - end - - def self.set_default_key_bindings(_) - end - - @@buf = [] - @@input = STDIN - - def self.input=(val) - @@input = val - end - - def self.with_raw_input - yield - end - - def self.getc(_timeout_second) - unless @@buf.empty? - return @@buf.shift - end - c = nil - loop do - Reline.core.line_editor.handle_signal - result = @@input.wait_readable(0.1) - next if result.nil? - c = @@input.read(1) - break - end - c&.ord - end - - def self.ungetc(c) - @@buf.unshift(c) - end - - def self.get_screen_size - [24, 80] - end - - def self.cursor_pos - Reline::CursorPos.new(1, 1) - end - - def self.hide_cursor - end - - def self.show_cursor - end - - def self.move_cursor_column(val) - end - - def self.move_cursor_up(val) - end - - def self.move_cursor_down(val) - end - - def self.erase_after_cursor - end - - def self.scroll_down(val) - end - - def self.clear_screen - end - - def self.set_screen_size(rows, columns) - end - - def self.set_winch_handler(&handler) - end - - @@pasting = false - - def self.in_pasting? - @@pasting - end - - def self.prep - end - - def self.deprep(otio) - end -end diff --git a/lib/reline/io.rb b/lib/reline/io.rb new file mode 100644 index 0000000000..c1dd1a56c8 --- /dev/null +++ b/lib/reline/io.rb @@ -0,0 +1,41 @@ + +module Reline + class IO + RESET_COLOR = "\e[0m" + + def self.decide_io_gate + if ENV['TERM'] == 'dumb' + Reline::Dumb.new + else + require 'reline/io/ansi' + + case RbConfig::CONFIG['host_os'] + when /mswin|msys|mingw|cygwin|bccwin|wince|emc/ + require 'reline/io/windows' + io = Reline::Windows.new + if io.msys_tty? + Reline::ANSI.new + else + io + end + else + Reline::ANSI.new + end + end + end + + def dumb? + false + end + + def win? + false + end + + def reset_color_sequence + self.class::RESET_COLOR + end + end +end + +require 'reline/io/dumb' diff --git a/lib/reline/ansi.rb b/lib/reline/io/ansi.rb index 5e1f7249e3..2b5a5c5786 100644 --- a/lib/reline/ansi.rb +++ b/lib/reline/io/ansi.rb @@ -1,10 +1,7 @@ require 'io/console' require 'io/wait' -require_relative 'terminfo' - -class Reline::ANSI - RESET_COLOR = "\e[0m" +class Reline::ANSI < Reline::IO CAPNAME_KEY_BINDINGS = { 'khome' => :ed_move_to_beg, 'kend' => :ed_move_to_end, @@ -36,15 +33,19 @@ class Reline::ANSI Reline::Terminfo.setupterm(0, 2) end - def self.encoding - Encoding.default_external + def initialize + @input = STDIN + @output = STDOUT + @buf = [] + @old_winch_handler = nil end - def self.win? - false + def encoding + Encoding.default_external end - def self.set_default_key_bindings(config, allow_terminfo: true) + def set_default_key_bindings(config, allow_terminfo: true) + set_bracketed_paste_key_bindings(config) set_default_key_bindings_ansi_cursor(config) if allow_terminfo && Reline::Terminfo.enabled? set_default_key_bindings_terminfo(config) @@ -66,7 +67,13 @@ class Reline::ANSI end end - def self.set_default_key_bindings_ansi_cursor(config) + def set_bracketed_paste_key_bindings(config) + [:emacs, :vi_insert, :vi_command].each do |keymap| + config.add_default_key_binding_by_keymap(keymap, START_BRACKETED_PASTE.bytes, :bracketed_paste_start) + end + end + + def set_default_key_bindings_ansi_cursor(config) ANSI_CURSOR_KEY_BINDINGS.each do |char, (default_func, modifiers)| bindings = [["\e[#{char}", default_func]] # CSI + char if modifiers[:ctrl] @@ -88,7 +95,7 @@ class Reline::ANSI end end - def self.set_default_key_bindings_terminfo(config) + def set_default_key_bindings_terminfo(config) key_bindings = CAPNAME_KEY_BINDINGS.map do |capname, key_binding| begin key_code = Reline::Terminfo.tigetstr(capname) @@ -105,7 +112,7 @@ class Reline::ANSI end end - def self.set_default_key_bindings_comprehensive_list(config) + def set_default_key_bindings_comprehensive_list(config) { # Console (80x25) [27, 91, 49, 126] => :ed_move_to_beg, # Home @@ -140,132 +147,107 @@ class Reline::ANSI end end - @@input = STDIN - def self.input=(val) - @@input = val + def input=(val) + @input = val end - @@output = STDOUT - def self.output=(val) - @@output = val + def output=(val) + @output = val end - def self.with_raw_input - if @@input.tty? - @@input.raw(intr: true) { yield } + def with_raw_input + if @input.tty? + @input.raw(intr: true) { yield } else yield end end - @@buf = [] - def self.inner_getc(timeout_second) - unless @@buf.empty? - return @@buf.shift + def inner_getc(timeout_second) + unless @buf.empty? + return @buf.shift end - until @@input.wait_readable(0.01) + until @input.wait_readable(0.01) timeout_second -= 0.01 return nil if timeout_second <= 0 Reline.core.line_editor.handle_signal end - c = @@input.getbyte - (c == 0x16 && @@input.raw(min: 0, time: 0, &:getbyte)) || c + c = @input.getbyte + (c == 0x16 && @input.tty? && @input.raw(min: 0, time: 0, &:getbyte)) || c rescue Errno::EIO # Maybe the I/O has been closed. nil - rescue Errno::ENOTTY - nil end - @@in_bracketed_paste_mode = false - START_BRACKETED_PASTE = String.new("\e[200~,", encoding: Encoding::ASCII_8BIT) - END_BRACKETED_PASTE = String.new("\e[200~.", encoding: Encoding::ASCII_8BIT) - def self.getc_with_bracketed_paste(timeout_second) + START_BRACKETED_PASTE = String.new("\e[200~", encoding: Encoding::ASCII_8BIT) + END_BRACKETED_PASTE = String.new("\e[201~", encoding: Encoding::ASCII_8BIT) + def read_bracketed_paste buffer = String.new(encoding: Encoding::ASCII_8BIT) - buffer << inner_getc(timeout_second) - while START_BRACKETED_PASTE.start_with?(buffer) or END_BRACKETED_PASTE.start_with?(buffer) do - if START_BRACKETED_PASTE == buffer - @@in_bracketed_paste_mode = true - return inner_getc(timeout_second) - elsif END_BRACKETED_PASTE == buffer - @@in_bracketed_paste_mode = false - ungetc(-1) - return inner_getc(timeout_second) - end - succ_c = inner_getc(Reline.core.config.keyseq_timeout) - - if succ_c - buffer << succ_c - else - break - end + until buffer.end_with?(END_BRACKETED_PASTE) + c = inner_getc(Float::INFINITY) + break unless c + buffer << c end - buffer.bytes.reverse_each do |ch| - ungetc ch - end - inner_getc(timeout_second) + string = buffer.delete_suffix(END_BRACKETED_PASTE).force_encoding(encoding) + string.valid_encoding? ? string : '' end # if the usage expects to wait indefinitely, use Float::INFINITY for timeout_second - def self.getc(timeout_second) - if Reline.core.config.enable_bracketed_paste - getc_with_bracketed_paste(timeout_second) - else - inner_getc(timeout_second) - end + def getc(timeout_second) + inner_getc(timeout_second) end - def self.in_pasting? - @@in_bracketed_paste_mode or (not empty_buffer?) + def in_pasting? + not empty_buffer? end - def self.empty_buffer? - unless @@buf.empty? + def empty_buffer? + unless @buf.empty? return false end - !@@input.wait_readable(0) + !@input.wait_readable(0) end - def self.ungetc(c) - @@buf.unshift(c) + def ungetc(c) + @buf.unshift(c) end - def self.retrieve_keybuffer + def retrieve_keybuffer begin - return unless @@input.wait_readable(0.001) - str = @@input.read_nonblock(1024) + return unless @input.wait_readable(0.001) + str = @input.read_nonblock(1024) str.bytes.each do |c| - @@buf.push(c) + @buf.push(c) end rescue EOFError end end - def self.get_screen_size - s = @@input.winsize + def get_screen_size + s = @input.winsize return s if s[0] > 0 && s[1] > 0 s = [ENV["LINES"].to_i, ENV["COLUMNS"].to_i] return s if s[0] > 0 && s[1] > 0 [24, 80] - rescue Errno::ENOTTY + rescue Errno::ENOTTY, Errno::ENODEV [24, 80] end - def self.set_screen_size(rows, columns) - @@input.winsize = [rows, columns] + def set_screen_size(rows, columns) + @input.winsize = [rows, columns] self - rescue Errno::ENOTTY + rescue Errno::ENOTTY, Errno::ENODEV self end - def self.cursor_pos - begin + def cursor_pos + if both_tty? res = +'' m = nil - @@input.raw do |stdin| - @@output << "\e[6n" - @@output.flush + @input.raw do |stdin| + @output << "\e[6n" + @output.flush loop do c = stdin.getc next if c.nil? @@ -279,9 +261,9 @@ class Reline::ANSI end column = m[:column].to_i - 1 row = m[:row].to_i - 1 - rescue Errno::ENOTTY + else begin - buf = @@output.pread(@@output.pos, 0) + buf = @output.pread(@output.pos, 0) row = buf.count("\n") column = buf.rindex("\n") ? (buf.size - buf.rindex("\n")) - 1 : 0 rescue Errno::ESPIPE, IOError @@ -294,30 +276,34 @@ class Reline::ANSI Reline::CursorPos.new(column, row) end - def self.move_cursor_column(x) - @@output.write "\e[#{x + 1}G" + def both_tty? + @input.tty? && @output.tty? + end + + def move_cursor_column(x) + @output.write "\e[#{x + 1}G" end - def self.move_cursor_up(x) + def move_cursor_up(x) if x > 0 - @@output.write "\e[#{x}A" + @output.write "\e[#{x}A" elsif x < 0 move_cursor_down(-x) end end - def self.move_cursor_down(x) + def move_cursor_down(x) if x > 0 - @@output.write "\e[#{x}B" + @output.write "\e[#{x}B" elsif x < 0 move_cursor_up(-x) end end - def self.hide_cursor + def hide_cursor if Reline::Terminfo.enabled? && Reline::Terminfo.term_supported? begin - @@output.write Reline::Terminfo.tigetstr('civis') + @output.write Reline::Terminfo.tigetstr('civis') rescue Reline::Terminfo::TerminfoError # civis is undefined end @@ -326,10 +312,10 @@ class Reline::ANSI end end - def self.show_cursor + def show_cursor if Reline::Terminfo.enabled? && Reline::Terminfo.term_supported? begin - @@output.write Reline::Terminfo.tigetstr('cnorm') + @output.write Reline::Terminfo.tigetstr('cnorm') rescue Reline::Terminfo::TerminfoError # cnorm is undefined end @@ -338,34 +324,37 @@ class Reline::ANSI end end - def self.erase_after_cursor - @@output.write "\e[K" + def erase_after_cursor + @output.write "\e[K" end # This only works when the cursor is at the bottom of the scroll range # For more details, see https://github.com/ruby/reline/pull/577#issuecomment-1646679623 - def self.scroll_down(x) + def scroll_down(x) return if x.zero? # We use `\n` instead of CSI + S because CSI + S would cause https://github.com/ruby/reline/issues/576 - @@output.write "\n" * x + @output.write "\n" * x end - def self.clear_screen - @@output.write "\e[2J" - @@output.write "\e[1;1H" + def clear_screen + @output.write "\e[2J" + @output.write "\e[1;1H" end - @@old_winch_handler = nil - def self.set_winch_handler(&handler) - @@old_winch_handler = Signal.trap('WINCH', &handler) + def set_winch_handler(&handler) + @old_winch_handler = Signal.trap('WINCH', &handler) end - def self.prep + def prep + # Enable bracketed paste + @output.write "\e[?2004h" if Reline.core.config.enable_bracketed_paste && both_tty? retrieve_keybuffer nil end - def self.deprep(otio) - Signal.trap('WINCH', @@old_winch_handler) if @@old_winch_handler + def deprep(otio) + # Disable bracketed paste + @output.write "\e[?2004l" if Reline.core.config.enable_bracketed_paste && both_tty? + Signal.trap('WINCH', @old_winch_handler) if @old_winch_handler end end diff --git a/lib/reline/io/dumb.rb b/lib/reline/io/dumb.rb new file mode 100644 index 0000000000..6ed69ffdfa --- /dev/null +++ b/lib/reline/io/dumb.rb @@ -0,0 +1,106 @@ +require 'io/wait' + +class Reline::Dumb < Reline::IO + RESET_COLOR = '' # Do not send color reset sequence + + def initialize(encoding: nil) + @input = STDIN + @buf = [] + @pasting = false + @encoding = encoding + @screen_size = [24, 80] + end + + def dumb? + true + end + + def encoding + if @encoding + @encoding + elsif RUBY_PLATFORM =~ /mswin|mingw/ + Encoding::UTF_8 + else + Encoding::default_external + end + end + + def set_default_key_bindings(_) + end + + def input=(val) + @input = val + end + + def with_raw_input + yield + end + + def getc(_timeout_second) + unless @buf.empty? + return @buf.shift + end + c = nil + loop do + Reline.core.line_editor.handle_signal + result = @input.wait_readable(0.1) + next if result.nil? + c = @input.read(1) + break + end + c&.ord + end + + def ungetc(c) + @buf.unshift(c) + end + + def get_screen_size + @screen_size + end + + def cursor_pos + Reline::CursorPos.new(1, 1) + end + + def hide_cursor + end + + def show_cursor + end + + def move_cursor_column(val) + end + + def move_cursor_up(val) + end + + def move_cursor_down(val) + end + + def erase_after_cursor + end + + def scroll_down(val) + end + + def clear_screen + end + + def set_screen_size(rows, columns) + @screen_size = [rows, columns] + end + + def set_winch_handler(&handler) + end + + def in_pasting? + @pasting + end + + def prep + end + + def deprep(otio) + end +end diff --git a/lib/reline/windows.rb b/lib/reline/io/windows.rb index ee3f73e383..6ba4b830d6 100644 --- a/lib/reline/windows.rb +++ b/lib/reline/io/windows.rb @@ -1,21 +1,49 @@ require 'fiddle/import' -class Reline::Windows - RESET_COLOR = "\e[0m" +class Reline::Windows < Reline::IO + def initialize + @input_buf = [] + @output_buf = [] + + @output = STDOUT + @hsg = nil + @getwch = Win32API.new('msvcrt', '_getwch', [], 'I') + @kbhit = Win32API.new('msvcrt', '_kbhit', [], 'I') + @GetKeyState = Win32API.new('user32', 'GetKeyState', ['L'], 'L') + @GetConsoleScreenBufferInfo = Win32API.new('kernel32', 'GetConsoleScreenBufferInfo', ['L', 'P'], 'L') + @SetConsoleCursorPosition = Win32API.new('kernel32', 'SetConsoleCursorPosition', ['L', 'L'], 'L') + @GetStdHandle = Win32API.new('kernel32', 'GetStdHandle', ['L'], 'L') + @FillConsoleOutputCharacter = Win32API.new('kernel32', 'FillConsoleOutputCharacter', ['L', 'L', 'L', 'L', 'P'], 'L') + @ScrollConsoleScreenBuffer = Win32API.new('kernel32', 'ScrollConsoleScreenBuffer', ['L', 'P', 'P', 'L', 'P'], 'L') + @hConsoleHandle = @GetStdHandle.call(STD_OUTPUT_HANDLE) + @hConsoleInputHandle = @GetStdHandle.call(STD_INPUT_HANDLE) + @GetNumberOfConsoleInputEvents = Win32API.new('kernel32', 'GetNumberOfConsoleInputEvents', ['L', 'P'], 'L') + @ReadConsoleInputW = Win32API.new('kernel32', 'ReadConsoleInputW', ['L', 'P', 'L', 'P'], 'L') + @GetFileType = Win32API.new('kernel32', 'GetFileType', ['L'], 'L') + @GetFileInformationByHandleEx = Win32API.new('kernel32', 'GetFileInformationByHandleEx', ['L', 'I', 'P', 'L'], 'I') + @FillConsoleOutputAttribute = Win32API.new('kernel32', 'FillConsoleOutputAttribute', ['L', 'L', 'L', 'L', 'P'], 'L') + @SetConsoleCursorInfo = Win32API.new('kernel32', 'SetConsoleCursorInfo', ['L', 'P'], 'L') + + @GetConsoleMode = Win32API.new('kernel32', 'GetConsoleMode', ['L', 'P'], 'L') + @SetConsoleMode = Win32API.new('kernel32', 'SetConsoleMode', ['L', 'L'], 'L') + @WaitForSingleObject = Win32API.new('kernel32', 'WaitForSingleObject', ['L', 'L'], 'L') + + @legacy_console = getconsolemode & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0 + end - def self.encoding + def encoding Encoding::UTF_8 end - def self.win? + def win? true end - def self.win_legacy_console? - @@legacy_console + def win_legacy_console? + @legacy_console end - def self.set_default_key_bindings(config) + def set_default_key_bindings(config) { [224, 72] => :ed_prev_history, # ↑ [224, 80] => :ed_next_history, # ↓ @@ -129,58 +157,32 @@ class Reline::Windows STD_OUTPUT_HANDLE = -11 FILE_TYPE_PIPE = 0x0003 FILE_NAME_INFO = 2 - @@getwch = Win32API.new('msvcrt', '_getwch', [], 'I') - @@kbhit = Win32API.new('msvcrt', '_kbhit', [], 'I') - @@GetKeyState = Win32API.new('user32', 'GetKeyState', ['L'], 'L') - @@GetConsoleScreenBufferInfo = Win32API.new('kernel32', 'GetConsoleScreenBufferInfo', ['L', 'P'], 'L') - @@SetConsoleCursorPosition = Win32API.new('kernel32', 'SetConsoleCursorPosition', ['L', 'L'], 'L') - @@GetStdHandle = Win32API.new('kernel32', 'GetStdHandle', ['L'], 'L') - @@FillConsoleOutputCharacter = Win32API.new('kernel32', 'FillConsoleOutputCharacter', ['L', 'L', 'L', 'L', 'P'], 'L') - @@ScrollConsoleScreenBuffer = Win32API.new('kernel32', 'ScrollConsoleScreenBuffer', ['L', 'P', 'P', 'L', 'P'], 'L') - @@hConsoleHandle = @@GetStdHandle.call(STD_OUTPUT_HANDLE) - @@hConsoleInputHandle = @@GetStdHandle.call(STD_INPUT_HANDLE) - @@GetNumberOfConsoleInputEvents = Win32API.new('kernel32', 'GetNumberOfConsoleInputEvents', ['L', 'P'], 'L') - @@ReadConsoleInputW = Win32API.new('kernel32', 'ReadConsoleInputW', ['L', 'P', 'L', 'P'], 'L') - @@GetFileType = Win32API.new('kernel32', 'GetFileType', ['L'], 'L') - @@GetFileInformationByHandleEx = Win32API.new('kernel32', 'GetFileInformationByHandleEx', ['L', 'I', 'P', 'L'], 'I') - @@FillConsoleOutputAttribute = Win32API.new('kernel32', 'FillConsoleOutputAttribute', ['L', 'L', 'L', 'L', 'P'], 'L') - @@SetConsoleCursorInfo = Win32API.new('kernel32', 'SetConsoleCursorInfo', ['L', 'P'], 'L') - - @@GetConsoleMode = Win32API.new('kernel32', 'GetConsoleMode', ['L', 'P'], 'L') - @@SetConsoleMode = Win32API.new('kernel32', 'SetConsoleMode', ['L', 'L'], 'L') - @@WaitForSingleObject = Win32API.new('kernel32', 'WaitForSingleObject', ['L', 'L'], 'L') ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4 - private_class_method def self.getconsolemode + private def getconsolemode mode = "\000\000\000\000" - @@GetConsoleMode.call(@@hConsoleHandle, mode) + @GetConsoleMode.call(@hConsoleHandle, mode) mode.unpack1('L') end - private_class_method def self.setconsolemode(mode) - @@SetConsoleMode.call(@@hConsoleHandle, mode) + private def setconsolemode(mode) + @SetConsoleMode.call(@hConsoleHandle, mode) end - @@legacy_console = (getconsolemode() & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0) - #if @@legacy_console + #if @legacy_console # setconsolemode(getconsolemode() | ENABLE_VIRTUAL_TERMINAL_PROCESSING) - # @@legacy_console = (getconsolemode() & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0) + # @legacy_console = (getconsolemode() & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0) #end - @@input_buf = [] - @@output_buf = [] - - @@output = STDOUT - - def self.msys_tty?(io = @@hConsoleInputHandle) + def msys_tty?(io = @hConsoleInputHandle) # check if fd is a pipe - if @@GetFileType.call(io) != FILE_TYPE_PIPE + if @GetFileType.call(io) != FILE_TYPE_PIPE return false end bufsize = 1024 p_buffer = "\0" * bufsize - res = @@GetFileInformationByHandleEx.call(io, FILE_NAME_INFO, p_buffer, bufsize - 2) + res = @GetFileInformationByHandleEx.call(io, FILE_NAME_INFO, p_buffer, bufsize - 2) return false if res == 0 # get pipe name: p_buffer layout is: @@ -217,65 +219,63 @@ class Reline::Windows [ { control_keys: :SHIFT, virtual_key_code: VK_TAB }, [27, 91, 90] ], ] - @@hsg = nil - - def self.process_key_event(repeat_count, virtual_key_code, virtual_scan_code, char_code, control_key_state) + def process_key_event(repeat_count, virtual_key_code, virtual_scan_code, char_code, control_key_state) # high-surrogate if 0xD800 <= char_code and char_code <= 0xDBFF - @@hsg = char_code + @hsg = char_code return end # low-surrogate if 0xDC00 <= char_code and char_code <= 0xDFFF - if @@hsg - char_code = 0x10000 + (@@hsg - 0xD800) * 0x400 + char_code - 0xDC00 - @@hsg = nil + if @hsg + char_code = 0x10000 + (@hsg - 0xD800) * 0x400 + char_code - 0xDC00 + @hsg = nil else # no high-surrogate. ignored. return end else # ignore high-surrogate without low-surrogate if there - @@hsg = nil + @hsg = nil end key = KeyEventRecord.new(virtual_key_code, char_code, control_key_state) match = KEY_MAP.find { |args,| key.matches?(**args) } unless match.nil? - @@output_buf.concat(match.last) + @output_buf.concat(match.last) return end # no char, only control keys return if key.char_code == 0 and key.control_keys.any? - @@output_buf.push("\e".ord) if key.control_keys.include?(:ALT) and !key.control_keys.include?(:CTRL) + @output_buf.push("\e".ord) if key.control_keys.include?(:ALT) and !key.control_keys.include?(:CTRL) - @@output_buf.concat(key.char.bytes) + @output_buf.concat(key.char.bytes) end - def self.check_input_event + def check_input_event num_of_events = 0.chr * 8 - while @@output_buf.empty? + while @output_buf.empty? Reline.core.line_editor.handle_signal - if @@WaitForSingleObject.(@@hConsoleInputHandle, 100) != 0 # max 0.1 sec + if @WaitForSingleObject.(@hConsoleInputHandle, 100) != 0 # max 0.1 sec # prevent for background consolemode change - @@legacy_console = (getconsolemode() & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0) + @legacy_console = getconsolemode & ENABLE_VIRTUAL_TERMINAL_PROCESSING == 0 next end - next if @@GetNumberOfConsoleInputEvents.(@@hConsoleInputHandle, num_of_events) == 0 or num_of_events.unpack1('L') == 0 + next if @GetNumberOfConsoleInputEvents.(@hConsoleInputHandle, num_of_events) == 0 or num_of_events.unpack1('L') == 0 input_records = 0.chr * 20 * 80 read_event = 0.chr * 4 - if @@ReadConsoleInputW.(@@hConsoleInputHandle, input_records, 80, read_event) != 0 + if @ReadConsoleInputW.(@hConsoleInputHandle, input_records, 80, read_event) != 0 read_events = read_event.unpack1('L') 0.upto(read_events) do |idx| input_record = input_records[idx * 20, 20] event = input_record[0, 2].unpack1('s*') case event when WINDOW_BUFFER_SIZE_EVENT - @@winch_handler.() + @winch_handler.() when KEY_EVENT key_down = input_record[4, 4].unpack1('l*') repeat_count = input_record[8, 2].unpack1('s*') @@ -293,34 +293,34 @@ class Reline::Windows end end - def self.with_raw_input + def with_raw_input yield end - def self.getc(_timeout_second) + def getc(_timeout_second) check_input_event - @@output_buf.shift + @output_buf.shift end - def self.ungetc(c) - @@output_buf.unshift(c) + def ungetc(c) + @output_buf.unshift(c) end - def self.in_pasting? - not self.empty_buffer? + def in_pasting? + not empty_buffer? end - def self.empty_buffer? - if not @@output_buf.empty? + def empty_buffer? + if not @output_buf.empty? false - elsif @@kbhit.call == 0 + elsif @kbhit.call == 0 true else false end end - def self.get_console_screen_buffer_info + def get_console_screen_buffer_info # CONSOLE_SCREEN_BUFFER_INFO # [ 0,2] dwSize.X # [ 2,2] dwSize.Y @@ -334,18 +334,18 @@ class Reline::Windows # [18,2] dwMaximumWindowSize.X # [20,2] dwMaximumWindowSize.Y csbi = 0.chr * 22 - return if @@GetConsoleScreenBufferInfo.call(@@hConsoleHandle, csbi) == 0 + return if @GetConsoleScreenBufferInfo.call(@hConsoleHandle, csbi) == 0 csbi end - def self.get_screen_size + def get_screen_size unless csbi = get_console_screen_buffer_info return [1, 1] end csbi[0, 4].unpack('SS').reverse end - def self.cursor_pos + def cursor_pos unless csbi = get_console_screen_buffer_info return Reline::CursorPos.new(0, 0) end @@ -354,49 +354,49 @@ class Reline::Windows Reline::CursorPos.new(x, y) end - def self.move_cursor_column(val) - @@SetConsoleCursorPosition.call(@@hConsoleHandle, cursor_pos.y * 65536 + val) + def move_cursor_column(val) + @SetConsoleCursorPosition.call(@hConsoleHandle, cursor_pos.y * 65536 + val) end - def self.move_cursor_up(val) + def move_cursor_up(val) if val > 0 y = cursor_pos.y - val y = 0 if y < 0 - @@SetConsoleCursorPosition.call(@@hConsoleHandle, y * 65536 + cursor_pos.x) + @SetConsoleCursorPosition.call(@hConsoleHandle, y * 65536 + cursor_pos.x) elsif val < 0 move_cursor_down(-val) end end - def self.move_cursor_down(val) + def move_cursor_down(val) if val > 0 return unless csbi = get_console_screen_buffer_info screen_height = get_screen_size.first y = cursor_pos.y + val y = screen_height - 1 if y > (screen_height - 1) - @@SetConsoleCursorPosition.call(@@hConsoleHandle, (cursor_pos.y + val) * 65536 + cursor_pos.x) + @SetConsoleCursorPosition.call(@hConsoleHandle, (cursor_pos.y + val) * 65536 + cursor_pos.x) elsif val < 0 move_cursor_up(-val) end end - def self.erase_after_cursor + def erase_after_cursor return unless csbi = get_console_screen_buffer_info attributes = csbi[8, 2].unpack1('S') cursor = csbi[4, 4].unpack1('L') written = 0.chr * 4 - @@FillConsoleOutputCharacter.call(@@hConsoleHandle, 0x20, get_screen_size.last - cursor_pos.x, cursor, written) - @@FillConsoleOutputAttribute.call(@@hConsoleHandle, attributes, get_screen_size.last - cursor_pos.x, cursor, written) + @FillConsoleOutputCharacter.call(@hConsoleHandle, 0x20, get_screen_size.last - cursor_pos.x, cursor, written) + @FillConsoleOutputAttribute.call(@hConsoleHandle, attributes, get_screen_size.last - cursor_pos.x, cursor, written) end - def self.scroll_down(val) + def scroll_down(val) return if val < 0 return unless csbi = get_console_screen_buffer_info buffer_width, buffer_lines, x, y, attributes, window_left, window_top, window_bottom = csbi.unpack('ssssSssx2s') screen_height = window_bottom - window_top + 1 val = screen_height if val > screen_height - if @@legacy_console || window_left != 0 + if @legacy_console || window_left != 0 # unless ENABLE_VIRTUAL_TERMINAL, # if srWindow.Left != 0 then it's conhost.exe hosted console # and puts "\n" causes horizontal scroll. its glitch. @@ -404,11 +404,11 @@ class Reline::Windows scroll_rectangle = [0, val, buffer_width, buffer_lines - val].pack('s4') destination_origin = 0 # y * 65536 + x fill = [' '.ord, attributes].pack('SS') - @@ScrollConsoleScreenBuffer.call(@@hConsoleHandle, scroll_rectangle, nil, destination_origin, fill) + @ScrollConsoleScreenBuffer.call(@hConsoleHandle, scroll_rectangle, nil, destination_origin, fill) else origin_x = x + 1 origin_y = y - window_top + 1 - @@output.write [ + @output.write [ (origin_y != screen_height) ? "\e[#{screen_height};H" : nil, "\n" * val, (origin_y != screen_height or !x.zero?) ? "\e[#{origin_y};#{origin_x}H" : nil @@ -416,49 +416,49 @@ class Reline::Windows end end - def self.clear_screen - if @@legacy_console + def clear_screen + if @legacy_console return unless csbi = get_console_screen_buffer_info buffer_width, _buffer_lines, attributes, window_top, window_bottom = csbi.unpack('ss@8S@12sx2s') fill_length = buffer_width * (window_bottom - window_top + 1) screen_topleft = window_top * 65536 written = 0.chr * 4 - @@FillConsoleOutputCharacter.call(@@hConsoleHandle, 0x20, fill_length, screen_topleft, written) - @@FillConsoleOutputAttribute.call(@@hConsoleHandle, attributes, fill_length, screen_topleft, written) - @@SetConsoleCursorPosition.call(@@hConsoleHandle, screen_topleft) + @FillConsoleOutputCharacter.call(@hConsoleHandle, 0x20, fill_length, screen_topleft, written) + @FillConsoleOutputAttribute.call(@hConsoleHandle, attributes, fill_length, screen_topleft, written) + @SetConsoleCursorPosition.call(@hConsoleHandle, screen_topleft) else - @@output.write "\e[2J" "\e[H" + @output.write "\e[2J" "\e[H" end end - def self.set_screen_size(rows, columns) + def set_screen_size(rows, columns) raise NotImplementedError end - def self.hide_cursor + def hide_cursor size = 100 visible = 0 # 0 means false cursor_info = [size, visible].pack('Li') - @@SetConsoleCursorInfo.call(@@hConsoleHandle, cursor_info) + @SetConsoleCursorInfo.call(@hConsoleHandle, cursor_info) end - def self.show_cursor + def show_cursor size = 100 visible = 1 # 1 means true cursor_info = [size, visible].pack('Li') - @@SetConsoleCursorInfo.call(@@hConsoleHandle, cursor_info) + @SetConsoleCursorInfo.call(@hConsoleHandle, cursor_info) end - def self.set_winch_handler(&handler) - @@winch_handler = handler + def set_winch_handler(&handler) + @winch_handler = handler end - def self.prep + def prep # do nothing nil end - def self.deprep(otio) + def deprep(otio) # do nothing end diff --git a/lib/reline/key_actor.rb b/lib/reline/key_actor.rb index ebe09d2009..0ac7604556 100644 --- a/lib/reline/key_actor.rb +++ b/lib/reline/key_actor.rb @@ -2,6 +2,7 @@ module Reline::KeyActor end require 'reline/key_actor/base' +require 'reline/key_actor/composite' require 'reline/key_actor/emacs' require 'reline/key_actor/vi_command' require 'reline/key_actor/vi_insert' diff --git a/lib/reline/key_actor/base.rb b/lib/reline/key_actor/base.rb index 194e98938c..ee28c7681e 100644 --- a/lib/reline/key_actor/base.rb +++ b/lib/reline/key_actor/base.rb @@ -1,15 +1,31 @@ class Reline::KeyActor::Base - MAPPING = Array.new(256) + def initialize(mapping = []) + @mapping = mapping + @matching_bytes = {} + @key_bindings = {} + end def get_method(key) - self.class::MAPPING[key] + @mapping[key] + end + + def add(key, func) + (1...key.size).each do |size| + @matching_bytes[key.take(size)] = true + end + @key_bindings[key] = func + end + + def matching?(key) + @matching_bytes[key] end - def initialize - @default_key_bindings = {} + def get(key) + @key_bindings[key] end - def default_key_bindings - @default_key_bindings + def clear + @matching_bytes.clear + @key_bindings.clear end end diff --git a/lib/reline/key_actor/composite.rb b/lib/reline/key_actor/composite.rb new file mode 100644 index 0000000000..37e94ce6cf --- /dev/null +++ b/lib/reline/key_actor/composite.rb @@ -0,0 +1,17 @@ +class Reline::KeyActor::Composite + def initialize(key_actors) + @key_actors = key_actors + end + + def matching?(key) + @key_actors.any? { |key_actor| key_actor.matching?(key) } + end + + def get(key) + @key_actors.each do |key_actor| + func = key_actor.get(key) + return func if func + end + nil + end +end diff --git a/lib/reline/key_actor/emacs.rb b/lib/reline/key_actor/emacs.rb index 5d0a7fb63d..ad84ee1d99 100644 --- a/lib/reline/key_actor/emacs.rb +++ b/lib/reline/key_actor/emacs.rb @@ -1,5 +1,5 @@ -class Reline::KeyActor::Emacs < Reline::KeyActor::Base - MAPPING = [ +module Reline::KeyActor + EMACS_MAPPING = [ # 0 ^@ :em_set_mark, # 1 ^A @@ -19,7 +19,7 @@ class Reline::KeyActor::Emacs < Reline::KeyActor::Base # 8 ^H :em_delete_prev_char, # 9 ^I - :ed_unassigned, + :complete, # 10 ^J :ed_newline, # 11 ^K @@ -63,7 +63,7 @@ class Reline::KeyActor::Emacs < Reline::KeyActor::Base # 30 ^^ :ed_unassigned, # 31 ^_ - :ed_unassigned, + :undo, # 32 SPACE :ed_insert, # 33 ! @@ -319,7 +319,7 @@ class Reline::KeyActor::Emacs < Reline::KeyActor::Base # 158 M-^^ :ed_unassigned, # 159 M-^_ - :ed_unassigned, + :redo, # 160 M-SPACE :em_set_mark, # 161 M-! diff --git a/lib/reline/key_actor/vi_command.rb b/lib/reline/key_actor/vi_command.rb index 06bb0ba8e4..d972c5e67f 100644 --- a/lib/reline/key_actor/vi_command.rb +++ b/lib/reline/key_actor/vi_command.rb @@ -1,5 +1,5 @@ -class Reline::KeyActor::ViCommand < Reline::KeyActor::Base - MAPPING = [ +module Reline::KeyActor + VI_COMMAND_MAPPING = [ # 0 ^@ :ed_unassigned, # 1 ^A diff --git a/lib/reline/key_actor/vi_insert.rb b/lib/reline/key_actor/vi_insert.rb index c3d7f9c12d..312df1646b 100644 --- a/lib/reline/key_actor/vi_insert.rb +++ b/lib/reline/key_actor/vi_insert.rb @@ -1,5 +1,5 @@ -class Reline::KeyActor::ViInsert < Reline::KeyActor::Base - MAPPING = [ +module Reline::KeyActor + VI_INSERT_MAPPING = [ # 0 ^@ :ed_unassigned, # 1 ^A @@ -19,7 +19,7 @@ class Reline::KeyActor::ViInsert < Reline::KeyActor::Base # 8 ^H :vi_delete_prev_char, # 9 ^I - :ed_insert, + :complete, # 10 ^J :ed_newline, # 11 ^K @@ -29,11 +29,11 @@ class Reline::KeyActor::ViInsert < Reline::KeyActor::Base # 13 ^M :ed_newline, # 14 ^N - :ed_insert, + :menu_complete, # 15 ^O :ed_insert, # 16 ^P - :ed_insert, + :menu_complete_backward, # 17 ^Q :ed_ignore, # 18 ^R diff --git a/lib/reline/key_stroke.rb b/lib/reline/key_stroke.rb index bceffbb53f..ba40899685 100644 --- a/lib/reline/key_stroke.rb +++ b/lib/reline/key_stroke.rb @@ -7,138 +7,99 @@ class Reline::KeyStroke @config = config end - def compress_meta_key(ary) - return ary unless @config.convert_meta - ary.inject([]) { |result, key| - if result.size > 0 and result.last == "\e".ord - result[result.size - 1] = Reline::Key.new(key, key | 0b10000000, true) - else - result << key - end - result - } - end + # Input exactly matches to a key sequence + MATCHING = :matching + # Input partially matches to a key sequence + MATCHED = :matched + # Input matches to a key sequence and the key sequence is a prefix of another key sequence + MATCHING_MATCHED = :matching_matched + # Input does not match to any key sequence + UNMATCHED = :unmatched - def start_with?(me, other) - compressed_me = compress_meta_key(me) - compressed_other = compress_meta_key(other) - i = 0 - loop do - my_c = compressed_me[i] - other_c = compressed_other[i] - other_is_last = (i + 1) == compressed_other.size - me_is_last = (i + 1) == compressed_me.size - if my_c != other_c - if other_c == "\e".ord and other_is_last and my_c.is_a?(Reline::Key) and my_c.with_meta - return true - else - return false - end - elsif other_is_last - return true - elsif me_is_last - return false - end - i += 1 - end - end + def match_status(input) + matching = key_mapping.matching?(input) + matched = key_mapping.get(input) - def equal?(me, other) - case me - when Array - compressed_me = compress_meta_key(me) - compressed_other = compress_meta_key(other) - compressed_me.size == compressed_other.size and [compressed_me, compressed_other].transpose.all?{ |i| equal?(i[0], i[1]) } - when Integer - if other.is_a?(Reline::Key) - if other.combined_char == "\e".ord - false - else - other.combined_char == me - end - else - me == other - end - when Reline::Key - if other.is_a?(Integer) - me.combined_char == other - else - me == other - end - end - end + # FIXME: Workaround for single byte. remove this after MAPPING is merged into KeyActor. + matched ||= input.size == 1 + matching ||= input == [ESC_BYTE] - def match_status(input) - key_mapping.keys.select { |lhs| - start_with?(lhs, input) - }.tap { |it| - return :matched if it.size == 1 && equal?(it[0], input) - return :matching if it.size == 1 && !equal?(it[0], input) - return :matched if it.max_by(&:size)&.size&.< input.size - return :matching if it.size > 1 - } - if key_mapping.keys.any? { |lhs| start_with?(input, lhs) } - :matched + if matching && matched + MATCHING_MATCHED + elsif matching + MATCHING + elsif matched + MATCHED + elsif input[0] == ESC_BYTE + match_unknown_escape_sequence(input, vi_mode: @config.editing_mode_is?(:vi_insert, :vi_command)) + elsif input.size == 1 + MATCHED else - match_unknown_escape_sequence(input).first + UNMATCHED end end def expand(input) - lhs = key_mapping.keys.select { |item| start_with?(input, item) }.sort_by(&:size).last - unless lhs - status, size = match_unknown_escape_sequence(input) - case status - when :matched - return [:ed_unassigned] + expand(input.drop(size)) - when :matching - return [:ed_unassigned] - else - return input - end + matched_bytes = nil + (1..input.size).each do |i| + bytes = input.take(i) + status = match_status(bytes) + matched_bytes = bytes if status == MATCHED || status == MATCHING_MATCHED end - rhs = key_mapping[lhs] + return [[], []] unless matched_bytes - case rhs - when String - rhs_bytes = rhs.bytes - expand(expand(rhs_bytes) + expand(input.drop(lhs.size))) - when Symbol - [rhs] + expand(input.drop(lhs.size)) - when Array - rhs + func = key_mapping.get(matched_bytes) + if func.is_a?(Array) + keys = func.map { |c| Reline::Key.new(c, c, false) } + elsif func + keys = [Reline::Key.new(func, func, false)] + elsif matched_bytes.size == 1 + keys = [Reline::Key.new(matched_bytes.first, matched_bytes.first, false)] + elsif matched_bytes.size == 2 && matched_bytes[0] == ESC_BYTE + keys = [Reline::Key.new(matched_bytes[1], matched_bytes[1] | 0b10000000, true)] + else + keys = [] end + + [keys, input.drop(matched_bytes.size)] end private # returns match status of CSI/SS3 sequence and matched length - def match_unknown_escape_sequence(input) + def match_unknown_escape_sequence(input, vi_mode: false) idx = 0 - return [:unmatched, nil] unless input[idx] == ESC_BYTE + return UNMATCHED unless input[idx] == ESC_BYTE idx += 1 idx += 1 if input[idx] == ESC_BYTE case input[idx] when nil - return [:matching, nil] + if idx == 1 # `ESC` + return MATCHING_MATCHED + else # `ESC ESC` + return MATCHING + end when 91 # == '['.ord - # CSI sequence + # CSI sequence `ESC [ ... char` idx += 1 idx += 1 while idx < input.size && CSI_PARAMETER_BYTES_RANGE.cover?(input[idx]) idx += 1 while idx < input.size && CSI_INTERMEDIATE_BYTES_RANGE.cover?(input[idx]) - input[idx] ? [:matched, idx + 1] : [:matching, nil] when 79 # == 'O'.ord - # SS3 sequence - input[idx + 1] ? [:matched, idx + 2] : [:matching, nil] + # SS3 sequence `ESC O char` + idx += 1 else - if idx == 1 - # `ESC char`, make it :unmatched so that it will be handled correctly in `read_2nd_character_of_key_sequence` - [:unmatched, nil] - else - # `ESC ESC char` - [:matched, idx + 1] - end + # `ESC char` or `ESC ESC char` + return UNMATCHED if vi_mode + end + + case input.size + when idx + MATCHING + when idx + 1 + MATCHED + else + UNMATCHED end end diff --git a/lib/reline/line_editor.rb b/lib/reline/line_editor.rb index 07a9cc6af0..d659378acd 100644 --- a/lib/reline/line_editor.rb +++ b/lib/reline/line_editor.rb @@ -4,7 +4,6 @@ require 'reline/unicode' require 'tempfile' class Reline::LineEditor - # TODO: undo # TODO: Use "private alias_method" idiom after drop Ruby 2.5. attr_reader :byte_pointer attr_accessor :confirm_multiline_termination_proc @@ -46,6 +45,7 @@ class Reline::LineEditor RenderedScreen = Struct.new(:base_y, :lines, :cursor_y, keyword_init: true) CompletionJourneyState = Struct.new(:line_index, :pre, :target, :post, :list, :pointer) + NullActionState = [nil, nil].freeze class MenuInfo attr_reader :list @@ -75,7 +75,7 @@ class Reline::LineEditor def initialize(config, encoding) @config = config @completion_append_character = '' - @screen_size = Reline::IOGate.get_screen_size + @screen_size = [0, 0] # Should be initialized with actual winsize in LineEditor#reset reset_variables(encoding: encoding) end @@ -235,11 +235,9 @@ class Reline::LineEditor @vi_waiting_operator_arg = nil @completion_journey_state = nil @completion_state = CompletionState::NORMAL - @completion_occurs = false @perfect_matched = nil @menu_info = nil @searching_prompt = nil - @first_char = true @just_cursor_moving = false @eof = false @continuous_insertion_buffer = String.new(encoding: @encoding) @@ -252,6 +250,11 @@ class Reline::LineEditor @resized = false @cache = {} @rendered_screen = RenderedScreen.new(base_y: 0, lines: [], cursor_y: 0) + @input_lines = [[[""], 0, 0]] + @input_lines_position = 0 + @undoing = false + @prev_action_state = NullActionState + @next_action_state = NullActionState reset_line end @@ -284,7 +287,7 @@ class Reline::LineEditor indent1 = @auto_indent_proc.(@buffer_of_lines.take(@line_index - 1).push(''), @line_index - 1, 0, true) indent2 = @auto_indent_proc.(@buffer_of_lines.take(@line_index), @line_index - 1, @buffer_of_lines[@line_index - 1].bytesize, false) indent = indent2 || indent1 - @buffer_of_lines[@line_index - 1] = ' ' * indent + @buffer_of_lines[@line_index - 1].gsub(/\A */, '') + @buffer_of_lines[@line_index - 1] = ' ' * indent + @buffer_of_lines[@line_index - 1].gsub(/\A\s*/, '') ) process_auto_indent @line_index, add_newline: true else @@ -411,7 +414,7 @@ class Reline::LineEditor # do nothing elsif level == :blank Reline::IOGate.move_cursor_column base_x - @output.write "#{Reline::IOGate::RESET_COLOR}#{' ' * width}" + @output.write "#{Reline::IOGate.reset_color_sequence}#{' ' * width}" else x, w, content = new_items[level] cover_begin = base_x != 0 && new_levels[base_x - 1] == level @@ -421,7 +424,7 @@ class Reline::LineEditor content, pos = Reline::Unicode.take_mbchar_range(content, base_x - x, width, cover_begin: cover_begin, cover_end: cover_end, padding: true) end Reline::IOGate.move_cursor_column x + pos - @output.write "#{Reline::IOGate::RESET_COLOR}#{content}#{Reline::IOGate::RESET_COLOR}" + @output.write "#{Reline::IOGate.reset_color_sequence}#{content}#{Reline::IOGate.reset_color_sequence}" end base_x += width end @@ -683,10 +686,8 @@ class Reline::LineEditor @trap_key.each do |t| @config.add_oneshot_key_binding(t, @name) end - elsif @trap_key.is_a?(Array) + else @config.add_oneshot_key_binding(@trap_key, @name) - elsif @trap_key.is_a?(Integer) or @trap_key.is_a?(Reline::Key) - @config.add_oneshot_key_binding([@trap_key], @name) end end dialog_render_info @@ -856,7 +857,7 @@ class Reline::LineEditor [target, preposing, completed, postposing] end - private def complete(list, just_show_list) + private def perform_completion(list, just_show_list) case @completion_state when CompletionState::NORMAL @completion_state = CompletionState::COMPLETION @@ -885,12 +886,12 @@ class Reline::LineEditor @completion_state = CompletionState::PERFECT_MATCH else @completion_state = CompletionState::MENU_WITH_PERFECT_MATCH - complete(list, true) if @config.show_all_if_ambiguous + perform_completion(list, true) if @config.show_all_if_ambiguous end @perfect_matched = completed else @completion_state = CompletionState::MENU - complete(list, true) if @config.show_all_if_ambiguous + perform_completion(list, true) if @config.show_all_if_ambiguous end if not just_show_list and target < completed @buffer_of_lines[@line_index] = (preposing + completed + completion_append_character.to_s + postposing).split("\n")[@line_index] || String.new(encoding: @encoding) @@ -949,7 +950,8 @@ class Reline::LineEditor unless @waiting_proc byte_pointer_diff = @byte_pointer - old_byte_pointer @byte_pointer = old_byte_pointer - send(@vi_waiting_operator, byte_pointer_diff) + method_obj = method(@vi_waiting_operator) + wrap_method_call(@vi_waiting_operator, method_obj, byte_pointer_diff) cleanup_waiting end else @@ -1010,7 +1012,8 @@ class Reline::LineEditor if @vi_waiting_operator byte_pointer_diff = @byte_pointer - old_byte_pointer @byte_pointer = old_byte_pointer - send(@vi_waiting_operator, byte_pointer_diff) + method_obj = method(@vi_waiting_operator) + wrap_method_call(@vi_waiting_operator, method_obj, byte_pointer_diff) cleanup_waiting end @kill_ring.process @@ -1065,10 +1068,6 @@ class Reline::LineEditor end private def normal_char(key) - if key.combined_char.is_a?(Symbol) - process_key(key.combined_char, key.combined_char) - return - end @multibyte_buffer << key.combined_char if @multibyte_buffer.size > 1 if @multibyte_buffer.dup.force_encoding(@encoding).valid_encoding? @@ -1081,17 +1080,7 @@ class Reline::LineEditor else # single byte return if key.char >= 128 # maybe, first byte of multi byte method_symbol = @config.editing_mode.get_method(key.combined_char) - if key.with_meta and method_symbol == :ed_unassigned - if @config.editing_mode_is?(:vi_command, :vi_insert) - # split ESC + key in vi mode - method_symbol = @config.editing_mode.get_method("\e".ord) - process_key("\e".ord, method_symbol) - method_symbol = @config.editing_mode.get_method(key.char) - process_key(key.char, method_symbol) - end - else - process_key(key.combined_char, method_symbol) - end + process_key(key.combined_char, method_symbol) @multibyte_buffer.clear end if @config.editing_mode_is?(:vi_command) and @byte_pointer > 0 and @byte_pointer == current_line.bytesize @@ -1111,6 +1100,7 @@ class Reline::LineEditor end def input_key(key) + save_old_buffer @config.reset_oneshot_key_bindings @dialogs.each do |dialog| if key.char.instance_of?(Symbol) and key.char == dialog.name @@ -1119,53 +1109,34 @@ class Reline::LineEditor end if key.char.nil? process_insert(force: true) - if @first_char - @eof = true - end + @eof = buffer_empty? finish return end - old_lines = @buffer_of_lines.dup - @first_char = false @completion_occurs = false - if @config.editing_mode_is?(:emacs, :vi_insert) and key.char == "\C-i".ord - if !@config.disable_completion - process_insert(force: true) - if @config.autocompletion - @completion_state = CompletionState::NORMAL - @completion_occurs = move_completed_list(:down) - else - @completion_journey_state = nil - result = call_completion_proc - if result.is_a?(Array) - @completion_occurs = true - complete(result, false) - end - end - end - elsif @config.editing_mode_is?(:vi_insert) and ["\C-p".ord, "\C-n".ord].include?(key.char) - # In vi mode, move completed list even if autocompletion is off - if not @config.disable_completion - process_insert(force: true) - @completion_state = CompletionState::NORMAL - @completion_occurs = move_completed_list("\C-p".ord == key.char ? :up : :down) - end - elsif Symbol === key.char and respond_to?(key.char, true) + + if key.char.is_a?(Symbol) process_key(key.char, key.char) else normal_char(key) end + + @prev_action_state, @next_action_state = @next_action_state, NullActionState + unless @completion_occurs @completion_state = CompletionState::NORMAL @completion_journey_state = nil end + push_input_lines unless @undoing + @undoing = false + if @in_pasting clear_dialogs return end - modified = old_lines != @buffer_of_lines + modified = @old_buffer_of_lines != @buffer_of_lines if !@completion_occurs && modified && !@config.disable_completion && @config.autocompletion # Auto complete starts only when edited process_insert(force: true) @@ -1174,6 +1145,29 @@ class Reline::LineEditor modified end + def save_old_buffer + @old_buffer_of_lines = @buffer_of_lines.dup + end + + def push_input_lines + if @old_buffer_of_lines == @buffer_of_lines + @input_lines[@input_lines_position] = [@buffer_of_lines.dup, @byte_pointer, @line_index] + else + @input_lines = @input_lines[0..@input_lines_position] + @input_lines_position += 1 + @input_lines.push([@buffer_of_lines.dup, @byte_pointer, @line_index]) + end + trim_input_lines + end + + MAX_INPUT_LINES = 100 + def trim_input_lines + if @input_lines.size > MAX_INPUT_LINES + @input_lines.shift + @input_lines_position -= 1 + end + end + def scroll_into_view _wrapped_cursor_x, wrapped_cursor_y = wrapped_cursor_position if wrapped_cursor_y < screen_scroll_top @@ -1250,6 +1244,18 @@ class Reline::LineEditor process_auto_indent end + def set_current_lines(lines, byte_pointer = nil, line_index = 0) + cursor = current_byte_pointer_cursor + @buffer_of_lines = lines + @line_index = line_index + if byte_pointer + @byte_pointer = byte_pointer + else + calculate_nearest_cursor(cursor) + end + process_auto_indent + end + def retrieve_completion_block(set_completion_quote_character = false) if Reline.completer_word_break_characters.empty? word_break_regexp = nil @@ -1331,6 +1337,18 @@ class Reline::LineEditor @confirm_multiline_termination_proc.(temp_buffer.join("\n") + "\n") end + def insert_pasted_text(text) + save_old_buffer + pre = @buffer_of_lines[@line_index].byteslice(0, @byte_pointer) + post = @buffer_of_lines[@line_index].byteslice(@byte_pointer..) + lines = (pre + text.gsub(/\r\n?/, "\n") + post).split("\n", -1) + lines << '' if lines.empty? + @buffer_of_lines[@line_index, 1] = lines + @line_index += lines.size - 1 + @byte_pointer = @buffer_of_lines[@line_index].bytesize - post.bytesize + push_input_lines + end + def insert_text(text) if @buffer_of_lines[@line_index].bytesize == @byte_pointer @buffer_of_lines[@line_index] += text @@ -1387,6 +1405,10 @@ class Reline::LineEditor whole_lines.join("\n") end + private def buffer_empty? + current_line.empty? and @buffer_of_lines.size == 1 + end + def finished? @finished end @@ -1429,13 +1451,42 @@ class Reline::LineEditor end end - private def completion_journey_up(key) - if not @config.disable_completion and @config.autocompletion + private def complete(_key) + return if @config.disable_completion + + process_insert(force: true) + if @config.autocompletion @completion_state = CompletionState::NORMAL - @completion_occurs = move_completed_list(:up) + @completion_occurs = move_completed_list(:down) + else + @completion_journey_state = nil + result = call_completion_proc + if result.is_a?(Array) + @completion_occurs = true + perform_completion(result, false) + end end end - alias_method :menu_complete_backward, :completion_journey_up + + private def completion_journey_move(direction) + return if @config.disable_completion + + process_insert(force: true) + @completion_state = CompletionState::NORMAL + @completion_occurs = move_completed_list(direction) + end + + private def menu_complete(_key) + completion_journey_move(:down) + end + + private def menu_complete_backward(_key) + completion_journey_move(:up) + end + + private def completion_journey_up(_key) + completion_journey_move(:up) if @config.autocompletion + end # Editline:: +ed-unassigned+ This editor command always results in an error. # GNU Readline:: There is no corresponding macro. @@ -1706,29 +1757,31 @@ class Reline::LineEditor end private def ed_search_prev_history(key, arg: 1) - substr = current_line.byteslice(0, @byte_pointer) + substr = prev_action_state_value(:search_history) == :empty ? '' : current_line.byteslice(0, @byte_pointer) return if @history_pointer == 0 return if @history_pointer.nil? && substr.empty? && !current_line.empty? history_range = 0...(@history_pointer || Reline::HISTORY.size) h_pointer, line_index = search_history(substr, history_range.reverse_each) return unless h_pointer - move_history(h_pointer, line: line_index || :start, cursor: @byte_pointer) + move_history(h_pointer, line: line_index || :start, cursor: substr.empty? ? :end : @byte_pointer) arg -= 1 + set_next_action_state(:search_history, :empty) if substr.empty? ed_search_prev_history(key, arg: arg) if arg > 0 end alias_method :history_search_backward, :ed_search_prev_history private def ed_search_next_history(key, arg: 1) - substr = current_line.byteslice(0, @byte_pointer) + substr = prev_action_state_value(:search_history) == :empty ? '' : current_line.byteslice(0, @byte_pointer) return if @history_pointer.nil? history_range = @history_pointer + 1...Reline::HISTORY.size h_pointer, line_index = search_history(substr, history_range) return if h_pointer.nil? and not substr.empty? - move_history(h_pointer, line: line_index || :start, cursor: @byte_pointer) + move_history(h_pointer, line: line_index || :start, cursor: substr.empty? ? :end : @byte_pointer) arg -= 1 + set_next_action_state(:search_history, :empty) if substr.empty? ed_search_next_history(key, arg: arg) if arg > 0 end alias_method :history_search_forward, :ed_search_next_history @@ -1884,7 +1937,7 @@ class Reline::LineEditor alias_method :kill_whole_line, :em_kill_line private def em_delete(key) - if current_line.empty? and @buffer_of_lines.size == 1 and key == "\C-d".ord + if buffer_empty? and key == "\C-d".ord @eof = true finish elsif @byte_pointer < current_line.bytesize @@ -1904,7 +1957,7 @@ class Reline::LineEditor elsif !@config.autocompletion # show completed list result = call_completion_proc if result.is_a?(Array) - complete(result, true) + perform_completion(result, true) end end end @@ -2232,8 +2285,7 @@ class Reline::LineEditor end private def vi_list_or_eof(key) - if current_line.empty? and @buffer_of_lines.size == 1 - set_current_line('', 0) + if buffer_empty? @eof = true finish else @@ -2474,4 +2526,32 @@ class Reline::LineEditor private def vi_editing_mode(key) @config.editing_mode = :vi_insert end + + private def undo(_key) + @undoing = true + + return if @input_lines_position <= 0 + + @input_lines_position -= 1 + target_lines, target_cursor_x, target_cursor_y = @input_lines[@input_lines_position] + set_current_lines(target_lines.dup, target_cursor_x, target_cursor_y) + end + + private def redo(_key) + @undoing = true + + return if @input_lines_position >= @input_lines.size - 1 + + @input_lines_position += 1 + target_lines, target_cursor_x, target_cursor_y = @input_lines[@input_lines_position] + set_current_lines(target_lines.dup, target_cursor_x, target_cursor_y) + end + + private def prev_action_state_value(type) + @prev_action_state[0] == type ? @prev_action_state[1] : nil + end + + private def set_next_action_state(type, value) + @next_action_state = [type, value] + end end diff --git a/lib/reline/unicode.rb b/lib/reline/unicode.rb index 82c9ec427c..d7460d6d4a 100644 --- a/lib/reline/unicode.rb +++ b/lib/reline/unicode.rb @@ -43,11 +43,13 @@ class Reline::Unicode def self.escape_for_print(str) str.chars.map! { |gr| - escaped = EscapedPairs[gr.ord] - if escaped && gr != -"\n" && gr != -"\t" - escaped - else + case gr + when -"\n" gr + when -"\t" + -' ' + else + EscapedPairs[gr.ord] || gr end }.join end diff --git a/lib/reline/version.rb b/lib/reline/version.rb index ab10713f45..680e5a9cca 100644 --- a/lib/reline/version.rb +++ b/lib/reline/version.rb @@ -1,3 +1,3 @@ module Reline - VERSION = '0.5.5' + VERSION = '0.5.8' end diff --git a/lib/ruby_vm/rjit/insn_compiler.rb b/lib/ruby_vm/rjit/insn_compiler.rb index f9450241c9..9e4b28f87a 100644 --- a/lib/ruby_vm/rjit/insn_compiler.rb +++ b/lib/ruby_vm/rjit/insn_compiler.rb @@ -504,7 +504,7 @@ module RubyVM::RJIT shape = C.rb_shape_get_shape_by_id(shape_id) current_capacity = shape.capacity - dest_shape = C.rb_shape_get_next(shape, comptime_receiver, ivar_name) + dest_shape = C.rb_shape_get_next_no_warnings(shape, comptime_receiver, ivar_name) new_shape_id = C.rb_shape_id(dest_shape) if new_shape_id == C::OBJ_TOO_COMPLEX_SHAPE_ID diff --git a/lib/rubygems.rb b/lib/rubygems.rb index ad7ab10756..ac225ca70a 100644 --- a/lib/rubygems.rb +++ b/lib/rubygems.rb @@ -1013,6 +1013,13 @@ An Array (#{env.inspect}) was passed in from #{caller[3]} end ## + # Is this platform FreeBSD + + def self.freebsd_platform? + RbConfig::CONFIG["host_os"].to_s.include?("bsd") + end + + ## # Load +plugins+ as Ruby files def self.load_plugin_files(plugins) # :nodoc: diff --git a/lib/rubygems/basic_specification.rb b/lib/rubygems/basic_specification.rb index 0380fceece..f25756f92c 100644 --- a/lib/rubygems/basic_specification.rb +++ b/lib/rubygems/basic_specification.rb @@ -144,6 +144,19 @@ class Gem::BasicSpecification end ## + # Returns the full name of this Gem (see `Gem::BasicSpecification#full_name`). + # Information about where the gem is installed is also included if not + # installed in the default GEM_HOME. + + def full_name_with_location + if base_dir != Gem.dir + "#{full_name} in #{base_dir}" + else + full_name + end + end + + ## # Full paths in the gem to add to <code>$LOAD_PATH</code> when this gem is # activated. diff --git a/lib/rubygems/commands/pristine_command.rb b/lib/rubygems/commands/pristine_command.rb index 456d897df2..999c9fef0f 100644 --- a/lib/rubygems/commands/pristine_command.rb +++ b/lib/rubygems/commands/pristine_command.rb @@ -57,7 +57,7 @@ class Gem::Commands::PristineCommand < Gem::Command end add_option("-i", "--install-dir DIR", - "Gem repository to get binstubs and plugins installed") do |value, options| + "Gem repository to get gems restored") do |value, options| options[:install_dir] = File.expand_path(value) end @@ -103,21 +103,25 @@ extensions will be restored. end def execute + install_dir = options[:install_dir] + + specification_record = install_dir ? Gem::SpecificationRecord.from_path(install_dir) : Gem::Specification.specification_record + specs = if options[:all] - Gem::Specification.map + specification_record.map # `--extensions` must be explicitly given to pristine only gems # with extensions. elsif options[:extensions_set] && options[:extensions] && options[:args].empty? - Gem::Specification.select do |spec| + specification_record.select do |spec| spec.extensions && !spec.extensions.empty? end elsif options[:only_missing_extensions] - Gem::Specification.select(&:missing_extensions?) + specification_record.select(&:missing_extensions?) else get_all_gem_names.sort.map do |gem_name| - Gem::Specification.find_all_by_name(gem_name, options[:version]).reverse + specification_record.find_all_by_name(gem_name, options[:version]).reverse end.flatten end @@ -144,7 +148,7 @@ extensions will be restored. end unless spec.extensions.empty? || options[:extensions] || options[:only_executables] || options[:only_plugins] - say "Skipped #{spec.full_name}, it needs to compile an extension" + say "Skipped #{spec.full_name_with_location}, it needs to compile an extension" next end @@ -153,7 +157,7 @@ extensions will be restored. unless File.exist?(gem) || options[:only_executables] || options[:only_plugins] require_relative "../remote_fetcher" - say "Cached gem for #{spec.full_name} not found, attempting to fetch..." + say "Cached gem for #{spec.full_name_with_location} not found, attempting to fetch..." dep = Gem::Dependency.new spec.name, spec.version found, _ = Gem::SpecFetcher.fetcher.spec_for_dependency dep @@ -176,7 +180,6 @@ extensions will be restored. end bin_dir = options[:bin_dir] if options[:bin_dir] - install_dir = options[:install_dir] if options[:install_dir] installer_options = { wrappers: true, @@ -198,7 +201,7 @@ extensions will be restored. installer.install end - say "Restored #{spec.full_name}" + say "Restored #{spec.full_name_with_location}" end end end diff --git a/lib/rubygems/commands/setup_command.rb b/lib/rubygems/commands/setup_command.rb index 3f38074280..9c633d6ef7 100644 --- a/lib/rubygems/commands/setup_command.rb +++ b/lib/rubygems/commands/setup_command.rb @@ -585,6 +585,8 @@ abort "#{deprecation_message}" args = %w[--all --only-executables --silent] args << "--bindir=#{bindir}" + args << "--install-dir=#{default_dir}" + if options[:env_shebang] args << "--env-shebang" end diff --git a/lib/rubygems/commands/uninstall_command.rb b/lib/rubygems/commands/uninstall_command.rb index 2a77ec72cf..283bc96ce3 100644 --- a/lib/rubygems/commands/uninstall_command.rb +++ b/lib/rubygems/commands/uninstall_command.rb @@ -184,7 +184,7 @@ that is a dependency of an existing gem. You can use the rescue Gem::GemNotInHomeException => e spec = e.spec alert("In order to remove #{spec.name}, please execute:\n" \ - "\tgem uninstall #{spec.name} --install-dir=#{spec.installation_path}") + "\tgem uninstall #{spec.name} --install-dir=#{spec.base_dir}") rescue Gem::UninstallError => e spec = e.spec alert_error("Error: unable to successfully uninstall '#{spec.name}' which is " \ diff --git a/lib/rubygems/dependency.rb b/lib/rubygems/dependency.rb index d1bf074441..5ce9c5e840 100644 --- a/lib/rubygems/dependency.rb +++ b/lib/rubygems/dependency.rb @@ -271,15 +271,7 @@ class Gem::Dependency end def matching_specs(platform_only = false) - env_req = Gem.env_requirement(name) - matches = Gem::Specification.stubs_for(name).find_all do |spec| - requirement.satisfied_by?(spec.version) && env_req.satisfied_by?(spec.version) - end.map(&:to_spec) - - if prioritizes_bundler? - require_relative "bundler_version_finder" - Gem::BundlerVersionFinder.prioritize!(matches) - end + matches = Gem::Specification.find_all_by_name(name, requirement) if platform_only matches.reject! do |spec| @@ -297,10 +289,6 @@ class Gem::Dependency @requirement.specific? end - def prioritizes_bundler? - name == "bundler" && !specific? - end - def to_specs matches = matching_specs true diff --git a/lib/rubygems/ext/cargo_builder.rb b/lib/rubygems/ext/cargo_builder.rb index 86a0e73f28..09ad1407c2 100644 --- a/lib/rubygems/ext/cargo_builder.rb +++ b/lib/rubygems/ext/cargo_builder.rb @@ -184,6 +184,7 @@ class Gem::Ext::CargoBuilder < Gem::Ext::Builder end def cargo_dylib_path(dest_path, crate_name) + so_ext = RbConfig::CONFIG["SOEXT"] prefix = so_ext == "dll" ? "" : "lib" path_parts = [dest_path] path_parts << ENV["CARGO_BUILD_TARGET"] if ENV["CARGO_BUILD_TARGET"] @@ -312,22 +313,6 @@ EOF deffile_path end - # We have to basically reimplement <code>RbConfig::CONFIG['SOEXT']</code> here to support - # Ruby < 2.5 - # - # @see https://github.com/ruby/ruby/blob/c87c027f18c005460746a74c07cd80ee355b16e4/configure.ac#L3185 - def so_ext - return RbConfig::CONFIG["SOEXT"] if RbConfig::CONFIG.key?("SOEXT") - - if win_target? - "dll" - elsif darwin_target? - "dylib" - else - "so" - end - end - # Corresponds to $(LIBPATH) in mkmf def mkmf_libpath ["-L", "native=#{makefile_config("libdir")}"] diff --git a/lib/rubygems/installer.rb b/lib/rubygems/installer.rb index 8f6f9a5aa8..844f292ba2 100644 --- a/lib/rubygems/installer.rb +++ b/lib/rubygems/installer.rb @@ -344,7 +344,7 @@ class Gem::Installer say spec.post_install_message if options[:post_install_message] && !spec.post_install_message.nil? - Gem::Specification.add_spec(spec) + Gem::Specification.add_spec(spec) unless @install_dir load_plugin diff --git a/lib/rubygems/package/tar_header.rb b/lib/rubygems/package/tar_header.rb index 087f13f6c9..dd5e835a1e 100644 --- a/lib/rubygems/package/tar_header.rb +++ b/lib/rubygems/package/tar_header.rb @@ -95,14 +95,14 @@ class Gem::Package::TarHeader attr_reader(*FIELDS) - EMPTY_HEADER = ("\0" * 512).freeze # :nodoc: + EMPTY_HEADER = ("\0" * 512).b.freeze # :nodoc: ## # Creates a tar header from IO +stream+ def self.from(stream) header = stream.read 512 - empty = (header == EMPTY_HEADER) + return EMPTY if header == EMPTY_HEADER fields = header.unpack UNPACK_FORMAT @@ -123,7 +123,7 @@ class Gem::Package::TarHeader devminor: strict_oct(fields.shift), prefix: fields.shift, - empty: empty + empty: false end def self.strict_oct(str) @@ -172,6 +172,22 @@ class Gem::Package::TarHeader @empty = vals[:empty] end + EMPTY = new({ # :nodoc: + checksum: 0, + gname: "", + linkname: "", + magic: "", + mode: 0, + name: "", + prefix: "", + size: 0, + uname: "", + version: 0, + + empty: true, + }).freeze + private_constant :EMPTY + ## # Is the tar entry empty? @@ -241,7 +257,7 @@ class Gem::Package::TarHeader header = header.pack PACK_FORMAT - header << ("\0" * ((512 - header.size) % 512)) + header.ljust 512, "\0" end def oct(num, len) diff --git a/lib/rubygems/platform.rb b/lib/rubygems/platform.rb index 48b7344aee..d54ad12880 100644 --- a/lib/rubygems/platform.rb +++ b/lib/rubygems/platform.rb @@ -134,6 +134,7 @@ class Gem::Platform when /netbsdelf/ then ["netbsdelf", nil] when /openbsd(\d+\.\d+)?/ then ["openbsd", $1] when /solaris(\d+\.\d+)?/ then ["solaris", $1] + when /wasi/ then ["wasi", nil] # test when /^(\w+_platform)(\d+)?/ then [$1, $2] else ["unknown", nil] diff --git a/lib/rubygems/specification.rb b/lib/rubygems/specification.rb index a1eaf1248e..da55a2e6d3 100644 --- a/lib/rubygems/specification.rb +++ b/lib/rubygems/specification.rb @@ -11,6 +11,7 @@ require_relative "deprecate" require_relative "basic_specification" require_relative "stub_specification" require_relative "platform" +require_relative "specification_record" require_relative "util/list" require "rbconfig" @@ -179,19 +180,9 @@ class Gem::Specification < Gem::BasicSpecification @@default_value[k].nil? end - def self.clear_specs # :nodoc: - @@all = nil - @@stubs = nil - @@stubs_by_name = {} - @@spec_with_requirable_file = {} - @@active_stub_with_requirable_file = {} - end - private_class_method :clear_specs - - clear_specs - # Sentinel object to represent "not found" stubs NOT_FOUND = Struct.new(:to_spec, :this).new # :nodoc: + deprecate_constant :NOT_FOUND # Tracking removed method calls to warn users during build time. REMOVED_METHODS = [:rubyforge_project=, :mark_version].freeze # :nodoc: @@ -770,7 +761,7 @@ class Gem::Specification < Gem::BasicSpecification attr_accessor :specification_version def self._all # :nodoc: - @@all ||= Gem.loaded_specs.values | stubs.map(&:to_spec) + specification_record.all end def self.clear_load_cache # :nodoc: @@ -788,26 +779,9 @@ class Gem::Specification < Gem::BasicSpecification end end - def self.gemspec_stubs_in(dir, pattern) + def self.gemspec_stubs_in(dir, pattern) # :nodoc: Gem::Util.glob_files_in_dir(pattern, dir).map {|path| yield path }.select(&:valid?) end - private_class_method :gemspec_stubs_in - - def self.installed_stubs(dirs, pattern) - map_stubs(dirs, pattern) do |path, base_dir, gems_dir| - Gem::StubSpecification.gemspec_stub(path, base_dir, gems_dir) - end - end - private_class_method :installed_stubs - - def self.map_stubs(dirs, pattern) # :nodoc: - dirs.flat_map do |dir| - base_dir = File.dirname dir - gems_dir = File.join base_dir, "gems" - gemspec_stubs_in(dir, pattern) {|path| yield path, base_dir, gems_dir } - end - end - private_class_method :map_stubs def self.each_spec(dirs) # :nodoc: each_gemspec(dirs) do |path| @@ -820,13 +794,7 @@ class Gem::Specification < Gem::BasicSpecification # Returns a Gem::StubSpecification for every installed gem def self.stubs - @@stubs ||= begin - pattern = "*.gemspec" - stubs = stubs_for_pattern(pattern, false) - - @@stubs_by_name = stubs.select {|s| Gem::Platform.match_spec? s }.group_by(&:name) - stubs - end + specification_record.stubs end ## @@ -845,13 +813,7 @@ class Gem::Specification < Gem::BasicSpecification # only returns stubs that match Gem.platforms def self.stubs_for(name) - if @@stubs - @@stubs_by_name[name] || [] - else - @@stubs_by_name[name] ||= stubs_for_pattern("#{name}-*.gemspec").select do |s| - s.name == name - end - end + specification_record.stubs_for(name) end ## @@ -859,12 +821,7 @@ class Gem::Specification < Gem::BasicSpecification # optionally filtering out specs not matching the current platform # def self.stubs_for_pattern(pattern, match_platform = true) # :nodoc: - installed_stubs = installed_stubs(Gem::Specification.dirs, pattern) - installed_stubs.select! {|s| Gem::Platform.match_spec? s } if match_platform - stubs = installed_stubs + default_stubs(pattern) - stubs = stubs.uniq(&:full_name) - _resort!(stubs) - stubs + specification_record.stubs_for_pattern(pattern, match_platform) end def self._resort!(specs) # :nodoc: @@ -873,7 +830,9 @@ class Gem::Specification < Gem::BasicSpecification next names if names.nonzero? versions = b.version <=> a.version next versions if versions.nonzero? - Gem::Platform.sort_priority(b.platform) + platforms = Gem::Platform.sort_priority(b.platform) <=> Gem::Platform.sort_priority(a.platform) + next platforms if platforms.nonzero? + b.base_dir == Gem.path.first ? 1 : -1 end end @@ -893,23 +852,14 @@ class Gem::Specification < Gem::BasicSpecification # properly sorted. def self.add_spec(spec) - return if _all.include? spec - - _all << spec - stubs << spec - (@@stubs_by_name[spec.name] ||= []) << spec - - _resort!(@@stubs_by_name[spec.name]) - _resort!(stubs) + specification_record.add_spec(spec) end ## # Removes +spec+ from the known specs. def self.remove_spec(spec) - _all.delete spec.to_spec - stubs.delete spec - (@@stubs_by_name[spec.name] || []).delete spec + specification_record.remove_spec(spec) end ## @@ -923,27 +873,17 @@ class Gem::Specification < Gem::BasicSpecification end ## - # Sets the known specs to +specs+. Not guaranteed to work for you in - # the future. Use at your own risk. Caveat emptor. Doomy doom doom. - # Etc etc. - # - #-- - # Makes +specs+ the known specs - # Listen, time is a river - # Winter comes, code breaks - # - # -- wilsonb + # Sets the known specs to +specs+. def self.all=(specs) - @@stubs_by_name = specs.group_by(&:name) - @@all = @@stubs = specs + specification_record.all = specs end ## # Return full names of all specs in sorted order. def self.all_names - _all.map(&:full_name) + specification_record.all_names end ## @@ -968,9 +908,7 @@ class Gem::Specification < Gem::BasicSpecification # Return the directories that Specification uses to find specs. def self.dirs - @@dirs ||= Gem.path.collect do |dir| - File.join dir, "specifications" - end + @@dirs ||= Gem::SpecificationRecord.dirs_from(Gem.path) end ## @@ -980,7 +918,7 @@ class Gem::Specification < Gem::BasicSpecification def self.dirs=(dirs) reset - @@dirs = Array(dirs).map {|dir| File.join dir, "specifications" } + @@dirs = Gem::SpecificationRecord.dirs_from(Array(dirs)) end extend Enumerable @@ -989,21 +927,15 @@ class Gem::Specification < Gem::BasicSpecification # Enumerate every known spec. See ::dirs= and ::add_spec to set the list of # specs. - def self.each - return enum_for(:each) unless block_given? - - _all.each do |x| - yield x - end + def self.each(&block) + specification_record.each(&block) end ## # Returns every spec that matches +name+ and optional +requirements+. def self.find_all_by_name(name, *requirements) - requirements = Gem::Requirement.default if requirements.empty? - - Gem::Dependency.new(name, *requirements).matching_specs + specification_record.find_all_by_name(name, *requirements) end ## @@ -1033,12 +965,7 @@ class Gem::Specification < Gem::BasicSpecification # Return the best specification that contains the file matching +path+. def self.find_by_path(path) - path = path.dup.freeze - spec = @@spec_with_requirable_file[path] ||= stubs.find do |s| - s.contains_requirable_file? path - end || NOT_FOUND - - spec.to_spec + specification_record.find_by_path(path) end ## @@ -1046,19 +973,15 @@ class Gem::Specification < Gem::BasicSpecification # amongst the specs that are not activated. def self.find_inactive_by_path(path) - stub = stubs.find do |s| - next if s.activated? - s.contains_requirable_file? path - end - stub&.to_spec + specification_record.find_inactive_by_path(path) end - def self.find_active_stub_by_path(path) - stub = @@active_stub_with_requirable_file[path] ||= stubs.find do |s| - s.activated? && s.contains_requirable_file?(path) - end || NOT_FOUND + ## + # Return the best specification that contains the file matching +path+, among + # those already activated. - stub.this + def self.find_active_stub_by_path(path) + specification_record.find_active_stub_by_path(path) end ## @@ -1125,14 +1048,14 @@ class Gem::Specification < Gem::BasicSpecification # +prerelease+ is true. def self.latest_specs(prerelease = false) - _latest_specs Gem::Specification.stubs, prerelease + specification_record.latest_specs(prerelease) end ## # Return the latest installed spec for gem +name+. def self.latest_spec_for(name) - latest_specs(true).find {|installed_spec| installed_spec.name == name } + specification_record.latest_spec_for(name) end def self._latest_specs(specs, prerelease = false) # :nodoc: @@ -1270,7 +1193,7 @@ class Gem::Specification < Gem::BasicSpecification def self.reset @@dirs = nil Gem.pre_reset_hooks.each(&:call) - clear_specs + @specification_record = nil clear_load_cache unresolved = unresolved_deps unless unresolved.empty? @@ -1291,6 +1214,13 @@ class Gem::Specification < Gem::BasicSpecification Gem.post_reset_hooks.each(&:call) end + ## + # Keeps track of all currently known specifications + + def self.specification_record + @specification_record ||= Gem::SpecificationRecord.new(dirs) + end + # DOC: This method needs documented or nodoc'd def self.unresolved_deps @unresolved_deps ||= Hash.new {|h, n| h[n] = Gem::Dependency.new n } diff --git a/lib/rubygems/specification_record.rb b/lib/rubygems/specification_record.rb new file mode 100644 index 0000000000..664d506265 --- /dev/null +++ b/lib/rubygems/specification_record.rb @@ -0,0 +1,212 @@ +# frozen_string_literal: true + +module Gem + class SpecificationRecord + def self.dirs_from(paths) + paths.map do |path| + File.join(path, "specifications") + end + end + + def self.from_path(path) + new(dirs_from([path])) + end + + def initialize(dirs) + @all = nil + @stubs = nil + @stubs_by_name = {} + @spec_with_requirable_file = {} + @active_stub_with_requirable_file = {} + + @dirs = dirs + end + + # Sentinel object to represent "not found" stubs + NOT_FOUND = Struct.new(:to_spec, :this).new + private_constant :NOT_FOUND + + ## + # Returns the list of all specifications in the record + + def all + @all ||= Gem.loaded_specs.values | stubs.map(&:to_spec) + end + + ## + # Returns a Gem::StubSpecification for every specification in the record + + def stubs + @stubs ||= begin + pattern = "*.gemspec" + stubs = stubs_for_pattern(pattern, false) + + @stubs_by_name = stubs.select {|s| Gem::Platform.match_spec? s }.group_by(&:name) + stubs + end + end + + ## + # Returns a Gem::StubSpecification for every specification in the record + # named +name+ only returns stubs that match Gem.platforms + + def stubs_for(name) + if @stubs + @stubs_by_name[name] || [] + else + @stubs_by_name[name] ||= stubs_for_pattern("#{name}-*.gemspec").select do |s| + s.name == name + end + end + end + + ## + # Finds stub specifications matching a pattern in the record, optionally + # filtering out specs not matching the current platform + + def stubs_for_pattern(pattern, match_platform = true) + installed_stubs = installed_stubs(pattern) + installed_stubs.select! {|s| Gem::Platform.match_spec? s } if match_platform + stubs = installed_stubs + Gem::Specification.default_stubs(pattern) + Gem::Specification._resort!(stubs) + stubs + end + + ## + # Adds +spec+ to the the record, keeping the collection properly sorted. + + def add_spec(spec) + return if all.include? spec + + all << spec + stubs << spec + (@stubs_by_name[spec.name] ||= []) << spec + + Gem::Specification._resort!(@stubs_by_name[spec.name]) + Gem::Specification._resort!(stubs) + end + + ## + # Removes +spec+ from the record. + + def remove_spec(spec) + all.delete spec.to_spec + stubs.delete spec + (@stubs_by_name[spec.name] || []).delete spec + end + + ## + # Sets the specs known by the record to +specs+. + + def all=(specs) + @stubs_by_name = specs.group_by(&:name) + @all = @stubs = specs + end + + ## + # Return full names of all specs in the record in sorted order. + + def all_names + all.map(&:full_name) + end + + include Enumerable + + ## + # Enumerate every known spec. + + def each + return enum_for(:each) unless block_given? + + all.each do |x| + yield x + end + end + + ## + # Returns every spec in the record that matches +name+ and optional +requirements+. + + def find_all_by_name(name, *requirements) + req = Gem::Requirement.create(*requirements) + env_req = Gem.env_requirement(name) + + matches = stubs_for(name).find_all do |spec| + req.satisfied_by?(spec.version) && env_req.satisfied_by?(spec.version) + end.map(&:to_spec) + + if name == "bundler" && !req.specific? + require_relative "bundler_version_finder" + Gem::BundlerVersionFinder.prioritize!(matches) + end + + matches + end + + ## + # Return the best specification in the record that contains the file matching +path+. + + def find_by_path(path) + path = path.dup.freeze + spec = @spec_with_requirable_file[path] ||= stubs.find do |s| + s.contains_requirable_file? path + end || NOT_FOUND + + spec.to_spec + end + + ## + # Return the best specification in the record that contains the file + # matching +path+ amongst the specs that are not activated. + + def find_inactive_by_path(path) + stub = stubs.find do |s| + next if s.activated? + s.contains_requirable_file? path + end + stub&.to_spec + end + + ## + # Return the best specification in the record that contains the file + # matching +path+, among those already activated. + + def find_active_stub_by_path(path) + stub = @active_stub_with_requirable_file[path] ||= stubs.find do |s| + s.activated? && s.contains_requirable_file?(path) + end || NOT_FOUND + + stub.this + end + + ## + # Return the latest specs in the record, optionally including prerelease + # specs if +prerelease+ is true. + + def latest_specs(prerelease) + Gem::Specification._latest_specs stubs, prerelease + end + + ## + # Return the latest installed spec in the record for gem +name+. + + def latest_spec_for(name) + latest_specs(true).find {|installed_spec| installed_spec.name == name } + end + + private + + def installed_stubs(pattern) + map_stubs(pattern) do |path, base_dir, gems_dir| + Gem::StubSpecification.gemspec_stub(path, base_dir, gems_dir) + end + end + + def map_stubs(pattern) + @dirs.flat_map do |dir| + base_dir = File.dirname dir + gems_dir = File.join base_dir, "gems" + Gem::Specification.gemspec_stubs_in(dir, pattern) {|path| yield path, base_dir, gems_dir } + end + end + end +end diff --git a/lib/rubygems/stub_specification.rb b/lib/rubygems/stub_specification.rb index 58748df5d6..ea66fbc3f6 100644 --- a/lib/rubygems/stub_specification.rb +++ b/lib/rubygems/stub_specification.rb @@ -210,4 +210,25 @@ class Gem::StubSpecification < Gem::BasicSpecification def stubbed? data.is_a? StubLine end + + def ==(other) # :nodoc: + self.class === other && + name == other.name && + version == other.version && + platform == other.platform + end + + alias_method :eql?, :== # :nodoc: + + def hash # :nodoc: + name.hash ^ version.hash ^ platform.hash + end + + def <=>(other) # :nodoc: + sort_obj <=> other.sort_obj + end + + def sort_obj # :nodoc: + [name, version, Gem::Platform.sort_priority(platform)] + end end diff --git a/lib/rubygems/uninstaller.rb b/lib/rubygems/uninstaller.rb index c96df2a085..23791313c8 100644 --- a/lib/rubygems/uninstaller.rb +++ b/lib/rubygems/uninstaller.rb @@ -32,7 +32,7 @@ class Gem::Uninstaller attr_reader :bin_dir ## - # The gem repository the gem will be installed into + # The gem repository the gem will be uninstalled from attr_reader :gem_home @@ -49,8 +49,9 @@ class Gem::Uninstaller # TODO: document the valid options @gem = gem @version = options[:version] || Gem::Requirement.default - @gem_home = File.realpath(options[:install_dir] || Gem.dir) - @plugins_dir = Gem.plugindir(@gem_home) + @install_dir = options[:install_dir] + @gem_home = File.realpath(@install_dir || Gem.dir) + @user_dir = File.exist?(Gem.user_dir) ? File.realpath(Gem.user_dir) : Gem.user_dir @force_executables = options[:executables] @force_all = options[:all] @force_ignore = options[:ignore] @@ -70,7 +71,7 @@ class Gem::Uninstaller # only add user directory if install_dir is not set @user_install = false - @user_install = options[:user_install] unless options[:install_dir] + @user_install = options[:user_install] unless @install_dir # Optimization: populated during #uninstall @default_specs_matching_uninstall_params = [] @@ -85,11 +86,7 @@ class Gem::Uninstaller list = [] - dirs = - Gem::Specification.dirs + - [Gem.default_specifications_dir] - - Gem::Specification.each_spec dirs do |spec| + specification_record.stubs.each do |spec| next unless dependency.matches_spec? spec list << spec @@ -101,11 +98,11 @@ class Gem::Uninstaller default_specs, list = list.partition(&:default_gem?) warn_cannot_uninstall_default_gems(default_specs - list) - @default_specs_matching_uninstall_params = default_specs + @default_specs_matching_uninstall_params = default_specs.map(&:to_spec) list, other_repo_specs = list.partition do |spec| @gem_home == spec.base_dir || - (@user_install && spec.base_dir == Gem.user_dir) + (@user_install && spec.base_dir == @user_dir) end list.sort! @@ -125,7 +122,7 @@ class Gem::Uninstaller remove_all list elsif list.size > 1 - gem_names = list.map(&:full_name) + gem_names = list.map(&:full_name_with_location) gem_names << "All versions" say @@ -146,7 +143,9 @@ class Gem::Uninstaller ## # Uninstalls gem +spec+ - def uninstall_gem(spec) + def uninstall_gem(stub) + spec = stub.to_spec + @spec = spec unless dependencies_ok? spec @@ -164,6 +163,8 @@ class Gem::Uninstaller remove_plugins @spec remove @spec + specification_record.remove_spec(stub) + regenerate_plugins Gem.post_uninstall_hooks.each do |hook| @@ -239,7 +240,7 @@ class Gem::Uninstaller def remove(spec) unless path_ok?(@gem_home, spec) || - (@user_install && path_ok?(Gem.user_dir, spec)) + (@user_install && path_ok?(@user_dir, spec)) e = Gem::GemNotInHomeException.new \ "Gem '#{spec.full_name}' is not installed in directory #{@gem_home}" e.spec = spec @@ -274,8 +275,6 @@ class Gem::Uninstaller safe_delete { FileUtils.rm_r gemspec } announce_deletion_of(spec) - - Gem::Specification.reset end ## @@ -284,17 +283,17 @@ class Gem::Uninstaller def remove_plugins(spec) # :nodoc: return if spec.plugins.empty? - remove_plugins_for(spec, @plugins_dir) + remove_plugins_for(spec, plugin_dir_for(spec)) end ## # Regenerates plugin wrappers after removal. def regenerate_plugins - latest = Gem::Specification.latest_spec_for(@spec.name) + latest = specification_record.latest_spec_for(@spec.name) return if latest.nil? - regenerate_plugins_for(latest, @plugins_dir) + regenerate_plugins_for(latest, plugin_dir_for(@spec)) end ## @@ -379,6 +378,10 @@ class Gem::Uninstaller private + def specification_record + @specification_record ||= @install_dir ? Gem::SpecificationRecord.from_path(@install_dir) : Gem::Specification.specification_record + end + def announce_deletion_of(spec) name = spec.full_name say "Successfully uninstalled #{name}" @@ -406,4 +409,8 @@ class Gem::Uninstaller say "Gem #{spec.full_name} cannot be uninstalled because it is a default gem" end end + + def plugin_dir_for(spec) + Gem.plugindir(spec.base_dir) + end end diff --git a/lib/rubygems/util/licenses.rb b/lib/rubygems/util/licenses.rb index f3c7201639..192ae30b9b 100644 --- a/lib/rubygems/util/licenses.rb +++ b/lib/rubygems/util/licenses.rb @@ -15,6 +15,7 @@ class Gem::Licenses # license identifiers LICENSE_IDENTIFIERS = %w[ 0BSD + 3D-Slicer-1.0 AAL ADSL AFL-1.1 @@ -26,6 +27,7 @@ class Gem::Licenses AGPL-1.0-or-later AGPL-3.0-only AGPL-3.0-or-later + AMD-newlib AMDPLPA AML AML-glslang @@ -62,6 +64,7 @@ class Gem::Licenses BSD-2-Clause-Darwin BSD-2-Clause-Patent BSD-2-Clause-Views + BSD-2-Clause-first-lines BSD-3-Clause BSD-3-Clause-Attribution BSD-3-Clause-Clear @@ -191,6 +194,7 @@ class Gem::Licenses CUA-OPL-1.0 Caldera Caldera-no-preamble + Catharon ClArtistic Clips Community-Spec-1.0 @@ -270,25 +274,32 @@ class Gem::Licenses Glide Glulxe Graphics-Gems + Gutmann HP-1986 HP-1989 HPND HPND-DEC HPND-Fenneberg-Livingston HPND-INRIA-IMAG + HPND-Intel HPND-Kevlin-Henney HPND-MIT-disclaimer HPND-Markus-Kuhn HPND-Pbmplus HPND-UC + HPND-UC-export-US HPND-doc HPND-doc-sell HPND-export-US + HPND-export-US-acknowledgement HPND-export-US-modify + HPND-export2-US + HPND-merchantability-variant HPND-sell-MIT-disclaimer-xserver HPND-sell-regexpr HPND-sell-variant HPND-sell-variant-MIT-disclaimer + HPND-sell-variant-MIT-disclaimer-rev HTMLTIDY HaskellReport Hippocratic-2.1 @@ -353,6 +364,7 @@ class Gem::Licenses MIT-0 MIT-CMU MIT-Festival + MIT-Khronos-old MIT-Modern-Variant MIT-Wu MIT-advertising @@ -386,7 +398,9 @@ class Gem::Licenses NAIST-2003 NASA-1.3 NBPL-1.0 + NCBI-PD NCGL-UK-2.0 + NCL NCSA NGPL NICTA-1.0 @@ -410,6 +424,7 @@ class Gem::Licenses Nokia Noweb O-UDA-1.0 + OAR OCCT-PL OCLC-2.0 ODC-By-1.0 @@ -463,6 +478,7 @@ class Gem::Licenses PDDL-1.0 PHP-3.0 PHP-3.01 + PPL PSF-2.0 Parity-6.0.0 Parity-7.0.0 @@ -518,6 +534,7 @@ class Gem::Licenses Spencer-99 SugarCRM-1.1.3 Sun-PPP + Sun-PPP-2000 SunPro Symlinks TAPR-OHL-1.0 @@ -574,6 +591,7 @@ class Gem::Licenses Zimbra-1.3 Zimbra-1.4 Zlib + any-OSI bcrypt-Solar-Designer blessing bzip2-1.0.6 @@ -582,6 +600,7 @@ class Gem::Licenses copyleft-next-0.3.0 copyleft-next-0.3.1 curl + cve-tou diffmark dtoa dvipdfm @@ -604,6 +623,7 @@ class Gem::Licenses mpi-permissive mpich2 mplus + pkgconf pnmstitch psfrag psutils @@ -613,12 +633,14 @@ class Gem::Licenses softSurfer ssh-keyscan swrule + threeparttable ulem w3m xinetd xkeyboard-config-Zinoviev xlock xpp + xzoom zlib-acknowledgement ].freeze @@ -660,6 +682,7 @@ class Gem::Licenses EXCEPTION_IDENTIFIERS = %w[ 389-exception Asterisk-exception + Asterisk-linking-protocols-exception Autoconf-exception-2.0 Autoconf-exception-3.0 Autoconf-exception-generic @@ -697,11 +720,13 @@ class Gem::Licenses OCCT-exception-1.0 OCaml-LGPL-linking-exception OpenJDK-assembly-exception-1.0 + PCRE2-exception PS-or-PDF-font-exception-20170817 QPL-1.0-INRIA-2004-exception Qt-GPL-exception-1.0 Qt-LGPL-exception-1.1 Qwt-exception-1.0 + RRDtool-FLOSS-exception-2.0 SANE-exception SHL-2.0 SHL-2.1 diff --git a/lib/rubygems/yaml_serializer.rb b/lib/rubygems/yaml_serializer.rb index 128becc1ce..af86c63ef7 100644 --- a/lib/rubygems/yaml_serializer.rb +++ b/lib/rubygems/yaml_serializer.rb @@ -60,7 +60,6 @@ module Gem indent, key, quote, val = match.captures val = strip_comment(val) - convert_to_backward_compatible_key!(key) depth = indent.size / 2 if quote.empty? && val.empty? new_hash = {} @@ -92,14 +91,8 @@ module Gem end end - # for settings' keys - def convert_to_backward_compatible_key!(key) - key << "/" if /https?:/i.match?(key) && !%r{/\Z}.match?(key) - key.gsub!(".", "__") - end - class << self - private :dump_hash, :convert_to_backward_compatible_key! + private :dump_hash end end end diff --git a/lib/tempfile.rb b/lib/tempfile.rb index 1d7b80a74d..bddef3bf22 100644 --- a/lib/tempfile.rb +++ b/lib/tempfile.rb @@ -8,18 +8,61 @@ require 'delegate' require 'tmpdir' -# A utility class for managing temporary files. When you create a Tempfile -# object, it will create a temporary file with a unique filename. A Tempfile -# objects behaves just like a File object, and you can perform all the usual -# file operations on it: reading data, writing data, changing its permissions, -# etc. So although this class does not explicitly document all instance methods -# supported by File, you can in fact call any File instance method on a -# Tempfile object. +# A utility class for managing temporary files. +# +# There are two kind of methods of creating a temporary file: +# +# - Tempfile.create (recommended) +# - Tempfile.new and Tempfile.open (mostly for backward compatibility, not recommended) +# +# Tempfile.create creates a usual \File object. +# The timing of file deletion is predictable. +# Also, it supports open-and-unlink technique which +# removes the temporary file immediately after creation. +# +# Tempfile.new and Tempfile.open creates a \Tempfile object. +# The created file is removed by the GC (finalizer). +# The timing of file deletion is not predictable. # # == Synopsis # # require 'tempfile' # +# # Tempfile.create with a block +# # The filename are choosen automatically. +# # (You can specify the prefix and suffix of the filename by an optional argument.) +# Tempfile.create {|f| +# f.puts "foo" +# f.rewind +# f.read # => "foo\n" +# } # The file is removed at block exit. +# +# # Tempfile.create without a block +# # You need to unlink the file in non-block form. +# f = Tempfile.create +# f.puts "foo" +# f.close +# File.unlink(f.path) # You need to unlink the file. +# +# # Tempfile.create(anonymous: true) without a block +# f = Tempfile.create(anonymous: true) +# # The file is already removed because anonymous. +# f.path # => "/tmp/" (no filename since no file) +# f.puts "foo" +# f.rewind +# f.read # => "foo\n" +# f.close +# +# # Tempfile.create(anonymous: true) with a block +# Tempfile.create(anonymous: true) {|f| +# # The file is already removed because anonymous. +# f.path # => "/tmp/" (no filename since no file) +# f.puts "foo" +# f.rewind +# f.read # => "foo\n" +# } +# +# # Not recommended: Tempfile.new without a block # file = Tempfile.new('foo') # file.path # => A unique filename in the OS's temp directory, # # e.g.: "/tmp/foo.24722.0" @@ -30,7 +73,27 @@ require 'tmpdir' # file.close # file.unlink # deletes the temp file # -# == Good practices +# == About Tempfile.new and Tempfile.open +# +# This section does not apply to Tempfile.create because +# it returns a File object (not a Tempfile object). +# +# When you create a Tempfile object, +# it will create a temporary file with a unique filename. A Tempfile +# objects behaves just like a File object, and you can perform all the usual +# file operations on it: reading data, writing data, changing its permissions, +# etc. So although this class does not explicitly document all instance methods +# supported by File, you can in fact call any File instance method on a +# Tempfile object. +# +# A Tempfile object has a finalizer to remove the temporary file. +# This means that the temporary file is removed via GC. +# This can cause several problems: +# +# - Long GC intervals and conservative GC can accumulate temporary files that are not removed. +# - Temporary files are not removed if Ruby exits abnormally (such as SIGKILL, SEGV). +# +# There are legacy good practices for Tempfile.new and Tempfile.open as follows. # # === Explicit close # @@ -71,12 +134,17 @@ require 'tmpdir' # be able to read from or write to the Tempfile, and you do not need to # know the Tempfile's filename either. # +# Also, this guarantees the temporary file is removed even if Ruby exits abnormally. +# The OS reclaims the storage for the temporary file when the file is closed or +# the Ruby process exits (normally or abnormally). +# # For example, a practical use case for unlink-after-creation would be this: # you need a large byte buffer that's too large to comfortably fit in RAM, # e.g. when you're writing a web server and you want to buffer the client's # file upload data. # -# Please refer to #unlink for more information and a code example. +# `Tempfile.create(anonymous: true)` supports this behavior. +# It also works on Windows. # # == Minor notes # @@ -392,8 +460,9 @@ end # see {File Permissions}[rdoc-ref:File@File+Permissions]. # - Mode is <tt>'w+'</tt> (read/write mode, positioned at the end). # -# With no block, the file is not removed automatically, -# and so should be explicitly removed. +# The temporary file removal depends on the keyword argument +anonymous+ and +# whether a block is given or not. +# See the description about the +anonymous+ keyword argument later. # # Example: # @@ -401,11 +470,36 @@ end # f.class # => File # f.path # => "/tmp/20220505-9795-17ky6f6" # f.stat.mode.to_s(8) # => "100600" +# f.close # File.exist?(f.path) # => true # File.unlink(f.path) # File.exist?(f.path) # => false # -# Argument +basename+, if given, may be one of: +# Tempfile.create {|f| +# f.puts "foo" +# f.rewind +# f.read # => "foo\n" +# f.path # => "/tmp/20240524-380207-oma0ny" +# File.exist?(f.path) # => true +# } # The file is removed at block exit. +# +# f = Tempfile.create(anonymous: true) +# # The file is already removed because anonymous +# f.path # => "/tmp/" (no filename since no file) +# f.puts "foo" +# f.rewind +# f.read # => "foo\n" +# f.close +# +# Tempfile.create(anonymous: true) {|f| +# # The file is already removed because anonymous +# f.path # => "/tmp/" (no filename since no file) +# f.puts "foo" +# f.rewind +# f.read # => "foo\n" +# } +# +# The argument +basename+, if given, may be one of the following: # # - A string: the generated filename begins with +basename+: # @@ -416,27 +510,57 @@ end # # Tempfile.create(%w/foo .jpg/) # => #<File:/tmp/foo20220505-17839-tnjchh.jpg> # -# With arguments +basename+ and +tmpdir+, the file is created in directory +tmpdir+: +# With arguments +basename+ and +tmpdir+, the file is created in the directory +tmpdir+: # # Tempfile.create('foo', '.') # => #<File:./foo20220505-9795-1emu6g8> # -# Keyword arguments +mode+ and +options+ are passed directly to method +# Keyword arguments +mode+ and +options+ are passed directly to the method # {File.open}[rdoc-ref:File.open]: # -# - The value given with +mode+ must be an integer, +# - The value given for +mode+ must be an integer # and may be expressed as the logical OR of constants defined in # {File::Constants}[rdoc-ref:File::Constants]. # - For +options+, see {Open Options}[rdoc-ref:IO@Open+Options]. # -# With a block given, creates the file as above, passes it to the block, -# and returns the block's value; -# before the return, the file object is closed and the underlying file is removed: +# The keyword argument +anonymous+ specifies when the file is removed. +# +# - +anonymous=false+ (default) without a block: the file is not removed. +# - +anonymous=false+ (default) with a block: the file is removed after the block exits. +# - +anonymous=true+ without a block: the file is removed before returning. +# - +anonymous=true+ with a block: the file is removed before the block is called. +# +# In the first case (+anonymous=false+ without a block), +# the file is not removed automatically. +# It should be explicitly closed. +# It can be used to rename to the desired filename. +# If the file is not needed, it should be explicitly removed. +# +# The +File#path+ method of the created file object returns the temporary directory with a trailing slash +# when +anonymous+ is true. +# +# When a block is given, it creates the file as described above, passes it to the block, +# and returns the block's value. +# Before the returning, the file object is closed and the underlying file is removed: # # Tempfile.create {|file| file.path } # => "/tmp/20220505-9795-rkists" # +# Implementation note: +# +# The keyword argument +anonymous=true+ is implemented using FILE_SHARE_DELETE on Windows. +# O_TMPFILE is used on Linux. +# # Related: Tempfile.new. # -def Tempfile.create(basename="", tmpdir=nil, mode: 0, **options) +def Tempfile.create(basename="", tmpdir=nil, mode: 0, anonymous: false, **options, &block) + if anonymous + create_anonymous(basename, tmpdir, mode: mode, **options, &block) + else + create_with_filename(basename, tmpdir, mode: mode, **options, &block) + end +end + +class << Tempfile +private def create_with_filename(basename="", tmpdir=nil, mode: 0, **options) tmpfile = nil Dir::Tmpname.create(basename, tmpdir, **options) do |tmpname, n, opts| mode |= File::RDWR|File::CREAT|File::EXCL @@ -464,3 +588,37 @@ def Tempfile.create(basename="", tmpdir=nil, mode: 0, **options) tmpfile end end + +private def create_anonymous(basename="", tmpdir=nil, mode: 0, **options, &block) + tmpfile = nil + tmpdir = Dir.tmpdir() if tmpdir.nil? + if defined?(File::TMPFILE) # O_TMPFILE since Linux 3.11 + begin + tmpfile = File.open(tmpdir, File::RDWR | File::TMPFILE, 0600) + rescue Errno::EISDIR, Errno::ENOENT, Errno::EOPNOTSUPP + # kernel or the filesystem does not support O_TMPFILE + # fallback to create-and-unlink + end + end + if tmpfile.nil? + mode |= File::SHARE_DELETE | File::BINARY # Windows needs them to unlink the opened file. + tmpfile = create_with_filename(basename, tmpdir, mode: mode, **options) + File.unlink(tmpfile.path) + end + path = File.join(tmpdir, '') + if tmpfile.path != path + # clear path. + tmpfile.autoclose = false + tmpfile = File.new(tmpfile.fileno, mode: File::RDWR, path: path) + end + if block + begin + yield tmpfile + ensure + tmpfile.close + end + else + tmpfile + end +end +end |