diff options
author | Hiroshi SHIBATA <hsbt@ruby-lang.org> | 2022-07-13 14:44:32 +0900 |
---|---|---|
committer | nagachika <nagachika@ruby-lang.org> | 2022-09-03 15:54:07 +0900 |
commit | a01f5ad1ec2455e97e27eb2758588ff5e63c4131 (patch) | |
tree | cd3fd955c118214e37017265687c48d231ae0cce /lib | |
parent | b9f6a09bd2127ea51612bd27bef5830831b48d4f (diff) |
Merge RubyGems-3.3.16 and Bundler-2.3.16
Diffstat (limited to 'lib')
34 files changed, 337 insertions, 293 deletions
diff --git a/lib/bundler/cli.rb b/lib/bundler/cli.rb index e1c284130b..3d93ce5e6f 100644 --- a/lib/bundler/cli.rb +++ b/lib/bundler/cli.rb @@ -251,9 +251,7 @@ module Bundler remembered_negative_flag_deprecation("no-deployment") require_relative "cli/install" - Bundler.settings.temporary(:no_install => false) do - Install.new(options.dup).run - end + Install.new(options.dup).run end map aliases_for("install") @@ -299,9 +297,7 @@ module Bundler def update(*gems) SharedHelpers.major_deprecation(2, "The `--force` option has been renamed to `--redownload`") if ARGV.include?("--force") require_relative "cli/update" - Bundler.settings.temporary(:no_install => false) do - Update.new(options, gems).run - end + Update.new(options, gems).run end desc "show GEM [OPTIONS]", "Shows all gems that are part of the bundle, or the path to a given gem" diff --git a/lib/bundler/cli/cache.rb b/lib/bundler/cli/cache.rb index c8698ed7e3..eb5dd23092 100644 --- a/lib/bundler/cli/cache.rb +++ b/lib/bundler/cli/cache.rb @@ -14,7 +14,7 @@ module Bundler Bundler.settings.set_command_option_if_given :cache_path, options["cache-path"] setup_cache_all - install + install unless Bundler.settings[:no_install] # TODO: move cache contents here now that all bundles are locked custom_path = Bundler.settings[:path] if options[:path] diff --git a/lib/bundler/cli/install.rb b/lib/bundler/cli/install.rb index e9b85f7f6f..acf92f28ad 100644 --- a/lib/bundler/cli/install.rb +++ b/lib/bundler/cli/install.rb @@ -161,8 +161,6 @@ module Bundler Bundler.settings.set_command_option_if_given :no_prune, options["no-prune"] - Bundler.settings.set_command_option_if_given :no_install, options["no-install"] - Bundler.settings.set_command_option_if_given :clean, options["clean"] normalize_groups if options[:without] || options[:with] diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb index 4fca763bcc..2e0f23a402 100644 --- a/lib/bundler/definition.rb +++ b/lib/bundler/definition.rb @@ -255,20 +255,18 @@ module Bundler # # @return [SpecSet] resolved dependencies def resolve - @resolve ||= begin - if Bundler.frozen_bundle? - Bundler.ui.debug "Frozen, using resolution from the lockfile" - @locked_specs - elsif !unlocking? && nothing_changed? - Bundler.ui.debug("Found no changes, using resolution from the lockfile") - SpecSet.new(filter_specs(@locked_specs, @dependencies.select {|dep| @locked_specs[dep].any? })) - else - last_resolve = converge_locked_specs - # Run a resolve against the locally available gems - Bundler.ui.debug("Found changes from the lockfile, re-resolving dependencies because #{change_reason}") - expanded_dependencies = expand_dependencies(dependencies + metadata_dependencies, true) - Resolver.resolve(expanded_dependencies, source_requirements, last_resolve, gem_version_promoter, additional_base_requirements_for_resolve, platforms) - end + @resolve ||= if Bundler.frozen_bundle? + Bundler.ui.debug "Frozen, using resolution from the lockfile" + @locked_specs + elsif !unlocking? && nothing_changed? + Bundler.ui.debug("Found no changes, using resolution from the lockfile") + SpecSet.new(filter_specs(@locked_specs, @dependencies.select {|dep| @locked_specs[dep].any? })) + else + last_resolve = converge_locked_specs + # Run a resolve against the locally available gems + Bundler.ui.debug("Found changes from the lockfile, re-resolving dependencies because #{change_reason}") + expanded_dependencies = expand_dependencies(dependencies + metadata_dependencies, true) + Resolver.resolve(expanded_dependencies, source_requirements, last_resolve, gem_version_promoter, additional_base_requirements_for_resolve, platforms) end end @@ -735,12 +733,10 @@ module Bundler end def metadata_dependencies - @metadata_dependencies ||= begin - [ - Dependency.new("Ruby\0", RubyVersion.system.gem_version), - Dependency.new("RubyGems\0", Gem::VERSION), - ] - end + @metadata_dependencies ||= [ + Dependency.new("Ruby\0", RubyVersion.system.gem_version), + Dependency.new("RubyGems\0", Gem::VERSION), + ] end def expand_dependencies(dependencies, remote = false) diff --git a/lib/bundler/dependency.rb b/lib/bundler/dependency.rb index d12b120bba..018a3182b9 100644 --- a/lib/bundler/dependency.rb +++ b/lib/bundler/dependency.rb @@ -9,6 +9,7 @@ module Bundler attr_reader :autorequire attr_reader :groups, :platforms, :gemfile, :git, :github, :branch, :ref + # rubocop:disable Naming/VariableNumber PLATFORM_MAP = { :ruby => Gem::Platform::RUBY, :ruby_18 => Gem::Platform::RUBY, @@ -91,6 +92,7 @@ module Bundler :x64_mingw_30 => Gem::Platform::X64_MINGW, :x64_mingw_31 => Gem::Platform::X64_MINGW, }.freeze + # rubocop:enable Naming/VariableNumber def initialize(name, version, options = {}, &blk) type = options["type"] || :runtime diff --git a/lib/bundler/dsl.rb b/lib/bundler/dsl.rb index 1ae19a46e9..bfa078046c 100644 --- a/lib/bundler/dsl.rb +++ b/lib/bundler/dsl.rb @@ -511,9 +511,7 @@ module Bundler # be raised. # def contents - @contents ||= begin - dsl_path && File.exist?(dsl_path) && File.read(dsl_path) - end + @contents ||= dsl_path && File.exist?(dsl_path) && File.read(dsl_path) end # The message of the exception reports the content of podspec for the diff --git a/lib/bundler/errors.rb b/lib/bundler/errors.rb index 0bc1a860df..f10b6cc68f 100644 --- a/lib/bundler/errors.rb +++ b/lib/bundler/errors.rb @@ -41,12 +41,14 @@ module Bundler class GemspecError < BundlerError; status_code(14); end class InvalidOption < BundlerError; status_code(15); end class ProductionError < BundlerError; status_code(16); end + class HTTPError < BundlerError status_code(17) def filter_uri(uri) URICredentialsFilter.credential_filtered_uri(uri) end end + class RubyVersionMismatch < BundlerError; status_code(18); end class SecurityError < BundlerError; status_code(19); end class LockfileError < BundlerError; status_code(20); end diff --git a/lib/bundler/fetcher.rb b/lib/bundler/fetcher.rb index 6fe047568f..e9d5dd505c 100644 --- a/lib/bundler/fetcher.rb +++ b/lib/bundler/fetcher.rb @@ -20,6 +20,7 @@ module Bundler class TooManyRequestsError < HTTPError; end # This error is raised if the API returns a 413 (only printed in verbose) class FallbackError < HTTPError; end + # This is the error raised if OpenSSL fails the cert verification class CertificateFailureError < HTTPError def initialize(remote_uri) @@ -33,6 +34,7 @@ module Bundler " sources and change 'https' to 'http'." end end + # This is the error raised when a source is HTTPS and OpenSSL didn't load class SSLError < HTTPError def initialize(msg = nil) @@ -42,6 +44,7 @@ module Bundler "using RVM are available at rvm.io/packages/openssl." end end + # This error is raised if HTTP authentication is required, but not provided. class AuthenticationRequiredError < HTTPError def initialize(remote_uri) @@ -52,6 +55,7 @@ module Bundler "or by storing the credentials in the `#{Settings.key_for(remote_uri)}` environment variable" end end + # This error is raised if HTTP authentication is provided, but incorrect. class BadAuthenticationError < HTTPError def initialize(remote_uri) diff --git a/lib/bundler/fetcher/base.rb b/lib/bundler/fetcher/base.rb index 16cc98273a..62cc75add8 100644 --- a/lib/bundler/fetcher/base.rb +++ b/lib/bundler/fetcher/base.rb @@ -19,14 +19,12 @@ module Bundler end def fetch_uri - @fetch_uri ||= begin - if remote_uri.host == "rubygems.org" - uri = remote_uri.dup - uri.host = "index.rubygems.org" - uri - else - remote_uri - end + @fetch_uri ||= if remote_uri.host == "rubygems.org" + uri = remote_uri.dup + uri.host = "index.rubygems.org" + uri + else + remote_uri end end diff --git a/lib/bundler/plugin/api/source.rb b/lib/bundler/plugin/api/source.rb index a6ae08237c..67c45bd204 100644 --- a/lib/bundler/plugin/api/source.rb +++ b/lib/bundler/plugin/api/source.rb @@ -258,7 +258,7 @@ module Bundler @dependencies |= Array(names) end - # Note: Do not override if you don't know what you are doing. + # NOTE: Do not override if you don't know what you are doing. def can_lock?(spec) spec.source == self end @@ -285,7 +285,7 @@ module Bundler end alias_method :identifier, :to_s - # Note: Do not override if you don't know what you are doing. + # NOTE: Do not override if you don't know what you are doing. def include?(other) other == self end @@ -294,7 +294,7 @@ module Bundler SharedHelpers.digest(:SHA1).hexdigest(uri) end - # Note: Do not override if you don't know what you are doing. + # NOTE: Do not override if you don't know what you are doing. def gem_install_dir Bundler.install_path end diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb index d749694952..18eb18160d 100644 --- a/lib/bundler/resolver.rb +++ b/lib/bundler/resolver.rb @@ -233,19 +233,17 @@ module Bundler # before dependencies that are unconstrained def amount_constrained(dependency) @amount_constrained ||= {} - @amount_constrained[dependency.name] ||= begin - if (base = @base[dependency.name]) && !base.empty? - dependency.requirement.satisfied_by?(base.first.version) ? 0 : 1 - else - all = index_for(dependency).search(dependency.name).size + @amount_constrained[dependency.name] ||= if (base = @base[dependency.name]) && !base.empty? + dependency.requirement.satisfied_by?(base.first.version) ? 0 : 1 + else + all = index_for(dependency).search(dependency.name).size - if all <= 1 - all - 1_000_000 - else - search = search_for(dependency) - search = @prerelease_specified[dependency.name] ? search.count : search.count {|s| !s.version.prerelease? } - search - all - end + if all <= 1 + all - 1_000_000 + else + search = search_for(dependency) + search = @prerelease_specified[dependency.name] ? search.count : search.count {|s| !s.version.prerelease? } + search - all end end end diff --git a/lib/bundler/rubygems_gem_installer.rb b/lib/bundler/rubygems_gem_installer.rb index 87b9772c27..df2dcdb454 100644 --- a/lib/bundler/rubygems_gem_installer.rb +++ b/lib/bundler/rubygems_gem_installer.rb @@ -90,6 +90,14 @@ module Bundler end end + def spec + if Bundler.rubygems.provides?("< 3.3.12") # RubyGems implementation rescues and re-raises errors before 3.3.12 and we don't want that + @package.spec + else + super + end + end + private def strict_rm_rf(dir) diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb index 1c2b374d8b..a6180d5160 100644 --- a/lib/bundler/rubygems_integration.rb +++ b/lib/bundler/rubygems_integration.rb @@ -203,20 +203,9 @@ module Bundler EXT_LOCK end - def spec_from_gem(path, policy = nil) - require "rubygems/security" - require "psych" - gem_from_path(path, security_policies[policy]).spec - rescue Exception, Gem::Exception, Gem::Security::Exception => e # rubocop:disable Lint/RescueException - if e.is_a?(Gem::Security::Exception) || - e.message =~ /unknown trust policy|unsigned gem/i || - e.message =~ /couldn't verify (meta)?data signature/i - raise SecurityError, - "The gem #{File.basename(path, ".gem")} can't be installed because " \ - "the security policy didn't allow it, with the message: #{e.message}" - else - raise e - end + def spec_from_gem(path) + require "rubygems/package" + Gem::Package.new(path).spec end def build_gem(gem_dir, spec) @@ -514,13 +503,6 @@ module Bundler Gem::RemoteFetcher.new(proxy) end - def gem_from_path(path, policy = nil) - require "rubygems/package" - p = Gem::Package.new(path) - p.security_policy = policy if policy - p - end - def build(spec, skip_validation = false) require "rubygems/package" Gem::Package.build(spec, skip_validation) diff --git a/lib/bundler/source/git.rb b/lib/bundler/source/git.rb index eb82544b86..ed66dcdc12 100644 --- a/lib/bundler/source/git.rb +++ b/lib/bundler/source/git.rb @@ -219,13 +219,11 @@ module Bundler # across different projects, this cache will be shared. # When using local git repos, this is set to the local repo. def cache_path - @cache_path ||= begin - if Bundler.requires_sudo? || Bundler.feature_flag.global_gem_cache? - Bundler.user_cache - else - Bundler.bundle_path.join("cache", "bundler") - end.join("git", git_scope) - end + @cache_path ||= if Bundler.requires_sudo? || Bundler.feature_flag.global_gem_cache? + Bundler.user_cache + else + Bundler.bundle_path.join("cache", "bundler") + end.join("git", git_scope) end def app_cache_dirname diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 5dceacbae4..f78e6a443b 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -139,13 +139,9 @@ module Bundler force = options[:force] ensure_builtin_gems_cached = options[:ensure_builtin_gems_cached] - if ensure_builtin_gems_cached && spec.default_gem? - if !cached_path(spec) - cached_built_in_gem(spec) unless spec.remote - force = true - else - spec.loaded_from = loaded_from(spec) - end + if ensure_builtin_gems_cached && spec.default_gem? && !cached_path(spec) + cached_built_in_gem(spec) unless spec.remote + force = true end if installed?(spec) && !force @@ -153,84 +149,90 @@ module Bundler return nil # no post-install message end - # Download the gem to get the spec, because some specs that are returned - # by rubygems.org are broken and wrong. if spec.remote # Check for this spec from other sources - uris = [spec.remote.anonymized_uri] - uris += remotes_for_spec(spec).map(&:anonymized_uri) - uris.uniq! + uris = [spec.remote, *remotes_for_spec(spec)].map(&:anonymized_uri).uniq Installer.ambiguous_gems << [spec.name, *uris] if uris.length > 1 path = fetch_gem(spec, options[:previous_spec]) - begin - s = Bundler.rubygems.spec_from_gem(path, Bundler.settings["trust-policy"]) - spec.__swap__(s) + else + path = cached_gem(spec) + raise GemNotFound, "Could not find #{spec.file_name} for installation" unless path + end + + if requires_sudo? + install_path = Bundler.tmp(spec.full_name) + bin_path = install_path.join("bin") + else + install_path = rubygems_dir + bin_path = Bundler.system_bindir + end + + Bundler.mkdir_p bin_path, :no_sudo => true unless spec.executables.empty? || Bundler.rubygems.provides?(">= 2.7.5") + + require_relative "../rubygems_gem_installer" + + installer = Bundler::RubyGemsGemInstaller.at( + path, + :security_policy => Bundler.rubygems.security_policies[Bundler.settings["trust-policy"]], + :install_dir => install_path.to_s, + :bin_dir => bin_path.to_s, + :ignore_dependencies => true, + :wrappers => true, + :env_shebang => true, + :build_args => options[:build_args], + :bundler_expected_checksum => spec.respond_to?(:checksum) && spec.checksum, + :bundler_extension_cache_path => extension_cache_path(spec) + ) + + if spec.remote + s = begin + installer.spec rescue Gem::Package::FormatError Bundler.rm_rf(path) raise + rescue Gem::Security::Exception => e + raise SecurityError, + "The gem #{File.basename(path, ".gem")} can't be installed because " \ + "the security policy didn't allow it, with the message: #{e.message}" end + + spec.__swap__(s) end - unless Bundler.settings[:no_install] - message = "Installing #{version_message(spec, options[:previous_spec])}" - message += " with native extensions" if spec.extensions.any? - Bundler.ui.confirm message + message = "Installing #{version_message(spec, options[:previous_spec])}" + message += " with native extensions" if spec.extensions.any? + Bundler.ui.confirm message - path = cached_gem(spec) - raise GemNotFound, "Could not find #{spec.file_name} for installation" unless path - if requires_sudo? - install_path = Bundler.tmp(spec.full_name) - bin_path = install_path.join("bin") - else - install_path = rubygems_dir - bin_path = Bundler.system_bindir - end + installed_spec = installer.install + + spec.full_gem_path = installed_spec.full_gem_path + spec.loaded_from = installed_spec.loaded_from - Bundler.mkdir_p bin_path, :no_sudo => true unless spec.executables.empty? || Bundler.rubygems.provides?(">= 2.7.5") - - require_relative "../rubygems_gem_installer" - - installed_spec = Bundler::RubyGemsGemInstaller.at( - path, - :install_dir => install_path.to_s, - :bin_dir => bin_path.to_s, - :ignore_dependencies => true, - :wrappers => true, - :env_shebang => true, - :build_args => options[:build_args], - :bundler_expected_checksum => spec.respond_to?(:checksum) && spec.checksum, - :bundler_extension_cache_path => extension_cache_path(spec) - ).install - spec.full_gem_path = installed_spec.full_gem_path - - # SUDO HAX - if requires_sudo? - Bundler.rubygems.repository_subdirectories.each do |name| - src = File.join(install_path, name, "*") - dst = File.join(rubygems_dir, name) - if name == "extensions" && Dir.glob(src).any? - src = File.join(src, "*/*") - ext_src = Dir.glob(src).first - ext_src.gsub!(src[0..-6], "") - dst = File.dirname(File.join(dst, ext_src)) - end - SharedHelpers.filesystem_access(dst) do |p| - Bundler.mkdir_p(p) - end - Bundler.sudo "cp -R #{src} #{dst}" if Dir[src].any? + # SUDO HAX + if requires_sudo? + Bundler.rubygems.repository_subdirectories.each do |name| + src = File.join(install_path, name, "*") + dst = File.join(rubygems_dir, name) + if name == "extensions" && Dir.glob(src).any? + src = File.join(src, "*/*") + ext_src = Dir.glob(src).first + ext_src.gsub!(src[0..-6], "") + dst = File.dirname(File.join(dst, ext_src)) end + SharedHelpers.filesystem_access(dst) do |p| + Bundler.mkdir_p(p) + end + Bundler.sudo "cp -R #{src} #{dst}" if Dir[src].any? + end - spec.executables.each do |exe| - SharedHelpers.filesystem_access(Bundler.system_bindir) do |p| - Bundler.mkdir_p(p) - end - Bundler.sudo "cp -R #{install_path}/bin/#{exe} #{Bundler.system_bindir}/" + spec.executables.each do |exe| + SharedHelpers.filesystem_access(Bundler.system_bindir) do |p| + Bundler.mkdir_p(p) end + Bundler.sudo "cp -R #{install_path}/bin/#{exe} #{Bundler.system_bindir}/" end - installed_spec.loaded_from = loaded_from(spec) end - spec.loaded_from = loaded_from(spec) spec.post_install_message ensure @@ -348,10 +350,6 @@ module Bundler end end - def loaded_from(spec) - "#{rubygems_dir}/specifications/#{spec.full_name}.gemspec" - end - def cached_gem(spec) if spec.default_gem? cached_built_in_gem(spec) @@ -364,10 +362,14 @@ module Bundler global_cache_path = download_cache_path(spec) @caches << global_cache_path if global_cache_path - possibilities = @caches.map {|p| "#{p}/#{spec.file_name}" } + possibilities = @caches.map {|p| package_path(p, spec) } possibilities.find {|p| File.exist?(p) } end + def package_path(cache_path, spec) + "#{cache_path}/#{spec.file_name}" + end + def normalize_uri(uri) uri = uri.to_s uri = "#{uri}/" unless uri =~ %r{/$} @@ -459,12 +461,11 @@ module Bundler end def fetch_gem(spec, previous_spec = nil) - return false unless spec.remote - spec.fetch_platform cache_path = download_cache_path(spec) || default_cache_path_for(rubygems_dir) - gem_path = "#{cache_path}/#{spec.file_name}" + gem_path = package_path(cache_path, spec) + return gem_path if File.exist?(gem_path) if requires_sudo? download_path = Bundler.tmp(spec.full_name) @@ -482,7 +483,7 @@ module Bundler SharedHelpers.filesystem_access(cache_path) do |p| Bundler.mkdir_p(p) end - Bundler.sudo "mv #{download_cache_path}/#{spec.file_name} #{gem_path}" + Bundler.sudo "mv #{package_path(download_cache_path, spec)} #{gem_path}" end gem_path diff --git a/lib/bundler/version.rb b/lib/bundler/version.rb index 9e8fa48a1d..a9a2934be8 100644 --- a/lib/bundler/version.rb +++ b/lib/bundler/version.rb @@ -1,7 +1,7 @@ # frozen_string_literal: false module Bundler - VERSION = "2.3.15".freeze + VERSION = "2.3.16".freeze def self.bundler_major_version @bundler_major_version ||= VERSION.split(".").first.to_i diff --git a/lib/rubygems.rb b/lib/rubygems.rb index 8c2c465ad4..1dd8e27486 100644 --- a/lib/rubygems.rb +++ b/lib/rubygems.rb @@ -8,7 +8,7 @@ require 'rbconfig' module Gem - VERSION = "3.3.15".freeze + VERSION = "3.3.16".freeze end # Must be first since it unloads the prelude from 1.9.2 @@ -607,7 +607,6 @@ An Array (#{env.inspect}) was passed in from #{caller[3]} return if @yaml_loaded require 'psych' - require_relative 'rubygems/psych_additions' require_relative 'rubygems/psych_tree' require_relative 'rubygems/safe_yaml' diff --git a/lib/rubygems/commands/install_command.rb b/lib/rubygems/commands/install_command.rb index adf2cdba84..87563accb0 100644 --- a/lib/rubygems/commands/install_command.rb +++ b/lib/rubygems/commands/install_command.rb @@ -261,7 +261,7 @@ You can use `i` command instead of `install`. return unless Gem::SourceFetchProblem === x require_relative "../uri" - msg = "Unable to pull data from '#{Gem::Uri.new(x.source.uri).redacted}': #{x.error.message}" + msg = "Unable to pull data from '#{Gem::Uri.redact(x.source.uri)}': #{x.error.message}" alert_warning msg end diff --git a/lib/rubygems/commands/sources_command.rb b/lib/rubygems/commands/sources_command.rb index 9e74f3c47d..35fba1bd04 100644 --- a/lib/rubygems/commands/sources_command.rb +++ b/lib/rubygems/commands/sources_command.rb @@ -62,7 +62,7 @@ class Gem::Commands::SourcesCommand < Gem::Command say "#{source_uri} is not a URI" terminate_interaction 1 rescue Gem::RemoteFetcher::FetchError => e - say "Error fetching #{source_uri}:\n\t#{e.message}" + say "Error fetching #{Gem::Uri.redact(source.uri)}:\n\t#{e.message}" terminate_interaction 1 end end diff --git a/lib/rubygems/core_ext/kernel_require.rb b/lib/rubygems/core_ext/kernel_require.rb index 4b867c55e9..23badd75d2 100644 --- a/lib/rubygems/core_ext/kernel_require.rb +++ b/lib/rubygems/core_ext/kernel_require.rb @@ -18,7 +18,7 @@ module Kernel end file = Gem::KERNEL_WARN_IGNORES_INTERNAL_ENTRIES ? "<internal:#{__FILE__}>" : __FILE__ - module_eval <<'RUBY', file, __LINE__ + 1 + module_eval <<'RUBY', file, __LINE__ + 1 # rubocop:disable Style/EvalWithLocation ## # When RubyGems is required, Kernel#require is replaced with our own which # is capable of loading gems on demand. diff --git a/lib/rubygems/errors.rb b/lib/rubygems/errors.rb index f115ce23d0..e6e222033d 100644 --- a/lib/rubygems/errors.rb +++ b/lib/rubygems/errors.rb @@ -168,7 +168,7 @@ module Gem # An English description of the error. def wordy - "Unable to download data from #{Gem::Uri.new(@source.uri).redacted} - #{@error.message}" + "Unable to download data from #{Gem::Uri.redact(@source.uri)} - #{@error.message}" end ## diff --git a/lib/rubygems/ext/builder.rb b/lib/rubygems/ext/builder.rb index d9aa68c067..e9986a3fc8 100644 --- a/lib/rubygems/ext/builder.rb +++ b/lib/rubygems/ext/builder.rb @@ -48,7 +48,7 @@ class Gem::Ext::Builder end end - def self.run(command, results, command_name = nil, dir = Dir.pwd) + def self.run(command, results, command_name = nil, dir = Dir.pwd, env = {}) verbose = Gem.configuration.really_verbose begin @@ -63,9 +63,9 @@ class Gem::Ext::Builder require "open3" # Set $SOURCE_DATE_EPOCH for the subprocess. - env = {'SOURCE_DATE_EPOCH' => Gem.source_date_epoch_string} + build_env = { 'SOURCE_DATE_EPOCH' => Gem.source_date_epoch_string }.merge(env) output, status = begin - Open3.capture2e(env, *command, :chdir => dir) + Open3.capture2e(build_env, *command, :chdir => dir) rescue => error raise Gem::InstallError, "#{command_name || class_name} failed#{error.message}" end diff --git a/lib/rubygems/ext/cargo_builder.rb b/lib/rubygems/ext/cargo_builder.rb index 4c16063224..8465e9ef05 100644 --- a/lib/rubygems/ext/cargo_builder.rb +++ b/lib/rubygems/ext/cargo_builder.rb @@ -4,48 +4,67 @@ # over the `cargo rustc` command which takes care of building Rust code in a way # that Ruby can use. class Gem::Ext::CargoBuilder < Gem::Ext::Builder - attr_reader :spec + attr_accessor :spec, :runner, :profile def initialize(spec) + require_relative "../command" + require_relative "cargo_builder/link_flag_converter" + @spec = spec + @runner = self.class.method(:run) + @profile = :release end def build(_extension, dest_path, results, args = [], lib_dir = nil, cargo_dir = Dir.pwd) - require "rubygems/command" require "fileutils" require "shellwords" build_crate(dest_path, results, args, cargo_dir) - ext_path = rename_cdylib_for_ruby_compatibility(dest_path) - finalize_directory(ext_path, dest_path, lib_dir, cargo_dir) + validate_cargo_build!(dest_path) + rename_cdylib_for_ruby_compatibility(dest_path) + finalize_directory(dest_path, lib_dir, cargo_dir) results end - private - def build_crate(dest_path, results, args, cargo_dir) - manifest = File.join(cargo_dir, "Cargo.toml") + env = build_env + cmd = cargo_command(cargo_dir, dest_path, args) + runner.call cmd, results, 'cargo', cargo_dir, env - given_ruby_static = ENV["RUBY_STATIC"] + results + end - ENV["RUBY_STATIC"] = "true" if ruby_static? && !given_ruby_static + def build_env + build_env = rb_config_env + build_env["RUBY_STATIC"] = "true" if ruby_static? && ENV.key?('RUBY_STATIC') + build_env + end + def cargo_command(cargo_dir, dest_path, args = []) + manifest = File.join(cargo_dir, "Cargo.toml") cargo = ENV.fetch("CARGO", "cargo") cmd = [] cmd += [cargo, "rustc"] + cmd += ["--target", ENV['CARGO_BUILD_TARGET']] if ENV['CARGO_BUILD_TARGET'] cmd += ["--target-dir", dest_path] cmd += ["--manifest-path", manifest] - cmd += ["--lib", "--release", "--locked"] - cmd += ["--"] - cmd += [*cargo_rustc_args(dest_path)] + cmd += ["--lib"] + cmd += ["--profile", profile.to_s] + cmd += ["--locked"] if profile == :release cmd += Gem::Command.build_args cmd += args + cmd += ["--"] + cmd += [*cargo_rustc_args(dest_path)] + cmd + end - self.class.run cmd, results, self.class.class_name, cargo_dir - results - ensure - ENV["RUBY_STATIC"] = given_ruby_static + private + + def rb_config_env + result = {} + RbConfig::CONFIG.each {|k, v| result["RBCONFIG_#{k}"] = v } + result end def cargo_rustc_args(dest_dir) @@ -92,7 +111,7 @@ class Gem::Ext::CargoBuilder < Gem::Ext::Builder def libruby_args(dest_dir) libs = makefile_config(ruby_static? ? "LIBRUBYARG_STATIC" : "LIBRUBYARG_SHARED") raw_libs = Shellwords.split(libs) - raw_libs.flat_map {|l| ldflag_to_link_modifier(l, dest_dir) } + raw_libs.flat_map {|l| ldflag_to_link_modifier(l) } end def ruby_static? @@ -103,22 +122,33 @@ class Gem::Ext::CargoBuilder < Gem::Ext::Builder # Ruby expects the dylib to follow a file name convention for loading def rename_cdylib_for_ruby_compatibility(dest_path) - dylib_path = validate_cargo_build!(dest_path) - dlext_name = "#{spec.name}.#{makefile_config("DLEXT")}" - new_name = dylib_path.gsub(File.basename(dylib_path), dlext_name) - FileUtils.cp(dylib_path, new_name) - new_name + new_path = final_extension_path(dest_path) + FileUtils.cp(cargo_dylib_path(dest_path), new_path) + new_path end def validate_cargo_build!(dir) - prefix = so_ext == "dll" ? "" : "lib" - dylib_path = File.join(dir, "release", "#{prefix}#{cargo_crate_name}.#{so_ext}") + dylib_path = cargo_dylib_path(dir) raise DylibNotFoundError, dir unless File.exist?(dylib_path) dylib_path end + def final_extension_path(dest_path) + dylib_path = cargo_dylib_path(dest_path) + dlext_name = "#{spec.name}.#{makefile_config("DLEXT")}" + dylib_path.gsub(File.basename(dylib_path), dlext_name) + end + + def cargo_dylib_path(dest_path) + prefix = so_ext == "dll" ? "" : "lib" + path_parts = [dest_path] + path_parts << ENV['CARGO_BUILD_TARGET'] if ENV['CARGO_BUILD_TARGET'] + path_parts += [profile_target_directory, "#{prefix}#{cargo_crate_name}.#{so_ext}"] + File.join(*path_parts) + end + def cargo_crate_name spec.metadata.fetch('cargo_crate_name', spec.name).tr('-', '_') end @@ -127,42 +157,19 @@ class Gem::Ext::CargoBuilder < Gem::Ext::Builder split_flags("DLDFLAGS") .map {|arg| maybe_resolve_ldflag_variable(arg, dest_dir) } .compact - .flat_map {|arg| ldflag_to_link_modifier(arg, dest_dir) } + .flat_map {|arg| ldflag_to_link_modifier(arg) } end def rustc_lib_flags(dest_dir) - split_flags("LIBS").flat_map {|arg| ldflag_to_link_modifier(arg, dest_dir) } + split_flags("LIBS").flat_map {|arg| ldflag_to_link_modifier(arg) } end def split_flags(var) Shellwords.split(RbConfig::CONFIG.fetch(var, "")) end - def ldflag_to_link_modifier(arg, dest_dir) - flag = arg[0..1] - val = arg[2..-1] - - case flag - when "-L" then ["-L", "native=#{val}"] - when "-l" then ["-l", val.to_s] - when "-F" then ["-l", "framework=#{val}"] - else ["-C", "link_arg=#{arg}"] - end - end - - def link_flag(link_name) - # These are provided by the CRT with MSVC - # @see https://github.com/rust-lang/pkg-config-rs/blob/49a4ac189aafa365167c72e8e503565a7c2697c2/src/lib.rs#L622 - return [] if msvc_target? && ["m", "c", "pthread"].include?(link_name) - - if link_name.include?("ruby") - # Specify the lib kind and give it the name "ruby" for linking - kind = ruby_static? ? "static" : "dylib" - - ["-l", "#{kind}=ruby:#{link_name}"] - else - ["-l", link_name] - end + def ldflag_to_link_modifier(arg) + LinkFlagConverter.convert(arg) end def msvc_target? @@ -182,20 +189,24 @@ class Gem::Ext::CargoBuilder < Gem::Ext::Builder !!Gem::WIN_PATTERNS.find {|r| target_platform =~ r } end - # Intepolate substition vars in the arg (i.e. $(DEFFILE)) + # Interpolate substition vars in the arg (i.e. $(DEFFILE)) def maybe_resolve_ldflag_variable(input_arg, dest_dir) - str = input_arg.gsub(/\$\((\w+)\)/) do |var_name| - case var_name - # On windows, it is assumed that mkmf has setup an exports file for the - # extension, so we have to to create one ourselves. - when "DEFFILE" - write_deffile(dest_dir) - else - RbConfig::CONFIG[var_name] - end - end.strip + var_matches = input_arg.match(/\$\((\w+)\)/) - str == "" ? nil : str + return input_arg unless var_matches + + var_name = var_matches[1] + + return input_arg if var_name.nil? || var_name.chomp.empty? + + case var_name + # On windows, it is assumed that mkmf has setup an exports file for the + # extension, so we have to to create one ourselves. + when "DEFFILE" + write_deffile(dest_dir) + else + RbConfig::CONFIG[var_name] + end end def write_deffile(dest_dir) @@ -241,14 +252,18 @@ class Gem::Ext::CargoBuilder < Gem::Ext::Builder # Good balance between binary size and debugability def debug_flags + return [] if profile == :dev + ["-C", "debuginfo=1"] end # Copied from ExtConfBuilder - def finalize_directory(ext_path, dest_path, lib_dir, extension_dir) + def finalize_directory(dest_path, lib_dir, extension_dir) require "fileutils" require "tempfile" + ext_path = final_extension_path(dest_path) + begin tmp_dest = Dir.mktmpdir(".gem.", extension_dir) @@ -287,6 +302,14 @@ class Gem::Ext::CargoBuilder < Gem::Ext::Builder path end + def profile_target_directory + case profile + when :release then 'release' + when :dev then 'debug' + else raise "unknown target directory for profile: #{profile}" + end + end + # Error raised when no cdylib artifact was created class DylibNotFoundError < StandardError def initialize(dir) diff --git a/lib/rubygems/ext/cargo_builder/link_flag_converter.rb b/lib/rubygems/ext/cargo_builder/link_flag_converter.rb new file mode 100644 index 0000000000..111bb05492 --- /dev/null +++ b/lib/rubygems/ext/cargo_builder/link_flag_converter.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class Gem::Ext::CargoBuilder < Gem::Ext::Builder + # Converts Ruby link flags into something cargo understands + class LinkFlagConverter + def self.convert(arg) + case arg.chomp + when /^-L\s*(.+)$/ + ["-L", "native=#{$1}"] + when /^--library=(\w+\S+)$/, /^-l\s*(\w+\S+)$/ + ["-l", $1] + when /^-l\s*:lib(\S+).a$/ + ["-l", "static=#{$1}"] + when /^-l\s*:lib(\S+).(so|dylib|dll)$/ + ["-l", "dylib=#{$1}"] + when /^-F\s*(.*)$/ + ["-l", "framework=#{$1}"] + else + ["-C", "link_arg=#{arg}"] + end + end + end +end diff --git a/lib/rubygems/gemcutter_utilities.rb b/lib/rubygems/gemcutter_utilities.rb index 7cb5fe9448..af0d957bc8 100644 --- a/lib/rubygems/gemcutter_utilities.rb +++ b/lib/rubygems/gemcutter_utilities.rb @@ -135,7 +135,7 @@ module Gem::GemcutterUtilities sign_in_host, scope: scope) do |request| request.basic_auth email, password request["OTP"] = otp if otp - request.body = URI.encode_www_form({:api_key => api_key }.merge(update_scope_params)) + request.body = URI.encode_www_form({ :api_key => api_key }.merge(update_scope_params)) end with_response response do |resp| diff --git a/lib/rubygems/psych_additions.rb b/lib/rubygems/psych_additions.rb deleted file mode 100644 index a13c1ec676..0000000000 --- a/lib/rubygems/psych_additions.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true -# This exists just to satisfy bugs in marshal'd gemspecs that -# contain a reference to Psych::PrivateType. We prune these out -# in Specification._load, but if we don't have the constant, Marshal -# blows up. - -module Psych # :nodoc: - class PrivateType # :nodoc: - end -end diff --git a/lib/rubygems/request.rb b/lib/rubygems/request.rb index 136b154bee..6d6c2d8de8 100644 --- a/lib/rubygems/request.rb +++ b/lib/rubygems/request.rb @@ -193,7 +193,7 @@ class Gem::Request begin @requests[connection.object_id] += 1 - verbose "#{request.method} #{Gem::Uri.new(@uri).redacted}" + verbose "#{request.method} #{Gem::Uri.redact(@uri)}" file_name = File.basename(@uri.path) # perform download progress reporter only for gems diff --git a/lib/rubygems/source.rb b/lib/rubygems/source.rb index 85c300a8f8..5f49a0d216 100644 --- a/lib/rubygems/source.rb +++ b/lib/rubygems/source.rb @@ -26,15 +26,8 @@ class Gem::Source # Creates a new Source which will use the index located at +uri+. def initialize(uri) - begin - unless uri.kind_of? URI - uri = URI.parse(uri.to_s) - end - rescue URI::InvalidURIError - raise if Gem::Source == self.class - end - - @uri = uri + require_relative "uri" + @uri = Gem::Uri.parse!(uri) @update_cache = nil end diff --git a/lib/rubygems/source/git.rb b/lib/rubygems/source/git.rb index 8d5dc1f69d..1d964eb59a 100644 --- a/lib/rubygems/source/git.rb +++ b/lib/rubygems/source/git.rb @@ -49,8 +49,8 @@ class Gem::Source::Git < Gem::Source # will be checked out when the gem is installed. def initialize(name, repository, reference, submodules = false) - super repository - + require_relative "../uri" + @uri = Gem::Uri.parse(repository) @name = name @repository = repository @reference = reference diff --git a/lib/rubygems/source_list.rb b/lib/rubygems/source_list.rb index 16e90e1ef7..7abe796409 100644 --- a/lib/rubygems/source_list.rb +++ b/lib/rubygems/source_list.rb @@ -48,15 +48,11 @@ class Gem::SourceList # String. def <<(obj) - require "uri" - src = case obj - when URI - Gem::Source.new(obj) when Gem::Source obj else - Gem::Source.new(URI.parse(obj)) + Gem::Source.new(obj) end @sources << src unless @sources.include?(src) diff --git a/lib/rubygems/specification.rb b/lib/rubygems/specification.rb index 33eb0a2bff..01aa8fd942 100644 --- a/lib/rubygems/specification.rb +++ b/lib/rubygems/specification.rb @@ -157,7 +157,7 @@ class Gem::Specification < Gem::BasicSpecification }.freeze # rubocop:disable Style/MutableConstant - INITIALIZE_CODE_FOR_DEFAULTS = { } # :nodoc: + INITIALIZE_CODE_FOR_DEFAULTS = {} # :nodoc: # rubocop:enable Style/MutableConstant @@default_value.each do |k,v| @@ -1082,6 +1082,7 @@ class Gem::Specification < Gem::BasicSpecification spec.specification_version ||= NONEXISTENT_SPECIFICATION_VERSION spec.reset_nil_attributes_to_default + spec.flatten_require_paths spec end @@ -1273,10 +1274,26 @@ class Gem::Specification < Gem::BasicSpecification array = begin Marshal.load str rescue ArgumentError => e - raise unless e.message.include?("YAML") + # + # Some very old marshaled specs included references to `YAML::PrivateType` + # and `YAML::Syck::DefaultKey` constants due to bugs in the old emitter + # that generated them. Workaround the issue by defining the necessary + # constants and retrying. + # + message = e.message + raise unless message.include?("YAML::") + + Object.const_set "YAML", Psych unless Object.const_defined?(:YAML) + + if message.include?("YAML::Syck::") + YAML.const_set "Syck", YAML unless YAML.const_defined?(:Syck) + + YAML::Syck.const_set "DefaultKey", Class.new if message.include?("YAML::Syck::DefaultKey") + elsif message.include?("YAML::PrivateType") + YAML.const_set "PrivateType", Class.new + end - Object.const_set "YAML", Psych - Marshal.load str + retry end spec = Gem::Specification.new @@ -2676,6 +2693,13 @@ class Gem::Specification < Gem::BasicSpecification @installed_by_version ||= nil end + def flatten_require_paths # :nodoc: + return unless raw_require_paths.first.is_a?(Array) + + warn "#{name} #{version} includes a gemspec with `require_paths` set to an array of arrays. Newer versions of this gem might've already fixed this" + raw_require_paths.flatten! + end + def raw_require_paths # :nodoc: @require_paths end diff --git a/lib/rubygems/specification_policy.rb b/lib/rubygems/specification_policy.rb index 002b3c20c4..8b5d01dda2 100644 --- a/lib/rubygems/specification_policy.rb +++ b/lib/rubygems/specification_policy.rb @@ -154,7 +154,7 @@ class Gem::SpecificationPolicy def validate_duplicate_dependencies # :nodoc: # NOTE: see REFACTOR note in Gem::Dependency about types - this might be brittle - seen = Gem::Dependency::TYPES.inject({}) {|types, type| types.merge({ type => {}}) } + seen = Gem::Dependency::TYPES.inject({}) {|types, type| types.merge({ type => {} }) } error_messages = [] @specification.dependencies.each do |dep| diff --git a/lib/rubygems/uri.rb b/lib/rubygems/uri.rb index ba30fac2f5..6acb9041f9 100644 --- a/lib/rubygems/uri.rb +++ b/lib/rubygems/uri.rb @@ -5,6 +5,44 @@ # class Gem::Uri + ## + # Parses and redacts uri + + def self.redact(uri) + new(uri).redacted + end + + ## + # Parses uri, raising if it's invalid + + def self.parse!(uri) + require "uri" + + raise URI::InvalidURIError unless uri + + return uri unless uri.is_a?(String) + + # Always escape URI's to deal with potential spaces and such + # It should also be considered that source_uri may already be + # a valid URI with escaped characters. e.g. "{DESede}" is encoded + # as "%7BDESede%7D". If this is escaped again the percentage + # symbols will be escaped. + begin + URI.parse(uri) + rescue URI::InvalidURIError + URI.parse(URI::DEFAULT_PARSER.escape(uri)) + end + end + + ## + # Parses uri, returning the original uri if it's invalid + + def self.parse(uri) + parse!(uri) + rescue URI::InvalidURIError + uri + end + def initialize(source_uri) @parsed_uri = parse(source_uri) end @@ -26,7 +64,7 @@ class Gem::Uri end def redact_credentials_from(text) - return text unless valid_uri? && password? + return text unless valid_uri? && password? && text.include?(to_s) text.sub(password, 'REDACTED') end @@ -50,35 +88,12 @@ class Gem::Uri private - ## - # Parses the #uri, raising if it's invalid - def parse!(uri) - require "uri" - - raise URI::InvalidURIError unless uri - - # Always escape URI's to deal with potential spaces and such - # It should also be considered that source_uri may already be - # a valid URI with escaped characters. e.g. "{DESede}" is encoded - # as "%7BDESede%7D". If this is escaped again the percentage - # symbols will be escaped. - begin - URI.parse(uri) - rescue URI::InvalidURIError - URI.parse(URI::DEFAULT_PARSER.escape(uri)) - end + self.class.parse!(uri) end - ## - # Parses the #uri, returning the original uri if it's invalid - def parse(uri) - return uri unless uri.is_a?(String) - - parse!(uri) - rescue URI::InvalidURIError - uri + self.class.parse(uri) end def with_redacted_user diff --git a/lib/rubygems/util.rb b/lib/rubygems/util.rb index 4363c5adce..9fd78ab2bc 100644 --- a/lib/rubygems/util.rb +++ b/lib/rubygems/util.rb @@ -60,7 +60,7 @@ module Gem::Util # Invokes system, but silences all output. def self.silent_system(*command) - opt = {:out => IO::NULL, :err => [:child, :out]} + opt = { :out => IO::NULL, :err => [:child, :out] } if Hash === command.last opt.update(command.last) cmds = command[0...-1] |