diff options
author | hsbt <hsbt@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2017-11-01 23:29:38 +0000 |
---|---|---|
committer | hsbt <hsbt@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2017-11-01 23:29:38 +0000 |
commit | be7b5929126cb3e696ef222339237faba9b8fe5a (patch) | |
tree | 51eae376f93c09bc82dde5a657a91df2c89062e4 /lib/bundler/source | |
parent | ae49dbd392083f69026f2a0fff4a1d5f42d172a7 (diff) |
Update bundled bundler to 1.16.0.
* lib/bundler, spec/bundler: Merge bundler-1.16.0.
* common.mk: rspec examples of bundler-1.16.0 needs require option.
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@60603 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'lib/bundler/source')
-rw-r--r-- | lib/bundler/source/gemspec.rb | 1 | ||||
-rw-r--r-- | lib/bundler/source/git.rb | 40 | ||||
-rw-r--r-- | lib/bundler/source/git/git_proxy.rb | 25 | ||||
-rw-r--r-- | lib/bundler/source/metadata.rb | 63 | ||||
-rw-r--r-- | lib/bundler/source/path.rb | 16 | ||||
-rw-r--r-- | lib/bundler/source/path/installer.rb | 2 | ||||
-rw-r--r-- | lib/bundler/source/rubygems.rb | 238 | ||||
-rw-r--r-- | lib/bundler/source/rubygems/remote.rb | 5 |
8 files changed, 269 insertions, 121 deletions
diff --git a/lib/bundler/source/gemspec.rb b/lib/bundler/source/gemspec.rb index 05e613277f..7e3447e776 100644 --- a/lib/bundler/source/gemspec.rb +++ b/lib/bundler/source/gemspec.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + module Bundler class Source class Gemspec < Path diff --git a/lib/bundler/source/git.rb b/lib/bundler/source/git.rb index b3e218e390..a1a59ddce5 100644 --- a/lib/bundler/source/git.rb +++ b/lib/bundler/source/git.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -require "fileutils" + +require "bundler/vendored_fileutils" require "uri" -require "digest/sha1" module Bundler class Source @@ -18,7 +18,7 @@ module Bundler @allow_remote = false # Stringify options that could be set as symbols - %w(ref branch tag revision).each {|k| options[k] = options[k].to_s if options[k] } + %w[ref branch tag revision].each {|k| options[k] = options[k].to_s if options[k] } @uri = options["uri"] || "" @branch = options["branch"] @@ -39,7 +39,7 @@ module Bundler out = String.new("GIT\n") out << " remote: #{@uri}\n" out << " revision: #{revision}\n" - %w(ref branch tag submodules).each do |opt| + %w[ref branch tag submodules].each do |opt| out << " #{opt}: #{options[opt]}\n" if options[opt] end out << " glob: #{@glob}\n" unless @glob == DEFAULT_GLOB @@ -169,15 +169,13 @@ module Bundler def install(spec, options = {}) force = options[:force] - Bundler.ui.info "Using #{version_message(spec)} from #{self}" + print_using_message "Using #{version_message(spec)} from #{self}" - if requires_checkout? && !@copied && !force + if (requires_checkout? && !@copied) || force Bundler.ui.debug " * Checking out revision: #{ref}" git_proxy.copy_to(install_path, submodules) serialize_gemspecs_in(install_path) @copied = true - elsif force - git_proxy.copy_to(install_path, submodules) end generate_bin_options = { :disable_extensions => !Bundler.rubygems.spec_missing_extensions?(spec), :build_args => options[:build_args] } @@ -188,7 +186,7 @@ module Bundler def cache(spec, custom_path = nil) app_cache_path = app_cache_path(custom_path) - return unless Bundler.settings[:cache_all] + return unless Bundler.feature_flag.cache_all? return if path == app_cache_path cached! FileUtils.rm_rf(app_cache_path) @@ -210,13 +208,11 @@ module Bundler # When using local git repos, this is set to the local repo. def cache_path @cache_path ||= begin - git_scope = "#{base_name}-#{uri_hash}" - - if Bundler.requires_sudo? - Bundler.user_bundle_path.join("cache/git", git_scope) + if Bundler.requires_sudo? || Bundler.feature_flag.global_gem_cache? + Bundler.user_cache else - Bundler.cache.join("git", git_scope) - end + Bundler.bundle_path.join("cache", "bundler") + end.join("git", git_scope) end end @@ -287,7 +283,7 @@ module Bundler # If there is no URI scheme, assume it is an ssh/git URI input = uri end - Digest::SHA1.hexdigest(input) + SharedHelpers.digest(:SHA1).hexdigest(input) end def cached_revision @@ -304,9 +300,9 @@ module Bundler def fetch git_proxy.checkout - rescue GitError + rescue GitError => e raise unless Bundler.feature_flag.allow_offline_install? - Bundler.ui.warn "Using cached git data because of network errors" + Bundler.ui.warn "Using cached git data because of network errors:\n#{e}" end # no-op, since we validate when re-serializing the gemspec @@ -319,6 +315,14 @@ module Bundler StubSpecification.from_stub(stub) end end + + def git_scope + "#{base_name}-#{uri_hash}" + end + + def extension_cache_slug(_) + extension_dir_name + end end end end diff --git a/lib/bundler/source/git/git_proxy.rb b/lib/bundler/source/git/git_proxy.rb index c05d7a5afa..c56dda66ea 100644 --- a/lib/bundler/source/git/git_proxy.rb +++ b/lib/bundler/source/git/git_proxy.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require "shellwords" require "tempfile" module Bundler @@ -62,7 +63,7 @@ module Bundler begin @revision ||= find_local_revision rescue GitCommandError - raise MissingGitRevisionError.new(ref, uri) + raise MissingGitRevisionError.new(ref, URICredentialsFilter.credential_filtered_uri(uri)) end @revision @@ -90,18 +91,21 @@ module Bundler end def checkout - if path.exist? - return if has_revision_cached? - Bundler.ui.info "Fetching #{URICredentialsFilter.credential_filtered_uri(uri)}" - in_path do - git_retry %(fetch --force --quiet --tags #{uri_escaped_with_configured_credentials} "refs/heads/*:refs/heads/*") - end - else - Bundler.ui.info "Fetching #{URICredentialsFilter.credential_filtered_uri(uri)}" + return if path.exist? && has_revision_cached? + extra_ref = "#{Shellwords.shellescape(ref)}:#{Shellwords.shellescape(ref)}" if ref && ref.start_with?("refs/") + + Bundler.ui.info "Fetching #{URICredentialsFilter.credential_filtered_uri(uri)}" + + unless path.exist? SharedHelpers.filesystem_access(path.dirname) do |p| FileUtils.mkdir_p(p) end git_retry %(clone #{uri_escaped_with_configured_credentials} "#{path}" --bare --no-hardlinks --quiet) + return unless extra_ref + end + + in_path do + git_retry %(fetch --force --quiet --tags #{uri_escaped_with_configured_credentials} "refs/heads/*:refs/heads/*" #{extra_ref}) end end @@ -149,7 +153,7 @@ module Bundler end def git_retry(command) - Bundler::Retry.new("`git #{command}`", GitNotAllowedError).attempts do + Bundler::Retry.new("`git #{URICredentialsFilter.credential_filtered_string(command, uri)}`", GitNotAllowedError).attempts do git(command) end end @@ -217,6 +221,7 @@ module Bundler def in_path(&blk) checkout unless path.exist? + _ = URICredentialsFilter # load it before we chdir SharedHelpers.chdir(path, &blk) end diff --git a/lib/bundler/source/metadata.rb b/lib/bundler/source/metadata.rb new file mode 100644 index 0000000000..93909002c7 --- /dev/null +++ b/lib/bundler/source/metadata.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Bundler + class Source + class Metadata < Source + def specs + @specs ||= Index.build do |idx| + idx << Gem::Specification.new("ruby\0", RubyVersion.system.to_gem_version_with_patchlevel) + idx << Gem::Specification.new("rubygems\0", Gem::VERSION) + + idx << Gem::Specification.new do |s| + s.name = "bundler" + s.version = VERSION + s.platform = Gem::Platform::RUBY + s.source = self + s.authors = ["bundler team"] + s.bindir = "exe" + s.executables = %w[bundle] + # can't point to the actual gemspec or else the require paths will be wrong + s.loaded_from = File.expand_path("..", __FILE__) + end + if loaded_spec = nil && Bundler.rubygems.loaded_specs("bundler") + idx << loaded_spec # this has to come after the fake gemspec, to override it + elsif local_spec = Bundler.rubygems.find_name("bundler").find {|s| s.version.to_s == VERSION } + idx << local_spec + end + + idx.each {|s| s.source = self } + end + end + + def cached!; end + + def remote!; end + + def options + {} + end + + def install(spec, _opts = {}) + print_using_message "Using #{version_message(spec)}" + nil + end + + def to_s + "the local ruby installation" + end + + def ==(other) + self.class == other.class + end + alias_method :eql?, :== + + def hash + self.class.hash + end + + def version_message(spec) + "#{spec.name} #{spec.version}" + end + end + end +end diff --git a/lib/bundler/source/path.rb b/lib/bundler/source/path.rb index 8dd0763cc1..ed734bf549 100644 --- a/lib/bundler/source/path.rb +++ b/lib/bundler/source/path.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + module Bundler class Source class Path < Source @@ -35,10 +36,12 @@ module Bundler end def remote! + @local_specs = nil @allow_remote = true end def cached! + @local_specs = nil @allow_cached = true end @@ -74,14 +77,14 @@ module Bundler end def install(spec, options = {}) - Bundler.ui.info "Using #{version_message(spec)} from #{self}" + print_using_message "Using #{version_message(spec)} from #{self}" generate_bin(spec, :disable_extensions => true) nil # no post-install message end def cache(spec, custom_path = nil) app_cache_path = app_cache_path(custom_path) - return unless Bundler.settings[:cache_all] + return unless Bundler.feature_flag.cache_all? return if expand(@original_path).to_s.index(root_path.to_s + "/") == 0 unless @original_path.exist? @@ -113,10 +116,6 @@ module Bundler Bundler.root end - def is_a_path? - instance_of?(Path) - end - def expanded_original_path @expanded_original_path ||= expand(original_path) end @@ -228,7 +227,8 @@ module Bundler spec, :env_shebang => false, :disable_extensions => options[:disable_extensions], - :build_args => options[:build_args] + :build_args => options[:build_args], + :bundler_extension_cache_path => extension_cache_path(spec) ) installer.post_install rescue Gem::InvalidSpecificationException => e @@ -242,7 +242,7 @@ module Bundler "to modify their .gemspec so it can work with `gem build`." end - Bundler.ui.warn "The validation message from Rubygems was:\n #{e.message}" + Bundler.ui.warn "The validation message from RubyGems was:\n #{e.message}" end end end diff --git a/lib/bundler/source/path/installer.rb b/lib/bundler/source/path/installer.rb index 9c2f74a31b..a0357ffa39 100644 --- a/lib/bundler/source/path/installer.rb +++ b/lib/bundler/source/path/installer.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + module Bundler class Source class Path @@ -6,6 +7,7 @@ module Bundler attr_reader :spec def initialize(spec, options = {}) + @options = options @spec = spec @gem_dir = Bundler.rubygems.path(spec.full_gem_path) @wrappers = true diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 353194f53f..6f4157364f 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require "uri" require "rubygems/user_interaction" @@ -31,6 +32,7 @@ module Bundler end def cached! + @specs = nil @allow_cached = true end @@ -49,6 +51,7 @@ module Bundler end def can_lock?(spec) + return super if Bundler.feature_flag.lockfile_uses_separate_rubygems_sources? spec.source.is_a?(Rubygems) end @@ -69,8 +72,12 @@ module Bundler end def to_s - remote_names = remotes.map(&:to_s).join(", ") - "rubygems repository #{remote_names}" + if remotes.empty? + "locally installed gems" + else + remote_names = remotes.map(&:to_s).join(", ") + "rubygems repository #{remote_names} or installed locally" + end end alias_method :name, :to_s @@ -99,8 +106,8 @@ module Bundler end end - if installed?(spec) && (!force || spec.name.eql?("bundler")) - Bundler.ui.info "Using #{version_message(spec)}" + if installed?(spec) && !force + print_using_message "Using #{version_message(spec)}" return nil # no post-install message end @@ -141,7 +148,8 @@ module Bundler :wrappers => true, :env_shebang => true, :build_args => opts[:build_args], - :bundler_expected_checksum => spec.respond_to?(:checksum) && spec.checksum + :bundler_expected_checksum => spec.respond_to?(:checksum) && spec.checksum, + :bundler_extension_cache_path => extension_cache_path(spec) ).install end spec.full_gem_path = installed_spec.full_gem_path @@ -212,13 +220,21 @@ module Bundler @remotes.unshift(uri) unless @remotes.include?(uri) end - def replace_remotes(other_remotes) + def equivalent_remotes?(other_remotes) + other_remotes.map(&method(:remove_auth)) == @remotes.map(&method(:remove_auth)) + end + + def replace_remotes(other_remotes, allow_equivalent = false) return false if other_remotes == @remotes + equivalent = allow_equivalent && equivalent_remotes?(other_remotes) + @remotes = [] other_remotes.reverse_each do |r| add_remote r.to_s end + + !equivalent end def unmet_deps @@ -236,6 +252,43 @@ module Bundler end end + def double_check_for(unmet_dependency_names, override_dupes = false, index = specs) + return unless @allow_remote + raise ArgumentError, "missing index" unless index + + return unless api_fetchers.any? + + unmet_dependency_names = unmet_dependency_names.call + unless unmet_dependency_names.nil? + if api_fetchers.size <= 1 + # can't do this when there are multiple fetchers because then we might not fetch from _all_ + # of them + unmet_dependency_names -= remote_specs.spec_names # avoid re-fetching things we've already gotten + end + return if unmet_dependency_names.empty? + end + + Bundler.ui.debug "Double checking for #{unmet_dependency_names || "all specs (due to the size of the request)"} in #{self}" + + fetch_names(api_fetchers, unmet_dependency_names, index, override_dupes) + end + + def dependency_names_to_double_check + names = [] + remote_specs.each do |spec| + case spec + when EndpointSpecification, Gem::Specification, StubSpecification, LazySpecification + names.concat(spec.runtime_dependencies) + when RemoteSpecification # from the full index + return nil + else + raise "unhandled spec type (#{spec.inspect})" + end + end + names.map!(&:name) if names + names + end + protected def credless_remotes @@ -276,7 +329,7 @@ module Bundler end def suppress_configured_credentials(remote) - remote_nouser = remote.dup.tap {|uri| uri.user = uri.password = nil }.to_s + remote_nouser = remove_auth(remote) if remote.userinfo && remote.userinfo == Bundler.settings[remote_nouser] remote_nouser else @@ -284,15 +337,14 @@ module Bundler end end + def remove_auth(remote) + remote.dup.tap {|uri| uri.user = uri.password = nil }.to_s + end + def installed_specs - @installed_specs ||= begin - idx = Index.new - have_bundler = false + @installed_specs ||= Index.build do |idx| Bundler.rubygems.all_specs.reverse_each do |spec| - if spec.name == "bundler" - next unless spec.version.to_s == VERSION - have_bundler = true - end + next if spec.name == "bundler" spec.source = self if Bundler.rubygems.spec_missing_extensions?(spec, false) Bundler.ui.debug "Source #{self} is ignoring #{spec} because it is missing extensions" @@ -300,23 +352,6 @@ module Bundler end idx << spec end - - # Always have bundler locally - unless have_bundler - # We're running bundler directly from the source - # so, let's create a fake gemspec for it (it's a path) - # gemspec - bundler = Gem::Specification.new do |s| - s.name = "bundler" - s.version = VERSION - s.platform = Gem::Platform::RUBY - s.source = self - s.authors = ["bundler team"] - s.loaded_from = File.expand_path("..", __FILE__) - end - idx << bundler - end - idx end end @@ -334,9 +369,9 @@ module Bundler end idx << s end - end - idx + idx + end end def api_fetchers @@ -348,71 +383,36 @@ module Bundler index_fetchers = fetchers - api_fetchers # gather lists from non-api sites - index_fetchers.each do |f| - Bundler.ui.info "Fetching source index from #{f.uri}" - idx.use f.specs_with_retry(nil, self) - end + fetch_names(index_fetchers, nil, idx, false) # because ensuring we have all the gems we need involves downloading # the gemspecs of those gems, if the non-api sites contain more than - # about 100 gems, we treat all sites as non-api for speed. + # about 500 gems, we treat all sites as non-api for speed. allow_api = idx.size < API_REQUEST_LIMIT && dependency_names.size < API_REQUEST_LIMIT Bundler.ui.debug "Need to query more than #{API_REQUEST_LIMIT} gems." \ " Downloading full index instead..." unless allow_api - if allow_api - api_fetchers.each do |f| - Bundler.ui.info "Fetching gem metadata from #{f.uri}", Bundler.ui.debug? - idx.use f.specs_with_retry(dependency_names, self) - Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over - end - - # Suppose the gem Foo depends on the gem Bar. Foo exists in Source A. Bar has some versions that exist in both - # sources A and B. At this point, the API request will have found all the versions of Bar in source A, - # but will not have found any versions of Bar from source B, which is a problem if the requested version - # of Foo specifically depends on a version of Bar that is only found in source B. This ensures that for - # each spec we found, we add all possible versions from all sources to the index. - loop do - idxcount = idx.size - api_fetchers.each do |f| - Bundler.ui.info "Fetching version metadata from #{f.uri}", Bundler.ui.debug? - idx.use f.specs_with_retry(idx.dependency_names, self), true - Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over - end - break if idxcount == idx.size - end - - if api_fetchers.any? - # it's possible that gems from one source depend on gems from some - # other source, so now we download gemspecs and iterate over those - # dependencies, looking for gems we don't have info on yet. - unmet = idx.unmet_dependency_names - - # if there are any cross-site gems we missed, get them now - api_fetchers.each do |f| - Bundler.ui.info "Fetching dependency metadata from #{f.uri}", Bundler.ui.debug? - idx.use f.specs_with_retry(unmet, self) - Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over - end if unmet.any? - else - allow_api = false - end - end + fetch_names(api_fetchers, allow_api && dependency_names, idx, false) + end + end - unless allow_api - api_fetchers.each do |f| - Bundler.ui.info "Fetching source index from #{f.uri}" - idx.use f.specs_with_retry(nil, self) - end + def fetch_names(fetchers, dependency_names, index, override_dupes) + fetchers.each do |f| + if dependency_names + Bundler.ui.info "Fetching gem metadata from #{f.uri}", Bundler.ui.debug? + index.use f.specs_with_retry(dependency_names, self), override_dupes + Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over + else + Bundler.ui.info "Fetching source index from #{f.uri}" + index.use f.specs_with_retry(nil, self), override_dupes end end end def fetch_gem(spec) return false unless spec.remote - uri = spec.remote.uri + spec.fetch_platform - Bundler.ui.confirm("Fetching #{version_message(spec)}") download_path = requires_sudo? ? Bundler.tmp(spec.full_name) : rubygems_dir gem_path = "#{rubygems_dir}/cache/#{spec.full_name}.gem" @@ -420,7 +420,7 @@ module Bundler SharedHelpers.filesystem_access("#{download_path}/cache") do |p| FileUtils.mkdir_p(p) end - Bundler.rubygems.download_gem(spec, uri, download_path) + download_gem(spec, download_path) if requires_sudo? SharedHelpers.filesystem_access("#{rubygems_dir}/cache") do |p| @@ -457,6 +457,76 @@ module Bundler def cache_path Bundler.app_cache end + + private + + # Checks if the requested spec exists in the global cache. If it does, + # we copy it to the download path, and if it does not, we download it. + # + # @param [Specification] spec + # the spec we want to download or retrieve from the cache. + # + # @param [String] download_path + # the local directory the .gem will end up in. + # + def download_gem(spec, download_path) + local_path = File.join(download_path, "cache/#{spec.full_name}.gem") + + if (cache_path = download_cache_path(spec)) && cache_path.file? + SharedHelpers.filesystem_access(local_path) do + FileUtils.cp(cache_path, local_path) + end + else + uri = spec.remote.uri + Bundler.ui.confirm("Fetching #{version_message(spec)}") + Bundler.rubygems.download_gem(spec, uri, download_path) + cache_globally(spec, local_path) + end + end + + # Checks if the requested spec exists in the global cache. If it does + # not, we create the relevant global cache subdirectory if it does not + # exist and copy the spec from the local cache to the global cache. + # + # @param [Specification] spec + # the spec we want to copy to the global cache. + # + # @param [String] local_cache_path + # the local directory from which we want to copy the .gem. + # + def cache_globally(spec, local_cache_path) + return unless cache_path = download_cache_path(spec) + return if cache_path.exist? + + SharedHelpers.filesystem_access(cache_path.dirname, &:mkpath) + SharedHelpers.filesystem_access(cache_path) do + FileUtils.cp(local_cache_path, cache_path) + end + end + + # Returns the global cache path of the calling Rubygems::Source object. + # + # Note that the Source determines the path's subdirectory. We use this + # subdirectory in the global cache path so that gems with the same name + # -- and possibly different versions -- from different sources are saved + # to their respective subdirectories and do not override one another. + # + # @param [Gem::Specification] specification + # + # @return [Pathname] The global cache path. + # + def download_cache_path(spec) + return unless Bundler.feature_flag.global_gem_cache? + return unless remote = spec.remote + return unless cache_slug = remote.cache_slug + + Bundler.user_cache.join("gems", cache_slug, spec.file_name) + end + + def extension_cache_slug(spec) + return unless remote = spec.remote + remote.cache_slug + end end end end diff --git a/lib/bundler/source/rubygems/remote.rb b/lib/bundler/source/rubygems/remote.rb index b49e645506..e73baaa992 100644 --- a/lib/bundler/source/rubygems/remote.rb +++ b/lib/bundler/source/rubygems/remote.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + module Bundler class Source class Rubygems @@ -20,10 +21,12 @@ module Bundler # def cache_slug @cache_slug ||= begin + return nil unless SharedHelpers.md5_available? + cache_uri = original_uri || uri uri_parts = [cache_uri.host, cache_uri.user, cache_uri.port, cache_uri.path] - uri_digest = Digest::MD5.hexdigest(uri_parts.compact.join(".")) + uri_digest = SharedHelpers.digest(:MD5).hexdigest(uri_parts.compact.join(".")) uri_parts[-1] = uri_digest uri_parts.compact.join(".") |