diff options
Diffstat (limited to 'lib/bundler/source')
| -rw-r--r-- | lib/bundler/source/gemspec.rb | 5 | ||||
| -rw-r--r-- | lib/bundler/source/git.rb | 115 | ||||
| -rw-r--r-- | lib/bundler/source/git/git_proxy.rb | 106 | ||||
| -rw-r--r-- | lib/bundler/source/metadata.rb | 9 | ||||
| -rw-r--r-- | lib/bundler/source/path.rb | 36 | ||||
| -rw-r--r-- | lib/bundler/source/path/installer.rb | 2 | ||||
| -rw-r--r-- | lib/bundler/source/rubygems.rb | 215 | ||||
| -rw-r--r-- | lib/bundler/source/rubygems/remote.rb | 28 | ||||
| -rw-r--r-- | lib/bundler/source/rubygems_aggregate.rb | 5 |
9 files changed, 362 insertions, 159 deletions
diff --git a/lib/bundler/source/gemspec.rb b/lib/bundler/source/gemspec.rb index 7e3447e776..ed766dbe74 100644 --- a/lib/bundler/source/gemspec.rb +++ b/lib/bundler/source/gemspec.rb @@ -4,14 +4,15 @@ module Bundler class Source class Gemspec < Path attr_reader :gemspec + attr_writer :checksum_store def initialize(options) super @gemspec = options["gemspec"] end - def as_path_source - Path.new(options) + def to_s + "gemspec at `#{@path}`" end end end diff --git a/lib/bundler/source/git.rb b/lib/bundler/source/git.rb index 3f69ea1e65..a002a2570a 100644 --- a/lib/bundler/source/git.rb +++ b/lib/bundler/source/git.rb @@ -70,13 +70,13 @@ module Bundler end def hash - [self.class, uri, ref, branch, name, version, glob, submodules].hash + [self.class, uri, ref, branch, name, glob, submodules].hash end def eql?(other) other.is_a?(Git) && uri == other.uri && ref == other.ref && branch == other.branch && name == other.name && - version == other.version && glob == other.glob && + glob == other.glob && submodules == other.submodules end @@ -102,7 +102,7 @@ module Bundler end def identifier - uri_with_specifiers([humanized_ref, cached_revision, glob_for_display]) + uri_with_specifiers([humanized_ref, locked_revision, glob_for_display]) end def uri_with_specifiers(specifiers) @@ -176,10 +176,10 @@ module Bundler "#{current_branch} but Gemfile specifies #{branch}" end - changed = cached_revision && cached_revision != revision + changed = locked_revision && locked_revision != revision - if !Bundler.settings[:disable_local_revision_check] && changed && !@unlocked && !git_proxy.contains?(cached_revision) - raise GitError, "The Gemfile lock is pointing to revision #{shortref_for_display(cached_revision)} " \ + if !Bundler.settings[:disable_local_revision_check] && changed && !@unlocked && !git_proxy.contains?(locked_revision) + raise GitError, "The Gemfile lock is pointing to revision #{shortref_for_display(locked_revision)} " \ "but the current branch in your local override for #{name} does not contain such commit. " \ "Please make sure your branch is up to date." end @@ -188,11 +188,16 @@ module Bundler end def specs(*) - set_cache_path!(app_cache_path) if has_app_cache? && !local? + set_cache_path!(app_cache_path) if use_app_cache? if requires_checkout? && !@copied - fetch - checkout + Plugin.hook(Plugin::Events::GIT_BEFORE_FETCH, self) + begin + fetch unless use_app_cache? + checkout + ensure + Plugin.hook(Plugin::Events::GIT_AFTER_FETCH, self) + end end local_specs @@ -208,22 +213,22 @@ module Bundler checkout end - generate_bin_options = { disable_extensions: !Bundler.rubygems.spec_missing_extensions?(spec), build_args: options[:build_args] } + generate_bin_options = { disable_extensions: !spec.missing_extensions?, build_args: options[:build_args] } generate_bin(spec, generate_bin_options) requires_checkout? ? spec.post_install_message : nil end + def migrate_cache(custom_path = nil, local: false) + if local + cache_to(custom_path, try_migrate: false) + else + cache_to(custom_path, try_migrate: true) + end + end + def cache(spec, custom_path = nil) - app_cache_path = app_cache_path(custom_path) - return unless Bundler.feature_flag.cache_all? - return if install_path == app_cache_path - return if cache_path == app_cache_path - cached! - FileUtils.rm_rf(app_cache_path) - git_proxy.checkout if requires_checkout? - FileUtils.cp_r("#{cache_path}/.", app_cache_path) - FileUtils.touch(app_cache_path.join(".bundlecache")) + cache_to(custom_path, try_migrate: false) end def load_spec_files @@ -238,7 +243,7 @@ module Bundler # across different projects, this cache will be shared. # When using local git repos, this is set to the local repo. def cache_path - @cache_path ||= if Bundler.feature_flag.global_gem_cache? + @cache_path ||= if Bundler.settings[:global_gem_cache] Bundler.user_cache else Bundler.bundle_path.join("cache", "bundler") @@ -246,7 +251,7 @@ module Bundler end def app_cache_dirname - "#{base_name}-#{shortref_for_path(cached_revision || revision)}" + "#{base_name}-#{shortref_for_path(locked_revision || revision)}" end def revision @@ -267,9 +272,39 @@ module Bundler private + def cache_to(custom_path, try_migrate: false) + return unless Bundler.settings[:cache_all] + + app_cache_path = app_cache_path(custom_path) + + migrate = try_migrate ? bare_repo?(app_cache_path) : false + + set_cache_path!(nil) if migrate + + return if cache_path == app_cache_path + + cached! + FileUtils.rm_rf(app_cache_path) + git_proxy.checkout if migrate || requires_checkout? + git_proxy.copy_to(app_cache_path, @submodules) + serialize_gemspecs_in(app_cache_path) + end + def checkout Bundler.ui.debug " * Checking out revision: #{ref}" - git_proxy.copy_to(install_path, submodules) + if use_app_cache? && !bare_repo?(app_cache_path) + SharedHelpers.filesystem_access(install_path.dirname) do |p| + FileUtils.mkdir_p(p) + end + FileUtils.cp_r("#{app_cache_path}/.", install_path) + else + if use_app_cache? && bare_repo?(app_cache_path) + Bundler.ui.warn "Installing from cache in old \"bare repository\" format for compatibility. " \ + "Please run `bundle cache` and commit the updated cache to migrate to the new format and get rid of this warning." + end + + git_proxy.copy_to(install_path, submodules) + end serialize_gemspecs_in(install_path) @copied = true end @@ -296,7 +331,7 @@ module Bundler # The gemspecs we cache should already be evaluated. spec = Bundler.load_gemspec(spec_path) next unless spec - Bundler.rubygems.set_installed_by_version(spec) + spec.installed_by_version = Gem::VERSION Bundler.rubygems.validate(spec) File.open(spec_path, "wb") {|file| file.write(spec.to_ruby) } end @@ -318,15 +353,23 @@ module Bundler end def has_app_cache? - cached_revision && super + locked_revision && super + end + + def use_app_cache? + has_app_cache? && !local? end def requires_checkout? - allow_git_ops? && !local? && !cached_revision_checked_out? + allow_git_ops? && !local? && !locked_revision_checked_out? + end + + def locked_revision_checked_out? + locked_revision && locked_revision == revision && installed? end - def cached_revision_checked_out? - cached_revision && cached_revision == revision && install_path.exist? + def installed? + git_proxy.installed_to?(install_path) end def base_name @@ -363,7 +406,7 @@ module Bundler Bundler::Digest.sha1(input) end - def cached_revision + def locked_revision options["revision"] end @@ -372,13 +415,12 @@ module Bundler end def git_proxy - @git_proxy ||= GitProxy.new(cache_path, uri, options, cached_revision, self) + @git_proxy ||= GitProxy.new(cache_path, uri, options, locked_revision, self) end def fetch git_proxy.checkout rescue GitError => e - raise unless Bundler.feature_flag.allow_offline_install? Bundler.ui.warn "Using cached git data because of network errors:\n#{e}" end @@ -386,9 +428,12 @@ module Bundler def validate_spec(_spec); end def load_gemspec(file) - stub = Gem::StubSpecification.gemspec_stub(file, install_path.parent, install_path.parent) - stub.full_gem_path = Pathname.new(file).dirname.expand_path(root).to_s - StubSpecification.from_stub(stub) + dirname = Pathname.new(file).dirname + SharedHelpers.chdir(dirname.to_s) do + stub = Gem::StubSpecification.gemspec_stub(file, install_path.parent, install_path.parent) + stub.full_gem_path = dirname.expand_path(root).to_s + StubSpecification.from_stub(stub) + end end def git_scope @@ -402,6 +447,10 @@ module Bundler def override_for(path) Bundler.settings.local_overrides.key(path) end + + def bare_repo?(path) + File.exist?(path.join("objects")) && File.exist?(path.join("HEAD")) + end end end end diff --git a/lib/bundler/source/git/git_proxy.rb b/lib/bundler/source/git/git_proxy.rb index 2fc9c6535f..8094dcaa9d 100644 --- a/lib/bundler/source/git/git_proxy.rb +++ b/lib/bundler/source/git/git_proxy.rb @@ -16,7 +16,7 @@ module Bundler def initialize(command) msg = String.new msg << "Bundler is trying to run `#{command}` at runtime. You probably need to run `bundle install`. However, " - msg << "this error message could probably be more useful. Please submit a ticket at https://github.com/rubygems/rubygems/issues/new?labels=Bundler&template=bundler-related-issue.md " + msg << "this error message could probably be more useful. Please submit a ticket at https://github.com/ruby/rubygems/issues/new?labels=Bundler&template=bundler-related-issue.md " msg << "with steps to reproduce as well as the following\n\nCALLER: #{caller.join("\n")}" super msg end @@ -57,6 +57,29 @@ module Bundler attr_accessor :path, :uri, :branch, :tag, :ref, :explicit_ref attr_writer :revision + def self.version + @version ||= full_version[/((\.?\d+)+).*/, 1] + end + + def self.full_version + @full_version ||= begin + raise GitNotInstalledError.new unless Bundler.git_present? + + require "open3" + out, err, status = Open3.capture3("git", "--version") + + raise GitCommandError.new("--version", SharedHelpers.pwd, err) unless status.success? + Bundler.ui.warn err unless err.empty? + + out.sub(/git version\s*/, "").strip + end + end + + def self.reset + @version = nil + @full_version = nil + end + def initialize(path, uri, options = {}, revision = nil, git = nil) @path = path @uri = uri @@ -92,11 +115,11 @@ module Bundler end def version - @version ||= full_version.match(/((\.?\d+)+).*/)[1] + self.class.version end def full_version - @full_version ||= git_local("--version").sub(/git version\s*/, "").strip + self.class.full_version end def checkout @@ -121,7 +144,7 @@ module Bundler FileUtils.rm_rf(p) end git "clone", "--no-checkout", "--quiet", path.to_s, destination.to_s - File.chmod(((File.stat(destination).mode | 0o777) & ~File.umask), destination) + File.chmod((File.stat(destination).mode | 0o777) & ~File.umask, destination) rescue Errno::EEXIST => e file_path = e.message[%r{.*?((?:[a-zA-Z]:)?/.*)}, 1] raise GitError, "Bundler could not install a gem because it needs to " \ @@ -137,7 +160,7 @@ module Bundler git "fetch", "--force", "--quiet", *extra_fetch_args(ref), dir: destination end - git "reset", "--hard", @revision, dir: destination + git "reset", "--hard", revision, dir: destination if submodules git_retry "submodule", "update", "--init", "--recursive", dir: destination @@ -147,10 +170,16 @@ module Bundler end end + def installed_to?(destination) + # if copy_to is interrupted, it may leave a partially installed directory that + # contains .git but no other files -- consider this not to be installed + Dir.exist?(destination) && (Dir.children(destination) - [".git"]).any? + end + private def git_remote_fetch(args) - command = ["fetch", "--force", "--quiet", "--no-tags", *args, "--", configured_uri, refspec].compact + command = fetch_command(args) command_with_no_credentials = check_allowed(command) Bundler::Retry.new("`#{command_with_no_credentials}` at #{path}", [MissingGitRevisionError]).attempts do @@ -160,6 +189,11 @@ module Bundler if err.include?("couldn't find remote ref") || err.include?("not our ref") raise MissingGitRevisionError.new(command_with_no_credentials, path, commit || explicit_ref, credential_filtered_uri) else + if shallow? + args -= depth_args + command = fetch_command(args) + command_with_no_credentials = check_allowed(command) + end raise GitCommandError.new(command_with_no_credentials, path, err) end end @@ -172,23 +206,22 @@ module Bundler FileUtils.mkdir_p(p) end - command = ["clone", "--bare", "--no-hardlinks", "--quiet", *extra_clone_args, "--", configured_uri, path.to_s] + clone_args = extra_clone_args + command = clone_command(clone_args) command_with_no_credentials = check_allowed(command) Bundler::Retry.new("`#{command_with_no_credentials}`", [MissingGitRevisionError]).attempts do _, err, status = capture(command, nil) return extra_ref if status.success? - if err.include?("Could not find remote branch") + if err.include?("Could not find remote branch") || # git up to 2.49 + err.include?("Remote branch #{branch_option} not found") # git 2.49 or higher raise MissingGitRevisionError.new(command_with_no_credentials, nil, explicit_ref, credential_filtered_uri) else - idx = command.index("--depth") - if idx - command.delete_at(idx) - command.delete_at(idx) + if shallow? + clone_args -= depth_args + command = clone_command(clone_args) command_with_no_credentials = check_allowed(command) - - err += "Retrying without --depth argument." end raise GitCommandError.new(command_with_no_credentials, path, err) end @@ -197,14 +230,14 @@ module Bundler def clone_needs_unshallow? return false unless path.join("shallow").exist? - return true if full_clone? + return true unless shallow? @revision && @revision != head_revision end def extra_ref return false if not_pinned? - return true unless full_clone? + return true if shallow? ref.start_with?("refs/") end @@ -256,7 +289,7 @@ module Bundler end def not_pinned? - branch || tag || ref.nil? + branch_option || ref.nil? end def pinned_to_full_sha? @@ -298,8 +331,8 @@ module Bundler end def has_revision_cached? - return unless @revision && path.exist? - git("cat-file", "-e", @revision, dir: path) + return unless commit && path.exist? + git("cat-file", "-e", commit, dir: path) true rescue GitError false @@ -332,8 +365,6 @@ module Bundler config_auth = Bundler.settings[remote.to_s] || Bundler.settings[remote.host] remote.userinfo ||= config_auth remote.to_s - elsif File.exist?(uri) - "file://#{uri}" else uri.to_s end @@ -401,13 +432,14 @@ module Bundler end def capture3_args_for(cmd, dir) - return ["git", *cmd] unless dir + # Disable automatic maintenance so a background commit-graph write in + # the source repo can't race the hardlinking local clone and fail with + # "hardlink different from source". + opts = ["-c", "gc.auto=0", "-c", "maintenance.auto=false"] - if Bundler.feature_flag.bundler_3_mode? || supports_minus_c? - ["git", "-C", dir.to_s, *cmd] - else - ["git", *cmd, { chdir: dir.to_s }] - end + return ["git", *opts, *cmd] unless dir + + ["git", "-C", dir.to_s, *opts, *cmd] end def extra_clone_args @@ -422,12 +454,20 @@ module Bundler # anyways. return args if @revision - args += ["--branch", branch || tag] if branch || tag + args += ["--branch", branch_option] if branch_option args end + def fetch_command(args) + ["fetch", "--force", "--quiet", "--no-tags", *args, "--", configured_uri, refspec].compact + end + + def clone_command(args) + ["clone", "--bare", "--no-hardlinks", "--quiet", *args, "--", configured_uri, path.to_s] + end + def depth_args - return [] if full_clone? + return [] unless shallow? ["--depth", depth.to_s] end @@ -438,12 +478,12 @@ module Bundler extra_args end - def full_clone? - depth.nil? + def branch_option + branch || tag end - def supports_minus_c? - @supports_minus_c ||= Gem::Version.new(version) >= Gem::Version.new("1.8.5") + def shallow? + !depth.nil? end def needs_allow_any_sha1_in_want? diff --git a/lib/bundler/source/metadata.rb b/lib/bundler/source/metadata.rb index 6b05e17727..ecf8895187 100644 --- a/lib/bundler/source/metadata.rb +++ b/lib/bundler/source/metadata.rb @@ -24,9 +24,8 @@ module Bundler s.bindir = "exe" s.homepage = "https://bundler.io" s.summary = "The best way to manage your application's dependencies" - s.executables = %w[bundle] - # can't point to the actual gemspec or else the require paths will be wrong - s.loaded_from = __dir__ + s.executables = %w[bundle bundler] + s.loaded_from = SharedHelpers.gemspec_path end end @@ -59,6 +58,10 @@ module Bundler def version_message(spec) "#{spec.name} #{spec.version}" end + + def checksum_store + @checksum_store ||= Checksum::Store.new + end end end end diff --git a/lib/bundler/source/path.rb b/lib/bundler/source/path.rb index 754eaa39c4..366a23aea7 100644 --- a/lib/bundler/source/path.rb +++ b/lib/bundler/source/path.rb @@ -24,7 +24,7 @@ module Bundler @path = Pathname.new(options["path"]) expanded_path = expand(@path) @path = if @path.relative? - expanded_path.relative_path_from(root_path.expand_path) + expanded_path.relative_path_from(File.expand_path(root_path)) else expanded_path end @@ -53,13 +53,17 @@ module Bundler "source at `#{@path}`" end + alias_method :identifier, :to_s + + alias_method :to_gemfile, :path + def hash [self.class, expanded_path, version].hash end def eql?(other) - return unless other.class == self.class - expanded_original_path == other.expanded_original_path && + [Gemspec, Path].include?(other.class) && + expanded_original_path == other.expanded_original_path && version == other.version end @@ -79,7 +83,7 @@ module Bundler def cache(spec, custom_path = nil) app_cache_path = app_cache_path(custom_path) - return unless Bundler.feature_flag.cache_all? + return unless Bundler.settings[:cache_all] return if expand(@original_path).to_s.index(root_path.to_s + "/") == 0 unless @original_path.exist? @@ -122,11 +126,7 @@ module Bundler end def expand(somepath) - if Bundler.current_ruby.jruby? # TODO: Unify when https://github.com/rubygems/bundler/issues/7598 fixed upstream and all supported jrubies include the fix - somepath.expand_path(root_path).expand_path - else - somepath.expand_path(root_path) - end + somepath.expand_path(root_path) rescue ArgumentError => e Bundler.ui.debug(e) raise PathError, "There was an error while trying to use the path " \ @@ -148,7 +148,7 @@ module Bundler def load_gemspec(file) return unless spec = Bundler.load_gemspec(file) - Bundler.rubygems.set_installed_by_version(spec) + spec.installed_by_version = Gem::VERSION spec end @@ -165,6 +165,13 @@ module Bundler next unless spec = load_gemspec(file) spec.source = self + # The ignore attribute is for ignoring installed gems that don't + # have extensions correctly compiled for activation. In the case of + # path sources, there's a single version of each gem in the path + # source available to Bundler, so we always certainly want to + # consider that for activation and never makes sense to ignore it. + spec.ignored = false + # Validation causes extension_dir to be calculated, which depends # on #source, so we validate here instead of load_gemspec validate_spec(spec) @@ -212,15 +219,16 @@ module Bundler # Some gem authors put absolute paths in their gemspec # and we have to save them from themselves - spec.files = spec.files.map do |path| - next path unless /\A#{Pathname::SEPARATOR_PAT}/o.match?(path) + spec.files = spec.files.filter_map do |path| + pathname = Pathname.new(path) + next path unless pathname.absolute? next if File.directory?(path) begin - Pathname.new(path).relative_path_from(gem_dir).to_s + pathname.relative_path_from(gem_dir).to_s rescue ArgumentError path end - end.compact + end installer = Path::Installer.new( spec, diff --git a/lib/bundler/source/path/installer.rb b/lib/bundler/source/path/installer.rb index 0af28fe770..39765e5da2 100644 --- a/lib/bundler/source/path/installer.rb +++ b/lib/bundler/source/path/installer.rb @@ -24,7 +24,7 @@ module Bundler def post_install run_hooks(:pre_install) - unless @disable_extensions + unless @disable_extensions || Bundler.settings[:no_build_extension] build_extensions run_hooks(:post_build) end diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 1085fdc2d8..ed864604fe 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -8,20 +8,26 @@ module Bundler autoload :Remote, File.expand_path("rubygems/remote", __dir__) # Ask for X gems per API request - API_REQUEST_SIZE = 50 + API_REQUEST_SIZE = 100 + REQUIRE_MUTEX = Mutex.new attr_accessor :remotes def initialize(options = {}) @options = options @remotes = [] + @remote_cooldowns = {} @dependency_names = [] @allow_remote = false @allow_cached = false @allow_local = options["allow_local"] || false + @prefer_local = false @checksum_store = Checksum::Store.new + @gem_installers = {} + @gem_installers_mutex = Mutex.new - Array(options["remotes"]).reverse_each {|r| add_remote(r) } + cooldown = options["cooldown"] + Array(options["remotes"]).reverse_each {|r| add_remote(r, cooldown: cooldown) } @lockfile_remotes = @remotes if options["from_lockfile"] end @@ -30,6 +36,10 @@ module Bundler @caches ||= [cache_path, *Bundler.rubygems.gem_cache] end + def prefer_local! + @prefer_local = true + end + def local_only! @specs = nil @allow_local = true @@ -37,6 +47,10 @@ module Bundler @allow_remote = false end + def local_only? + @allow_local && !@allow_remote + end + def local! return if @allow_local @@ -136,73 +150,77 @@ module Bundler # sources, and large_idx.merge! small_idx is way faster than # small_idx.merge! large_idx. index = @allow_remote ? remote_specs.dup : Index.new + + # Snapshot per-version `created_at` from the remote info before installed + # / cached specs overwrite the EndpointSpecification objects that carry + # it. The cooldown filter consults `created_at` on every candidate, so + # local stubs need the published date back-filled to participate. + remote_created_at = collect_remote_created_at(index) + index.merge!(cached_specs) if @allow_cached index.merge!(installed_specs) if @allow_local - # complete with default specs, only if not already available in the - # index through remote, cached, or installed specs - index.use(default_specs) if @allow_local + if @allow_local + if @prefer_local + index.merge!(default_specs) + else + # complete with default specs, only if not already available in the + # index through remote, cached, or installed specs + index.use(default_specs) + end + end + + backfill_created_at(index, remote_created_at) unless remote_created_at.empty? index end end - def install(spec, options = {}) - if (spec.default_gem? && !cached_built_in_gem(spec)) || (installed?(spec) && !options[:force]) - print_using_message "Using #{version_message(spec, options[:previous_spec])}" - return nil # no post-install message - end - - if spec.remote - # Check for this spec from other sources - uris = [spec.remote, *remotes_for_spec(spec)].map(&:anonymized_uri).uniq - Installer.ambiguous_gems << [spec.name, *uris] if uris.length > 1 + def download(spec, options = {}) + if (spec.default_gem? && !cached_built_in_gem(spec, local: options[:local])) || (installed?(spec) && !options[:force]) + return true end - path = fetch_gem_if_possible(spec, options[:previous_spec]) - raise GemNotFound, "Could not find #{spec.file_name} for installation" unless path - - return if Bundler.settings[:no_install] - - install_path = rubygems_dir - bin_path = Bundler.system_bindir - - require_relative "../rubygems_gem_installer" - - installer = Bundler::RubyGemsGemInstaller.at( - path, - security_policy: Bundler.rubygems.security_policies[Bundler.settings["trust-policy"]], - install_dir: install_path.to_s, - bin_dir: bin_path.to_s, - ignore_dependencies: true, - wrappers: true, - env_shebang: true, - build_args: options[:build_args], - bundler_extension_cache_path: extension_cache_path(spec) - ) + installer = rubygems_gem_installer(spec, options) if spec.remote s = begin installer.spec rescue Gem::Package::FormatError - Bundler.rm_rf(path) + Bundler.rm_rf(installer.gem) raise rescue Gem::Security::Exception => e raise SecurityError, - "The gem #{File.basename(path, ".gem")} can't be installed because " \ + "The gem #{installer.gem} can't be installed because " \ "the security policy didn't allow it, with the message: #{e.message}" end spec.__swap__(s) end + spec + end + + def install(spec, options = {}) + if (spec.default_gem? && !cached_built_in_gem(spec, local: options[:local])) || (installed?(spec) && !options[:force]) + print_using_message "Using #{version_message(spec, options[:previous_spec])}" + return nil # no post-install message + end + + return if Bundler.settings[:no_install] + + installer = rubygems_gem_installer(spec, options) spec.source.checksum_store.register(spec, installer.gem_checksum) message = "Installing #{version_message(spec, options[:previous_spec])}" message += " with native extensions" if spec.extensions.any? Bundler.ui.confirm message - installed_spec = installer.install + installed_spec = nil + + Gem.time("Installed #{spec.name} in", 0, true) do + installed_spec = installer.install + end spec.full_gem_path = installed_spec.full_gem_path spec.loaded_from = installed_spec.loaded_from @@ -222,12 +240,13 @@ module Bundler raise InstallError, e.message end - def cached_built_in_gem(spec) - cached_path = cached_path(spec) - if cached_path.nil? + def cached_built_in_gem(spec, local: false) + cached_path = cached_gem(spec) + if cached_path.nil? && !local remote_spec = remote_specs.search(spec).first if remote_spec cached_path = fetch_gem(remote_spec) + spec.remote = remote_spec.remote else Bundler.ui.warn "#{spec.full_name} is built in to Ruby, and can't be cached because your Gemfile doesn't have any sources that contain it." end @@ -235,9 +254,14 @@ module Bundler cached_path end - def add_remote(source) + def add_remote(source, cooldown: nil) uri = normalize_uri(source) @remotes.unshift(uri) unless @remotes.include?(uri) + @remote_cooldowns[uri] = cooldown if cooldown + end + + def cooldown_for(uri) + @remote_cooldowns[uri] end def spec_names @@ -258,7 +282,7 @@ module Bundler def remote_fetchers @remote_fetchers ||= remotes.to_h do |uri| - remote = Source::Rubygems::Remote.new(uri) + remote = Source::Rubygems::Remote.new(uri, cooldown: cooldown_for(uri)) [remote, Bundler::Fetcher.new(remote)] end.freeze end @@ -306,6 +330,13 @@ module Bundler @allow_remote && api_fetchers.any? end + def clear_cache + @specs = nil + @installed_specs = nil + @default_specs = nil + @cached_specs = nil + end + protected def remote_names @@ -316,22 +347,7 @@ module Bundler remotes.map(&method(:remove_auth)) end - def remotes_for_spec(spec) - specs.search_all(spec.name).inject([]) do |uris, s| - uris << s.remote if s.remote - uris - end - end - def cached_gem(spec) - if spec.default_gem? - cached_built_in_gem(spec) - else - cached_path(spec) - end - end - - def cached_path(spec) global_cache_path = download_cache_path(spec) caches << global_cache_path if global_cache_path @@ -364,10 +380,7 @@ module Bundler @installed_specs ||= Index.build do |idx| Bundler.rubygems.installed_specs.reverse_each do |spec| spec.source = self - if Bundler.rubygems.spec_missing_extensions?(spec, false) - Bundler.ui.debug "Source #{self} is ignoring #{spec} because it is missing extensions" - next - end + next if spec.ignored? idx << spec end end @@ -449,7 +462,7 @@ module Bundler end def installed?(spec) - installed_specs[spec].any? && !spec.deleted_gem? + installed_specs[spec].any? && !spec.installation_missing? end def rubygems_dir @@ -466,6 +479,31 @@ module Bundler private + def collect_remote_created_at(index) + return {} unless @allow_remote + + snapshot = {} + index.each do |spec| + next unless spec.respond_to?(:created_at) && spec.created_at + # Remember the remote that supplied the date too: when a source has + # several remotes with different per-URI cooldown settings we must + # restore the same one during backfill so `effective_cooldown` agrees. + snapshot[[spec.name, spec.version]] = [spec.created_at, spec.remote] + end + snapshot + end + + def backfill_created_at(index, snapshot) + index.each do |spec| + next unless spec.respond_to?(:created_at=) + next if spec.created_at + remote_created_at, remote = snapshot[[spec.name, spec.version]] + next unless remote_created_at + spec.created_at = remote_created_at + spec.remote ||= remote if remote && spec.respond_to?(:remote=) + end + end + def lockfile_remotes @lockfile_remotes || credless_remotes end @@ -486,7 +524,15 @@ module Bundler uri = spec.remote.uri Bundler.ui.confirm("Fetching #{version_message(spec, previous_spec)}") gem_remote_fetcher = remote_fetchers.fetch(spec.remote).gem_remote_fetcher - Bundler.rubygems.download_gem(spec, uri, download_cache_path, gem_remote_fetcher) + + Plugin.hook(Plugin::Events::GEM_BEFORE_FETCH, spec) + begin + Gem.time("Downloaded #{spec.name} in", 0, true) do + Bundler.rubygems.download_gem(spec, uri, download_cache_path, gem_remote_fetcher) + end + ensure + Plugin.hook(Plugin::Events::GEM_AFTER_FETCH, spec) + end end # Returns the global cache path of the calling Rubygems::Source object. @@ -501,17 +547,52 @@ module Bundler # @return [Pathname] The global cache path. # def download_cache_path(spec) - return unless Bundler.feature_flag.global_gem_cache? + return unless Bundler.settings[:global_gem_cache] return unless remote = spec.remote return unless cache_slug = remote.cache_slug - Bundler.user_cache.join("gems", cache_slug) + if Gem.respond_to?(:global_gem_cache_path) + Pathname.new(Gem.global_gem_cache_path).join(cache_slug) + else + # Fall back to old location for older RubyGems versions + Bundler.user_cache.join("gems", cache_slug) + end end def extension_cache_slug(spec) return unless remote = spec.remote remote.cache_slug end + + # We are using a mutex to read and write from/to the hash. + # The reason this double synchronization was added is for performance + # and to lock the mutex for the shortest possible amount of time. Otherwise, + # all threads are fighting over this mutex and when it gets acquired it gets locked + # until a thread finishes downloading a gem, leaving the other threads waiting + # doing nothing. + def rubygems_gem_installer(spec, options) + @gem_installers_mutex.synchronize { @gem_installers[spec.name] } || begin + path = fetch_gem_if_possible(spec, options[:previous_spec]) + raise GemNotFound, "Could not find #{spec.file_name} for installation" unless path + + REQUIRE_MUTEX.synchronize { require_relative "../rubygems_gem_installer" } + + installer = Bundler::RubyGemsGemInstaller.at( + path, + security_policy: Bundler.rubygems.security_policies[Bundler.settings["trust-policy"]], + install_dir: rubygems_dir.to_s, + bin_dir: Bundler.system_bindir.to_s, + ignore_dependencies: true, + wrappers: true, + env_shebang: true, + build_args: options[:build_args], + bundler_extension_cache_path: extension_cache_path(spec), + build_extension: Bundler.settings[:no_build_extension] ? false : nil, + install_plugin: Bundler.settings[:no_install_plugin] ? false : nil + ) + @gem_installers_mutex.synchronize { @gem_installers[spec.name] ||= installer } + end + end end end end diff --git a/lib/bundler/source/rubygems/remote.rb b/lib/bundler/source/rubygems/remote.rb index 9c5c06de24..3d847424b7 100644 --- a/lib/bundler/source/rubygems/remote.rb +++ b/lib/bundler/source/rubygems/remote.rb @@ -4,9 +4,9 @@ module Bundler class Source class Rubygems class Remote - attr_reader :uri, :anonymized_uri, :original_uri + attr_reader :uri, :anonymized_uri, :original_uri, :cooldown - def initialize(uri) + def initialize(uri, cooldown: nil) orig_uri = uri uri = Bundler.settings.mirror_for(uri) @original_uri = orig_uri if orig_uri != uri @@ -14,8 +14,21 @@ module Bundler @uri = apply_auth(uri, fallback_auth).freeze @anonymized_uri = remove_auth(@uri).freeze + @cooldown = cooldown end + # Returns the cooldown days that apply to this remote, resolving the + # precedence CLI > config > Gemfile per-source. Returns nil if no + # cooldown applies. + def effective_cooldown + override = Bundler.settings[:cooldown] + return override if override + @cooldown + end + + MAX_CACHE_SLUG_HOST_SIZE = 255 - 1 - 32 # 255 minus dot minus MD5 length + private_constant :MAX_CACHE_SLUG_HOST_SIZE + # @return [String] A slug suitable for use as a cache key for this # remote. # @@ -28,10 +41,15 @@ module Bundler host = cache_uri.to_s.start_with?("file://") ? nil : cache_uri.host uri_parts = [host, cache_uri.user, cache_uri.port, cache_uri.path] - uri_digest = SharedHelpers.digest(:MD5).hexdigest(uri_parts.compact.join(".")) + uri_parts.compact! + uri_digest = SharedHelpers.digest(:MD5).hexdigest(uri_parts.join(".")) + + uri_parts.pop + host_parts = uri_parts.join(".") + return uri_digest if host_parts.empty? - uri_parts[-1] = uri_digest - uri_parts.compact.join(".") + shortened_host_parts = host_parts[0...MAX_CACHE_SLUG_HOST_SIZE] + [shortened_host_parts, uri_digest].join(".") end end diff --git a/lib/bundler/source/rubygems_aggregate.rb b/lib/bundler/source/rubygems_aggregate.rb index 99ef81ad54..8aeaa375fa 100644 --- a/lib/bundler/source/rubygems_aggregate.rb +++ b/lib/bundler/source/rubygems_aggregate.rb @@ -5,9 +5,10 @@ module Bundler class RubygemsAggregate attr_reader :source_map, :sources - def initialize(sources, source_map) + def initialize(sources, source_map, excluded_sources = []) @sources = sources @source_map = source_map + @excluded_sources = excluded_sources @index = build_index end @@ -31,6 +32,8 @@ module Bundler dependency_names = source_map.pinned_spec_names sources.all_sources.each do |source| + next if @excluded_sources.include?(source) + source.dependency_names = dependency_names - source_map.pinned_spec_names(source) idx.add_source source.specs dependency_names.concat(source.unmet_deps).uniq! |