Merge rubygems/bundler HEAD

  Pick from https://github.com/rubygems/rubygems/commit/6b3a5a9ab0453463381a8164efb6298ea9eb776f

16 files changed, 225 insertions, 90 deletions

diff --git a/lib/bundler.rb b/lib/bundler.rb
index b24d47c6d0..79f65ccbe1 100644
--- a/lib/bundler.rb
+++ b/lib/bundler.rb
@@ -53,12 +53,12 @@ module Bundler
   autoload :GemHelpers,             File.expand_path("bundler/gem_helpers", __dir__)
   autoload :GemVersionPromoter,     File.expand_path("bundler/gem_version_promoter", __dir__)
   autoload :Graph,                  File.expand_path("bundler/graph", __dir__)
-  autoload :IncompleteSpecification, File.expand_path("bundler/incomplete_specification", __dir__)
   autoload :Index,                  File.expand_path("bundler/index", __dir__)
   autoload :Injector,               File.expand_path("bundler/injector", __dir__)
   autoload :Installer,              File.expand_path("bundler/installer", __dir__)
   autoload :LazySpecification,      File.expand_path("bundler/lazy_specification", __dir__)
   autoload :LockfileParser,         File.expand_path("bundler/lockfile_parser", __dir__)
+  autoload :MatchRemoteMetadata,    File.expand_path("bundler/match_remote_metadata", __dir__)
   autoload :ProcessLock,            File.expand_path("bundler/process_lock", __dir__)
   autoload :RemoteSpecification,    File.expand_path("bundler/remote_specification", __dir__)
   autoload :Resolver,               File.expand_path("bundler/resolver", __dir__)
diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb
index 7d28375bb5..66efd82b53 100644
--- a/lib/bundler/definition.rb
+++ b/lib/bundler/definition.rb
@@ -145,8 +145,6 @@ module Bundler
       @dependency_changes = converge_dependencies
       @local_changes = converge_locals
 
-      @reresolve = nil
-
       @requires = compute_requires
     end
 
@@ -218,6 +216,7 @@ module Bundler
       true
     rescue BundlerError => e
       @resolve = nil
+      @resolver = nil
       @specs = nil
       @gem_version_promoter = nil
 
@@ -288,7 +287,7 @@ module Bundler
         end
       else
         Bundler.ui.debug("Found changes from the lockfile, re-resolving dependencies because #{change_reason}")
-        @reresolve = reresolve
+        resolver.start(expanded_dependencies)
       end
     end
 
@@ -482,11 +481,18 @@ module Bundler
 
     private
 
-    def reresolve
-      last_resolve = converge_locked_specs
-      remove_ruby_from_platforms_if_necessary!(dependencies)
-      expanded_dependencies = expand_dependencies(dependencies + metadata_dependencies, true)
-      Resolver.resolve(expanded_dependencies, source_requirements, last_resolve, gem_version_promoter, additional_base_requirements_for_resolve, platforms)
+    def resolver
+      @resolver ||= begin
+        last_resolve = converge_locked_specs
+        Resolver.new(source_requirements, last_resolve, gem_version_promoter, additional_base_requirements_for_resolve, platforms)
+      end
+    end
+
+    def expanded_dependencies
+      @expanded_dependencies ||= begin
+        remove_ruby_from_platforms_if_necessary!(dependencies)
+        expand_dependencies(dependencies + metadata_dependencies, true)
+      end
     end
 
     def filter_specs(specs, deps)
@@ -514,15 +520,13 @@ module Bundler
         raise GemNotFound, "Could not find #{missing_specs_list.join(" nor ")}"
       end
 
-      if @reresolve.nil?
+      loop do
         incomplete_specs = specs.incomplete_specs
+        break if incomplete_specs.empty?
 
-        if incomplete_specs.any?
-          Bundler.ui.debug("The lockfile does not have all gems needed for the current platform though, Bundler will still re-resolve dependencies")
-          @unlock[:gems].concat(incomplete_specs.map(&:name))
-          @resolve = reresolve
-          specs = resolve.materialize(dependencies)
-        end
+        Bundler.ui.debug("The lockfile does not have all gems needed for the current platform though, Bundler will still re-resolve dependencies")
+        @resolve = resolver.start(expanded_dependencies, :exclude_specs => incomplete_specs)
+        specs = resolve.materialize(dependencies)
       end
 
       bundler = sources.metadata_source.specs.search(Gem::Dependency.new("bundler", VERSION)).last
@@ -879,10 +883,8 @@ module Bundler
     def additional_base_requirements_for_resolve
       return [] unless @locked_gems && unlocking? && !sources.expired_sources?(@locked_gems.sources)
       converge_specs(@originally_locked_specs).map do |locked_spec|
-        name = locked_spec.name
-        dep = Dependency.new(name, ">= #{locked_spec.version}")
-        DepProxy.get_proxy(dep, locked_spec.platform)
-      end
+        Dependency.new(locked_spec.name, ">= #{locked_spec.version}")
+      end.uniq
     end
 
     def remove_ruby_from_platforms_if_necessary!(dependencies)
@@ -894,6 +896,7 @@ module Bundler
 
       remove_platform(Gem::Platform::RUBY)
       add_current_platform
+      resolver.platforms = @platforms
     end
 
     def source_map
diff --git a/lib/bundler/endpoint_specification.rb b/lib/bundler/endpoint_specification.rb
index 368534ba6d..ea197328ba 100644
--- a/lib/bundler/endpoint_specification.rb
+++ b/lib/bundler/endpoint_specification.rb
@@ -3,6 +3,8 @@
 module Bundler
   # used for Creating Specifications from the Gemcutter Endpoint
   class EndpointSpecification < Gem::Specification
+    include MatchRemoteMetadata
+
     attr_reader :name, :version, :platform, :checksum
     attr_accessor :source, :remote, :dependencies
 
@@ -20,17 +22,6 @@ module Bundler
       parse_metadata(metadata)
     end
 
-    def required_ruby_version
-      @required_ruby_version ||= _remote_specification.required_ruby_version
-    end
-
-    # A fallback is included because the original version of the specification
-    # API didn't include that field, so some marshalled specs in the index have it
-    # set to +nil+.
-    def required_rubygems_version
-      @required_rubygems_version ||= _remote_specification.required_rubygems_version || Gem::Requirement.default
-    end
-
     def fetch_platform
       @platform
     end
diff --git a/lib/bundler/gem_version_promoter.rb b/lib/bundler/gem_version_promoter.rb
index 3cce3f2139..ddf7446dd2 100644
--- a/lib/bundler/gem_version_promoter.rb
+++ b/lib/bundler/gem_version_promoter.rb
@@ -88,6 +88,10 @@ module Bundler
       end
     end
 
+    def reset
+      @sort_versions = {}
+    end
+
     # @return [bool] Convenience method for testing value of level variable.
     def major?
       level == :major
diff --git a/lib/bundler/incomplete_specification.rb b/lib/bundler/incomplete_specification.rb
deleted file mode 100644
index 6d0b9b901c..0000000000
--- a/lib/bundler/incomplete_specification.rb
+++ /dev/null
@@ -1,12 +0,0 @@
-# frozen_string_literal: true
-
-module Bundler
-  class IncompleteSpecification
-    attr_reader :name, :platform
-
-    def initialize(name, platform)
-      @name = name
-      @platform = platform
-    end
-  end
-end
diff --git a/lib/bundler/installer.rb b/lib/bundler/installer.rb
index b7b0e36dfd..1b17de5d4e 100644
--- a/lib/bundler/installer.rb
+++ b/lib/bundler/installer.rb
@@ -238,19 +238,14 @@ module Bundler
     end
 
     def ensure_specs_are_compatible!
-      system_ruby = Bundler::RubyVersion.system
-      rubygems_version = Bundler.rubygems.version
       @definition.specs.each do |spec|
-        if required_ruby_version = spec.required_ruby_version
-          unless required_ruby_version.satisfied_by?(system_ruby.gem_version)
-            raise InstallError, "#{spec.full_name} requires ruby version #{required_ruby_version}, " \
-              "which is incompatible with the current version, #{system_ruby}"
-          end
+        unless spec.matches_current_ruby?
+          raise InstallError, "#{spec.full_name} requires ruby version #{spec.required_ruby_version}, " \
+            "which is incompatible with the current version, #{Gem.ruby_version}"
         end
-        next unless required_rubygems_version = spec.required_rubygems_version
-        unless required_rubygems_version.satisfied_by?(rubygems_version)
-          raise InstallError, "#{spec.full_name} requires rubygems version #{required_rubygems_version}, " \
-            "which is incompatible with the current version, #{rubygems_version}"
+        unless spec.matches_current_rubygems?
+          raise InstallError, "#{spec.full_name} requires rubygems version #{spec.required_rubygems_version}, " \
+            "which is incompatible with the current version, #{Gem.rubygems_version}"
         end
       end
     end
diff --git a/lib/bundler/lazy_specification.rb b/lib/bundler/lazy_specification.rb
index 78d2c22f81..ec141cfa27 100644
--- a/lib/bundler/lazy_specification.rb
+++ b/lib/bundler/lazy_specification.rb
@@ -95,8 +95,8 @@ module Bundler
       @specification = begin
         search = candidates.reverse.find do |spec|
           spec.is_a?(StubSpecification) ||
-            (spec.required_ruby_version.satisfied_by?(Gem.ruby_version) &&
-              spec.required_rubygems_version.satisfied_by?(Gem.rubygems_version))
+            (spec.matches_current_ruby? &&
+              spec.matches_current_rubygems?)
         end
         if search.nil? && Bundler.frozen_bundle?
           search = candidates.last
diff --git a/lib/bundler/match_metadata.rb b/lib/bundler/match_metadata.rb
new file mode 100644
index 0000000000..499036ca93
--- /dev/null
+++ b/lib/bundler/match_metadata.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Bundler
+  module MatchMetadata
+    def matches_current_ruby?
+      @required_ruby_version.satisfied_by?(Gem.ruby_version)
+    end
+
+    def matches_current_rubygems?
+      @required_rubygems_version.satisfied_by?(Gem.rubygems_version)
+    end
+  end
+end
diff --git a/lib/bundler/match_remote_metadata.rb b/lib/bundler/match_remote_metadata.rb
new file mode 100644
index 0000000000..e1b2f4d0e2
--- /dev/null
+++ b/lib/bundler/match_remote_metadata.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Bundler
+  module FetchMetadata
+    def matches_current_ruby?
+      @required_ruby_version ||= _remote_specification.required_ruby_version
+
+      super
+    end
+
+    def matches_current_rubygems?
+      # A fallback is included because the original version of the specification
+      # API didn't include that field, so some marshalled specs in the index have it
+      # set to +nil+.
+      @required_rubygems_version ||= _remote_specification.required_rubygems_version || Gem::Requirement.default
+
+      super
+    end
+  end
+
+  module MatchRemoteMetadata
+    include MatchMetadata
+
+    prepend FetchMetadata
+  end
+end
diff --git a/lib/bundler/remote_specification.rb b/lib/bundler/remote_specification.rb
index b5d7e3a6c9..601957746f 100644
--- a/lib/bundler/remote_specification.rb
+++ b/lib/bundler/remote_specification.rb
@@ -6,6 +6,7 @@ module Bundler
   # be seeded with what we're given from the source's abbreviated index - the
   # full specification will only be fetched when necessary.
   class RemoteSpecification
+    include MatchRemoteMetadata
     include MatchPlatform
     include Comparable
 
@@ -28,13 +29,6 @@ module Bundler
       @platform = _remote_specification.platform
     end
 
-    # A fallback is included because the original version of the specification
-    # API didn't include that field, so some marshalled specs in the index have it
-    # set to +nil+.
-    def required_rubygems_version
-      @required_rubygems_version ||= _remote_specification.required_rubygems_version || Gem::Requirement.default
-    end
-
     def full_name
       if @original_platform == Gem::Platform::RUBY
         "#{@name}-#{@version}"
diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb
index ca1bdbda7b..e382319112 100644
--- a/lib/bundler/resolver.rb
+++ b/lib/bundler/resolver.rb
@@ -7,6 +7,8 @@ module Bundler
 
     include GemHelpers
 
+    attr_writer :platforms
+
     # Figures out the best possible configuration of gems that satisfies
     # the list of passed dependencies and any child dependencies without
     # causing any gem activation errors.
@@ -19,41 +21,48 @@ module Bundler
     #   collection of gemspecs is returned. Otherwise, nil is returned.
     def self.resolve(requirements, source_requirements = {}, base = [], gem_version_promoter = GemVersionPromoter.new, additional_base_requirements = [], platforms = nil)
       base = SpecSet.new(base) unless base.is_a?(SpecSet)
-      metadata_requirements, regular_requirements = requirements.partition {|dep| dep.name.end_with?("\0") }
-      resolver = new(source_requirements, base, gem_version_promoter, additional_base_requirements, platforms, metadata_requirements)
-      result = resolver.start(requirements)
-      SpecSet.new(SpecSet.new(result).for(regular_requirements, false, platforms))
+      resolver = new(source_requirements, base, gem_version_promoter, additional_base_requirements, platforms)
+      resolver.start(requirements)
     end
 
-    def initialize(source_requirements, base, gem_version_promoter, additional_base_requirements, platforms, metadata_requirements)
+    def initialize(source_requirements, base, gem_version_promoter, additional_base_requirements, platforms)
       @source_requirements = source_requirements
-      @metadata_requirements = metadata_requirements
       @base = base
       @resolver = Molinillo::Resolver.new(self, self)
+      @results_for = {}
       @search_for = {}
-      @base_dg = Molinillo::DependencyGraph.new
-      base.each do |ls|
-        dep = Dependency.new(ls.name, ls.version)
-        @base_dg.add_vertex(ls.name, DepProxy.get_proxy(dep, ls.platform), true)
-      end
-      additional_base_requirements.each {|d| @base_dg.add_vertex(d.name, d) }
-      @platforms = platforms.reject {|p| p != Gem::Platform::RUBY && (platforms - [p]).any? {|pl| generic(pl) == p } }
+      @additional_base_requirements = additional_base_requirements
+      @platforms = platforms
       @resolving_only_for_ruby = platforms == [Gem::Platform::RUBY]
       @gem_version_promoter = gem_version_promoter
       @use_gvp = Bundler.feature_flag.use_gem_version_promoter_for_major_updates? || !@gem_version_promoter.major?
     end
 
-    def start(requirements)
+    def start(requirements, exclude_specs: [])
+      @metadata_requirements, regular_requirements = requirements.partition {|dep| dep.name.end_with?("\0") }
+
+      exclude_specs.each do |spec|
+        remove_from_candidates(spec)
+      end
+
+      @base_dg = Molinillo::DependencyGraph.new
+      @base.each do |ls|
+        dep = Dependency.new(ls.name, ls.version)
+        @base_dg.add_vertex(ls.name, DepProxy.get_proxy(dep, ls.platform), true)
+      end
+      @additional_base_requirements.each {|d| @base_dg.add_vertex(d.name, d) }
+
       @gem_version_promoter.prerelease_specified = @prerelease_specified = {}
       requirements.each {|dep| @prerelease_specified[dep.name] ||= dep.prerelease? }
 
       verify_gemfile_dependencies_are_found!(requirements)
-      dg = @resolver.resolve(requirements, @base_dg)
-      dg.
+      result = @resolver.resolve(requirements, @base_dg).
         map(&:payload).
         reject {|sg| sg.name.end_with?("\0") }.
         map(&:to_specs).
         flatten
+
+      SpecSet.new(SpecSet.new(result).for(regular_requirements, false, @platforms))
     rescue Molinillo::VersionConflict => e
       message = version_conflict_message(e)
       raise VersionConflict.new(e.conflicts.keys.uniq, message)
@@ -177,7 +186,7 @@ module Bundler
     end
 
     def results_for(dependency)
-      index_for(dependency).search(dependency)
+      @results_for[dependency] ||= index_for(dependency).search(dependency)
     end
 
     def name_for(dependency)
@@ -228,6 +237,19 @@ module Bundler
 
     private
 
+    def remove_from_candidates(spec)
+      @base.delete(spec)
+      @gem_version_promoter.reset
+
+      @results_for.keys.each do |dep|
+        next unless dep.name == spec.name
+
+        @results_for[dep].reject {|s| s.name == spec.name && s.version == spec.version }
+      end
+
+      @search_for = {}
+    end
+
     # returns an integer \in (-\infty, 0]
     # a number closer to 0 means the dependency is less constraining
     #
diff --git a/lib/bundler/resolver/spec_group.rb b/lib/bundler/resolver/spec_group.rb
index 1c016fc55d..4e5b0082d3 100644
--- a/lib/bundler/resolver/spec_group.rb
+++ b/lib/bundler/resolver/spec_group.rb
@@ -105,7 +105,7 @@ module Bundler
       end
 
       def metadata_dependency(name, requirement, platform)
-        return if requirement.none?
+        return if requirement.nil? || requirement.none?
 
         DepProxy.get_proxy(Dependency.new("#{name}\0", requirement), platform)
       end
diff --git a/lib/bundler/rubygems_ext.rb b/lib/bundler/rubygems_ext.rb
index 938c58e64d..dee15f8ac2 100644
--- a/lib/bundler/rubygems_ext.rb
+++ b/lib/bundler/rubygems_ext.rb
@@ -15,6 +15,7 @@ require "rubygems/specification"
 # `Gem::Source` from the redefined `Gem::Specification#source`.
 require "rubygems/source"
 
+require_relative "match_metadata"
 require_relative "match_platform"
 
 # Cherry-pick fixes to `Gem.ruby_version` to be useful for modern Bundler
@@ -28,6 +29,7 @@ end
 
 module Gem
   class Specification
+    include ::Bundler::MatchMetadata
     include ::Bundler::MatchPlatform
 
     attr_accessor :remote, :location, :relative_loaded_from
diff --git a/lib/bundler/spec_set.rb b/lib/bundler/spec_set.rb
index 14733269d6..4965ca9e60 100644
--- a/lib/bundler/spec_set.rb
+++ b/lib/bundler/spec_set.rb
@@ -7,8 +7,11 @@ module Bundler
     include Enumerable
     include TSort
 
-    def initialize(specs)
+    attr_reader :incomplete_specs
+
+    def initialize(specs, incomplete_specs = [])
       @specs = specs
+      @incomplete_specs = incomplete_specs
     end
 
     def for(dependencies, check = false, platforms = [nil])
@@ -19,7 +22,10 @@ module Bundler
       loop do
         break unless dep = deps.shift
 
-        key = [dep[0].name, dep[1]]
+        name = dep[0].name
+        platform = dep[1]
+
+        key = [name, platform]
         next if handled.key?(key)
 
         handled[key] = true
@@ -33,7 +39,7 @@ module Bundler
             deps << [d, dep[1]]
           end
         elsif check
-          specs << IncompleteSpecification.new(*key)
+          @incomplete_specs += lookup[name]
         end
       end
 
@@ -51,6 +57,12 @@ module Bundler
       @sorted = nil
     end
 
+    def delete(spec)
+      @specs.delete(spec)
+      @lookup = nil
+      @sorted = nil
+    end
+
     def sort!
       self
     end
@@ -66,7 +78,7 @@ module Bundler
     def materialize(deps)
       materialized = self.for(deps, true)
 
-      SpecSet.new(materialized)
+      SpecSet.new(materialized, incomplete_specs)
     end
 
     # Materialize for all the specs in the spec set, regardless of what platform they're for
@@ -83,17 +95,15 @@ module Bundler
     end
 
     def incomplete_ruby_specs?(deps)
-      self.class.new(self.for(deps, true, [Gem::Platform::RUBY])).incomplete_specs.any?
+      self.for(deps, true, [Gem::Platform::RUBY])
+
+      @incomplete_specs.any?
     end
 
     def missing_specs
       @specs.select {|s| s.is_a?(LazySpecification) }
     end
 
-    def incomplete_specs
-      @specs.select {|s| s.is_a?(IncompleteSpecification) }
-    end
-
     def merge(set)
       arr = sorted.dup
       set.each do |set_spec|
diff --git a/spec/bundler/install/gems/dependency_api_spec.rb b/spec/bundler/install/gems/dependency_api_spec.rb
index 79317a7fad..a3c5bc32aa 100644
--- a/spec/bundler/install/gems/dependency_api_spec.rb
+++ b/spec/bundler/install/gems/dependency_api_spec.rb
@@ -443,6 +443,22 @@ RSpec.describe "gemcutter's dependency API" do
     expect(the_bundle).to include_gems "back_deps 1.0"
   end
 
+  it "does not fetch all marshaled specs" do
+    build_repo2 do
+      build_gem "foo", "1.0"
+      build_gem "foo", "2.0"
+    end
+
+    install_gemfile <<-G, :artifice => "endpoint", :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo2.to_s }, :verbose => true
+      source "#{source_uri}"
+
+      gem "foo"
+    G
+
+    expect(out).to include("foo-2.0.gemspec.rz")
+    expect(out).not_to include("foo-1.0.gemspec.rz")
+  end
+
   it "does not refetch if the only unmet dependency is bundler" do
     build_repo2 do
       build_gem "bundler_dep" do |s|
diff --git a/spec/bundler/install/gems/resolving_spec.rb b/spec/bundler/install/gems/resolving_spec.rb
index 9c0d6bfe56..9405f146b9 100644
--- a/spec/bundler/install/gems/resolving_spec.rb
+++ b/spec/bundler/install/gems/resolving_spec.rb
@@ -305,6 +305,77 @@ RSpec.describe "bundle install with install-time dependencies" do
         end
       end
 
+      context "in a transitive dependencies in a lockfile" do
+        before do
+          build_repo2 do
+            build_gem "rubocop", "1.28.2" do |s|
+              s.required_ruby_version = ">= #{current_ruby_minor}"
+
+              s.add_dependency "rubocop-ast", ">= 1.17.0", "< 2.0"
+            end
+
+            build_gem "rubocop", "1.35.0" do |s|
+              s.required_ruby_version = ">= #{next_ruby_minor}"
+
+              s.add_dependency "rubocop-ast", ">= 1.20.1", "< 2.0"
+            end
+
+            build_gem "rubocop-ast", "1.17.0" do |s|
+              s.required_ruby_version = ">= #{current_ruby_minor}"
+            end
+
+            build_gem "rubocop-ast", "1.21.0" do |s|
+              s.required_ruby_version = ">= #{next_ruby_minor}"
+            end
+          end
+
+          gemfile <<-G
+            source "http://localgemserver.test/"
+            gem 'rubocop'
+          G
+
+          lockfile <<~L
+            GEM
+              remote: http://localgemserver.test/
+              specs:
+                rubocop (1.35.0)
+                  rubocop-ast (>= 1.20.1, < 2.0)
+                rubocop-ast (1.21.0)
+
+            PLATFORMS
+              #{lockfile_platforms}
+
+            DEPENDENCIES
+              parallel_tests
+
+            BUNDLED WITH
+               #{Bundler::VERSION}
+          L
+        end
+
+        it "automatically updates lockfile to use the older compatible versions" do
+          bundle "install --verbose", :artifice => "compact_index", :env => { "BUNDLER_SPEC_GEM_REPO" => gem_repo2.to_s }
+
+          expect(lockfile).to eq <<~L
+            GEM
+              remote: http://localgemserver.test/
+              specs:
+                rubocop (1.28.2)
+                  rubocop-ast (>= 1.17.0, < 2.0)
+                rubocop-ast (1.17.0)
+
+            PLATFORMS
+              #{lockfile_platforms}
+
+            DEPENDENCIES
+              rubocop
+
+            BUNDLED WITH
+               #{Bundler::VERSION}
+          L
+        end
+      end
+
       it "gives a meaningful error on ruby version mismatches between dependencies" do
         build_repo4 do
           build_gem "requires-old-ruby" do |s|