Merge RubyGems/Bundler master.

  Pick from https://github.com/rubygems/rubygems/commit/5ace20dbecfeaf09fba5f616193f3cfcff70ba00

10 files changed, 162 insertions, 104 deletions

diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb
index 09a756abc7..9763298504 100644
--- a/lib/bundler/definition.rb
+++ b/lib/bundler/definition.rb
@@ -159,13 +159,6 @@ module Bundler
       resolve
     end
 
-    def resolve_prefering_local!
-      @prefer_local = true
-      @remote = true
-      sources.remote!
-      resolve
-    end
-
     def resolve_with_cache!
       sources.cached!
       resolve
@@ -177,6 +170,23 @@ module Bundler
       resolve
     end
 
+    def resolution_mode=(options)
+      if options["local"]
+        @remote = false
+      else
+        @remote = true
+        @prefer_local = options["prefer-local"]
+      end
+    end
+
+    def setup_sources_for_resolve
+      if @remote == false
+        sources.cached!
+      else
+        sources.remote!
+      end
+    end
+
     # For given dependency list returns a SpecSet with Gemspec of all the required
     # dependencies.
     #  1. The method first resolves the dependencies specified in Gemfile
@@ -473,31 +483,19 @@ module Bundler
     private
 
     def resolver
-      @resolver ||= begin
-        last_resolve = converge_locked_specs
-        remove_ruby_from_platforms_if_necessary!(current_dependencies)
-        Resolver.new(source_requirements, last_resolve, gem_version_promoter, additional_base_requirements_for_resolve(last_resolve))
-      end
+      @resolver ||= Resolver.new(resolution_packages, gem_version_promoter)
     end
 
     def expanded_dependencies
-      @expanded_dependencies ||= dependencies + metadata_dependencies
+      dependencies + metadata_dependencies
     end
 
     def resolution_packages
       @resolution_packages ||= begin
-        packages = Hash.new do |h, k|
-          h[k] = Resolver::Package.new(k, @platforms, @originally_locked_specs, @unlock[:gems])
-        end
-
-        expanded_dependencies.each do |dep|
-          name = dep.name
-          platforms = dep.gem_platforms(@platforms)
-
-          packages[name] = Resolver::Package.new(name, platforms, @originally_locked_specs, @unlock[:gems], :dependency => dep)
-        end
-
-        packages
+        last_resolve = converge_locked_specs
+        remove_ruby_from_platforms_if_necessary!(current_dependencies)
+        packages = Resolver::Base.new(source_requirements, expanded_dependencies, last_resolve, @platforms, :locked_specs => @originally_locked_specs, :unlock => @unlock[:gems], :prerelease => gem_version_promoter.pre?)
+        additional_base_requirements_for_resolve(packages, last_resolve)
       end
     end
 
@@ -531,13 +529,15 @@ module Bundler
         break if incomplete_specs.empty?
 
         Bundler.ui.debug("The lockfile does not have all gems needed for the current platform though, Bundler will still re-resolve dependencies")
-        @resolve = start_resolution(:exclude_specs => incomplete_specs)
+        setup_sources_for_resolve
+        resolution_packages.delete(incomplete_specs)
+        @resolve = start_resolution
         specs = resolve.materialize(dependencies)
 
         still_incomplete_specs = specs.incomplete_specs
 
         if still_incomplete_specs == incomplete_specs
-          package = resolution_packages[incomplete_specs.first.name]
+          package = resolution_packages.get_package(incomplete_specs.first.name)
           resolver.raise_not_found! package
         end
 
@@ -550,14 +550,14 @@ module Bundler
       specs
     end
 
-    def start_resolution(exclude_specs: [])
-      result = resolver.start(expanded_dependencies, resolution_packages, :exclude_specs => exclude_specs)
+    def start_resolution
+      result = resolver.start
 
       SpecSet.new(SpecSet.new(result).for(dependencies, false, @platforms))
     end
 
     def precompute_source_requirements_for_indirect_dependencies?
-      @remote && sources.non_global_rubygems_sources.all?(&:dependency_api_available?) && !sources.aggregate_global_source?
+      sources.non_global_rubygems_sources.all?(&:dependency_api_available?) && !sources.aggregate_global_source?
     end
 
     def pin_locally_available_names(source_requirements)
@@ -885,11 +885,12 @@ module Bundler
       current == proposed
     end
 
-    def additional_base_requirements_for_resolve(last_resolve)
-      return [] unless @locked_gems && unlocking? && !sources.expired_sources?(@locked_gems.sources)
-      converge_specs(@originally_locked_specs - last_resolve).map do |locked_spec|
-        Dependency.new(locked_spec.name, ">= #{locked_spec.version}")
-      end.uniq
+    def additional_base_requirements_for_resolve(resolution_packages, last_resolve)
+      return resolution_packages unless @locked_gems && unlocking? && !sources.expired_sources?(@locked_gems.sources)
+      converge_specs(@originally_locked_specs - last_resolve).each do |locked_spec|
+        resolution_packages.base_requirements[locked_spec.name] = Gem::Requirement.new(">= #{locked_spec.version}")
+      end
+      resolution_packages
     end
 
     def remove_ruby_from_platforms_if_necessary!(dependencies)
diff --git a/lib/bundler/inline.rb b/lib/bundler/inline.rb
index 855ae5526a..5c184f67a1 100644
--- a/lib/bundler/inline.rb
+++ b/lib/bundler/inline.rb
@@ -31,6 +31,7 @@
 #
 def gemfile(install = false, options = {}, &gemfile)
   require_relative "../bundler"
+  Bundler.reset!
 
   opts = options.dup
   ui = opts.delete(:ui) { Bundler::UI::Shell.new }
diff --git a/lib/bundler/installer.rb b/lib/bundler/installer.rb
index 04d7dc6f23..680e812ac8 100644
--- a/lib/bundler/installer.rb
+++ b/lib/bundler/installer.rb
@@ -249,17 +249,13 @@ module Bundler
 
     # returns whether or not a re-resolve was needed
     def resolve_if_needed(options)
+      @definition.resolution_mode = options
+
       if !@definition.unlocking? && !options["force"] && !Bundler.settings[:inline] && Bundler.default_lockfile.file?
         return false if @definition.nothing_changed? && !@definition.missing_specs?
       end
 
-      if options["local"]
-        @definition.resolve_with_cache!
-      elsif options["prefer-local"]
-        @definition.resolve_prefering_local!
-      else
-        @definition.resolve_remotely!
-      end
+      @definition.setup_sources_for_resolve
 
       true
     end
diff --git a/lib/bundler/lazy_specification.rb b/lib/bundler/lazy_specification.rb
index dccfe48bba..6749892930 100644
--- a/lib/bundler/lazy_specification.rb
+++ b/lib/bundler/lazy_specification.rb
@@ -89,7 +89,7 @@ module Bundler
 
         installable_candidates = GemHelpers.select_best_platform_match(matching_specs, target_platform)
 
-        specification = __materialize__(installable_candidates)
+        specification = __materialize__(installable_candidates, :fallback_to_non_installable => false)
         return specification unless specification.nil?
 
         if target_platform != platform
@@ -102,13 +102,18 @@ module Bundler
       __materialize__(candidates)
     end
 
-    def __materialize__(candidates)
+    # If in frozen mode, we fallback to a non-installable candidate because by
+    # doing this we avoid re-resolving and potentially end up changing the
+    # lock file, which is not allowed. In that case, we will give a proper error
+    # about the mismatch higher up the stack, right before trying to install the
+    # bad gem.
+    def __materialize__(candidates, fallback_to_non_installable: Bundler.frozen_bundle?)
       search = candidates.reverse.find do |spec|
         spec.is_a?(StubSpecification) ||
           (spec.matches_current_ruby? &&
             spec.matches_current_rubygems?)
       end
-      if search.nil? && Bundler.frozen_bundle?
+      if search.nil? && fallback_to_non_installable
         search = candidates.last
       else
         search.dependencies = dependencies if search && search.full_name == full_name && (search.is_a?(RemoteSpecification) || search.is_a?(EndpointSpecification))
diff --git a/lib/bundler/resolver.rb b/lib/bundler/resolver.rb
index df9ce2ff3f..8237ff53fe 100644
--- a/lib/bundler/resolver.rb
+++ b/lib/bundler/resolver.rb
@@ -9,26 +9,21 @@ module Bundler
   class Resolver
     require_relative "vendored_pub_grub"
     require_relative "resolver/base"
-    require_relative "resolver/package"
     require_relative "resolver/candidate"
     require_relative "resolver/incompatibility"
     require_relative "resolver/root"
 
     include GemHelpers
 
-    def initialize(source_requirements, base, gem_version_promoter, additional_base_requirements)
-      @source_requirements = source_requirements
-      @base = Resolver::Base.new(base, additional_base_requirements)
+    def initialize(base, gem_version_promoter)
+      @source_requirements = base.source_requirements
+      @base = base
       @gem_version_promoter = gem_version_promoter
     end
 
-    def start(requirements, packages, exclude_specs: [])
-      exclude_specs.each do |spec|
-        remove_from_candidates(spec)
-      end
-
-      @requirements = requirements
-      @packages = packages
+    def start
+      @requirements = @base.requirements
+      @packages = @base.packages
 
       root, logger = setup_solver
 
@@ -78,33 +73,26 @@ module Bundler
     rescue PubGrub::SolveFailure => e
       incompatibility = e.incompatibility
 
-      names_to_unlock = []
-      extended_explanation = nil
+      names_to_unlock, names_to_allow_prereleases_for, extended_explanation = find_names_to_relax(incompatibility)
 
-      while incompatibility.conflict?
-        cause = incompatibility.cause
-        incompatibility = cause.incompatibility
-
-        incompatibility.terms.each do |term|
-          name = term.package.name
-          names_to_unlock << name if base_requirements[name]
+      names_to_relax = names_to_unlock + names_to_allow_prereleases_for
 
-          no_versions_incompat = [cause.incompatibility, cause.satisfier].find {|incompat| incompat.cause.is_a?(PubGrub::Incompatibility::NoVersions) }
-          next unless no_versions_incompat
+      if names_to_relax.any?
+        if names_to_unlock.any?
+          Bundler.ui.debug "Found conflicts with locked dependencies. Will retry with #{names_to_unlock.join(", ")} unlocked...", true
 
-          extended_explanation = no_versions_incompat.extended_explanation
+          @base.unlock_names(names_to_unlock)
         end
-      end
-
-      names_to_unlock.uniq!
 
-      if names_to_unlock.any?
-        Bundler.ui.debug "Found conflicts with locked dependencies. Retrying with #{names_to_unlock.join(", ")} unlocked...", true
+        if names_to_allow_prereleases_for.any?
+          Bundler.ui.debug "Found conflicts with dependencies with prereleases. Will retrying considering prereleases for #{names_to_allow_prereleases_for.join(", ")}...", true
 
-        @base.unlock_names(names_to_unlock)
+          @base.include_prereleases(names_to_allow_prereleases_for)
+        end
 
         root, logger = setup_solver
 
+        Bundler.ui.debug "Retrying resolution...", true
         retry
       end
 
@@ -118,6 +106,35 @@ module Bundler
       raise SolveFailure.new(explanation)
     end
 
+    def find_names_to_relax(incompatibility)
+      names_to_unlock = []
+      names_to_allow_prereleases_for = []
+      extended_explanation = nil
+
+      while incompatibility.conflict?
+        cause = incompatibility.cause
+        incompatibility = cause.incompatibility
+
+        incompatibility.terms.each do |term|
+          package = term.package
+          name = package.name
+
+          if base_requirements[name]
+            names_to_unlock << name
+          elsif package.ignores_prereleases?
+            names_to_allow_prereleases_for << name
+          end
+
+          no_versions_incompat = [cause.incompatibility, cause.satisfier].find {|incompat| incompat.cause.is_a?(PubGrub::Incompatibility::NoVersions) }
+          next unless no_versions_incompat
+
+          extended_explanation = no_versions_incompat.extended_explanation
+        end
+      end
+
+      [names_to_unlock.uniq, names_to_allow_prereleases_for.uniq, extended_explanation]
+    end
+
     def parse_dependency(package, dependency)
       range = if repository_for(package).is_a?(Source::Gemspec)
         PubGrub::VersionRange.any
@@ -215,7 +232,7 @@ module Bundler
 
     def all_versions_for(package)
       name = package.name
-      results = (@base[name] + @all_specs[name]).uniq {|spec| [spec.version.hash, spec.platform] }
+      results = (@base[name] + filter_prereleases(@all_specs[name], package)).uniq {|spec| [spec.version.hash, spec.platform] }
       locked_requirement = base_requirements[name]
       results = filter_matching_specs(results, locked_requirement) if locked_requirement
 
@@ -284,6 +301,12 @@ module Bundler
       end
     end
 
+    def filter_prereleases(specs, package)
+      return specs unless package.ignores_prereleases?
+
+      specs.reject {|s| s.version.prerelease? }
+    end
+
     def requirement_satisfied_by?(requirement, spec)
       requirement.satisfied_by?(spec.version) || spec.source.is_a?(Source::Gemspec)
     end
@@ -304,25 +327,20 @@ module Bundler
       @base.base_requirements
     end
 
-    def remove_from_candidates(spec)
-      @base.delete(spec)
-    end
-
     def prepare_dependencies(requirements, packages)
       to_dependency_hash(requirements, packages).map do |dep_package, dep_constraint|
         name = dep_package.name
 
-        # If a dependency is scoped to a platform different from the current
-        # one, we ignore it. However, it may reappear during resolution as a
-        # transitive dependency of another package, so we need to reset the
-        # package so the proper versions are considered if reintroduced later.
-        if dep_package.platforms.empty?
-          @packages.delete(name)
-          next
+        next [dep_package, dep_constraint] if name == "bundler"
+
+        versions = versions_for(dep_package, dep_constraint.range)
+        if versions.empty? && dep_package.ignores_prereleases?
+          @sorted_versions.delete(dep_package)
+          dep_package.consider_prereleases!
+          versions = versions_for(dep_package, dep_constraint.range)
         end
+        next [dep_package, dep_constraint] unless versions.empty?
 
-        next [dep_package, dep_constraint] if name == "bundler"
-        next [dep_package, dep_constraint] unless versions_for(dep_package, dep_constraint.range).empty?
         next unless dep_package.current_platform?
 
         raise_not_found!(dep_package)
diff --git a/lib/bundler/resolver/base.rb b/lib/bundler/resolver/base.rb
index 78b798f4ec..6921c047a7 100644
--- a/lib/bundler/resolver/base.rb
+++ b/lib/bundler/resolver/base.rb
@@ -1,19 +1,47 @@
 # frozen_string_literal: true
 
+require_relative "package"
+
 module Bundler
   class Resolver
     class Base
-      def initialize(base, additional_base_requirements)
+      attr_reader :packages, :requirements, :source_requirements
+
+      def initialize(source_requirements, dependencies, base, platforms, options)
+        @source_requirements = source_requirements
+
         @base = base
-        @additional_base_requirements = additional_base_requirements
+
+        @packages = Hash.new do |hash, name|
+          hash[name] = Package.new(name, platforms, **options)
+        end
+
+        @requirements = dependencies.map do |dep|
+          dep_platforms = dep.gem_platforms(platforms)
+
+          # Dependencies scoped to external platforms are ignored
+          next if dep_platforms.empty?
+
+          name = dep.name
+
+          @packages[name] = Package.new(name, dep_platforms, **options.merge(:dependency => dep))
+
+          dep
+        end.compact
       end
 
       def [](name)
         @base[name]
       end
 
-      def delete(spec)
-        @base.delete(spec)
+      def delete(specs)
+        specs.each do |spec|
+          @base.delete(spec)
+        end
+      end
+
+      def get_package(name)
+        @packages[name]
       end
 
       def base_requirements
@@ -24,10 +52,14 @@ module Bundler
         names.each do |name|
           @base.delete_by_name(name)
 
-          @additional_base_requirements.reject! {|dep| dep.name == name }
+          @base_requirements.delete(name)
         end
+      end
 
-        @base_requirements = nil
+      def include_prereleases(names)
+        names.each do |name|
+          get_package(name).consider_prereleases!
+        end
       end
 
       private
@@ -38,7 +70,6 @@ module Bundler
           req = Gem::Requirement.new(ls.version)
           base_requirements[ls.name] = req
         end
-        @additional_base_requirements.each {|d| base_requirements[d.name] = d.requirement }
         base_requirements
       end
     end
diff --git a/lib/bundler/resolver/candidate.rb b/lib/bundler/resolver/candidate.rb
index 681f9aca73..e695ef08ee 100644
--- a/lib/bundler/resolver/candidate.rb
+++ b/lib/bundler/resolver/candidate.rb
@@ -26,9 +26,8 @@ module Bundler
 
       def initialize(version, specs: [])
         @spec_group = Resolver::SpecGroup.new(specs)
-        @platforms = specs.map(&:platform).sort_by(&:to_s).uniq
         @version = Gem::Version.new(version)
-        @ruby_only = @platforms == [Gem::Platform::RUBY]
+        @ruby_only = specs.map(&:platform).uniq == [Gem::Platform::RUBY]
       end
 
       def dependencies
@@ -88,9 +87,7 @@ module Bundler
       end
 
       def to_s
-        return @version.to_s if @platforms.empty? || @ruby_only
-
-        "#{@version} (#{@platforms.join(", ")})"
+        @version.to_s
       end
     end
   end
diff --git a/lib/bundler/resolver/package.rb b/lib/bundler/resolver/package.rb
index 7d64632860..7499a75006 100644
--- a/lib/bundler/resolver/package.rb
+++ b/lib/bundler/resolver/package.rb
@@ -15,12 +15,13 @@ module Bundler
     class Package
       attr_reader :name, :platforms, :dependency, :locked_version
 
-      def initialize(name, platforms, locked_specs, unlock, dependency: nil)
+      def initialize(name, platforms, locked_specs:, unlock:, prerelease: false, dependency: nil)
         @name = name
         @platforms = platforms
         @locked_version = locked_specs[name].first&.version
         @unlock = unlock
         @dependency = dependency || Dependency.new(name, @locked_version)
+        @prerelease = @dependency.prerelease? || @locked_version&.prerelease? || prerelease ? :consider_first : :ignore
       end
 
       def to_s
@@ -47,8 +48,16 @@ module Bundler
         @unlock.empty? || @unlock.include?(name)
       end
 
+      def ignores_prereleases?
+        @prerelease == :ignore
+      end
+
       def prerelease_specified?
-        @dependency.prerelease?
+        @prerelease == :consider_first
+      end
+
+      def consider_prereleases!
+        @prerelease = :consider_last
       end
 
       def force_ruby_platform?
diff --git a/lib/bundler/shared_helpers.rb b/lib/bundler/shared_helpers.rb
index 0a6afe0e5a..794a03e62d 100644
--- a/lib/bundler/shared_helpers.rb
+++ b/lib/bundler/shared_helpers.rb
@@ -284,7 +284,7 @@ module Bundler
       Bundler::SharedHelpers.set_env "BUNDLE_BIN_PATH", exe_file
       Bundler::SharedHelpers.set_env "BUNDLE_GEMFILE", find_gemfile.to_s
       Bundler::SharedHelpers.set_env "BUNDLER_VERSION", Bundler::VERSION
-      Bundler::SharedHelpers.set_env "BUNDLER_SETUP", File.expand_path("setup", __dir__)
+      Bundler::SharedHelpers.set_env "BUNDLER_SETUP", File.expand_path("setup", __dir__) unless RUBY_VERSION < "2.7"
     end
 
     def set_path
diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb
index 7eefd291a2..c39071705a 100644
--- a/lib/bundler/source/rubygems.rb
+++ b/lib/bundler/source/rubygems.rb
@@ -292,7 +292,7 @@ module Bundler
       end
 
       def dependency_api_available?
-        api_fetchers.any?
+        @allow_remote && api_fetchers.any?
       end
 
       protected