Update bundled bundler to 1.16.0.

  * lib/bundler, spec/bundler: Merge bundler-1.16.0.
  * common.mk: rspec examples of bundler-1.16.0 needs require option.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@60603 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 155 insertions, 111 deletions

diff --git a/lib/bundler/definition.rb b/lib/bundler/definition.rb
index 3e5b1bc447..f93ed76226 100644
--- a/lib/bundler/definition.rb
+++ b/lib/bundler/definition.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
+
 require "bundler/lockfile_parser"
-require "digest/sha1"
 require "set"
 
 module Bundler
@@ -14,7 +14,9 @@ module Bundler
       :locked_gems,
       :platforms,
       :requires,
-      :ruby_version
+      :ruby_version,
+      :lockfile,
+      :gemfiles
     )
 
     # Given a gemfile and lockfile creates a Bundler definition
@@ -51,8 +53,16 @@ module Bundler
     #   to be updated or true if all gems should be updated
     # @param ruby_version [Bundler::RubyVersion, nil] Requested Ruby Version
     # @param optional_groups [Array(String)] A list of optional groups
-    def initialize(lockfile, dependencies, sources, unlock, ruby_version = nil, optional_groups = [])
-      @unlocking = unlock == true || !unlock.empty?
+    def initialize(lockfile, dependencies, sources, unlock, ruby_version = nil, optional_groups = [], gemfiles = [])
+      if [true, false].include?(unlock)
+        @unlocking_bundler = false
+        @unlocking = unlock
+      else
+        unlock = unlock.dup
+        @unlocking_bundler = unlock.delete(:bundler)
+        unlock.delete_if {|_k, v| Array(v).empty? }
+        @unlocking = !unlock.empty?
+      end
 
       @dependencies    = dependencies
       @sources         = sources
@@ -61,6 +71,7 @@ module Bundler
       @remote          = false
       @specs           = nil
       @ruby_version    = ruby_version
+      @gemfiles        = gemfiles
 
       @lockfile               = lockfile
       @lockfile_contents      = String.new
@@ -102,7 +113,7 @@ module Bundler
       end
       @unlocking ||= @unlock[:ruby] ||= (!@locked_ruby_version ^ !@ruby_version)
 
-      add_current_platform unless Bundler.settings[:frozen]
+      add_current_platform unless Bundler.frozen?
 
       converge_path_sources_to_gemspec_sources
       @path_changes = converge_paths
@@ -167,9 +178,8 @@ module Bundler
                              "to a different version of #{locked_gem} that hasn't been removed in order to install."
         end
         unless specs["bundler"].any?
-          local = Bundler.settings[:frozen] ? rubygems_index : index
-          bundler = local.search(Gem::Dependency.new("bundler", VERSION)).last
-          specs["bundler"] = bundler if bundler
+          bundler = sources.metadata_source.specs.search(Gem::Dependency.new("bundler", VERSION)).last
+          specs["bundler"] = bundler
         end
 
         specs
@@ -194,10 +204,19 @@ module Bundler
       missing
     end
 
-    def missing_dependencies
-      missing = []
-      resolve.materialize(current_dependencies, missing)
-      missing
+    def missing_specs?
+      missing = missing_specs
+      return false if missing.empty?
+      Bundler.ui.debug "The definition is missing #{missing.map(&:full_name)}"
+      true
+    rescue BundlerError => e
+      @index = nil
+      @resolve = nil
+      @specs = nil
+      @gem_version_promoter = create_gem_version_promoter
+
+      Bundler.ui.debug "The definition is missing dependencies, failed to resolve & materialize locally (#{e})"
+      true
     end
 
     def requested_specs
@@ -226,7 +245,10 @@ module Bundler
     def resolve
       @resolve ||= begin
         last_resolve = converge_locked_specs
-        if Bundler.settings[:frozen] || (!unlocking? && nothing_changed?)
+        if Bundler.frozen?
+          Bundler.ui.debug "Frozen, using resolution from the lockfile"
+          last_resolve
+        elsif !unlocking? && nothing_changed?
           Bundler.ui.debug("Found no changes, using resolution from the lockfile")
           last_resolve
         else
@@ -242,25 +264,44 @@ module Bundler
         dependency_names = @dependencies.map(&:name)
 
         sources.all_sources.each do |source|
-          source.dependency_names = dependency_names.dup
+          source.dependency_names = dependency_names - pinned_spec_names(source)
           idx.add_source source.specs
-          dependency_names -= pinned_spec_names(source.specs)
           dependency_names.concat(source.unmet_deps).uniq!
         end
-        idx << Gem::Specification.new("ruby\0", RubyVersion.system.to_gem_version_with_patchlevel)
-        idx << Gem::Specification.new("rubygems\0", Gem::VERSION)
-      end
-    end
 
-    # used when frozen is enabled so we can find the bundler
-    # spec, even if (say) a git gem is not checked out.
-    def rubygems_index
-      @rubygems_index ||= Index.build do |idx|
-        sources.rubygems_sources.each do |rubygems|
-          idx.add_source rubygems.specs
+        double_check_for_index(idx, dependency_names)
+      end
+    end
+
+    # Suppose the gem Foo depends on the gem Bar.  Foo exists in Source A.  Bar has some versions that exist in both
+    # sources A and B.  At this point, the API request will have found all the versions of Bar in source A,
+    # but will not have found any versions of Bar from source B, which is a problem if the requested version
+    # of Foo specifically depends on a version of Bar that is only found in source B. This ensures that for
+    # each spec we found, we add all possible versions from all sources to the index.
+    def double_check_for_index(idx, dependency_names)
+      pinned_names = pinned_spec_names
+      loop do
+        idxcount = idx.size
+
+        names = :names # do this so we only have to traverse to get dependency_names from the index once
+        unmet_dependency_names = lambda do
+          return names unless names == :names
+          new_names = sources.all_sources.map(&:dependency_names_to_double_check)
+          return names = nil if new_names.compact!
+          names = new_names.flatten(1).concat(dependency_names)
+          names.uniq!
+          names -= pinned_names
+          names
         end
+
+        sources.all_sources.each do |source|
+          source.double_check_for(unmet_dependency_names, :override_dupes)
+        end
+
+        break if idxcount == idx.size
       end
     end
+    private :double_check_for_index
 
     def has_rubygems_remotes?
       sources.rubygems_sources.any? {|s| s.remotes.any? }
@@ -295,10 +336,10 @@ module Bundler
         end
       end
 
-      preserve_unknown_sections ||= !updating_major && (Bundler.settings[:frozen] || !unlocking?)
+      preserve_unknown_sections ||= !updating_major && (Bundler.frozen? || !(unlocking? || @unlocking_bundler))
       return if lockfiles_equal?(@lockfile_contents, contents, preserve_unknown_sections)
 
-      if Bundler.settings[:frozen]
+      if Bundler.frozen?
         Bundler.ui.error "Cannot write a changed lockfile while frozen."
         return
       end
@@ -338,51 +379,8 @@ module Bundler
     end
 
     def to_lock
-      out = String.new
-
-      sources.lock_sources.each do |source|
-        # Add the source header
-        out << source.to_lock
-        # Find all specs for this source
-        resolve.
-          select {|s| source.can_lock?(s) }.
-          # This needs to be sorted by full name so that
-          # gems with the same name, but different platform
-          # are ordered consistently
-          sort_by(&:full_name).
-          each do |spec|
-            next if spec.name == "bundler"
-            out << spec.to_lock
-          end
-        out << "\n"
-      end
-
-      out << "PLATFORMS\n"
-
-      platforms.map(&:to_s).sort.each do |p|
-        out << "  #{p}\n"
-      end
-
-      out << "\n"
-      out << "DEPENDENCIES\n"
-
-      handled = []
-      dependencies.sort_by(&:to_s).each do |dep|
-        next if handled.include?(dep.name)
-        out << dep.to_lock
-        handled << dep.name
-      end
-
-      if locked_ruby_version
-        out << "\nRUBY VERSION\n"
-        out << "   #{locked_ruby_version}\n"
-      end
-
-      # Record the version of Bundler that was used to create the lockfile
-      out << "\nBUNDLED WITH\n"
-      out << "   #{locked_bundler_version}\n"
-
-      out
+      require "bundler/lockfile_generator"
+      LockfileGenerator.generate(self)
     end
 
     def ensure_equivalent_gemfile_and_lockfile(explicit_flag = false)
@@ -392,8 +390,13 @@ module Bundler
              "updated #{Bundler.default_lockfile.relative_path_from(SharedHelpers.pwd)} to version control."
 
       unless explicit_flag
-
-        suggested_command = Bundler.settings.locations("frozen")[:global] == "1" ? "bundle config --delete frozen" : "bundle install --no-deployment"
+        suggested_command = if Bundler.settings.locations("frozen")[:global]
+          "bundle config --delete frozen"
+        elsif Bundler.settings.locations("deployment").keys.&([:global, :local]).any?
+          "bundle config --delete deployment"
+        else
+          "bundle install --no-deployment"
+        end
         msg << "\n\nIf this is a development machine, remove the #{Bundler.default_gemfile} " \
                "freeze \nby running `#{suggested_command}`."
       end
@@ -417,8 +420,8 @@ module Bundler
 
       # Check if it is possible that the source is only changed thing
       if (new_deps.empty? && deleted_deps.empty?) && (!new_sources.empty? && !deleted_sources.empty?)
-        new_sources.reject! {|source| source.is_a_path? && source.path.exist? }
-        deleted_sources.reject! {|source| source.is_a_path? && source.path.exist? }
+        new_sources.reject! {|source| (source.path? && source.path.exist?) || equivalent_rubygems_remotes?(source) }
+        deleted_sources.reject! {|source| (source.path? && source.path.exist?) || equivalent_rubygems_remotes?(source) }
       end
 
       if @locked_sources != gemfile_sources
@@ -511,7 +514,7 @@ module Bundler
 
     def add_current_platform
       current_platform = Bundler.local_platform
-      add_platform(current_platform) if Bundler.settings[:specific_platform]
+      add_platform(current_platform) if Bundler.feature_flag.specific_platform?
       add_platform(generic(current_platform))
     end
 
@@ -558,10 +561,7 @@ module Bundler
     end
 
     def pretty_dep(dep, source = false)
-      msg = String.new(dep.name)
-      msg << " (#{dep.requirement})" unless dep.requirement == Gem::Requirement.default
-      msg << " from the `#{dep.source}` source" if source && dep.source
-      msg
+      SharedHelpers.pretty_dependency(dep, source)
     end
 
     # Check if the specs of the given source changed
@@ -585,6 +585,9 @@ module Bundler
 
       # order here matters, since Index#== is checking source.specs.include?(locked_index)
       locked_index != source.specs
+    rescue PathError, GitError => e
+      Bundler.ui.debug "Assuming that #{source} has not changed since fetching its specs errored (#{e})"
+      false
     end
 
     # Get all locals and override their matching sources.
@@ -632,22 +635,32 @@ module Bundler
       end
     end
 
-    def converge_sources
+    def converge_rubygems_sources
+      return false if Bundler.feature_flag.lockfile_uses_separate_rubygems_sources?
+
       changes = false
 
-      # Get the Rubygems sources from the Gemfile.lock
+      # Get the RubyGems sources from the Gemfile.lock
       locked_gem_sources = @locked_sources.select {|s| s.is_a?(Source::Rubygems) }
-      # Get the Rubygems remotes from the Gemfile
+      # Get the RubyGems remotes from the Gemfile
       actual_remotes = sources.rubygems_remotes
 
-      # If there is a Rubygems source in both
+      # If there is a RubyGems source in both
       if !locked_gem_sources.empty? && !actual_remotes.empty?
         locked_gem_sources.each do |locked_gem|
           # Merge the remotes from the Gemfile into the Gemfile.lock
-          changes |= locked_gem.replace_remotes(actual_remotes)
+          changes |= locked_gem.replace_remotes(actual_remotes, Bundler.settings[:allow_deployment_source_credential_changes])
         end
       end
 
+      changes
+    end
+
+    def converge_sources
+      changes = false
+
+      changes |= converge_rubygems_sources
+
       # Replace the sources from the Gemfile with the sources from the Gemfile.lock,
       # if they exist in the Gemfile.lock and are `==`. If you can't find an equivalent
       # source in the Gemfile.lock, use the one from the Gemfile.
@@ -669,7 +682,7 @@ module Bundler
     end
 
     def converge_dependencies
-      frozen = Bundler.settings[:frozen]
+      frozen = Bundler.frozen?
       (@dependencies + @locked_deps.values).each do |dep|
         locked_source = @locked_deps[dep.name]
         # This is to make sure that if bundler is installing in deployment mode and
@@ -739,6 +752,8 @@ module Bundler
         end
       end
 
+      unlock_source_unlocks_spec = Bundler.feature_flag.unlock_source_unlocks_spec?
+
       converged = []
       @locked_specs.each do |s|
         # Replace the locked dependency's source with the equivalent source from the Gemfile
@@ -746,21 +761,33 @@ module Bundler
         s.source = (dep && dep.source) || sources.get(s.source)
 
         # Don't add a spec to the list if its source is expired. For example,
-        # if you change a Git gem to Rubygems.
+        # if you change a Git gem to RubyGems.
         next if s.source.nil?
         next if @unlock[:sources].include?(s.source.name)
 
         # XXX This is a backwards-compatibility fix to preserve the ability to
         # unlock a single gem by passing its name via `--source`. See issue #3759
         # TODO: delete in Bundler 2
-        next if @unlock[:sources].include?(s.name)
+        next if unlock_source_unlocks_spec && @unlock[:sources].include?(s.name)
 
         # If the spec is from a path source and it doesn't exist anymore
         # then we unlock it.
 
         # Path sources have special logic
         if s.source.instance_of?(Source::Path) || s.source.instance_of?(Source::Gemspec)
-          other = s.source.specs[s].first
+          other_sources_specs = begin
+            s.source.specs
+          rescue PathError, GitError
+            # if we won't need the source (according to the lockfile),
+            # don't error if the path/git source isn't available
+            next if @locked_specs.
+                    for(requested_dependencies, [], false, true, false).
+                    none? {|locked_spec| locked_spec.source == s.source }
+
+            raise
+          end
+
+          other = other_sources_specs[s].first
 
           # If the spec is no longer in the path source, unlock it. This
           # commonly happens if the version changed in the gemspec
@@ -807,17 +834,21 @@ module Bundler
     # the metadata dependencies here
     def expanded_dependencies
       @expanded_dependencies ||= begin
+        expand_dependencies(dependencies + metadata_dependencies, @remote)
+      end
+    end
+
+    def metadata_dependencies
+      @metadata_dependencies ||= begin
         ruby_versions = concat_ruby_version_requirements(@ruby_version)
         if ruby_versions.empty? || !@ruby_version.exact?
           concat_ruby_version_requirements(RubyVersion.system)
           concat_ruby_version_requirements(locked_ruby_version_object) unless @unlock[:ruby]
         end
-
-        metadata_dependencies = [
+        [
           Dependency.new("ruby\0", ruby_versions),
           Dependency.new("rubygems\0", Gem::VERSION),
         ]
-        expand_dependencies(dependencies + metadata_dependencies, @remote)
       end
     end
 
@@ -838,11 +869,12 @@ module Bundler
     end
 
     def expand_dependencies(dependencies, remote = false)
+      sorted_platforms = Resolver.sort_platforms(@platforms)
       deps = []
       dependencies.each do |dep|
         dep = Dependency.new(dep, ">= 0") unless dep.respond_to?(:name)
         next if !remote && !dep.current_platform?
-        platforms = dep.gem_platforms(@platforms)
+        platforms = dep.gem_platforms(sorted_platforms)
         if platforms.empty?
           mapped_platforms = dep.platforms.map {|p| Dependency::PLATFORM_MAP[p] }
           Bundler.ui.warn \
@@ -872,30 +904,33 @@ module Bundler
       # Record the specs available in each gem's source, so that those
       # specs will be available later when the resolver knows where to
       # look for that gemspec (or its dependencies)
-      source_requirements = {}
+      default = sources.default_source
+      source_requirements = { :default => default }
+      default = nil unless Bundler.feature_flag.lockfile_uses_separate_rubygems_sources?
       dependencies.each do |dep|
-        next unless dep.source
-        source_requirements[dep.name] = dep.source.specs
+        next unless source = dep.source || default
+        source_requirements[dep.name] = source
       end
+      metadata_dependencies.each do |dep|
+        source_requirements[dep.name] = sources.metadata_source
+      end
+      source_requirements["bundler"] = sources.metadata_source # needs to come last to override
       source_requirements
     end
 
-    def pinned_spec_names(specs)
-      names = []
-      specs.each do |s|
-        # TODO: when two sources without blocks is an error, we can change
-        # this check to !s.source.is_a?(Source::LocalRubygems). For now,
-        # we need to ask every Rubygems for every gem name.
-        if s.source.is_a?(Source::Git) || s.source.is_a?(Source::Path)
-          names << s.name
-        end
+    def pinned_spec_names(skip = nil)
+      pinned_names = []
+      default = Bundler.feature_flag.lockfile_uses_separate_rubygems_sources? && sources.default_source
+      @dependencies.each do |dep|
+        next unless dep_source = dep.source || default
+        next if dep_source == skip
+        pinned_names << dep.name
       end
-      names.uniq!
-      names
+      pinned_names
     end
 
     def requested_groups
-      groups - Bundler.settings.without - @optional_groups + Bundler.settings.with
+      groups - Bundler.settings[:without] - @optional_groups + Bundler.settings[:with]
     end
 
     def lockfiles_equal?(current, proposed, preserve_unknown_sections)
@@ -930,11 +965,20 @@ module Bundler
 
     def additional_base_requirements_for_resolve
       return [] unless @locked_gems && Bundler.feature_flag.only_update_to_newer_versions?
+      dependencies_by_name = dependencies.group_by(&:name)
       @locked_gems.specs.reduce({}) do |requirements, locked_spec|
-        dep = Gem::Dependency.new(locked_spec.name, ">= #{locked_spec.version}")
-        requirements[locked_spec.name] = DepProxy.new(dep, locked_spec.platform)
+        name = locked_spec.name
+        next requirements if @locked_deps[name] != dependencies_by_name[name]
+        dep = Gem::Dependency.new(name, ">= #{locked_spec.version}")
+        requirements[name] = DepProxy.new(dep, locked_spec.platform)
         requirements
       end.values
     end
+
+    def equivalent_rubygems_remotes?(source)
+      return false unless source.is_a?(Source::Rubygems)
+
+      Bundler.settings[:allow_deployment_source_credential_changes] && source.equivalent_remotes?(sources.rubygems_remotes)
+    end
   end
 end