Added bundler as default gems. Revisit [Feature #12733]

  * bin/*, lib/bundler/*, lib/bundler.rb, spec/bundler, man/*:
    Merge from latest stable branch of bundler/bundler repository and
    added workaround patches. I will backport them into upstream.
  * common.mk, defs/gmake.mk: Added `test-bundler` task for test suite
    of bundler.
  * tool/sync_default_gems.rb: Added sync task for bundler.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@65509 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

5 files changed, 396 insertions, 0 deletions

diff --git a/lib/bundler/fetcher/base.rb b/lib/bundler/fetcher/base.rb
new file mode 100644
index 0000000000..27987f670a
--- /dev/null
+++ b/lib/bundler/fetcher/base.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Bundler
+  class Fetcher
+    class Base
+      attr_reader :downloader
+      attr_reader :display_uri
+      attr_reader :remote
+
+      def initialize(downloader, remote, display_uri)
+        raise "Abstract class" if self.class == Base
+        @downloader = downloader
+        @remote = remote
+        @display_uri = display_uri
+      end
+
+      def remote_uri
+        @remote.uri
+      end
+
+      def fetch_uri
+        @fetch_uri ||= begin
+          if remote_uri.host == "rubygems.org"
+            uri = remote_uri.dup
+            uri.host = "index.rubygems.org"
+            uri
+          else
+            remote_uri
+          end
+        end
+      end
+
+      def available?
+        true
+      end
+
+      def api_fetcher?
+        false
+      end
+
+    private
+
+      def log_specs(debug_msg)
+        if Bundler.ui.debug?
+          Bundler.ui.debug debug_msg
+        else
+          Bundler.ui.info ".", false
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb
new file mode 100644
index 0000000000..cfc74d642c
--- /dev/null
+++ b/lib/bundler/fetcher/compact_index.rb
@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "bundler/fetcher/base"
+require "bundler/worker"
+
+module Bundler
+  autoload :CompactIndexClient, "bundler/compact_index_client"
+
+  class Fetcher
+    class CompactIndex < Base
+      def self.compact_index_request(method_name)
+        method = instance_method(method_name)
+        undef_method(method_name)
+        define_method(method_name) do |*args, &blk|
+          begin
+            method.bind(self).call(*args, &blk)
+          rescue NetworkDownError, CompactIndexClient::Updater::MisMatchedChecksumError => e
+            raise HTTPError, e.message
+          rescue AuthenticationRequiredError
+            # Fail since we got a 401 from the server.
+            raise
+          rescue HTTPError => e
+            Bundler.ui.trace(e)
+            nil
+          end
+        end
+      end
+
+      def specs(gem_names)
+        specs_for_names(gem_names)
+      end
+      compact_index_request :specs
+
+      def specs_for_names(gem_names)
+        gem_info = []
+        complete_gems = []
+        remaining_gems = gem_names.dup
+
+        until remaining_gems.empty?
+          log_specs "Looking up gems #{remaining_gems.inspect}"
+
+          deps = compact_index_client.dependencies(remaining_gems)
+          next_gems = deps.map {|d| d[3].map(&:first).flatten(1) }.flatten(1).uniq
+          deps.each {|dep| gem_info << dep }
+          complete_gems.concat(deps.map(&:first)).uniq!
+          remaining_gems = next_gems - complete_gems
+        end
+        @bundle_worker.stop if @bundle_worker
+        @bundle_worker = nil # reset it.  Not sure if necessary
+
+        gem_info
+      end
+
+      def fetch_spec(spec)
+        spec -= [nil, "ruby", ""]
+        contents = compact_index_client.spec(*spec)
+        return nil if contents.nil?
+        contents.unshift(spec.first)
+        contents[3].map! {|d| Gem::Dependency.new(*d) }
+        EndpointSpecification.new(*contents)
+      end
+      compact_index_request :fetch_spec
+
+      def available?
+        return nil unless SharedHelpers.md5_available?
+        user_home = Bundler.user_home
+        return nil unless user_home.directory? && user_home.writable?
+        # Read info file checksums out of /versions, so we can know if gems are up to date
+        fetch_uri.scheme != "file" && compact_index_client.update_and_parse_checksums!
+      rescue CompactIndexClient::Updater::MisMatchedChecksumError => e
+        Bundler.ui.debug(e.message)
+        nil
+      end
+      compact_index_request :available?
+
+      def api_fetcher?
+        true
+      end
+
+    private
+
+      def compact_index_client
+        @compact_index_client ||= begin
+          SharedHelpers.filesystem_access(cache_path) do
+            CompactIndexClient.new(cache_path, client_fetcher)
+          end.tap do |client|
+            client.in_parallel = lambda do |inputs, &blk|
+              func = lambda {|object, _index| blk.call(object) }
+              worker = bundle_worker(func)
+              inputs.each {|input| worker.enq(input) }
+              inputs.map { worker.deq }
+            end
+          end
+        end
+      end
+
+      def bundle_worker(func = nil)
+        @bundle_worker ||= begin
+          worker_name = "Compact Index (#{display_uri.host})"
+          Bundler::Worker.new(Bundler.current_ruby.rbx? ? 1 : 25, worker_name, func)
+        end
+        @bundle_worker.tap do |worker|
+          worker.instance_variable_set(:@func, func) if func
+        end
+      end
+
+      def cache_path
+        Bundler.user_cache.join("compact_index", remote.cache_slug)
+      end
+
+      def client_fetcher
+        ClientFetcher.new(self, Bundler.ui)
+      end
+
+      ClientFetcher = Struct.new(:fetcher, :ui) do
+        def call(path, headers)
+          fetcher.downloader.fetch(fetcher.fetch_uri + path, headers)
+        rescue NetworkDownError => e
+          raise unless Bundler.feature_flag.allow_offline_install? && headers["If-None-Match"]
+          ui.warn "Using the cached data for the new index because of a network error: #{e}"
+          Net::HTTPNotModified.new(nil, nil, nil)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bundler/fetcher/dependency.rb b/lib/bundler/fetcher/dependency.rb
new file mode 100644
index 0000000000..1430d1ebeb
--- /dev/null
+++ b/lib/bundler/fetcher/dependency.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require "bundler/fetcher/base"
+require "cgi"
+
+module Bundler
+  class Fetcher
+    class Dependency < Base
+      def available?
+        @available ||= fetch_uri.scheme != "file" && downloader.fetch(dependency_api_uri)
+      rescue NetworkDownError => e
+        raise HTTPError, e.message
+      rescue AuthenticationRequiredError
+        # Fail since we got a 401 from the server.
+        raise
+      rescue HTTPError
+        false
+      end
+
+      def api_fetcher?
+        true
+      end
+
+      def specs(gem_names, full_dependency_list = [], last_spec_list = [])
+        query_list = gem_names.uniq - full_dependency_list
+
+        log_specs "Query List: #{query_list.inspect}"
+
+        return last_spec_list if query_list.empty?
+
+        spec_list, deps_list = Bundler::Retry.new("dependency api", FAIL_ERRORS).attempts do
+          dependency_specs(query_list)
+        end
+
+        returned_gems = spec_list.map(&:first).uniq
+        specs(deps_list, full_dependency_list + returned_gems, spec_list + last_spec_list)
+      rescue MarshalError
+        Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over
+        Bundler.ui.debug "could not fetch from the dependency API, trying the full index"
+        nil
+      rescue HTTPError, GemspecError
+        Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over
+        Bundler.ui.debug "could not fetch from the dependency API\nit's suggested to retry using the full index via `bundle install --full-index`"
+        nil
+      end
+
+      def dependency_specs(gem_names)
+        Bundler.ui.debug "Query Gemcutter Dependency Endpoint API: #{gem_names.join(",")}"
+
+        gem_list = unmarshalled_dep_gems(gem_names)
+        get_formatted_specs_and_deps(gem_list)
+      end
+
+      def unmarshalled_dep_gems(gem_names)
+        gem_list = []
+        gem_names.each_slice(Source::Rubygems::API_REQUEST_SIZE) do |names|
+          marshalled_deps = downloader.fetch(dependency_api_uri(names)).body
+          gem_list.concat(Bundler.load_marshal(marshalled_deps))
+        end
+        gem_list
+      end
+
+      def get_formatted_specs_and_deps(gem_list)
+        deps_list = []
+        spec_list = []
+
+        gem_list.each do |s|
+          deps_list.concat(s[:dependencies].map(&:first))
+          deps = s[:dependencies].map {|n, d| [n, d.split(", ")] }
+          spec_list.push([s[:name], s[:number], s[:platform], deps])
+        end
+        [spec_list, deps_list]
+      end
+
+      def dependency_api_uri(gem_names = [])
+        uri = fetch_uri + "api/v1/dependencies"
+        uri.query = "gems=#{CGI.escape(gem_names.sort.join(","))}" if gem_names.any?
+        uri
+      end
+    end
+  end
+end
diff --git a/lib/bundler/fetcher/downloader.rb b/lib/bundler/fetcher/downloader.rb
new file mode 100644
index 0000000000..e0e0cbf1c9
--- /dev/null
+++ b/lib/bundler/fetcher/downloader.rb
@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Bundler
+  class Fetcher
+    class Downloader
+      attr_reader :connection
+      attr_reader :redirect_limit
+
+      def initialize(connection, redirect_limit)
+        @connection = connection
+        @redirect_limit = redirect_limit
+      end
+
+      def fetch(uri, headers = {}, counter = 0)
+        raise HTTPError, "Too many redirects" if counter >= redirect_limit
+
+        response = request(uri, headers)
+        Bundler.ui.debug("HTTP #{response.code} #{response.message} #{uri}")
+
+        case response
+        when Net::HTTPSuccess, Net::HTTPNotModified
+          response
+        when Net::HTTPRedirection
+          new_uri = URI.parse(response["location"])
+          if new_uri.host == uri.host
+            new_uri.user = uri.user
+            new_uri.password = uri.password
+          end
+          fetch(new_uri, headers, counter + 1)
+        when Net::HTTPRequestedRangeNotSatisfiable
+          new_headers = headers.dup
+          new_headers.delete("Range")
+          new_headers["Accept-Encoding"] = "gzip"
+          fetch(uri, new_headers)
+        when Net::HTTPRequestEntityTooLarge
+          raise FallbackError, response.body
+        when Net::HTTPUnauthorized
+          raise AuthenticationRequiredError, uri.host
+        when Net::HTTPNotFound
+          raise FallbackError, "Net::HTTPNotFound"
+        else
+          raise HTTPError, "#{response.class}#{": #{response.body}" unless response.body.empty?}"
+        end
+      end
+
+      def request(uri, headers)
+        validate_uri_scheme!(uri)
+
+        Bundler.ui.debug "HTTP GET #{uri}"
+        req = Net::HTTP::Get.new uri.request_uri, headers
+        if uri.user
+          user = CGI.unescape(uri.user)
+          password = uri.password ? CGI.unescape(uri.password) : nil
+          req.basic_auth(user, password)
+        end
+        connection.request(uri, req)
+      rescue NoMethodError => e
+        raise unless ["undefined method", "use_ssl="].all? {|snippet| e.message.include? snippet }
+        raise LoadError.new("cannot load such file -- openssl")
+      rescue OpenSSL::SSL::SSLError
+        raise CertificateFailureError.new(uri)
+      rescue *HTTP_ERRORS => e
+        Bundler.ui.trace e
+        case e.message
+        when /host down:/, /getaddrinfo: nodename nor servname provided/
+          raise NetworkDownError, "Could not reach host #{uri.host}. Check your network " \
+            "connection and try again."
+        else
+          raise HTTPError, "Network error while fetching #{URICredentialsFilter.credential_filtered_uri(uri)}" \
+            " (#{e})"
+        end
+      end
+
+    private
+
+      def validate_uri_scheme!(uri)
+        return if uri.scheme =~ /\Ahttps?\z/
+        raise InvalidOption,
+          "The request uri `#{uri}` has an invalid scheme (`#{uri.scheme}`). " \
+          "Did you mean `http` or `https`?"
+      end
+    end
+  end
+end
diff --git a/lib/bundler/fetcher/index.rb b/lib/bundler/fetcher/index.rb
new file mode 100644
index 0000000000..1a8064624d
--- /dev/null
+++ b/lib/bundler/fetcher/index.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "bundler/fetcher/base"
+require "rubygems/remote_fetcher"
+
+module Bundler
+  class Fetcher
+    class Index < Base
+      def specs(_gem_names)
+        Bundler.rubygems.fetch_all_remote_specs(remote)
+      rescue Gem::RemoteFetcher::FetchError, OpenSSL::SSL::SSLError, Net::HTTPFatalError => e
+        case e.message
+        when /certificate verify failed/
+          raise CertificateFailureError.new(display_uri)
+        when /401/
+          raise AuthenticationRequiredError, remote_uri
+        when /403/
+          raise BadAuthenticationError, remote_uri if remote_uri.userinfo
+          raise AuthenticationRequiredError, remote_uri
+        else
+          Bundler.ui.trace e
+          raise HTTPError, "Could not fetch specs from #{display_uri}"
+        end
+      end
+
+      def fetch_spec(spec)
+        spec -= [nil, "ruby", ""]
+        spec_file_name = "#{spec.join "-"}.gemspec"
+
+        uri = URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz")
+        if uri.scheme == "file"
+          Bundler.load_marshal Bundler.rubygems.inflate(Gem.read_binary(uri.path))
+        elsif cached_spec_path = gemspec_cached_path(spec_file_name)
+          Bundler.load_gemspec(cached_spec_path)
+        else
+          Bundler.load_marshal Bundler.rubygems.inflate(downloader.fetch(uri).body)
+        end
+      rescue MarshalError
+        raise HTTPError, "Gemspec #{spec} contained invalid data.\n" \
+          "Your network or your gem server is probably having issues right now."
+      end
+
+    private
+
+      # cached gem specification path, if one exists
+      def gemspec_cached_path(spec_file_name)
+        paths = Bundler.rubygems.spec_cache_dirs.map {|dir| File.join(dir, spec_file_name) }
+        paths.find {|path| File.file? path }
+      end
+    end
+  end
+end