diff options
author | hsbt <hsbt@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2017-12-22 23:08:05 +0000 |
---|---|---|
committer | hsbt <hsbt@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2017-12-22 23:08:05 +0000 |
commit | 7825e8363d4b2ccad8e2d3f5eeba9e26f6656911 (patch) | |
tree | 83cbcf419e0feeb2ab0fd063ed85e0776eb0081b /lib/bundler/fetcher | |
parent | 73bed0312895322e0fd18310e840356c8e6af812 (diff) |
Postponing the Bundler merge.
I faced a big issue about Bundler with ruby core.
I have no time to resolve it issue before 2.5 final release.
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@61416 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'lib/bundler/fetcher')
-rw-r--r-- | lib/bundler/fetcher/base.rb | 52 | ||||
-rw-r--r-- | lib/bundler/fetcher/compact_index.rb | 126 | ||||
-rw-r--r-- | lib/bundler/fetcher/dependency.rb | 82 | ||||
-rw-r--r-- | lib/bundler/fetcher/downloader.rb | 79 | ||||
-rw-r--r-- | lib/bundler/fetcher/index.rb | 52 |
5 files changed, 0 insertions, 391 deletions
diff --git a/lib/bundler/fetcher/base.rb b/lib/bundler/fetcher/base.rb deleted file mode 100644 index 27987f670a..0000000000 --- a/lib/bundler/fetcher/base.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -module Bundler - class Fetcher - class Base - attr_reader :downloader - attr_reader :display_uri - attr_reader :remote - - def initialize(downloader, remote, display_uri) - raise "Abstract class" if self.class == Base - @downloader = downloader - @remote = remote - @display_uri = display_uri - end - - def remote_uri - @remote.uri - end - - def fetch_uri - @fetch_uri ||= begin - if remote_uri.host == "rubygems.org" - uri = remote_uri.dup - uri.host = "index.rubygems.org" - uri - else - remote_uri - end - end - end - - def available? - true - end - - def api_fetcher? - false - end - - private - - def log_specs(debug_msg) - if Bundler.ui.debug? - Bundler.ui.debug debug_msg - else - Bundler.ui.info ".", false - end - end - end - end -end diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb deleted file mode 100644 index cfc74d642c..0000000000 --- a/lib/bundler/fetcher/compact_index.rb +++ /dev/null @@ -1,126 +0,0 @@ -# frozen_string_literal: true - -require "bundler/fetcher/base" -require "bundler/worker" - -module Bundler - autoload :CompactIndexClient, "bundler/compact_index_client" - - class Fetcher - class CompactIndex < Base - def self.compact_index_request(method_name) - method = instance_method(method_name) - undef_method(method_name) - define_method(method_name) do |*args, &blk| - begin - method.bind(self).call(*args, &blk) - rescue NetworkDownError, CompactIndexClient::Updater::MisMatchedChecksumError => e - raise HTTPError, e.message - rescue AuthenticationRequiredError - # Fail since we got a 401 from the server. - raise - rescue HTTPError => e - Bundler.ui.trace(e) - nil - end - end - end - - def specs(gem_names) - specs_for_names(gem_names) - end - compact_index_request :specs - - def specs_for_names(gem_names) - gem_info = [] - complete_gems = [] - remaining_gems = gem_names.dup - - until remaining_gems.empty? - log_specs "Looking up gems #{remaining_gems.inspect}" - - deps = compact_index_client.dependencies(remaining_gems) - next_gems = deps.map {|d| d[3].map(&:first).flatten(1) }.flatten(1).uniq - deps.each {|dep| gem_info << dep } - complete_gems.concat(deps.map(&:first)).uniq! - remaining_gems = next_gems - complete_gems - end - @bundle_worker.stop if @bundle_worker - @bundle_worker = nil # reset it. Not sure if necessary - - gem_info - end - - def fetch_spec(spec) - spec -= [nil, "ruby", ""] - contents = compact_index_client.spec(*spec) - return nil if contents.nil? - contents.unshift(spec.first) - contents[3].map! {|d| Gem::Dependency.new(*d) } - EndpointSpecification.new(*contents) - end - compact_index_request :fetch_spec - - def available? - return nil unless SharedHelpers.md5_available? - user_home = Bundler.user_home - return nil unless user_home.directory? && user_home.writable? - # Read info file checksums out of /versions, so we can know if gems are up to date - fetch_uri.scheme != "file" && compact_index_client.update_and_parse_checksums! - rescue CompactIndexClient::Updater::MisMatchedChecksumError => e - Bundler.ui.debug(e.message) - nil - end - compact_index_request :available? - - def api_fetcher? - true - end - - private - - def compact_index_client - @compact_index_client ||= begin - SharedHelpers.filesystem_access(cache_path) do - CompactIndexClient.new(cache_path, client_fetcher) - end.tap do |client| - client.in_parallel = lambda do |inputs, &blk| - func = lambda {|object, _index| blk.call(object) } - worker = bundle_worker(func) - inputs.each {|input| worker.enq(input) } - inputs.map { worker.deq } - end - end - end - end - - def bundle_worker(func = nil) - @bundle_worker ||= begin - worker_name = "Compact Index (#{display_uri.host})" - Bundler::Worker.new(Bundler.current_ruby.rbx? ? 1 : 25, worker_name, func) - end - @bundle_worker.tap do |worker| - worker.instance_variable_set(:@func, func) if func - end - end - - def cache_path - Bundler.user_cache.join("compact_index", remote.cache_slug) - end - - def client_fetcher - ClientFetcher.new(self, Bundler.ui) - end - - ClientFetcher = Struct.new(:fetcher, :ui) do - def call(path, headers) - fetcher.downloader.fetch(fetcher.fetch_uri + path, headers) - rescue NetworkDownError => e - raise unless Bundler.feature_flag.allow_offline_install? && headers["If-None-Match"] - ui.warn "Using the cached data for the new index because of a network error: #{e}" - Net::HTTPNotModified.new(nil, nil, nil) - end - end - end - end -end diff --git a/lib/bundler/fetcher/dependency.rb b/lib/bundler/fetcher/dependency.rb deleted file mode 100644 index 1430d1ebeb..0000000000 --- a/lib/bundler/fetcher/dependency.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true - -require "bundler/fetcher/base" -require "cgi" - -module Bundler - class Fetcher - class Dependency < Base - def available? - @available ||= fetch_uri.scheme != "file" && downloader.fetch(dependency_api_uri) - rescue NetworkDownError => e - raise HTTPError, e.message - rescue AuthenticationRequiredError - # Fail since we got a 401 from the server. - raise - rescue HTTPError - false - end - - def api_fetcher? - true - end - - def specs(gem_names, full_dependency_list = [], last_spec_list = []) - query_list = gem_names.uniq - full_dependency_list - - log_specs "Query List: #{query_list.inspect}" - - return last_spec_list if query_list.empty? - - spec_list, deps_list = Bundler::Retry.new("dependency api", FAIL_ERRORS).attempts do - dependency_specs(query_list) - end - - returned_gems = spec_list.map(&:first).uniq - specs(deps_list, full_dependency_list + returned_gems, spec_list + last_spec_list) - rescue MarshalError - Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over - Bundler.ui.debug "could not fetch from the dependency API, trying the full index" - nil - rescue HTTPError, GemspecError - Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over - Bundler.ui.debug "could not fetch from the dependency API\nit's suggested to retry using the full index via `bundle install --full-index`" - nil - end - - def dependency_specs(gem_names) - Bundler.ui.debug "Query Gemcutter Dependency Endpoint API: #{gem_names.join(",")}" - - gem_list = unmarshalled_dep_gems(gem_names) - get_formatted_specs_and_deps(gem_list) - end - - def unmarshalled_dep_gems(gem_names) - gem_list = [] - gem_names.each_slice(Source::Rubygems::API_REQUEST_SIZE) do |names| - marshalled_deps = downloader.fetch(dependency_api_uri(names)).body - gem_list.concat(Bundler.load_marshal(marshalled_deps)) - end - gem_list - end - - def get_formatted_specs_and_deps(gem_list) - deps_list = [] - spec_list = [] - - gem_list.each do |s| - deps_list.concat(s[:dependencies].map(&:first)) - deps = s[:dependencies].map {|n, d| [n, d.split(", ")] } - spec_list.push([s[:name], s[:number], s[:platform], deps]) - end - [spec_list, deps_list] - end - - def dependency_api_uri(gem_names = []) - uri = fetch_uri + "api/v1/dependencies" - uri.query = "gems=#{CGI.escape(gem_names.sort.join(","))}" if gem_names.any? - uri - end - end - end -end diff --git a/lib/bundler/fetcher/downloader.rb b/lib/bundler/fetcher/downloader.rb deleted file mode 100644 index cbc5e220bd..0000000000 --- a/lib/bundler/fetcher/downloader.rb +++ /dev/null @@ -1,79 +0,0 @@ -# frozen_string_literal: true - -module Bundler - class Fetcher - class Downloader - attr_reader :connection - attr_reader :redirect_limit - - def initialize(connection, redirect_limit) - @connection = connection - @redirect_limit = redirect_limit - end - - def fetch(uri, options = {}, counter = 0) - raise HTTPError, "Too many redirects" if counter >= redirect_limit - - response = request(uri, options) - Bundler.ui.debug("HTTP #{response.code} #{response.message} #{uri}") - - case response - when Net::HTTPSuccess, Net::HTTPNotModified - response - when Net::HTTPRedirection - new_uri = URI.parse(response["location"]) - if new_uri.host == uri.host - new_uri.user = uri.user - new_uri.password = uri.password - end - fetch(new_uri, options, counter + 1) - when Net::HTTPRequestEntityTooLarge - raise FallbackError, response.body - when Net::HTTPUnauthorized - raise AuthenticationRequiredError, uri.host - when Net::HTTPNotFound - raise FallbackError, "Net::HTTPNotFound" - else - raise HTTPError, "#{response.class}#{": #{response.body}" unless response.body.empty?}" - end - end - - def request(uri, options) - validate_uri_scheme!(uri) - - Bundler.ui.debug "HTTP GET #{uri}" - req = Net::HTTP::Get.new uri.request_uri, options - if uri.user - user = CGI.unescape(uri.user) - password = uri.password ? CGI.unescape(uri.password) : nil - req.basic_auth(user, password) - end - connection.request(uri, req) - rescue NoMethodError => e - raise unless ["undefined method", "use_ssl="].all? {|snippet| e.message.include? snippet } - raise LoadError.new("cannot load such file -- openssl") - rescue OpenSSL::SSL::SSLError - raise CertificateFailureError.new(uri) - rescue *HTTP_ERRORS => e - Bundler.ui.trace e - case e.message - when /host down:/, /getaddrinfo: nodename nor servname provided/ - raise NetworkDownError, "Could not reach host #{uri.host}. Check your network " \ - "connection and try again." - else - raise HTTPError, "Network error while fetching #{URICredentialsFilter.credential_filtered_uri(uri)}" \ - " (#{e})" - end - end - - private - - def validate_uri_scheme!(uri) - return if uri.scheme =~ /\Ahttps?\z/ - raise InvalidOption, - "The request uri `#{uri}` has an invalid scheme (`#{uri.scheme}`). " \ - "Did you mean `http` or `https`?" - end - end - end -end diff --git a/lib/bundler/fetcher/index.rb b/lib/bundler/fetcher/index.rb deleted file mode 100644 index 9529944391..0000000000 --- a/lib/bundler/fetcher/index.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -require "bundler/fetcher/base" -require "rubygems/remote_fetcher" - -module Bundler - class Fetcher - class Index < Base - def specs(_gem_names) - Bundler.rubygems.fetch_all_remote_specs(remote) - rescue Gem::RemoteFetcher::FetchError, OpenSSL::SSL::SSLError, Net::HTTPFatalError => e - case e.message - when /certificate verify failed/ - raise CertificateFailureError.new(display_uri) - when /401/ - raise AuthenticationRequiredError, remote_uri - when /403/ - raise BadAuthenticationError, remote_uri if remote_uri.userinfo - raise AuthenticationRequiredError, remote_uri - else - Bundler.ui.trace e - raise HTTPError, "Could not fetch specs from #{display_uri}" - end - end - - def fetch_spec(spec) - spec -= [nil, "ruby", ""] - spec_file_name = "#{spec.join "-"}.gemspec" - - uri = URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz") - if uri.scheme == "file" - Bundler.load_marshal Gem.inflate(Gem.read_binary(uri.path)) - elsif cached_spec_path = gemspec_cached_path(spec_file_name) - Bundler.load_gemspec(cached_spec_path) - else - Bundler.load_marshal Gem.inflate(downloader.fetch(uri).body) - end - rescue MarshalError - raise HTTPError, "Gemspec #{spec} contained invalid data.\n" \ - "Your network or your gem server is probably having issues right now." - end - - private - - # cached gem specification path, if one exists - def gemspec_cached_path(spec_file_name) - paths = Bundler.rubygems.spec_cache_dirs.map {|dir| File.join(dir, spec_file_name) } - paths.find {|path| File.file? path } - end - end - end -end |