summaryrefslogtreecommitdiff
path: root/lib/bundler/fetcher
diff options
context:
space:
mode:
authorhsbt <hsbt@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2017-12-22 23:08:05 +0000
committerhsbt <hsbt@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2017-12-22 23:08:05 +0000
commit7825e8363d4b2ccad8e2d3f5eeba9e26f6656911 (patch)
tree83cbcf419e0feeb2ab0fd063ed85e0776eb0081b /lib/bundler/fetcher
parent73bed0312895322e0fd18310e840356c8e6af812 (diff)
Postponing the Bundler merge.
I faced a big issue about Bundler with ruby core. I have no time to resolve it issue before 2.5 final release. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@61416 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'lib/bundler/fetcher')
-rw-r--r--lib/bundler/fetcher/base.rb52
-rw-r--r--lib/bundler/fetcher/compact_index.rb126
-rw-r--r--lib/bundler/fetcher/dependency.rb82
-rw-r--r--lib/bundler/fetcher/downloader.rb79
-rw-r--r--lib/bundler/fetcher/index.rb52
5 files changed, 0 insertions, 391 deletions
diff --git a/lib/bundler/fetcher/base.rb b/lib/bundler/fetcher/base.rb
deleted file mode 100644
index 27987f670a..0000000000
--- a/lib/bundler/fetcher/base.rb
+++ /dev/null
@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-module Bundler
- class Fetcher
- class Base
- attr_reader :downloader
- attr_reader :display_uri
- attr_reader :remote
-
- def initialize(downloader, remote, display_uri)
- raise "Abstract class" if self.class == Base
- @downloader = downloader
- @remote = remote
- @display_uri = display_uri
- end
-
- def remote_uri
- @remote.uri
- end
-
- def fetch_uri
- @fetch_uri ||= begin
- if remote_uri.host == "rubygems.org"
- uri = remote_uri.dup
- uri.host = "index.rubygems.org"
- uri
- else
- remote_uri
- end
- end
- end
-
- def available?
- true
- end
-
- def api_fetcher?
- false
- end
-
- private
-
- def log_specs(debug_msg)
- if Bundler.ui.debug?
- Bundler.ui.debug debug_msg
- else
- Bundler.ui.info ".", false
- end
- end
- end
- end
-end
diff --git a/lib/bundler/fetcher/compact_index.rb b/lib/bundler/fetcher/compact_index.rb
deleted file mode 100644
index cfc74d642c..0000000000
--- a/lib/bundler/fetcher/compact_index.rb
+++ /dev/null
@@ -1,126 +0,0 @@
-# frozen_string_literal: true
-
-require "bundler/fetcher/base"
-require "bundler/worker"
-
-module Bundler
- autoload :CompactIndexClient, "bundler/compact_index_client"
-
- class Fetcher
- class CompactIndex < Base
- def self.compact_index_request(method_name)
- method = instance_method(method_name)
- undef_method(method_name)
- define_method(method_name) do |*args, &blk|
- begin
- method.bind(self).call(*args, &blk)
- rescue NetworkDownError, CompactIndexClient::Updater::MisMatchedChecksumError => e
- raise HTTPError, e.message
- rescue AuthenticationRequiredError
- # Fail since we got a 401 from the server.
- raise
- rescue HTTPError => e
- Bundler.ui.trace(e)
- nil
- end
- end
- end
-
- def specs(gem_names)
- specs_for_names(gem_names)
- end
- compact_index_request :specs
-
- def specs_for_names(gem_names)
- gem_info = []
- complete_gems = []
- remaining_gems = gem_names.dup
-
- until remaining_gems.empty?
- log_specs "Looking up gems #{remaining_gems.inspect}"
-
- deps = compact_index_client.dependencies(remaining_gems)
- next_gems = deps.map {|d| d[3].map(&:first).flatten(1) }.flatten(1).uniq
- deps.each {|dep| gem_info << dep }
- complete_gems.concat(deps.map(&:first)).uniq!
- remaining_gems = next_gems - complete_gems
- end
- @bundle_worker.stop if @bundle_worker
- @bundle_worker = nil # reset it. Not sure if necessary
-
- gem_info
- end
-
- def fetch_spec(spec)
- spec -= [nil, "ruby", ""]
- contents = compact_index_client.spec(*spec)
- return nil if contents.nil?
- contents.unshift(spec.first)
- contents[3].map! {|d| Gem::Dependency.new(*d) }
- EndpointSpecification.new(*contents)
- end
- compact_index_request :fetch_spec
-
- def available?
- return nil unless SharedHelpers.md5_available?
- user_home = Bundler.user_home
- return nil unless user_home.directory? && user_home.writable?
- # Read info file checksums out of /versions, so we can know if gems are up to date
- fetch_uri.scheme != "file" && compact_index_client.update_and_parse_checksums!
- rescue CompactIndexClient::Updater::MisMatchedChecksumError => e
- Bundler.ui.debug(e.message)
- nil
- end
- compact_index_request :available?
-
- def api_fetcher?
- true
- end
-
- private
-
- def compact_index_client
- @compact_index_client ||= begin
- SharedHelpers.filesystem_access(cache_path) do
- CompactIndexClient.new(cache_path, client_fetcher)
- end.tap do |client|
- client.in_parallel = lambda do |inputs, &blk|
- func = lambda {|object, _index| blk.call(object) }
- worker = bundle_worker(func)
- inputs.each {|input| worker.enq(input) }
- inputs.map { worker.deq }
- end
- end
- end
- end
-
- def bundle_worker(func = nil)
- @bundle_worker ||= begin
- worker_name = "Compact Index (#{display_uri.host})"
- Bundler::Worker.new(Bundler.current_ruby.rbx? ? 1 : 25, worker_name, func)
- end
- @bundle_worker.tap do |worker|
- worker.instance_variable_set(:@func, func) if func
- end
- end
-
- def cache_path
- Bundler.user_cache.join("compact_index", remote.cache_slug)
- end
-
- def client_fetcher
- ClientFetcher.new(self, Bundler.ui)
- end
-
- ClientFetcher = Struct.new(:fetcher, :ui) do
- def call(path, headers)
- fetcher.downloader.fetch(fetcher.fetch_uri + path, headers)
- rescue NetworkDownError => e
- raise unless Bundler.feature_flag.allow_offline_install? && headers["If-None-Match"]
- ui.warn "Using the cached data for the new index because of a network error: #{e}"
- Net::HTTPNotModified.new(nil, nil, nil)
- end
- end
- end
- end
-end
diff --git a/lib/bundler/fetcher/dependency.rb b/lib/bundler/fetcher/dependency.rb
deleted file mode 100644
index 1430d1ebeb..0000000000
--- a/lib/bundler/fetcher/dependency.rb
+++ /dev/null
@@ -1,82 +0,0 @@
-# frozen_string_literal: true
-
-require "bundler/fetcher/base"
-require "cgi"
-
-module Bundler
- class Fetcher
- class Dependency < Base
- def available?
- @available ||= fetch_uri.scheme != "file" && downloader.fetch(dependency_api_uri)
- rescue NetworkDownError => e
- raise HTTPError, e.message
- rescue AuthenticationRequiredError
- # Fail since we got a 401 from the server.
- raise
- rescue HTTPError
- false
- end
-
- def api_fetcher?
- true
- end
-
- def specs(gem_names, full_dependency_list = [], last_spec_list = [])
- query_list = gem_names.uniq - full_dependency_list
-
- log_specs "Query List: #{query_list.inspect}"
-
- return last_spec_list if query_list.empty?
-
- spec_list, deps_list = Bundler::Retry.new("dependency api", FAIL_ERRORS).attempts do
- dependency_specs(query_list)
- end
-
- returned_gems = spec_list.map(&:first).uniq
- specs(deps_list, full_dependency_list + returned_gems, spec_list + last_spec_list)
- rescue MarshalError
- Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over
- Bundler.ui.debug "could not fetch from the dependency API, trying the full index"
- nil
- rescue HTTPError, GemspecError
- Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over
- Bundler.ui.debug "could not fetch from the dependency API\nit's suggested to retry using the full index via `bundle install --full-index`"
- nil
- end
-
- def dependency_specs(gem_names)
- Bundler.ui.debug "Query Gemcutter Dependency Endpoint API: #{gem_names.join(",")}"
-
- gem_list = unmarshalled_dep_gems(gem_names)
- get_formatted_specs_and_deps(gem_list)
- end
-
- def unmarshalled_dep_gems(gem_names)
- gem_list = []
- gem_names.each_slice(Source::Rubygems::API_REQUEST_SIZE) do |names|
- marshalled_deps = downloader.fetch(dependency_api_uri(names)).body
- gem_list.concat(Bundler.load_marshal(marshalled_deps))
- end
- gem_list
- end
-
- def get_formatted_specs_and_deps(gem_list)
- deps_list = []
- spec_list = []
-
- gem_list.each do |s|
- deps_list.concat(s[:dependencies].map(&:first))
- deps = s[:dependencies].map {|n, d| [n, d.split(", ")] }
- spec_list.push([s[:name], s[:number], s[:platform], deps])
- end
- [spec_list, deps_list]
- end
-
- def dependency_api_uri(gem_names = [])
- uri = fetch_uri + "api/v1/dependencies"
- uri.query = "gems=#{CGI.escape(gem_names.sort.join(","))}" if gem_names.any?
- uri
- end
- end
- end
-end
diff --git a/lib/bundler/fetcher/downloader.rb b/lib/bundler/fetcher/downloader.rb
deleted file mode 100644
index cbc5e220bd..0000000000
--- a/lib/bundler/fetcher/downloader.rb
+++ /dev/null
@@ -1,79 +0,0 @@
-# frozen_string_literal: true
-
-module Bundler
- class Fetcher
- class Downloader
- attr_reader :connection
- attr_reader :redirect_limit
-
- def initialize(connection, redirect_limit)
- @connection = connection
- @redirect_limit = redirect_limit
- end
-
- def fetch(uri, options = {}, counter = 0)
- raise HTTPError, "Too many redirects" if counter >= redirect_limit
-
- response = request(uri, options)
- Bundler.ui.debug("HTTP #{response.code} #{response.message} #{uri}")
-
- case response
- when Net::HTTPSuccess, Net::HTTPNotModified
- response
- when Net::HTTPRedirection
- new_uri = URI.parse(response["location"])
- if new_uri.host == uri.host
- new_uri.user = uri.user
- new_uri.password = uri.password
- end
- fetch(new_uri, options, counter + 1)
- when Net::HTTPRequestEntityTooLarge
- raise FallbackError, response.body
- when Net::HTTPUnauthorized
- raise AuthenticationRequiredError, uri.host
- when Net::HTTPNotFound
- raise FallbackError, "Net::HTTPNotFound"
- else
- raise HTTPError, "#{response.class}#{": #{response.body}" unless response.body.empty?}"
- end
- end
-
- def request(uri, options)
- validate_uri_scheme!(uri)
-
- Bundler.ui.debug "HTTP GET #{uri}"
- req = Net::HTTP::Get.new uri.request_uri, options
- if uri.user
- user = CGI.unescape(uri.user)
- password = uri.password ? CGI.unescape(uri.password) : nil
- req.basic_auth(user, password)
- end
- connection.request(uri, req)
- rescue NoMethodError => e
- raise unless ["undefined method", "use_ssl="].all? {|snippet| e.message.include? snippet }
- raise LoadError.new("cannot load such file -- openssl")
- rescue OpenSSL::SSL::SSLError
- raise CertificateFailureError.new(uri)
- rescue *HTTP_ERRORS => e
- Bundler.ui.trace e
- case e.message
- when /host down:/, /getaddrinfo: nodename nor servname provided/
- raise NetworkDownError, "Could not reach host #{uri.host}. Check your network " \
- "connection and try again."
- else
- raise HTTPError, "Network error while fetching #{URICredentialsFilter.credential_filtered_uri(uri)}" \
- " (#{e})"
- end
- end
-
- private
-
- def validate_uri_scheme!(uri)
- return if uri.scheme =~ /\Ahttps?\z/
- raise InvalidOption,
- "The request uri `#{uri}` has an invalid scheme (`#{uri.scheme}`). " \
- "Did you mean `http` or `https`?"
- end
- end
- end
-end
diff --git a/lib/bundler/fetcher/index.rb b/lib/bundler/fetcher/index.rb
deleted file mode 100644
index 9529944391..0000000000
--- a/lib/bundler/fetcher/index.rb
+++ /dev/null
@@ -1,52 +0,0 @@
-# frozen_string_literal: true
-
-require "bundler/fetcher/base"
-require "rubygems/remote_fetcher"
-
-module Bundler
- class Fetcher
- class Index < Base
- def specs(_gem_names)
- Bundler.rubygems.fetch_all_remote_specs(remote)
- rescue Gem::RemoteFetcher::FetchError, OpenSSL::SSL::SSLError, Net::HTTPFatalError => e
- case e.message
- when /certificate verify failed/
- raise CertificateFailureError.new(display_uri)
- when /401/
- raise AuthenticationRequiredError, remote_uri
- when /403/
- raise BadAuthenticationError, remote_uri if remote_uri.userinfo
- raise AuthenticationRequiredError, remote_uri
- else
- Bundler.ui.trace e
- raise HTTPError, "Could not fetch specs from #{display_uri}"
- end
- end
-
- def fetch_spec(spec)
- spec -= [nil, "ruby", ""]
- spec_file_name = "#{spec.join "-"}.gemspec"
-
- uri = URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz")
- if uri.scheme == "file"
- Bundler.load_marshal Gem.inflate(Gem.read_binary(uri.path))
- elsif cached_spec_path = gemspec_cached_path(spec_file_name)
- Bundler.load_gemspec(cached_spec_path)
- else
- Bundler.load_marshal Gem.inflate(downloader.fetch(uri).body)
- end
- rescue MarshalError
- raise HTTPError, "Gemspec #{spec} contained invalid data.\n" \
- "Your network or your gem server is probably having issues right now."
- end
-
- private
-
- # cached gem specification path, if one exists
- def gemspec_cached_path(spec_file_name)
- paths = Bundler.rubygems.spec_cache_dirs.map {|dir| File.join(dir, spec_file_name) }
- paths.find {|path| File.file? path }
- end
- end
- end
-end