diff options
Diffstat (limited to 'lib/bundler/fetcher.rb')
-rw-r--r-- | lib/bundler/fetcher.rb | 150 |
1 files changed, 80 insertions, 70 deletions
diff --git a/lib/bundler/fetcher.rb b/lib/bundler/fetcher.rb index e9d5dd505c..6288b22dcd 100644 --- a/lib/bundler/fetcher.rb +++ b/lib/bundler/fetcher.rb @@ -1,14 +1,15 @@ # frozen_string_literal: true require_relative "vendored_persistent" +require_relative "vendored_timeout" require "cgi" require "securerandom" require "zlib" -require "rubygems/request" module Bundler # Handles all the fetching with the rubygems server class Fetcher + autoload :Base, File.expand_path("fetcher/base", __dir__) autoload :CompactIndex, File.expand_path("fetcher/compact_index", __dir__) autoload :Downloader, File.expand_path("fetcher/downloader", __dir__) autoload :Dependency, File.expand_path("fetcher/dependency", __dir__) @@ -29,9 +30,7 @@ module Bundler " is a chance you are experiencing a man-in-the-middle attack, but" \ " most likely your system doesn't have the CA certificates needed" \ " for verification. For information about OpenSSL certificates, see" \ - " https://railsapps.github.io/openssl-certificate-verify-failed.html." \ - " To connect without using SSL, edit your Gemfile" \ - " sources and change 'https' to 'http'." + " https://railsapps.github.io/openssl-certificate-verify-failed.html." end end @@ -39,9 +38,7 @@ module Bundler class SSLError < HTTPError def initialize(msg = nil) super msg || "Could not load OpenSSL.\n" \ - "You must recompile Ruby with OpenSSL support or change the sources in your " \ - "Gemfile from 'https' to 'http'. Instructions for compiling with OpenSSL " \ - "using RVM are available at rvm.io/packages/openssl." + "You must recompile Ruby with OpenSSL support." end end @@ -65,6 +62,16 @@ module Bundler end end + # This error is raised if HTTP authentication is correct, but lacks + # necessary permissions. + class AuthenticationForbiddenError < HTTPError + def initialize(remote_uri) + remote_uri = filter_uri(remote_uri) + super "Access token could not be authenticated for #{remote_uri}.\n" \ + "Make sure it's valid and has the necessary scopes configured." + end + end + # Exceptions classes that should bypass retry attempts. If your password didn't work the # first time, it's not going to the third time. NET_ERRORS = [:HTTPBadGateway, :HTTPBadRequest, :HTTPFailedDependency, @@ -74,9 +81,9 @@ module Bundler :HTTPRequestURITooLong, :HTTPUnauthorized, :HTTPUnprocessableEntity, :HTTPUnsupportedMediaType, :HTTPVersionNotSupported].freeze FAIL_ERRORS = begin - fail_errors = [AuthenticationRequiredError, BadAuthenticationError, FallbackError] + fail_errors = [AuthenticationRequiredError, BadAuthenticationError, AuthenticationForbiddenError, FallbackError, SecurityError] fail_errors << Gem::Requirement::BadRequirementError - fail_errors.concat(NET_ERRORS.map {|e| Net.const_get(e) }) + fail_errors.concat(NET_ERRORS.map {|e| Gem::Net.const_get(e) }) end.freeze class << self @@ -88,6 +95,7 @@ module Bundler self.max_retries = Bundler.settings[:retry] # How many retries for the API call def initialize(remote) + @cis = nil @remote = remote Socket.do_not_reverse_lookup = true @@ -103,15 +111,17 @@ module Bundler spec -= [nil, "ruby", ""] spec_file_name = "#{spec.join "-"}.gemspec" - uri = Bundler::URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz") - if uri.scheme == "file" - path = Bundler.rubygems.correct_for_windows_path(uri.path) - Bundler.load_marshal Bundler.rubygems.inflate(Gem.read_binary(path)) + uri = Gem::URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz") + spec = if uri.scheme == "file" + path = Gem::Util.correct_for_windows_path(uri.path) + Bundler.safe_load_marshal Bundler.rubygems.inflate(Gem.read_binary(path)) elsif cached_spec_path = gemspec_cached_path(spec_file_name) Bundler.load_gemspec(cached_spec_path) else - Bundler.load_marshal Bundler.rubygems.inflate(downloader.fetch(uri).body) + Bundler.safe_load_marshal Bundler.rubygems.inflate(downloader.fetch(uri).body) end + raise MarshalError, "is #{spec.inspect}" unless spec.is_a?(Gem::Specification) + spec rescue MarshalError raise HTTPError, "Gemspec #{spec} contained invalid data.\n" \ "Your network or your gem server is probably having issues right now." @@ -128,20 +138,11 @@ module Bundler def specs(gem_names, source) index = Bundler::Index.new - if Bundler::Fetcher.disable_endpoint - @use_api = false - specs = fetchers.last.specs(gem_names) - else - specs = [] - @fetchers = fetchers.drop_while do |f| - !f.available? || (f.api_fetcher? && !gem_names) || !specs = f.specs(gem_names) - end - @use_api = false if fetchers.none?(&:api_fetcher?) - end - - specs.each do |name, version, platform, dependencies, metadata| + fetch_specs(gem_names).each do |name, version, platform, dependencies, metadata| spec = if dependencies - EndpointSpecification.new(name, version, platform, self, dependencies, metadata) + EndpointSpecification.new(name, version, platform, self, dependencies, metadata).tap do |es| + source.checksum_store.replace(es, es.checksum) + end else RemoteSpecification.new(name, version, platform, self) end @@ -152,22 +153,10 @@ module Bundler index rescue CertificateFailureError - Bundler.ui.info "" if gem_names && use_api # newline after dots + Bundler.ui.info "" if gem_names && api_fetcher? # newline after dots raise end - def use_api - return @use_api if defined?(@use_api) - - fetchers.shift until fetchers.first.available? - - @use_api = if remote_uri.scheme == "file" || Bundler::Fetcher.disable_endpoint - false - else - fetchers.first.api_fetcher? - end - end - def user_agent @user_agent ||= begin ruby = Bundler::RubyVersion.system @@ -203,10 +192,6 @@ module Bundler end end - def fetchers - @fetchers ||= FETCHERS.map {|f| f.new(downloader, @remote, uri) } - end - def http_proxy return unless uri = connection.proxy_uri uri.to_s @@ -216,37 +201,61 @@ module Bundler "#<#{self.class}:0x#{object_id} uri=#{uri}>" end + def api_fetcher? + fetchers.first.api_fetcher? + end + + def gem_remote_fetcher + @gem_remote_fetcher ||= begin + require_relative "fetcher/gem_remote_fetcher" + fetcher = GemRemoteFetcher.new Gem.configuration[:http_proxy] + fetcher.headers["User-Agent"] = user_agent + fetcher.headers["X-Gemfile-Source"] = @remote.original_uri.to_s if @remote.original_uri + fetcher + end + end + private - FETCHERS = [CompactIndex, Dependency, Index].freeze + def available_fetchers + if Bundler::Fetcher.disable_endpoint + [Index] + elsif remote_uri.scheme == "file" + Bundler.ui.debug("Using a local server, bundler won't use the CompactIndex API") + [Index] + else + [CompactIndex, Dependency, Index] + end + end + + def fetchers + @fetchers ||= available_fetchers.map {|f| f.new(downloader, @remote, uri, gem_remote_fetcher) }.drop_while {|f| !f.available? } + end + + def fetch_specs(gem_names) + fetchers.reject!(&:api_fetcher?) unless gem_names + fetchers.reject! do |f| + specs = f.specs(gem_names) + return specs if specs + true + end + [] + end def cis - env_cis = { - "TRAVIS" => "travis", - "CIRCLECI" => "circle", - "SEMAPHORE" => "semaphore", - "JENKINS_URL" => "jenkins", - "BUILDBOX" => "buildbox", - "GO_SERVER_URL" => "go", - "SNAP_CI" => "snap", - "GITLAB_CI" => "gitlab", - "GITHUB_ACTIONS" => "github", - "CI_NAME" => ENV["CI_NAME"], - "CI" => "ci", - } - env_cis.find_all {|env, _| ENV[env] }.map {|_, ci| ci } + @cis ||= Bundler::CIDetector.ci_strings end def connection @connection ||= begin needs_ssl = remote_uri.scheme == "https" || - Bundler.settings[:ssl_verify_mode] || - Bundler.settings[:ssl_client_cert] + Bundler.settings[:ssl_verify_mode] || + Bundler.settings[:ssl_client_cert] raise SSLError if needs_ssl && !defined?(OpenSSL::SSL) - con = PersistentHTTP.new :name => "bundler", :proxy => :ENV + con = Gem::Net::HTTP::Persistent.new name: "bundler", proxy: :ENV if gem_proxy = Gem.configuration[:http_proxy] - con.proxy = Bundler::URI.parse(gem_proxy) if gem_proxy != :no_proxy + con.proxy = Gem::URI.parse(gem_proxy) if gem_proxy != :no_proxy end if remote_uri.scheme == "https" @@ -256,8 +265,8 @@ module Bundler end ssl_client_cert = Bundler.settings[:ssl_client_cert] || - (Gem.configuration.ssl_client_cert if - Gem.configuration.respond_to?(:ssl_client_cert)) + (Gem.configuration.ssl_client_cert if + Gem.configuration.respond_to?(:ssl_client_cert)) if ssl_client_cert pem = File.read(ssl_client_cert) con.cert = OpenSSL::X509::Certificate.new(pem) @@ -279,17 +288,17 @@ module Bundler end HTTP_ERRORS = [ - Timeout::Error, EOFError, SocketError, Errno::ENETDOWN, Errno::ENETUNREACH, + Gem::Timeout::Error, EOFError, SocketError, Errno::ENETDOWN, Errno::ENETUNREACH, Errno::EINVAL, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EAGAIN, - Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, - PersistentHTTP::Error, Zlib::BufError, Errno::EHOSTUNREACH + Gem::Net::HTTPBadResponse, Gem::Net::HTTPHeaderSyntaxError, Gem::Net::ProtocolError, + Gem::Net::HTTP::Persistent::Error, Zlib::BufError, Errno::EHOSTUNREACH ].freeze def bundler_cert_store store = OpenSSL::X509::Store.new ssl_ca_cert = Bundler.settings[:ssl_ca_cert] || - (Gem.configuration.ssl_ca_cert if - Gem.configuration.respond_to?(:ssl_ca_cert)) + (Gem.configuration.ssl_ca_cert if + Gem.configuration.respond_to?(:ssl_ca_cert)) if ssl_ca_cert if File.directory? ssl_ca_cert store.add_path ssl_ca_cert @@ -298,6 +307,7 @@ module Bundler end else store.set_default_paths + require "rubygems/request" Gem::Request.get_cert_files.each {|c| store.add_file c } end store |