summaryrefslogtreecommitdiff
path: root/lib/bundler/fetcher.rb
blob: 8397f7b72b02183f9a3b66fde0c09b1a5f53800b (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
# frozen_string_literal: true

require "bundler/vendored_persistent"
require "cgi"
require "securerandom"
require "zlib"
require "rubygems/request"

module Bundler
  # Handles all the fetching with the rubygems server
  class Fetcher
    autoload :CompactIndex, "bundler/fetcher/compact_index"
    autoload :Downloader, "bundler/fetcher/downloader"
    autoload :Dependency, "bundler/fetcher/dependency"
    autoload :Index, "bundler/fetcher/index"

    # This error is raised when it looks like the network is down
    class NetworkDownError < HTTPError; end
    # This error is raised if we should rate limit our requests to the API
    class TooManyRequestsError < HTTPError; end
    # This error is raised if the API returns a 413 (only printed in verbose)
    class FallbackError < HTTPError; end
    # This is the error raised if OpenSSL fails the cert verification
    class CertificateFailureError < HTTPError
      def initialize(remote_uri)
        remote_uri = filter_uri(remote_uri)
        super "Could not verify the SSL certificate for #{remote_uri}.\nThere" \
          " is a chance you are experiencing a man-in-the-middle attack, but" \
          " most likely your system doesn't have the CA certificates needed" \
          " for verification. For information about OpenSSL certificates, see" \
          " http://bit.ly/ruby-ssl. To connect without using SSL, edit your Gemfile" \
          " sources and change 'https' to 'http'."
      end
    end
    # This is the error raised when a source is HTTPS and OpenSSL didn't load
    class SSLError < HTTPError
      def initialize(msg = nil)
        super msg || "Could not load OpenSSL.\n" \
            "You must recompile Ruby with OpenSSL support or change the sources in your " \
            "Gemfile from 'https' to 'http'. Instructions for compiling with OpenSSL " \
            "using RVM are available at rvm.io/packages/openssl."
      end
    end
    # This error is raised if HTTP authentication is required, but not provided.
    class AuthenticationRequiredError < HTTPError
      def initialize(remote_uri)
        remote_uri = filter_uri(remote_uri)
        super "Authentication is required for #{remote_uri}.\n" \
          "Please supply credentials for this source. You can do this by running:\n" \
          " bundle config set #{remote_uri} username:password"
      end
    end
    # This error is raised if HTTP authentication is provided, but incorrect.
    class BadAuthenticationError < HTTPError
      def initialize(remote_uri)
        remote_uri = filter_uri(remote_uri)
        super "Bad username or password for #{remote_uri}.\n" \
          "Please double-check your credentials and correct them."
      end
    end

    # Exceptions classes that should bypass retry attempts. If your password didn't work the
    # first time, it's not going to the third time.
    NET_ERRORS = [:HTTPBadGateway, :HTTPBadRequest, :HTTPFailedDependency,
                  :HTTPForbidden, :HTTPInsufficientStorage, :HTTPMethodNotAllowed,
                  :HTTPMovedPermanently, :HTTPNoContent, :HTTPNotFound,
                  :HTTPNotImplemented, :HTTPPreconditionFailed, :HTTPRequestEntityTooLarge,
                  :HTTPRequestURITooLong, :HTTPUnauthorized, :HTTPUnprocessableEntity,
                  :HTTPUnsupportedMediaType, :HTTPVersionNotSupported].freeze
    FAIL_ERRORS = begin
      fail_errors = [AuthenticationRequiredError, BadAuthenticationError, FallbackError]
      fail_errors << Gem::Requirement::BadRequirementError if defined?(Gem::Requirement::BadRequirementError)
      fail_errors.concat(NET_ERRORS.map {|e| SharedHelpers.const_get_safely(e, Net) }.compact)
    end.freeze

    class << self
      attr_accessor :disable_endpoint, :api_timeout, :redirect_limit, :max_retries
    end

    self.redirect_limit = Bundler.settings[:redirect] # How many redirects to allow in one request
    self.api_timeout    = Bundler.settings[:timeout] # How long to wait for each API call
    self.max_retries    = Bundler.settings[:retry] # How many retries for the API call

    def initialize(remote)
      @remote = remote

      Socket.do_not_reverse_lookup = true
      connection # create persistent connection
    end

    def uri
      @remote.anonymized_uri
    end

    # fetch a gem specification
    def fetch_spec(spec)
      spec -= [nil, "ruby", ""]
      spec_file_name = "#{spec.join "-"}.gemspec"

      uri = URI.parse("#{remote_uri}#{Gem::MARSHAL_SPEC_DIR}#{spec_file_name}.rz")
      if uri.scheme == "file"
        Bundler.load_marshal Bundler.rubygems.inflate(Gem.read_binary(uri.path))
      elsif cached_spec_path = gemspec_cached_path(spec_file_name)
        Bundler.load_gemspec(cached_spec_path)
      else
        Bundler.load_marshal Bundler.rubygems.inflate(downloader.fetch(uri).body)
      end
    rescue MarshalError
      raise HTTPError, "Gemspec #{spec} contained invalid data.\n" \
        "Your network or your gem server is probably having issues right now."
    end

    # return the specs in the bundler format as an index with retries
    def specs_with_retry(gem_names, source)
      Bundler::Retry.new("fetcher", FAIL_ERRORS).attempts do
        specs(gem_names, source)
      end
    end

    # return the specs in the bundler format as an index
    def specs(gem_names, source)
      old = Bundler.rubygems.sources
      index = Bundler::Index.new

      if Bundler::Fetcher.disable_endpoint
        @use_api = false
        specs = fetchers.last.specs(gem_names)
      else
        specs = []
        fetchers.shift until fetchers.first.available? || fetchers.empty?
        fetchers.dup.each do |f|
          break unless f.api_fetcher? && !gem_names || !specs = f.specs(gem_names)
          fetchers.delete(f)
        end
        @use_api = false if fetchers.none?(&:api_fetcher?)
      end

      specs.each do |name, version, platform, dependencies, metadata|
        next if name == "bundler"
        spec = if dependencies
          EndpointSpecification.new(name, version, platform, dependencies, metadata)
        else
          RemoteSpecification.new(name, version, platform, self)
        end
        spec.source = source
        spec.remote = @remote
        index << spec
      end

      index
    rescue CertificateFailureError
      Bundler.ui.info "" if gem_names && use_api # newline after dots
      raise
    ensure
      Bundler.rubygems.sources = old
    end

    def use_api
      return @use_api if defined?(@use_api)

      fetchers.shift until fetchers.first.available?

      @use_api = if remote_uri.scheme == "file" || Bundler::Fetcher.disable_endpoint
        false
      else
        fetchers.first.api_fetcher?
      end
    end

    def user_agent
      @user_agent ||= begin
        ruby = Bundler::RubyVersion.system

        agent = String.new("bundler/#{Bundler::VERSION}")
        agent << " rubygems/#{Gem::VERSION}"
        agent << " ruby/#{ruby.versions_string(ruby.versions)}"
        agent << " (#{ruby.host})"
        agent << " command/#{ARGV.first}"

        if ruby.engine != "ruby"
          # engine_version raises on unknown engines
          engine_version = begin
                             ruby.engine_versions
                           rescue RuntimeError
                             "???"
                           end
          agent << " #{ruby.engine}/#{ruby.versions_string(engine_version)}"
        end

        agent << " options/#{Bundler.settings.all.join(",")}"

        agent << " ci/#{cis.join(",")}" if cis.any?

        # add a random ID so we can consolidate runs server-side
        agent << " " << SecureRandom.hex(8)

        # add any user agent strings set in the config
        extra_ua = Bundler.settings[:user_agent]
        agent << " " << extra_ua if extra_ua

        agent
      end
    end

    def fetchers
      @fetchers ||= FETCHERS.map {|f| f.new(downloader, @remote, uri) }
    end

    def http_proxy
      return unless uri = connection.proxy_uri
      uri.to_s
    end

    def inspect
      "#<#{self.class}:0x#{object_id} uri=#{uri}>"
    end

  private

    FETCHERS = [CompactIndex, Dependency, Index].freeze

    def cis
      env_cis = {
        "TRAVIS" => "travis",
        "CIRCLECI" => "circle",
        "SEMAPHORE" => "semaphore",
        "JENKINS_URL" => "jenkins",
        "BUILDBOX" => "buildbox",
        "GO_SERVER_URL" => "go",
        "SNAP_CI" => "snap",
        "CI_NAME" => ENV["CI_NAME"],
        "CI" => "ci",
      }
      env_cis.find_all {|env, _| ENV[env] }.map {|_, ci| ci }
    end

    def connection
      @connection ||= begin
        needs_ssl = remote_uri.scheme == "https" ||
          Bundler.settings[:ssl_verify_mode] ||
          Bundler.settings[:ssl_client_cert]
        raise SSLError if needs_ssl && !defined?(OpenSSL::SSL)

        con = PersistentHTTP.new "bundler", :ENV
        if gem_proxy = Bundler.rubygems.configuration[:http_proxy]
          con.proxy = URI.parse(gem_proxy) if gem_proxy != :no_proxy
        end

        if remote_uri.scheme == "https"
          con.verify_mode = (Bundler.settings[:ssl_verify_mode] ||
            OpenSSL::SSL::VERIFY_PEER)
          con.cert_store = bundler_cert_store
        end

        ssl_client_cert = Bundler.settings[:ssl_client_cert] ||
          (Bundler.rubygems.configuration.ssl_client_cert if
            Bundler.rubygems.configuration.respond_to?(:ssl_client_cert))
        if ssl_client_cert
          pem = File.read(ssl_client_cert)
          con.cert = OpenSSL::X509::Certificate.new(pem)
          con.key  = OpenSSL::PKey::RSA.new(pem)
        end

        con.read_timeout = Fetcher.api_timeout
        con.open_timeout = Fetcher.api_timeout
        con.override_headers["User-Agent"] = user_agent
        con.override_headers["X-Gemfile-Source"] = @remote.original_uri.to_s if @remote.original_uri
        con
      end
    end

    # cached gem specification path, if one exists
    def gemspec_cached_path(spec_file_name)
      paths = Bundler.rubygems.spec_cache_dirs.map {|dir| File.join(dir, spec_file_name) }
      paths = paths.select {|path| File.file? path }
      paths.first
    end

    HTTP_ERRORS = [
      Timeout::Error, EOFError, SocketError, Errno::ENETDOWN, Errno::ENETUNREACH,
      Errno::EINVAL, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EAGAIN,
      Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError,
      PersistentHTTP::Error, Zlib::BufError, Errno::EHOSTUNREACH
    ].freeze

    def bundler_cert_store
      store = OpenSSL::X509::Store.new
      ssl_ca_cert = Bundler.settings[:ssl_ca_cert] ||
        (Bundler.rubygems.configuration.ssl_ca_cert if
          Bundler.rubygems.configuration.respond_to?(:ssl_ca_cert))
      if ssl_ca_cert
        if File.directory? ssl_ca_cert
          store.add_path ssl_ca_cert
        else
          store.add_file ssl_ca_cert
        end
      else
        store.set_default_paths
        Gem::Request.get_cert_files.each {|c| store.add_file c }
      end
      store
    end

  private

    def remote_uri
      @remote.uri
    end

    def downloader
      @downloader ||= Downloader.new(connection, self.class.redirect_limit)
    end
  end
end