From 97f3ceeaa5c66ad6a6a5f3f37339c4b1cbe71677 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Rodr=C3=ADguez?= Date: Tue, 11 Jun 2019 12:48:34 +0200 Subject: [bundler/bundler] Bump net-http-persistent to 3.0.1 * Adds an extra artifice task to vendorize new `connection_pool` dependency. * Cherry-pick's needed Windows fix not yet merged into master branch of `net-http-persistent`. * Update bundler usages to be compatible with the new version, and fix unit specs. https://github.com/bundler/bundler/commit/0575baa6bb --- lib/bundler/fetcher.rb | 2 +- .../vendor/connection_pool/lib/connection_pool.rb | 161 +++++++ .../lib/connection_pool/monotonic_time.rb | 66 +++ .../lib/connection_pool/timed_stack.rb | 176 +++++++ .../connection_pool/lib/connection_pool/version.rb | 3 + .../net-http-persistent/lib/net/http/faster.rb | 27 -- .../net-http-persistent/lib/net/http/persistent.rb | 518 ++++++++++----------- .../lib/net/http/persistent/connection.rb | 40 ++ .../lib/net/http/persistent/pool.rb | 50 ++ .../lib/net/http/persistent/ssl_reuse.rb | 129 ----- .../lib/net/http/persistent/timed_stack_multi.rb | 69 +++ lib/bundler/vendored_persistent.rb | 10 +- spec/bundler/bundler/vendored_persistent_spec.rb | 4 +- 13 files changed, 813 insertions(+), 442 deletions(-) create mode 100644 lib/bundler/vendor/connection_pool/lib/connection_pool.rb create mode 100644 lib/bundler/vendor/connection_pool/lib/connection_pool/monotonic_time.rb create mode 100644 lib/bundler/vendor/connection_pool/lib/connection_pool/timed_stack.rb create mode 100644 lib/bundler/vendor/connection_pool/lib/connection_pool/version.rb delete mode 100644 lib/bundler/vendor/net-http-persistent/lib/net/http/faster.rb create mode 100644 lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/connection.rb create mode 100644 lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/pool.rb delete mode 100644 lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/ssl_reuse.rb create mode 100644 lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi.rb diff --git a/lib/bundler/fetcher.rb b/lib/bundler/fetcher.rb index 8118c2b2e7..7bda26770b 100644 --- a/lib/bundler/fetcher.rb +++ b/lib/bundler/fetcher.rb @@ -242,7 +242,7 @@ module Bundler Bundler.settings[:ssl_client_cert] raise SSLError if needs_ssl && !defined?(OpenSSL::SSL) - con = PersistentHTTP.new "bundler", :ENV + con = PersistentHTTP.new :name => "bundler", :proxy => :ENV if gem_proxy = Bundler.rubygems.configuration[:http_proxy] con.proxy = URI.parse(gem_proxy) if gem_proxy != :no_proxy end diff --git a/lib/bundler/vendor/connection_pool/lib/connection_pool.rb b/lib/bundler/vendor/connection_pool/lib/connection_pool.rb new file mode 100644 index 0000000000..fbcd26c765 --- /dev/null +++ b/lib/bundler/vendor/connection_pool/lib/connection_pool.rb @@ -0,0 +1,161 @@ +require_relative 'connection_pool/version' +require_relative 'connection_pool/timed_stack' + + +# Generic connection pool class for e.g. sharing a limited number of network connections +# among many threads. Note: Connections are lazily created. +# +# Example usage with block (faster): +# +# @pool = Bundler::ConnectionPool.new { Redis.new } +# +# @pool.with do |redis| +# redis.lpop('my-list') if redis.llen('my-list') > 0 +# end +# +# Using optional timeout override (for that single invocation) +# +# @pool.with(timeout: 2.0) do |redis| +# redis.lpop('my-list') if redis.llen('my-list') > 0 +# end +# +# Example usage replacing an existing connection (slower): +# +# $redis = Bundler::ConnectionPool.wrap { Redis.new } +# +# def do_work +# $redis.lpop('my-list') if $redis.llen('my-list') > 0 +# end +# +# Accepts the following options: +# - :size - number of connections to pool, defaults to 5 +# - :timeout - amount of time to wait for a connection if none currently available, defaults to 5 seconds +# +class Bundler::ConnectionPool + DEFAULTS = {size: 5, timeout: 5} + + class Error < RuntimeError + end + + def self.wrap(options, &block) + Wrapper.new(options, &block) + end + + def initialize(options = {}, &block) + raise ArgumentError, 'Connection pool requires a block' unless block + + options = DEFAULTS.merge(options) + + @size = options.fetch(:size) + @timeout = options.fetch(:timeout) + + @available = TimedStack.new(@size, &block) + @key = :"current-#{@available.object_id}" + @key_count = :"current-#{@available.object_id}-count" + end + +if Thread.respond_to?(:handle_interrupt) + + # MRI + def with(options = {}) + Thread.handle_interrupt(Exception => :never) do + conn = checkout(options) + begin + Thread.handle_interrupt(Exception => :immediate) do + yield conn + end + ensure + checkin + end + end + end + +else + + # jruby 1.7.x + def with(options = {}) + conn = checkout(options) + begin + yield conn + ensure + checkin + end + end + +end + + def checkout(options = {}) + if ::Thread.current[@key] + ::Thread.current[@key_count]+= 1 + ::Thread.current[@key] + else + ::Thread.current[@key_count]= 1 + ::Thread.current[@key]= @available.pop(options[:timeout] || @timeout) + end + end + + def checkin + if ::Thread.current[@key] + if ::Thread.current[@key_count] == 1 + @available.push(::Thread.current[@key]) + ::Thread.current[@key]= nil + else + ::Thread.current[@key_count]-= 1 + end + else + raise Bundler::ConnectionPool::Error, 'no connections are checked out' + end + + nil + end + + def shutdown(&block) + @available.shutdown(&block) + end + + # Size of this connection pool + def size + @size + end + + # Number of pool entries available for checkout at this instant. + def available + @available.length + end + + private + + class Wrapper < ::BasicObject + METHODS = [:with, :pool_shutdown] + + def initialize(options = {}, &block) + @pool = options.fetch(:pool) { ::Bundler::ConnectionPool.new(options, &block) } + end + + def with(&block) + @pool.with(&block) + end + + def pool_shutdown(&block) + @pool.shutdown(&block) + end + + def pool_size + @pool.size + end + + def pool_available + @pool.available + end + + def respond_to?(id, *args) + METHODS.include?(id) || with { |c| c.respond_to?(id, *args) } + end + + def method_missing(name, *args, &block) + with do |connection| + connection.send(name, *args, &block) + end + end + end +end diff --git a/lib/bundler/vendor/connection_pool/lib/connection_pool/monotonic_time.rb b/lib/bundler/vendor/connection_pool/lib/connection_pool/monotonic_time.rb new file mode 100644 index 0000000000..5a9c4a27bb --- /dev/null +++ b/lib/bundler/vendor/connection_pool/lib/connection_pool/monotonic_time.rb @@ -0,0 +1,66 @@ +# Global monotonic clock from Concurrent Ruby 1.0. +# Copyright (c) Jerry D'Antonio -- released under the MIT license. +# Slightly modified; used with permission. +# https://github.com/ruby-concurrency/concurrent-ruby + +require 'thread' + +class Bundler::ConnectionPool + + class_definition = Class.new do + + if defined?(Process::CLOCK_MONOTONIC) + + # @!visibility private + def get_time + Process.clock_gettime(Process::CLOCK_MONOTONIC) + end + + elsif defined?(RUBY_ENGINE) && RUBY_ENGINE == 'jruby' + + # @!visibility private + def get_time + java.lang.System.nanoTime() / 1_000_000_000.0 + end + + else + + # @!visibility private + def initialize + @mutex = Mutex.new + @last_time = Time.now.to_f + end + + # @!visibility private + def get_time + @mutex.synchronize do + now = Time.now.to_f + if @last_time < now + @last_time = now + else # clock has moved back in time + @last_time += 0.000_001 + end + end + end + end + end + + ## + # Clock that cannot be set and represents monotonic time since + # some unspecified starting point. + # + # @!visibility private + GLOBAL_MONOTONIC_CLOCK = class_definition.new + private_constant :GLOBAL_MONOTONIC_CLOCK + + class << self + ## + # Returns the current time a tracked by the application monotonic clock. + # + # @return [Float] The current monotonic time when `since` not given else + # the elapsed monotonic time between `since` and the current time + def monotonic_time + GLOBAL_MONOTONIC_CLOCK.get_time + end + end +end diff --git a/lib/bundler/vendor/connection_pool/lib/connection_pool/timed_stack.rb b/lib/bundler/vendor/connection_pool/lib/connection_pool/timed_stack.rb new file mode 100644 index 0000000000..f3fe1e04ad --- /dev/null +++ b/lib/bundler/vendor/connection_pool/lib/connection_pool/timed_stack.rb @@ -0,0 +1,176 @@ +require 'thread' +require 'timeout' +require_relative 'monotonic_time' + +## +# Raised when you attempt to retrieve a connection from a pool that has been +# shut down. + +class Bundler::ConnectionPool::PoolShuttingDownError < RuntimeError; end + +## +# The TimedStack manages a pool of homogeneous connections (or any resource +# you wish to manage). Connections are created lazily up to a given maximum +# number. + +# Examples: +# +# ts = TimedStack.new(1) { MyConnection.new } +# +# # fetch a connection +# conn = ts.pop +# +# # return a connection +# ts.push conn +# +# conn = ts.pop +# ts.pop timeout: 5 +# #=> raises Timeout::Error after 5 seconds + +class Bundler::ConnectionPool::TimedStack + attr_reader :max + + ## + # Creates a new pool with +size+ connections that are created from the given + # +block+. + + def initialize(size = 0, &block) + @create_block = block + @created = 0 + @que = [] + @max = size + @mutex = Mutex.new + @resource = ConditionVariable.new + @shutdown_block = nil + end + + ## + # Returns +obj+ to the stack. +options+ is ignored in TimedStack but may be + # used by subclasses that extend TimedStack. + + def push(obj, options = {}) + @mutex.synchronize do + if @shutdown_block + @shutdown_block.call(obj) + else + store_connection obj, options + end + + @resource.broadcast + end + end + alias_method :<<, :push + + ## + # Retrieves a connection from the stack. If a connection is available it is + # immediately returned. If no connection is available within the given + # timeout a Timeout::Error is raised. + # + # +:timeout+ is the only checked entry in +options+ and is preferred over + # the +timeout+ argument (which will be removed in a future release). Other + # options may be used by subclasses that extend TimedStack. + + def pop(timeout = 0.5, options = {}) + options, timeout = timeout, 0.5 if Hash === timeout + timeout = options.fetch :timeout, timeout + + deadline = Bundler::ConnectionPool.monotonic_time + timeout + @mutex.synchronize do + loop do + raise Bundler::ConnectionPool::PoolShuttingDownError if @shutdown_block + return fetch_connection(options) if connection_stored?(options) + + connection = try_create(options) + return connection if connection + + to_wait = deadline - Bundler::ConnectionPool.monotonic_time + raise Timeout::Error, "Waited #{timeout} sec" if to_wait <= 0 + @resource.wait(@mutex, to_wait) + end + end + end + + ## + # Shuts down the TimedStack which prevents connections from being checked + # out. The +block+ is called once for each connection on the stack. + + def shutdown(&block) + raise ArgumentError, "shutdown must receive a block" unless block_given? + + @mutex.synchronize do + @shutdown_block = block + @resource.broadcast + + shutdown_connections + end + end + + ## + # Returns +true+ if there are no available connections. + + def empty? + (@created - @que.length) >= @max + end + + ## + # The number of connections available on the stack. + + def length + @max - @created + @que.length + end + + private + + ## + # This is an extension point for TimedStack and is called with a mutex. + # + # This method must returns true if a connection is available on the stack. + + def connection_stored?(options = nil) + !@que.empty? + end + + ## + # This is an extension point for TimedStack and is called with a mutex. + # + # This method must return a connection from the stack. + + def fetch_connection(options = nil) + @que.pop + end + + ## + # This is an extension point for TimedStack and is called with a mutex. + # + # This method must shut down all connections on the stack. + + def shutdown_connections(options = nil) + while connection_stored?(options) + conn = fetch_connection(options) + @shutdown_block.call(conn) + end + end + + ## + # This is an extension point for TimedStack and is called with a mutex. + # + # This method must return +obj+ to the stack. + + def store_connection(obj, options = nil) + @que.push obj + end + + ## + # This is an extension point for TimedStack and is called with a mutex. + # + # This method must create a connection if and only if the total number of + # connections allowed has not been met. + + def try_create(options = nil) + unless @created == @max + object = @create_block.call + @created += 1 + object + end + end +end diff --git a/lib/bundler/vendor/connection_pool/lib/connection_pool/version.rb b/lib/bundler/vendor/connection_pool/lib/connection_pool/version.rb new file mode 100644 index 0000000000..b149c0e242 --- /dev/null +++ b/lib/bundler/vendor/connection_pool/lib/connection_pool/version.rb @@ -0,0 +1,3 @@ +class Bundler::ConnectionPool + VERSION = "2.2.2" +end diff --git a/lib/bundler/vendor/net-http-persistent/lib/net/http/faster.rb b/lib/bundler/vendor/net-http-persistent/lib/net/http/faster.rb deleted file mode 100644 index e5e09080c2..0000000000 --- a/lib/bundler/vendor/net-http-persistent/lib/net/http/faster.rb +++ /dev/null @@ -1,27 +0,0 @@ -require 'net/protocol' - -## -# Aaron Patterson's monkeypatch (accepted into 1.9.1) to fix Net::HTTP's speed -# problems. -# -# http://gist.github.com/251244 - -class Net::BufferedIO #:nodoc: - alias :old_rbuf_fill :rbuf_fill - - def rbuf_fill - if @io.respond_to? :read_nonblock then - begin - @rbuf << @io.read_nonblock(65536) - rescue Errno::EWOULDBLOCK, Errno::EAGAIN => e - retry if IO.select [@io], nil, nil, @read_timeout - raise Timeout::Error, e.message - end - else # SSL sockets do not have read_nonblock - timeout @read_timeout do - @rbuf << @io.sysread(65536) - end - end - end -end if RUBY_VERSION < '1.9' - diff --git a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb index 7cbca5bc06..0d4bb350dd 100644 --- a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb +++ b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent.rb @@ -1,12 +1,7 @@ require 'net/http' -begin - require 'net/https' -rescue LoadError - # net/https or openssl -end if RUBY_VERSION < '1.9' # but only for 1.8 -require 'bundler/vendor/net-http-persistent/lib/net/http/faster' require 'uri' require 'cgi' # for escaping +require_relative '../../../../connection_pool/lib/connection_pool' begin require 'net/http/pipeline' @@ -38,7 +33,7 @@ autoload :OpenSSL, 'openssl' # # uri = URI 'http://example.com/awesome/web/service' # -# http = Bundler::Persistent::Net::HTTP::Persistent.new 'my_app_name' +# http = Bundler::Persistent::Net::HTTP::Persistent.new name: 'my_app_name' # # # perform a GET # response = http.request uri @@ -70,13 +65,17 @@ autoload :OpenSSL, 'openssl' # Here are the SSL settings, see the individual methods for documentation: # # #certificate :: This client's certificate -# #ca_file :: The certificate-authority +# #ca_file :: The certificate-authorities +# #ca_path :: Directory with certificate-authorities # #cert_store :: An SSL certificate store +# #ciphers :: List of SSl ciphers allowed # #private_key :: The client's SSL private key # #reuse_ssl_sessions :: Reuse a previously opened SSL session for a new # connection +# #ssl_timeout :: SSL session lifetime # #ssl_version :: Which specific SSL version to use # #verify_callback :: For server certificate verification +# #verify_depth :: Depth of certificate verification # #verify_mode :: How connections should be verified # # == Proxies @@ -154,7 +153,7 @@ autoload :OpenSSL, 'openssl' # uri = URI 'http://example.com/awesome/web/service' # post_uri = uri + 'create' # -# http = Bundler::Persistent::Net::HTTP::Persistent.new 'my_app_name' +# http = Bundler::Persistent::Net::HTTP::Persistent.new name: 'my_app_name' # # post = Net::HTTP::Post.new post_uri.path # # ... fill in POST request @@ -200,10 +199,19 @@ class Bundler::Persistent::Net::HTTP::Persistent HAVE_OPENSSL = defined? OpenSSL::SSL # :nodoc: + ## + # The default connection pool size is 1/4 the allowed open files. + + if Gem.win_platform? then + DEFAULT_POOL_SIZE = 256 + else + DEFAULT_POOL_SIZE = Process.getrlimit(Process::RLIMIT_NOFILE).first / 4 + end + ## # The version of Bundler::Persistent::Net::HTTP::Persistent you are using - VERSION = '2.9.4' + VERSION = '3.0.1' ## # Exceptions rescued for automatic retry on ruby 2.0.0. This overlaps with @@ -248,31 +256,31 @@ class Bundler::Persistent::Net::HTTP::Persistent http = new 'net-http-persistent detect_idle_timeout' - connection = http.connection_for uri + http.connection_for uri do |connection| + sleep_time = 0 - sleep_time = 0 + http = connection.http - loop do - response = connection.request req + loop do + response = http.request req - $stderr.puts "HEAD #{uri} => #{response.code}" if $DEBUG + $stderr.puts "HEAD #{uri} => #{response.code}" if $DEBUG - unless Net::HTTPOK === response then - raise Error, "bad response code #{response.code} detecting idle timeout" - end + unless Net::HTTPOK === response then + raise Error, "bad response code #{response.code} detecting idle timeout" + end - break if sleep_time >= max + break if sleep_time >= max - sleep_time += 1 + sleep_time += 1 - $stderr.puts "sleeping #{sleep_time}" if $DEBUG - sleep sleep_time + $stderr.puts "sleeping #{sleep_time}" if $DEBUG + sleep sleep_time + end end rescue # ignore StandardErrors, we've probably found the idle timeout. ensure - http.shutdown - return sleep_time unless $! end @@ -281,7 +289,9 @@ class Bundler::Persistent::Net::HTTP::Persistent attr_reader :certificate + ## # For Net::HTTP parity + alias cert certificate ## @@ -290,12 +300,23 @@ class Bundler::Persistent::Net::HTTP::Persistent attr_reader :ca_file + ## + # A directory of SSL certificates to be used as certificate authorities. + # Setting this will set verify_mode to VERIFY_PEER. + + attr_reader :ca_path + ## # An SSL certificate store. Setting this will override the default # certificate store. See verify_mode for more information. attr_reader :cert_store + ## + # The ciphers allowed for SSL connections + + attr_reader :ciphers + ## # Sends debug_output to this IO via Net::HTTP#set_debug_output. # @@ -309,11 +330,6 @@ class Bundler::Persistent::Net::HTTP::Persistent attr_reader :generation # :nodoc: - ## - # Where this instance's connections live in the thread local variables - - attr_reader :generation_key # :nodoc: - ## # Headers that are added to every request using Net::HTTP#add_field @@ -369,7 +385,9 @@ class Bundler::Persistent::Net::HTTP::Persistent attr_reader :private_key + ## # For Net::HTTP parity + alias key private_key ## @@ -383,14 +401,14 @@ class Bundler::Persistent::Net::HTTP::Persistent attr_reader :no_proxy ## - # Seconds to wait until reading one block. See Net::HTTP#read_timeout + # Test-only accessor for the connection pool - attr_accessor :read_timeout + attr_reader :pool # :nodoc: ## - # Where this instance's request counts live in the thread local variables + # Seconds to wait until reading one block. See Net::HTTP#read_timeout - attr_reader :request_key # :nodoc: + attr_accessor :read_timeout ## # By default SSL sessions are reused to avoid extra SSL handshakes. Set @@ -418,17 +436,33 @@ class Bundler::Persistent::Net::HTTP::Persistent attr_reader :ssl_generation # :nodoc: ## - # Where this instance's SSL connections live in the thread local variables + # SSL session lifetime - attr_reader :ssl_generation_key # :nodoc: + attr_reader :ssl_timeout ## # SSL version to use. # # By default, the version will be negotiated automatically between client - # and server. Ruby 1.9 and newer only. + # and server. Ruby 1.9 and newer only. Deprecated since Ruby 2.5. + + attr_reader :ssl_version + + ## + # Minimum SSL version to use, e.g. :TLS1_1 + # + # By default, the version will be negotiated automatically between client + # and server. Ruby 2.5 and newer only. + + attr_reader :min_version + + ## + # Maximum SSL version to use, e.g. :TLS1_2 + # + # By default, the version will be negotiated automatically between client + # and server. Ruby 2.5 and newer only. - attr_reader :ssl_version if RUBY_VERSION > '1.9' + attr_reader :max_version ## # Where this instance's last-use times live in the thread local variables @@ -436,16 +470,21 @@ class Bundler::Persistent::Net::HTTP::Persistent attr_reader :timeout_key # :nodoc: ## - # SSL verification callback. Used when ca_file is set. + # SSL verification callback. Used when ca_file or ca_path is set. attr_reader :verify_callback + ## + # Sets the depth of SSL certificate verification + + attr_reader :verify_depth + ## # HTTPS verify mode. Defaults to OpenSSL::SSL::VERIFY_PEER which verifies # the server certificate. # - # If no ca_file or cert_store is set the default system certificate store is - # used. + # If no ca_file, ca_path or cert_store is set the default system certificate + # store is used. # # You can use +verify_mode+ to override any default values. @@ -478,8 +517,12 @@ class Bundler::Persistent::Net::HTTP::Persistent # proxy = URI 'http://proxy.example' # proxy.user = 'AzureDiamond' # proxy.password = 'hunter2' + # + # Set +pool_size+ to limit the maximum number of connections allowed. + # Defaults to 1/4 the number of allowed file handles. You can have no more + # than this many threads with active HTTP transactions. - def initialize name = nil, proxy = nil + def initialize name: nil, proxy: nil, pool_size: DEFAULT_POOL_SIZE @name = name @debug_output = nil @@ -494,26 +537,30 @@ class Bundler::Persistent::Net::HTTP::Persistent @idle_timeout = 5 @max_requests = nil @socket_options = [] + @ssl_generation = 0 # incremented when SSL session variables change @socket_options << [Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1] if Socket.const_defined? :TCP_NODELAY - key = ['net_http_persistent', name].compact - @generation_key = [key, 'generations' ].join('_').intern - @ssl_generation_key = [key, 'ssl_generations'].join('_').intern - @request_key = [key, 'requests' ].join('_').intern - @timeout_key = [key, 'timeouts' ].join('_').intern + @pool = Bundler::Persistent::Net::HTTP::Persistent::Pool.new size: pool_size do |http_args| + Bundler::Persistent::Net::HTTP::Persistent::Connection.new Net::HTTP, http_args, @ssl_generation + end @certificate = nil @ca_file = nil + @ca_path = nil + @ciphers = nil @private_key = nil + @ssl_timeout = nil @ssl_version = nil + @min_version = nil + @max_version = nil @verify_callback = nil + @verify_depth = nil @verify_mode = nil @cert_store = nil @generation = 0 # incremented when proxy URI changes - @ssl_generation = 0 # incremented when SSL session variables change if HAVE_OPENSSL then @verify_mode = OpenSSL::SSL::VERIFY_PEER @@ -522,9 +569,6 @@ class Bundler::Persistent::Net::HTTP::Persistent @retry_change_requests = false - @ruby_1 = RUBY_VERSION < '2' - @retried_on_ruby_2 = !@ruby_1 - self.proxy = proxy if proxy end @@ -549,6 +593,15 @@ class Bundler::Persistent::Net::HTTP::Persistent reconnect_ssl end + ## + # Sets the SSL certificate authority path. + + def ca_path= path + @ca_path = path + + reconnect_ssl + end + ## # Overrides the default SSL certificate store used for verifying # connections. @@ -560,92 +613,55 @@ class Bundler::Persistent::Net::HTTP::Persistent end ## - # Finishes all connections on the given +thread+ that were created before - # the given +generation+ in the threads +generation_key+ list. - # - # See #shutdown for a bunch of scary warning about misusing this method. - - def cleanup(generation, thread = Thread.current, - generation_key = @generation_key) # :nodoc: - timeouts = thread[@timeout_key] + # The ciphers allowed for SSL connections - (0...generation).each do |old_generation| - next unless thread[generation_key] + def ciphers= ciphers + @ciphers = ciphers - conns = thread[generation_key].delete old_generation - - conns.each_value do |conn| - finish conn, thread - - timeouts.delete conn.object_id if timeouts - end if conns - end + reconnect_ssl end ## # Creates a new connection for +uri+ def connection_for uri - Thread.current[@generation_key] ||= Hash.new { |h,k| h[k] = {} } - Thread.current[@ssl_generation_key] ||= Hash.new { |h,k| h[k] = {} } - Thread.current[@request_key] ||= Hash.new 0 - Thread.current[@timeout_key] ||= Hash.new EPOCH - use_ssl = uri.scheme.downcase == 'https' - if use_ssl then - raise Bundler::Persistent::Net::HTTP::Persistent::Error, 'OpenSSL is not available' unless - HAVE_OPENSSL - - ssl_generation = @ssl_generation - - ssl_cleanup ssl_generation - - connections = Thread.current[@ssl_generation_key][ssl_generation] - else - generation = @generation - - cleanup generation + net_http_args = [uri.host, uri.port] - connections = Thread.current[@generation_key][generation] - end + net_http_args.concat @proxy_args if + @proxy_uri and not proxy_bypass? uri.host, uri.port - net_http_args = [uri.host, uri.port] - connection_id = net_http_args.join ':' + connection = @pool.checkout net_http_args - if @proxy_uri and not proxy_bypass? uri.host, uri.port then - connection_id << @proxy_connection_id - net_http_args.concat @proxy_args - else - net_http_args.concat [nil, nil, nil, nil] - end + http = connection.http - connection = connections[connection_id] + connection.ressl @ssl_generation if + connection.ssl_generation != @ssl_generation - unless connection = connections[connection_id] then - connections[connection_id] = http_class.new(*net_http_args) - connection = connections[connection_id] - ssl connection if use_ssl - else - reset connection if expired? connection + if not http.started? then + ssl http if use_ssl + start http + elsif expired? connection then + reset connection end - start connection unless connection.started? - - connection.read_timeout = @read_timeout if @read_timeout - connection.keep_alive_timeout = @idle_timeout if @idle_timeout && connection.respond_to?(:keep_alive_timeout=) + http.read_timeout = @read_timeout if @read_timeout + http.keep_alive_timeout = @idle_timeout if @idle_timeout - connection + return yield connection rescue Errno::ECONNREFUSED - address = connection.proxy_address || connection.address - port = connection.proxy_port || connection.port + address = http.proxy_address || http.address + port = http.proxy_port || http.port raise Error, "connection refused: #{address}:#{port}" rescue Errno::EHOSTDOWN - address = connection.proxy_address || connection.address - port = connection.proxy_port || connection.port + address = http.proxy_address || http.address + port = http.proxy_port || http.port raise Error, "host down: #{address}:#{port}" + ensure + @pool.checkin net_http_args end ## @@ -653,12 +669,11 @@ class Bundler::Persistent::Net::HTTP::Persistent # this connection def error_message connection - requests = Thread.current[@request_key][connection.object_id] - 1 # fixup - last_use = Thread.current[@timeout_key][connection.object_id] + connection.requests -= 1 # fixup - age = Time.now - last_use + age = Time.now - connection.last_use - "after #{requests} requests on #{connection.object_id}, " \ + "after #{connection.requests} requests on #{connection.http.object_id}, " \ "last used #{age} seconds ago" end @@ -682,26 +697,23 @@ class Bundler::Persistent::Net::HTTP::Persistent # maximum request count, false otherwise. def expired? connection - requests = Thread.current[@request_key][connection.object_id] - return true if @max_requests && requests >= @max_requests + return true if @max_requests && connection.requests >= @max_requests return false unless @idle_timeout return true if @idle_timeout.zero? - last_used = Thread.current[@timeout_key][connection.object_id] - - Time.now - last_used > @idle_timeout + Time.now - connection.last_use > @idle_timeout end ## # Starts the Net::HTTP +connection+ - def start connection - connection.set_debug_output @debug_output if @debug_output - connection.open_timeout = @open_timeout if @open_timeout + def start http + http.set_debug_output @debug_output if @debug_output + http.open_timeout = @open_timeout if @open_timeout - connection.start + http.start - socket = connection.instance_variable_get :@socket + socket = http.instance_variable_get :@socket if socket then # for fakeweb @socket_options.each do |option| @@ -713,25 +725,11 @@ class Bundler::Persistent::Net::HTTP::Persistent ## # Finishes the Net::HTTP +connection+ - def finish connection, thread = Thread.current - if requests = thread[@request_key] then - requests.delete connection.object_id - end - + def finish connection connection.finish - rescue IOError - end - def http_class # :nodoc: - if RUBY_VERSION > '2.0' then - Net::HTTP - elsif [:Artifice, :FakeWeb, :WebMock].any? { |klass| - Object.const_defined?(klass) - } or not @reuse_ssl_sessions then - Net::HTTP - else - Bundler::Persistent::Net::HTTP::Persistent::SSLReuse - end + connection.http.instance_variable_set :@ssl_session, nil unless + @reuse_ssl_sessions end ## @@ -754,55 +752,9 @@ class Bundler::Persistent::Net::HTTP::Persistent ## # Is the request +req+ idempotent or is retry_change_requests allowed. - # - # If +retried_on_ruby_2+ is true, true will be returned if we are on ruby, - # retry_change_requests is allowed and the request is not idempotent. - - def can_retry? req, retried_on_ruby_2 = false - return @retry_change_requests && !idempotent?(req) if retried_on_ruby_2 - - @retry_change_requests || idempotent?(req) - end - - if RUBY_VERSION > '1.9' then - ## - # Workaround for missing Net::HTTPHeader#connection_close? on Ruby 1.8 - - def connection_close? header - header.connection_close? - end - - ## - # Workaround for missing Net::HTTPHeader#connection_keep_alive? on Ruby 1.8 - - def connection_keep_alive? header - header.connection_keep_alive? - end - else - ## - # Workaround for missing Net::HTTPRequest#connection_close? on Ruby 1.8 - - def connection_close? header - header['connection'] =~ /close/ or header['proxy-connection'] =~ /close/ - end - ## - # Workaround for missing Net::HTTPRequest#connection_keep_alive? on Ruby - # 1.8 - - def connection_keep_alive? header - header['connection'] =~ /keep-alive/ or - header['proxy-connection'] =~ /keep-alive/ - end - end - - ## - # Deprecated in favor of #expired? - - def max_age # :nodoc: - return Time.now + 1 unless @idle_timeout - - Time.now - @idle_timeout + def can_retry? req + @retry_change_requests && !idempotent?(req) end ## @@ -824,9 +776,9 @@ class Bundler::Persistent::Net::HTTP::Persistent # net-http-persistent #pipeline will be present. def pipeline uri, requests, &block # :yields: responses - connection = connection_for uri - - connection.pipeline requests, &block + connection_for uri do |connection| + connection.http.pipeline requests, &block + end end ## @@ -959,18 +911,17 @@ class Bundler::Persistent::Net::HTTP::Persistent # Finishes then restarts the Net::HTTP +connection+ def reset connection - Thread.current[@request_key].delete connection.object_id - Thread.current[@timeout_key].delete connection.object_id + http = connection.http finish connection - start connection + start http rescue Errno::ECONNREFUSED - e = Error.new "connection refused: #{connection.address}:#{connection.port}" + e = Error.new "connection refused: #{http.address}:#{http.port}" e.set_backtrace $@ raise e rescue Errno::EHOSTDOWN - e = Error.new "host down: #{connection.address}:#{connection.port}" + e = Error.new "host down: #{http.address}:#{http.port}" e.set_backtrace $@ raise e end @@ -991,52 +942,56 @@ class Bundler::Persistent::Net::HTTP::Persistent retried = false bad_response = false - req = request_setup req || uri + uri = URI uri + req = request_setup req || uri + response = nil - connection = connection_for uri - connection_id = connection.object_id + connection_for uri do |connection| + http = connection.http - begin - Thread.current[@request_key][connection_id] += 1 - response = connection.request req, &block + begin + connection.requests += 1 - if connection_close?(req) or - (response.http_version <= '1.0' and - not connection_keep_alive?(response)) or - connection_close?(response) then - connection.finish - end - rescue Net::HTTPBadResponse => e - message = error_message connection + response = http.request req, &block + + if req.connection_close? or + (response.http_version <= '1.0' and + not response.connection_keep_alive?) or + response.connection_close? then + finish connection + end + rescue Net::HTTPBadResponse => e + message = error_message connection - finish connection + finish connection - raise Error, "too many bad responses #{message}" if + raise Error, "too many bad responses #{message}" if bad_response or not can_retry? req - bad_response = true - retry - rescue *RETRIED_EXCEPTIONS => e # retried on ruby 2 - request_failed e, req, connection if - retried or not can_retry? req, @retried_on_ruby_2 + bad_response = true + retry + rescue *RETRIED_EXCEPTIONS => e + request_failed e, req, connection if + retried or not can_retry? req - reset connection + reset connection - retried = true - retry - rescue Errno::EINVAL, Errno::ETIMEDOUT => e # not retried on ruby 2 - request_failed e, req, connection if retried or not can_retry? req + retried = true + retry + rescue Errno::EINVAL, Errno::ETIMEDOUT => e # not retried on ruby 2 + request_failed e, req, connection if retried or not can_retry? req - reset connection + reset connection - retried = true - retry - rescue Exception => e - finish connection + retried = true + retry + rescue Exception => e + finish connection - raise - ensure - Thread.current[@timeout_key][connection_id] = Time.now + raise + ensure + connection.last_use = Time.now + end end @http_versions["#{uri.host}:#{uri.port}"] ||= response.http_version @@ -1056,7 +1011,6 @@ class Bundler::Persistent::Net::HTTP::Persistent finish connection - raise Error, message, exception.backtrace end @@ -1090,45 +1044,15 @@ class Bundler::Persistent::Net::HTTP::Persistent end ## - # Shuts down all connections for +thread+. - # - # Uses the current thread by default. - # - # If you've used Bundler::Persistent::Net::HTTP::Persistent across multiple threads you should - # call this in each thread when you're done making HTTP requests. + # Shuts down all connections # - # *NOTE*: Calling shutdown for another thread can be dangerous! + # *NOTE*: Calling shutdown for can be dangerous! # - # If the thread is still using the connection it may cause an error! It is - # best to call #shutdown in the thread at the appropriate time instead! + # If any thread is still using a connection it may cause an error! Call + # #shutdown when you are completely done making requests! - def shutdown thread = Thread.current - generation = reconnect - cleanup generation, thread, @generation_key - - ssl_generation = reconnect_ssl - cleanup ssl_generation, thread, @ssl_generation_key - - thread[@request_key] = nil - thread[@timeout_key] = nil - end - - ## - # Shuts down all connections in all threads - # - # *NOTE*: THIS METHOD IS VERY DANGEROUS! - # - # Do not call this method if other threads are still using their - # connections! Call #shutdown at the appropriate time instead! - # - # Use this method only as a last resort! - - def shutdown_in_all_threads - Thread.list.each do |thread| - shutdown thread - end - - nil + def shutdown + @pool.shutdown { |http| http.finish } end ## @@ -1137,9 +1061,14 @@ class Bundler::Persistent::Net::HTTP::Persistent def ssl connection connection.use_ssl = true + connection.ciphers = @ciphers if @ciphers + connection.ssl_timeout = @ssl_timeout if @ssl_timeout connection.ssl_version = @ssl_version if @ssl_version + connection.min_version = @min_version if @min_version + connection.max_version = @max_version if @max_version - connection.verify_mode = @verify_mode + connection.verify_depth = @verify_depth + connection.verify_mode = @verify_mode if OpenSSL::SSL::VERIFY_PEER == OpenSSL::SSL::VERIFY_NONE and not Object.const_defined?(:I_KNOW_THAT_OPENSSL_VERIFY_PEER_EQUALS_VERIFY_NONE_IS_WRONG) then @@ -1168,8 +1097,10 @@ application: WARNING end - if @ca_file then - connection.ca_file = @ca_file + connection.ca_file = @ca_file if @ca_file + connection.ca_path = @ca_path if @ca_path + + if @ca_file or @ca_path then connection.verify_mode = OpenSSL::SSL::VERIFY_PEER connection.verify_callback = @verify_callback if @verify_callback end @@ -1189,11 +1120,12 @@ application: end ## - # Finishes all connections that existed before the given SSL parameter - # +generation+. + # SSL session lifetime + + def ssl_timeout= ssl_timeout + @ssl_timeout = ssl_timeout - def ssl_cleanup generation # :nodoc: - cleanup generation, Thread.current, @ssl_generation_key + reconnect_ssl end ## @@ -1203,7 +1135,34 @@ application: @ssl_version = ssl_version reconnect_ssl - end if RUBY_VERSION > '1.9' + end + + ## + # Minimum SSL version to use + + def min_version= min_version + @min_version = min_version + + reconnect_ssl + end + + ## + # maximum SSL version to use + + def max_version= max_version + @max_version = max_version + + reconnect_ssl + end + + ## + # Sets the depth of SSL certificate verification + + def verify_depth= verify_depth + @verify_depth = verify_depth + + reconnect_ssl + end ## # Sets the HTTPS verify mode. Defaults to OpenSSL::SSL::VERIFY_PEER. @@ -1229,5 +1188,6 @@ application: end -require 'bundler/vendor/net-http-persistent/lib/net/http/persistent/ssl_reuse' +require 'bundler/vendor/net-http-persistent/lib/net/http/persistent/connection' +require 'bundler/vendor/net-http-persistent/lib/net/http/persistent/pool' diff --git a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/connection.rb b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/connection.rb new file mode 100644 index 0000000000..a57a5d1352 --- /dev/null +++ b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/connection.rb @@ -0,0 +1,40 @@ +## +# A Net::HTTP connection wrapper that holds extra information for managing the +# connection's lifetime. + +class Bundler::Persistent::Net::HTTP::Persistent::Connection # :nodoc: + + attr_accessor :http + + attr_accessor :last_use + + attr_accessor :requests + + attr_accessor :ssl_generation + + def initialize http_class, http_args, ssl_generation + @http = http_class.new(*http_args) + @ssl_generation = ssl_generation + + reset + end + + def finish + @http.finish + rescue IOError + ensure + reset + end + + def reset + @last_use = Bundler::Persistent::Net::HTTP::Persistent::EPOCH + @requests = 0 + end + + def ressl ssl_generation + @ssl_generation = ssl_generation + + finish + end + +end diff --git a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/pool.rb b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/pool.rb new file mode 100644 index 0000000000..9af0b328b2 --- /dev/null +++ b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/pool.rb @@ -0,0 +1,50 @@ +class Bundler::Persistent::Net::HTTP::Persistent::Pool < Bundler::ConnectionPool # :nodoc: + + attr_reader :available # :nodoc: + attr_reader :key # :nodoc: + + def initialize(options = {}, &block) + super + + @available = Bundler::Persistent::Net::HTTP::Persistent::TimedStackMulti.new(@size, &block) + @key = "current-#{@available.object_id}" + end + + def checkin net_http_args + stack = Thread.current[@key][net_http_args] ||= [] + + raise Bundler::ConnectionPool::Error, 'no connections are checked out' if + stack.empty? + + conn = stack.pop + + if stack.empty? + @available.push conn, connection_args: net_http_args + end + + nil + end + + def checkout net_http_args + stacks = Thread.current[@key] ||= {} + stack = stacks[net_http_args] ||= [] + + if stack.empty? then + conn = @available.pop connection_args: net_http_args + else + conn = stack.last + end + + stack.push conn + + conn + end + + def shutdown + Thread.current[@key] = nil + super + end +end + +require 'bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi' + diff --git a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/ssl_reuse.rb b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/ssl_reuse.rb deleted file mode 100644 index 1b6b789f6d..0000000000 --- a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/ssl_reuse.rb +++ /dev/null @@ -1,129 +0,0 @@ -## -# This Net::HTTP subclass adds SSL session reuse and Server Name Indication -# (SNI) RFC 3546. -# -# DO NOT DEPEND UPON THIS CLASS -# -# This class is an implementation detail and is subject to change or removal -# at any time. - -class Bundler::Persistent::Net::HTTP::Persistent::SSLReuse < Net::HTTP - - @is_proxy_class = false - @proxy_addr = nil - @proxy_port = nil - @proxy_user = nil - @proxy_pass = nil - - def initialize address, port = nil # :nodoc: - super - - @ssl_session = nil - end - - ## - # From ruby trunk r33086 including http://redmine.ruby-lang.org/issues/5341 - - def connect # :nodoc: - D "opening connection to #{conn_address()}..." - s = timeout(@open_timeout) { TCPSocket.open(conn_address(), conn_port()) } - D "opened" - if use_ssl? - ssl_parameters = Hash.new - iv_list = instance_variables - SSL_ATTRIBUTES.each do |name| - ivname = "@#{name}".intern - if iv_list.include?(ivname) and - value = instance_variable_get(ivname) - ssl_parameters[name] = value - end - end - unless @ssl_context then - @ssl_context = OpenSSL::SSL::SSLContext.new - @ssl_context.set_params(ssl_parameters) - end - s = OpenSSL::SSL::SSLSocket.new(s, @ssl_context) - s.sync_close = true - end - @socket = Net::BufferedIO.new(s) - @socket.read_timeout = @read_timeout - @socket.continue_timeout = @continue_timeout if - @socket.respond_to? :continue_timeout - @socket.debug_output = @debug_output - if use_ssl? - begin - if proxy? - @socket.writeline sprintf('CONNECT %s:%s HTTP/%s', - @address, @port, HTTPVersion) - @socket.writeline "Host: #{@address}:#{@port}" - if proxy_user - credential = ["#{proxy_user}:#{proxy_pass}"].pack('m') - credential.delete!("\r\n") - @socket.writeline "Proxy-Authorization: Basic #{credential}" - end - @socket.writeline '' - Net::HTTPResponse.read_new(@socket).value - end - s.session = @ssl_session if @ssl_session - # Server Name Indication (SNI) RFC 3546 - s.hostname = @address if s.respond_to? :hostname= - timeout(@open_timeout) { s.connect } - if @ssl_context.verify_mode != OpenSSL::SSL::VERIFY_NONE - s.post_connection_check(@address) - end - @ssl_session = s.session - rescue => exception - D "Conn close because of connect error #{exception}" - @socket.close if @socket and not @socket.closed? - raise exception - end - end - on_connect - end if RUBY_VERSION > '1.9' - - ## - # From ruby_1_8_7 branch r29865 including a modified - # http://redmine.ruby-lang.org/issues/5341 - - def connect # :nodoc: - D "opening connection to #{conn_address()}..." - s = timeout(@open_timeout) { TCPSocket.open(conn_address(), conn_port()) } - D "opened" - if use_ssl? - unless @ssl_context.verify_mode - warn "warning: peer certificate won't be verified in this SSL session" - @ssl_context.verify_mode = OpenSSL::SSL::VERIFY_NONE - end - s = OpenSSL::SSL::SSLSocket.new(s, @ssl_context) - s.sync_close = true - end - @socket = Net::BufferedIO.new(s) - @socket.read_timeout = @read_timeout - @socket.debug_output = @debug_output - if use_ssl? - if proxy? - @socket.writeline sprintf('CONNECT %s:%s HTTP/%s', - @address, @port, HTTPVersion) - @socket.writeline "Host: #{@address}:#{@port}" - if proxy_user - credential = ["#{proxy_user}:#{proxy_pass}"].pack('m') - credential.delete!("\r\n") - @socket.writeline "Proxy-Authorization: Basic #{credential}" - end - @socket.writeline '' - Net::HTTPResponse.read_new(@socket).value - end - s.session = @ssl_session if @ssl_session - s.connect - if @ssl_context.verify_mode != OpenSSL::SSL::VERIFY_NONE - s.post_connection_check(@address) - end - @ssl_session = s.session - end - on_connect - end if RUBY_VERSION < '1.9' - - private :connect - -end - diff --git a/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi.rb b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi.rb new file mode 100644 index 0000000000..280997e32a --- /dev/null +++ b/lib/bundler/vendor/net-http-persistent/lib/net/http/persistent/timed_stack_multi.rb @@ -0,0 +1,69 @@ +class Bundler::Persistent::Net::HTTP::Persistent::TimedStackMulti < Bundler::ConnectionPool::TimedStack # :nodoc: + + def initialize(size = 0, &block) + super + + @enqueued = 0 + @ques = Hash.new { |h, k| h[k] = [] } + @lru = {} + @key = :"connection_args-#{object_id}" + end + + def empty? + (@created - @enqueued) >= @max + end + + def length + @max - @created + @enqueued + end + + private + + def connection_stored? options = {} # :nodoc: + !@ques[options[:connection_args]].empty? + end + + def fetch_connection options = {} # :nodoc: + connection_args = options[:connection_args] + + @enqueued -= 1 + lru_update connection_args + @ques[connection_args].pop + end + + def lru_update connection_args # :nodoc: + @lru.delete connection_args + @lru[connection_args] = true + end + + def shutdown_connections # :nodoc: + @ques.each_key do |key| + super connection_args: key + end + end + + def store_connection obj, options = {} # :nodoc: + @ques[options[:connection_args]].push obj + @enqueued += 1 + end + + def try_create options = {} # :nodoc: + connection_args = options[:connection_args] + + if @created >= @max && @enqueued >= 1 + oldest, = @lru.first + @lru.delete oldest + @ques[oldest].pop + + @created -= 1 + end + + if @created < @max + @created += 1 + lru_update connection_args + return @create_block.call(connection_args) + end + end + +end + diff --git a/lib/bundler/vendored_persistent.rb b/lib/bundler/vendored_persistent.rb index 7670b83992..045a761dac 100644 --- a/lib/bundler/vendored_persistent.rb +++ b/lib/bundler/vendored_persistent.rb @@ -20,13 +20,15 @@ require_relative "vendor/net-http-persistent/lib/net/http/persistent" module Bundler class PersistentHTTP < Persistent::Net::HTTP::Persistent def connection_for(uri) - connection = super - warn_old_tls_version_rubygems_connection(uri, connection) - connection + super(uri) do |connection| + result = yield connection + warn_old_tls_version_rubygems_connection(uri, connection) + result + end end def warn_old_tls_version_rubygems_connection(uri, connection) - return unless connection.use_ssl? + return unless connection.http.use_ssl? return unless (uri.host || "").end_with?("rubygems.org") socket = connection.instance_variable_get(:@socket) diff --git a/spec/bundler/bundler/vendored_persistent_spec.rb b/spec/bundler/bundler/vendored_persistent_spec.rb index c760c067e0..b4d68c2ea0 100644 --- a/spec/bundler/bundler/vendored_persistent_spec.rb +++ b/spec/bundler/bundler/vendored_persistent_spec.rb @@ -5,13 +5,13 @@ require "bundler/vendored_persistent" RSpec.describe Bundler::PersistentHTTP do describe "#warn_old_tls_version_rubygems_connection" do let(:uri) { "https://index.rubygems.org" } - let(:connection) { instance_double(subject.http_class) } + let(:connection) { instance_double(Bundler::Persistent::Net::HTTP::Persistent::Connection) } let(:tls_version) { "TLSv1.2" } let(:socket) { double("Socket") } let(:socket_io) { double("SocketIO") } before do - allow(connection).to receive(:use_ssl?).and_return(!tls_version.nil?) + allow(connection).to receive_message_chain(:http, :use_ssl?).and_return(!tls_version.nil?) allow(socket).to receive(:io).and_return(socket_io) if socket connection.instance_variable_set(:@socket, socket) -- cgit v1.2.3