diff options
author | Samuel Giddins <segiddins@segiddins.me> | 2023-10-22 13:25:07 -0700 |
---|---|---|
committer | git <svn-admin@ruby-lang.org> | 2023-11-15 08:33:14 +0000 |
commit | b69bbf588a3dd167d62dbb89f0cef25ebae4a7ea (patch) | |
tree | 613203d44c4b91ee854e006b5f1cae705df75403 /lib/bundler | |
parent | 536649f819ed8f2bb0f8f44b1a0ca5c6d1753b24 (diff) |
[rubygems/rubygems] User bundler UA when downloading gems
Gem::RemoteFetcher uses Gem::Request, which adds the RubyGems UA.
Gem::RemoteFetcher is used to download gems, as well as the full index.
We would like the bundler UA to be used whenever bundler is making
requests.
This PR also avoids unsafely mutating the headers hash on the shared
`Gem::RemoteFetcher.fetcher` instance, which could cause corruption or
incorrect headers when making parallel requests. Instead, we create one
remote fetcher per rubygems remote, which is similar to the connection
segregation bundler is already doing
https://github.com/rubygems/rubygems/commit/f0e8dacdec
Diffstat (limited to 'lib/bundler')
-rw-r--r-- | lib/bundler/cli.rb | 2 | ||||
-rw-r--r-- | lib/bundler/fetcher.rb | 12 | ||||
-rw-r--r-- | lib/bundler/fetcher/base.rb | 4 | ||||
-rw-r--r-- | lib/bundler/fetcher/gem_remote_fetcher.rb | 16 | ||||
-rw-r--r-- | lib/bundler/fetcher/index.rb | 2 | ||||
-rw-r--r-- | lib/bundler/rubygems_integration.rb | 19 | ||||
-rw-r--r-- | lib/bundler/source/rubygems.rb | 15 |
7 files changed, 47 insertions, 23 deletions
diff --git a/lib/bundler/cli.rb b/lib/bundler/cli.rb index dd91038d64..c27ce896c8 100644 --- a/lib/bundler/cli.rb +++ b/lib/bundler/cli.rb @@ -844,7 +844,7 @@ module Bundler return unless SharedHelpers.md5_available? latest = Fetcher::CompactIndex. - new(nil, Source::Rubygems::Remote.new(Bundler::URI("https://rubygems.org")), nil). + new(nil, Source::Rubygems::Remote.new(Bundler::URI("https://rubygems.org")), nil, nil). send(:compact_index_client). instance_variable_get(:@cache). dependencies("bundler"). diff --git a/lib/bundler/fetcher.rb b/lib/bundler/fetcher.rb index e5384c5679..dd91b7ceb4 100644 --- a/lib/bundler/fetcher.rb +++ b/lib/bundler/fetcher.rb @@ -204,6 +204,16 @@ module Bundler fetchers.first.api_fetcher? end + def gem_remote_fetcher + @gem_remote_fetcher ||= begin + require_relative "fetcher/gem_remote_fetcher" + fetcher = GemRemoteFetcher.new Gem.configuration[:http_proxy] + fetcher.headers["User-Agent"] = user_agent + fetcher.headers["X-Gemfile-Source"] = @remote.original_uri.to_s if @remote.original_uri + fetcher + end + end + private def available_fetchers @@ -218,7 +228,7 @@ module Bundler end def fetchers - @fetchers ||= available_fetchers.map {|f| f.new(downloader, @remote, uri) }.drop_while {|f| !f.available? } + @fetchers ||= available_fetchers.map {|f| f.new(downloader, @remote, uri, gem_remote_fetcher) }.drop_while {|f| !f.available? } end def fetch_specs(gem_names) diff --git a/lib/bundler/fetcher/base.rb b/lib/bundler/fetcher/base.rb index 99c6343c9a..cfec2f8e94 100644 --- a/lib/bundler/fetcher/base.rb +++ b/lib/bundler/fetcher/base.rb @@ -6,12 +6,14 @@ module Bundler attr_reader :downloader attr_reader :display_uri attr_reader :remote + attr_reader :gem_remote_fetcher - def initialize(downloader, remote, display_uri) + def initialize(downloader, remote, display_uri, gem_remote_fetcher) raise "Abstract class" if self.class == Base @downloader = downloader @remote = remote @display_uri = display_uri + @gem_remote_fetcher = gem_remote_fetcher end def remote_uri diff --git a/lib/bundler/fetcher/gem_remote_fetcher.rb b/lib/bundler/fetcher/gem_remote_fetcher.rb new file mode 100644 index 0000000000..3fc7b68263 --- /dev/null +++ b/lib/bundler/fetcher/gem_remote_fetcher.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require "rubygems/remote_fetcher" + +module Bundler + class Fetcher + class GemRemoteFetcher < Gem::RemoteFetcher + def request(*args) + super do |req| + req.delete("User-Agent") if headers["User-Agent"] + yield req if block_given? + end + end + end + end +end diff --git a/lib/bundler/fetcher/index.rb b/lib/bundler/fetcher/index.rb index c623647f01..6e37e1e5d1 100644 --- a/lib/bundler/fetcher/index.rb +++ b/lib/bundler/fetcher/index.rb @@ -6,7 +6,7 @@ module Bundler class Fetcher class Index < Base def specs(_gem_names) - Bundler.rubygems.fetch_all_remote_specs(remote) + Bundler.rubygems.fetch_all_remote_specs(remote, gem_remote_fetcher) rescue Gem::RemoteFetcher::FetchError => e case e.message when /certificate verify failed/ diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb index f420934ceb..bde2b39416 100644 --- a/lib/bundler/rubygems_integration.rb +++ b/lib/bundler/rubygems_integration.rb @@ -467,11 +467,9 @@ module Bundler Gem::Specification.all = specs end - def fetch_specs(remote, name) + def fetch_specs(remote, name, fetcher) require "rubygems/remote_fetcher" path = remote.uri.to_s + "#{name}.#{Gem.marshal_version}.gz" - fetcher = gem_remote_fetcher - fetcher.headers = { "X-Gemfile-Source" => remote.original_uri.to_s } if remote.original_uri string = fetcher.fetch_path(path) specs = Bundler.safe_load_marshal(string) raise MarshalError, "Specs #{name} from #{remote} is expected to be an Array but was unexpected class #{specs.class}" unless specs.is_a?(Array) @@ -481,18 +479,16 @@ module Bundler raise unless name == "prerelease_specs" end - def fetch_all_remote_specs(remote) - specs = fetch_specs(remote, "specs") - pres = fetch_specs(remote, "prerelease_specs") || [] + def fetch_all_remote_specs(remote, gem_remote_fetcher) + specs = fetch_specs(remote, "specs", gem_remote_fetcher) + pres = fetch_specs(remote, "prerelease_specs", gem_remote_fetcher) || [] specs.concat(pres) end - def download_gem(spec, uri, cache_dir) + def download_gem(spec, uri, cache_dir, fetcher) require "rubygems/remote_fetcher" uri = Bundler.settings.mirror_for(uri) - fetcher = gem_remote_fetcher - fetcher.headers = { "X-Gemfile-Source" => spec.remote.original_uri.to_s } if spec.remote.original_uri Bundler::Retry.new("download gem from #{uri}").attempts do gem_file_name = spec.file_name local_gem_path = File.join cache_dir, gem_file_name @@ -519,11 +515,6 @@ module Bundler raise Bundler::HTTPError, "Could not download gem from #{uri} due to underlying error <#{e.message}>" end - def gem_remote_fetcher - require "rubygems/remote_fetcher" - Gem::RemoteFetcher.fetcher - end - def build(spec, skip_validation = false) require "rubygems/package" Gem::Package.build(spec, skip_validation) diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb index 6d6f36e298..e67e7a97ff 100644 --- a/lib/bundler/source/rubygems.rb +++ b/lib/bundler/source/rubygems.rb @@ -255,11 +255,15 @@ module Bundler end end - def fetchers - @fetchers ||= remotes.map do |uri| + def remote_fetchers + @remote_fetchers ||= remotes.to_h do |uri| remote = Source::Rubygems::Remote.new(uri) - Bundler::Fetcher.new(remote) - end + [remote, Bundler::Fetcher.new(remote)] + end.freeze + end + + def fetchers + @fetchers ||= remote_fetchers.values.freeze end def double_check_for(unmet_dependency_names) @@ -480,7 +484,8 @@ module Bundler def download_gem(spec, download_cache_path, previous_spec = nil) uri = spec.remote.uri Bundler.ui.confirm("Fetching #{version_message(spec, previous_spec)}") - Bundler.rubygems.download_gem(spec, uri, download_cache_path) + gem_remote_fetcher = remote_fetchers.fetch(spec.remote).gem_remote_fetcher + Bundler.rubygems.download_gem(spec, uri, download_cache_path, gem_remote_fetcher) end # Returns the global cache path of the calling Rubygems::Source object. |