[rubygems/rubygems] User bundler UA when downloading gems

Gem::RemoteFetcher uses Gem::Request, which adds the RubyGems UA.
Gem::RemoteFetcher is used to download gems, as well as the full index.
We would like the bundler UA to be used whenever bundler is making
requests.

This PR also avoids unsafely mutating the headers hash on the shared
`Gem::RemoteFetcher.fetcher` instance, which could cause corruption or
incorrect headers when making parallel requests. Instead, we create one
remote fetcher per rubygems remote, which is similar to the connection
segregation bundler is already doing

https://github.com/rubygems/rubygems/commit/f0e8dacdec

7 files changed, 47 insertions, 23 deletions

diff --git a/lib/bundler/cli.rb b/lib/bundler/cli.rb
index dd91038d64..c27ce896c8 100644
--- a/lib/bundler/cli.rb
+++ b/lib/bundler/cli.rb
@@ -844,7 +844,7 @@ module Bundler
       return unless SharedHelpers.md5_available?
 
       latest = Fetcher::CompactIndex.
-               new(nil, Source::Rubygems::Remote.new(Bundler::URI("https://rubygems.org")), nil).
+               new(nil, Source::Rubygems::Remote.new(Bundler::URI("https://rubygems.org")), nil, nil).
                send(:compact_index_client).
                instance_variable_get(:@cache).
                dependencies("bundler").
diff --git a/lib/bundler/fetcher.rb b/lib/bundler/fetcher.rb
index e5384c5679..dd91b7ceb4 100644
--- a/lib/bundler/fetcher.rb
+++ b/lib/bundler/fetcher.rb
@@ -204,6 +204,16 @@ module Bundler
       fetchers.first.api_fetcher?
     end
 
+    def gem_remote_fetcher
+      @gem_remote_fetcher ||= begin
+        require_relative "fetcher/gem_remote_fetcher"
+        fetcher = GemRemoteFetcher.new Gem.configuration[:http_proxy]
+        fetcher.headers["User-Agent"] = user_agent
+        fetcher.headers["X-Gemfile-Source"] = @remote.original_uri.to_s if @remote.original_uri
+        fetcher
+      end
+    end
+
     private
 
     def available_fetchers
@@ -218,7 +228,7 @@ module Bundler
     end
 
     def fetchers
-      @fetchers ||= available_fetchers.map {|f| f.new(downloader, @remote, uri) }.drop_while {|f| !f.available? }
+      @fetchers ||= available_fetchers.map {|f| f.new(downloader, @remote, uri, gem_remote_fetcher) }.drop_while {|f| !f.available? }
     end
 
     def fetch_specs(gem_names)
diff --git a/lib/bundler/fetcher/base.rb b/lib/bundler/fetcher/base.rb
index 99c6343c9a..cfec2f8e94 100644
--- a/lib/bundler/fetcher/base.rb
+++ b/lib/bundler/fetcher/base.rb
@@ -6,12 +6,14 @@ module Bundler
       attr_reader :downloader
       attr_reader :display_uri
       attr_reader :remote
+      attr_reader :gem_remote_fetcher
 
-      def initialize(downloader, remote, display_uri)
+      def initialize(downloader, remote, display_uri, gem_remote_fetcher)
         raise "Abstract class" if self.class == Base
         @downloader = downloader
         @remote = remote
         @display_uri = display_uri
+        @gem_remote_fetcher = gem_remote_fetcher
       end
 
       def remote_uri
diff --git a/lib/bundler/fetcher/gem_remote_fetcher.rb b/lib/bundler/fetcher/gem_remote_fetcher.rb
new file mode 100644
index 0000000000..3fc7b68263
--- /dev/null
+++ b/lib/bundler/fetcher/gem_remote_fetcher.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "rubygems/remote_fetcher"
+
+module Bundler
+  class Fetcher
+    class GemRemoteFetcher < Gem::RemoteFetcher
+      def request(*args)
+        super do |req|
+          req.delete("User-Agent") if headers["User-Agent"]
+          yield req if block_given?
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bundler/fetcher/index.rb b/lib/bundler/fetcher/index.rb
index c623647f01..6e37e1e5d1 100644
--- a/lib/bundler/fetcher/index.rb
+++ b/lib/bundler/fetcher/index.rb
@@ -6,7 +6,7 @@ module Bundler
   class Fetcher
     class Index < Base
       def specs(_gem_names)
-        Bundler.rubygems.fetch_all_remote_specs(remote)
+        Bundler.rubygems.fetch_all_remote_specs(remote, gem_remote_fetcher)
       rescue Gem::RemoteFetcher::FetchError => e
         case e.message
         when /certificate verify failed/
diff --git a/lib/bundler/rubygems_integration.rb b/lib/bundler/rubygems_integration.rb
index f420934ceb..bde2b39416 100644
--- a/lib/bundler/rubygems_integration.rb
+++ b/lib/bundler/rubygems_integration.rb
@@ -467,11 +467,9 @@ module Bundler
       Gem::Specification.all = specs
     end
 
-    def fetch_specs(remote, name)
+    def fetch_specs(remote, name, fetcher)
       require "rubygems/remote_fetcher"
       path = remote.uri.to_s + "#{name}.#{Gem.marshal_version}.gz"
-      fetcher = gem_remote_fetcher
-      fetcher.headers = { "X-Gemfile-Source" => remote.original_uri.to_s } if remote.original_uri
       string = fetcher.fetch_path(path)
       specs = Bundler.safe_load_marshal(string)
       raise MarshalError, "Specs #{name} from #{remote} is expected to be an Array but was unexpected class #{specs.class}" unless specs.is_a?(Array)
@@ -481,18 +479,16 @@ module Bundler
       raise unless name == "prerelease_specs"
     end
 
-    def fetch_all_remote_specs(remote)
-      specs = fetch_specs(remote, "specs")
-      pres = fetch_specs(remote, "prerelease_specs") || []
+    def fetch_all_remote_specs(remote, gem_remote_fetcher)
+      specs = fetch_specs(remote, "specs", gem_remote_fetcher)
+      pres = fetch_specs(remote, "prerelease_specs", gem_remote_fetcher) || []
 
       specs.concat(pres)
     end
 
-    def download_gem(spec, uri, cache_dir)
+    def download_gem(spec, uri, cache_dir, fetcher)
       require "rubygems/remote_fetcher"
       uri = Bundler.settings.mirror_for(uri)
-      fetcher = gem_remote_fetcher
-      fetcher.headers = { "X-Gemfile-Source" => spec.remote.original_uri.to_s } if spec.remote.original_uri
       Bundler::Retry.new("download gem from #{uri}").attempts do
         gem_file_name = spec.file_name
         local_gem_path = File.join cache_dir, gem_file_name
@@ -519,11 +515,6 @@ module Bundler
       raise Bundler::HTTPError, "Could not download gem from #{uri} due to underlying error <#{e.message}>"
     end
 
-    def gem_remote_fetcher
-      require "rubygems/remote_fetcher"
-      Gem::RemoteFetcher.fetcher
-    end
-
     def build(spec, skip_validation = false)
       require "rubygems/package"
       Gem::Package.build(spec, skip_validation)
diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb
index 6d6f36e298..e67e7a97ff 100644
--- a/lib/bundler/source/rubygems.rb
+++ b/lib/bundler/source/rubygems.rb
@@ -255,11 +255,15 @@ module Bundler
         end
       end
 
-      def fetchers
-        @fetchers ||= remotes.map do |uri|
+      def remote_fetchers
+        @remote_fetchers ||= remotes.to_h do |uri|
           remote = Source::Rubygems::Remote.new(uri)
-          Bundler::Fetcher.new(remote)
-        end
+          [remote, Bundler::Fetcher.new(remote)]
+        end.freeze
+      end
+
+      def fetchers
+        @fetchers ||= remote_fetchers.values.freeze
       end
 
       def double_check_for(unmet_dependency_names)
@@ -480,7 +484,8 @@ module Bundler
       def download_gem(spec, download_cache_path, previous_spec = nil)
         uri = spec.remote.uri
         Bundler.ui.confirm("Fetching #{version_message(spec, previous_spec)}")
-        Bundler.rubygems.download_gem(spec, uri, download_cache_path)
+        gem_remote_fetcher = remote_fetchers.fetch(spec.remote).gem_remote_fetcher
+        Bundler.rubygems.download_gem(spec, uri, download_cache_path, gem_remote_fetcher)
       end
 
       # Returns the global cache path of the calling Rubygems::Source object.