diff options
Diffstat (limited to 'spec/bundler/support')
38 files changed, 708 insertions, 339 deletions
diff --git a/spec/bundler/support/activate.rb b/spec/bundler/support/activate.rb new file mode 100644 index 0000000000..143b77833d --- /dev/null +++ b/spec/bundler/support/activate.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require "rubygems" +Gem.instance_variable_set(:@ruby, ENV["RUBY"]) if ENV["RUBY"] + +require_relative "path" +bundler_gemspec = Spec::Path.loaded_gemspec +bundler_gemspec.instance_variable_set(:@full_gem_path, Spec::Path.source_root.to_s) +bundler_gemspec.activate if bundler_gemspec.respond_to?(:activate) diff --git a/spec/bundler/support/artifice/compact_index_checksum_mismatch.rb b/spec/bundler/support/artifice/compact_index_checksum_mismatch.rb index a6545b9ee4..83b147d2ae 100644 --- a/spec/bundler/support/artifice/compact_index_checksum_mismatch.rb +++ b/spec/bundler/support/artifice/compact_index_checksum_mismatch.rb @@ -4,10 +4,10 @@ require_relative "helpers/compact_index" class CompactIndexChecksumMismatch < CompactIndexAPI get "/versions" do - headers "ETag" => quote("123") + headers "Repr-Digest" => "sha-256=:ungWv48Bz+pBQUDeXa4iI7ADYaOWF3qctBD/YfIAFa0=:" headers "Surrogate-Control" => "max-age=2592000, stale-while-revalidate=60" content_type "text/plain" - body "" + body "content does not match the checksum" end end diff --git a/spec/bundler/support/artifice/compact_index_concurrent_download.rb b/spec/bundler/support/artifice/compact_index_concurrent_download.rb index 35548f278c..5d55b8a72b 100644 --- a/spec/bundler/support/artifice/compact_index_concurrent_download.rb +++ b/spec/bundler/support/artifice/compact_index_concurrent_download.rb @@ -7,11 +7,12 @@ class CompactIndexConcurrentDownload < CompactIndexAPI versions = File.join(Bundler.rubygems.user_home, ".bundle", "cache", "compact_index", "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions") - # Verify the original (empty) content hasn't been deleted, e.g. on a retry - File.binread(versions) == "" || raise("Original file should be present and empty") + # Verify the original content hasn't been deleted, e.g. on a retry + data = File.binread(versions) + data == "created_at" || raise("Original file should be present with expected content") # Verify this is only requested once for a partial download - env["HTTP_RANGE"] || raise("Missing Range header for expected partial download") + env["HTTP_RANGE"] == "bytes=#{data.bytesize - 1}-" || raise("Missing Range header for expected partial download") # Overwrite the file in parallel, which should be then overwritten # after a successful download to prevent corruption diff --git a/spec/bundler/support/artifice/compact_index_etag_match.rb b/spec/bundler/support/artifice/compact_index_etag_match.rb new file mode 100644 index 0000000000..08d7b5ec53 --- /dev/null +++ b/spec/bundler/support/artifice/compact_index_etag_match.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require_relative "helpers/compact_index" + +class CompactIndexEtagMatch < CompactIndexAPI + get "/versions" do + raise "ETag header should be present" unless env["HTTP_IF_NONE_MATCH"] + headers "ETag" => env["HTTP_IF_NONE_MATCH"] + status 304 + body "" + end +end + +require_relative "helpers/artifice" + +Artifice.activate_with(CompactIndexEtagMatch) diff --git a/spec/bundler/support/artifice/compact_index_host_redirect.rb b/spec/bundler/support/artifice/compact_index_host_redirect.rb index 9a711186db..4f82bf3812 100644 --- a/spec/bundler/support/artifice/compact_index_host_redirect.rb +++ b/spec/bundler/support/artifice/compact_index_host_redirect.rb @@ -3,7 +3,7 @@ require_relative "helpers/compact_index" class CompactIndexHostRedirect < CompactIndexAPI - get "/fetch/actual/gem/:id", :host_name => "localgemserver.test" do + get "/fetch/actual/gem/:id", host_name: "localgemserver.test" do redirect "http://bundler.localgemserver.test#{request.path_info}" end diff --git a/spec/bundler/support/artifice/compact_index_partial_update.rb b/spec/bundler/support/artifice/compact_index_partial_update.rb index 8c73011346..f111d91ef9 100644 --- a/spec/bundler/support/artifice/compact_index_partial_update.rb +++ b/spec/bundler/support/artifice/compact_index_partial_update.rb @@ -23,7 +23,7 @@ class CompactIndexPartialUpdate < CompactIndexAPI # Verify that a partial request is made, starting from the index of the # final byte of the cached file. unless env["HTTP_RANGE"] == "bytes=#{File.binread(cached_versions_path).bytesize - 1}-" - raise("Range header should be present, and start from the index of the final byte of the cache.") + raise("Range header should be present, and start from the index of the final byte of the cache. #{env["HTTP_RANGE"].inspect}") end etag_response do diff --git a/spec/bundler/support/artifice/compact_index_partial_update_bad_digest.rb b/spec/bundler/support/artifice/compact_index_partial_update_bad_digest.rb new file mode 100644 index 0000000000..ac04336636 --- /dev/null +++ b/spec/bundler/support/artifice/compact_index_partial_update_bad_digest.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require_relative "helpers/compact_index" + +# The purpose of this Artifice is to test that an incremental response is invalidated +# and a second request is issued for the full content. +class CompactIndexPartialUpdateBadDigest < CompactIndexAPI + def partial_update_bad_digest + response_body = yield + if request.env["HTTP_RANGE"] + headers "Repr-Digest" => "sha-256=:#{Digest::SHA256.base64digest("wrong digest on ranged request")}:" + else + headers "Repr-Digest" => "sha-256=:#{Digest::SHA256.base64digest(response_body)}:" + end + headers "Surrogate-Control" => "max-age=2592000, stale-while-revalidate=60" + content_type "text/plain" + requested_range_for(response_body) + end + + get "/versions" do + partial_update_bad_digest do + file = tmp("versions.list") + FileUtils.rm_f(file) + file = CompactIndex::VersionsFile.new(file.to_s) + file.create(gems) + file.contents([], calculate_info_checksums: true) + end + end + + get "/info/:name" do + partial_update_bad_digest do + gem = gems.find {|g| g.name == params[:name] } + CompactIndex.info(gem ? gem.versions : []) + end + end +end + +require_relative "helpers/artifice" + +Artifice.activate_with(CompactIndexPartialUpdateBadDigest) diff --git a/spec/bundler/support/artifice/compact_index_partial_update_no_etag_not_incremental.rb b/spec/bundler/support/artifice/compact_index_partial_update_no_digest_not_incremental.rb index 20546ba4c3..99bae039f0 100644 --- a/spec/bundler/support/artifice/compact_index_partial_update_no_etag_not_incremental.rb +++ b/spec/bundler/support/artifice/compact_index_partial_update_no_digest_not_incremental.rb @@ -2,8 +2,10 @@ require_relative "helpers/compact_index" -class CompactIndexPartialUpdateNoEtagNotIncremental < CompactIndexAPI - def partial_update_no_etag +# The purpose of this Artifice is to test that an incremental response is ignored +# when the digest is not present to verify that the partial response is valid. +class CompactIndexPartialUpdateNoDigestNotIncremental < CompactIndexAPI + def partial_update_no_digest response_body = yield headers "Surrogate-Control" => "max-age=2592000, stale-while-revalidate=60" content_type "text/plain" @@ -11,12 +13,12 @@ class CompactIndexPartialUpdateNoEtagNotIncremental < CompactIndexAPI end get "/versions" do - partial_update_no_etag do + partial_update_no_digest do file = tmp("versions.list") FileUtils.rm_f(file) file = CompactIndex::VersionsFile.new(file.to_s) file.create(gems) - lines = file.contents([], :calculate_info_checksums => true).split("\n") + lines = file.contents([], calculate_info_checksums: true).split("\n") name, versions, checksum = lines.last.split(" ") # shuffle versions so new versions are not appended to the end @@ -25,7 +27,7 @@ class CompactIndexPartialUpdateNoEtagNotIncremental < CompactIndexAPI end get "/info/:name" do - partial_update_no_etag do + partial_update_no_digest do gem = gems.find {|g| g.name == params[:name] } lines = CompactIndex.info(gem ? gem.versions : []).split("\n") @@ -37,4 +39,4 @@ end require_relative "helpers/artifice" -Artifice.activate_with(CompactIndexPartialUpdateNoEtagNotIncremental) +Artifice.activate_with(CompactIndexPartialUpdateNoDigestNotIncremental) diff --git a/spec/bundler/support/artifice/compact_index_range_ignored.rb b/spec/bundler/support/artifice/compact_index_range_ignored.rb new file mode 100644 index 0000000000..2303682c1f --- /dev/null +++ b/spec/bundler/support/artifice/compact_index_range_ignored.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require_relative "helpers/compact_index" + +class CompactIndexRangeIgnored < CompactIndexAPI + # Stub the server to not return 304 so that we don't bypass all the logic + def not_modified?(_checksum) + false + end + + get "/versions" do + cached_versions_path = File.join( + Bundler.rubygems.user_home, ".bundle", "cache", "compact_index", + "localgemserver.test.80.dd34752a738ee965a2a4298dc16db6c5", "versions" + ) + + # Verify a cached copy of the versions file exists + unless File.binread(cached_versions_path).size > 0 + raise("Cached versions file should be present and have content") + end + + # Verify that a partial request is made, starting from the index of the + # final byte of the cached file. + unless env.delete("HTTP_RANGE") + raise("Expected client to write the full response on the first try") + end + + etag_response do + file = tmp("versions.list") + FileUtils.rm_f(file) + file = CompactIndex::VersionsFile.new(file.to_s) + file.create(gems) + file.contents + end + end +end + +require_relative "helpers/artifice" + +Artifice.activate_with(CompactIndexRangeIgnored) diff --git a/spec/bundler/support/artifice/compact_index_strict_basic_authentication.rb b/spec/bundler/support/artifice/compact_index_strict_basic_authentication.rb index fa25c4eca1..96259385e7 100644 --- a/spec/bundler/support/artifice/compact_index_strict_basic_authentication.rb +++ b/spec/bundler/support/artifice/compact_index_strict_basic_authentication.rb @@ -10,7 +10,7 @@ class CompactIndexStrictBasicAuthentication < CompactIndexAPI # Only accepts password == "password" unless env["HTTP_AUTHORIZATION"] == "Basic dXNlcjpwYXNz" - halt 403, "Authentication failed" + halt 401, "Authentication failed" end end end diff --git a/spec/bundler/support/artifice/compact_index_wrong_gem_checksum.rb b/spec/bundler/support/artifice/compact_index_wrong_gem_checksum.rb index acc13a56ff..9bd2ca0a9d 100644 --- a/spec/bundler/support/artifice/compact_index_wrong_gem_checksum.rb +++ b/spec/bundler/support/artifice/compact_index_wrong_gem_checksum.rb @@ -7,7 +7,8 @@ class CompactIndexWrongGemChecksum < CompactIndexAPI etag_response do name = params[:name] gem = gems.find {|g| g.name == name } - checksum = ENV.fetch("BUNDLER_SPEC_#{name.upcase}_CHECKSUM") { "ab" * 22 } + # This generates the hexdigest "2222222222222222222222222222222222222222222222222222222222222222" + checksum = ENV.fetch("BUNDLER_SPEC_#{name.upcase}_CHECKSUM") { "IiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiI=" } versions = gem ? gem.versions : [] versions.each {|v| v.checksum = checksum } CompactIndex.info(versions) diff --git a/spec/bundler/support/artifice/endpoint_500.rb b/spec/bundler/support/artifice/endpoint_500.rb index d8ab6b65bc..b1ed1964c8 100644 --- a/spec/bundler/support/artifice/endpoint_500.rb +++ b/spec/bundler/support/artifice/endpoint_500.rb @@ -2,7 +2,7 @@ require_relative "../path" -$LOAD_PATH.unshift(*Dir[Spec::Path.base_system_gem_path.join("gems/{mustermann,rack,tilt,sinatra,ruby2_keywords}-*/lib")].map(&:to_s)) +$LOAD_PATH.unshift(*Dir[Spec::Path.base_system_gem_path.join("gems/{mustermann,rack,tilt,sinatra,ruby2_keywords,base64}-*/lib")].map(&:to_s)) require "sinatra/base" diff --git a/spec/bundler/support/artifice/endpoint_host_redirect.rb b/spec/bundler/support/artifice/endpoint_host_redirect.rb index 0efb6cda02..6ce51bed93 100644 --- a/spec/bundler/support/artifice/endpoint_host_redirect.rb +++ b/spec/bundler/support/artifice/endpoint_host_redirect.rb @@ -3,7 +3,7 @@ require_relative "helpers/endpoint" class EndpointHostRedirect < Endpoint - get "/fetch/actual/gem/:id", :host_name => "localgemserver.test" do + get "/fetch/actual/gem/:id", host_name: "localgemserver.test" do redirect "http://bundler.localgemserver.test#{request.path_info}" end diff --git a/spec/bundler/support/artifice/endpoint_mirror_source.rb b/spec/bundler/support/artifice/endpoint_mirror_source.rb index 6ea1a77eca..fed7a746b9 100644 --- a/spec/bundler/support/artifice/endpoint_mirror_source.rb +++ b/spec/bundler/support/artifice/endpoint_mirror_source.rb @@ -4,7 +4,7 @@ require_relative "helpers/endpoint" class EndpointMirrorSource < Endpoint get "/gems/:id" do - if request.env["HTTP_X_GEMFILE_SOURCE"] == "https://server.example.org/" + if request.env["HTTP_X_GEMFILE_SOURCE"] == "https://server.example.org/" && request.env["HTTP_USER_AGENT"].start_with?("bundler") File.binread("#{gem_repo1}/gems/#{params[:id]}") else halt 500 diff --git a/spec/bundler/support/artifice/endpoint_strict_basic_authentication.rb b/spec/bundler/support/artifice/endpoint_strict_basic_authentication.rb index 8ce1bdd4ad..dff360c5c5 100644 --- a/spec/bundler/support/artifice/endpoint_strict_basic_authentication.rb +++ b/spec/bundler/support/artifice/endpoint_strict_basic_authentication.rb @@ -10,7 +10,7 @@ class EndpointStrictBasicAuthentication < Endpoint # Only accepts password == "password" unless env["HTTP_AUTHORIZATION"] == "Basic dXNlcjpwYXNz" - halt 403, "Authentication failed" + halt 401, "Authentication failed" end end end diff --git a/spec/bundler/support/artifice/fail.rb b/spec/bundler/support/artifice/fail.rb index 6286e43fbd..8822e5b8e2 100644 --- a/spec/bundler/support/artifice/fail.rb +++ b/spec/bundler/support/artifice/fail.rb @@ -1,11 +1,11 @@ # frozen_string_literal: true -require "net/http" +require "bundler/vendored_net_http" -class Fail < Net::HTTP - # Net::HTTP uses a @newimpl instance variable to decide whether +class Fail < Gem::Net::HTTP + # Gem::Net::HTTP uses a @newimpl instance variable to decide whether # to use a legacy implementation. Since we are subclassing - # Net::HTTP, we must set it + # Gem::Net::HTTP, we must set it @newimpl = true def request(req, body = nil, &block) @@ -17,13 +17,11 @@ class Fail < Net::HTTP end def exception(req) - name = ENV.fetch("BUNDLER_SPEC_EXCEPTION") { "Errno::ENETUNREACH" } - const = name.split("::").reduce(Object) {|mod, sym| mod.const_get(sym) } - const.new("host down: Bundler spec artifice fail! #{req["PATH_INFO"]}") + Errno::ENETUNREACH.new("host down: Bundler spec artifice fail! #{req["PATH_INFO"]}") end end require_relative "helpers/artifice" -# Replace Net::HTTP with our failing subclass +# Replace Gem::Net::HTTP with our failing subclass Artifice.replace_net_http(::Fail) diff --git a/spec/bundler/support/artifice/helpers/artifice.rb b/spec/bundler/support/artifice/helpers/artifice.rb index b8c78614fb..788268295c 100644 --- a/spec/bundler/support/artifice/helpers/artifice.rb +++ b/spec/bundler/support/artifice/helpers/artifice.rb @@ -4,7 +4,7 @@ module Artifice # Activate Artifice with a particular Rack endpoint. # - # Calling this method will replace the Net::HTTP system + # Calling this method will replace the Gem::Net::HTTP system # with a replacement that routes all requests to the # Rack endpoint. # @@ -18,11 +18,11 @@ module Artifice # Deactivate the Artifice replacement. def self.deactivate - replace_net_http(::Net::HTTP) + replace_net_http(::Gem::Net::HTTP) end def self.replace_net_http(value) - ::Net.class_eval do + ::Gem::Net.class_eval do remove_const(:HTTP) const_set(:HTTP, value) end diff --git a/spec/bundler/support/artifice/helpers/compact_index.rb b/spec/bundler/support/artifice/helpers/compact_index.rb index 4df47a9659..3fc1ce7fef 100644 --- a/spec/bundler/support/artifice/helpers/compact_index.rb +++ b/spec/bundler/support/artifice/helpers/compact_index.rb @@ -4,6 +4,7 @@ require_relative "endpoint" $LOAD_PATH.unshift Dir[Spec::Path.base_system_gem_path.join("gems/compact_index*/lib")].first.to_s require "compact_index" +require "digest" class CompactIndexAPI < Endpoint helpers do @@ -17,9 +18,10 @@ class CompactIndexAPI < Endpoint def etag_response response_body = yield - checksum = Digest(:MD5).hexdigest(response_body) - return if not_modified?(checksum) - headers "ETag" => quote(checksum) + etag = Digest::MD5.hexdigest(response_body) + headers "ETag" => quote(etag) + return if not_modified?(etag) + headers "Repr-Digest" => "sha-256=:#{Digest::SHA256.base64digest(response_body)}:" headers "Surrogate-Control" => "max-age=2592000, stale-while-revalidate=60" content_type "text/plain" requested_range_for(response_body) @@ -29,17 +31,16 @@ class CompactIndexAPI < Endpoint raise end - def not_modified?(checksum) + def not_modified?(etag) etags = parse_etags(request.env["HTTP_IF_NONE_MATCH"]) - return unless etags.include?(checksum) - headers "ETag" => quote(checksum) + return unless etags.include?(etag) status 304 body "" end def requested_range_for(response_body) - ranges = Rack::Utils.byte_ranges(env, response_body.bytesize) + ranges = Rack::Utils.get_byte_ranges(env["HTTP_RANGE"], response_body.bytesize) if ranges status 206 @@ -75,15 +76,17 @@ class CompactIndexAPI < Endpoint specs.group_by(&:name).map do |name, versions| gem_versions = versions.map do |spec| - deps = spec.dependencies.select {|d| d.type == :runtime }.map do |d| + deps = spec.runtime_dependencies.map do |d| reqs = d.requirement.requirements.map {|r| r.join(" ") }.join(", ") CompactIndex::Dependency.new(d.name, reqs) end - checksum = begin - Digest(:SHA256).file("#{gem_repo}/gems/#{spec.original_name}.gem").base64digest - rescue StandardError - nil - end + begin + checksum = ENV.fetch("BUNDLER_SPEC_#{name.upcase}_CHECKSUM") do + Digest(:SHA256).file("#{gem_repo}/gems/#{spec.original_name}.gem").hexdigest + end + rescue StandardError + checksum = nil + end CompactIndex::GemVersion.new(spec.version.version, spec.platform.to_s, checksum, nil, deps, spec.required_ruby_version.to_s, spec.required_rubygems_version.to_s) end diff --git a/spec/bundler/support/artifice/helpers/endpoint.rb b/spec/bundler/support/artifice/helpers/endpoint.rb index fc0381dc38..83ba1be0fc 100644 --- a/spec/bundler/support/artifice/helpers/endpoint.rb +++ b/spec/bundler/support/artifice/helpers/endpoint.rb @@ -2,7 +2,7 @@ require_relative "../../path" -$LOAD_PATH.unshift(*Dir[Spec::Path.base_system_gem_path.join("gems/{mustermann,rack,tilt,sinatra,ruby2_keywords}-*/lib")].map(&:to_s)) +$LOAD_PATH.unshift(*Dir[Spec::Path.base_system_gem_path.join("gems/{mustermann,rack,tilt,sinatra,ruby2_keywords,base64}-*/lib")].map(&:to_s)) require "sinatra/base" @@ -69,10 +69,10 @@ class Endpoint < Sinatra::Base spec = load_spec(name, version, platform, gem_repo) next unless gem_names.include?(spec.name) { - :name => spec.name, - :number => spec.version.version, - :platform => spec.platform.to_s, - :dependencies => spec.dependencies.select {|dep| dep.type == :runtime }.map do |dep| + name: spec.name, + number: spec.version.version, + platform: spec.platform.to_s, + dependencies: spec.runtime_dependencies.map do |dep| [dep.name, dep.requirement.requirements.map {|a| a.join(" ") }.join(", ")] end, } diff --git a/spec/bundler/support/artifice/helpers/rack_request.rb b/spec/bundler/support/artifice/helpers/rack_request.rb index c4a07812a6..f419bacb8c 100644 --- a/spec/bundler/support/artifice/helpers/rack_request.rb +++ b/spec/bundler/support/artifice/helpers/rack_request.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require "rack/test" -require "net/http" +require "bundler/vendored_net_http" module Artifice module Net @@ -16,25 +16,25 @@ module Artifice end end - class HTTP < ::Net::HTTP + class HTTP < ::Gem::Net::HTTP class << self attr_accessor :endpoint end - # Net::HTTP uses a @newimpl instance variable to decide whether + # Gem::Net::HTTP uses a @newimpl instance variable to decide whether # to use a legacy implementation. Since we are subclassing - # Net::HTTP, we must set it + # Gem::Net::HTTP, we must set it @newimpl = true # We don't need to connect, so blank out this method def connect end - # Replace the Net::HTTP request method with a method + # Replace the Gem::Net::HTTP request method with a method # that converts the request into a Rack request and # dispatches it to the Rack endpoint. # - # @param [Net::HTTPRequest] req A Net::HTTPRequest + # @param [Net::HTTPRequest] req A Gem::Net::HTTPRequest # object, or one if its subclasses # @param [optional, String, #read] body This should # be sent as "rack.input". If it's a String, it will @@ -42,7 +42,7 @@ module Artifice # @return [Net::HTTPResponse] # # @yield [Net::HTTPResponse] If a block is provided, - # this method will yield the Net::HTTPResponse to + # this method will yield the Gem::Net::HTTPResponse to # it after the body is read. def request(req, body = nil, &block) rack_request = RackRequest.new(self.class.endpoint) @@ -56,17 +56,17 @@ module Artifice body_stream_contents = req.body_stream.read if req.body_stream response = rack_request.request("#{prefix}#{req.path}", - { :method => req.method, :input => body || req.body || body_stream_contents }) + { method: req.method, input: body || req.body || body_stream_contents }) make_net_http_response(response, &block) end private - # This method takes a Rack response and creates a Net::HTTPResponse + # This method takes a Rack response and creates a Gem::Net::HTTPResponse # Instead of trying to mock HTTPResponse directly, we just convert # the Rack response into a String that looks like a normal HTTP - # response and call Net::HTTPResponse.read_new + # response and call Gem::Net::HTTPResponse.read_new # # @param [Array(#to_i, Hash, #each)] response a Rack response # @return [Net::HTTPResponse] @@ -86,8 +86,8 @@ module Artifice response_string << "" << body - response_io = ::Net::BufferedIO.new(StringIO.new(response_string.join("\n"))) - res = ::Net::HTTPResponse.read_new(response_io) + response_io = ::Gem::Net::BufferedIO.new(StringIO.new(response_string.join("\n"))) + res = ::Gem::Net::HTTPResponse.read_new(response_io) res.reading_body(response_io, true) do yield res if block_given? diff --git a/spec/bundler/support/artifice/vcr.rb b/spec/bundler/support/artifice/vcr.rb index 6a346f1ff9..7b9a8bdeaf 100644 --- a/spec/bundler/support/artifice/vcr.rb +++ b/spec/bundler/support/artifice/vcr.rb @@ -1,13 +1,13 @@ # frozen_string_literal: true -require "net/http" +require "bundler/vendored_net_http" require_relative "../path" -CASSETTE_PATH = "#{Spec::Path.spec_dir}/support/artifice/vcr_cassettes" -USED_CASSETTES_PATH = "#{Spec::Path.spec_dir}/support/artifice/used_cassettes.txt" +CASSETTE_PATH = "#{Spec::Path.spec_dir}/support/artifice/vcr_cassettes".freeze +USED_CASSETTES_PATH = "#{Spec::Path.spec_dir}/support/artifice/used_cassettes.txt".freeze CASSETTE_NAME = ENV.fetch("BUNDLER_SPEC_VCR_CASSETTE_NAME") { "realworld" } -class BundlerVCRHTTP < Net::HTTP +class BundlerVCRHTTP < Gem::Net::HTTP class RequestHandler attr_reader :http, :request, :body, :response_block def initialize(http, request, body = nil, &response_block) @@ -41,8 +41,8 @@ class BundlerVCRHTTP < Net::HTTP def recorded_response File.open(request_pair_paths.last, "rb:ASCII-8BIT") do |response_file| - response_io = ::Net::BufferedIO.new(response_file) - ::Net::HTTPResponse.read_new(response_io).tap do |response| + response_io = ::Gem::Net::BufferedIO.new(response_file) + ::Gem::Net::HTTPResponse.read_new(response_io).tap do |response| response.decode_content = request.decode_content if request.respond_to?(:decode_content) response.uri = request.uri @@ -148,5 +148,5 @@ end require_relative "helpers/artifice" -# Replace Net::HTTP with our VCR subclass +# Replace Gem::Net::HTTP with our VCR subclass Artifice.replace_net_http(BundlerVCRHTTP) diff --git a/spec/bundler/support/artifice/windows.rb b/spec/bundler/support/artifice/windows.rb index 4d90e0a426..fea991c071 100644 --- a/spec/bundler/support/artifice/windows.rb +++ b/spec/bundler/support/artifice/windows.rb @@ -2,7 +2,7 @@ require_relative "../path" -$LOAD_PATH.unshift(*Dir[Spec::Path.base_system_gem_path.join("gems/{mustermann,rack,tilt,sinatra,ruby2_keywords}-*/lib")].map(&:to_s)) +$LOAD_PATH.unshift(*Dir[Spec::Path.base_system_gem_path.join("gems/{mustermann,rack,tilt,sinatra,ruby2_keywords,base64}-*/lib")].map(&:to_s)) require "sinatra/base" diff --git a/spec/bundler/support/build_metadata.rb b/spec/bundler/support/build_metadata.rb index 98d8ac23c8..189100edb7 100644 --- a/spec/bundler/support/build_metadata.rb +++ b/spec/bundler/support/build_metadata.rb @@ -10,20 +10,20 @@ module Spec def write_build_metadata(dir: source_root) build_metadata = { - :git_commit_sha => git_commit_sha, - :built_at => loaded_gemspec.date.utc.strftime("%Y-%m-%d"), - :release => true, + git_commit_sha: git_commit_sha, + built_at: loaded_gemspec.date.utc.strftime("%Y-%m-%d"), + release: true, } - replace_build_metadata(build_metadata, dir: dir) # rubocop:disable Style/HashSyntax + replace_build_metadata(build_metadata, dir: dir) end def reset_build_metadata(dir: source_root) build_metadata = { - :release => false, + release: false, } - replace_build_metadata(build_metadata, dir: dir) # rubocop:disable Style/HashSyntax + replace_build_metadata(build_metadata, dir: dir) end private @@ -41,7 +41,7 @@ module Spec end def git_commit_sha - ruby_core_tarball? ? "unknown" : sys_exec("git rev-parse --short HEAD", :dir => source_root).strip + ruby_core_tarball? ? "unknown" : git("rev-parse --short HEAD", source_root).strip end extend self diff --git a/spec/bundler/support/builders.rb b/spec/bundler/support/builders.rb index 7c16470153..8f646b9358 100644 --- a/spec/bundler/support/builders.rb +++ b/spec/bundler/support/builders.rb @@ -17,6 +17,10 @@ module Spec Gem::Platform.new(platform) end + def rake_version + "13.2.1" + end + def build_repo1 rake_path = Dir["#{Path.base_system_gems}/**/rake*.gem"].first @@ -49,7 +53,7 @@ module Spec build_gem "rails", "2.3.2" do |s| s.executables = "rails" - s.add_dependency "rake", "13.0.1" + s.add_dependency "rake", rake_version s.add_dependency "actionpack", "2.3.2" s.add_dependency "activerecord", "2.3.2" s.add_dependency "actionmailer", "2.3.2" @@ -73,11 +77,11 @@ module Spec s.add_dependency "activesupport", ">= 2.0.0" end - build_gem "rspec", "1.2.7", :no_default => true do |s| + build_gem "rspec", "1.2.7", no_default: true do |s| s.write "lib/spec.rb", "SPEC = '1.2.7'" end - build_gem "rack-test", :no_default => true do |s| + build_gem "rack-test", no_default: true do |s| s.write "lib/rack/test.rb", "RACK_TEST = '1.0'" end @@ -191,27 +195,25 @@ module Spec end end - def build_repo2(&blk) + def build_repo2(**kwargs, &blk) FileUtils.rm_rf gem_repo2 FileUtils.cp_r gem_repo1, gem_repo2 - update_repo2(&blk) if block_given? + update_repo2(**kwargs, &blk) if block_given? end # A repo that has no pre-installed gems included. (The caller completely # determines the contents with the block.) - def build_repo4(&blk) + def build_repo4(**kwargs, &blk) FileUtils.rm_rf gem_repo4 - build_repo(gem_repo4, &blk) + build_repo(gem_repo4, **kwargs, &blk) end def update_repo4(&blk) update_repo(gem_repo4, &blk) end - def update_repo2 - update_repo gem_repo2 do - yield if block_given? - end + def update_repo2(**kwargs, &blk) + update_repo(gem_repo2, **kwargs, &blk) end def build_security_repo @@ -229,12 +231,12 @@ module Spec end end - def build_repo(path, &blk) + def build_repo(path, **kwargs, &blk) return if File.directory?(path) FileUtils.mkdir_p("#{path}/gems") - update_repo(path, &blk) + update_repo(path,**kwargs, &blk) end def check_test_gems! @@ -251,7 +253,7 @@ module Spec end end - def update_repo(path) + def update_repo(path, build_compact_index: true) if path == gem_repo1 && caller.first.split(" ").last == "`build_repo`" raise "Updating gem_repo1 is unsupported -- use gem_repo2 instead" end @@ -260,7 +262,12 @@ module Spec @_build_repo = File.basename(path) yield with_gem_path_as Path.base_system_gem_path do - gem_command :generate_index, :dir => path + Dir[Spec::Path.base_system_gem_path.join("gems/rubygems-generate_index*/lib")].first || + raise("Could not find rubygems-generate_index lib directory in #{Spec::Path.base_system_gem_path}") + + command = "generate_index" + command += " --no-compact" if !build_compact_index && gem_command(command + " --help").include?("--[no-]compact") + gem_command command, dir: path end ensure @_build_path = nil @@ -290,6 +297,10 @@ module Spec build_with(LibBuilder, name, args, &blk) end + def build_bundler(*args, &blk) + build_with(BundlerBuilder, "bundler", args, &blk) + end + def build_gem(name, *args, &blk) build_with(GemBuilder, name, args, &blk) end @@ -395,6 +406,49 @@ module Spec alias_method :dep, :runtime end + class BundlerBuilder + attr_writer :required_ruby_version + + def initialize(context, name, version) + raise "can only build bundler" unless name == "bundler" + + @context = context + @version = version || Bundler::VERSION + end + + def _build(options = {}) + full_name = "bundler-#{@version}" + build_path = @context.tmp + full_name + bundler_path = build_path + "#{full_name}.gem" + + FileUtils.mkdir_p build_path + + @context.shipped_files.each do |shipped_file| + target_shipped_file = shipped_file + target_shipped_file = shipped_file.sub(/\Alibexec/, "exe") if @context.ruby_core? + target_shipped_file = build_path + target_shipped_file + target_shipped_dir = File.dirname(target_shipped_file) + FileUtils.mkdir_p target_shipped_dir unless File.directory?(target_shipped_dir) + FileUtils.cp shipped_file, target_shipped_file, preserve: true + end + + @context.replace_version_file(@version, dir: build_path) + @context.replace_required_ruby_version(@required_ruby_version, dir: build_path) if @required_ruby_version + + Spec::BuildMetadata.write_build_metadata(dir: build_path) + + @context.gem_command "build #{@context.relative_gemspec}", dir: build_path + + if block_given? + yield(bundler_path) + else + FileUtils.mv bundler_path, options[:path] + end + ensure + build_path.rmtree + end + end + class LibBuilder def initialize(context, name, version) @context = context @@ -440,8 +494,6 @@ module Spec write "ext/extconf.rb", <<-RUBY require "mkmf" - $extout = "$(topdir)/" + RbConfig::CONFIG["EXTOUT"] - extension_name = "#{name}_c" if extra_lib_dir = with_config("ext-lib") # add extra libpath if --with-ext-lib is @@ -466,7 +518,6 @@ module Spec if options[:rubygems_version] @spec.rubygems_version = options[:rubygems_version] - def @spec.mark_version; end def @spec.validate(*); end end @@ -523,7 +574,7 @@ module Spec default_branch = options[:default_branch] || "main" path = options[:path] || _default_path source = options[:source] || "git@#{path}" - super(options.merge(:path => path, :source => source)) + super(options.merge(path: path, source: source)) @context.git("config --global init.defaultBranch #{default_branch}", path) @context.git("init", path) @context.git("add *", path) @@ -537,7 +588,7 @@ module Spec class GitBareBuilder < LibBuilder def _build(options) path = options[:path] || _default_path - super(options.merge(:path => path)) + super(options.merge(path: path)) @context.git("init --bare", path) end end @@ -562,7 +613,7 @@ module Spec _default_files.keys.each do |path| _default_files[path] += "\n#{Builders.constantize(name)}_PREV_REF = '#{current_ref}'" end - super(options.merge(:path => libpath, :gemspec => update_gemspec, :source => source)) + super(options.merge(path: libpath, gemspec: update_gemspec, source: source)) @context.git("commit -am BUMP", libpath) end end @@ -585,7 +636,7 @@ module Spec class GemBuilder < LibBuilder def _build(opts) lib_path = opts[:lib_path] || @context.tmp(".tmp/#{@spec.full_name}") - lib_path = super(opts.merge(:path => lib_path, :no_default => opts[:no_default])) + lib_path = super(opts.merge(path: lib_path, no_default: opts[:no_default])) destination = opts[:path] || _default_path FileUtils.mkdir_p(lib_path.join(destination)) @@ -594,16 +645,16 @@ module Spec Bundler.rubygems.build(@spec, opts[:skip_validation]) end elsif opts[:skip_validation] - @context.gem_command "build --force #{@spec.name}", :dir => lib_path + @context.gem_command "build --force #{@spec.name}", dir: lib_path else - @context.gem_command "build #{@spec.name}", :dir => lib_path + @context.gem_command "build #{@spec.name}", dir: lib_path end gem_path = File.expand_path("#{@spec.full_name}.gem", lib_path) if opts[:to_system] - @context.system_gems gem_path, :default => opts[:default] + @context.system_gems gem_path, default: opts[:default] elsif opts[:to_bundle] - @context.system_gems gem_path, :path => @context.default_bundle_path + @context.system_gems gem_path, path: @context.default_bundle_path else FileUtils.mv(gem_path, destination) end diff --git a/spec/bundler/support/bundle.rb b/spec/bundler/support/bundle.rb index 5f808531ff..5d6d658040 100644 --- a/spec/bundler/support/bundle.rb +++ b/spec/bundler/support/bundle.rb @@ -1,10 +1,5 @@ # frozen_string_literal: true -require "rubygems" -Gem.instance_variable_set(:@ruby, ENV["RUBY"]) if ENV["RUBY"] +require_relative "activate" -require_relative "path" -bundler_gemspec = Spec::Path.loaded_gemspec -bundler_gemspec.instance_variable_set(:@full_gem_path, Spec::Path.source_root) -bundler_gemspec.activate if bundler_gemspec.respond_to?(:activate) load File.expand_path("bundle", Spec::Path.bindir) diff --git a/spec/bundler/support/checksums.rb b/spec/bundler/support/checksums.rb new file mode 100644 index 0000000000..f758559b3b --- /dev/null +++ b/spec/bundler/support/checksums.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Spec + module Checksums + class ChecksumsBuilder + def initialize(enabled = true, &block) + @enabled = enabled + @checksums = {} + yield self if block_given? + end + + def initialize_copy(original) + super + @checksums = @checksums.dup + end + + def checksum(repo, name, version, platform = Gem::Platform::RUBY) + name_tuple = Gem::NameTuple.new(name, version, platform) + gem_file = File.join(repo, "gems", "#{name_tuple.full_name}.gem") + File.open(gem_file, "rb") do |f| + register(name_tuple, Bundler::Checksum.from_gem(f, "#{gem_file} (via ChecksumsBuilder#checksum)")) + end + end + + def no_checksum(name, version, platform = Gem::Platform::RUBY) + name_tuple = Gem::NameTuple.new(name, version, platform) + register(name_tuple, nil) + end + + def delete(name, platform = nil) + @checksums.reject! {|k, _| k.name == name && (platform.nil? || k.platform == platform) } + end + + def to_s + return "" unless @enabled + + locked_checksums = @checksums.map do |name_tuple, checksum| + checksum &&= " #{checksum.to_lock}" + " #{name_tuple.lock_name}#{checksum}\n" + end + + "\nCHECKSUMS\n#{locked_checksums.sort.join}" + end + + private + + def register(name_tuple, checksum) + delete(name_tuple.name, name_tuple.platform) + @checksums[name_tuple] = checksum + end + end + + def checksums_section(enabled = true, &block) + ChecksumsBuilder.new(enabled, &block) + end + + def checksums_section_when_existing(&block) + begin + enabled = lockfile.match?(/^CHECKSUMS$/) + rescue Errno::ENOENT + enabled = false + end + checksums_section(enabled, &block) + end + + def checksum_to_lock(*args) + checksums_section do |c| + c.checksum(*args) + end.to_s.sub(/^CHECKSUMS\n/, "").strip + end + + def checksum_digest(*args) + checksum_to_lock(*args).split(Bundler::Checksum::ALGO_SEPARATOR, 2).last + end + + # if prefixes is given, removes all checksums where the line + # has any of the prefixes on the line before the checksum + # otherwise, removes all checksums from the lockfile + def remove_checksums_from_lockfile(lockfile, *prefixes) + head, remaining = lockfile.split(/^CHECKSUMS$/, 2) + return lockfile unless remaining + checksums, tail = remaining.split("\n\n", 2) + + prefixes = + if prefixes.empty? + nil + else + /(#{prefixes.map {|p| Regexp.escape(p) }.join("|")})/ + end + + checksums = checksums.each_line.map do |line| + if prefixes.nil? || line.match?(prefixes) + line.gsub(/ sha256=[a-f0-9]{64}/i, "") + else + line + end + end + + head.concat( + "CHECKSUMS", + checksums.join, + "\n\n", + tail + ) + end + + def remove_checksums_section_from_lockfile(lockfile) + head, remaining = lockfile.split(/^CHECKSUMS$/, 2) + return lockfile unless remaining + _checksums, tail = remaining.split("\n\n", 2) + head.concat(tail) + end + end +end diff --git a/spec/bundler/support/command_execution.rb b/spec/bundler/support/command_execution.rb index 68e5c56c75..02726744d3 100644 --- a/spec/bundler/support/command_execution.rb +++ b/spec/bundler/support/command_execution.rb @@ -1,7 +1,37 @@ # frozen_string_literal: true module Spec - CommandExecution = Struct.new(:command, :working_directory, :exitstatus, :stdout, :stderr) do + class CommandExecution + def initialize(command, working_directory:, timeout:) + @command = command + @working_directory = working_directory + @timeout = timeout + @original_stdout = String.new + @original_stderr = String.new + end + + attr_accessor :exitstatus, :command, :original_stdout, :original_stderr + attr_reader :timeout + attr_writer :failure_reason + + def raise_error! + return unless failure? + + error_header = if failure_reason == :timeout + "Invoking `#{command}` was aborted after #{timeout} seconds with output:" + else + "Invoking `#{command}` failed with output:" + end + + raise <<~ERROR + #{error_header} + + ---------------------------------------------------------------------- + #{stdboth} + ---------------------------------------------------------------------- + ERROR + end + def to_s "$ #{command}" end @@ -11,6 +41,14 @@ module Spec @stdboth ||= [stderr, stdout].join("\n").strip end + def stdout + normalize(original_stdout) + end + + def stderr + normalize(original_stderr) + end + def to_s_verbose [ to_s, @@ -29,5 +67,13 @@ module Spec return true unless exitstatus exitstatus > 0 end + + private + + attr_reader :failure_reason + + def normalize(string) + string.force_encoding(Encoding::UTF_8).strip.gsub("\r\n", "\n") + end end end diff --git a/spec/bundler/support/env.rb b/spec/bundler/support/env.rb new file mode 100644 index 0000000000..4d99c892cd --- /dev/null +++ b/spec/bundler/support/env.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Spec + module Env + def ruby_core? + !ENV["GEM_COMMAND"].nil? + end + end +end diff --git a/spec/bundler/support/filters.rb b/spec/bundler/support/filters.rb index 78545d2e64..e1683ae75b 100644 --- a/spec/bundler/support/filters.rb +++ b/spec/bundler/support/filters.rb @@ -14,25 +14,25 @@ class RequirementChecker < Proc attr_accessor :provided def inspect - "\"!= #{provided}\"" + "\"#{provided}\"" end end RSpec.configure do |config| - config.filter_run_excluding :realworld => true + config.filter_run_excluding realworld: true git_version = Bundler::Source::Git::GitProxy.new(nil, nil).version - config.filter_run_excluding :git => RequirementChecker.against(git_version) - config.filter_run_excluding :bundler => RequirementChecker.against(Bundler::VERSION.split(".")[0]) - config.filter_run_excluding :rubygems => RequirementChecker.against(Gem::VERSION) - config.filter_run_excluding :ruby_repo => !ENV["GEM_COMMAND"].nil? - config.filter_run_excluding :no_color_tty => Gem.win_platform? || !ENV["GITHUB_ACTION"].nil? - config.filter_run_excluding :permissions => Gem.win_platform? - config.filter_run_excluding :readline => Gem.win_platform? - config.filter_run_excluding :jruby_only => RUBY_ENGINE != "jruby" - config.filter_run_excluding :truffleruby_only => RUBY_ENGINE != "truffleruby" - config.filter_run_excluding :man => Gem.win_platform? + config.filter_run_excluding git: RequirementChecker.against(git_version) + config.filter_run_excluding bundler: RequirementChecker.against(Bundler::VERSION.split(".")[0]) + config.filter_run_excluding rubygems: RequirementChecker.against(Gem::VERSION) + config.filter_run_excluding ruby_repo: !ENV["GEM_COMMAND"].nil? + config.filter_run_excluding no_color_tty: Gem.win_platform? || !ENV["GITHUB_ACTION"].nil? + config.filter_run_excluding permissions: Gem.win_platform? + config.filter_run_excluding readline: Gem.win_platform? + config.filter_run_excluding jruby_only: RUBY_ENGINE != "jruby" + config.filter_run_excluding truffleruby_only: RUBY_ENGINE != "truffleruby" + config.filter_run_excluding man: Gem.win_platform? config.filter_run_when_matching :focus unless ENV["CI"] end diff --git a/spec/bundler/support/helpers.rb b/spec/bundler/support/helpers.rb index 7b8c56b6ad..b65686c6f4 100644 --- a/spec/bundler/support/helpers.rb +++ b/spec/bundler/support/helpers.rb @@ -1,12 +1,15 @@ # frozen_string_literal: true -require_relative "command_execution" require_relative "the_bundle" require_relative "path" +require_relative "options" +require_relative "subprocess" module Spec module Helpers include Spec::Path + include Spec::Options + include Spec::Subprocess def reset! Dir.glob("#{tmp}/{gems/*,*}", File::FNM_DOTMATCH).each do |dir| @@ -27,23 +30,7 @@ module Spec TheBundle.new(*args) end - def command_executions - @command_executions ||= [] - end - - def last_command - command_executions.last || raise("There is no last command") - end - - def out - last_command.stdout - end - - def err - last_command.stderr - end - - MAJOR_DEPRECATION = /^\[DEPRECATED\]\s*/.freeze + MAJOR_DEPRECATION = /^\[DEPRECATED\]\s*/ def err_without_deprecations err.gsub(/#{MAJOR_DEPRECATION}.+[\n]?/, "") @@ -53,14 +40,10 @@ module Spec err.split("\n").select {|l| l =~ MAJOR_DEPRECATION }.join("\n").split(MAJOR_DEPRECATION) end - def exitstatus - last_command.exitstatus - end - def run(cmd, *args) opts = args.last.is_a?(Hash) ? args.pop : {} groups = args.map(&:inspect).join(", ") - setup = "require '#{entrypoint}' ; Bundler.ui.silence { Bundler.setup(#{groups}) }" + setup = "require 'bundler' ; Bundler.ui.silence { Bundler.setup(#{groups}) }" ruby([setup, cmd].join(" ; "), opts) end @@ -116,13 +99,13 @@ module Spec end end.join - ruby_cmd = build_ruby_cmd({ :load_path => load_path, :requires => requires }) + ruby_cmd = build_ruby_cmd({ load_path: load_path, requires: requires, env: env }) cmd = "#{ruby_cmd} #{bundle_bin} #{cmd}#{args}" - sys_exec(cmd, { :env => env, :dir => dir, :raise_on_error => raise_on_error }, &block) + sys_exec(cmd, { env: env, dir: dir, raise_on_error: raise_on_error }, &block) end def bundler(cmd, options = {}) - options[:bundle_bin] = system_gem_path.join("bin/bundler") + options[:bundle_bin] = system_gem_path("bin/bundler") bundle(cmd, options) end @@ -147,7 +130,13 @@ module Spec lib_option = libs ? "-I#{libs.join(File::PATH_SEPARATOR)}" : [] requires = options.delete(:requires) || [] - requires << "#{Path.spec_dir}/support/hax.rb" + + hax_path = "#{Path.spec_dir}/support/hax.rb" + + # For specs that need to ignore the default Bundler gem, load hax before + # anything else since other stuff may actually load bundler and not skip + # the default version + options[:env]&.include?("BUNDLER_IGNORE_DEFAULT_GEM") ? requires.prepend(hax_path) : requires.append(hax_path) require_option = requires.map {|r| "-r#{r}" } [Gem.ruby, *lib_option, *require_option].compact.join(" ") @@ -169,63 +158,30 @@ module Spec "#{Gem.ruby} -S #{ENV["GEM_PATH"]}/bin/rake" end - def git(cmd, path, options = {}) - sys_exec("git #{cmd}", options.merge(:dir => path)) - end - - def sys_exec(cmd, options = {}) + def sys_exec(cmd, options = {}, &block) env = options[:env] || {} env["RUBYOPT"] = opt_add(opt_add("-r#{spec_dir}/support/switch_rubygems.rb", env["RUBYOPT"]), ENV["RUBYOPT"]) - dir = options[:dir] || bundled_app - command_execution = CommandExecution.new(cmd.to_s, dir) - - require "open3" - require "shellwords" - Open3.popen3(env, *cmd.shellsplit, :chdir => dir) do |stdin, stdout, stderr, wait_thr| - yield stdin, stdout, wait_thr if block_given? - stdin.close - - stdout_read_thread = Thread.new { stdout.read } - stderr_read_thread = Thread.new { stderr.read } - command_execution.stdout = stdout_read_thread.value.strip - command_execution.stderr = stderr_read_thread.value.strip - - status = wait_thr.value - command_execution.exitstatus = if status.exited? - status.exitstatus - elsif status.signaled? - exit_status_for_signal(status.termsig) - end - end - - unless options[:raise_on_error] == false || command_execution.success? - raise <<~ERROR - - Invoking `#{cmd}` failed with output: - ---------------------------------------------------------------------- - #{command_execution.stdboth} - ---------------------------------------------------------------------- - ERROR - end - - command_executions << command_execution + options[:env] = env + options[:dir] ||= bundled_app - command_execution.stdout + sh(cmd, options, &block) end - def all_commands_output - return "" if command_executions.empty? + def config(config = nil, path = bundled_app(".bundle/config")) + current = File.exist?(path) ? Psych.load_file(path) : {} + return current unless config - "\n\nCommands:\n#{command_executions.map(&:to_s_verbose).join("\n\n")}" - end + current = {} if current.empty? - def config(config = nil, path = bundled_app(".bundle/config")) - return Psych.load_file(path) unless config FileUtils.mkdir_p(File.dirname(path)) - File.open(path, "w") do |f| - f.puts config.to_yaml + + new_config = current.merge(config).compact + + File.open(path, "w+") do |f| + f.puts new_config.to_yaml end - config + + new_config end def global_config(config = nil) @@ -244,7 +200,7 @@ module Spec contents = args.pop if contents.nil? - File.open(bundled_app_gemfile, "r", &:read) + read_gemfile else create_file(args.pop || "Gemfile", contents) end @@ -254,12 +210,24 @@ module Spec contents = args.pop if contents.nil? - File.open(bundled_app_lock, "r", &:read) + read_lockfile else create_file(args.pop || "Gemfile.lock", contents) end end + def read_gemfile(file = "Gemfile") + read_bundled_app_file(file) + end + + def read_lockfile(file = "Gemfile.lock") + read_bundled_app_file(file) + end + + def read_bundled_app_file(file) + bundled_app(file).read + end + def strip_whitespace(str) # Trim the leading spaces spaces = str[/\A\s+/, 0] || "" @@ -281,59 +249,35 @@ module Spec def system_gems(*gems) gems = gems.flatten options = gems.last.is_a?(Hash) ? gems.pop : {} - path = options.fetch(:path, system_gem_path) + install_dir = options.fetch(:path, system_gem_path) default = options.fetch(:default, false) - with_gem_path_as(path) do + with_gem_path_as(install_dir) do gem_repo = options.fetch(:gem_repo, gem_repo1) gems.each do |g| gem_name = g.to_s if gem_name.start_with?("bundler") version = gem_name.match(/\Abundler-(?<version>.*)\z/)[:version] if gem_name != "bundler" - with_built_bundler(version) {|gem_path| install_gem(gem_path, default) } + with_built_bundler(version) {|gem_path| install_gem(gem_path, install_dir, default) } elsif %r{\A(?:[a-zA-Z]:)?/.*\.gem\z}.match?(gem_name) - install_gem(gem_name, default) + install_gem(gem_name, install_dir, default) else - install_gem("#{gem_repo}/gems/#{gem_name}.gem", default) + install_gem("#{gem_repo}/gems/#{gem_name}.gem", install_dir, default) end end end end - def install_gem(path, default = false) + def install_gem(path, install_dir, default = false) raise "OMG `#{path}` does not exist!" unless File.exist?(path) - args = "--no-document --ignore-dependencies --verbose --local" - args += " --default --install-dir #{system_gem_path}" if default + args = "--no-document --ignore-dependencies --verbose --local --install-dir #{install_dir}" + args += " --default" if default gem_command "install #{args} '#{path}'" end - def with_built_bundler(version = nil) - version ||= Bundler::VERSION - full_name = "bundler-#{version}" - build_path = tmp + full_name - bundler_path = build_path + "#{full_name}.gem" - - Dir.mkdir build_path - - begin - shipped_files.each do |shipped_file| - target_shipped_file = build_path + shipped_file - target_shipped_dir = File.dirname(target_shipped_file) - FileUtils.mkdir_p target_shipped_dir unless File.directory?(target_shipped_dir) - FileUtils.cp shipped_file, target_shipped_file, :preserve => true - end - - replace_version_file(version, dir: build_path) # rubocop:disable Style/HashSyntax - - Spec::BuildMetadata.write_build_metadata(dir: build_path) # rubocop:disable Style/HashSyntax - - gem_command "build #{relative_gemspec}", :dir => build_path - - yield(bundler_path) - ensure - build_path.rmtree - end + def with_built_bundler(version = nil, &block) + Builders::BundlerBuilder.new(self, "bundler", version)._build(&block) end def with_gem_path_as(path) @@ -367,16 +311,6 @@ module Spec end end - def opt_add(option, options) - [option.strip, options].compact.reject(&:empty?).join(" ") - end - - def opt_remove(option, options) - return unless options - - options.split(" ").reject {|opt| opt.strip == option.strip }.join(" ") - end - def break_git! FileUtils.mkdir_p(tmp("broken_path")) File.open(tmp("broken_path/git"), "w", 0o755) do |f| @@ -452,32 +386,12 @@ module Spec old = ENV["BUNDLER_SPEC_WINDOWS"] ENV["BUNDLER_SPEC_WINDOWS"] = "true" simulate_platform platform do - simulate_bundler_version_when_missing_prerelease_default_gem_activation do - yield - end + yield end ensure ENV["BUNDLER_SPEC_WINDOWS"] = old end - def simulate_bundler_version_when_missing_prerelease_default_gem_activation - return yield unless rubygems_version_failing_to_activate_bundler_prereleases - - old = ENV["BUNDLER_VERSION"] - ENV["BUNDLER_VERSION"] = Bundler::VERSION - yield - ensure - ENV["BUNDLER_VERSION"] = old - end - - def env_for_missing_prerelease_default_gem_activation - if rubygems_version_failing_to_activate_bundler_prereleases - { "BUNDLER_VERSION" => Bundler::VERSION } - else - {} - end - end - def current_ruby_minor Gem.ruby_version.segments.tap {|s| s.delete_at(2) }.join(".") end @@ -496,14 +410,8 @@ module Spec Gem.ruby_version.segments[0..1] end - # versions not including - # https://github.com/rubygems/rubygems/commit/929e92d752baad3a08f3ac92eaec162cb96aedd1 - def rubygems_version_failing_to_activate_bundler_prereleases - Gem.rubygems_version < Gem::Version.new("3.1.0.pre.1") - end - def revision_for(path) - sys_exec("git rev-parse HEAD", :dir => path).strip + git("rev-parse HEAD", path).strip end def with_read_only(pattern) diff --git a/spec/bundler/support/indexes.rb b/spec/bundler/support/indexes.rb index 78372302f1..086a311551 100644 --- a/spec/bundler/support/indexes.rb +++ b/spec/bundler/support/indexes.rb @@ -14,10 +14,10 @@ module Spec alias_method :platforms, :platform - def resolve(args = []) + def resolve(args = [], dependency_api_available: true) @platforms ||= ["ruby"] - default_source = instance_double("Bundler::Source::Rubygems", :specs => @index, :to_s => "locally install gems") - source_requirements = { :default => default_source } + default_source = instance_double("Bundler::Source::Rubygems", specs: @index, to_s: "locally install gems", dependency_api_available?: dependency_api_available) + source_requirements = { default: default_source } base = args[0] || Bundler::SpecSet.new([]) base.each {|ls| ls.source = default_source } gem_version_promoter = args[1] || Bundler::GemVersionPromoter.new @@ -27,7 +27,7 @@ module Spec name = d.name source_requirements[name] = d.source = default_source end - packages = Bundler::Resolver::Base.new(source_requirements, @deps, base, @platforms, :locked_specs => originally_locked, :unlock => unlock) + packages = Bundler::Resolver::Base.new(source_requirements, @deps, base, @platforms, locked_specs: originally_locked, unlock: unlock) Bundler::Resolver.new(packages, gem_version_promoter).start end @@ -41,6 +41,12 @@ module Spec expect(got).to eq(specs.sort) end + def should_resolve_without_dependency_api(specs) + got = resolve(dependency_api_available: false) + got = got.map(&:full_name).sort + expect(got).to eq(specs.sort) + end + def should_resolve_and_include(specs, args = []) got = resolve(args) got = got.map(&:full_name).sort @@ -298,7 +304,7 @@ module Spec end end - def a_unresovable_child_index + def a_unresolvable_child_index build_index do gem "json", %w[1.8.0] diff --git a/spec/bundler/support/matchers.rb b/spec/bundler/support/matchers.rb index ea7c784683..0f027dcf04 100644 --- a/spec/bundler/support/matchers.rb +++ b/spec/bundler/support/matchers.rb @@ -97,18 +97,6 @@ module Spec end end - RSpec::Matchers.define :take_less_than do |seconds| - match do |actual| - start_time = Time.now - - actual.call - - (Time.now - start_time).to_f < seconds - end - - supports_block_expectations - end - define_compound_matcher :read_as, [exist] do |file_contents| diffable diff --git a/spec/bundler/support/options.rb b/spec/bundler/support/options.rb new file mode 100644 index 0000000000..551fa1acd8 --- /dev/null +++ b/spec/bundler/support/options.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Spec + module Options + def opt_add(option, options) + [option.strip, options].compact.reject(&:empty?).join(" ") + end + + def opt_remove(option, options) + return unless options + + options.split(" ").reject {|opt| opt.strip == option.strip }.join(" ") + end + end +end diff --git a/spec/bundler/support/path.rb b/spec/bundler/support/path.rb index 8d1807b56c..26be5488c3 100644 --- a/spec/bundler/support/path.rb +++ b/spec/bundler/support/path.rb @@ -3,8 +3,12 @@ require "pathname" require "rbconfig" +require_relative "env" + module Spec module Path + include Spec::Env + def source_root @source_root ||= Pathname.new(ruby_core? ? "../../.." : "../..").expand_path(__dir__) end @@ -42,8 +46,7 @@ module Spec end def dev_gemfile - name = RUBY_VERSION.start_with?("2.6") ? "dev26_gems.rb" : "dev_gems.rb" - @dev_gemfile ||= tool_dir.join(name) + @dev_gemfile ||= tool_dir.join("dev_gems.rb") end def bindir @@ -59,7 +62,7 @@ module Spec end def gem_bin - @gem_bin ||= ruby_core? ? ENV["GEM_COMMAND"] : "gem" + @gem_bin ||= ENV["GEM_COMMAND"] || "gem" end def path @@ -81,7 +84,13 @@ module Spec end def shipped_files - @shipped_files ||= loaded_gemspec.files + @shipped_files ||= if ruby_core_tarball? + loaded_gemspec.files.map {|f| f.gsub(%r{^exe/}, "libexec/") } + elsif ruby_core? + tracked_files + else + loaded_gemspec.files + end end def lib_tracked_files @@ -104,7 +113,7 @@ module Spec end def home(*path) - tmp.join("home", *path) + tmp("home", *path) end def default_bundle_path(*path) @@ -124,13 +133,13 @@ module Spec end def bundled_app(*path) - root = tmp.join("bundled_app") + root = tmp("bundled_app") FileUtils.mkdir_p(root) root.join(*path) end def bundled_app2(*path) - root = tmp.join("bundled_app2") + root = tmp("bundled_app2") FileUtils.mkdir_p(root) root.join(*path) end @@ -156,15 +165,15 @@ module Spec end def base_system_gems - tmp.join("gems/base") + tmp("gems/base") end def rubocop_gems - tmp.join("gems/rubocop") + tmp("gems/rubocop") end def standard_gems - tmp.join("gems/standard") + tmp("gems/standard") end def file_uri_for(path) @@ -226,13 +235,6 @@ module Spec root.join("lib") end - # Sometimes rubygems version under test does not include - # https://github.com/rubygems/rubygems/pull/2728 and will not always end up - # activating the current bundler. In that case, require bundler absolutely. - def entrypoint - Gem.rubygems_version < Gem::Version.new("3.1.a") ? "#{lib_dir}/bundler" : "bundler" - end - def global_plugin_gem(*args) home ".bundle", "plugin", "gems", *args end @@ -252,15 +254,11 @@ module Spec File.open(version_file, "w") {|f| f << contents } end - def ruby_core? - # avoid to warnings - @ruby_core ||= nil - - if @ruby_core.nil? - @ruby_core = true & ENV["GEM_COMMAND"] - else - @ruby_core - end + def replace_required_ruby_version(version, dir:) + gemspec_file = File.expand_path("bundler.gemspec", dir) + contents = File.read(gemspec_file) + contents.sub!(/(^\s+s\.required_ruby_version\s*=\s*)"[^"]+"/, %(\\1"#{version}")) + File.open(gemspec_file, "w") {|f| f << contents } end def git_root @@ -272,11 +270,11 @@ module Spec def git_ls_files(glob) skip "Not running on a git context, since running tests from a tarball" if ruby_core_tarball? - sys_exec("git ls-files -z -- #{glob}", :dir => source_root).split("\x0") + git("ls-files -z -- #{glob}", source_root).split("\x0") end def tracked_files_glob - ruby_core? ? "lib/bundler lib/bundler.rb spec/bundler man/bundle*" : "" + ruby_core? ? "libexec/bundle* lib/bundler lib/bundler.rb spec/bundler man/bundle*" : "lib exe spec CHANGELOG.md LICENSE.md README.md bundler.gemspec" end def lib_tracked_files_glob diff --git a/spec/bundler/support/platforms.rb b/spec/bundler/support/platforms.rb index eca1b2e60d..526e1c09a9 100644 --- a/spec/bundler/support/platforms.rb +++ b/spec/bundler/support/platforms.rb @@ -95,12 +95,17 @@ module Spec 9999 end - def lockfile_platforms(*extra) - formatted_lockfile_platforms(local_platform, *extra) + def default_platform_list(*extra, defaults: default_locked_platforms) + defaults.concat(extra).uniq end - def formatted_lockfile_platforms(*platforms) + def lockfile_platforms(*extra, defaults: default_locked_platforms) + platforms = default_platform_list(*extra, defaults: defaults) platforms.map(&:to_s).sort.join("\n ") end + + def default_locked_platforms + [local_platform, generic_local_platform] + end end end diff --git a/spec/bundler/support/rubygems_ext.rb b/spec/bundler/support/rubygems_ext.rb index 4553c0606e..7748234abc 100644 --- a/spec/bundler/support/rubygems_ext.rb +++ b/spec/bundler/support/rubygems_ext.rb @@ -24,9 +24,9 @@ module Spec gem_load_activate_and_possibly_install(gem_name, bin_container) end - def gem_require(gem_name) + def gem_require(gem_name, entrypoint) gem_activate(gem_name) - require gem_name + require entrypoint end def test_setup @@ -38,6 +38,10 @@ module Spec FileUtils.mkdir_p(Path.tmpdir) ENV["HOME"] = Path.home.to_s + # Remove "RUBY_CODESIGN", which is used by mkmf-generated Makefile to + # sign extension bundles on macOS, to avoid trying to find the specified key + # from the fake $HOME/Library/Keychains directory. + ENV.delete "RUBY_CODESIGN" ENV["TMPDIR"] = Path.tmpdir.to_s require "rubygems/user_interaction" @@ -66,7 +70,7 @@ module Spec ENV["BUNDLE_PATH"] = nil ENV["GEM_HOME"] = ENV["GEM_PATH"] = Path.base_system_gem_path.to_s - ENV["PATH"] = [Path.system_gem_path.join("bin"), ENV["PATH"]].join(File::PATH_SEPARATOR) + ENV["PATH"] = [Path.system_gem_path("bin"), ENV["PATH"]].join(File::PATH_SEPARATOR) ENV["PATH"] = [Path.bindir, ENV["PATH"]].join(File::PATH_SEPARATOR) if Path.ruby_core? end @@ -86,7 +90,7 @@ module Spec puts success_message puts else - system("git status --porcelain") + system("git diff") puts puts error_message @@ -113,11 +117,19 @@ module Spec def gem_activate_and_possibly_install(gem_name) gem_activate(gem_name) rescue Gem::LoadError => e - Gem.install(gem_name, e.requirement) + # Windows 3.0 puts a Windows stub script as `rake` while it should be + # named `rake.bat`. RubyGems does not like that and avoids overwriting it + # unless explicitly instructed to do so with `force`. + if RUBY_VERSION.start_with?("3.0") && Gem.win_platform? + Gem.install(gem_name, e.requirement, force: true) + else + Gem.install(gem_name, e.requirement) + end retry end def gem_activate(gem_name) + require_relative "activate" require "bundler" gem_requirement = Bundler::LockfileParser.new(File.read(dev_lockfile)).specs.find {|spec| spec.name == gem_name }.version gem gem_name, gem_requirement diff --git a/spec/bundler/support/rubygems_version_manager.rb b/spec/bundler/support/rubygems_version_manager.rb index 5653601ae8..c174c461f0 100644 --- a/spec/bundler/support/rubygems_version_manager.rb +++ b/spec/bundler/support/rubygems_version_manager.rb @@ -1,12 +1,13 @@ # frozen_string_literal: true -require "pathname" -require_relative "helpers" -require_relative "path" +require_relative "options" +require_relative "env" +require_relative "subprocess" class RubygemsVersionManager - include Spec::Helpers - include Spec::Path + include Spec::Options + include Spec::Env + include Spec::Subprocess def initialize(source) @source = source @@ -30,11 +31,10 @@ class RubygemsVersionManager rubygems_default_path = rubygems_path + "/defaults" bundler_path = rubylibdir + "/bundler" - bundler_exemptions = Gem.rubygems_version < Gem::Version.new("3.2.0") ? [bundler_path + "/errors.rb"] : [] bad_loaded_features = $LOADED_FEATURES.select do |loaded_feature| (loaded_feature.start_with?(rubygems_path) && !loaded_feature.start_with?(rubygems_default_path)) || - (loaded_feature.start_with?(bundler_path) && !bundler_exemptions.any? {|bundler_exemption| loaded_feature.start_with?(bundler_exemption) }) + loaded_feature.start_with?(bundler_path) end errors = if bad_loaded_features.any? @@ -58,7 +58,7 @@ class RubygemsVersionManager cmd = [RbConfig.ruby, $0, *ARGV].compact - ENV["RUBYOPT"] = opt_add("-I#{local_copy_path.join("lib")}", opt_remove("--disable-gems", ENV["RUBYOPT"])) + ENV["RUBYOPT"] = opt_add("-I#{File.join(local_copy_path, "lib")}", opt_remove("--disable-gems", ENV["RUBYOPT"])) exec(ENV, *cmd) end @@ -66,14 +66,14 @@ class RubygemsVersionManager def switch_local_copy_if_needed return unless local_copy_switch_needed? - sys_exec("git checkout #{target_tag}", :dir => local_copy_path) + git("checkout #{target_tag}", local_copy_path) - ENV["RGV"] = local_copy_path.to_s + ENV["RGV"] = local_copy_path end def rubygems_unrequire_needed? require "rubygems" - !$LOADED_FEATURES.include?(local_copy_path.join("lib/rubygems.rb").to_s) + !$LOADED_FEATURES.include?(File.join(local_copy_path, "lib/rubygems.rb")) end def local_copy_switch_needed? @@ -85,7 +85,7 @@ class RubygemsVersionManager end def local_copy_tag - sys_exec("git rev-parse --abbrev-ref HEAD", :dir => local_copy_path) + git("rev-parse --abbrev-ref HEAD", local_copy_path) end def local_copy_path @@ -95,21 +95,25 @@ class RubygemsVersionManager def resolve_local_copy_path return expanded_source if source_is_path? - rubygems_path = source_root.join("tmp/rubygems") + rubygems_path = File.join(source_root, "tmp/rubygems") - unless rubygems_path.directory? - sys_exec("git clone .. #{rubygems_path}", :dir => source_root) + unless File.directory?(rubygems_path) + git("clone .. #{rubygems_path}", source_root) end rubygems_path end def source_is_path? - expanded_source.directory? + File.directory?(expanded_source) end def expanded_source - @expanded_source ||= Pathname.new(@source).expand_path(source_root) + @expanded_source ||= File.expand_path(@source, source_root) + end + + def source_root + @source_root ||= File.expand_path(ruby_core? ? "../../.." : "../..", __dir__) end def resolve_target_tag diff --git a/spec/bundler/support/subprocess.rb b/spec/bundler/support/subprocess.rb new file mode 100644 index 0000000000..ade18e7805 --- /dev/null +++ b/spec/bundler/support/subprocess.rb @@ -0,0 +1,108 @@ +# frozen_string_literal: true + +require_relative "command_execution" + +module Spec + module Subprocess + class TimeoutExceeded < StandardError; end + + def command_executions + @command_executions ||= [] + end + + def last_command + command_executions.last || raise("There is no last command") + end + + def out + last_command.stdout + end + + def err + last_command.stderr + end + + def exitstatus + last_command.exitstatus + end + + def git(cmd, path = Dir.pwd, options = {}) + sh("git #{cmd}", options.merge(dir: path)) + end + + def sh(cmd, options = {}) + dir = options[:dir] + env = options[:env] || {} + + command_execution = CommandExecution.new(cmd.to_s, working_directory: dir, timeout: 60) + + require "open3" + require "shellwords" + Open3.popen3(env, *cmd.shellsplit, chdir: dir) do |stdin, stdout, stderr, wait_thr| + yield stdin, stdout, wait_thr if block_given? + stdin.close + + stdout_handler = ->(data) { command_execution.original_stdout << data } + stderr_handler = ->(data) { command_execution.original_stderr << data } + + stdout_thread = read_stream(stdout, stdout_handler, timeout: command_execution.timeout) + stderr_thread = read_stream(stderr, stderr_handler, timeout: command_execution.timeout) + + stdout_thread.join + stderr_thread.join + + status = wait_thr.value + command_execution.exitstatus = if status.exited? + status.exitstatus + elsif status.signaled? + exit_status_for_signal(status.termsig) + end + rescue TimeoutExceeded + command_execution.failure_reason = :timeout + command_execution.exitstatus = exit_status_for_signal(Signal.list["INT"]) + end + + unless options[:raise_on_error] == false || command_execution.success? + command_execution.raise_error! + end + + command_executions << command_execution + + command_execution.stdout + end + + # Mostly copied from https://github.com/piotrmurach/tty-command/blob/49c37a895ccea107e8b78d20e4cb29de6a1a53c8/lib/tty/command/process_runner.rb#L165-L193 + def read_stream(stream, handler, timeout:) + Thread.new do + Thread.current.report_on_exception = false + cmd_start = Time.now + readers = [stream] + + while readers.any? + ready = IO.select(readers, nil, readers, timeout) + raise TimeoutExceeded if ready.nil? + + ready[0].each do |reader| + chunk = reader.readpartial(16 * 1024) + handler.call(chunk) + + # control total time spent reading + runtime = Time.now - cmd_start + time_left = timeout - runtime + raise TimeoutExceeded if time_left < 0.0 + rescue Errno::EAGAIN, Errno::EINTR + rescue EOFError, Errno::EPIPE, Errno::EIO + readers.delete(reader) + reader.close + end + end + end + end + + def all_commands_output + return "" if command_executions.empty? + + "\n\nCommands:\n#{command_executions.map(&:to_s_verbose).join("\n\n")}" + end + end +end |