diff options
Diffstat (limited to 'lib/bundler/source/git/git_proxy.rb')
| -rw-r--r-- | lib/bundler/source/git/git_proxy.rb | 478 |
1 files changed, 361 insertions, 117 deletions
diff --git a/lib/bundler/source/git/git_proxy.rb b/lib/bundler/source/git/git_proxy.rb index 7612eb16c6..8094dcaa9d 100644 --- a/lib/bundler/source/git/git_proxy.rb +++ b/lib/bundler/source/git/git_proxy.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require "shellwords" - module Bundler class Source class Git @@ -17,8 +15,8 @@ module Bundler class GitNotAllowedError < GitError def initialize(command) msg = String.new - msg << "Bundler is trying to run a `git #{command}` at runtime. You probably need to run `bundle install`. However, " - msg << "this error message could probably be more useful. Please submit a ticket at https://github.com/bundler/bundler/issues " + msg << "Bundler is trying to run `#{command}` at runtime. You probably need to run `bundle install`. However, " + msg << "this error message could probably be more useful. Please submit a ticket at https://github.com/ruby/rubygems/issues/new?labels=Bundler&template=bundler-related-issue.md " msg << "with steps to reproduce as well as the following\n\nCALLER: #{caller.join("\n")}" super msg end @@ -27,21 +25,28 @@ module Bundler class GitCommandError < GitError attr_reader :command - def initialize(command, path = nil, extra_info = nil) + def initialize(command, path, extra_info = nil) @command = command - msg = String.new - msg << "Git error: command `git #{command}` in directory #{SharedHelpers.pwd} has failed." + msg = String.new("Git error: command `#{command}`") + msg << " in directory #{path}" if path + msg << " has failed." msg << "\n#{extra_info}" if extra_info - msg << "\nIf this error persists you could try removing the cache directory '#{path}'" if path && path.exist? super msg end end class MissingGitRevisionError < GitCommandError - def initialize(command, path, ref, repo) + def initialize(command, destination_path, ref, repo) msg = "Revision #{ref} does not exist in the repository #{repo}. Maybe you misspelled it?" - super command, path, msg + super command, destination_path, msg + end + end + + class AmbiguousGitReference < GitError + def initialize(options) + msg = "Specification of branch or ref with tag is ambiguous. You specified #{options.inspect}" + super msg end end @@ -49,72 +54,87 @@ module Bundler # All actions required by the Git source is encapsulated in this # object. class GitProxy - attr_accessor :path, :uri, :ref + attr_accessor :path, :uri, :branch, :tag, :ref, :explicit_ref attr_writer :revision - def initialize(path, uri, ref, revision = nil, git = nil) + def self.version + @version ||= full_version[/((\.?\d+)+).*/, 1] + end + + def self.full_version + @full_version ||= begin + raise GitNotInstalledError.new unless Bundler.git_present? + + require "open3" + out, err, status = Open3.capture3("git", "--version") + + raise GitCommandError.new("--version", SharedHelpers.pwd, err) unless status.success? + Bundler.ui.warn err unless err.empty? + + out.sub(/git version\s*/, "").strip + end + end + + def self.reset + @version = nil + @full_version = nil + end + + def initialize(path, uri, options = {}, revision = nil, git = nil) @path = path @uri = uri - @ref = ref + @tag = options["tag"] + @branch = options["branch"] + @ref = options["ref"] + if @tag + raise AmbiguousGitReference.new(options) if @branch || @ref + @explicit_ref = @tag + else + @explicit_ref = @ref || @branch + end @revision = revision @git = git - raise GitNotInstalledError.new if allow? && !Bundler.git_present? + @commit_ref = nil end def revision - return @revision if @revision - - begin - @revision ||= find_local_revision - rescue GitCommandError => e - raise MissingGitRevisionError.new(e.command, path, ref, URICredentialsFilter.credential_filtered_uri(uri)) - end - - @revision + @revision ||= allowed_with_path { find_local_revision } end - def branch - @branch ||= allowed_in_path do - git("rev-parse --abbrev-ref HEAD").strip + def current_branch + @current_branch ||= with_path do + git_local("rev-parse", "--abbrev-ref", "HEAD", dir: path).strip end end def contains?(commit) - allowed_in_path do - result, status = git_null("branch --contains #{commit}") - status.success? && result =~ /^\* (.*)$/ + allowed_with_path do + result, status = git_null("branch", "--contains", commit, dir: path) + status.success? && result.match?(/^\* (.*)$/) end end def version - git("--version").match(/(git version\s*)?((\.?\d+)+).*/)[2] + self.class.version end def full_version - git("--version").sub("git version", "").strip + self.class.full_version end def checkout - return if path.exist? && has_revision_cached? - extra_ref = "#{Shellwords.shellescape(ref)}:#{Shellwords.shellescape(ref)}" if ref && ref.start_with?("refs/") + return if has_revision_cached? - Bundler.ui.info "Fetching #{URICredentialsFilter.credential_filtered_uri(uri)}" + Bundler.ui.info "Fetching #{credential_filtered_uri}" - unless path.exist? - SharedHelpers.filesystem_access(path.dirname) do |p| - FileUtils.mkdir_p(p) - end - git_retry %(clone #{uri_escaped_with_configured_credentials} "#{path}" --bare --no-hardlinks --quiet) - return unless extra_ref - end + extra_fetch_needed = clone_needs_extra_fetch? + unshallow_needed = clone_needs_unshallow? + return unless extra_fetch_needed || unshallow_needed - in_path do - git_retry %(fetch --force --quiet --tags #{uri_escaped_with_configured_credentials} "refs/heads/*:refs/heads/*" #{extra_ref}) - end + git_remote_fetch(unshallow_needed ? ["--unshallow"] : depth_args) end def copy_to(destination, submodules = false) - # method 1 unless File.exist?(destination.join(".git")) begin SharedHelpers.filesystem_access(destination.dirname) do |p| @@ -123,135 +143,359 @@ module Bundler SharedHelpers.filesystem_access(destination) do |p| FileUtils.rm_rf(p) end - git_retry %(clone --no-checkout --quiet "#{path}" "#{destination}") - File.chmod(((File.stat(destination).mode | 0o777) & ~File.umask), destination) + git "clone", "--no-checkout", "--quiet", path.to_s, destination.to_s + File.chmod((File.stat(destination).mode | 0o777) & ~File.umask, destination) rescue Errno::EEXIST => e - file_path = e.message[%r{.*?(/.*)}, 1] + file_path = e.message[%r{.*?((?:[a-zA-Z]:)?/.*)}, 1] raise GitError, "Bundler could not install a gem because it needs to " \ "create a directory, but a file exists - #{file_path}. Please delete " \ "this file and try again." end end - # method 2 - SharedHelpers.chdir(destination) do - git_retry %(fetch --force --quiet --tags "#{path}") - begin - git "reset --hard #{@revision}" - rescue GitCommandError => e - raise MissingGitRevisionError.new(e.command, path, @revision, URICredentialsFilter.credential_filtered_uri(uri)) + ref = @commit_ref || (locked_to_full_sha? && @revision) + if ref + git "config", "uploadpack.allowAnySHA1InWant", "true", dir: path.to_s if @commit_ref.nil? && needs_allow_any_sha1_in_want? + + git "fetch", "--force", "--quiet", *extra_fetch_args(ref), dir: destination + end + + git "reset", "--hard", revision, dir: destination + + if submodules + git_retry "submodule", "update", "--init", "--recursive", dir: destination + elsif Gem::Version.create(version) >= Gem::Version.create("2.9.0") + inner_command = "git -C $toplevel submodule deinit --force $sm_path" + git_retry "submodule", "foreach", "--quiet", inner_command, dir: destination + end + end + + def installed_to?(destination) + # if copy_to is interrupted, it may leave a partially installed directory that + # contains .git but no other files -- consider this not to be installed + Dir.exist?(destination) && (Dir.children(destination) - [".git"]).any? + end + + private + + def git_remote_fetch(args) + command = fetch_command(args) + command_with_no_credentials = check_allowed(command) + + Bundler::Retry.new("`#{command_with_no_credentials}` at #{path}", [MissingGitRevisionError]).attempts do + out, err, status = capture(command, path) + return out if status.success? + + if err.include?("couldn't find remote ref") || err.include?("not our ref") + raise MissingGitRevisionError.new(command_with_no_credentials, path, commit || explicit_ref, credential_filtered_uri) + else + if shallow? + args -= depth_args + command = fetch_command(args) + command_with_no_credentials = check_allowed(command) + end + raise GitCommandError.new(command_with_no_credentials, path, err) end + end + end + + def clone_needs_extra_fetch? + return true if path.exist? - if submodules - git_retry "submodule update --init --recursive" - elsif Gem::Version.create(version) >= Gem::Version.create("2.9.0") - git_retry "submodule deinit --all --force" + SharedHelpers.filesystem_access(path.dirname) do |p| + FileUtils.mkdir_p(p) + end + + clone_args = extra_clone_args + command = clone_command(clone_args) + command_with_no_credentials = check_allowed(command) + + Bundler::Retry.new("`#{command_with_no_credentials}`", [MissingGitRevisionError]).attempts do + _, err, status = capture(command, nil) + return extra_ref if status.success? + + if err.include?("Could not find remote branch") || # git up to 2.49 + err.include?("Remote branch #{branch_option} not found") # git 2.49 or higher + raise MissingGitRevisionError.new(command_with_no_credentials, nil, explicit_ref, credential_filtered_uri) + else + if shallow? + clone_args -= depth_args + command = clone_command(clone_args) + command_with_no_credentials = check_allowed(command) + end + raise GitCommandError.new(command_with_no_credentials, path, err) end end end - private + def clone_needs_unshallow? + return false unless path.join("shallow").exist? + return true unless shallow? - def git_null(command) - command_with_no_credentials = URICredentialsFilter.credential_filtered_string(command, uri) - raise GitNotAllowedError.new(command_with_no_credentials) unless allow? + @revision && @revision != head_revision + end + + def extra_ref + return false if not_pinned? + return true if shallow? + + ref.start_with?("refs/") + end - out, status = SharedHelpers.with_clean_git_env do - capture_and_ignore_stderr("git #{command}") + def depth + return @depth if defined?(@depth) + + @depth = if !supports_fetching_unreachable_refs? + nil + elsif not_pinned? || pinned_to_full_sha? + 1 + elsif ref.include?("~") + parsed_depth = ref.split("~").last + parsed_depth.to_i + 1 + end + end + + def refspec + if commit + @commit_ref = "refs/#{commit}-sha" + return "#{commit}:#{@commit_ref}" + end + + reference = fully_qualified_ref + + reference ||= if ref.include?("~") + ref.split("~").first + elsif ref.start_with?("refs/") + ref + else + "refs/*" end - [URICredentialsFilter.credential_filtered_string(out, uri), status] + "#{reference}:#{reference}" end - def git_retry(command) - Bundler::Retry.new("`git #{URICredentialsFilter.credential_filtered_string(command, uri)}`", GitNotAllowedError).attempts do - git(command) + def commit + @commit ||= pinned_to_full_sha? ? ref : @revision + end + + def fully_qualified_ref + if branch + "refs/heads/#{branch}" + elsif tag + "refs/tags/#{tag}" + elsif ref.nil? + "refs/heads/#{current_branch}" end end - def git(command, check_errors = true, error_msg = nil) - command_with_no_credentials = URICredentialsFilter.credential_filtered_string(command, uri) - raise GitNotAllowedError.new(command_with_no_credentials) unless allow? + def not_pinned? + branch_option || ref.nil? + end + + def pinned_to_full_sha? + full_sha_revision?(ref) + end + + def locked_to_full_sha? + full_sha_revision?(@revision) + end - out, status = SharedHelpers.with_clean_git_env do - capture_and_filter_stderr(uri, "git #{command}") + def full_sha_revision?(ref) + ref&.match?(/\A\h{40}\z/) + end + + def git_null(*command, dir: nil) + check_allowed(command) + + capture(command, dir, ignore_err: true) + end + + def git_retry(*command, dir: nil) + command_with_no_credentials = check_allowed(command) + + Bundler::Retry.new("`#{command_with_no_credentials}` at #{dir || SharedHelpers.pwd}").attempts do + git(*command, dir: dir) + end + end + + def git(*command, dir: nil) + run_command(*command, dir: dir) do |unredacted_command| + check_allowed(unredacted_command) end + end - stdout_with_no_credentials = URICredentialsFilter.credential_filtered_string(out, uri) - raise GitCommandError.new(command_with_no_credentials, path, error_msg) if check_errors && !status.success? - stdout_with_no_credentials + def git_local(*command, dir: nil) + run_command(*command, dir: dir) do |unredacted_command| + redact_and_check_presence(unredacted_command) + end end def has_revision_cached? - return unless @revision - in_path { git("cat-file -e #{@revision}") } + return unless commit && path.exist? + git("cat-file", "-e", commit, dir: path) true rescue GitError false end - def remove_cache - FileUtils.rm_rf(path) + def find_local_revision + return head_revision if explicit_ref.nil? + + find_revision_for(explicit_ref) end - def find_local_revision - allowed_in_path do - git("rev-parse --verify #{Shellwords.shellescape(ref)}", true).strip - end + def head_revision + verify("HEAD") end - # Escape the URI for git commands - def uri_escaped_with_configured_credentials - remote = configured_uri_for(uri) - if Bundler::WINDOWS - # Windows quoting requires double quotes only, with double quotes - # inside the string escaped by being doubled. - '"' + remote.gsub('"') { '""' } + '"' - else - # Bash requires single quoted strings, with the single quotes escaped - # by ending the string, escaping the quote, and restarting the string. - "'" + remote.gsub("'") { "'\\''" } + "'" - end + def find_revision_for(reference) + verify(reference) + rescue GitCommandError => e + raise MissingGitRevisionError.new(e.command, path, reference, credential_filtered_uri) + end + + def verify(reference) + git("rev-parse", "--verify", reference, dir: path).strip end - # Adds credentials to the URI as Fetcher#configured_uri_for does - def configured_uri_for(uri) - if /https?:/ =~ uri - remote = Bundler::URI(uri) + # Adds credentials to the URI + def configured_uri + if /https?:/.match?(uri) + remote = Gem::URI(uri) config_auth = Bundler.settings[remote.to_s] || Bundler.settings[remote.host] remote.userinfo ||= config_auth remote.to_s else - uri + uri.to_s end end + # Removes credentials from the URI + def credential_filtered_uri + URICredentialsFilter.credential_filtered_uri(uri) + end + def allow? - @git ? @git.allow_git_ops? : true + allowed = @git ? @git.allow_git_ops? : true + + raise GitNotInstalledError.new if allowed && !Bundler.git_present? + + allowed end - def in_path(&blk) + def with_path(&blk) checkout unless path.exist? - _ = URICredentialsFilter # load it before we chdir - SharedHelpers.chdir(path, &blk) + blk.call end - def allowed_in_path - return in_path { yield } if allow? + def allowed_with_path + return with_path { yield } if allow? raise GitError, "The git source #{uri} is not yet checked out. Please run `bundle install` before trying to start your application" end - def capture_and_filter_stderr(uri, cmd) - require "open3" - return_value, captured_err, status = Open3.capture3(cmd) - Bundler.ui.warn URICredentialsFilter.credential_filtered_string(captured_err, uri) if uri && !captured_err.empty? - [return_value, status] + def check_allowed(command) + command_with_no_credentials = redact_and_check_presence(command) + raise GitNotAllowedError.new(command_with_no_credentials) unless allow? + command_with_no_credentials + end + + def redact_and_check_presence(command) + raise GitNotInstalledError.new unless Bundler.git_present? + + require "shellwords" + URICredentialsFilter.credential_filtered_string("git #{command.shelljoin}", uri) + end + + def run_command(*command, dir: nil) + command_with_no_credentials = yield(command) + + out, err, status = capture(command, dir) + + raise GitCommandError.new(command_with_no_credentials, dir || SharedHelpers.pwd, err) unless status.success? + + Bundler.ui.warn err unless err.empty? + + out + end + + def capture(cmd, dir, ignore_err: false) + SharedHelpers.with_clean_git_env do + require "open3" + out, err, status = Open3.capture3(*capture3_args_for(cmd, dir)) + + filtered_out = URICredentialsFilter.credential_filtered_string(out, uri) + return [filtered_out, status] if ignore_err + + filtered_err = URICredentialsFilter.credential_filtered_string(err, uri) + [filtered_out, filtered_err, status] + end + end + + def capture3_args_for(cmd, dir) + # Disable automatic maintenance so a background commit-graph write in + # the source repo can't race the hardlinking local clone and fail with + # "hardlink different from source". + opts = ["-c", "gc.auto=0", "-c", "maintenance.auto=false"] + + return ["git", *opts, *cmd] unless dir + + ["git", "-C", dir.to_s, *opts, *cmd] + end + + def extra_clone_args + args = depth_args + return [] if args.empty? + + args += ["--single-branch"] + args.unshift("--no-tags") if supports_cloning_with_no_tags? + + # If there's a locked revision, no need to clone any specific branch + # or tag, since we will end up checking out that locked revision + # anyways. + return args if @revision + + args += ["--branch", branch_option] if branch_option + args + end + + def fetch_command(args) + ["fetch", "--force", "--quiet", "--no-tags", *args, "--", configured_uri, refspec].compact + end + + def clone_command(args) + ["clone", "--bare", "--no-hardlinks", "--quiet", *args, "--", configured_uri, path.to_s] + end + + def depth_args + return [] unless shallow? + + ["--depth", depth.to_s] + end + + def extra_fetch_args(ref) + extra_args = [path.to_s, *depth_args] + extra_args.push(ref) + extra_args + end + + def branch_option + branch || tag + end + + def shallow? + !depth.nil? + end + + def needs_allow_any_sha1_in_want? + @needs_allow_any_sha1_in_want ||= Gem::Version.new(version) <= Gem::Version.new("2.13.7") + end + + def supports_fetching_unreachable_refs? + @supports_fetching_unreachable_refs ||= Gem::Version.new(version) >= Gem::Version.new("2.5.0") end - def capture_and_ignore_stderr(cmd) - require "open3" - return_value, _, status = Open3.capture3(cmd) - [return_value, status] + def supports_cloning_with_no_tags? + @supports_cloning_with_no_tags ||= Gem::Version.new(version) >= Gem::Version.new("2.14.0-rc0") end end end |