diff options
author | hsbt <hsbt@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2017-09-08 08:45:41 +0000 |
---|---|---|
committer | hsbt <hsbt@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2017-09-08 08:45:41 +0000 |
commit | 8598f8c2dc78c6d1ae87cb6ae19c34ba2cb29241 (patch) | |
tree | 0bbd28f684e745cb212761b7c74fe08668f85cc8 /lib/bundler/source | |
parent | f2e04b77aa8a363d7e36ce5a9cdb60714a537a3c (diff) |
Merge bundler to standard libraries.
rubygems 2.7.x depends bundler-1.15.x. This is preparation for
rubygems and bundler migration.
* lib/bundler.rb, lib/bundler/*: files of bundler-1.15.4
* spec/bundler/*: rspec examples of bundler-1.15.4. I applied patches.
* https://github.com/bundler/bundler/pull/6007
* Exclude not working examples on ruby repository.
* Fake ruby interpriter instead of installed ruby.
* Makefile.in: Added test task named `test-bundler`. This task is only
working macOS/linux yet. I'm going to support Windows environment later.
* tool/sync_default_gems.rb: Added sync task for bundler.
[Feature #12733][ruby-core:77172]
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59779 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'lib/bundler/source')
-rw-r--r-- | lib/bundler/source/gemspec.rb | 17 | ||||
-rw-r--r-- | lib/bundler/source/git.rb | 324 | ||||
-rw-r--r-- | lib/bundler/source/git/git_proxy.rb | 252 | ||||
-rw-r--r-- | lib/bundler/source/path.rb | 249 | ||||
-rw-r--r-- | lib/bundler/source/path/installer.rb | 72 | ||||
-rw-r--r-- | lib/bundler/source/rubygems.rb | 462 | ||||
-rw-r--r-- | lib/bundler/source/rubygems/remote.rb | 63 |
7 files changed, 1439 insertions, 0 deletions
diff --git a/lib/bundler/source/gemspec.rb b/lib/bundler/source/gemspec.rb new file mode 100644 index 0000000000..05e613277f --- /dev/null +++ b/lib/bundler/source/gemspec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true +module Bundler + class Source + class Gemspec < Path + attr_reader :gemspec + + def initialize(options) + super + @gemspec = options["gemspec"] + end + + def as_path_source + Path.new(options) + end + end + end +end diff --git a/lib/bundler/source/git.rb b/lib/bundler/source/git.rb new file mode 100644 index 0000000000..b3e218e390 --- /dev/null +++ b/lib/bundler/source/git.rb @@ -0,0 +1,324 @@ +# frozen_string_literal: true +require "fileutils" +require "uri" +require "digest/sha1" + +module Bundler + class Source + class Git < Path + autoload :GitProxy, "bundler/source/git/git_proxy" + + attr_reader :uri, :ref, :branch, :options, :submodules + + def initialize(options) + @options = options + @glob = options["glob"] || DEFAULT_GLOB + + @allow_cached = false + @allow_remote = false + + # Stringify options that could be set as symbols + %w(ref branch tag revision).each {|k| options[k] = options[k].to_s if options[k] } + + @uri = options["uri"] || "" + @branch = options["branch"] + @ref = options["ref"] || options["branch"] || options["tag"] || "master" + @submodules = options["submodules"] + @name = options["name"] + @version = options["version"].to_s.strip.gsub("-", ".pre.") + + @copied = false + @local = false + end + + def self.from_lock(options) + new(options.merge("uri" => options.delete("remote"))) + end + + def to_lock + out = String.new("GIT\n") + out << " remote: #{@uri}\n" + out << " revision: #{revision}\n" + %w(ref branch tag submodules).each do |opt| + out << " #{opt}: #{options[opt]}\n" if options[opt] + end + out << " glob: #{@glob}\n" unless @glob == DEFAULT_GLOB + out << " specs:\n" + end + + def hash + [self.class, uri, ref, branch, name, version, submodules].hash + end + + def eql?(other) + other.is_a?(Git) && uri == other.uri && ref == other.ref && + branch == other.branch && name == other.name && + version == other.version && submodules == other.submodules + end + + alias_method :==, :eql? + + def to_s + at = if local? + path + elsif user_ref = options["ref"] + if ref =~ /\A[a-z0-9]{4,}\z/i + shortref_for_display(user_ref) + else + user_ref + end + else + ref + end + + rev = begin + "@#{shortref_for_display(revision)}" + rescue GitError + nil + end + + "#{uri} (at #{at}#{rev})" + end + + def name + File.basename(@uri, ".git") + end + + # This is the path which is going to contain a specific + # checkout of the git repository. When using local git + # repos, this is set to the local repo. + def install_path + @install_path ||= begin + git_scope = "#{base_name}-#{shortref_for_path(revision)}" + + path = Bundler.install_path.join(git_scope) + + if !path.exist? && Bundler.requires_sudo? + Bundler.user_bundle_path.join(Bundler.ruby_scope).join(git_scope) + else + path + end + end + end + + alias_method :path, :install_path + + def extension_dir_name + "#{base_name}-#{shortref_for_path(revision)}" + end + + def unlock! + git_proxy.revision = nil + options["revision"] = nil + + @unlocked = true + end + + def local_override!(path) + return false if local? + + path = Pathname.new(path) + path = path.expand_path(Bundler.root) unless path.relative? + + unless options["branch"] || Bundler.settings[:disable_local_branch_check] + raise GitError, "Cannot use local override for #{name} at #{path} because " \ + ":branch is not specified in Gemfile. Specify a branch or use " \ + "`bundle config --delete` to remove the local override" + end + + unless path.exist? + raise GitError, "Cannot use local override for #{name} because #{path} " \ + "does not exist. Check `bundle config --delete` to remove the local override" + end + + set_local!(path) + + # Create a new git proxy without the cached revision + # so the Gemfile.lock always picks up the new revision. + @git_proxy = GitProxy.new(path, uri, ref) + + if git_proxy.branch != options["branch"] && !Bundler.settings[:disable_local_branch_check] + raise GitError, "Local override for #{name} at #{path} is using branch " \ + "#{git_proxy.branch} but Gemfile specifies #{options["branch"]}" + end + + changed = cached_revision && cached_revision != git_proxy.revision + + if changed && !@unlocked && !git_proxy.contains?(cached_revision) + raise GitError, "The Gemfile lock is pointing to revision #{shortref_for_display(cached_revision)} " \ + "but the current branch in your local override for #{name} does not contain such commit. " \ + "Please make sure your branch is up to date." + end + + changed + end + + def specs(*) + set_local!(app_cache_path) if has_app_cache? && !local? + + if requires_checkout? && !@copied + fetch + git_proxy.copy_to(install_path, submodules) + serialize_gemspecs_in(install_path) + @copied = true + end + + local_specs + end + + def install(spec, options = {}) + force = options[:force] + + Bundler.ui.info "Using #{version_message(spec)} from #{self}" + + if requires_checkout? && !@copied && !force + Bundler.ui.debug " * Checking out revision: #{ref}" + git_proxy.copy_to(install_path, submodules) + serialize_gemspecs_in(install_path) + @copied = true + elsif force + git_proxy.copy_to(install_path, submodules) + end + + generate_bin_options = { :disable_extensions => !Bundler.rubygems.spec_missing_extensions?(spec), :build_args => options[:build_args] } + generate_bin(spec, generate_bin_options) + + requires_checkout? ? spec.post_install_message : nil + end + + def cache(spec, custom_path = nil) + app_cache_path = app_cache_path(custom_path) + return unless Bundler.settings[:cache_all] + return if path == app_cache_path + cached! + FileUtils.rm_rf(app_cache_path) + git_proxy.checkout if requires_checkout? + git_proxy.copy_to(app_cache_path, @submodules) + serialize_gemspecs_in(app_cache_path) + end + + def load_spec_files + super + rescue PathError => e + Bundler.ui.trace e + raise GitError, "#{self} is not yet checked out. Run `bundle install` first." + end + + # This is the path which is going to contain a cache + # of the git repository. When using the same git repository + # across different projects, this cache will be shared. + # When using local git repos, this is set to the local repo. + def cache_path + @cache_path ||= begin + git_scope = "#{base_name}-#{uri_hash}" + + if Bundler.requires_sudo? + Bundler.user_bundle_path.join("cache/git", git_scope) + else + Bundler.cache.join("git", git_scope) + end + end + end + + def app_cache_dirname + "#{base_name}-#{shortref_for_path(cached_revision || revision)}" + end + + def revision + git_proxy.revision + end + + def allow_git_ops? + @allow_remote || @allow_cached + end + + private + + def serialize_gemspecs_in(destination) + destination = destination.expand_path(Bundler.root) if destination.relative? + Dir["#{destination}/#{@glob}"].each do |spec_path| + # Evaluate gemspecs and cache the result. Gemspecs + # in git might require git or other dependencies. + # The gemspecs we cache should already be evaluated. + spec = Bundler.load_gemspec(spec_path) + next unless spec + Bundler.rubygems.set_installed_by_version(spec) + Bundler.rubygems.validate(spec) + File.open(spec_path, "wb") {|file| file.write(spec.to_ruby) } + end + end + + def set_local!(path) + @local = true + @local_specs = @git_proxy = nil + @cache_path = @install_path = path + end + + def has_app_cache? + cached_revision && super + end + + def local? + @local + end + + def requires_checkout? + allow_git_ops? && !local? + end + + def base_name + File.basename(uri.sub(%r{^(\w+://)?([^/:]+:)?(//\w*/)?(\w*/)*}, ""), ".git") + end + + def shortref_for_display(ref) + ref[0..6] + end + + def shortref_for_path(ref) + ref[0..11] + end + + def uri_hash + if uri =~ %r{^\w+://(\w+@)?} + # Downcase the domain component of the URI + # and strip off a trailing slash, if one is present + input = URI.parse(uri).normalize.to_s.sub(%r{/$}, "") + else + # If there is no URI scheme, assume it is an ssh/git URI + input = uri + end + Digest::SHA1.hexdigest(input) + end + + def cached_revision + options["revision"] + end + + def cached? + cache_path.exist? + end + + def git_proxy + @git_proxy ||= GitProxy.new(cache_path, uri, ref, cached_revision, self) + end + + def fetch + git_proxy.checkout + rescue GitError + raise unless Bundler.feature_flag.allow_offline_install? + Bundler.ui.warn "Using cached git data because of network errors" + end + + # no-op, since we validate when re-serializing the gemspec + def validate_spec(_spec); end + + if Bundler.rubygems.stubs_provide_full_functionality? + def load_gemspec(file) + stub = Gem::StubSpecification.gemspec_stub(file, install_path.parent, install_path.parent) + stub.full_gem_path = Pathname.new(file).dirname.expand_path(root).to_s.untaint + StubSpecification.from_stub(stub) + end + end + end + end +end diff --git a/lib/bundler/source/git/git_proxy.rb b/lib/bundler/source/git/git_proxy.rb new file mode 100644 index 0000000000..c05d7a5afa --- /dev/null +++ b/lib/bundler/source/git/git_proxy.rb @@ -0,0 +1,252 @@ +# frozen_string_literal: true +require "shellwords" +require "tempfile" +module Bundler + class Source + class Git + class GitNotInstalledError < GitError + def initialize + msg = String.new + msg << "You need to install git to be able to use gems from git repositories. " + msg << "For help installing git, please refer to GitHub's tutorial at https://help.github.com/articles/set-up-git" + super msg + end + end + + class GitNotAllowedError < GitError + def initialize(command) + msg = String.new + msg << "Bundler is trying to run a `git #{command}` at runtime. You probably need to run `bundle install`. However, " + msg << "this error message could probably be more useful. Please submit a ticket at http://github.com/bundler/bundler/issues " + msg << "with steps to reproduce as well as the following\n\nCALLER: #{caller.join("\n")}" + super msg + end + end + + class GitCommandError < GitError + def initialize(command, path = nil, extra_info = nil) + msg = String.new + msg << "Git error: command `git #{command}` in directory #{SharedHelpers.pwd} has failed." + msg << "\n#{extra_info}" if extra_info + msg << "\nIf this error persists you could try removing the cache directory '#{path}'" if path && path.exist? + super msg + end + end + + class MissingGitRevisionError < GitError + def initialize(ref, repo) + msg = "Revision #{ref} does not exist in the repository #{repo}. Maybe you misspelled it?" + super msg + end + end + + # The GitProxy is responsible to interact with git repositories. + # All actions required by the Git source is encapsulated in this + # object. + class GitProxy + attr_accessor :path, :uri, :ref + attr_writer :revision + + def initialize(path, uri, ref, revision = nil, git = nil) + @path = path + @uri = uri + @ref = ref + @revision = revision + @git = git + raise GitNotInstalledError.new if allow? && !Bundler.git_present? + end + + def revision + return @revision if @revision + + begin + @revision ||= find_local_revision + rescue GitCommandError + raise MissingGitRevisionError.new(ref, uri) + end + + @revision + end + + def branch + @branch ||= allowed_in_path do + git("rev-parse --abbrev-ref HEAD").strip + end + end + + def contains?(commit) + allowed_in_path do + result = git_null("branch --contains #{commit}") + $? == 0 && result =~ /^\* (.*)$/ + end + end + + def version + git("--version").match(/(git version\s*)?((\.?\d+)+).*/)[2] + end + + def full_version + git("--version").sub("git version", "").strip + end + + def checkout + if path.exist? + return if has_revision_cached? + Bundler.ui.info "Fetching #{URICredentialsFilter.credential_filtered_uri(uri)}" + in_path do + git_retry %(fetch --force --quiet --tags #{uri_escaped_with_configured_credentials} "refs/heads/*:refs/heads/*") + end + else + Bundler.ui.info "Fetching #{URICredentialsFilter.credential_filtered_uri(uri)}" + SharedHelpers.filesystem_access(path.dirname) do |p| + FileUtils.mkdir_p(p) + end + git_retry %(clone #{uri_escaped_with_configured_credentials} "#{path}" --bare --no-hardlinks --quiet) + end + end + + def copy_to(destination, submodules = false) + # method 1 + unless File.exist?(destination.join(".git")) + begin + SharedHelpers.filesystem_access(destination.dirname) do |p| + FileUtils.mkdir_p(p) + end + SharedHelpers.filesystem_access(destination) do |p| + FileUtils.rm_rf(p) + end + git_retry %(clone --no-checkout --quiet "#{path}" "#{destination}") + File.chmod(((File.stat(destination).mode | 0o777) & ~File.umask), destination) + rescue Errno::EEXIST => e + file_path = e.message[%r{.*?(/.*)}, 1] + raise GitError, "Bundler could not install a gem because it needs to " \ + "create a directory, but a file exists - #{file_path}. Please delete " \ + "this file and try again." + end + end + # method 2 + SharedHelpers.chdir(destination) do + git_retry %(fetch --force --quiet --tags "#{path}") + git "reset --hard #{@revision}" + + if submodules + git_retry "submodule update --init --recursive" + elsif Gem::Version.create(version) >= Gem::Version.create("2.9.0") + git_retry "submodule deinit --all --force" + end + end + end + + private + + # TODO: Do not rely on /dev/null. + # Given that open3 is not cross platform until Ruby 1.9.3, + # the best solution is to pipe to /dev/null if it exists. + # If it doesn't, everything will work fine, but the user + # will get the $stderr messages as well. + def git_null(command) + git("#{command} 2>#{Bundler::NULL}", false) + end + + def git_retry(command) + Bundler::Retry.new("`git #{command}`", GitNotAllowedError).attempts do + git(command) + end + end + + def git(command, check_errors = true, error_msg = nil) + command_with_no_credentials = URICredentialsFilter.credential_filtered_string(command, uri) + raise GitNotAllowedError.new(command_with_no_credentials) unless allow? + + out = SharedHelpers.with_clean_git_env do + capture_and_filter_stderr(uri) { `git #{command}` } + end + + stdout_with_no_credentials = URICredentialsFilter.credential_filtered_string(out, uri) + raise GitCommandError.new(command_with_no_credentials, path, error_msg) if check_errors && !$?.success? + stdout_with_no_credentials + end + + def has_revision_cached? + return unless @revision + in_path { git("cat-file -e #{@revision}") } + true + rescue GitError + false + end + + def remove_cache + FileUtils.rm_rf(path) + end + + def find_local_revision + allowed_in_path do + git("rev-parse --verify #{Shellwords.shellescape(ref)}", true).strip + end + end + + # Escape the URI for git commands + def uri_escaped_with_configured_credentials + remote = configured_uri_for(uri) + if Bundler::WINDOWS + # Windows quoting requires double quotes only, with double quotes + # inside the string escaped by being doubled. + '"' + remote.gsub('"') { '""' } + '"' + else + # Bash requires single quoted strings, with the single quotes escaped + # by ending the string, escaping the quote, and restarting the string. + "'" + remote.gsub("'") { "'\\''" } + "'" + end + end + + # Adds credentials to the URI as Fetcher#configured_uri_for does + def configured_uri_for(uri) + if /https?:/ =~ uri + remote = URI(uri) + config_auth = Bundler.settings[remote.to_s] || Bundler.settings[remote.host] + remote.userinfo ||= config_auth + remote.to_s + else + uri + end + end + + def allow? + @git ? @git.allow_git_ops? : true + end + + def in_path(&blk) + checkout unless path.exist? + SharedHelpers.chdir(path, &blk) + end + + def allowed_in_path + return in_path { yield } if allow? + raise GitError, "The git source #{uri} is not yet checked out. Please run `bundle install` before trying to start your application" + end + + # TODO: Replace this with Open3 when upgrading to bundler 2 + # Similar to #git_null, as Open3 is not cross-platform, + # a temporary way is to use Tempfile to capture the stderr. + # When replacing this using Open3, make sure git_null is + # also replaced by Open3, so stdout and stderr all got handled properly. + def capture_and_filter_stderr(uri) + return_value, captured_err = "" + backup_stderr = STDERR.dup + begin + Tempfile.open("captured_stderr") do |f| + STDERR.reopen(f) + return_value = yield + f.rewind + captured_err = f.read + end + ensure + STDERR.reopen backup_stderr + end + $stderr.puts URICredentialsFilter.credential_filtered_string(captured_err, uri) if uri && !captured_err.empty? + return_value + end + end + end + end +end diff --git a/lib/bundler/source/path.rb b/lib/bundler/source/path.rb new file mode 100644 index 0000000000..8dd0763cc1 --- /dev/null +++ b/lib/bundler/source/path.rb @@ -0,0 +1,249 @@ +# frozen_string_literal: true +module Bundler + class Source + class Path < Source + autoload :Installer, "bundler/source/path/installer" + + attr_reader :path, :options, :root_path, :original_path + attr_writer :name + attr_accessor :version + + protected :original_path + + DEFAULT_GLOB = "{,*,*/*}.gemspec".freeze + + def initialize(options) + @options = options.dup + @glob = options["glob"] || DEFAULT_GLOB + + @allow_cached = false + @allow_remote = false + + @root_path = options["root_path"] || Bundler.root + + if options["path"] + @path = Pathname.new(options["path"]) + @path = expand(@path) unless @path.relative? + end + + @name = options["name"] + @version = options["version"] + + # Stores the original path. If at any point we move to the + # cached directory, we still have the original path to copy from. + @original_path = @path + end + + def remote! + @allow_remote = true + end + + def cached! + @allow_cached = true + end + + def self.from_lock(options) + new(options.merge("path" => options.delete("remote"))) + end + + def to_lock + out = String.new("PATH\n") + out << " remote: #{lockfile_path}\n" + out << " glob: #{@glob}\n" unless @glob == DEFAULT_GLOB + out << " specs:\n" + end + + def to_s + "source at `#{@path}`" + end + + def hash + [self.class, expanded_path, version].hash + end + + def eql?(other) + return unless other.class == self.class + expanded_original_path == other.expanded_original_path && + version == other.version + end + + alias_method :==, :eql? + + def name + File.basename(expanded_path.to_s) + end + + def install(spec, options = {}) + Bundler.ui.info "Using #{version_message(spec)} from #{self}" + generate_bin(spec, :disable_extensions => true) + nil # no post-install message + end + + def cache(spec, custom_path = nil) + app_cache_path = app_cache_path(custom_path) + return unless Bundler.settings[:cache_all] + return if expand(@original_path).to_s.index(root_path.to_s + "/") == 0 + + unless @original_path.exist? + raise GemNotFound, "Can't cache gem #{version_message(spec)} because #{self} is missing!" + end + + FileUtils.rm_rf(app_cache_path) + FileUtils.cp_r("#{@original_path}/.", app_cache_path) + FileUtils.touch(app_cache_path.join(".bundlecache")) + end + + def local_specs(*) + @local_specs ||= load_spec_files + end + + def specs + if has_app_cache? + @path = app_cache_path + @expanded_path = nil # Invalidate + end + local_specs + end + + def app_cache_dirname + name + end + + def root + Bundler.root + end + + def is_a_path? + instance_of?(Path) + end + + def expanded_original_path + @expanded_original_path ||= expand(original_path) + end + + private + + def expanded_path + @expanded_path ||= expand(path) + end + + def expand(somepath) + somepath.expand_path(root_path) + rescue ArgumentError => e + Bundler.ui.debug(e) + raise PathError, "There was an error while trying to use the path " \ + "`#{somepath}`.\nThe error message was: #{e.message}." + end + + def lockfile_path + return relative_path(original_path) if original_path.absolute? + expand(original_path).relative_path_from(Bundler.root) + end + + def app_cache_path(custom_path = nil) + @app_cache_path ||= Bundler.app_cache(custom_path).join(app_cache_dirname) + end + + def has_app_cache? + SharedHelpers.in_bundle? && app_cache_path.exist? + end + + def load_gemspec(file) + return unless spec = Bundler.load_gemspec(file) + Bundler.rubygems.set_installed_by_version(spec) + spec + end + + def validate_spec(spec) + Bundler.rubygems.validate(spec) + end + + def load_spec_files + index = Index.new + + if File.directory?(expanded_path) + # We sort depth-first since `<<` will override the earlier-found specs + Dir["#{expanded_path}/#{@glob}"].sort_by {|p| -p.split(File::SEPARATOR).size }.each do |file| + next unless spec = load_gemspec(file) + spec.source = self + + # Validation causes extension_dir to be calculated, which depends + # on #source, so we validate here instead of load_gemspec + validate_spec(spec) + index << spec + end + + if index.empty? && @name && @version + index << Gem::Specification.new do |s| + s.name = @name + s.source = self + s.version = Gem::Version.new(@version) + s.platform = Gem::Platform::RUBY + s.summary = "Fake gemspec for #{@name}" + s.relative_loaded_from = "#{@name}.gemspec" + s.authors = ["no one"] + if expanded_path.join("bin").exist? + executables = expanded_path.join("bin").children + executables.reject! {|p| File.directory?(p) } + s.executables = executables.map {|c| c.basename.to_s } + end + end + end + else + message = String.new("The path `#{expanded_path}` ") + message << if File.exist?(expanded_path) + "is not a directory." + else + "does not exist." + end + raise PathError, message + end + + index + end + + def relative_path(path = self.path) + if path.to_s.start_with?(root_path.to_s) + return path.relative_path_from(root_path) + end + path + end + + def generate_bin(spec, options = {}) + gem_dir = Pathname.new(spec.full_gem_path) + + # Some gem authors put absolute paths in their gemspec + # and we have to save them from themselves + spec.files = spec.files.map do |p| + next p unless p =~ /\A#{Pathname::SEPARATOR_PAT}/ + next if File.directory?(p) + begin + Pathname.new(p).relative_path_from(gem_dir).to_s + rescue ArgumentError + p + end + end.compact + + installer = Path::Installer.new( + spec, + :env_shebang => false, + :disable_extensions => options[:disable_extensions], + :build_args => options[:build_args] + ) + installer.post_install + rescue Gem::InvalidSpecificationException => e + Bundler.ui.warn "\n#{spec.name} at #{spec.full_gem_path} did not have a valid gemspec.\n" \ + "This prevents bundler from installing bins or native extensions, but " \ + "that may not affect its functionality." + + if !spec.extensions.empty? && !spec.email.empty? + Bundler.ui.warn "If you need to use this package without installing it from a gem " \ + "repository, please contact #{spec.email} and ask them " \ + "to modify their .gemspec so it can work with `gem build`." + end + + Bundler.ui.warn "The validation message from Rubygems was:\n #{e.message}" + end + end + end +end diff --git a/lib/bundler/source/path/installer.rb b/lib/bundler/source/path/installer.rb new file mode 100644 index 0000000000..9c2f74a31b --- /dev/null +++ b/lib/bundler/source/path/installer.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true +module Bundler + class Source + class Path + class Installer < Bundler::RubyGemsGemInstaller + attr_reader :spec + + def initialize(spec, options = {}) + @spec = spec + @gem_dir = Bundler.rubygems.path(spec.full_gem_path) + @wrappers = true + @env_shebang = true + @format_executable = options[:format_executable] || false + @build_args = options[:build_args] || Bundler.rubygems.build_args + @gem_bin_dir = "#{Bundler.rubygems.gem_dir}/bin" + @disable_extensions = options[:disable_extensions] + + if Bundler.requires_sudo? + @tmp_dir = Bundler.tmp(spec.full_name).to_s + @bin_dir = "#{@tmp_dir}/bin" + else + @bin_dir = @gem_bin_dir + end + end + + def post_install + SharedHelpers.chdir(@gem_dir) do + run_hooks(:pre_install) + + unless @disable_extensions + build_extensions + run_hooks(:post_build) + end + + generate_bin unless spec.executables.nil? || spec.executables.empty? + + run_hooks(:post_install) + end + ensure + Bundler.rm_rf(@tmp_dir) if Bundler.requires_sudo? + end + + private + + def generate_bin + super + + if Bundler.requires_sudo? + SharedHelpers.filesystem_access(@gem_bin_dir) do |p| + Bundler.mkdir_p(p) + end + spec.executables.each do |exe| + Bundler.sudo "cp -R #{@bin_dir}/#{exe} #{@gem_bin_dir}" + end + end + end + + def run_hooks(type) + hooks_meth = "#{type}_hooks" + return unless Gem.respond_to?(hooks_meth) + Gem.send(hooks_meth).each do |hook| + result = hook.call(self) + next unless result == false + location = " at #{$1}" if hook.inspect =~ /@(.*:\d+)/ + message = "#{type} hook#{location} failed for #{spec.full_name}" + raise InstallHookError, message + end + end + end + end + end +end diff --git a/lib/bundler/source/rubygems.rb b/lib/bundler/source/rubygems.rb new file mode 100644 index 0000000000..353194f53f --- /dev/null +++ b/lib/bundler/source/rubygems.rb @@ -0,0 +1,462 @@ +# frozen_string_literal: true +require "uri" +require "rubygems/user_interaction" + +module Bundler + class Source + class Rubygems < Source + autoload :Remote, "bundler/source/rubygems/remote" + + # Use the API when installing less than X gems + API_REQUEST_LIMIT = 500 + # Ask for X gems per API request + API_REQUEST_SIZE = 50 + + attr_reader :remotes, :caches + + def initialize(options = {}) + @options = options + @remotes = [] + @dependency_names = [] + @allow_remote = false + @allow_cached = false + @caches = [cache_path, *Bundler.rubygems.gem_cache] + + Array(options["remotes"] || []).reverse_each {|r| add_remote(r) } + end + + def remote! + @specs = nil + @allow_remote = true + end + + def cached! + @allow_cached = true + end + + def hash + @remotes.hash + end + + def eql?(other) + other.is_a?(Rubygems) && other.credless_remotes == credless_remotes + end + + alias_method :==, :eql? + + def include?(o) + o.is_a?(Rubygems) && (o.credless_remotes - credless_remotes).empty? + end + + def can_lock?(spec) + spec.source.is_a?(Rubygems) + end + + def options + { "remotes" => @remotes.map(&:to_s) } + end + + def self.from_lock(options) + new(options) + end + + def to_lock + out = String.new("GEM\n") + remotes.reverse_each do |remote| + out << " remote: #{suppress_configured_credentials remote}\n" + end + out << " specs:\n" + end + + def to_s + remote_names = remotes.map(&:to_s).join(", ") + "rubygems repository #{remote_names}" + end + alias_method :name, :to_s + + def specs + @specs ||= begin + # remote_specs usually generates a way larger Index than the other + # sources, and large_idx.use small_idx is way faster than + # small_idx.use large_idx. + idx = @allow_remote ? remote_specs.dup : Index.new + idx.use(cached_specs, :override_dupes) if @allow_cached || @allow_remote + idx.use(installed_specs, :override_dupes) + idx + end + end + + def install(spec, opts = {}) + force = opts[:force] + ensure_builtin_gems_cached = opts[:ensure_builtin_gems_cached] + + if ensure_builtin_gems_cached && builtin_gem?(spec) + if !cached_path(spec) + cached_built_in_gem(spec) unless spec.remote + force = true + else + spec.loaded_from = loaded_from(spec) + end + end + + if installed?(spec) && (!force || spec.name.eql?("bundler")) + Bundler.ui.info "Using #{version_message(spec)}" + return nil # no post-install message + end + + # Download the gem to get the spec, because some specs that are returned + # by rubygems.org are broken and wrong. + if spec.remote + # Check for this spec from other sources + uris = [spec.remote.anonymized_uri] + uris += remotes_for_spec(spec).map(&:anonymized_uri) + uris.uniq! + Installer.ambiguous_gems << [spec.name, *uris] if uris.length > 1 + + s = Bundler.rubygems.spec_from_gem(fetch_gem(spec), Bundler.settings["trust-policy"]) + spec.__swap__(s) + end + + unless Bundler.settings[:no_install] + message = "Installing #{version_message(spec)}" + message += " with native extensions" if spec.extensions.any? + Bundler.ui.confirm message + + path = cached_gem(spec) + if requires_sudo? + install_path = Bundler.tmp(spec.full_name) + bin_path = install_path.join("bin") + else + install_path = rubygems_dir + bin_path = Bundler.system_bindir + end + + installed_spec = nil + Bundler.rubygems.preserve_paths do + installed_spec = Bundler::RubyGemsGemInstaller.at( + path, + :install_dir => install_path.to_s, + :bin_dir => bin_path.to_s, + :ignore_dependencies => true, + :wrappers => true, + :env_shebang => true, + :build_args => opts[:build_args], + :bundler_expected_checksum => spec.respond_to?(:checksum) && spec.checksum + ).install + end + spec.full_gem_path = installed_spec.full_gem_path + + # SUDO HAX + if requires_sudo? + Bundler.rubygems.repository_subdirectories.each do |name| + src = File.join(install_path, name, "*") + dst = File.join(rubygems_dir, name) + if name == "extensions" && Dir.glob(src).any? + src = File.join(src, "*/*") + ext_src = Dir.glob(src).first + ext_src.gsub!(src[0..-6], "") + dst = File.dirname(File.join(dst, ext_src)) + end + SharedHelpers.filesystem_access(dst) do |p| + Bundler.mkdir_p(p) + end + Bundler.sudo "cp -R #{src} #{dst}" if Dir[src].any? + end + + spec.executables.each do |exe| + SharedHelpers.filesystem_access(Bundler.system_bindir) do |p| + Bundler.mkdir_p(p) + end + Bundler.sudo "cp -R #{install_path}/bin/#{exe} #{Bundler.system_bindir}/" + end + end + installed_spec.loaded_from = loaded_from(spec) + end + spec.loaded_from = loaded_from(spec) + + spec.post_install_message + ensure + Bundler.rm_rf(install_path) if requires_sudo? + end + + def cache(spec, custom_path = nil) + if builtin_gem?(spec) + cached_path = cached_built_in_gem(spec) + else + cached_path = cached_gem(spec) + end + raise GemNotFound, "Missing gem file '#{spec.full_name}.gem'." unless cached_path + return if File.dirname(cached_path) == Bundler.app_cache.to_s + Bundler.ui.info " * #{File.basename(cached_path)}" + FileUtils.cp(cached_path, Bundler.app_cache(custom_path)) + rescue Errno::EACCES => e + Bundler.ui.debug(e) + raise InstallError, e.message + end + + def cached_built_in_gem(spec) + cached_path = cached_path(spec) + if cached_path.nil? + remote_spec = remote_specs.search(spec).first + if remote_spec + cached_path = fetch_gem(remote_spec) + else + Bundler.ui.warn "#{spec.full_name} is built in to Ruby, and can't be cached because your Gemfile doesn't have any sources that contain it." + end + end + cached_path + end + + def add_remote(source) + uri = normalize_uri(source) + @remotes.unshift(uri) unless @remotes.include?(uri) + end + + def replace_remotes(other_remotes) + return false if other_remotes == @remotes + + @remotes = [] + other_remotes.reverse_each do |r| + add_remote r.to_s + end + end + + def unmet_deps + if @allow_remote && api_fetchers.any? + remote_specs.unmet_dependency_names + else + [] + end + end + + def fetchers + @fetchers ||= remotes.map do |uri| + remote = Source::Rubygems::Remote.new(uri) + Bundler::Fetcher.new(remote) + end + end + + protected + + def credless_remotes + remotes.map(&method(:suppress_configured_credentials)) + end + + def remotes_for_spec(spec) + specs.search_all(spec.name).inject([]) do |uris, s| + uris << s.remote if s.remote + uris + end + end + + def loaded_from(spec) + "#{rubygems_dir}/specifications/#{spec.full_name}.gemspec" + end + + def cached_gem(spec) + cached_gem = cached_path(spec) + unless cached_gem + raise Bundler::GemNotFound, "Could not find #{spec.file_name} for installation" + end + cached_gem + end + + def cached_path(spec) + possibilities = @caches.map {|p| "#{p}/#{spec.file_name}" } + possibilities.find {|p| File.exist?(p) } + end + + def normalize_uri(uri) + uri = uri.to_s + uri = "#{uri}/" unless uri =~ %r{/$} + uri = URI(uri) + raise ArgumentError, "The source must be an absolute URI. For example:\n" \ + "source 'https://rubygems.org'" if !uri.absolute? || (uri.is_a?(URI::HTTP) && uri.host.nil?) + uri + end + + def suppress_configured_credentials(remote) + remote_nouser = remote.dup.tap {|uri| uri.user = uri.password = nil }.to_s + if remote.userinfo && remote.userinfo == Bundler.settings[remote_nouser] + remote_nouser + else + remote + end + end + + def installed_specs + @installed_specs ||= begin + idx = Index.new + have_bundler = false + Bundler.rubygems.all_specs.reverse_each do |spec| + if spec.name == "bundler" + next unless spec.version.to_s == VERSION + have_bundler = true + end + spec.source = self + if Bundler.rubygems.spec_missing_extensions?(spec, false) + Bundler.ui.debug "Source #{self} is ignoring #{spec} because it is missing extensions" + next + end + idx << spec + end + + # Always have bundler locally + unless have_bundler + # We're running bundler directly from the source + # so, let's create a fake gemspec for it (it's a path) + # gemspec + bundler = Gem::Specification.new do |s| + s.name = "bundler" + s.version = VERSION + s.platform = Gem::Platform::RUBY + s.source = self + s.authors = ["bundler team"] + s.loaded_from = File.expand_path("..", __FILE__) + end + idx << bundler + end + idx + end + end + + def cached_specs + @cached_specs ||= begin + idx = installed_specs.dup + + Dir["#{cache_path}/*.gem"].each do |gemfile| + next if gemfile =~ /^bundler\-[\d\.]+?\.gem/ + s ||= Bundler.rubygems.spec_from_gem(gemfile) + s.source = self + if Bundler.rubygems.spec_missing_extensions?(s, false) + Bundler.ui.debug "Source #{self} is ignoring #{s} because it is missing extensions" + next + end + idx << s + end + end + + idx + end + + def api_fetchers + fetchers.select {|f| f.use_api && f.fetchers.first.api_fetcher? } + end + + def remote_specs + @remote_specs ||= Index.build do |idx| + index_fetchers = fetchers - api_fetchers + + # gather lists from non-api sites + index_fetchers.each do |f| + Bundler.ui.info "Fetching source index from #{f.uri}" + idx.use f.specs_with_retry(nil, self) + end + + # because ensuring we have all the gems we need involves downloading + # the gemspecs of those gems, if the non-api sites contain more than + # about 100 gems, we treat all sites as non-api for speed. + allow_api = idx.size < API_REQUEST_LIMIT && dependency_names.size < API_REQUEST_LIMIT + Bundler.ui.debug "Need to query more than #{API_REQUEST_LIMIT} gems." \ + " Downloading full index instead..." unless allow_api + + if allow_api + api_fetchers.each do |f| + Bundler.ui.info "Fetching gem metadata from #{f.uri}", Bundler.ui.debug? + idx.use f.specs_with_retry(dependency_names, self) + Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over + end + + # Suppose the gem Foo depends on the gem Bar. Foo exists in Source A. Bar has some versions that exist in both + # sources A and B. At this point, the API request will have found all the versions of Bar in source A, + # but will not have found any versions of Bar from source B, which is a problem if the requested version + # of Foo specifically depends on a version of Bar that is only found in source B. This ensures that for + # each spec we found, we add all possible versions from all sources to the index. + loop do + idxcount = idx.size + api_fetchers.each do |f| + Bundler.ui.info "Fetching version metadata from #{f.uri}", Bundler.ui.debug? + idx.use f.specs_with_retry(idx.dependency_names, self), true + Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over + end + break if idxcount == idx.size + end + + if api_fetchers.any? + # it's possible that gems from one source depend on gems from some + # other source, so now we download gemspecs and iterate over those + # dependencies, looking for gems we don't have info on yet. + unmet = idx.unmet_dependency_names + + # if there are any cross-site gems we missed, get them now + api_fetchers.each do |f| + Bundler.ui.info "Fetching dependency metadata from #{f.uri}", Bundler.ui.debug? + idx.use f.specs_with_retry(unmet, self) + Bundler.ui.info "" unless Bundler.ui.debug? # new line now that the dots are over + end if unmet.any? + else + allow_api = false + end + end + + unless allow_api + api_fetchers.each do |f| + Bundler.ui.info "Fetching source index from #{f.uri}" + idx.use f.specs_with_retry(nil, self) + end + end + end + end + + def fetch_gem(spec) + return false unless spec.remote + uri = spec.remote.uri + spec.fetch_platform + Bundler.ui.confirm("Fetching #{version_message(spec)}") + + download_path = requires_sudo? ? Bundler.tmp(spec.full_name) : rubygems_dir + gem_path = "#{rubygems_dir}/cache/#{spec.full_name}.gem" + + SharedHelpers.filesystem_access("#{download_path}/cache") do |p| + FileUtils.mkdir_p(p) + end + Bundler.rubygems.download_gem(spec, uri, download_path) + + if requires_sudo? + SharedHelpers.filesystem_access("#{rubygems_dir}/cache") do |p| + Bundler.mkdir_p(p) + end + Bundler.sudo "mv #{download_path}/cache/#{spec.full_name}.gem #{gem_path}" + end + + gem_path + ensure + Bundler.rm_rf(download_path) if requires_sudo? + end + + def builtin_gem?(spec) + # Ruby 2.1, where all included gems have this summary + return true if spec.summary =~ /is bundled with Ruby/ + + # Ruby 2.0, where gemspecs are stored in specifications/default/ + spec.loaded_from && spec.loaded_from.include?("specifications/default/") + end + + def installed?(spec) + installed_specs[spec].any? + end + + def requires_sudo? + Bundler.requires_sudo? + end + + def rubygems_dir + Bundler.rubygems.gem_dir + end + + def cache_path + Bundler.app_cache + end + end + end +end diff --git a/lib/bundler/source/rubygems/remote.rb b/lib/bundler/source/rubygems/remote.rb new file mode 100644 index 0000000000..b49e645506 --- /dev/null +++ b/lib/bundler/source/rubygems/remote.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true +module Bundler + class Source + class Rubygems + class Remote + attr_reader :uri, :anonymized_uri, :original_uri + + def initialize(uri) + orig_uri = uri + uri = Bundler.settings.mirror_for(uri) + @original_uri = orig_uri if orig_uri != uri + fallback_auth = Bundler.settings.credentials_for(uri) + + @uri = apply_auth(uri, fallback_auth).freeze + @anonymized_uri = remove_auth(@uri).freeze + end + + # @return [String] A slug suitable for use as a cache key for this + # remote. + # + def cache_slug + @cache_slug ||= begin + cache_uri = original_uri || uri + + uri_parts = [cache_uri.host, cache_uri.user, cache_uri.port, cache_uri.path] + uri_digest = Digest::MD5.hexdigest(uri_parts.compact.join(".")) + + uri_parts[-1] = uri_digest + uri_parts.compact.join(".") + end + end + + def to_s + "rubygems remote at #{anonymized_uri}" + end + + private + + def apply_auth(uri, auth) + if auth && uri.userinfo.nil? + uri = uri.dup + uri.userinfo = auth + end + + uri + rescue URI::InvalidComponentError + error_message = "Please CGI escape your usernames and passwords before " \ + "setting them for authentication." + raise HTTPError.new(error_message) + end + + def remove_auth(uri) + if uri.userinfo + uri = uri.dup + uri.user = uri.password = nil + end + + uri + end + end + end + end +end |