diff options
Diffstat (limited to 'lib/rubygems/package.rb')
| -rw-r--r-- | lib/rubygems/package.rb | 464 |
1 files changed, 302 insertions, 162 deletions
diff --git a/lib/rubygems/package.rb b/lib/rubygems/package.rb index 77811ed5ec..7e41b18f66 100644 --- a/lib/rubygems/package.rb +++ b/lib/rubygems/package.rb @@ -1,14 +1,22 @@ -# -*- coding: utf-8 -*- # frozen_string_literal: true -#-- + +# rubocop:disable Style/AsciiComments + # Copyright (C) 2004 Mauricio Julio Fernández Pradier # See LICENSE.txt for additional licensing information. -#++ -# + +# rubocop:enable Style/AsciiComments + +require_relative "win_platform" +require_relative "security" +require_relative "user_interaction" + +## # Example using a Gem::Package # # Builds a .gem file given a Gem::Specification. A .gem file is a tarball -# which contains a data.tar.gz and metadata.gz, and possibly signatures. +# which contains a data.tar.gz, metadata.gz, checksums.yaml.gz and possibly +# signatures. # # require 'rubygems' # require 'rubygems/package' @@ -41,13 +49,7 @@ # #files are the files in the .gem tar file, not the Ruby files in the gem # #extract_files and #contents automatically call #verify -require 'rubygems/security' -require 'rubygems/specification' -require 'rubygems/user_interaction' -require 'zlib' - class Gem::Package - include Gem::UserInteraction class Error < Gem::Exception; end @@ -55,22 +57,26 @@ class Gem::Package class FormatError < Error attr_reader :path - def initialize message, source = nil + def initialize(message, source = nil) if source - @path = source.path + @path = source.is_a?(String) ? source : source.path - message = message + " in #{path}" if path + message += " in #{path}" if path end super message end - end class PathError < Error - def initialize destination, destination_dir - super "installing into parent path %s of %s is not allowed" % - [destination, destination_dir] + def initialize(destination, destination_dir) + super format("installing into parent path %s of %s is not allowed", destination, destination_dir) + end + end + + class SymlinkError < Error + def initialize(name, destination, destination_dir) + super format("installing symlink '%s' pointing to parent path %s of %s is not allowed", name, destination, destination_dir) end end @@ -83,7 +89,6 @@ class Gem::Package class TarInvalidError < Error; end - attr_accessor :build_time # :nodoc: ## @@ -98,6 +103,11 @@ class Gem::Package attr_reader :files ## + # Reference to the gem being packaged. + + attr_reader :gem + + ## # The security policy used for verifying the contents of this package. attr_accessor :security_policy @@ -107,12 +117,24 @@ class Gem::Package attr_writer :spec - def self.build spec, skip_validation=false - gem_file = spec.file_name + ## + # Permission for directories + attr_accessor :dir_mode + + ## + # Permission for program files + attr_accessor :prog_mode + + ## + # Permission for other files + attr_accessor :data_mode + + def self.build(spec, skip_validation = false, strict_validation = false, file_name = nil) + gem_file = file_name || spec.file_name package = new gem_file package.spec = spec - package.build skip_validation + package.build skip_validation, strict_validation gem_file end @@ -124,34 +146,62 @@ class Gem::Package # If +gem+ is an existing file in the old format a Gem::Package::Old will be # returned. - def self.new gem, security_policy = nil + def self.new(gem, security_policy = nil) gem = if gem.is_a?(Gem::Package::Source) - gem - elsif gem.respond_to? :read - Gem::Package::IOSource.new gem - else - Gem::Package::FileSource.new gem - end + gem + elsif gem.respond_to? :read + Gem::Package::IOSource.new gem + else + Gem::Package::FileSource.new gem + end - return super unless Gem::Package == self + return super unless self == Gem::Package return super unless gem.present? return super unless gem.start - return super unless gem.start.include? 'MD5SUM =' + return super unless gem.start.include? "MD5SUM =" Gem::Package::Old.new gem end ## + # Extracts the Gem::Specification and raw metadata from the .gem file at + # +path+. + #-- + + def self.raw_spec(path, security_policy = nil) + format = new(path, security_policy) + spec = format.spec + + metadata = nil + + File.open path, Gem.binary_mode do |io| + tar = Gem::Package::TarReader.new io + tar.each_entry do |entry| + case entry.full_name + when "metadata" then + metadata = entry.read + when "metadata.gz" then + metadata = Gem::Util.gunzip entry.read + end + end + end + + [spec, metadata] + end + + ## # Creates a new package that will read or write to the file +gem+. - def initialize gem, security_policy # :notnew: + def initialize(gem, security_policy) # :notnew: + require "zlib" + @gem = gem - @build_time = Time.now + @build_time = Gem.source_date_epoch @checksums = {} @contents = nil - @digests = Hash.new { |h, algorithm| h[algorithm] = {} } + @digests = Hash.new {|h, algorithm| h[algorithm] = {} } @files = nil @security_policy = security_policy @signatures = {} @@ -162,17 +212,17 @@ class Gem::Package ## # Copies this package to +path+ (if possible) - def copy_to path + def copy_to(path) FileUtils.cp @gem.path, path unless File.exist? path end ## # Adds a checksum for each entry in the gem to checksums.yaml.gz. - def add_checksums tar + def add_checksums(tar) Gem.load_yaml - checksums_by_algorithm = Hash.new { |h, algorithm| h[algorithm] = {} } + checksums_by_algorithm = Hash.new {|h, algorithm| h[algorithm] = {} } @checksums.each do |name, digests| digests.each do |algorithm, digest| @@ -180,9 +230,13 @@ class Gem::Package end end - tar.add_file_signed 'checksums.yaml.gz', 0444, @signer do |io| + tar.add_file_signed "checksums.yaml.gz", 0o444, @signer do |io| gzip_to io do |gz_io| - YAML.dump checksums_by_algorithm, gz_io + if Gem.use_psych? + Psych.dump checksums_by_algorithm, gz_io + else + gz_io.write Gem::YAMLSerializer.dump(checksums_by_algorithm) + end end end end @@ -191,8 +245,8 @@ class Gem::Package # Adds the files listed in the packages's Gem::Specification to data.tar.gz # and adds this file to the +tar+. - def add_contents tar # :nodoc: - digests = tar.add_file_signed 'data.tar.gz', 0444, @signer do |io| + def add_contents(tar) # :nodoc: + digests = tar.add_file_signed "data.tar.gz", 0o444, @signer do |io| gzip_to io do |gz_io| Gem::Package::TarWriter.new gz_io do |data_tar| add_files data_tar @@ -200,27 +254,25 @@ class Gem::Package end end - @checksums['data.tar.gz'] = digests + @checksums["data.tar.gz"] = digests end ## # Adds files included the package's Gem::Specification to the +tar+ file - def add_files tar # :nodoc: + def add_files(tar) # :nodoc: @spec.files.each do |file| stat = File.lstat file if stat.symlink? - relative_dir = File.dirname(file).sub("#{Dir.pwd}/", '') - target_path = File.join(relative_dir, File.readlink(file)) - tar.add_symlink file, target_path, stat.mode + tar.add_symlink file, File.readlink(file), stat.mode end next unless stat.file? tar.add_file_simple file, stat.mode, stat.size do |dst_io| - open file, 'rb' do |src_io| - dst_io.write src_io.read 16384 until src_io.eof? + File.open file, "rb" do |src_io| + copy_stream(src_io, dst_io, stat.size) end end end @@ -229,27 +281,31 @@ class Gem::Package ## # Adds the package's Gem::Specification to the +tar+ file - def add_metadata tar # :nodoc: - digests = tar.add_file_signed 'metadata.gz', 0444, @signer do |io| + def add_metadata(tar) # :nodoc: + digests = tar.add_file_signed "metadata.gz", 0o444, @signer do |io| gzip_to io do |gz_io| gz_io.write @spec.to_yaml end end - @checksums['metadata.gz'] = digests + @checksums["metadata.gz"] = digests end ## # Builds this package based on the specification set by #spec= - def build skip_validation = false + def build(skip_validation = false, strict_validation = false) + raise ArgumentError, "skip_validation = true and strict_validation = true are incompatible" if skip_validation && strict_validation + Gem.load_yaml - require 'rubygems/security' - @spec.mark_version - @spec.validate unless skip_validation + @spec.validate true, strict_validation unless skip_validation - setup_signer + setup_signer( + signer_options: { + expiration_length_days: Gem.configuration.cert_expiration_length_days, + } + ) @gem.with_write_io do |gem_io| Gem::Package::TarWriter.new gem_io do |gem| @@ -263,7 +319,7 @@ class Gem::Package Successfully built RubyGem Name: #{@spec.name} Version: #{@spec.version} - File: #{File.basename @spec.cache_file} + File: #{File.basename @gem.path} EOM ensure @signer = nil @@ -283,7 +339,7 @@ EOM gem_tar = Gem::Package::TarReader.new io gem_tar.each do |entry| - next unless entry.full_name == 'data.tar.gz' + next unless entry.full_name == "data.tar.gz" open_tar_gz entry do |pkg_tar| pkg_tar.each do |contents_entry| @@ -294,31 +350,31 @@ EOM return @contents end end + rescue Zlib::GzipFile::Error, EOFError, Gem::Package::TarInvalidError => e + raise Gem::Package::FormatError.new e.message, @gem end ## # Creates a digest of the TarEntry +entry+ from the digest algorithm set by # the security policy. - def digest entry # :nodoc: - algorithms = if @checksums then - @checksums.keys - else - [Gem::Security::DIGEST_NAME].compact - end - - algorithms.each do |algorithm| - digester = - if defined?(OpenSSL::Digest) then - OpenSSL::Digest.new algorithm - else - Digest.const_get(algorithm).new - end + def digest(entry) # :nodoc: + algorithms = if @checksums + @checksums.to_h {|algorithm, _| [algorithm, Gem::Security.create_digest(algorithm)] } + elsif Gem::Security::DIGEST_NAME + { Gem::Security::DIGEST_NAME => Gem::Security.create_digest(Gem::Security::DIGEST_NAME) } + end - digester << entry.read(16384) until entry.eof? + return @digests if algorithms.nil? || algorithms.empty? - entry.rewind + buf = String.new(capacity: 16_384, encoding: Encoding::BINARY) + until entry.eof? + entry.readpartial(16_384, buf) + algorithms.each_value {|digester| digester << buf } + end + entry.rewind + algorithms.each do |algorithm, digester| @digests[algorithm][entry.full_name] = digester end @@ -331,22 +387,24 @@ EOM # If +pattern+ is specified, only entries matching that glob will be # extracted. - def extract_files destination_dir, pattern = "*" + def extract_files(destination_dir, pattern = "*") verify unless @spec - FileUtils.mkdir_p destination_dir + FileUtils.mkdir_p destination_dir, mode: dir_mode && 0o755 @gem.with_read_io do |io| reader = Gem::Package::TarReader.new io reader.each do |entry| - next unless entry.full_name == 'data.tar.gz' + next unless entry.full_name == "data.tar.gz" extract_tar_gz entry, destination_dir, pattern - return # ignore further entries + break # ignore further entries end end + rescue Zlib::GzipFile::Error, EOFError, Gem::Package::TarInvalidError => e + raise Gem::Package::FormatError.new e.message, @gem end ## @@ -360,36 +418,81 @@ EOM # If +pattern+ is specified, only entries matching that glob will be # extracted. - def extract_tar_gz io, destination_dir, pattern = "*" # :nodoc: + def extract_tar_gz(io, destination_dir, pattern = "*") # :nodoc: + destination_dir = File.realpath(destination_dir) + + directories = [] + symlinks = [] + open_tar_gz io do |tar| tar.each do |entry| - next unless File.fnmatch pattern, entry.full_name, File::FNM_DOTMATCH + full_name = entry.full_name + next unless File.fnmatch pattern, full_name, File::FNM_DOTMATCH + + destination = install_location full_name, destination_dir - destination = install_location entry.full_name, destination_dir + if entry.symlink? + link_target = entry.header.linkname + real_destination = link_target.start_with?("/") ? link_target : File.expand_path(link_target, File.dirname(destination)) - FileUtils.rm_rf destination + raise Gem::Package::SymlinkError.new(full_name, real_destination, destination_dir) unless + normalize_path(real_destination).start_with? normalize_path(destination_dir + "/") + + symlinks << [full_name, link_target, destination, real_destination] + end - mkdir_options = {} - mkdir_options[:mode] = entry.header.mode if entry.directory? mkdir = - if entry.directory? then + if entry.directory? destination else File.dirname destination end - FileUtils.mkdir_p mkdir, mkdir_options + unless directories.include?(mkdir) + FileUtils.mkdir_p mkdir, mode: dir_mode ? 0o755 : (entry.header.mode if entry.directory?) + directories << mkdir + end - open destination, 'wb' do |out| - out.write entry.read - FileUtils.chmod entry.header.mode, destination - end if entry.file? + real_mkdir = File.realpath(mkdir) + unless real_mkdir == destination_dir || normalize_path(real_mkdir).start_with?(normalize_path(destination_dir + "/")) + raise Gem::Package::PathError.new(real_mkdir, destination_dir) + end - File.symlink(entry.header.linkname, destination) if entry.symlink? + if entry.file? + File.open(destination, "wb") do |out| + copy_stream(tar.io, out, entry.size) + # Flush needs to happen before chmod because there could be data + # in the IO buffer that needs to be written, and that could be + # written after the chmod (on close) which would mess up the perms + out.flush + out.chmod file_mode(entry.header.mode) & ~File.umask + end + end verbose destination end end + + symlinks.each do |name, target, destination, real_destination| + if File.exist?(real_destination) + create_symlink(target, destination) + else + alert_warning "#{@spec.full_name} ships with a dangling symlink named #{name} pointing to missing #{target} file. Ignoring" + end + end + + if dir_mode + File.chmod(dir_mode, *directories) + end + end + + def file_mode(mode) # :nodoc: + ((mode & 0o111).zero? ? data_mode : prog_mode) || + # If we're not using one of the default modes, then we're going to fall + # back to the mode from the tarball. In this case we need to mask it down + # to fit into 2^16 bits (the maximum value for a mode in CRuby since it + # gets put into an unsigned short). + (mode & ((1 << 16) - 1)) end ## @@ -398,7 +501,7 @@ EOM # Also sets the gzip modification time to the package build time to ease # testing. - def gzip_to io # :yields: gz_io + def gzip_to(io) # :yields: gz_io gz_io = Zlib::GzipWriter.new io, Zlib::BEST_COMPRESSION gz_io.mtime = @build_time @@ -412,39 +515,40 @@ EOM # # If +filename+ is not inside +destination_dir+ an exception is raised. - def install_location filename, destination_dir # :nodoc: + def install_location(filename, destination_dir) # :nodoc: raise Gem::Package::PathError.new(filename, destination_dir) if - filename.start_with? '/' + filename.start_with? "/" - destination_dir = File.realpath destination_dir if - File.respond_to? :realpath - destination_dir = File.expand_path destination_dir - - destination = File.join destination_dir, filename - destination = File.expand_path destination + destination_dir = File.realpath(destination_dir) + destination = File.expand_path(filename, destination_dir) raise Gem::Package::PathError.new(destination, destination_dir) unless - destination.start_with? destination_dir + normalize_path(destination).start_with? normalize_path(destination_dir + "/") - destination.untaint destination end + if Gem.win_platform? + def normalize_path(pathname) # :nodoc: + pathname.downcase + end + else + def normalize_path(pathname) # :nodoc: + pathname + end + end + ## # Loads a Gem::Specification from the TarEntry +entry+ - def load_spec entry # :nodoc: + def load_spec_from_metadata(entry) # :nodoc: + limit = 10 * 1024 * 1024 case entry.full_name - when 'metadata' then - @spec = Gem::Specification.from_yaml entry.read - when 'metadata.gz' then - args = [entry] - args << { :external_encoding => Encoding::UTF_8 } if - Object.const_defined?(:Encoding) && - Zlib::GzipReader.method(:wrap).arity != 1 - - Zlib::GzipReader.wrap(*args) do |gzio| - @spec = Gem::Specification.from_yaml gzio.read + when "metadata" then + @spec = Gem::Specification.from_yaml limit_read(entry, "metadata", limit) + when "metadata.gz" then + Zlib::GzipReader.wrap(entry, external_encoding: Encoding::UTF_8) do |gzio| + @spec = Gem::Specification.from_yaml limit_read(gzio, "metadata.gz", limit) end end end @@ -452,23 +556,32 @@ EOM ## # Opens +io+ as a gzipped tar archive - def open_tar_gz io # :nodoc: + def open_tar_gz(io) # :nodoc: Zlib::GzipReader.wrap io do |gzio| tar = Gem::Package::TarReader.new gzio yield tar + ensure + # Consume remaining gzip data to prevent the + # "attempt to close unfinished zstream; reset forced" warning + # when the GzipReader is closed with unconsumed compressed data. + begin + IO.copy_stream(gzio, IO::NULL) + rescue Zlib::GzipFile::Error, IOError + nil + end end end ## # Reads and loads checksums.yaml.gz from the tar file +gem+ - def read_checksums gem + def read_checksums(gem) Gem.load_yaml - @checksums = gem.seek 'checksums.yaml.gz' do |entry| + @checksums = gem.seek "checksums.yaml.gz" do |entry| Zlib::GzipReader.wrap entry do |gz_io| - Gem::SafeYAML.safe_load gz_io.read + Gem::SafeYAML.safe_load limit_read(gz_io, "checksums.yaml.gz", 10 * 1024 * 1024) end end end @@ -477,15 +590,22 @@ EOM # Prepares the gem for signing and checksum generation. If a signing # certificate and key are not present only checksum generation is set up. - def setup_signer - passphrase = ENV['GEM_PRIVATE_KEY_PASSPHRASE'] - if @spec.signing_key then - @signer = Gem::Security::Signer.new @spec.signing_key, @spec.cert_chain, passphrase + def setup_signer(signer_options: {}) + passphrase = ENV["GEM_PRIVATE_KEY_PASSPHRASE"] + if @spec.signing_key + @signer = + Gem::Security::Signer.new( + @spec.signing_key, + @spec.cert_chain, + passphrase, + signer_options + ) + @spec.signing_key = nil - @spec.cert_chain = @signer.cert_chain.map { |cert| cert.to_s } + @spec.cert_chain = @signer.cert_chain.map(&:to_s) else @signer = Gem::Security::Signer.new nil, nil, passphrase - @spec.cert_chain = @signer.cert_chain.map { |cert| cert.to_pem } if + @spec.cert_chain = @signer.cert_chain.map(&:to_pem) if @signer.cert_chain end end @@ -527,8 +647,7 @@ EOM verify_checksums @digests, @checksums - @security_policy.verify_signatures @spec, @digests, @signatures if - @security_policy + @security_policy&.verify_signatures @spec, @digests, @signatures true rescue Gem::Security::Exception @@ -537,22 +656,24 @@ EOM raise rescue Errno::ENOENT => e raise Gem::Package::FormatError.new e.message - rescue Gem::Package::TarInvalidError => e + rescue Zlib::GzipFile::Error, EOFError, Gem::Package::TarInvalidError => e raise Gem::Package::FormatError.new e.message, @gem end + private + ## # Verifies the +checksums+ against the +digests+. This check is not # cryptographically secure. Missing checksums are ignored. - def verify_checksums digests, checksums # :nodoc: + def verify_checksums(digests, checksums) # :nodoc: return unless checksums checksums.sort.each do |algorithm, gem_digests| gem_digests.sort.each do |file_name, gem_hexdigest| computed_digest = digests[algorithm][file_name] - unless computed_digest.hexdigest == gem_hexdigest then + unless computed_digest.hexdigest == gem_hexdigest raise Gem::Package::FormatError.new \ "#{algorithm} checksum mismatch for #{file_name}", @gem end @@ -563,67 +684,86 @@ EOM ## # Verifies +entry+ in a .gem file. - def verify_entry entry + def verify_entry(entry) file_name = entry.full_name @files << file_name case file_name when /\.sig$/ then - @signatures[$`] = entry.read if @security_policy + @signatures[$`] = limit_read(entry, file_name, 1024 * 1024) if @security_policy return else digest entry end - case file_name - when /^metadata(.gz)?$/ then - load_spec entry - when 'data.tar.gz' then - verify_gz entry - end - rescue => e - message = "package is corrupt, exception while verifying: " + - "#{e.message} (#{e.class})" - raise Gem::Package::FormatError.new message, @gem + load_spec_from_metadata entry + rescue StandardError + warn "Exception while verifying #{@gem.path}" + raise end ## # Verifies the files of the +gem+ - def verify_files gem + def verify_files(gem) gem.each do |entry| verify_entry entry end - unless @spec then - raise Gem::Package::FormatError.new 'package metadata is missing', @gem + unless @spec + raise Gem::Package::FormatError.new "package metadata is missing", @gem end - unless @files.include? 'data.tar.gz' then + unless @files.include? "data.tar.gz" raise Gem::Package::FormatError.new \ - 'package content (data.tar.gz) is missing', @gem + "package content (data.tar.gz) is missing", @gem end - end - ## - # Verifies that +entry+ is a valid gzipped file. + if (duplicates = @files.group_by {|f| f }.select {|_k,v| v.size > 1 }.map(&:first)) && duplicates.any? + raise Gem::Security::Exception, "duplicate files in the package: (#{duplicates.map(&:inspect).join(", ")})" + end + end - def verify_gz entry # :nodoc: - Zlib::GzipReader.wrap entry do |gzio| - gzio.read 16384 until gzio.eof? # gzip checksum verification + if RUBY_ENGINE == "truffleruby" + def copy_stream(src, dst, size) # :nodoc: + dst.write src.read(size) + end + else + def copy_stream(src, dst, size) # :nodoc: + IO.copy_stream(src, dst, size) end - rescue Zlib::GzipFile::Error => e - raise Gem::Package::FormatError.new(e.message, entry.full_name) end + def limit_read(io, name, limit) + bytes = io.read(limit + 1) + raise Gem::Package::FormatError, "#{name} is too big (over #{limit} bytes)" if bytes.size > limit + bytes + end + + if Gem.win_platform? + # Create a symlink and fallback to copy the file or directory on Windows, + # where symlink creation needs special privileges in form of the Developer Mode. + # JRuby on Windows raises TypeError from the wincode path-conversion helper + # when it cannot create the symlink, so fall back to copy in that case too. + def create_symlink(old_name, new_name) + File.symlink(old_name, new_name) + rescue Errno::EACCES, TypeError + from = File.expand_path(old_name, File.dirname(new_name)) + FileUtils.cp_r(from, new_name) + end + else + def create_symlink(old_name, new_name) + File.symlink(old_name, new_name) + end + end end -require 'rubygems/package/digest_io' -require 'rubygems/package/source' -require 'rubygems/package/file_source' -require 'rubygems/package/io_source' -require 'rubygems/package/old' -require 'rubygems/package/tar_header' -require 'rubygems/package/tar_reader' -require 'rubygems/package/tar_reader/entry' -require 'rubygems/package/tar_writer' +require_relative "package/digest_io" +require_relative "package/source" +require_relative "package/file_source" +require_relative "package/io_source" +require_relative "package/old" +require_relative "package/tar_header" +require_relative "package/tar_reader" +require_relative "package/tar_reader/entry" +require_relative "package/tar_writer" |