diff options
Diffstat (limited to 'lib/rubygems')
| -rw-r--r-- | lib/rubygems/package.rb | 4 | ||||
| -rw-r--r-- | lib/rubygems/request_set.rb | 65 | ||||
| -rw-r--r-- | lib/rubygems/request_set/lockfile.rb | 2 | ||||
| -rw-r--r-- | lib/rubygems/request_set/lockfile/parser.rb | 344 | ||||
| -rw-r--r-- | lib/rubygems/request_set/lockfile/tokenizer.rb | 122 | ||||
| -rw-r--r-- | lib/rubygems/yaml_serializer.rb | 60 |
6 files changed, 97 insertions, 500 deletions
diff --git a/lib/rubygems/package.rb b/lib/rubygems/package.rb index d36b6dfb5e..7e41b18f66 100644 --- a/lib/rubygems/package.rb +++ b/lib/rubygems/package.rb @@ -743,9 +743,11 @@ EOM if Gem.win_platform? # Create a symlink and fallback to copy the file or directory on Windows, # where symlink creation needs special privileges in form of the Developer Mode. + # JRuby on Windows raises TypeError from the wincode path-conversion helper + # when it cannot create the symlink, so fall back to copy in that case too. def create_symlink(old_name, new_name) File.symlink(old_name, new_name) - rescue Errno::EACCES + rescue Errno::EACCES, TypeError from = File.expand_path(old_name, File.dirname(new_name)) FileUtils.cp_r(from, new_name) end diff --git a/lib/rubygems/request_set.rb b/lib/rubygems/request_set.rb index dbebd1af0c..c06ef32da9 100644 --- a/lib/rubygems/request_set.rb +++ b/lib/rubygems/request_set.rb @@ -322,11 +322,8 @@ class Gem::RequestSet @git_set.root_dir = @install_dir lock_file = "#{File.expand_path(path)}.lock" - begin - tokenizer = Gem::RequestSet::Lockfile::Tokenizer.from_file lock_file - parser = tokenizer.make_parser self, [] - parser.parse - rescue Errno::ENOENT + if File.exist?(lock_file) + load_lockfile lock_file end gf = Gem::RequestSet::GemDependencyAPI.new self, path @@ -335,6 +332,63 @@ class Gem::RequestSet gf.load end + def load_lockfile(lock_file) # :nodoc: + require "bundler" + require "bundler/lockfile_parser" + + # Bundler::Source::Path resolves relative `remote:` paths against + # Bundler.root, which raises when there is no Gemfile in the working + # directory. Anchor it to the lockfile's directory so PATH sections in a + # `gem install -g` lockfile can be parsed without a Bundler environment. + previous_root = Bundler.instance_variable_get(:@root) + Bundler.instance_variable_set(:@root, Pathname.new(File.expand_path(File.dirname(lock_file)))) + + parser = Bundler::LockfileParser.new(File.read(lock_file), lockfile_path: lock_file) + + parser.specs.group_by(&:source).each do |source, specs| + case source + when Bundler::Source::Rubygems + remotes = source.remotes.map {|remote| Gem::Source.new(remote.to_s) } + remotes << Gem::Source.new(Gem::DEFAULT_HOST) if remotes.empty? + lock_set = Gem::Resolver::LockSet.new(remotes) + specs.each do |spec| + added = lock_set.add(spec.name, spec.version.to_s, spec.platform) + spec.dependencies.each do |dep| + added.each {|s| s.add_dependency dep } + end + end + @sets << lock_set + when Bundler::Source::Git + git_set = Gem::Resolver::GitSet.new + git_set.root_dir = @install_dir + specs.each do |spec| + git_spec = git_set.add_git_spec( + spec.name, + spec.version.to_s, + source.uri.to_s, + source.revision, + source.submodules || false + ) + spec.dependencies.each {|dep| git_spec.add_dependency dep } + end + @sets << git_set + when Bundler::Source::Path + vendor_set = Gem::Resolver::VendorSet.new + specs.each do |spec| + loaded = vendor_set.add_vendor_gem(spec.name, source.path.to_s) + spec.dependencies.each {|dep| loaded.dependencies << dep } + end + @sets << vendor_set + end + end + + parser.dependencies.each_value do |dep| + gem dep.name, *dep.requirement.as_list + end + ensure + Bundler.instance_variable_set(:@root, previous_root) if defined?(previous_root) + end + def pretty_print(q) # :nodoc: q.group 2, "[RequestSet:", "]" do q.breakable @@ -462,4 +516,3 @@ end require_relative "request_set/gem_dependency_api" require_relative "request_set/lockfile" -require_relative "request_set/lockfile/tokenizer" diff --git a/lib/rubygems/request_set/lockfile.rb b/lib/rubygems/request_set/lockfile.rb index da6dd329bc..8b9c9690d6 100644 --- a/lib/rubygems/request_set/lockfile.rb +++ b/lib/rubygems/request_set/lockfile.rb @@ -231,5 +231,3 @@ class Gem::RequestSet::Lockfile @set.sorted_requests end end - -require_relative "lockfile/tokenizer" diff --git a/lib/rubygems/request_set/lockfile/parser.rb b/lib/rubygems/request_set/lockfile/parser.rb deleted file mode 100644 index e751a1445e..0000000000 --- a/lib/rubygems/request_set/lockfile/parser.rb +++ /dev/null @@ -1,344 +0,0 @@ -# frozen_string_literal: true - -class Gem::RequestSet::Lockfile::Parser - ### - # Parses lockfiles - - def initialize(tokenizer, set, platforms, filename = nil) - @tokens = tokenizer - @filename = filename - @set = set - @platforms = platforms - end - - def parse - until @tokens.empty? do - token = get - - case token.type - when :section then - @tokens.skip :newline - - case token.value - when "DEPENDENCIES" then - parse_DEPENDENCIES - when "GIT" then - parse_GIT - when "GEM" then - parse_GEM - when "PATH" then - parse_PATH - when "PLATFORMS" then - parse_PLATFORMS - else - token = get until @tokens.empty? || peek.first == :section - end - else - raise "BUG: unhandled token #{token.type} (#{token.value.inspect}) at line #{token.line} column #{token.column}" - end - end - end - - ## - # Gets the next token for a Lockfile - - def get(expected_types = nil, expected_value = nil) # :nodoc: - token = @tokens.shift - - if expected_types && !Array(expected_types).include?(token.type) - unget token - - message = "unexpected token [#{token.type.inspect}, #{token.value.inspect}], " \ - "expected #{expected_types.inspect}" - - raise Gem::RequestSet::Lockfile::ParseError.new message, token.column, token.line, @filename - end - - if expected_value && expected_value != token.value - unget token - - message = "unexpected token [#{token.type.inspect}, #{token.value.inspect}], " \ - "expected [#{expected_types.inspect}, " \ - "#{expected_value.inspect}]" - - raise Gem::RequestSet::Lockfile::ParseError.new message, token.column, token.line, @filename - end - - token - end - - def parse_DEPENDENCIES # :nodoc: - while !@tokens.empty? && peek.type == :text do - token = get :text - - requirements = [] - - case peek[0] - when :bang then - get :bang - - requirements << pinned_requirement(token.value) - when :l_paren then - get :l_paren - - loop do - op = get(:requirement).value - version = get(:text).value - - requirements << "#{op} #{version}" - - break unless peek.type == :comma - - get :comma - end - - get :r_paren - - if peek[0] == :bang - requirements.clear - requirements << pinned_requirement(token.value) - - get :bang - end - end - - @set.gem token.value, *requirements - - skip :newline - end - end - - def parse_GEM # :nodoc: - sources = [] - - while peek.first(2) == [:entry, "remote"] do - get :entry, "remote" - data = get(:text).value - skip :newline - - sources << Gem::Source.new(data) - end - - sources << Gem::Source.new(Gem::DEFAULT_HOST) if sources.empty? - - get :entry, "specs" - - skip :newline - - set = Gem::Resolver::LockSet.new sources - last_specs = nil - - while !@tokens.empty? && peek.type == :text do - token = get :text - name = token.value - column = token.column - - case peek[0] - when :newline then - last_specs.each do |spec| - spec.add_dependency Gem::Dependency.new name if column == 6 - end - when :l_paren then - get :l_paren - - token = get [:text, :requirement] - type = token.type - data = token.value - - if type == :text && column == 4 - version, platform = data.split "-", 2 - - platform = - platform ? Gem::Platform.new(platform) : Gem::Platform::RUBY - - last_specs = set.add name, version, platform - else - dependency = parse_dependency name, data - - last_specs.each do |spec| - spec.add_dependency dependency - end - end - - get :r_paren - else - raise "BUG: unknown token #{peek}" - end - - skip :newline - end - - @set.sets << set - end - - def parse_GIT # :nodoc: - get :entry, "remote" - repository = get(:text).value - - skip :newline - - get :entry, "revision" - revision = get(:text).value - - skip :newline - - type = peek.type - value = peek.value - if type == :entry && %w[branch ref tag].include?(value) - get - get :text - - skip :newline - end - - get :entry, "specs" - - skip :newline - - set = Gem::Resolver::GitSet.new - set.root_dir = @set.install_dir - - last_spec = nil - - while !@tokens.empty? && peek.type == :text do - token = get :text - name = token.value - column = token.column - - case peek[0] - when :newline then - last_spec.add_dependency Gem::Dependency.new name if column == 6 - when :l_paren then - get :l_paren - - token = get [:text, :requirement] - type = token.type - data = token.value - - if type == :text && column == 4 - last_spec = set.add_git_spec name, data, repository, revision, true - else - dependency = parse_dependency name, data - - last_spec.add_dependency dependency - end - - get :r_paren - else - raise "BUG: unknown token #{peek}" - end - - skip :newline - end - - @set.sets << set - end - - def parse_PATH # :nodoc: - get :entry, "remote" - directory = get(:text).value - - skip :newline - - get :entry, "specs" - - skip :newline - - set = Gem::Resolver::VendorSet.new - last_spec = nil - - while !@tokens.empty? && peek.first == :text do - token = get :text - name = token.value - column = token.column - - case peek[0] - when :newline then - last_spec.add_dependency Gem::Dependency.new name if column == 6 - when :l_paren then - get :l_paren - - token = get [:text, :requirement] - type = token.type - data = token.value - - if type == :text && column == 4 - last_spec = set.add_vendor_gem name, directory - else - dependency = parse_dependency name, data - - last_spec.dependencies << dependency - end - - get :r_paren - else - raise "BUG: unknown token #{peek}" - end - - skip :newline - end - - @set.sets << set - end - - def parse_PLATFORMS # :nodoc: - while !@tokens.empty? && peek.first == :text do - name = get(:text).value - - @platforms << name - - skip :newline - end - end - - ## - # Parses the requirements following the dependency +name+ and the +op+ for - # the first token of the requirements and returns a Gem::Dependency object. - - def parse_dependency(name, op) # :nodoc: - return Gem::Dependency.new name, op unless peek[0] == :text - - version = get(:text).value - - requirements = ["#{op} #{version}"] - - while peek.type == :comma do - get :comma - op = get(:requirement).value - version = get(:text).value - - requirements << "#{op} #{version}" - end - - Gem::Dependency.new name, requirements - end - - private - - def skip(type) # :nodoc: - @tokens.skip type - end - - ## - # Peeks at the next token for Lockfile - - def peek # :nodoc: - @tokens.peek - end - - def pinned_requirement(name) # :nodoc: - requirement = Gem::Dependency.new name - specification = @set.sets.flat_map do |set| - set.find_all(requirement) - end.compact.first - - specification&.version - end - - ## - # Ungets the last token retrieved by #get - - def unget(token) # :nodoc: - @tokens.unshift token - end -end diff --git a/lib/rubygems/request_set/lockfile/tokenizer.rb b/lib/rubygems/request_set/lockfile/tokenizer.rb deleted file mode 100644 index 65cef3baa0..0000000000 --- a/lib/rubygems/request_set/lockfile/tokenizer.rb +++ /dev/null @@ -1,122 +0,0 @@ -# frozen_string_literal: true - -# ) frozen_string_literal: true -require_relative "parser" - -class Gem::RequestSet::Lockfile::Tokenizer - Token = Struct.new :type, :value, :column, :line - EOF = Token.new :EOF - - def self.from_file(file) - new File.read(file), file - end - - def initialize(input, filename = nil, line = 0, pos = 0) - @line = line - @line_pos = pos - @tokens = [] - @filename = filename - tokenize input - end - - def make_parser(set, platforms) - Gem::RequestSet::Lockfile::Parser.new self, set, platforms, @filename - end - - def to_a - @tokens.map {|token| [token.type, token.value, token.column, token.line] } - end - - def skip(type) - @tokens.shift while !@tokens.empty? && peek.type == type - end - - ## - # Calculates the column (by byte) and the line of the current token based on - # +byte_offset+. - - def token_pos(byte_offset) # :nodoc: - [byte_offset - @line_pos, @line] - end - - def empty? - @tokens.empty? - end - - def unshift(token) - @tokens.unshift token - end - - def next_token - @tokens.shift - end - alias_method :shift, :next_token - - def peek - @tokens.first || EOF - end - - private - - def tokenize(input) - require "strscan" - s = StringScanner.new input - - until s.eos? do - pos = s.pos - - pos = s.pos if leading_whitespace = s.scan(/ +/) - - if s.scan(/[<|=>]{7}/) - message = "your #{@filename} contains merge conflict markers" - column, line = token_pos pos - - raise Gem::RequestSet::Lockfile::ParseError.new message, column, line, @filename - end - - @tokens << - if s.scan(/\r?\n/) - - token = Token.new(:newline, nil, *token_pos(pos)) - @line_pos = s.pos - @line += 1 - token - elsif s.scan(/[A-Z]+/) - - if leading_whitespace - text = s.matched - text += s.scan(/[^\s)]*/).to_s # in case of no match - Token.new(:text, text, *token_pos(pos)) - else - Token.new(:section, s.matched, *token_pos(pos)) - end - elsif s.scan(/([a-z]+):\s/) - - s.pos -= 1 # rewind for possible newline - Token.new(:entry, s[1], *token_pos(pos)) - elsif s.scan(/\(/) - - Token.new(:l_paren, nil, *token_pos(pos)) - elsif s.scan(/\)/) - - Token.new(:r_paren, nil, *token_pos(pos)) - elsif s.scan(/<=|>=|=|~>|<|>|!=/) - - Token.new(:requirement, s.matched, *token_pos(pos)) - elsif s.scan(/,/) - - Token.new(:comma, nil, *token_pos(pos)) - elsif s.scan(/!/) - - Token.new(:bang, nil, *token_pos(pos)) - elsif s.scan(/[^\s),!]*/) - - Token.new(:text, s.matched, *token_pos(pos)) - else - raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}" - end - end - - @tokens - end -end diff --git a/lib/rubygems/yaml_serializer.rb b/lib/rubygems/yaml_serializer.rb index a1d5c94fc5..b2547b136b 100644 --- a/lib/rubygems/yaml_serializer.rb +++ b/lib/rubygems/yaml_serializer.rb @@ -72,9 +72,7 @@ module Gem def parse_node(base_indent) @depth += 1 - if @depth > MAX_NESTING_DEPTH - raise Psych::SyntaxError, "exceeded maximum nesting depth (#{MAX_NESTING_DEPTH})" - end + raise_max_nesting! if @depth > MAX_NESTING_DEPTH skip_blank_and_comments return nil if @lines.empty? @@ -285,7 +283,9 @@ module Gem Scalar.new(value: result) end - def coerce(val) + def coerce(val, depth = 0) + raise_max_nesting! if depth > MAX_NESTING_DEPTH + val = val.sub(/^! /, "") if val.start_with?("! ") if val =~ /^"(.*)"$/ @@ -311,7 +311,7 @@ module Gem elsif val =~ /^\[(.*)\]$/ inner = $1.strip return Sequence.new if inner.empty? - items = inner.split(/\s*,\s*/).reject(&:empty?).map {|e| Scalar.new(value: coerce(e)) } + items = inner.split(/\s*,\s*/).reject(&:empty?).map {|e| Scalar.new(value: coerce(e, depth + 1)) } Sequence.new(items: items) elsif /\A\d{4}-\d{2}-\d{2}([ T]\d{2}:\d{2}:\d{2})?/.match?(val) begin @@ -392,6 +392,15 @@ module Gem node end + def raise_max_nesting! + message = "exceeded maximum nesting depth (#{MAX_NESTING_DEPTH})" + if defined?(Psych::VERSION) + raise Psych::SyntaxError.new(nil, 0, 0, 0, message, nil) + else + raise Psych::SyntaxError, message + end + end + def skip_blank_and_comments while @lines.any? line = @lines[0] @@ -454,9 +463,7 @@ module Gem result = build_node(node) - if result.is_a?(Hash) && - (result[:tag] == "!ruby/object:Gem::Specification" || - result["tag"] == "!ruby/object:Gem::Specification") + if result.is_a?(Hash) && result[:tag] == "!ruby/object:Gem::Specification" build_specification(result) else result @@ -482,7 +489,11 @@ module Gem if @alias_count > MAX_ALIAS_RESOLUTIONS raise Psych::BadAlias, "exceeded maximum alias resolutions (#{MAX_ALIAS_RESOLUTIONS})" end - @anchor_values.fetch(node.name, nil) + unless @anchor_values.key?(node.name) + klass = defined?(Psych::AnchorNotDefined) ? Psych::AnchorNotDefined : Psych::BadAlias + raise klass, "An alias referenced an unknown anchor: #{node.name}" + end + @anchor_values.fetch(node.name) end def store_anchor(name, value) @@ -491,7 +502,6 @@ module Gem end def build_mapping(node) - check_anchor!(node) validate_tag!(node.tag) if node.tag result = case node.tag @@ -592,9 +602,14 @@ module Gem d.instance_variable_set(:@requirement, hash["requirement"] || hash["version_requirements"]) - type = hash["type"] - type = type ? type.to_s.sub(/^:/, "").to_sym : :runtime - validate_symbol!(type) + raw_type = hash["type"] + if raw_type + name = raw_type.to_s.sub(/^:/, "") + validate_symbol!(name) + type = name.to_sym + else + type = :runtime + end d.instance_variable_set(:@type, type) d.instance_variable_set(:@prerelease, ["true", true].include?(hash["prerelease"])) @@ -634,19 +649,14 @@ module Gem end end - def validate_symbol!(sym) - if @permitted_symbols.any? && !@permitted_symbols.include?(sym.to_s) - if defined?(Psych::VERSION) - raise Psych::DisallowedClass.new("load", sym.inspect) - else - raise Psych::DisallowedClass, "Tried to load unspecified class: #{sym.inspect}" - end - end - end + def validate_symbol!(name) + return if @permitted_symbols.empty? || @permitted_symbols.include?(name) - def check_anchor!(node) - if node.anchor - raise Psych::AliasesNotEnabled unless @aliases + label = ":#{name}" + if defined?(Psych::VERSION) + raise Psych::DisallowedClass.new("load", label) + else + raise Psych::DisallowedClass, "Tried to load unspecified class: #{label}" end end |