summaryrefslogtreecommitdiff
path: root/lib/rubygems/request_set/lockfile/tokenizer.rb
diff options
context:
space:
mode:
Diffstat (limited to 'lib/rubygems/request_set/lockfile/tokenizer.rb')
-rw-r--r--lib/rubygems/request_set/lockfile/tokenizer.rb108
1 files changed, 108 insertions, 0 deletions
diff --git a/lib/rubygems/request_set/lockfile/tokenizer.rb b/lib/rubygems/request_set/lockfile/tokenizer.rb
new file mode 100644
index 0000000..73c9a83
--- /dev/null
+++ b/lib/rubygems/request_set/lockfile/tokenizer.rb
@@ -0,0 +1,108 @@
+require 'strscan'
+require 'rubygems/request_set/lockfile/parser'
+
+class Gem::RequestSet::Lockfile::Tokenizer
+ def self.from_file file
+ new File.read(file), file
+ end
+
+ def initialize input, filename = nil, line = 0, pos = 0
+ @line = line
+ @line_pos = pos
+ @tokens = []
+ @filename = filename
+ tokenize input
+ end
+
+ def make_parser set, platforms
+ Gem::RequestSet::Lockfile::Parser.new self, set, platforms, @filename
+ end
+
+ def to_a
+ @tokens
+ end
+
+ def skip type
+ @tokens.shift while not @tokens.empty? and peek.first == type
+ end
+
+ ##
+ # Calculates the column (by byte) and the line of the current token based on
+ # +byte_offset+.
+
+ def token_pos byte_offset # :nodoc:
+ [byte_offset - @line_pos, @line]
+ end
+
+ def empty?
+ @tokens.empty?
+ end
+
+ def unshift token
+ @tokens.unshift token
+ end
+
+ def next_token
+ @tokens.shift
+ end
+ alias :shift :next_token
+
+ def peek
+ @tokens.first || [:EOF]
+ end
+
+ private
+
+ def tokenize input
+ s = StringScanner.new input
+
+ until s.eos? do
+ pos = s.pos
+
+ pos = s.pos if leading_whitespace = s.scan(/ +/)
+
+ if s.scan(/[<|=>]{7}/) then
+ message = "your #{@filename} contains merge conflict markers"
+ column, line = token_pos pos
+
+ raise Gem::RequestSet::Lockfile::ParseError.new message, column, line, @filename
+ end
+
+ @tokens <<
+ case
+ when s.scan(/\r?\n/) then
+ token = [:newline, nil, *token_pos(pos)]
+ @line_pos = s.pos
+ @line += 1
+ token
+ when s.scan(/[A-Z]+/) then
+ if leading_whitespace then
+ text = s.matched
+ text += s.scan(/[^\s)]*/).to_s # in case of no match
+ [:text, text, *token_pos(pos)]
+ else
+ [:section, s.matched, *token_pos(pos)]
+ end
+ when s.scan(/([a-z]+):\s/) then
+ s.pos -= 1 # rewind for possible newline
+ [:entry, s[1], *token_pos(pos)]
+ when s.scan(/\(/) then
+ [:l_paren, nil, *token_pos(pos)]
+ when s.scan(/\)/) then
+ [:r_paren, nil, *token_pos(pos)]
+ when s.scan(/<=|>=|=|~>|<|>|!=/) then
+ [:requirement, s.matched, *token_pos(pos)]
+ when s.scan(/,/) then
+ [:comma, nil, *token_pos(pos)]
+ when s.scan(/!/) then
+ [:bang, nil, *token_pos(pos)]
+ when s.scan(/[^\s),!]*/) then
+ [:text, s.matched, *token_pos(pos)]
+ else
+ raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}"
+ end
+ end
+
+ @tokens
+ end
+end