summaryrefslogtreecommitdiff
path: root/ext/ripper/lib/ripper/lexer.rb
diff options
context:
space:
mode:
authorNobuyoshi Nakada <nobu@ruby-lang.org>2019-05-29 16:02:09 +0900
committerNobuyoshi Nakada <nobu@ruby-lang.org>2019-05-29 18:21:28 +0900
commit5ceff480c2c1696e9893a1e5e09e39f8425c6c5a (patch)
tree877283a4b6f0364cb3d81110a38b0239092c78ba /ext/ripper/lib/ripper/lexer.rb
parent7c0639f3f83f85557aff87e94da1afd373555bcb (diff)
ripper: Ripper::Lexer#scan
* ext/ripper/lib/ripper/lexer.rb (Ripper::Lexer#scan): parses the code and returns the result elements including errors. [EXPERIMENTAL]
Diffstat (limited to 'ext/ripper/lib/ripper/lexer.rb')
-rw-r--r--ext/ripper/lib/ripper/lexer.rb33
1 files changed, 29 insertions, 4 deletions
diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index a5ba5b6890..d1d257582d 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -60,13 +60,13 @@ class Ripper
def nobits?(i) to_int.nobits?(i) end
end
- Elem = Struct.new(:pos, :event, :tok, :state) do
- def initialize(pos, event, tok, state)
- super(pos, event, tok, State.new(state))
+ Elem = Struct.new(:pos, :event, :tok, :state, :message) do
+ def initialize(pos, event, tok, state, message = nil)
+ super(pos, event, tok, State.new(state), message)
end
def inspect
- "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}>"
+ "#<#{self.class}: #{event}@#{pos[0]}:#{pos[1]}:#{state}: #{tok.inspect}#{": " if message}#{message}>"
end
def pretty_print(q)
@@ -76,10 +76,23 @@ class Ripper
q.breakable
q.text("token: ")
tok.pretty_print(q)
+ if message
+ q.breakable
+ q.text("message: ")
+ q.text(message)
+ end
}
end
+
+ def to_a
+ a = super
+ a.pop unless a.last
+ a
+ end
end
+ attr_reader :errors
+
def tokenize
parse().sort_by(&:pos).map(&:tok)
end
@@ -88,7 +101,13 @@ class Ripper
parse().sort_by(&:pos).map(&:to_a)
end
+ # parse the code and returns elements including errors.
+ def scan
+ (parse() + errors).sort_by {|e| [*e.pos, (e.message ? -1 : 0)]}
+ end
+
def parse
+ @errors = []
@buf = []
@stack = []
super
@@ -144,6 +163,12 @@ class Ripper
@buf.push Elem.new([lineno(), column()], __callee__, tok, state())
end
+ def on_error(mesg)
+ @errors.push Elem.new([lineno(), column()], __callee__, token(), state(), mesg)
+ end
+ alias on_parse_error on_error
+ alias compile_error on_error
+
(SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|
alias_method event, :_push_token
end