summaryrefslogtreecommitdiff
path: root/ext/ripper/lib
diff options
context:
space:
mode:
authornobu <nobu@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2017-09-14 10:53:47 +0000
committernobu <nobu@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2017-09-14 10:53:47 +0000
commit7df1e45bb6b4326da5c799dcf58d38f0a14362af (patch)
treec742cebc6f6a1bc37d74c7a24b9d59b68971afdd /ext/ripper/lib
parenta61ae940c68c4401a42487ee2b670e145dd56dcd (diff)
ripper: add states of scanner
* parse.y (ripper_state): add states of scanner to tokens from Ripper.lex and Ripper::Filter#on_*. based on the patch by aycabta (Code Ahss) at [ruby-core:81789]. [Feature #13686] * ext/ripper/tools/preproc.rb (prelude, usercode): generate EXPR_* constants from enums. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59896 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'ext/ripper/lib')
-rw-r--r--ext/ripper/lib/ripper/filter.rb10
-rw-r--r--ext/ripper/lib/ripper/lexer.rb33
2 files changed, 26 insertions, 17 deletions
diff --git a/ext/ripper/lib/ripper/filter.rb b/ext/ripper/lib/ripper/filter.rb
index 455197f6906..9955d30550b 100644
--- a/ext/ripper/lib/ripper/filter.rb
+++ b/ext/ripper/lib/ripper/filter.rb
@@ -25,6 +25,7 @@ class Ripper
@__lexer = Lexer.new(src, filename, lineno)
@__line = nil
@__col = nil
+ @__state = nil
end
# The file name of the input.
@@ -46,13 +47,20 @@ class Ripper
@__col
end
+ # The scanner's state of the current token.
+ # This value is the bitwise OR of zero or more of the +Ripper::EXPR_*+ constants.
+ def state
+ @__state
+ end
+
# Starts the parser.
# +init+ is a data accumulator and is passed to the next event handler (as
# of Enumerable#inject).
def parse(init = nil)
data = init
- @__lexer.lex.each do |pos, event, tok|
+ @__lexer.lex.each do |pos, event, tok, state|
@__line, @__col = *pos
+ @__state = state
data = if respond_to?(event, true)
then __send__(event, tok, data)
else on_default(event, tok, data)
diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb
index 300c8eef518..4e82a1275c2 100644
--- a/ext/ripper/lib/ripper/lexer.rb
+++ b/ext/ripper/lib/ripper/lexer.rb
@@ -23,29 +23,30 @@ class Ripper
end
# Tokenizes the Ruby program and returns an array of an array,
- # which is formatted like <code>[[lineno, column], type, token]</code>.
+ # which is formatted like
+ # <code>[[lineno, column], type, token, state]</code>.
#
# require 'ripper'
# require 'pp'
#
# pp Ripper.lex("def m(a) nil end")
- # #=> [[[1, 0], :on_kw, "def"],
- # [[1, 3], :on_sp, " " ],
- # [[1, 4], :on_ident, "m" ],
- # [[1, 5], :on_lparen, "(" ],
- # [[1, 6], :on_ident, "a" ],
- # [[1, 7], :on_rparen, ")" ],
- # [[1, 8], :on_sp, " " ],
- # [[1, 9], :on_kw, "nil"],
- # [[1, 12], :on_sp, " " ],
- # [[1, 13], :on_kw, "end"]]
+ # #=> [[[1, 0], :on_kw, "def", Ripper::EXPR_FNAME ],
+ # [[1, 3], :on_sp, " ", Ripper::EXPR_FNAME ],
+ # [[1, 4], :on_ident, "m", Ripper::EXPR_ENDFN ],
+ # [[1, 5], :on_lparen, "(", Ripper::EXPR_LABEL | Ripper::EXPR_BEG],
+ # [[1, 6], :on_ident, "a", Ripper::EXPR_ARG ],
+ # [[1, 7], :on_rparen, ")", Ripper::EXPR_ENDFN ],
+ # [[1, 8], :on_sp, " ", Ripper::EXPR_BEG ],
+ # [[1, 9], :on_kw, "nil", Ripper::EXPR_END ],
+ # [[1, 12], :on_sp, " ", Ripper::EXPR_END ],
+ # [[1, 13], :on_kw, "end", Ripper::EXPR_END ]]
#
def Ripper.lex(src, filename = '-', lineno = 1)
Lexer.new(src, filename, lineno).lex
end
class Lexer < ::Ripper #:nodoc: internal use only
- Elem = Struct.new(:pos, :event, :tok)
+ Elem = Struct.new(:pos, :event, :tok, :state)
def tokenize
parse().sort_by(&:pos).map(&:tok)
@@ -77,7 +78,7 @@ class Ripper
e.event = :on_ignored_sp
next
end
- ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n])]
+ ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n], e.state)]
e.pos[1] += n
end
end
@@ -93,16 +94,16 @@ class Ripper
buf = []
@buf << buf
@buf = buf
- @buf.push Elem.new([lineno(), column()], __callee__, tok)
+ @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
end
def on_heredoc_end(tok)
- @buf.push Elem.new([lineno(), column()], __callee__, tok)
+ @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
@buf = @stack.pop
end
def _push_token(tok)
- @buf.push Elem.new([lineno(), column()], __callee__, tok)
+ @buf.push Elem.new([lineno(), column()], __callee__, tok, state())
end
(SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event|