diff options
Diffstat (limited to 'ext/ripper/lib/ripper/lexer.rb')
-rw-r--r-- | ext/ripper/lib/ripper/lexer.rb | 33 |
1 files changed, 17 insertions, 16 deletions
diff --git a/ext/ripper/lib/ripper/lexer.rb b/ext/ripper/lib/ripper/lexer.rb index 300c8eef51..4e82a1275c 100644 --- a/ext/ripper/lib/ripper/lexer.rb +++ b/ext/ripper/lib/ripper/lexer.rb @@ -23,29 +23,30 @@ class Ripper end # Tokenizes the Ruby program and returns an array of an array, - # which is formatted like <code>[[lineno, column], type, token]</code>. + # which is formatted like + # <code>[[lineno, column], type, token, state]</code>. # # require 'ripper' # require 'pp' # # pp Ripper.lex("def m(a) nil end") - # #=> [[[1, 0], :on_kw, "def"], - # [[1, 3], :on_sp, " " ], - # [[1, 4], :on_ident, "m" ], - # [[1, 5], :on_lparen, "(" ], - # [[1, 6], :on_ident, "a" ], - # [[1, 7], :on_rparen, ")" ], - # [[1, 8], :on_sp, " " ], - # [[1, 9], :on_kw, "nil"], - # [[1, 12], :on_sp, " " ], - # [[1, 13], :on_kw, "end"]] + # #=> [[[1, 0], :on_kw, "def", Ripper::EXPR_FNAME ], + # [[1, 3], :on_sp, " ", Ripper::EXPR_FNAME ], + # [[1, 4], :on_ident, "m", Ripper::EXPR_ENDFN ], + # [[1, 5], :on_lparen, "(", Ripper::EXPR_LABEL | Ripper::EXPR_BEG], + # [[1, 6], :on_ident, "a", Ripper::EXPR_ARG ], + # [[1, 7], :on_rparen, ")", Ripper::EXPR_ENDFN ], + # [[1, 8], :on_sp, " ", Ripper::EXPR_BEG ], + # [[1, 9], :on_kw, "nil", Ripper::EXPR_END ], + # [[1, 12], :on_sp, " ", Ripper::EXPR_END ], + # [[1, 13], :on_kw, "end", Ripper::EXPR_END ]] # def Ripper.lex(src, filename = '-', lineno = 1) Lexer.new(src, filename, lineno).lex end class Lexer < ::Ripper #:nodoc: internal use only - Elem = Struct.new(:pos, :event, :tok) + Elem = Struct.new(:pos, :event, :tok, :state) def tokenize parse().sort_by(&:pos).map(&:tok) @@ -77,7 +78,7 @@ class Ripper e.event = :on_ignored_sp next end - ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n])] + ignored_sp << [i, Elem.new(e.pos.dup, :on_ignored_sp, tok[0, n], e.state)] e.pos[1] += n end end @@ -93,16 +94,16 @@ class Ripper buf = [] @buf << buf @buf = buf - @buf.push Elem.new([lineno(), column()], __callee__, tok) + @buf.push Elem.new([lineno(), column()], __callee__, tok, state()) end def on_heredoc_end(tok) - @buf.push Elem.new([lineno(), column()], __callee__, tok) + @buf.push Elem.new([lineno(), column()], __callee__, tok, state()) @buf = @stack.pop end def _push_token(tok) - @buf.push Elem.new([lineno(), column()], __callee__, tok) + @buf.push Elem.new([lineno(), column()], __callee__, tok, state()) end (SCANNER_EVENTS.map {|event|:"on_#{event}"} - private_instance_methods(false)).each do |event| |