ext/ripper/lib/ripper/tokenizer.rb


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35

#
# ripper/tokenizer.rb
#
# Copyright (C) 2004 Minero Aoki
#
# This program is free software.
# You can distribute and/or modify this program under the Ruby License.
# For details of Ruby License, see ruby/COPYING.
#

require 'ripper'

class Ripper

  def Ripper.tokenize(str)
    Tokenizer.tokenize(str)
  end

  class Tokenizer < ::Ripper
    def Tokenizer.tokenize(str)
      new(str).tokenize
    end

    def tokenize
      @tokens = []
      parse
      @tokens
    end

    def on__scan(type, tok)
      @tokens.push tok
    end
  end

end