summaryrefslogtreecommitdiff
path: root/ext/ripper/lib
diff options
context:
space:
mode:
authoraamine <aamine@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2004-09-20 06:18:28 +0000
committeraamine <aamine@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2004-09-20 06:18:28 +0000
commitd781183f420b9fd144d877994cd4d4851fb520fd (patch)
treeec8aa11b81620e00564039d0c30b6abbd20b1180 /ext/ripper/lib
parentb1e943f465193fe4d834ba4b8e0e8dc142cfcaef (diff)
* ext/ripper/lib/ripper/tokenizer.rb: new file.
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6934 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'ext/ripper/lib')
-rw-r--r--ext/ripper/lib/ripper/tokenizer.rb35
1 files changed, 35 insertions, 0 deletions
diff --git a/ext/ripper/lib/ripper/tokenizer.rb b/ext/ripper/lib/ripper/tokenizer.rb
new file mode 100644
index 0000000..0bd966e
--- /dev/null
+++ b/ext/ripper/lib/ripper/tokenizer.rb
@@ -0,0 +1,35 @@
+#
+# ripper/tokenizer.rb
+#
+# Copyright (C) 2004 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper'
+
+class Ripper
+
+ def Ripper.tokenize(str)
+ Tokenizer.tokenize(str)
+ end
+
+ class Tokenizer < ::Ripper
+ def Tokenizer.tokenize(str)
+ new(str).tokenize
+ end
+
+ def tokenize
+ @tokens = []
+ parse
+ @tokens
+ end
+
+ def on__scan(type, tok)
+ @tokens.push tok
+ end
+ end
+
+end