From d781183f420b9fd144d877994cd4d4851fb520fd Mon Sep 17 00:00:00 2001
From: aamine <aamine@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>
Date: Mon, 20 Sep 2004 06:18:28 +0000
Subject: * ext/ripper/lib/ripper/tokenizer.rb: new file.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6934 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
---
 ext/ripper/lib/ripper/tokenizer.rb | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100644 ext/ripper/lib/ripper/tokenizer.rb

(limited to 'ext/ripper/lib')

diff --git a/ext/ripper/lib/ripper/tokenizer.rb b/ext/ripper/lib/ripper/tokenizer.rb
new file mode 100644
index 0000000000..0bd966ea51
--- /dev/null
+++ b/ext/ripper/lib/ripper/tokenizer.rb
@@ -0,0 +1,35 @@
+#
+# ripper/tokenizer.rb
+#
+# Copyright (C) 2004 Minero Aoki
+#
+# This program is free software.
+# You can distribute and/or modify this program under the Ruby License.
+# For details of Ruby License, see ruby/COPYING.
+#
+
+require 'ripper'
+
+class Ripper
+
+  def Ripper.tokenize(str)
+    Tokenizer.tokenize(str)
+  end
+
+  class Tokenizer < ::Ripper
+    def Tokenizer.tokenize(str)
+      new(str).tokenize
+    end
+
+    def tokenize
+      @tokens = []
+      parse
+      @tokens
+    end
+
+    def on__scan(type, tok)
+      @tokens.push tok
+    end
+  end
+
+end
-- 
cgit v1.2.3