From 2c179216717d75837de7a6ff092fd773670f4d11 Mon Sep 17 00:00:00 2001
From: aamine <aamine@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>
Date: Wed, 22 Sep 2004 05:22:50 +0000
Subject: * parse.y [ripper]: on__scan event removed. * parse.y [ripper]: event
 name is changed: on__XXX -> on_XXX. * ext/ripper/eventids2.c: ditto. *
 ext/ripper/ripper.rb.in: ditto. * ext/ripper/lib/ripper.rb: sync with
 ripper.rb.in. * ext/ripper/lib/ripper/tokenizer: ditto. *
 ext/ripper/lib/ripper/filter: new file. * sample/ripper/colorize.rb: new
 file. * sample/ripper/strip-comment.rb: new file.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6952 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
---
 ext/ripper/lib/ripper/tokenizer.rb | 89 +++++++++++++++-----------------------
 1 file changed, 34 insertions(+), 55 deletions(-)

(limited to 'ext/ripper/lib/ripper/tokenizer.rb')

diff --git a/ext/ripper/lib/ripper/tokenizer.rb b/ext/ripper/lib/ripper/tokenizer.rb
index de10d46664..4d90f16aa1 100644
--- a/ext/ripper/lib/ripper/tokenizer.rb
+++ b/ext/ripper/lib/ripper/tokenizer.rb
@@ -8,76 +8,55 @@
 # For details of Ruby License, see ruby/COPYING.
 #
 
-require 'ripper'
-
 class Ripper
 
-  def Ripper.tokenize(str)
-    Tokenizer.tokenize(str)
+  # Tokenizes Ruby program and returns an Array of String.
+  def Ripper.tokenize(src, filename = '-', lineno = 1)
+    Tokenizer.new(src, filename, lineno).tokenize
   end
 
+  # Tokenizes Ruby program and returns an Array of Array,
+  # which is formatted like [[lineno, column], type, token].
+  #
+  #   require 'ripper'
+  #   require 'pp'
+  #
+  #   p Ripper.scan("def m(a) nil end")
+  #     #=> [[[1,  0], :on_kw,     "def"],
+  #          [[1,  3], :on_sp,     " "  ],
+  #          [[1,  4], :on_ident,  "m"  ],
+  #          [[1,  5], :on_lparen, "("  ],
+  #          [[1,  6], :on_ident,  "a"  ],
+  #          [[1,  7], :on_rparen, ")"  ],
+  #          [[1,  8], :on_sp,     " "  ],
+  #          [[1,  9], :on_kw,     "nil"],
+  #          [[1, 12], :on_sp,     " "  ],
+  #          [[1, 13], :on_kw,     "end"]]
+  #
+  def Ripper.scan(src, filename = '-', lineno = 1)
+    Tokenizer.new(src, filename, lineno).parse
+  end
 
-  class Tokenizer < ::Ripper
-
-    def Tokenizer.tokenize(str, filename = '-', lineno = 1)
-      new(str, filename, lineno).tokenize
-    end
-
-    def initialize(src, filename = '-', lineno = 1)
-      @src = src
-      @__filename = filename
-      @__linestart = lineno
-      @__line = nil
-      @__col = nil
-    end
-
-    def filename
-      @__filename
-    end
-
-    def lineno
-      @__line
-    end
-
-    def column
-      @__col
-    end
-
+  class Tokenizer < ::Ripper   #:nodoc: internal use only
     def tokenize
-      _exec_tokenizer().map {|pos, event, tok| tok }
+      parse().map {|pos, event, tok| tok }
     end
 
     def parse
-      _exec_tokenizer().each do |pos, event, tok|
-        @__line, @__col = *pos
-        on__scan(event, tok)
-        __send__(event, tok)
-      end
-    end
-
-    private
-
-    def _exec_tokenizer
-      TokenSorter.new(@src, @__filename, @__linestart).parse
-    end
-
-  end
-
-
-  class TokenSorter < ::Ripper   #:nodoc: internal use only
-
-    def parse
-      @data = []
+      @buf = []
       super
-      @data.sort_by {|pos, event, tok| pos }
+      @buf.sort_by {|pos, event, tok| pos }
     end
 
     private
 
-    def on__scan(event, tok)
-      @data.push [[lineno(),column()], event, tok]
+    SCANNER_EVENTS.each do |event|
+      module_eval(<<-End)
+        def on_#{event}(tok)
+          @buf.push [[lineno(), column()], :on_#{event}, tok]
+        end
+      End
     end
-
   end
 
 end
-- 
cgit v1.2.3