* parse.y [ripper]: do not delay dispatching.

* ext/ripper/lib/ripper/tokenizer.rb: sort tokens by right order.


git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6936 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

1 files changed, 4 insertions, 2 deletions

diff --git a/ext/ripper/lib/ripper/tokenizer.rb b/ext/ripper/lib/ripper/tokenizer.rb
index 0bd966ea51..134de9d1e0 100644
--- a/ext/ripper/lib/ripper/tokenizer.rb
+++ b/ext/ripper/lib/ripper/tokenizer.rb
@@ -24,11 +24,13 @@ class Ripper
     def tokenize
       @tokens = []
       parse
-      @tokens
+      @tokens.sort_by {|tok, pos| pos }.map {|tok,| tok }
     end
 
+    private
+
     def on__scan(type, tok)
-      @tokens.push tok
+      @tokens.push [tok, [lineno(),column()]]
     end
   end