diff options
Diffstat (limited to 'trunk/ext/ripper/lib/ripper')
| -rw-r--r-- | trunk/ext/ripper/lib/ripper/core.rb | 70 | ||||
| -rw-r--r-- | trunk/ext/ripper/lib/ripper/filter.rb | 70 | ||||
| -rw-r--r-- | trunk/ext/ripper/lib/ripper/lexer.rb | 179 | ||||
| -rw-r--r-- | trunk/ext/ripper/lib/ripper/sexp.rb | 99 |
4 files changed, 418 insertions, 0 deletions
diff --git a/trunk/ext/ripper/lib/ripper/core.rb b/trunk/ext/ripper/lib/ripper/core.rb new file mode 100644 index 0000000000..35aa54d090 --- /dev/null +++ b/trunk/ext/ripper/lib/ripper/core.rb @@ -0,0 +1,70 @@ +# +# $Id$ +# +# Copyright (c) 2003-2005 Minero Aoki +# +# This program is free software. +# You can distribute and/or modify this program under the Ruby License. +# For details of Ruby License, see ruby/COPYING. +# + +require 'ripper.so' + +class Ripper + + # Parses Ruby program read from _src_. + # _src_ must be a String or a IO or a object which has #gets method. + def Ripper.parse(src, filename = '(ripper)', lineno = 1) + new(src, filename, lineno).parse + end + + # This array contains name of parser events. + PARSER_EVENTS = PARSER_EVENT_TABLE.keys + + # This array contains name of scanner events. + SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys + + # This array contains name of all ripper events. + EVENTS = PARSER_EVENTS + SCANNER_EVENTS + + private + + # + # Parser Events + # + + PARSER_EVENT_TABLE.each do |id, arity| + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{id}(#{ ('a'..'z').to_a[0, arity].join(', ') }) + #{arity == 0 ? 'nil' : 'a'} + end + End + end + + # This method is called when weak warning is produced by the parser. + # _fmt_ and _args_ is printf style. + def warn(fmt, *args) + end + + # This method is called when strong warning is produced by the parser. + # _fmt_ and _args_ is printf style. + def warning(fmt, *args) + end + + # This method is called when the parser found syntax error. + def compile_error(msg) + end + + # + # Scanner Events + # + + SCANNER_EVENTS.each do |id| + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{id}(token) + token + end + End + end + +end diff --git a/trunk/ext/ripper/lib/ripper/filter.rb b/trunk/ext/ripper/lib/ripper/filter.rb new file mode 100644 index 0000000000..898501b23c --- /dev/null +++ b/trunk/ext/ripper/lib/ripper/filter.rb @@ -0,0 +1,70 @@ +# +# $Id$ +# +# Copyright (c) 2004,2005 Minero Aoki +# +# This program is free software. +# You can distribute and/or modify this program under the Ruby License. +# For details of Ruby License, see ruby/COPYING. +# + +require 'ripper/lexer' + +class Ripper + + # This class handles only scanner events, + # and they are dispatched in the `right' order (same with input). + class Filter + + def initialize(src, filename = '-', lineno = 1) + @__lexer = Lexer.new(src, filename, lineno) + @__line = nil + @__col = nil + end + + # The file name of the input. + def filename + @__lexer.filename + end + + # The line number of the current token. + # This value starts from 1. + # This method is valid only in event handlers. + def lineno + @__line + end + + # The column number of the current token. + # This value starts from 0. + # This method is valid only in event handlers. + def column + @__col + end + + # Starts parsing. _init_ is a data accumulator. + # It is passed to the next event handler (as of Enumerable#inject). + def parse(init = nil) + data = init + @__lexer.lex.each do |pos, event, tok| + @__line, @__col = *pos + data = if respond_to?(event, true) + then __send__(event, tok, data) + else on_default(event, tok, data) + end + end + data + end + + private + + # This method is called when some event handler have not defined. + # _event_ is :on_XXX, _token_ is scanned token, _data_ is a data + # accumulator. The return value of this method is passed to the + # next event handler (as of Enumerable#inject). + def on_default(event, token, data) + data + end + + end + +end diff --git a/trunk/ext/ripper/lib/ripper/lexer.rb b/trunk/ext/ripper/lib/ripper/lexer.rb new file mode 100644 index 0000000000..14ef99f034 --- /dev/null +++ b/trunk/ext/ripper/lib/ripper/lexer.rb @@ -0,0 +1,179 @@ +# +# $Id$ +# +# Copyright (c) 2004,2005 Minero Aoki +# +# This program is free software. +# You can distribute and/or modify this program under the Ruby License. +# For details of Ruby License, see ruby/COPYING. +# + +require 'ripper/core' + +class Ripper + + # Tokenizes Ruby program and returns an Array of String. + def Ripper.tokenize(src, filename = '-', lineno = 1) + Lexer.new(src, filename, lineno).tokenize + end + + # Tokenizes Ruby program and returns an Array of Array, + # which is formatted like [[lineno, column], type, token]. + # + # require 'ripper' + # require 'pp' + # + # p Ripper.lex("def m(a) nil end") + # #=> [[[1, 0], :on_kw, "def"], + # [[1, 3], :on_sp, " " ], + # [[1, 4], :on_ident, "m" ], + # [[1, 5], :on_lparen, "(" ], + # [[1, 6], :on_ident, "a" ], + # [[1, 7], :on_rparen, ")" ], + # [[1, 8], :on_sp, " " ], + # [[1, 9], :on_kw, "nil"], + # [[1, 12], :on_sp, " " ], + # [[1, 13], :on_kw, "end"]] + # + def Ripper.lex(src, filename = '-', lineno = 1) + Lexer.new(src, filename, lineno).lex + end + + class Lexer < ::Ripper #:nodoc: internal use only + def tokenize + lex().map {|pos, event, tok| tok } + end + + def lex + parse().sort_by {|pos, event, tok| pos } + end + + def parse + @buf = [] + super + @buf + end + + private + + SCANNER_EVENTS.each do |event| + module_eval(<<-End, __FILE__+'/module_eval', __LINE__ + 1) + def on_#{event}(tok) + @buf.push [[lineno(), column()], :on_#{event}, tok] + end + End + end + end + + # [EXPERIMENTAL] + # Parses +src+ and return a string which was matched to +pattern+. + # +pattern+ should be described as Regexp. + # + # require 'ripper' + # + # p Ripper.slice('def m(a) nil end', 'ident') #=> "m" + # p Ripper.slice('def m(a) nil end', '[ident lparen rparen]+') #=> "m(a)" + # p Ripper.slice("<<EOS\nstring\nEOS", + # 'heredoc_beg nl $(tstring_content*) heredoc_end', 1) + # #=> "string\n" + # + def Ripper.slice(src, pattern, n = 0) + if m = token_match(src, pattern) + then m.string(n) + else nil + end + end + + def Ripper.token_match(src, pattern) #:nodoc: + TokenPattern.compile(pattern).match(src) + end + + class TokenPattern #:nodoc: + + class Error < ::StandardError; end + class CompileError < Error; end + class MatchError < Error; end + + class << self + alias compile new + end + + def initialize(pattern) + @source = pattern + @re = compile(pattern) + end + + def match(str) + match_list(::Ripper.lex(str)) + end + + def match_list(tokens) + if m = @re.match(map_tokens(tokens)) + then MatchData.new(tokens, m) + else nil + end + end + + private + + def compile(pattern) + if m = /[^\w\s$()\[\]{}?*+\.]/.match(pattern) + raise CompileError, "invalid char in pattern: #{m[0].inspect}" + end + buf = '' + pattern.scan(/(?:\w+|\$\(|[()\[\]\{\}?*+\.]+)/) do |tok| + case tok + when /\w/ + buf.concat map_token(tok) + when '$(' + buf.concat '(' + when '(' + buf.concat '(?:' + when /[?*\[\])\.]/ + buf.concat tok + else + raise 'must not happen' + end + end + Regexp.compile(buf) + rescue RegexpError => err + raise CompileError, err.message + end + + def map_tokens(tokens) + tokens.map {|pos,type,str| map_token(type.to_s.sub(/\Aon_/,'')) }.join + end + + MAP = {} + seed = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a + SCANNER_EVENT_TABLE.each do |ev, | + raise CompileError, "[RIPPER FATAL] too many system token" if seed.empty? + MAP[ev.to_s.sub(/\Aon_/,'')] = seed.shift + end + + def map_token(tok) + MAP[tok] or raise CompileError, "unknown token: #{tok}" + end + + class MatchData + def initialize(tokens, match) + @tokens = tokens + @match = match + end + + def string(n = 0) + return nil unless @match + match(n).join + end + + private + + def match(n = 0) + return [] unless @match + @tokens[@match.begin(n)...@match.end(n)].map {|pos,type,str| str } + end + end + + end + +end diff --git a/trunk/ext/ripper/lib/ripper/sexp.rb b/trunk/ext/ripper/lib/ripper/sexp.rb new file mode 100644 index 0000000000..f2260fe8dd --- /dev/null +++ b/trunk/ext/ripper/lib/ripper/sexp.rb @@ -0,0 +1,99 @@ +# +# $Id$ +# +# Copyright (c) 2004,2005 Minero Aoki +# +# This program is free software. +# You can distribute and/or modify this program under the Ruby License. +# For details of Ruby License, see ruby/COPYING. +# + +require 'ripper/core' + +class Ripper + + # [EXPERIMENTAL] + # Parses +src+ and create S-exp tree. + # This method is for mainly developper use. + # + # require 'ripper' + # require 'pp + # + # pp Ripper.sexp("def m(a) nil end") + # #=> [:program, + # [:stmts_add, + # [:stmts_new], + # [:def, + # [:@ident, "m", [1, 4]], + # [:paren, [:params, [[:@ident, "a", [1, 6]]], nil, nil, nil]], + # [:bodystmt, + # [:stmts_add, [:stmts_new], [:var_ref, [:@kw, "nil", [1, 9]]]], + # nil, + # nil, + # nil]]]] + # + def Ripper.sexp(src, filename = '-', lineno = 1) + SexpBuilderPP.new(src, filename, lineno).parse + end + + def Ripper.sexp_raw(src, filename = '-', lineno = 1) + SexpBuilder.new(src, filename, lineno).parse + end + + class SexpBuilderPP < ::Ripper #:nodoc: + private + + PARSER_EVENT_TABLE.each do |event, arity| + if /_new\z/ =~ event.to_s and arity == 0 + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{event} + [] + end + End + elsif /_add\z/ =~ event.to_s + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{event}(list, item) + list.push item + list + end + End + else + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{event}(*args) + [:#{event}, *args] + end + End + end + end + + SCANNER_EVENTS.each do |event| + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{event}(tok) + [:@#{event}, tok, [lineno(), column()]] + end + End + end + end + + class SexpBuilder < ::Ripper #:nodoc: + private + + PARSER_EVENTS.each do |event| + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{event}(*args) + args.unshift :#{event} + args + end + End + end + + SCANNER_EVENTS.each do |event| + module_eval(<<-End, __FILE__, __LINE__ + 1) + def on_#{event}(tok) + [:@#{event}, tok, [lineno(), column()]] + end + End + end + end + +end |