diff options
author | aamine <aamine@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2004-09-22 05:22:50 +0000 |
---|---|---|
committer | aamine <aamine@b2dd03c8-39d4-4d8f-98ff-823fe69b080e> | 2004-09-22 05:22:50 +0000 |
commit | 2c179216717d75837de7a6ff092fd773670f4d11 (patch) | |
tree | 6a275ae9914fc4157c21a36402ce9d87a250fa8a /ext/ripper/lib | |
parent | 61d1dce5a226d3bf8393fd10777db09b046f68e6 (diff) |
* parse.y [ripper]: on__scan event removed.
* parse.y [ripper]: event name is changed: on__XXX -> on_XXX.
* ext/ripper/eventids2.c: ditto.
* ext/ripper/ripper.rb.in: ditto.
* ext/ripper/lib/ripper.rb: sync with ripper.rb.in.
* ext/ripper/lib/ripper/tokenizer: ditto.
* ext/ripper/lib/ripper/filter: new file.
* sample/ripper/colorize.rb: new file.
* sample/ripper/strip-comment.rb: new file.
git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6952 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
Diffstat (limited to 'ext/ripper/lib')
-rw-r--r-- | ext/ripper/lib/ripper.rb | 334 | ||||
-rw-r--r-- | ext/ripper/lib/ripper/filter.rb | 55 | ||||
-rw-r--r-- | ext/ripper/lib/ripper/tokenizer.rb | 89 |
3 files changed, 255 insertions, 223 deletions
diff --git a/ext/ripper/lib/ripper.rb b/ext/ripper/lib/ripper.rb index c0715de988..cb4924d0cd 100644 --- a/ext/ripper/lib/ripper.rb +++ b/ext/ripper/lib/ripper.rb @@ -146,7 +146,6 @@ class Ripper # This table contains name of scanner events and its arity # (arity is always 1 for all scanner events). - # on__scan is NOT a scanner event. SCANNER_EVENT_TABLE = { :CHAR => 1, :__end__ => 1, @@ -196,7 +195,7 @@ class Ripper # This array contains name of scanner events. SCANNER_EVENTS = SCANNER_EVENT_TABLE.keys - # This table contains name of all ripper events, except on__scan. + # This table contains name of all ripper events. EVENTS = PARSER_EVENTS + SCANNER_EVENTS ### ### @@ -218,483 +217,483 @@ class Ripper # Parser Events # - def on__BEGIN(a) + def on_BEGIN(a) a end - def on__END(a) + def on_END(a) a end - def on__alias(a, b) + def on_alias(a, b) a end - def on__alias_error(a) + def on_alias_error(a) a end - def on__aref(a, b) + def on_aref(a, b) a end - def on__aref_field(a, b) + def on_aref_field(a, b) a end - def on__arg_ambiguous + def on_arg_ambiguous nil end - def on__arg_paren(a) + def on_arg_paren(a) a end - def on__arglist_add(a, b) + def on_arglist_add(a, b) a end - def on__arglist_add_block(a, b) + def on_arglist_add_block(a, b) a end - def on__arglist_add_star(a, b) + def on_arglist_add_star(a, b) a end - def on__arglist_new + def on_arglist_new nil end - def on__arglist_prepend(a, b) + def on_arglist_prepend(a, b) a end - def on__array(a) + def on_array(a) a end - def on__assign(a, b) + def on_assign(a, b) a end - def on__assign_error(a) + def on_assign_error(a) a end - def on__assoc_new(a, b) + def on_assoc_new(a, b) a end - def on__assoclist_from_args(a) + def on_assoclist_from_args(a) a end - def on__bare_assoc_hash(a) + def on_bare_assoc_hash(a) a end - def on__begin(a) + def on_begin(a) a end - def on__binary(a, b, c) + def on_binary(a, b, c) a end - def on__blockvar_add_block(a, b) + def on_blockvar_add_block(a, b) a end - def on__blockvar_add_star(a, b) + def on_blockvar_add_star(a, b) a end - def on__blockvar_new(a) + def on_blockvar_new(a) a end - def on__bodystmt(a, b, c, d) + def on_bodystmt(a, b, c, d) a end - def on__brace_block(a, b) + def on_brace_block(a, b) a end - def on__break(a) + def on_break(a) a end - def on__call(a, b, c) + def on_call(a, b, c) a end - def on__case(a, b) + def on_case(a, b) a end - def on__class(a, b, c) + def on_class(a, b, c) a end - def on__class_name_error(a) + def on_class_name_error(a) a end - def on__command(a, b) + def on_command(a, b) a end - def on__command_call(a, b, c, d) + def on_command_call(a, b, c, d) a end - def on__const_ref(a) + def on_const_ref(a) a end - def on__constpath_field(a, b) + def on_constpath_field(a, b) a end - def on__constpath_ref(a, b) + def on_constpath_ref(a, b) a end - def on__def(a, b, c) + def on_def(a, b, c) a end - def on__defined(a) + def on_defined(a) a end - def on__defs(a, b, c, d, e) + def on_defs(a, b, c, d, e) a end - def on__do_block(a, b) + def on_do_block(a, b) a end - def on__dot2(a, b) + def on_dot2(a, b) a end - def on__dot3(a, b) + def on_dot3(a, b) a end - def on__dyna_symbol(a) + def on_dyna_symbol(a) a end - def on__else(a) + def on_else(a) a end - def on__elsif(a, b, c) + def on_elsif(a, b, c) a end - def on__ensure(a) + def on_ensure(a) a end - def on__fcall(a) + def on_fcall(a) a end - def on__field(a, b, c) + def on_field(a, b, c) a end - def on__for(a, b, c) + def on_for(a, b, c) a end - def on__hash(a) + def on_hash(a) a end - def on__if(a, b, c) + def on_if(a, b, c) a end - def on__if_mod(a, b) + def on_if_mod(a, b) a end - def on__ifop(a, b, c) + def on_ifop(a, b, c) a end - def on__iter_block(a, b) + def on_iter_block(a, b) a end - def on__massign(a, b) + def on_massign(a, b) a end - def on__method_add_arg(a, b) + def on_method_add_arg(a, b) a end - def on__mlhs_add(a, b) + def on_mlhs_add(a, b) a end - def on__mlhs_add_star(a, b) + def on_mlhs_add_star(a, b) a end - def on__mlhs_new + def on_mlhs_new nil end - def on__mlhs_paren(a) + def on_mlhs_paren(a) a end - def on__module(a, b) + def on_module(a, b) a end - def on__mrhs_add(a, b) + def on_mrhs_add(a, b) a end - def on__mrhs_add_star(a, b) + def on_mrhs_add_star(a, b) a end - def on__mrhs_new + def on_mrhs_new nil end - def on__mrhs_new_from_arglist(a) + def on_mrhs_new_from_arglist(a) a end - def on__next(a) + def on_next(a) a end - def on__opassign(a, b, c) + def on_opassign(a, b, c) a end - def on__param_error(a) + def on_param_error(a) a end - def on__params(a, b, c, d) + def on_params(a, b, c, d) a end - def on__paren(a) + def on_paren(a) a end - def on__parse_error(a) + def on_parse_error(a) a end - def on__program(a) + def on_program(a) a end - def on__qwords_add(a, b) + def on_qwords_add(a, b) a end - def on__qwords_new + def on_qwords_new nil end - def on__redo + def on_redo nil end - def on__regexp_literal(a) + def on_regexp_literal(a) a end - def on__rescue(a, b, c, d) + def on_rescue(a, b, c, d) a end - def on__rescue_mod(a, b) + def on_rescue_mod(a, b) a end - def on__restparam(a) + def on_restparam(a) a end - def on__retry + def on_retry nil end - def on__return(a) + def on_return(a) a end - def on__return0 + def on_return0 nil end - def on__sclass(a, b) + def on_sclass(a, b) a end - def on__space(a) + def on_space(a) a end - def on__stmts_add(a, b) + def on_stmts_add(a, b) a end - def on__stmts_new + def on_stmts_new nil end - def on__string_add(a, b) + def on_string_add(a, b) a end - def on__string_concat(a, b) + def on_string_concat(a, b) a end - def on__string_content + def on_string_content nil end - def on__string_dvar(a) + def on_string_dvar(a) a end - def on__string_embexpr(a) + def on_string_embexpr(a) a end - def on__string_literal(a) + def on_string_literal(a) a end - def on__super(a) + def on_super(a) a end - def on__symbol(a) + def on_symbol(a) a end - def on__symbol_literal(a) + def on_symbol_literal(a) a end - def on__topconst_field(a) + def on_topconst_field(a) a end - def on__topconst_ref(a) + def on_topconst_ref(a) a end - def on__unary(a, b) + def on_unary(a, b) a end - def on__undef(a) + def on_undef(a) a end - def on__unless(a, b, c) + def on_unless(a, b, c) a end - def on__unless_mod(a, b) + def on_unless_mod(a, b) a end - def on__until(a, b) + def on_until(a, b) a end - def on__until_mod(a, b) + def on_until_mod(a, b) a end - def on__var_alias(a, b) + def on_var_alias(a, b) a end - def on__var_field(a) + def on_var_field(a) a end - def on__var_ref(a) + def on_var_ref(a) a end - def on__void_stmt + def on_void_stmt nil end - def on__when(a, b, c) + def on_when(a, b, c) a end - def on__while(a, b) + def on_while(a, b) a end - def on__while_mod(a, b) + def on_while_mod(a, b) a end - def on__word_add(a, b) + def on_word_add(a, b) a end - def on__word_new + def on_word_new nil end - def on__words_add(a, b) + def on_words_add(a, b) a end - def on__words_new + def on_words_new nil end - def on__xstring_add(a, b) + def on_xstring_add(a, b) a end - def on__xstring_literal(a) + def on_xstring_literal(a) a end - def on__xstring_new + def on_xstring_new nil end - def on__yield(a) + def on_yield(a) a end - def on__yield0 + def on_yield0 nil end - def on__zsuper + def on_zsuper nil end @@ -702,178 +701,177 @@ class Ripper # Lexer Events # - def on__scan(event, token) - end - - def on__CHAR(token) + def on_CHAR(token) token end - def on____end__(token) + def on___end__(token) token end - def on__backref(token) + def on_backref(token) token end - def on__backtick(token) + def on_backtick(token) token end - def on__comma(token) + def on_comma(token) token end - def on__comment(token) + def on_comment(token) token end - def on__const(token) + def on_const(token) token end - def on__cvar(token) + def on_cvar(token) token end - def on__embdoc(token) + def on_embdoc(token) token end - def on__embdoc_beg(token) + def on_embdoc_beg(token) token end - def on__embdoc_end(token) + def on_embdoc_end(token) token end - def on__embexpr_beg(token) + def on_embexpr_beg(token) token end - def on__embexpr_end(token) + def on_embexpr_end(token) token end - def on__embvar(token) + def on_embvar(token) token end - def on__float(token) + def on_float(token) token end - def on__gvar(token) + def on_gvar(token) token end - def on__heredoc_beg(token) + def on_heredoc_beg(token) token end - def on__heredoc_end(token) + def on_heredoc_end(token) token end - def on__ident(token) + def on_ident(token) token end - def on__ignored_nl(token) + def on_ignored_nl(token) token end - def on__int(token) + def on_int(token) token end - def on__ivar(token) + def on_ivar(token) token end - def on__kw(token) + def on_kw(token) token end - def on__lbrace(token) + def on_lbrace(token) token end - def on__lbracket(token) + def on_lbracket(token) token end - def on__lparen(token) + def on_lparen(token) token end - def on__nl(token) + def on_nl(token) token end - def on__op(token) + def on_op(token) token end - def on__period(token) + def on_period(token) token end - def on__qwords_beg(token) + def on_qwords_beg(token) token end - def on__rbrace(token) + def on_rbrace(token) token end - def on__rbracket(token) + def on_rbracket(token) token end - def on__regexp_beg(token) + def on_regexp_beg(token) token end - def on__regexp_end(token) + def on_regexp_end(token) token end - def on__rparen(token) + def on_rparen(token) token end - def on__semicolon(token) + def on_semicolon(token) token end - def on__sp(token) + def on_sp(token) token end - def on__symbeg(token) + def on_symbeg(token) token end - def on__tstring_beg(token) + def on_tstring_beg(token) token end - def on__tstring_content(token) + def on_tstring_content(token) token end - def on__tstring_end(token) + def on_tstring_end(token) token end - def on__words_beg(token) + def on_words_beg(token) token end - def on__words_sep(token) + def on_words_sep(token) token end end + +require 'ripper/tokenizer' diff --git a/ext/ripper/lib/ripper/filter.rb b/ext/ripper/lib/ripper/filter.rb new file mode 100644 index 0000000000..399c4c64bc --- /dev/null +++ b/ext/ripper/lib/ripper/filter.rb @@ -0,0 +1,55 @@ +# +# ripper/filter.rb +# +# Copyright (C) 2004 Minero Aoki +# +# This program is free software. +# You can distribute and/or modify this program under the Ruby License. +# For details of Ruby License, see ruby/COPYING. +# + +require 'ripper' + +class Ripper + + class Filter + + def initialize(src, filename = '-', lineno = 1) + @__parser = Tokenizer.new(src, filename, lineno) + @__line = nil + @__col = nil + end + + def filename + @__parser.filename + end + + def lineno + @__line + end + + def column + @__col + end + + def parse(init) + data = init + @__parser.parse.each do |pos, event, tok| + @__line, @__col = *pos + data = if respond_to?(event, true) + then __send__(event, tok, data) + else on_default(event, tok, data) + end + end + data + end + + private + + def on_default(event, tok, data) + data + end + + end + +end diff --git a/ext/ripper/lib/ripper/tokenizer.rb b/ext/ripper/lib/ripper/tokenizer.rb index de10d46664..4d90f16aa1 100644 --- a/ext/ripper/lib/ripper/tokenizer.rb +++ b/ext/ripper/lib/ripper/tokenizer.rb @@ -8,76 +8,55 @@ # For details of Ruby License, see ruby/COPYING. # -require 'ripper' - class Ripper - def Ripper.tokenize(str) - Tokenizer.tokenize(str) + # Tokenizes Ruby program and returns an Array of String. + def Ripper.tokenize(src, filename = '-', lineno = 1) + Tokenizer.new(src, filename, lineno).tokenize end + # Tokenizes Ruby program and returns an Array of Array, + # which is formatted like [[lineno, column], type, token]. + # + # require 'ripper' + # require 'pp' + # + # p Ripper.scan("def m(a) nil end") + # #=> [[[1, 0], :on_kw, "def"], + # [[1, 3], :on_sp, " " ], + # [[1, 4], :on_ident, "m" ], + # [[1, 5], :on_lparen, "(" ], + # [[1, 6], :on_ident, "a" ], + # [[1, 7], :on_rparen, ")" ], + # [[1, 8], :on_sp, " " ], + # [[1, 9], :on_kw, "nil"], + # [[1, 12], :on_sp, " " ], + # [[1, 13], :on_kw, "end"]] + # + def Ripper.scan(src, filename = '-', lineno = 1) + Tokenizer.new(src, filename, lineno).parse + end - class Tokenizer < ::Ripper - - def Tokenizer.tokenize(str, filename = '-', lineno = 1) - new(str, filename, lineno).tokenize - end - - def initialize(src, filename = '-', lineno = 1) - @src = src - @__filename = filename - @__linestart = lineno - @__line = nil - @__col = nil - end - - def filename - @__filename - end - - def lineno - @__line - end - - def column - @__col - end - + class Tokenizer < ::Ripper #:nodoc: internal use only def tokenize - _exec_tokenizer().map {|pos, event, tok| tok } + parse().map {|pos, event, tok| tok } end def parse - _exec_tokenizer().each do |pos, event, tok| - @__line, @__col = *pos - on__scan(event, tok) - __send__(event, tok) - end - end - - private - - def _exec_tokenizer - TokenSorter.new(@src, @__filename, @__linestart).parse - end - - end - - - class TokenSorter < ::Ripper #:nodoc: internal use only - - def parse - @data = [] + @buf = [] super - @data.sort_by {|pos, event, tok| pos } + @buf.sort_by {|pos, event, tok| pos } end private - def on__scan(event, tok) - @data.push [[lineno(),column()], event, tok] + SCANNER_EVENTS.each do |event| + module_eval(<<-End) + def on_#{event}(tok) + @buf.push [[lineno(), column()], :on_#{event}, tok] + end + End end - end end |