diff options
25 files changed, 3911 insertions, 0 deletions
@@ -1,3 +1,55 @@ +Mon Sep 13 02:26:31 2004 Minero Aoki <aamine@loveruby.net> + + * ext/ripper: ripper extention added. + + * ext/ripper/MANIFEST: new file. + + * ext/ripper/README: new file. + + * ext/ripper/depend: new file. + + * ext/ripper/extconf.rb: new file. + + * ext/ripper/eventids2.c: new file. + + * ext/ripper/ripper.rb.in: new file. + + * ext/ripper/lib/ripper.rb: new file. + + * ext/ripper/test/check-event-arity.rb: new file. + + * ext/ripper/test/check-event-coverage.sh: new file. + + * ext/ripper/test/check-scanner-event-coverage.rb: new file. + + * ext/ripper/test/list-called-events.rb: new file. + + * ext/ripper/test/src_rb: new file. + + * ext/ripper/test/validate.rb: new file. + + * ext/ripper/tools/generate-eventids1.rb: new file. + + * ext/ripper/tools/generate-param-macros.rb: new file. + + * ext/ripper/tools/generate-ripper_rb.rb: new file. + + * ext/ripper/tools/list-parse-event-ids.rb: new file. + + * ext/ripper/tools/list-scan-event-ids.rb: new file. + + * ext/ripper/tools/preproc.rb: new file. + + * ext/ripper/tools/strip.rb: new file. + + * test/ripper: ripper tests added. + + * test/ripper/dummyparser.rb: new file. + + * test/ripper/test_parser_events.rb: new file. + + * test/ripper/test_scanner_events.rb: new file. + Mon Sep 13 01:03:02 2004 Hidetoshi NAGAI <nagai@ai.kyutech.ac.jp> * ext/tcltklib/tcltklib.c: improve control of preserv/release tcltkip @@ -878,6 +878,9 @@ test/optparse/test_placearg.rb test/optparse/test_reqarg.rb test/ostruct/test_ostruct.rb test/rinda/test_rinda.rb +test/ripper/dummyparser.rb +test/ripper/test_parser_events.rb +test/ripper/test_scanner_events.rb test/rss/rss-assertions.rb test/rss/rss-testcase.rb test/rss/test_1.0.rb diff --git a/ext/ripper/MANIFEST b/ext/ripper/MANIFEST new file mode 100644 index 0000000000..fe1e25e620 --- /dev/null +++ b/ext/ripper/MANIFEST @@ -0,0 +1,20 @@ +MANIFEST +README +depend +eventids2.c +extconf.rb +ripper.rb.in +lib/ripper.rb +test/check-event-arity.rb +test/check-event-coverage.sh +test/check-scanner-event-coverage.rb +test/list-called-events.rb +test/src.rb +test/validate.rb +tools/generate-eventids1.rb +tools/generate-param-macros.rb +tools/generate-ripper_rb.rb +tools/list-parse-event-ids.rb +tools/list-scan-event-ids.rb +tools/preproc.rb +tools/strip.rb diff --git a/ext/ripper/README b/ext/ripper/README new file mode 100644 index 0000000000..5242c4256d --- /dev/null +++ b/ext/ripper/README @@ -0,0 +1,31 @@ +Ripper README +============= + + Ripper is a Ruby script parser. You can get information + by event-based style from the parser. + + !! WARNING !! + + Ripper is still early-alpha version. + I never assure any kind of backward compatibility. + +Requirements +------------ + + * ruby 1.9 (support CVS HEAD only) + * bison 1.28 or later (You MUST use bison) + +Usage +----- + + See test/ripper/* and sample/ripper/*. + To know thrown events, see lib/ripper.rb. + +License +------- + + Ruby License. + + Minero Aoki + aamine@loveruby.net + http://i.loveruby.net diff --git a/ext/ripper/depend b/ext/ripper/depend new file mode 100644 index 0000000000..b2333a2fdf --- /dev/null +++ b/ext/ripper/depend @@ -0,0 +1,37 @@ +ripper.o: ripper.c lex.c eventids1.c eventids2.c + $(CC) $(CFLAGS) $(CPPFLAGS) -c ripper.c + +ripper.c: ripper.y + bison -t -v -o$@ ripper.y + +ripper.y: $(hdrdir)/parse.y $(srcdir)/tools/preproc.rb + $(RUBY) $(srcdir)/tools/preproc.rb $(hdrdir)/parse.y > $@ + +eventids1.c: $(hdrdir)/parse.y $(srcdir)/tools/list-parse-event-ids.rb $(srcdir)/tools/generate-eventids1.rb + $(RUBY) $(srcdir)/tools/list-parse-event-ids.rb $(hdrdir)/parse.y | $(RUBY) $(srcdir)/tools/generate-eventids1.rb > $@ + +lex.c: + cp $(hdrdir)/lex.c . || cp $(srcdir)/lex.c . + +# +# Entries for ripper developpers (no srcdir/objdir support) +# +lib/ripper.rb: ripper.rb.in ids1 ids2 tools/generate-ripper_rb.rb + $(RUBY) tools/generate-ripper_rb.rb ripper.rb.in ids1 ids2 > $@ + +ids1: tools/list-parse-event-ids.rb ../../parse.y + $(RUBY) tools/list-parse-event-ids.rb -a ../../parse.y > $@ + +ids2: $(srcdir)/tools/list-scan-event-ids.rb eventids2.c + $(RUBY) tools/list-scan-event-ids.rb -a eventids2.c > $@ + +test: ripper.so test/*.* + $(RUBY) test/check-event-arity.rb $(hdrdir)/parse.y + sh test/check-event-coverage.sh + $(RUBY) test/check-scanner-event-coverage.rb + $(RUBY) test/validate.rb > /dev/null + @echo OK + +preproc: ripper.E +ripper.E: ripper.c + $(CC) -E $(CPPFLAGS) ripper.c | ruby tools/strip.rb > $@ diff --git a/ext/ripper/eventids2.c b/ext/ripper/eventids2.c new file mode 100644 index 0000000000..28519eb8e6 --- /dev/null +++ b/ext/ripper/eventids2.c @@ -0,0 +1,272 @@ +#define tIGNORED_NL (tLAST_TOKEN + 1) +#define tCOMMENT (tLAST_TOKEN + 2) +#define tEMBDOC_BEG (tLAST_TOKEN + 3) +#define tEMBDOC (tLAST_TOKEN + 4) +#define tEMBDOC_END (tLAST_TOKEN + 5) +#define tLWSP (tLAST_TOKEN + 6) +#define tHEREDOC_BEG (tLAST_TOKEN + 7) +#define tHEREDOC_CONTENT (tLAST_TOKEN + 8) +#define tHEREDOC_END (tLAST_TOKEN + 9) +#define k__END__ (tLAST_TOKEN + 10) +#define tCHAR (tLAST_TOKEN + 11) + +static ID ripper_id_scan; + +static ID ripper_id_backref; +static ID ripper_id_backtick; +static ID ripper_id_comma; +static ID ripper_id_const; +static ID ripper_id_cvar; +static ID ripper_id_embexpr_beg; +static ID ripper_id_embexpr_end; +static ID ripper_id_embvar; +static ID ripper_id_float; +static ID ripper_id_gvar; +static ID ripper_id_ident; +static ID ripper_id_int; +static ID ripper_id_ivar; +static ID ripper_id_kw; +static ID ripper_id_lbrace; +static ID ripper_id_lbracket; +static ID ripper_id_lparen; +static ID ripper_id_nl; +static ID ripper_id_op; +static ID ripper_id_period; +static ID ripper_id_rbrace; +static ID ripper_id_rbracket; +static ID ripper_id_rparen; +static ID ripper_id_semicolon; +static ID ripper_id_symbeg; +static ID ripper_id_tstring_beg; +static ID ripper_id_tstring_content; +static ID ripper_id_tstring_end; +static ID ripper_id_words_beg; +static ID ripper_id_qwords_beg; +static ID ripper_id_words_sep; +static ID ripper_id_regexp_beg; +static ID ripper_id_regexp_end; + +static ID ripper_id_ignored_nl; +static ID ripper_id_comment; +static ID ripper_id_embdoc_beg; +static ID ripper_id_embdoc; +static ID ripper_id_embdoc_end; +static ID ripper_id_lwsp; +static ID ripper_id_heredoc_beg; +static ID ripper_id_heredoc_content; +static ID ripper_id_heredoc_end; +static ID ripper_id___end__; +static ID ripper_id_CHAR; + +static void +ripper_init_eventids2() +{ + ripper_id_scan = rb_intern("on__scan"); + + ripper_id_backref = rb_intern("on__backref"); + ripper_id_backtick = rb_intern("on__backtick"); + ripper_id_comma = rb_intern("on__comma"); + ripper_id_const = rb_intern("on__const"); + ripper_id_cvar = rb_intern("on__cvar"); + ripper_id_embexpr_beg = rb_intern("on__embexpr_beg"); + ripper_id_embexpr_end = rb_intern("on__embexpr_end"); + ripper_id_embvar = rb_intern("on__embvar"); + ripper_id_float = rb_intern("on__float"); + ripper_id_gvar = rb_intern("on__gvar"); + ripper_id_ident = rb_intern("on__ident"); + ripper_id_int = rb_intern("on__int"); + ripper_id_ivar = rb_intern("on__ivar"); + ripper_id_kw = rb_intern("on__kw"); + ripper_id_lbrace = rb_intern("on__lbrace"); + ripper_id_lbracket = rb_intern("on__lbracket"); + ripper_id_lparen = rb_intern("on__lparen"); + ripper_id_nl = rb_intern("on__nl"); + ripper_id_op = rb_intern("on__op"); + ripper_id_period = rb_intern("on__period"); + ripper_id_rbrace = rb_intern("on__rbrace"); + ripper_id_rbracket = rb_intern("on__rbracket"); + ripper_id_rparen = rb_intern("on__rparen"); + ripper_id_semicolon = rb_intern("on__semicolon"); + ripper_id_symbeg = rb_intern("on__symbeg"); + ripper_id_tstring_beg = rb_intern("on__tstring_beg"); + ripper_id_tstring_content = rb_intern("on__tstring_content"); + ripper_id_tstring_end = rb_intern("on__tstring_end"); + ripper_id_words_beg = rb_intern("on__words_beg"); + ripper_id_qwords_beg = rb_intern("on__qwords_beg"); + ripper_id_words_sep = rb_intern("on__words_sep"); + ripper_id_regexp_beg = rb_intern("on__regexp_beg"); + ripper_id_regexp_end = rb_intern("on__regexp_end"); + + ripper_id_ignored_nl = rb_intern("on__ignored_nl"); + ripper_id_comment = rb_intern("on__comment"); + ripper_id_embdoc_beg = rb_intern("on__embdoc_beg"); + ripper_id_embdoc = rb_intern("on__embdoc"); + ripper_id_embdoc_end = rb_intern("on__embdoc_end"); + ripper_id_lwsp = rb_intern("on__sp"); + ripper_id_heredoc_beg = rb_intern("on__heredoc_beg"); + ripper_id_heredoc_content = rb_intern("on__heredoc_content"); + ripper_id_heredoc_end = rb_intern("on__heredoc_end"); + ripper_id___end__ = rb_intern("on____end__"); + ripper_id_CHAR = rb_intern("on__CHAR"); +} + +static struct token_assoc { + int token; + ID *id; +} token_to_eventid[] = { + {' ', &ripper_id_words_sep}, + {'!', &ripper_id_op}, + {'%', &ripper_id_op}, + {'&', &ripper_id_op}, + {'*', &ripper_id_op}, + {'+', &ripper_id_op}, + {'-', &ripper_id_op}, + {'/', &ripper_id_op}, + {'<', &ripper_id_op}, + {'=', &ripper_id_op}, + {'>', &ripper_id_op}, + {'?', &ripper_id_op}, + {'^', &ripper_id_op}, + {'|', &ripper_id_op}, + {'~', &ripper_id_op}, + {':', &ripper_id_op}, + {',', &ripper_id_comma}, + {'.', &ripper_id_period}, + {';', &ripper_id_semicolon}, + {'`', &ripper_id_backtick}, + {'\n', &ripper_id_nl}, + {kALIAS, &ripper_id_kw}, + {kAND, &ripper_id_kw}, + {kBEGIN, &ripper_id_kw}, + {kBREAK, &ripper_id_kw}, + {kCASE, &ripper_id_kw}, + {kCLASS, &ripper_id_kw}, + {kDEF, &ripper_id_kw}, + {kDEFINED, &ripper_id_kw}, + {kDO, &ripper_id_kw}, + {kDO_BLOCK, &ripper_id_kw}, + {kDO_COND, &ripper_id_kw}, + {kELSE, &ripper_id_kw}, + {kELSIF, &ripper_id_kw}, + {kEND, &ripper_id_kw}, + {kENSURE, &ripper_id_kw}, + {kFALSE, &ripper_id_kw}, + {kFOR, &ripper_id_kw}, + {kIF, &ripper_id_kw}, + {kIF_MOD, &ripper_id_kw}, + {kIN, &ripper_id_kw}, + {kMODULE, &ripper_id_kw}, + {kNEXT, &ripper_id_kw}, + {kNIL, &ripper_id_kw}, + {kNOT, &ripper_id_kw}, + {kOR, &ripper_id_kw}, + {kREDO, &ripper_id_kw}, + {kRESCUE, &ripper_id_kw}, + {kRESCUE_MOD, &ripper_id_kw}, + {kRETRY, &ripper_id_kw}, + {kRETURN, &ripper_id_kw}, + {kSELF, &ripper_id_kw}, + {kSUPER, &ripper_id_kw}, + {kTHEN, &ripper_id_kw}, + {kTRUE, &ripper_id_kw}, + {kUNDEF, &ripper_id_kw}, + {kUNLESS, &ripper_id_kw}, + {kUNLESS_MOD, &ripper_id_kw}, + {kUNTIL, &ripper_id_kw}, + {kUNTIL_MOD, &ripper_id_kw}, + {kWHEN, &ripper_id_kw}, + {kWHILE, &ripper_id_kw}, + {kWHILE_MOD, &ripper_id_kw}, + {kYIELD, &ripper_id_kw}, + {k__FILE__, &ripper_id_kw}, + {k__LINE__, &ripper_id_kw}, + {klBEGIN, &ripper_id_kw}, + {klEND, &ripper_id_kw}, + {tAMPER, &ripper_id_op}, + {tANDOP, &ripper_id_op}, + {tAREF, &ripper_id_op}, + {tASET, &ripper_id_op}, + {tASSOC, &ripper_id_op}, + {tBACK_REF, &ripper_id_backref}, + {tCMP, &ripper_id_op}, + {tCOLON2, &ripper_id_op}, + {tCOLON3, &ripper_id_op}, + {tCONSTANT, &ripper_id_const}, + {tCVAR, &ripper_id_cvar}, + {tDOT2, &ripper_id_op}, + {tDOT3, &ripper_id_op}, + {tEQ, &ripper_id_op}, + {tEQQ, &ripper_id_op}, + {tFID, &ripper_id_ident}, + {tFLOAT, &ripper_id_float}, + {tGEQ, &ripper_id_op}, + {tGVAR, &ripper_id_gvar}, + {tIDENTIFIER, &ripper_id_ident}, + {tINTEGER, &ripper_id_int}, + {tIVAR, &ripper_id_ivar}, + {tLBRACE, &ripper_id_lbrace}, + {tLBRACE_ARG, &ripper_id_lbrace}, + {'{', &ripper_id_lbrace}, + {'}', &ripper_id_rbrace}, + {tLBRACK, &ripper_id_lbracket}, + {'[', &ripper_id_lbracket}, + {']', &ripper_id_rbracket}, + {tLEQ, &ripper_id_op}, + {tLPAREN, &ripper_id_lparen}, + {tLPAREN_ARG, &ripper_id_lparen}, + {'(', &ripper_id_lparen}, + {')', &ripper_id_rparen}, + {tLSHFT, &ripper_id_op}, + {tMATCH, &ripper_id_op}, + {tNEQ, &ripper_id_op}, + {tNMATCH, &ripper_id_op}, + {tNTH_REF, &ripper_id_backref}, + {tOP_ASGN, &ripper_id_op}, + {tOROP, &ripper_id_op}, + {tPOW, &ripper_id_op}, + {tQWORDS_BEG, &ripper_id_qwords_beg}, + {tREGEXP_BEG, &ripper_id_regexp_beg}, + {tREGEXP_END, &ripper_id_regexp_end}, + {tRPAREN, &ripper_id_rparen}, + {tRSHFT, &ripper_id_op}, + {tSTAR, &ripper_id_op}, + {tSTRING_BEG, &ripper_id_tstring_beg}, + {tSTRING_CONTENT, &ripper_id_tstring_content}, + {tSTRING_DBEG, &ripper_id_embexpr_beg}, + {tSTRING_DVAR, &ripper_id_embvar}, + {tSTRING_END, &ripper_id_tstring_end}, + {tSYMBEG, &ripper_id_symbeg}, + {tUMINUS, &ripper_id_op}, + {tUMINUS_NUM, &ripper_id_op}, + {tUPLUS, &ripper_id_op}, + {tWORDS_BEG, &ripper_id_words_beg}, + {tXSTRING_BEG, &ripper_id_backtick}, +/* ripper specific tokens */ + {tIGNORED_NL, &ripper_id_ignored_nl}, + {tCOMMENT, &ripper_id_comment}, + {tEMBDOC_BEG, &ripper_id_embdoc_beg}, + {tEMBDOC, &ripper_id_embdoc}, + {tEMBDOC_END, &ripper_id_embdoc_end}, + {tLWSP, &ripper_id_lwsp}, + {tHEREDOC_BEG, &ripper_id_heredoc_beg}, + {tHEREDOC_CONTENT, &ripper_id_heredoc_content}, + {tHEREDOC_END, &ripper_id_heredoc_end}, + {k__END__, &ripper_id___end__}, + {0, NULL} +}; + +static ID +ripper_token2eventid(tok) + int tok; +{ + struct token_assoc *a; + + for (a = token_to_eventid; a->id != NULL; a++) { + if (a->token == tok) + return *a->id; + } + if (tok < 256) { + return ripper_id_CHAR; + } + rb_raise(rb_eRuntimeError, "[Ripper FATAL] unknown token %d", tok); +} diff --git a/ext/ripper/extconf.rb b/ext/ripper/extconf.rb new file mode 100644 index 0000000000..536e23d3ad --- /dev/null +++ b/ext/ripper/extconf.rb @@ -0,0 +1,24 @@ +#!ruby -s + +require 'mkmf' +require 'rbconfig' + +def main + unless have_command('bison') + $stderr.puts 'Ripper requires bison; abort' + exit 1 + end + $objs = %w(ripper.o) + $cleanfiles.concat %w(ripper.y ripper.c ripper.output ids1 ids2) + $CPPFLAGS += ' -DRIPPER' + $CPPFLAGS += ' -DRIPPER_DEBUG' if $debug + create_makefile 'ripper' +end + +def have_command(cmd) + ENV['PATH'].split(File::PATH_SEPARATOR).any? {|path| + File.executable?("#{path}/#{cmd}#{Config::CONFIG['EXEEXT']}") + } +end + +main diff --git a/ext/ripper/lib/ripper.rb b/ext/ripper/lib/ripper.rb new file mode 100644 index 0000000000..b34147ee2a --- /dev/null +++ b/ext/ripper/lib/ripper.rb @@ -0,0 +1,945 @@ +# +# This file is automatically generated from ripper.rb.in and parse.y. +# DO NOT MODIFY!!!!!! +# + +# +# ripper.rb +# +# Copyright (C) 2003,2004 Minero Aoki +# + +require 'ripper.so' + +class Ripper + def Ripper.parse(str, *args) + new.parse(str, *args) + end + + private + + def warn(fmt, *args) + end + + def warning(fmt, *args) + end + + def compile_error(msg) + end + + PARSER_EVENTS = [ + :BEGIN, + :END, + :alias, + :alias_error, + :aref, + :aref_field, + :arg_ambiguous, + :arg_paren, + :arglist_add, + :arglist_add_block, + :arglist_add_star, + :arglist_new, + :arglist_prepend, + :array, + :assign, + :assign_error, + :assoc_new, + :assoclist_from_args, + :bare_assoc_hash, + :begin, + :binary, + :blockvar_add_block, + :blockvar_add_star, + :blockvar_new, + :bodystmt, + :brace_block, + :break, + :call, + :case, + :class, + :class_name_error, + :command, + :command_call, + :const_ref, + :constpath_field, + :constpath_ref, + :def, + :defined, + :defs, + :do_block, + :dot2, + :dot3, + :dyna_symbol, + :else, + :elsif, + :ensure, + :fcall, + :field, + :for, + :hash, + :if, + :if_mod, + :ifop, + :iter_block, + :massign, + :method_add_arg, + :mlhs_add, + :mlhs_add_star, + :mlhs_new, + :mlhs_paren, + :module, + :mrhs_add, + :mrhs_add_star, + :mrhs_new, + :mrhs_new_from_arglist, + :next, + :opassign, + :param_error, + :params, + :paren, + :parse_error, + :program, + :qwords_add, + :qwords_new, + :redo, + :regexp_literal, + :rescue, + :rescue_mod, + :restparam, + :retry, + :return, + :return0, + :sclass, + :space, + :stmts_add, + :stmts_new, + :string_add, + :string_concat, + :string_content, + :string_dvar, + :string_embexpr, + :string_literal, + :super, + :symbol, + :symbol_literal, + :topconst_field, + :topconst_ref, + :unary, + :undef, + :unless, + :unless_mod, + :until, + :until_mod, + :var_alias, + :var_field, + :var_ref, + :void_stmt, + :when, + :while, + :while_mod, + :word_add, + :word_new, + :words_add, + :words_new, + :xstring_add, + :xstring_literal, + :xstring_new, + :yield, + :yield0, + :zsuper + ] + + SCANNER_EVENTS = [ + :BEGIN, + :END, + :alias, + :alias_error, + :aref, + :aref_field, + :arg_ambiguous, + :arg_paren, + :arglist_add, + :arglist_add_block, + :arglist_add_star, + :arglist_new, + :arglist_prepend, + :array, + :assign, + :assign_error, + :assoc_new, + :assoclist_from_args, + :bare_assoc_hash, + :begin, + :binary, + :blockvar_add_block, + :blockvar_add_star, + :blockvar_new, + :bodystmt, + :brace_block, + :break, + :call, + :case, + :class, + :class_name_error, + :command, + :command_call, + :const_ref, + :constpath_field, + :constpath_ref, + :def, + :defined, + :defs, + :do_block, + :dot2, + :dot3, + :dyna_symbol, + :else, + :elsif, + :ensure, + :fcall, + :field, + :for, + :hash, + :if, + :if_mod, + :ifop, + :iter_block, + :massign, + :method_add_arg, + :mlhs_add, + :mlhs_add_star, + :mlhs_new, + :mlhs_paren, + :module, + :mrhs_add, + :mrhs_add_star, + :mrhs_new, + :mrhs_new_from_arglist, + :next, + :opassign, + :param_error, + :params, + :paren, + :parse_error, + :program, + :qwords_add, + :qwords_new, + :redo, + :regexp_literal, + :rescue, + :rescue_mod, + :restparam, + :retry, + :return, + :return0, + :sclass, + :space, + :stmts_add, + :stmts_new, + :string_add, + :string_concat, + :string_content, + :string_dvar, + :string_embexpr, + :string_literal, + :super, + :symbol, + :symbol_literal, + :topconst_field, + :topconst_ref, + :unary, + :undef, + :unless, + :unless_mod, + :until, + :until_mod, + :var_alias, + :var_field, + :var_ref, + :void_stmt, + :when, + :while, + :while_mod, + :word_add, + :word_new, + :words_add, + :words_new, + :xstring_add, + :xstring_literal, + :xstring_new, + :yield, + :yield0, + :zsuper + ] + LEXER_EVENTS = SCANNER_EVENTS + + EVENTS = PARSER_EVENTS + SCANNER_EVENTS + + # + # Parser Events + # + + def on__BEGIN(a) + a + end + + def on__END(a) + a + end + + def on__alias(a, b) + a + end + + def on__alias_error(a) + a + end + + def on__aref(a, b) + a + end + + def on__aref_field(a, b) + a + end + + def on__arg_ambiguous() + nil + end + + def on__arg_paren(a) + a + end + + def on__arglist_add(a, b) + a + end + + def on__arglist_add_block(a, b) + a + end + + def on__arglist_add_star(a, b) + a + end + + def on__arglist_new() + nil + end + + def on__arglist_prepend(a, b) + a + end + + def on__array(a) + a + end + + def on__assign(a, b) + a + end + + def on__assign_error(a) + a + end + + def on__assoc_new(a, b) + a + end + + def on__assoclist_from_args(a) + a + end + + def on__bare_assoc_hash(a) + a + end + + def on__begin(a) + a + end + + def on__binary(a, b, c) + a + end + + def on__blockvar_add_block(a, b) + a + end + + def on__blockvar_add_star(a, b) + a + end + + def on__blockvar_new(a) + a + end + + def on__bodystmt(a, b, c, d) + a + end + + def on__brace_block(a, b) + a + end + + def on__break(a) + a + end + + def on__call(a, b, c) + a + end + + def on__case(a, b) + a + end + + def on__class(a, b, c) + a + end + + def on__class_name_error(a) + a + end + + def on__command(a, b) + a + end + + def on__command_call(a, b, c, d) + a + end + + def on__const_ref(a) + a + end + + def on__constpath_field(a, b) + a + end + + def on__constpath_ref(a, b) + a + end + + def on__def(a, b, c) + a + end + + def on__defined(a) + a + end + + def on__defs(a, b, c, d, e) + a + end + + def on__do_block(a, b) + a + end + + def on__dot2(a, b) + a + end + + def on__dot3(a, b) + a + end + + def on__dyna_symbol(a) + a + end + + def on__else(a) + a + end + + def on__elsif(a, b, c) + a + end + + def on__ensure(a) + a + end + + def on__fcall(a) + a + end + + def on__field(a, b, c) + a + end + + def on__for(a, b, c) + a + end + + def on__hash(a) + a + end + + def on__if(a, b, c) + a + end + + def on__if_mod(a, b) + a + end + + def on__ifop(a, b, c) + a + end + + def on__iter_block(a, b) + a + end + + def on__massign(a, b) + a + end + + def on__method_add_arg(a, b) + a + end + + def on__mlhs_add(a, b) + a + end + + def on__mlhs_add_star(a, b) + a + end + + def on__mlhs_new() + nil + end + + def on__mlhs_paren(a) + a + end + + def on__module(a, b) + a + end + + def on__mrhs_add(a, b) + a + end + + def on__mrhs_add_star(a, b) + a + end + + def on__mrhs_new() + nil + end + + def on__mrhs_new_from_arglist(a) + a + end + + def on__next(a) + a + end + + def on__opassign(a, b, c) + a + end + + def on__param_error(a) + a + end + + def on__params(a, b, c, d) + a + end + + def on__paren(a) + a + end + + def on__parse_error(a) + a + end + + def on__program(a) + a + end + + def on__qwords_add(a, b) + a + end + + def on__qwords_new() + nil + end + + def on__redo() + nil + end + + def on__regexp_literal(a) + a + end + + def on__rescue(a, b, c, d) + a + end + + def on__rescue_mod(a, b) + a + end + + def on__restparam(a) + a + end + + def on__retry() + nil + end + + def on__return(a) + a + end + + def on__return0() + nil + end + + def on__sclass(a, b) + a + end + + def on__space(a) + a + end + + def on__stmts_add(a, b) + a + end + + def on__stmts_new() + nil + end + + def on__string_add(a, b) + a + end + + def on__string_concat(a, b) + a + end + + def on__string_content() + nil + end + + def on__string_dvar(a) + a + end + + def on__string_embexpr(a) + a + end + + def on__string_literal(a) + a + end + + def on__super(a) + a + end + + def on__symbol(a) + a + end + + def on__symbol_literal(a) + a + end + + def on__topconst_field(a) + a + end + + def on__topconst_ref(a) + a + end + + def on__unary(a, b) + a + end + + def on__undef(a) + a + end + + def on__unless(a, b, c) + a + end + + def on__unless_mod(a, b) + a + end + + def on__until(a, b) + a + end + + def on__until_mod(a, b) + a + end + + def on__var_alias(a, b) + a + end + + def on__var_field(a) + a + end + + def on__var_ref(a) + a + end + + def on__void_stmt() + nil + end + + def on__when(a, b, c) + a + end + + def on__while(a, b) + a + end + + def on__while_mod(a, b) + a + end + + def on__word_add(a, b) + a + end + + def on__word_new() + nil + end + + def on__words_add(a, b) + a + end + + def on__words_new() + nil + end + + def on__xstring_add(a, b) + a + end + + def on__xstring_literal(a) + a + end + + def on__xstring_new() + nil + end + + def on__yield(a) + a + end + + def on__yield0() + nil + end + + def on__zsuper() + nil + end + + # + # Lexer Events + # + + def on__scan(event, token) + end + + def on__CHAR(token) + token + end + + def on____end__(token) + token + end + + def on__backref(token) + token + end + + def on__backtick(token) + token + end + + def on__comma(token) + token + end + + def on__comment(token) + token + end + + def on__const(token) + token + end + + def on__cvar(token) + token + end + + def on__embdoc(token) + token + end + + def on__embdoc_beg(token) + token + end + + def on__embdoc_end(token) + token + end + + def on__embexpr_beg(token) + token + end + + def on__embexpr_end(token) + token + end + + def on__embvar(token) + token + end + + def on__float(token) + token + end + + def on__gvar(token) + token + end + + def on__heredoc_beg(token) + token + end + + def on__heredoc_content(token) + token + end + + def on__heredoc_end(token) + token + end + + def on__ident(token) + token + end + + def on__ignored_nl(token) + token + end + + def on__int(token) + token + end + + def on__ivar(token) + token + end + + def on__kw(token) + token + end + + def on__lbrace(token) + token + end + + def on__lbracket(token) + token + end + + def on__lparen(token) + token + end + + def on__lwsp(token) + token + end + + def on__nl(token) + token + end + + def on__op(token) + token + end + + def on__period(token) + token + end + + def on__qwords_beg(token) + token + end + + def on__rbrace(token) + token + end + + def on__rbracket(token) + token + end + + def on__regexp_beg(token) + token + end + + def on__regexp_end(token) + token + end + + def on__rparen(token) + token + end + + def on__semicolon(token) + token + end + + def on__symbeg(token) + token + end + + def on__tstring_beg(token) + token + end + + def on__tstring_content(token) + token + end + + def on__tstring_end(token) + token + end + + def on__words_beg(token) + token + end + + def on__words_sep(token) + token + end +end diff --git a/ext/ripper/ripper.rb.in b/ext/ripper/ripper.rb.in new file mode 100644 index 0000000000..c6fa554296 --- /dev/null +++ b/ext/ripper/ripper.rb.in @@ -0,0 +1,53 @@ +# +# This file is automatically generated from ripper.rb.in and parse.y. +# DO NOT MODIFY!!!!!! +# + +# +# ripper.rb +# +# Copyright (C) 2003,2004 Minero Aoki +# + +require 'ripper.so' + +class Ripper + def Ripper.parse(str, *args) + new.parse(str, *args) + end + + private + + def warn(fmt, *args) + end + + def warning(fmt, *args) + end + + def compile_error(msg) + end + + PARSER_EVENTS = [ +#include ids1 + ] + + SCANNER_EVENTS = [ +#include ids2 + ] + LEXER_EVENTS = SCANNER_EVENTS + + EVENTS = PARSER_EVENTS + SCANNER_EVENTS + + # + # Parser Events + # +#include handlers1 + + # + # Lexer Events + # + + def on__scan(event, token) + end +#include handlers2 +end diff --git a/ext/ripper/test/check-event-arity.rb b/ext/ripper/test/check-event-arity.rb new file mode 100644 index 0000000000..00e1b98dbc --- /dev/null +++ b/ext/ripper/test/check-event-arity.rb @@ -0,0 +1,24 @@ +def main + invalid = false + table = {} + ARGF.each do |line| + next if /\A\#\s*define\s+s?dispatch\d/ === line + next if /ripper_dispatch\d/ === line + line.scan(/dispatch(\d)\((\w+)/) do |num, ev| + num = num.to_i + if data = table[ev] + locations, arity = data + unless num == arity + invalid = true + puts "arity differ [#{ev}]: #{ARGF.lineno}->#{num}; #{locations.join(',')}->#{arity}" + end + locations.push ARGF.lineno + else + table[ev] = [[ARGF.lineno], num.to_i] + end + end + end + exit 1 if invalid +end + +main diff --git a/ext/ripper/test/check-event-coverage.sh b/ext/ripper/test/check-event-coverage.sh new file mode 100755 index 0000000000..727c3b9837 --- /dev/null +++ b/ext/ripper/test/check-event-coverage.sh @@ -0,0 +1,15 @@ +# $Id$ + +RUBY=${RUBY:-ruby} +status=0 + +$RUBY tools/list-parse-event-ids.rb parse.y | awk '{print "on__" $1}' > list_a +$RUBY test/list-called-events.rb | sort -u > list_b +diff -u list_a list_b | grep '^-on' | sed 's/^-on__//' > list_diff +if [ -s list_diff ] +then + cat list_diff + status=1 +fi +rm -f list_a list_b list_diff +exit $status diff --git a/ext/ripper/test/check-scanner-event-coverage.rb b/ext/ripper/test/check-scanner-event-coverage.rb new file mode 100644 index 0000000000..4de01e35f4 --- /dev/null +++ b/ext/ripper/test/check-scanner-event-coverage.rb @@ -0,0 +1,18 @@ +def main + not_tested = eventids() - tested_ids() + unless not_tested.empty? + puts not_tested + exit 1 + end + exit 0 +end + +def eventids + File.read('eventids2.c').scan(/on__(\w+)/).flatten.uniq +end + +def tested_ids + File.read('test/test_scanner_events.rb').scan(/def test_(\S+)/).flatten.uniq +end + +main diff --git a/ext/ripper/test/list-called-events.rb b/ext/ripper/test/list-called-events.rb new file mode 100644 index 0000000000..d799efc979 --- /dev/null +++ b/ext/ripper/test/list-called-events.rb @@ -0,0 +1,12 @@ +require 'ripper.so' + +class R < Ripper + def method_missing(mid, *args) + puts mid + args[0] + end + undef :warn +end + +fname = (ARGV[0] || 'test/src_rb') +R.new(File.read(fname), fname, 1).parse diff --git a/ext/ripper/test/src_rb b/ext/ripper/test/src_rb new file mode 100644 index 0000000000..e7894362f3 --- /dev/null +++ b/ext/ripper/test/src_rb @@ -0,0 +1,363 @@ +# comment +=begin + embedded document +=end + +# literal +1 +1000000000000000000000 +1.0 +1.234e5 +1..2 +1...3 +:symbol +:"dynamic #{sym_embexpr} symbol" +[1,2,3] +{1 => 2} +'string' +"string" +"before #{str_embexpr} after" +"str #@ivar str" +"str #$gvar str" +"string" "concat" +`/bin/true` +{1, 2, 3, 4} +/regexp/ +/regexp_with_opt/mioe +/regexp #{regexp_embexpr} after/ +%q[string] +%Q[str#{str_embexpr}ing] +%r[regexp] +%w( a b c ) +%W( a#{w_emb}b c d ) +<<HERE +heredoc line 1 +heredoc line 2 +heredoc line 3 +HERE + +# special variables +true +false +nil +self + +# def +def a +end +def b() +end +def c(a) +end +def d(a,*rest) +end +def e(a,&block) +end +def f(a,*rest,&block) +end +def g(*rest) +end +def h(&block) +end +def i(*rest,&block) +end +def j(CONST) +end +def k(@ivar) +end +def l($gvar) +end +def n(@@cvar) +end + +# alias +alias x b +alias $rest $' # error +alias $nth $1 # error + +# undef +undef warn + +# class, module +class C +end +module M +end +class cname +end +class << Object.new + def self.a + end +end + +# field +$a = 1 +$' = 0 # error +$1 = 0 # error +@a = 2 +@@a = 3 +a = 4 +a += 1 +a -= 1 +a *= 1 +a /= 1 +a &&= 1 +a ||= 1 +m.a = 5 +m.a += 1 +m.a &&= 1 +m.a ||= 1 +a[1] = 2 +a[1] += 1 +a[1] &&= 1 +a[1] ||= 1 +C = 1 +C::C = 1 +::C = 1 +def m + C = 1 # dynamic const assignment + C::C = 1 # dynamic const assignment + ::C = 1 # dynamic const assignment +end + +# ref +lvar = $a +lvar = @a +lvar = @@a +lvar = Object +lvar = C +lvar = C::C +lvar = ::C +lvar = a[1] + +# unary operator ++1 +-1 +not 1 +!1 +~str + +# binary operator +1 + 1 +1 - 1 +1 * 1 +1 / 1 +1 ** 1 +1 ^ 1 +1 & 1 +1 | 1 +1 && 1 +1 || 1 + +# mlhs, mrhs +a, b, c = list +a, = list +a, * = list +a, *b = list +a, (b, c), d, *e = list +mlhs = 1, 2 +mlhs = 1, 2, 3, *list +mlhs = *list + +# method call +m +m() +m(a) +m(a,a) +m(*a) +m(&b) +m(a,*a) +m(a,&b) +m(a,*a,&b) +m(1=>2) +m(1=>2,*a) +m(1=>2,&b) +m(1=>2,*a,&b) +m () +m (a) +m (a,a) +m (*a) +m (&b) +m (a,*a) +m (a,&b) +m (a,*a,&b) +m (1=>2) +m (1=>2,*a) +m (1=>2,&b) +m (1=>2,*a,&b) +m a +m a,a +m *a +m &b +m a,*a +m a,&b +m a,*a,&b +m 1=>2 +m 1=>2,*a +m 1=>2,&b +m 1=>2,*a,&b +obj.m +obj.m() +obj.m(a) +obj.m(a,a) +obj.m(*a) +obj.m(&b) +obj.m(a,*a) +obj.m(a,&b) +obj.m(a,*a,&b) +obj.m(1=>2) +obj.m(1=>2,*a) +obj.m(1=>2,&b) +obj.m(1=>2,*a,&b) +obj.m () +obj.m (a) +obj.m (a,a) +obj.m (*a) +obj.m (&b) +obj.m (a,*a) +obj.m (a,&b) +obj.m (a,*a,&b) +obj.m (1=>2) +obj.m (1=>2) +obj.m (1=>2,*a) +obj.m (1=>2,&b) +obj.m (1=>2,*a,&b) +obj.m a +obj.m a,a +obj.m *a +obj.m &b +obj.m a,*a +obj.m a,&b +obj.m a,*a,&b +obj.m 1=>2 +obj.m 1=>2,*a +obj.m 1=>2,&b +obj.m 1=>2,*a,&b + +# ambiguous argument +m +1 +m /r/ + +# iterator +[1,2,3].each do |i| + print i +end +{1=>true}.each do |k,v| + puts k +end +[1,2,3].each {|i| print i } +[1].each {|a,| } +[1].each {|*b| } +[1].each {|a,*b| } +[1].each {|&block| } +[1].each {|a,&block| } +[1].each {|a,*b,&block| } +a = lambda() {|n| n * n } +a = lambda () {|n| n * n } +a = lambda (a) {|n| n * n } +a = lambda (a,b) {|n| n * n } + +# BEGIN, END +BEGIN { } +END { } + +# if, unless +1 if true +2 unless false +if false + 5 +elsif false + 6 +elsif false then 7 +else + 8 +end +if m +end +unless 1 + 2 +end +unless m +end +0 ? 1 : 2 + +# case +case 'a' +when 'b' + ; +when 'c' then 1 +else + 2 +end +case +when 1 +when 2 +when 3 +else +end +case 1 +else +end +case +else +end + +# while, until, for +while true + break + next + redo +end +begin + break +end while true +until false + break + next + redo +end +begin + break +end until false +for x in m() + break + next + redo +end +0 until true +1 while false + +# begin, rescue, else, ensure +begin + 1 +rescue StandardError => er + 2 +rescue => er + 3 + retry +else + 4 +ensure + 5 +end +a = 1 rescue 2 + +# jumps +def m + redo + yield + yield nil + super + super 1 + return + return nil +end + +# defined +defined? f +defined?(f) + +n = 1 \ ++ 1 + +__END__ diff --git a/ext/ripper/test/validate.rb b/ext/ripper/test/validate.rb new file mode 100644 index 0000000000..808d321043 --- /dev/null +++ b/ext/ripper/test/validate.rb @@ -0,0 +1,68 @@ +require 'ripper.so' + +class R < Ripper + def initialize(*args) + super + @lineno = 0 + end + + def parse + result = super + puts "#{@lineno}:result: #{rawVALUE(result)}" + validate_object result + p result + result + end + + def on__nl(str) + @lineno += 1 + end + + def on__ignored_nl(str) + @lineno += 1 + end + + def on__comment(cmt) + @lineno += 1 + end + + def on__embdoc_beg(str) + @lineno += 1 + end + + def on__embdoc(str) + @lineno += 1 + end + + def on__embdoc_end(str) + @lineno += 1 + end + + def method_missing(mid, *args) + puts mid + args.each_with_index do |a,idx| + puts "#{@lineno}:#{mid}\##{idx+1}: #{rawVALUE(a)}" + validate_object a + p a + end + args[0] + end + + def warn(*args) + end + + def warning(*args) + end + + unless respond_to?(:validate_object) + def validate_object(x) + x + end + def rawVALUE(x) + x.object_id + end + end +end + +fname = (ARGV[0] || 'test/src_rb') +R.new(File.read(fname), fname, 1).parse diff --git a/ext/ripper/tools/generate-eventids1.rb b/ext/ripper/tools/generate-eventids1.rb new file mode 100755 index 0000000000..de0e27b89c --- /dev/null +++ b/ext/ripper/tools/generate-eventids1.rb @@ -0,0 +1,18 @@ +# +# generate-eventids1.rb +# + +ids = ARGF.map {|s| s.strip } + +ids.each do |id| + puts "static ID ripper_id_#{id};" +end + +puts +puts 'static void' +puts 'ripper_init_eventids1()' +puts '{' +ids.each do |id| + puts %Q[ ripper_id_#{id} = rb_intern("on__#{id}");] +end +puts '}' diff --git a/ext/ripper/tools/generate-param-macros.rb b/ext/ripper/tools/generate-param-macros.rb new file mode 100755 index 0000000000..b19f6e8d5c --- /dev/null +++ b/ext/ripper/tools/generate-param-macros.rb @@ -0,0 +1,14 @@ +off = true +ARGF.each do |line| + case line + when /RIPPER_PARAMS_DECL_BEGIN/ + off = false + when /RIPPER_PARAMS_DECL_END/ + exit + when /ripper/ + next if off + var = line.scan(/\w+/).last or next + base = var.sub(/ripper_/, '') + puts %"\#define #{base}\t\t(parser->ripper_#{base})" + end +end diff --git a/ext/ripper/tools/generate-ripper_rb.rb b/ext/ripper/tools/generate-ripper_rb.rb new file mode 100755 index 0000000000..c6b2aea13f --- /dev/null +++ b/ext/ripper/tools/generate-ripper_rb.rb @@ -0,0 +1,44 @@ +# +# generate-ripper_rb.rb +# Creates ripper.rb, filling in default event handlers, given a basic +# template, the list of parser events (ids1), and a list of lexer +# events (ids2). +# + +def main + template, ids1, ids2 = *ARGV + File.foreach(template) do |line| + case line + when /\A\#include handlers1/ + File.foreach(ids1) do |line| + id, arity = line.split + arity = arity.to_i + puts + puts " def on__#{id}(#{argdecl(arity)})" + puts " #{arity == 0 ? 'nil' : 'a'}" + puts " end" + end + when /\A\#include handlers2/ + File.foreach(ids2) do |line| + id, arity = line.split + arity = arity.to_i + puts + puts " def on__#{id}(token)" + puts " token" + puts " end" + end + when /\A\#include (.*)/ + raise "unknown operation: #include #{$1}" + else + print line + end + end +end + +# Generate generic arg list depending on arity (n) +# n:: [Integer] arity of method +def argdecl( n ) + %w(a b c d e f g h i j k l m)[0, n].join(', ') +end + +main diff --git a/ext/ripper/tools/list-parse-event-ids.rb b/ext/ripper/tools/list-parse-event-ids.rb new file mode 100755 index 0000000000..2936f9b092 --- /dev/null +++ b/ext/ripper/tools/list-parse-event-ids.rb @@ -0,0 +1,38 @@ +# +# list-parse-event-ids.rb +# + +require 'getopts' + +def usage( status ) + (status == 0 ? $stdout : $stderr).print(<<EOS) +Usage: #{File.basename($0)} [-a] filename +EOS + exit status +end + +def main + getopts('a') or usage(1) + extract_ids(ARGF).each do |id, arity| + if $OPT_a + then puts "#{id} #{arity}" + else puts id + end + end +end + +def extract_ids( f ) + results = [] + f.each do |line| + next if /\A\#\s*define\s+s?dispatch/ === line + next if /ripper_dispatch/ === line + if a = line.scan(/dispatch(\d)\((\w+)/) + a.each do |arity, event| + results.push [event, arity.to_i] + end + end + end + results.uniq.sort +end + +main diff --git a/ext/ripper/tools/list-scan-event-ids.rb b/ext/ripper/tools/list-scan-event-ids.rb new file mode 100755 index 0000000000..6f25362b5d --- /dev/null +++ b/ext/ripper/tools/list-scan-event-ids.rb @@ -0,0 +1,32 @@ +# +# list-scan-event-ids.rb +# + +require 'getopts' + +def usage(status) + (status == 0 ? $stdout : $stderr).puts(<<EOS) +Usage: #{File.basename($0)} eventids2.c + -a print IDs with arity. +EOS + exit status +end + +def main + ok = getopts('a', 'help') + usage 0 if $OPT_help + usage 1 unless ok + extract_ids(ARGF).sort.each do |id| + if $OPT_a + puts "#{id} 1" + else + puts id + end + end +end + +def extract_ids(f) + (f.read.scan(/ripper_id_(\w+)/).flatten - ['scan']).uniq +end + +main diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb new file mode 100755 index 0000000000..a2dba36e02 --- /dev/null +++ b/ext/ripper/tools/preproc.rb @@ -0,0 +1,57 @@ +def main + prelude + grammar + usercode +end + +def prelude + while line = ARGF.gets + case line + when %r</\*%%%\*/> + puts '/*' + when %r</\*%> + puts '*/' + when %r<%\*/> + puts + when /\A%%/ + puts '%%' + return + when /\A%token/ + puts line.sub(/<\w+>/, '<val>') + when /\A%type/ + puts line.sub(/<\w+>/, '<val>') + else + print line + end + end +end + +def grammar + while line = ARGF.gets + case line + when %r</\*%%%\*/> + puts '#if 0' + when %r</\*%c%\*/> + puts '/*' + when %r</\*%c> + puts '*/' + when %r</\*%> + puts '#endif' + when %r<%\*/> + puts + when /\A%%/ + puts '%%' + return + else + print line + end + end +end + +def usercode + while line = ARGF.gets + print line + end +end + +main diff --git a/ext/ripper/tools/strip.rb b/ext/ripper/tools/strip.rb new file mode 100755 index 0000000000..99413c361d --- /dev/null +++ b/ext/ripper/tools/strip.rb @@ -0,0 +1,12 @@ +last_is_void = false +ARGF.each do |line| + if line.strip.empty? + #puts() unless last_is_void + last_is_void = true + elsif /\A\#/ === line + ; + else + print line + last_is_void = false + end +end diff --git a/test/ripper/dummyparser.rb b/test/ripper/dummyparser.rb new file mode 100644 index 0000000000..b95ebdb5d1 --- /dev/null +++ b/test/ripper/dummyparser.rb @@ -0,0 +1,526 @@ +# +# dummyparser.rb +# + +class Node + + def initialize( name, *nodes ) + @name = name + @children = nodes + end + + attr_reader :children + + def to_s + "#{@name}(#{@children.map {|n| n.to_s }.join(',')})" + end + +end + + +class NodeList + + def initialize + @list = [] + end + + attr_reader :list + + def push( item ) + @list.push item + self + end + + def prepend( items ) + @list[0,0] = items + end + + def to_s + '[' + @list.join(',') + ']' + end + +end + + +class DummyParser < Ripper + + def method_missing( mid, *args ) + raise NoMethodError, "wrong method: #{mid}" unless /\Aon__/ === mid.to_s + args[0] + end + + def on__program( stmts ) + $thru_program = true + stmts + end + + def on__stmts_new + NodeList.new + end + + def on__stmts_add( stmts, st ) + stmts.push st + stmts + end + + def on__void_stmt + Node.new('void') + end + + def on__BEGIN( stmts ) + Node.new('BEGIN', stmts) + end + + def on__END( stmts ) + Node.new('END', stmts) + end + + def on__var_ref( name ) + Node.new('ref', name) + end + + def on__alias(a, b) + Node.new('alias', a, b) + end + + def on__var_alias(a, b) + Node.new('valias', a, b) + end + + def on__alias_error(a) + Node.new('aliaserr', a) + end + + def on__aref(a, b) + Node.new('aref', a, b) + end + + def on__aref_field(a, b) + Node.new('aref_field', a, b) + end + + def on__arg_ambiguous + Node.new('arg_ambiguous') + end + + def on__arg_paren( args ) + args + end + + def on__arglist_new + NodeList.new + end + + def on__arglist_add( list, arg ) + list.push(arg) + end + + def on__arglist_add_block( list, blk ) + list.push('&' + blk.to_s) + end + + def on__arglist_add_star( list, arg ) + list.push('*' + arg.to_s) + end + + def on__arglist_prepend( list, args ) + list.prepend args + list + end + + def on__method_add_arg( m, arg ) + m.children.push arg + m + end + + def on__assoc_new(a, b) + Node.new('assoc', a, b) + end + + def on__bare_assoc_hash( assoc_list ) + Node.new('assocs', *assoc_list) + end + + def on__assoclist_from_args(a) + Node.new('assocs', *a.list) + end + + ######## untested + + def on__array(a) + Node.new('array', a) + end + + def on__assign(a, b) + Node.new('assign', a, b) + end + + def on__assign_error(a) + Node.new('assign_error', a) + end + + def on__begin(a) + Node.new('begin', a) + end + + def on__binary(a, b, c) + Node.new('binary', a, b, c) + end + + def on__block_var(a) + Node.new('block_var', a) + end + + def on__bodystmt(a, b, c, d) + Node.new('bodystmt', a, b, c, d) + end + + def on__brace_block(a, b) + Node.new('brace_block', a, b) + end + + def on__break(a) + Node.new('break', a) + end + + def on__call(a, b, c) + Node.new('call', a, b, c) + end + + def on__case(a, b) + Node.new('case', a, b) + end + + def on__class(a, b, c) + Node.new('class', a, b, c) + end + + def on__class_name_error(a) + Node.new('class_name_error', a) + end + + def on__command(a, b) + Node.new('command', a, b) + end + + def on__command_call(a, b, c, d) + Node.new('command_call', a, b, c, d) + end + + def on__const_ref(a) + Node.new('const_ref', a) + end + + def on__constpath_field(a, b) + Node.new('constpath_field', a, b) + end + + def on__constpath_ref(a, b) + Node.new('constpath_ref', a, b) + end + + def on__def(a, b, c) + Node.new('def', a, b, c) + end + + def on__defined(a) + Node.new('defined', a) + end + + def on__defs(a, b, c, d, e) + Node.new('defs', a, b, c, d, e) + end + + def on__do_block(a, b) + Node.new('do_block', a, b) + end + + def on__dot2(a, b) + Node.new('dot2', a, b) + end + + def on__dot3(a, b) + Node.new('dot3', a, b) + end + + def on__dyna_symbol(a) + Node.new('dyna_symbol', a) + end + + def on__else(a) + Node.new('else', a) + end + + def on__elsif(a, b, c) + Node.new('elsif', a, b, c) + end + + def on__ensure(a) + Node.new('ensure', a) + end + + def on__fcall(a) + Node.new('fcall', a) + end + + def on__field(a, b, c) + Node.new('field', a, b, c) + end + + def on__for(a, b, c) + Node.new('for', a, b, c) + end + + def on__hash(a) + Node.new('hash', a) + end + + def on__if(a, b, c) + Node.new('if', a, b, c) + end + + def on__if_mod(a, b) + Node.new('if_mod', a, b) + end + + def on__ifop(a, b, c) + Node.new('ifop', a, b, c) + end + + def on__iter_block(a, b) + Node.new('iter_block', a, b) + end + + def on__massign(a, b) + Node.new('massign', a, b) + end + + def on__mlhs_add(a, b) + Node.new('mlhs_add', a, b) + end + + def on__mlhs_add_star(a, b) + Node.new('mlhs_add_star', a, b) + end + + def on__mlhs_new + Node.new('mlhs_new') + end + + def on__mlhs_paren(a) + Node.new('mlhs_paren', a) + end + + def on__module(a, b) + Node.new('module', a, b) + end + + def on__mrhs_add(a, b) + Node.new('mrhs_add', a, b) + end + + def on__mrhs_add_star(a, b) + Node.new('mrhs_add_star', a, b) + end + + def on__mrhs_new + Node.new('mrhs_new') + end + + def on__mrhs_new_from_arglist(a) + Node.new('mrhs_new_from_arglist', a) + end + + def on__next(a) + Node.new('next', a) + end + + def on__opassign(a, b, c) + Node.new('opassign', a, b, c) + end + + def on__param_error(a) + Node.new('param_error', a) + end + + def on__params(a, b, c, d) + Node.new('params', a, b, c, d) + end + + def on__paren(a) + Node.new('paren', a) + end + + def on__parse_error(a) + Node.new('parse_error', a) + end + + def on__qwords_add(a, b) + Node.new('qwords_add', a, b) + end + + def on__qwords_new + Node.new('qwords_new') + end + + def on__redo + Node.new('redo') + end + + def on__regexp_literal(a) + Node.new('regexp_literal', a) + end + + def on__rescue(a, b, c, d) + Node.new('rescue', a, b, c, d) + end + + def on__rescue_mod(a, b) + Node.new('rescue_mod', a, b) + end + + def on__restparam(a) + Node.new('restparam', a) + end + + def on__retry + Node.new('retry') + end + + def on__return(a) + Node.new('return', a) + end + + def on__return0 + Node.new('return0') + end + + def on__sclass(a, b) + Node.new('sclass', a, b) + end + + def on__space(a) + Node.new('space', a) + end + + def on__string_add(a, b) + Node.new('string_add', a, b) + end + + def on__string_concat(a, b) + Node.new('string_concat', a, b) + end + + def on__string_content + Node.new('string_content') + end + + def on__string_dvar(a) + Node.new('string_dvar', a) + end + + def on__string_embexpr(a) + Node.new('string_embexpr', a) + end + + def on__string_literal(a) + Node.new('string_literal', a) + end + + def on__super(a) + Node.new('super', a) + end + + def on__symbol(a) + Node.new('symbol', a) + end + + def on__symbol_literal(a) + Node.new('symbol_literal', a) + end + + def on__topconst_field(a) + Node.new('topconst_field', a) + end + + def on__topconst_ref(a) + Node.new('topconst_ref', a) + end + + def on__unary(a, b) + Node.new('unary', a, b) + end + + def on__undef(a) + Node.new('undef', a) + end + + def on__unless(a, b, c) + Node.new('unless', a, b, c) + end + + def on__unless_mod(a, b) + Node.new('unless_mod', a, b) + end + + def on__until_mod(a, b) + Node.new('until_mod', a, b) + end + + def on__var_field(a) + Node.new('var_field', a) + end + + def on__when(a, b, c) + Node.new('when', a, b, c) + end + + def on__while(a, b) + Node.new('while', a, b) + end + + def on__while_mod(a, b) + Node.new('while_mod', a, b) + end + + def on__word_add(a, b) + Node.new('word_add', a, b) + end + + def on__word_new + Node.new('word_new') + end + + def on__words_add(a, b) + Node.new('words_add', a, b) + end + + def on__words_new + Node.new('words_new') + end + + def on__xstring_add(a, b) + Node.new('xstring_add', a, b) + end + + def on__xstring_literal(a) + Node.new('xstring_literal', a) + end + + def on__xstring_new + Node.new('xstring_new') + end + + def on__yield(a) + Node.new('yield', a) + end + + def on__yield0 + Node.new('yield0') + end + + def on__zsuper + Node.new('zsuper') + end + +end diff --git a/test/ripper/test_parser_events.rb b/test/ripper/test_parser_events.rb new file mode 100644 index 0000000000..512cf377e7 --- /dev/null +++ b/test/ripper/test_parser_events.rb @@ -0,0 +1,489 @@ +# +# test_parser_events.rb +# + +require 'ripper.so' +require 'dummyparser' +raise "ripper version differ" unless Ripper::Version == '0.1.0' +require 'test/unit' + + +class TestRipper_ParserEvents < Test::Unit::TestCase + + def parse(str) + DummyParser.new(str).parse.to_s + end + + $thru_program = false + + def test_program + assert_equal '[void()]', parse('') + assert_equal true, $thru_program + end + + def test_stmts_new + assert_equal '[void()]', parse('') + end + + def test_stmts_add + assert_equal '[ref(nil)]', parse('nil') + assert_equal '[ref(nil),ref(nil)]', parse('nil;nil') + assert_equal '[ref(nil),ref(nil),ref(nil)]', parse('nil;nil;nil') + end + + def test_void_stmt + assert_equal '[void()]', parse('') + assert_equal '[void()]', parse(';;') + end + + def test_var_ref + assert_equal '[ref(a)]', parse('a') + assert_equal '[ref(nil)]', parse('nil') + assert_equal '[ref(true)]', parse('true') + end + + def test_BEGIN + assert_equal '[BEGIN([void()])]', parse('BEGIN{}') + assert_equal '[BEGIN([ref(nil)])]', parse('BEGIN{nil}') + end + + def test_END + assert_equal '[END([void()])]', parse('END{}') + assert_equal '[END([ref(nil)])]', parse('END{nil}') + end + + def test_alias + assert_equal '[alias(a,b)]', parse('alias a b') + end + + def test_var_alias + assert_equal '[valias($a,$g)]', parse('alias $a $g') + end + + def test_alias_error + assert_equal '[aliaserr(valias($a,$1))]', parse('alias $a $1') + end + + def test_arglist + assert_equal '[fcall(m,[])]', parse('m()') + assert_equal '[fcall(m,[1])]', parse('m(1)') + assert_equal '[fcall(m,[1,2])]', parse('m(1,2)') + assert_equal '[fcall(m,[*ref(r)])]', parse('m(*r)') + assert_equal '[fcall(m,[1,*ref(r)])]', parse('m(1,*r)') + assert_equal '[fcall(m,[1,2,*ref(r)])]', parse('m(1,2,*r)') + assert_equal '[fcall(m,[&ref(r)])]', parse('m(&r)') + assert_equal '[fcall(m,[1,&ref(r)])]', parse('m(1,&r)') + assert_equal '[fcall(m,[1,2,&ref(r)])]', parse('m(1,2,&r)') + assert_equal '[fcall(m,[*ref(a),&ref(b)])]', parse('m(*a,&b)') + assert_equal '[fcall(m,[1,*ref(a),&ref(b)])]', parse('m(1,*a,&b)') + assert_equal '[fcall(m,[1,2,*ref(a),&ref(b)])]', parse('m(1,2,*a,&b)') + end + + def test_arg_paren + # FIXME + end + + def test_aref + assert_equal '[aref(ref(v),[1])]', parse('v[1]') + assert_equal '[aref(ref(v),[1,2])]', parse('v[1,2]') + end + + def test_assocs + assert_equal '[fcall(m,[assocs(assoc(1,2))])]', parse('m(1=>2)') + assert_equal '[fcall(m,[assocs(assoc(1,2),assoc(3,4))])]', parse('m(1=>2,3=>4)') + assert_equal '[fcall(m,[3,assocs(assoc(1,2))])]', parse('m(3,1=>2)') + end + + def test_aref_field + assert_equal '[assign(aref_field(ref(a),[1]),2)]', parse('a[1]=2') + end + +=begin + def test_arg_ambiguous + assert_equal true, $thru__arg_ambiguous + end +=end + + def test_array # array literal + assert_equal '[array([1,2,3])]', parse('[1,2,3]') + end + + def test_assign # generic assignment + assert_equal '[assign(var_field(v),1)]', parse('v=1') + end + +=begin + def test_assign_error + assert_equal true, $thru__assign_error + end + + def test_begin + assert_equal true, $thru__begin + end + + def test_binary + assert_equal true, $thru__binary + end + + def test_block_var + assert_equal true, $thru__block_var + end + + def test_bodystmt + assert_equal true, $thru__bodystmt + end + + def test_brace_block + assert_equal true, $thru__brace_block + end + + def test_break + assert_equal true, $thru__break + end + + def test_call + assert_equal true, $thru__call + end + + def test_case + assert_equal true, $thru__case + end + + def test_class + assert_equal true, $thru__class + end + + def test_class_name_error + assert_equal true, $thru__class_name_error + end + + def test_command + assert_equal true, $thru__command + end + + def test_command_call + assert_equal true, $thru__command_call + end + + def test_const_ref + assert_equal true, $thru__const_ref + end + + def test_constpath_field + assert_equal true, $thru__constpath_field + end + + def test_constpath_ref + assert_equal true, $thru__constpath_ref + end + + def test_def + assert_equal true, $thru__def + end + + def test_defined + assert_equal true, $thru__defined + end + + def test_defs + assert_equal true, $thru__defs + end + + def test_do_block + assert_equal true, $thru__do_block + end + + def test_dot2 + assert_equal true, $thru__dot2 + end + + def test_dot3 + assert_equal true, $thru__dot3 + end + + def test_dyna_symbol + assert_equal true, $thru__dyna_symbol + end + + def test_else + assert_equal true, $thru__else + end + + def test_elsif + assert_equal true, $thru__elsif + end + + def test_ensure + assert_equal true, $thru__ensure + end + + def test_fcall + assert_equal true, $thru__fcall + end + + def test_field + assert_equal true, $thru__field + end + + def test_for + assert_equal true, $thru__for + end + + def test_hash + assert_equal true, $thru__hash + end + + def test_if + assert_equal true, $thru__if + end + + def test_if_mod + assert_equal true, $thru__if_mod + end + + def test_ifop + assert_equal true, $thru__ifop + end + + def test_iter_block + assert_equal true, $thru__iter_block + end + + def test_massign + assert_equal true, $thru__massign + end + + def test_method_add_arg + assert_equal true, $thru__method_add_arg + end + + def test_mlhs_add + assert_equal true, $thru__mlhs_add + end + + def test_mlhs_add_star + assert_equal true, $thru__mlhs_add_star + end + + def test_mlhs_new + assert_equal true, $thru__mlhs_new + end + + def test_mlhs_paren + assert_equal true, $thru__mlhs_paren + end + + def test_module + assert_equal true, $thru__module + end + + def test_mrhs_add + assert_equal true, $thru__mrhs_add + end + + def test_mrhs_add_star + assert_equal true, $thru__mrhs_add_star + end + + def test_mrhs_new + assert_equal true, $thru__mrhs_new + end + + def test_mrhs_new_from_arglist + assert_equal true, $thru__mrhs_new_from_arglist + end + + def test_next + assert_equal true, $thru__next + end + + def test_opassign + assert_equal true, $thru__opassign + end + + def test_param_error + assert_equal true, $thru__param_error + end + + def test_params + assert_equal true, $thru__params + end + + def test_paren + assert_equal true, $thru__paren + end + + def test_parse_error + assert_equal true, $thru__parse_error + end + + def test_qwords_add + assert_equal true, $thru__qwords_add + end + + def test_qwords_new + assert_equal true, $thru__qwords_new + end + + def test_redo + assert_equal true, $thru__redo + end + + def test_regexp_literal + assert_equal true, $thru__regexp_literal + end + + def test_rescue + assert_equal true, $thru__rescue + end + + def test_rescue_mod + assert_equal true, $thru__rescue_mod + end + + def test_restparam + assert_equal true, $thru__restparam + end + + def test_retry + assert_equal true, $thru__retry + end + + def test_return + assert_equal true, $thru__return + end + + def test_return0 + assert_equal true, $thru__return0 + end + + def test_sclass + assert_equal true, $thru__sclass + end + + def test_space + assert_equal true, $thru__space + end + + def test_string_add + assert_equal true, $thru__string_add + end + + def test_string_concat + assert_equal true, $thru__string_concat + end + + def test_string_content + assert_equal true, $thru__string_content + end + + def test_string_dvar + assert_equal true, $thru__string_dvar + end + + def test_string_embexpr + assert_equal true, $thru__string_embexpr + end + + def test_string_literal + assert_equal true, $thru__string_literal + end + + def test_super + assert_equal true, $thru__super + end + + def test_symbol + assert_equal true, $thru__symbol + end + + def test_symbol_literal + assert_equal true, $thru__symbol_literal + end + + def test_topconst_field + assert_equal true, $thru__topconst_field + end + + def test_topconst_ref + assert_equal true, $thru__topconst_ref + end + + def test_unary + assert_equal true, $thru__unary + end + + def test_undef + assert_equal true, $thru__undef + end + + def test_unless + assert_equal true, $thru__unless + end + + def test_unless_mod + assert_equal true, $thru__unless_mod + end + + def test_until_mod + assert_equal true, $thru__until_mod + end + + def test_var_field + assert_equal true, $thru__var_field + end + + def test_when + assert_equal true, $thru__when + end + + def test_while + assert_equal true, $thru__while + end + + def test_while_mod + assert_equal true, $thru__while_mod + end + + def test_word_add + assert_equal true, $thru__word_add + end + + def test_word_new + assert_equal true, $thru__word_new + end + + def test_words_add + assert_equal true, $thru__words_add + end + + def test_words_new + assert_equal true, $thru__words_new + end + + def test_xstring_add + assert_equal true, $thru__xstring_add + end + + def test_xstring_literal + assert_equal true, $thru__xstring_literal + end + + def test_xstring_new + assert_equal true, $thru__xstring_new + end + + def test_yield + assert_equal true, $thru__yield + end + + def test_yield0 + assert_equal true, $thru__yield0 + end + + def test_zsuper + assert_equal true, $thru__zsuper + end +=end + +end diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb new file mode 100644 index 0000000000..28b65eb1b6 --- /dev/null +++ b/test/ripper/test_scanner_events.rb @@ -0,0 +1,744 @@ +# +# test_scanner_events.rb +# + +require 'ripper.so' +raise 'ripper version differ' unless Ripper::Version == '0.1.0' +require 'test/unit' + +class TestRipper_ScannerEvents < Test::Unit::TestCase + + class R < Ripper + def R.scan(target, src) + r = new(src, target) + r.parse + r.tokens.map {|id, tok| tok } + end + + def R.lex(src) + r = new(src, 'scan') + r.parse + r.tokens + end + + def initialize(src, target) + super src + @target = ('on__' + target).intern + @tokens = [] + end + + attr_reader :tokens + + def method_missing(mid, *args) + case mid.to_s + when /\Aon__scan/ + if @target == :on__scan + @tokens.push args + else + @tokens.push args if @target == args[0] + end + when /\Aon__/ + ; + else + raise NoMethodError, "no such method: #{mid}" + end + end + + def warn(fmt, *args) + #p [fmt, args] + end + + def warning(fmt, *args) + #p [fmt, args] + end + + def compile_error(msg) + end + end + + def test_scan + assert_equal [], + R.scan('scan', '') + assert_equal ['a'], + R.scan('scan', 'a') + assert_equal ['1'], + R.scan('scan', '1') + assert_equal ['1', ';', 'def', ' ', 'm', '(', 'arg', ')', 'end'], + R.scan('scan', "1;def m(arg)end") + end + + def test_backref + assert_equal ["$`", "$&", "$'", '$1', '$2', '$3'], + R.scan('backref', %q[m($~, $`, $&, $', $1, $2, $3)]) + end + + def test_backtick + assert_equal ["`"], + R.scan('backtick', %q[p `make all`]) + end + + def test_comma + assert_equal [','] * 6, + R.scan('comma', %q[ m(0,1,2,3,4,5,6) ]) + assert_equal [], + R.scan('comma', %q[".,.,.,.,.,.,.."]) + assert_equal [], + R.scan('comma', %Q[<<EOS\n,,,,,,,,,,\nEOS]) + end + + def test_period + assert_equal [], + R.scan('period', '') + assert_equal ['.'], + R.scan('period', 'a.b') + assert_equal ['.'], + R.scan('period', 'Object.new') + assert_equal [], + R.scan('period', '"."') + assert_equal [], + R.scan('period', '1..2') + assert_equal [], + R.scan('period', '1...3') + end + + def test_const + assert_equal ['CONST'], + R.scan('const', 'CONST') + assert_equal ['C'], + R.scan('const', 'C') + assert_equal ['CONST_A'], + R.scan('const', 'CONST_A') + assert_equal ['Const', 'Const2', 'Const3'], + R.scan('const', 'Const; Const2; Const3') + assert_equal ['Const'], + R.scan('const', 'Const(a)') + assert_equal ['M', 'A', 'A2'], + R.scan('const', 'M(A,A2)') + assert_equal [], + R.scan('const', '') + assert_equal [], + R.scan('const', 'm(lvar, @ivar, @@cvar, $gvar)') + end + + def test_cvar + assert_equal [], + R.scan('cvar', '') + assert_equal ['@@cvar'], + R.scan('cvar', '@@cvar') + assert_equal ['@@__cvar__'], + R.scan('cvar', '@@__cvar__') + assert_equal ['@@CVAR'], + R.scan('cvar', '@@CVAR') + assert_equal ['@@cvar'], + R.scan('cvar', ' @@cvar#comment') + assert_equal ['@@cvar'], + R.scan('cvar', ':@@cvar') + assert_equal ['@@cvar'], + R.scan('cvar', 'm(lvar, @ivar, @@cvar, $gvar)') + assert_equal [], + R.scan('cvar', '"@@cvar"') + end + + def test_embexpr_beg + assert_equal [], + R.scan('embexpr_beg', '') + assert_equal ['#{'], + R.scan('embexpr_beg', '"#{expr}"') + assert_equal [], + R.scan('embexpr_beg', '%q[#{expr}]') + assert_equal ['#{'], + R.scan('embexpr_beg', '%Q[#{expr}]') + assert_equal ['#{'], + R.scan('embexpr_beg', "m(<<EOS)\n\#{expr}\nEOS") + end + +=begin + def test_embexpr_end + assert_equal [], + R.scan('embexpr_end', '') + assert_equal ['}'], + R.scan('embexpr_end', '"#{expr}"') + assert_equal [], + R.scan('embexpr_end', '%q[#{expr}]') + assert_equal ['}'], + R.scan('embexpr_end', '%Q[#{expr}]') + assert_equal ['}'], + R.scan('embexpr_end', "m(<<EOS)\n\#{expr}\nEOS") + end +=end + + def test_embvar + assert_equal [], + R.scan('embvar', '') + assert_equal ['#'], + R.scan('embvar', '"#$gvar"') + assert_equal ['#'], + R.scan('embvar', '"#@ivar"') + assert_equal ['#'], + R.scan('embvar', '"#@@cvar"') + assert_equal [], + R.scan('embvar', '"#lvar"') + assert_equal [], + R.scan('embvar', '"#"') + assert_equal [], + R.scan('embvar', '"\#$gvar"') + assert_equal [], + R.scan('embvar', '"\#@ivar"') + assert_equal [], + R.scan('embvar', '%q[#@ivar]') + assert_equal ['#'], + R.scan('embvar', '%Q[#@ivar]') + end + + def test_float + assert_equal [], + R.scan('float', '') + assert_equal ['1.000'], + R.scan('float', '1.000') + assert_equal ['123.456'], + R.scan('float', '123.456') + assert_equal ['1.2345678901234567890123456789'], + R.scan('float', '1.2345678901234567890123456789') + assert_equal ['1.000'], + R.scan('float', ' 1.000# comment') + assert_equal ['1.234e5'], + R.scan('float', '1.234e5') + assert_equal ['1.234e1234567890'], + R.scan('float', '1.234e1234567890') + assert_equal ['1.0'], + R.scan('float', 'm(a,b,1.0,c,d)') + end + + def test_gvar + assert_equal [], + R.scan('gvar', '') + assert_equal ['$a'], + R.scan('gvar', '$a') + assert_equal ['$A'], + R.scan('gvar', '$A') + assert_equal ['$gvar'], + R.scan('gvar', 'm(lvar, @ivar, @@cvar, $gvar)') + assert_equal %w($_ $~ $* $$ $? $! $@ $/ $\\ $; $, $. $= $: $< $> $"), + R.scan('gvar', 'm($_, $~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $")') + end + + def test_ident + assert_equal [], + R.scan('ident', '') + assert_equal ['lvar'], + R.scan('ident', 'lvar') + assert_equal ['m', 'lvar'], + R.scan('ident', 'm(lvar, @ivar, @@cvar, $gvar)') + end + + def test_int + assert_equal [], + R.scan('int', '') + assert_equal ['1', '10', '100000000000000'], + R.scan('int', 'm(1,10,100000000000000)') + end + + def test_ivar + assert_equal [], + R.scan('ivar', '') + assert_equal ['@ivar'], + R.scan('ivar', '@ivar') + assert_equal ['@__ivar__'], + R.scan('ivar', '@__ivar__') + assert_equal ['@IVAR'], + R.scan('ivar', '@IVAR') + assert_equal ['@ivar'], + R.scan('ivar', 'm(lvar, @ivar, @@cvar, $gvar)') + end + + def test_kw + assert_equal [], + R.scan('kw', '') + assert_equal %w(not), + R.scan('kw', 'not 1') + assert_equal %w(and), + R.scan('kw', '1 and 2') + assert_equal %w(or), + R.scan('kw', '1 or 2') + assert_equal %w(if then else end), + R.scan('kw', 'if 1 then 2 else 3 end') + assert_equal %w(if then elsif else end), + R.scan('kw', 'if 1 then 2 elsif 3 else 4 end') + assert_equal %w(unless then end), + R.scan('kw', 'unless 1 then end') + assert_equal %w(if true), + R.scan('kw', '1 if true') + assert_equal %w(unless false), + R.scan('kw', '2 unless false') + assert_equal %w(case when when else end), + R.scan('kw', 'case n; when 1; when 2; else 3 end') + assert_equal %w(while do nil end), + R.scan('kw', 'while 1 do nil end') + assert_equal %w(until do nil end), + R.scan('kw', 'until 1 do nil end') + assert_equal %w(while), + R.scan('kw', '1 while 2') + assert_equal %w(until), + R.scan('kw', '1 until 2') + assert_equal %w(while break next retry end), + R.scan('kw', 'while 1; break; next; retry end') + assert_equal %w(for in next break end), + R.scan('kw', 'for x in obj; next 1; break 2 end') + assert_equal %w(begin rescue retry end), + R.scan('kw', 'begin 1; rescue; retry; end') + assert_equal %w(rescue), + R.scan('kw', '1 rescue 2') + assert_equal %w(def redo return end), + R.scan('kw', 'def m() redo; return end') + assert_equal %w(def yield yield end), + R.scan('kw', 'def m() yield; yield 1 end') + assert_equal %w(def super super super end), + R.scan('kw', 'def m() super; super(); super(1) end') + assert_equal %w(alias), + R.scan('kw', 'alias a b') + assert_equal %w(undef), + R.scan('kw', 'undef public') + assert_equal %w(class end), + R.scan('kw', 'class A < Object; end') + assert_equal %w(module end), + R.scan('kw', 'module M; end') + assert_equal %w(class end), + R.scan('kw', 'class << obj; end') + assert_equal %w(BEGIN), + R.scan('kw', 'BEGIN { }') + assert_equal %w(END), + R.scan('kw', 'END { }') + assert_equal %w(self), + R.scan('kw', 'self.class') + assert_equal %w(nil true false), + R.scan('kw', 'p(nil, true, false)') + assert_equal %w(__FILE__ __LINE__), + R.scan('kw', 'p __FILE__, __LINE__') + assert_equal %w(defined?), + R.scan('kw', 'defined?(Object)') + end + + def test_lbrace + assert_equal [], + R.scan('lbrace', '') + assert_equal ['{'], + R.scan('lbrace', '3.times{ }') + assert_equal ['{'], + R.scan('lbrace', '3.times { }') + assert_equal ['{'], + R.scan('lbrace', '3.times{}') + assert_equal [], + R.scan('lbrace', '"{}"') + assert_equal ['{'], + R.scan('lbrace', '{1=>2}') + end + + def test_rbrace + assert_equal [], + R.scan('rbrace', '') + assert_equal ['}'], + R.scan('rbrace', '3.times{ }') + assert_equal ['}'], + R.scan('rbrace', '3.times { }') + assert_equal ['}'], + R.scan('rbrace', '3.times{}') + assert_equal [], + R.scan('rbrace', '"{}"') + assert_equal ['}'], + R.scan('rbrace', '{1=>2}') + end + + def test_lbracket + assert_equal [], + R.scan('lbracket', '') + assert_equal ['['], + R.scan('lbracket', '[]') + assert_equal ['['], + R.scan('lbracket', 'a[1]') + assert_equal [], + R.scan('lbracket', 'm(%q[])') + end + + def test_rbracket + assert_equal [], + R.scan('rbracket', '') + assert_equal [']'], + R.scan('rbracket', '[]') + assert_equal [']'], + R.scan('rbracket', 'a[1]') + assert_equal [], + R.scan('rbracket', 'm(%q[])') + end + + def test_lparen + assert_equal [], + R.scan('lparen', '') + assert_equal ['('], + R.scan('lparen', '()') + assert_equal ['('], + R.scan('lparen', 'm()') + assert_equal ['('], + R.scan('lparen', 'm (a)') + assert_equal [], + R.scan('lparen', '"()"') + assert_equal [], + R.scan('lparen', '"%w()"') + end + + def test_rparen + assert_equal [], + R.scan('rparen', '') + assert_equal [')'], + R.scan('rparen', '()') + assert_equal [')'], + R.scan('rparen', 'm()') + assert_equal [')'], + R.scan('rparen', 'm (a)') + assert_equal [], + R.scan('rparen', '"()"') + assert_equal [], + R.scan('rparen', '"%w()"') + end + + def test_op + assert_equal [], + R.scan('op', '') + assert_equal ['|'], + R.scan('op', '1 | 1') + assert_equal ['^'], + R.scan('op', '1 ^ 1') + assert_equal ['&'], + R.scan('op', '1 & 1') + assert_equal ['<=>'], + R.scan('op', '1 <=> 1') + assert_equal ['=='], + R.scan('op', '1 == 1') + assert_equal ['==='], + R.scan('op', '1 === 1') + assert_equal ['=~'], + R.scan('op', '1 =~ 1') + assert_equal ['>'], + R.scan('op', '1 > 1') + assert_equal ['>='], + R.scan('op', '1 >= 1') + assert_equal ['<'], + R.scan('op', '1 < 1') + assert_equal ['<='], + R.scan('op', '1 <= 1') + assert_equal ['<<'], + R.scan('op', '1 << 1') + assert_equal ['>>'], + R.scan('op', '1 >> 1') + assert_equal ['+'], + R.scan('op', '1 + 1') + assert_equal ['-'], + R.scan('op', '1 - 1') + assert_equal ['*'], + R.scan('op', '1 * 1') + assert_equal ['/'], + R.scan('op', '1 / 1') + assert_equal ['%'], + R.scan('op', '1 % 1') + assert_equal ['**'], + R.scan('op', '1 ** 1') + assert_equal ['~'], + R.scan('op', '~1') + assert_equal ['-'], + R.scan('op', '-a') + assert_equal ['+'], + R.scan('op', '+a') + assert_equal ['[]'], + R.scan('op', ':[]') + assert_equal ['[]='], + R.scan('op', ':[]=') + assert_equal [], + R.scan('op', %q[`make all`]) + end + + def test_symbeg + assert_equal [], + R.scan('symbeg', '') + assert_equal [':'], + R.scan('symbeg', ':sym') + assert_equal [':'], + R.scan('symbeg', '[1,2,3,:sym]') + assert_equal [], + R.scan('symbeg', '":sym"') + assert_equal [], + R.scan('symbeg', 'a ? b : c') + end + + def test_tstring_beg + assert_equal [], + R.scan('tstring_beg', '') + assert_equal ['"'], + R.scan('tstring_beg', '"abcdef"') + assert_equal ['%q['], + R.scan('tstring_beg', '%q[abcdef]') + assert_equal ['%Q['], + R.scan('tstring_beg', '%Q[abcdef]') + end + + def test_tstring_content + assert_equal [], + R.scan('tstring_content', '') + assert_equal ['abcdef'], + R.scan('tstring_content', '"abcdef"') + assert_equal ['abcdef'], + R.scan('tstring_content', '%q[abcdef]') + assert_equal ['abcdef'], + R.scan('tstring_content', '%Q[abcdef]') + assert_equal ['abc', 'def'], + R.scan('tstring_content', '"abc#{1}def"') + assert_equal ['sym'], + R.scan('tstring_content', ':"sym"') + end + + def test_tstring_end + assert_equal [], + R.scan('tstring_end', '') + assert_equal ['"'], + R.scan('tstring_end', '"abcdef"') + assert_equal [']'], + R.scan('tstring_end', '%q[abcdef]') + assert_equal [']'], + R.scan('tstring_end', '%Q[abcdef]') + end + + def test_regexp_beg + assert_equal [], + R.scan('regexp_beg', '') + assert_equal ['/'], + R.scan('regexp_beg', '/re/') + assert_equal ['%r<'], + R.scan('regexp_beg', '%r<re>') + assert_equal [], + R.scan('regexp_beg', '5 / 5') + end + + def test_regexp_end + assert_equal [], + R.scan('regexp_end', '') + assert_equal ['/'], + R.scan('regexp_end', '/re/') + assert_equal ['>'], + R.scan('regexp_end', '%r<re>') + end + + def test_words_beg + assert_equal [], + R.scan('words_beg', '') + assert_equal ['%W('], + R.scan('words_beg', '%W()') + assert_equal ['%W('], + R.scan('words_beg', '%W(w w w)') + assert_equal ['%W( '], + R.scan('words_beg', '%W( w w w )') + end + + def test_qwords_beg + assert_equal [], + R.scan('qwords_beg', '') + assert_equal ['%w('], + R.scan('qwords_beg', '%w()') + assert_equal ['%w('], + R.scan('qwords_beg', '%w(w w w)') + assert_equal ['%w( '], + R.scan('qwords_beg', '%w( w w w )') + end + + # FIXME: Close paren must not present (`words_end' scanner event?). + def test_words_sep + assert_equal [], + R.scan('words_sep', '') + assert_equal [')'], + R.scan('words_sep', '%w()') + assert_equal [' ', ' ', ')'], + R.scan('words_sep', '%w(w w w)') + assert_equal [' ', ' ', ' )'], + R.scan('words_sep', '%w( w w w )') + assert_equal ["\n", ' ', ' )'], + R.scan('words_sep', "%w( w\nw w )") + end + + def test_heredoc_beg + assert_equal [], + R.scan('heredoc_beg', '') + assert_equal ['<<EOS'], + R.scan('heredoc_beg', "<<EOS\nheredoc\nEOS") + assert_equal ['<<EOS'], + R.scan('heredoc_beg', "<<EOS\nheredoc\nEOS\n") + assert_equal ['<<EOS'], + R.scan('heredoc_beg', "<<EOS\nheredoc\nEOS \n") + assert_equal ['<<-EOS'], + R.scan('heredoc_beg', "<<-EOS\nheredoc\n\tEOS \n") + assert_equal ['<<"EOS"'], + R.scan('heredoc_beg', %Q[<<"EOS"\nheredoc\nEOS]) + assert_equal [%q(<<'EOS')], + R.scan('heredoc_beg', "<<'EOS'\nheredoc\nEOS") + assert_equal [%q(<<`EOS`)], + R.scan('heredoc_beg', "<<`EOS`\nheredoc\nEOS") + end + + def test_heredoc_content + assert_equal [], + R.scan('heredoc_content', '') + assert_equal ["heredoc\n"], + R.scan('heredoc_content', "<<EOS\nheredoc\nEOS") + assert_equal ["heredoc\n"], + R.scan('heredoc_content', "<<EOS\nheredoc\nEOS\n") + assert_equal ["heredoc \n"], + R.scan('heredoc_content', "<<EOS\nheredoc \nEOS \n") + assert_equal ["heredoc\n"], + R.scan('heredoc_content', "<<-EOS\nheredoc\n\tEOS \n") + end + + def test_heredoc_end + assert_equal [], + R.scan('heredoc_end', '') + assert_equal ["EOS"], + R.scan('heredoc_end', "<<EOS\nheredoc\nEOS") + assert_equal ["EOS\n"], + R.scan('heredoc_end', "<<EOS\nheredoc\nEOS\n") + assert_equal ["EOS \n"], + R.scan('heredoc_end', "<<EOS\nheredoc\nEOS \n") + assert_equal ["\tEOS \n"], + R.scan('heredoc_end', "<<-EOS\nheredoc\n\tEOS \n") + end + + def test_semicolon + assert_equal [], + R.scan('semicolon', '') + assert_equal %w(;), + R.scan('semicolon', ';') + assert_equal %w(; ;), + R.scan('semicolon', ';;') + assert_equal %w(; ; ;), + R.scan('semicolon', 'nil;nil;nil;') + assert_equal %w(; ; ;), + R.scan('semicolon', 'nil;nil;nil;nil') + assert_equal [], + R.scan('semicolon', '";"') + assert_equal [], + R.scan('semicolon', '%w(;)') + assert_equal [], + R.scan('semicolon', '/;/') + end + + def test_comment + assert_equal [], + R.scan('comment', '') + assert_equal ['# comment'], + R.scan('comment', '# comment') + assert_equal ["# comment\n"], + R.scan('comment', "# comment\n") + assert_equal ["# comment\n"], + R.scan('comment', "# comment\n1 + 1") + assert_equal ["# comment\n"], + R.scan('comment', "1 + 1 + 1# comment\n1 + 1") + end + + def test_embdoc_beg + assert_equal [], + R.scan('embdoc_beg', '') + assert_equal ["=begin\n"], + R.scan('embdoc_beg', "=begin\ndoc\n=end") + assert_equal ["=begin \n"], + R.scan('embdoc_beg', "=begin \ndoc\n=end\n") + assert_equal ["=begin comment\n"], + R.scan('embdoc_beg', "=begin comment\ndoc\n=end\n") + end + + def test_embdoc + assert_equal [], + R.scan('embdoc', '') + assert_equal ["doc\n"], + R.scan('embdoc', "=begin\ndoc\n=end") + assert_equal ["doc\n"], + R.scan('embdoc', "=begin\ndoc\n=end\n") + end + + def test_embdoc_end + assert_equal [], + R.scan('embdoc_end', '') + assert_equal ["=end"], + R.scan('embdoc_end', "=begin\ndoc\n=end") + assert_equal ["=end\n"], + R.scan('embdoc_end', "=begin\ndoc\n=end\n") + end + + def test_sp + assert_equal [], + R.scan('sp', '') + assert_equal [' '], + R.scan('sp', ' ') + assert_equal [' '], + R.scan('sp', ' 1') + assert_equal [], + R.scan('sp', "\n") + assert_equal [' '], + R.scan('sp', " \n") + assert_equal [' ', ' '], + R.scan('sp', "1 + 1") + assert_equal [], + R.scan('sp', "' '") + assert_equal [], + R.scan('sp', "%w( )") + assert_equal [], + R.scan('sp', "%w( w )") + assert_equal [], + R.scan('sp', "p(/ /)") + end + + # `nl' event always means End-Of-Statement. + def test_nl + assert_equal [], + R.scan('nl', '') + assert_equal [], + R.scan('nl', "\n") + assert_equal ["\n"], + R.scan('nl', "1 + 1\n") + assert_equal ["\n", "\n"], + R.scan('nl', "1 + 1\n2 + 2\n") + assert_equal [], + R.scan('nl', "1 +\n1") + assert_equal [], + R.scan('nl', "1;\n") + assert_equal ["\n"], + R.scan('nl', "1 + 1\r\n") + assert_equal [], + R.scan('nl', "1;\r\n") + end + + def test_ignored_nl + assert_equal [], + R.scan('ignored_nl', '') + assert_equal ["\n"], + R.scan('ignored_nl', "\n") + assert_equal [], + R.scan('ignored_nl', "1 + 1\n") + assert_equal [], + R.scan('ignored_nl', "1 + 1\n2 + 2\n") + assert_equal ["\n"], + R.scan('ignored_nl', "1 +\n1") + assert_equal ["\n"], + R.scan('ignored_nl', "1;\n") + assert_equal [], + R.scan('ignored_nl', "1 + 1\r\n") + assert_equal ["\n"], + R.scan('ignored_nl', "1;\r\n") + end + + def test___end__ + end + + def test_CHAR + assert_equal [], + R.scan('CHAR', "") + assert_equal ["@"], + R.scan('CHAR', "@") + assert_equal [], + R.scan('CHAR', "@ivar") + end + +end |