summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authoraamine <aamine@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2004-09-12 17:34:30 +0000
committeraamine <aamine@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>2004-09-12 17:34:30 +0000
commitc971589817150d1850b06a49e11d11edfc489bbf (patch)
tree49ad381864fc0561cc3844ecb05600ae5dbb649b
parent5bea219a9d95e58a1b2ad77637776a8e0cade1bf (diff)
* ext/ripper: ripper extention added.
* ext/ripper/MANIFEST: new file. * ext/ripper/README: new file. * ext/ripper/depend: new file. * ext/ripper/extconf.rb: new file. * ext/ripper/eventids2.c: new file. * ext/ripper/ripper.rb.in: new file. * ext/ripper/lib/ripper.rb: new file. * ext/ripper/test/check-event-arity.rb: new file. * ext/ripper/test/check-event-coverage.sh: new file. * ext/ripper/test/check-scanner-event-coverage.rb: new file. * ext/ripper/test/list-called-events.rb: new file. * ext/ripper/test/src_rb: new file. * ext/ripper/test/validate.rb: new file. * ext/ripper/tools/generate-eventids1.rb: new file. * ext/ripper/tools/generate-param-macros.rb: new file. * ext/ripper/tools/generate-ripper_rb.rb: new file. * ext/ripper/tools/list-parse-event-ids.rb: new file. * ext/ripper/tools/list-scan-event-ids.rb: new file. * ext/ripper/tools/preproc.rb: new file. * ext/ripper/tools/strip.rb: new file. * test/ripper: ripper tests added. * test/ripper/dummyparser.rb: new file. * test/ripper/test_parser_events.rb: new file. * test/ripper/test_scanner_events.rb: new file. git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6891 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
-rw-r--r--ChangeLog52
-rw-r--r--MANIFEST3
-rw-r--r--ext/ripper/MANIFEST20
-rw-r--r--ext/ripper/README31
-rw-r--r--ext/ripper/depend37
-rw-r--r--ext/ripper/eventids2.c272
-rw-r--r--ext/ripper/extconf.rb24
-rw-r--r--ext/ripper/lib/ripper.rb945
-rw-r--r--ext/ripper/ripper.rb.in53
-rw-r--r--ext/ripper/test/check-event-arity.rb24
-rwxr-xr-xext/ripper/test/check-event-coverage.sh15
-rw-r--r--ext/ripper/test/check-scanner-event-coverage.rb18
-rw-r--r--ext/ripper/test/list-called-events.rb12
-rw-r--r--ext/ripper/test/src_rb363
-rw-r--r--ext/ripper/test/validate.rb68
-rwxr-xr-xext/ripper/tools/generate-eventids1.rb18
-rwxr-xr-xext/ripper/tools/generate-param-macros.rb14
-rwxr-xr-xext/ripper/tools/generate-ripper_rb.rb44
-rwxr-xr-xext/ripper/tools/list-parse-event-ids.rb38
-rwxr-xr-xext/ripper/tools/list-scan-event-ids.rb32
-rwxr-xr-xext/ripper/tools/preproc.rb57
-rwxr-xr-xext/ripper/tools/strip.rb12
-rw-r--r--test/ripper/dummyparser.rb526
-rw-r--r--test/ripper/test_parser_events.rb489
-rw-r--r--test/ripper/test_scanner_events.rb744
25 files changed, 3911 insertions, 0 deletions
diff --git a/ChangeLog b/ChangeLog
index 552684784d..41f4e7c0bb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,55 @@
+Mon Sep 13 02:26:31 2004 Minero Aoki <aamine@loveruby.net>
+
+ * ext/ripper: ripper extention added.
+
+ * ext/ripper/MANIFEST: new file.
+
+ * ext/ripper/README: new file.
+
+ * ext/ripper/depend: new file.
+
+ * ext/ripper/extconf.rb: new file.
+
+ * ext/ripper/eventids2.c: new file.
+
+ * ext/ripper/ripper.rb.in: new file.
+
+ * ext/ripper/lib/ripper.rb: new file.
+
+ * ext/ripper/test/check-event-arity.rb: new file.
+
+ * ext/ripper/test/check-event-coverage.sh: new file.
+
+ * ext/ripper/test/check-scanner-event-coverage.rb: new file.
+
+ * ext/ripper/test/list-called-events.rb: new file.
+
+ * ext/ripper/test/src_rb: new file.
+
+ * ext/ripper/test/validate.rb: new file.
+
+ * ext/ripper/tools/generate-eventids1.rb: new file.
+
+ * ext/ripper/tools/generate-param-macros.rb: new file.
+
+ * ext/ripper/tools/generate-ripper_rb.rb: new file.
+
+ * ext/ripper/tools/list-parse-event-ids.rb: new file.
+
+ * ext/ripper/tools/list-scan-event-ids.rb: new file.
+
+ * ext/ripper/tools/preproc.rb: new file.
+
+ * ext/ripper/tools/strip.rb: new file.
+
+ * test/ripper: ripper tests added.
+
+ * test/ripper/dummyparser.rb: new file.
+
+ * test/ripper/test_parser_events.rb: new file.
+
+ * test/ripper/test_scanner_events.rb: new file.
+
Mon Sep 13 01:03:02 2004 Hidetoshi NAGAI <nagai@ai.kyutech.ac.jp>
* ext/tcltklib/tcltklib.c: improve control of preserv/release tcltkip
diff --git a/MANIFEST b/MANIFEST
index eaa424ffd3..47072cd648 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -878,6 +878,9 @@ test/optparse/test_placearg.rb
test/optparse/test_reqarg.rb
test/ostruct/test_ostruct.rb
test/rinda/test_rinda.rb
+test/ripper/dummyparser.rb
+test/ripper/test_parser_events.rb
+test/ripper/test_scanner_events.rb
test/rss/rss-assertions.rb
test/rss/rss-testcase.rb
test/rss/test_1.0.rb
diff --git a/ext/ripper/MANIFEST b/ext/ripper/MANIFEST
new file mode 100644
index 0000000000..fe1e25e620
--- /dev/null
+++ b/ext/ripper/MANIFEST
@@ -0,0 +1,20 @@
+MANIFEST
+README
+depend
+eventids2.c
+extconf.rb
+ripper.rb.in
+lib/ripper.rb
+test/check-event-arity.rb
+test/check-event-coverage.sh
+test/check-scanner-event-coverage.rb
+test/list-called-events.rb
+test/src.rb
+test/validate.rb
+tools/generate-eventids1.rb
+tools/generate-param-macros.rb
+tools/generate-ripper_rb.rb
+tools/list-parse-event-ids.rb
+tools/list-scan-event-ids.rb
+tools/preproc.rb
+tools/strip.rb
diff --git a/ext/ripper/README b/ext/ripper/README
new file mode 100644
index 0000000000..5242c4256d
--- /dev/null
+++ b/ext/ripper/README
@@ -0,0 +1,31 @@
+Ripper README
+=============
+
+ Ripper is a Ruby script parser. You can get information
+ by event-based style from the parser.
+
+ !! WARNING !!
+
+ Ripper is still early-alpha version.
+ I never assure any kind of backward compatibility.
+
+Requirements
+------------
+
+ * ruby 1.9 (support CVS HEAD only)
+ * bison 1.28 or later (You MUST use bison)
+
+Usage
+-----
+
+ See test/ripper/* and sample/ripper/*.
+ To know thrown events, see lib/ripper.rb.
+
+License
+-------
+
+ Ruby License.
+
+ Minero Aoki
+ aamine@loveruby.net
+ http://i.loveruby.net
diff --git a/ext/ripper/depend b/ext/ripper/depend
new file mode 100644
index 0000000000..b2333a2fdf
--- /dev/null
+++ b/ext/ripper/depend
@@ -0,0 +1,37 @@
+ripper.o: ripper.c lex.c eventids1.c eventids2.c
+ $(CC) $(CFLAGS) $(CPPFLAGS) -c ripper.c
+
+ripper.c: ripper.y
+ bison -t -v -o$@ ripper.y
+
+ripper.y: $(hdrdir)/parse.y $(srcdir)/tools/preproc.rb
+ $(RUBY) $(srcdir)/tools/preproc.rb $(hdrdir)/parse.y > $@
+
+eventids1.c: $(hdrdir)/parse.y $(srcdir)/tools/list-parse-event-ids.rb $(srcdir)/tools/generate-eventids1.rb
+ $(RUBY) $(srcdir)/tools/list-parse-event-ids.rb $(hdrdir)/parse.y | $(RUBY) $(srcdir)/tools/generate-eventids1.rb > $@
+
+lex.c:
+ cp $(hdrdir)/lex.c . || cp $(srcdir)/lex.c .
+
+#
+# Entries for ripper developpers (no srcdir/objdir support)
+#
+lib/ripper.rb: ripper.rb.in ids1 ids2 tools/generate-ripper_rb.rb
+ $(RUBY) tools/generate-ripper_rb.rb ripper.rb.in ids1 ids2 > $@
+
+ids1: tools/list-parse-event-ids.rb ../../parse.y
+ $(RUBY) tools/list-parse-event-ids.rb -a ../../parse.y > $@
+
+ids2: $(srcdir)/tools/list-scan-event-ids.rb eventids2.c
+ $(RUBY) tools/list-scan-event-ids.rb -a eventids2.c > $@
+
+test: ripper.so test/*.*
+ $(RUBY) test/check-event-arity.rb $(hdrdir)/parse.y
+ sh test/check-event-coverage.sh
+ $(RUBY) test/check-scanner-event-coverage.rb
+ $(RUBY) test/validate.rb > /dev/null
+ @echo OK
+
+preproc: ripper.E
+ripper.E: ripper.c
+ $(CC) -E $(CPPFLAGS) ripper.c | ruby tools/strip.rb > $@
diff --git a/ext/ripper/eventids2.c b/ext/ripper/eventids2.c
new file mode 100644
index 0000000000..28519eb8e6
--- /dev/null
+++ b/ext/ripper/eventids2.c
@@ -0,0 +1,272 @@
+#define tIGNORED_NL (tLAST_TOKEN + 1)
+#define tCOMMENT (tLAST_TOKEN + 2)
+#define tEMBDOC_BEG (tLAST_TOKEN + 3)
+#define tEMBDOC (tLAST_TOKEN + 4)
+#define tEMBDOC_END (tLAST_TOKEN + 5)
+#define tLWSP (tLAST_TOKEN + 6)
+#define tHEREDOC_BEG (tLAST_TOKEN + 7)
+#define tHEREDOC_CONTENT (tLAST_TOKEN + 8)
+#define tHEREDOC_END (tLAST_TOKEN + 9)
+#define k__END__ (tLAST_TOKEN + 10)
+#define tCHAR (tLAST_TOKEN + 11)
+
+static ID ripper_id_scan;
+
+static ID ripper_id_backref;
+static ID ripper_id_backtick;
+static ID ripper_id_comma;
+static ID ripper_id_const;
+static ID ripper_id_cvar;
+static ID ripper_id_embexpr_beg;
+static ID ripper_id_embexpr_end;
+static ID ripper_id_embvar;
+static ID ripper_id_float;
+static ID ripper_id_gvar;
+static ID ripper_id_ident;
+static ID ripper_id_int;
+static ID ripper_id_ivar;
+static ID ripper_id_kw;
+static ID ripper_id_lbrace;
+static ID ripper_id_lbracket;
+static ID ripper_id_lparen;
+static ID ripper_id_nl;
+static ID ripper_id_op;
+static ID ripper_id_period;
+static ID ripper_id_rbrace;
+static ID ripper_id_rbracket;
+static ID ripper_id_rparen;
+static ID ripper_id_semicolon;
+static ID ripper_id_symbeg;
+static ID ripper_id_tstring_beg;
+static ID ripper_id_tstring_content;
+static ID ripper_id_tstring_end;
+static ID ripper_id_words_beg;
+static ID ripper_id_qwords_beg;
+static ID ripper_id_words_sep;
+static ID ripper_id_regexp_beg;
+static ID ripper_id_regexp_end;
+
+static ID ripper_id_ignored_nl;
+static ID ripper_id_comment;
+static ID ripper_id_embdoc_beg;
+static ID ripper_id_embdoc;
+static ID ripper_id_embdoc_end;
+static ID ripper_id_lwsp;
+static ID ripper_id_heredoc_beg;
+static ID ripper_id_heredoc_content;
+static ID ripper_id_heredoc_end;
+static ID ripper_id___end__;
+static ID ripper_id_CHAR;
+
+static void
+ripper_init_eventids2()
+{
+ ripper_id_scan = rb_intern("on__scan");
+
+ ripper_id_backref = rb_intern("on__backref");
+ ripper_id_backtick = rb_intern("on__backtick");
+ ripper_id_comma = rb_intern("on__comma");
+ ripper_id_const = rb_intern("on__const");
+ ripper_id_cvar = rb_intern("on__cvar");
+ ripper_id_embexpr_beg = rb_intern("on__embexpr_beg");
+ ripper_id_embexpr_end = rb_intern("on__embexpr_end");
+ ripper_id_embvar = rb_intern("on__embvar");
+ ripper_id_float = rb_intern("on__float");
+ ripper_id_gvar = rb_intern("on__gvar");
+ ripper_id_ident = rb_intern("on__ident");
+ ripper_id_int = rb_intern("on__int");
+ ripper_id_ivar = rb_intern("on__ivar");
+ ripper_id_kw = rb_intern("on__kw");
+ ripper_id_lbrace = rb_intern("on__lbrace");
+ ripper_id_lbracket = rb_intern("on__lbracket");
+ ripper_id_lparen = rb_intern("on__lparen");
+ ripper_id_nl = rb_intern("on__nl");
+ ripper_id_op = rb_intern("on__op");
+ ripper_id_period = rb_intern("on__period");
+ ripper_id_rbrace = rb_intern("on__rbrace");
+ ripper_id_rbracket = rb_intern("on__rbracket");
+ ripper_id_rparen = rb_intern("on__rparen");
+ ripper_id_semicolon = rb_intern("on__semicolon");
+ ripper_id_symbeg = rb_intern("on__symbeg");
+ ripper_id_tstring_beg = rb_intern("on__tstring_beg");
+ ripper_id_tstring_content = rb_intern("on__tstring_content");
+ ripper_id_tstring_end = rb_intern("on__tstring_end");
+ ripper_id_words_beg = rb_intern("on__words_beg");
+ ripper_id_qwords_beg = rb_intern("on__qwords_beg");
+ ripper_id_words_sep = rb_intern("on__words_sep");
+ ripper_id_regexp_beg = rb_intern("on__regexp_beg");
+ ripper_id_regexp_end = rb_intern("on__regexp_end");
+
+ ripper_id_ignored_nl = rb_intern("on__ignored_nl");
+ ripper_id_comment = rb_intern("on__comment");
+ ripper_id_embdoc_beg = rb_intern("on__embdoc_beg");
+ ripper_id_embdoc = rb_intern("on__embdoc");
+ ripper_id_embdoc_end = rb_intern("on__embdoc_end");
+ ripper_id_lwsp = rb_intern("on__sp");
+ ripper_id_heredoc_beg = rb_intern("on__heredoc_beg");
+ ripper_id_heredoc_content = rb_intern("on__heredoc_content");
+ ripper_id_heredoc_end = rb_intern("on__heredoc_end");
+ ripper_id___end__ = rb_intern("on____end__");
+ ripper_id_CHAR = rb_intern("on__CHAR");
+}
+
+static struct token_assoc {
+ int token;
+ ID *id;
+} token_to_eventid[] = {
+ {' ', &ripper_id_words_sep},
+ {'!', &ripper_id_op},
+ {'%', &ripper_id_op},
+ {'&', &ripper_id_op},
+ {'*', &ripper_id_op},
+ {'+', &ripper_id_op},
+ {'-', &ripper_id_op},
+ {'/', &ripper_id_op},
+ {'<', &ripper_id_op},
+ {'=', &ripper_id_op},
+ {'>', &ripper_id_op},
+ {'?', &ripper_id_op},
+ {'^', &ripper_id_op},
+ {'|', &ripper_id_op},
+ {'~', &ripper_id_op},
+ {':', &ripper_id_op},
+ {',', &ripper_id_comma},
+ {'.', &ripper_id_period},
+ {';', &ripper_id_semicolon},
+ {'`', &ripper_id_backtick},
+ {'\n', &ripper_id_nl},
+ {kALIAS, &ripper_id_kw},
+ {kAND, &ripper_id_kw},
+ {kBEGIN, &ripper_id_kw},
+ {kBREAK, &ripper_id_kw},
+ {kCASE, &ripper_id_kw},
+ {kCLASS, &ripper_id_kw},
+ {kDEF, &ripper_id_kw},
+ {kDEFINED, &ripper_id_kw},
+ {kDO, &ripper_id_kw},
+ {kDO_BLOCK, &ripper_id_kw},
+ {kDO_COND, &ripper_id_kw},
+ {kELSE, &ripper_id_kw},
+ {kELSIF, &ripper_id_kw},
+ {kEND, &ripper_id_kw},
+ {kENSURE, &ripper_id_kw},
+ {kFALSE, &ripper_id_kw},
+ {kFOR, &ripper_id_kw},
+ {kIF, &ripper_id_kw},
+ {kIF_MOD, &ripper_id_kw},
+ {kIN, &ripper_id_kw},
+ {kMODULE, &ripper_id_kw},
+ {kNEXT, &ripper_id_kw},
+ {kNIL, &ripper_id_kw},
+ {kNOT, &ripper_id_kw},
+ {kOR, &ripper_id_kw},
+ {kREDO, &ripper_id_kw},
+ {kRESCUE, &ripper_id_kw},
+ {kRESCUE_MOD, &ripper_id_kw},
+ {kRETRY, &ripper_id_kw},
+ {kRETURN, &ripper_id_kw},
+ {kSELF, &ripper_id_kw},
+ {kSUPER, &ripper_id_kw},
+ {kTHEN, &ripper_id_kw},
+ {kTRUE, &ripper_id_kw},
+ {kUNDEF, &ripper_id_kw},
+ {kUNLESS, &ripper_id_kw},
+ {kUNLESS_MOD, &ripper_id_kw},
+ {kUNTIL, &ripper_id_kw},
+ {kUNTIL_MOD, &ripper_id_kw},
+ {kWHEN, &ripper_id_kw},
+ {kWHILE, &ripper_id_kw},
+ {kWHILE_MOD, &ripper_id_kw},
+ {kYIELD, &ripper_id_kw},
+ {k__FILE__, &ripper_id_kw},
+ {k__LINE__, &ripper_id_kw},
+ {klBEGIN, &ripper_id_kw},
+ {klEND, &ripper_id_kw},
+ {tAMPER, &ripper_id_op},
+ {tANDOP, &ripper_id_op},
+ {tAREF, &ripper_id_op},
+ {tASET, &ripper_id_op},
+ {tASSOC, &ripper_id_op},
+ {tBACK_REF, &ripper_id_backref},
+ {tCMP, &ripper_id_op},
+ {tCOLON2, &ripper_id_op},
+ {tCOLON3, &ripper_id_op},
+ {tCONSTANT, &ripper_id_const},
+ {tCVAR, &ripper_id_cvar},
+ {tDOT2, &ripper_id_op},
+ {tDOT3, &ripper_id_op},
+ {tEQ, &ripper_id_op},
+ {tEQQ, &ripper_id_op},
+ {tFID, &ripper_id_ident},
+ {tFLOAT, &ripper_id_float},
+ {tGEQ, &ripper_id_op},
+ {tGVAR, &ripper_id_gvar},
+ {tIDENTIFIER, &ripper_id_ident},
+ {tINTEGER, &ripper_id_int},
+ {tIVAR, &ripper_id_ivar},
+ {tLBRACE, &ripper_id_lbrace},
+ {tLBRACE_ARG, &ripper_id_lbrace},
+ {'{', &ripper_id_lbrace},
+ {'}', &ripper_id_rbrace},
+ {tLBRACK, &ripper_id_lbracket},
+ {'[', &ripper_id_lbracket},
+ {']', &ripper_id_rbracket},
+ {tLEQ, &ripper_id_op},
+ {tLPAREN, &ripper_id_lparen},
+ {tLPAREN_ARG, &ripper_id_lparen},
+ {'(', &ripper_id_lparen},
+ {')', &ripper_id_rparen},
+ {tLSHFT, &ripper_id_op},
+ {tMATCH, &ripper_id_op},
+ {tNEQ, &ripper_id_op},
+ {tNMATCH, &ripper_id_op},
+ {tNTH_REF, &ripper_id_backref},
+ {tOP_ASGN, &ripper_id_op},
+ {tOROP, &ripper_id_op},
+ {tPOW, &ripper_id_op},
+ {tQWORDS_BEG, &ripper_id_qwords_beg},
+ {tREGEXP_BEG, &ripper_id_regexp_beg},
+ {tREGEXP_END, &ripper_id_regexp_end},
+ {tRPAREN, &ripper_id_rparen},
+ {tRSHFT, &ripper_id_op},
+ {tSTAR, &ripper_id_op},
+ {tSTRING_BEG, &ripper_id_tstring_beg},
+ {tSTRING_CONTENT, &ripper_id_tstring_content},
+ {tSTRING_DBEG, &ripper_id_embexpr_beg},
+ {tSTRING_DVAR, &ripper_id_embvar},
+ {tSTRING_END, &ripper_id_tstring_end},
+ {tSYMBEG, &ripper_id_symbeg},
+ {tUMINUS, &ripper_id_op},
+ {tUMINUS_NUM, &ripper_id_op},
+ {tUPLUS, &ripper_id_op},
+ {tWORDS_BEG, &ripper_id_words_beg},
+ {tXSTRING_BEG, &ripper_id_backtick},
+/* ripper specific tokens */
+ {tIGNORED_NL, &ripper_id_ignored_nl},
+ {tCOMMENT, &ripper_id_comment},
+ {tEMBDOC_BEG, &ripper_id_embdoc_beg},
+ {tEMBDOC, &ripper_id_embdoc},
+ {tEMBDOC_END, &ripper_id_embdoc_end},
+ {tLWSP, &ripper_id_lwsp},
+ {tHEREDOC_BEG, &ripper_id_heredoc_beg},
+ {tHEREDOC_CONTENT, &ripper_id_heredoc_content},
+ {tHEREDOC_END, &ripper_id_heredoc_end},
+ {k__END__, &ripper_id___end__},
+ {0, NULL}
+};
+
+static ID
+ripper_token2eventid(tok)
+ int tok;
+{
+ struct token_assoc *a;
+
+ for (a = token_to_eventid; a->id != NULL; a++) {
+ if (a->token == tok)
+ return *a->id;
+ }
+ if (tok < 256) {
+ return ripper_id_CHAR;
+ }
+ rb_raise(rb_eRuntimeError, "[Ripper FATAL] unknown token %d", tok);
+}
diff --git a/ext/ripper/extconf.rb b/ext/ripper/extconf.rb
new file mode 100644
index 0000000000..536e23d3ad
--- /dev/null
+++ b/ext/ripper/extconf.rb
@@ -0,0 +1,24 @@
+#!ruby -s
+
+require 'mkmf'
+require 'rbconfig'
+
+def main
+ unless have_command('bison')
+ $stderr.puts 'Ripper requires bison; abort'
+ exit 1
+ end
+ $objs = %w(ripper.o)
+ $cleanfiles.concat %w(ripper.y ripper.c ripper.output ids1 ids2)
+ $CPPFLAGS += ' -DRIPPER'
+ $CPPFLAGS += ' -DRIPPER_DEBUG' if $debug
+ create_makefile 'ripper'
+end
+
+def have_command(cmd)
+ ENV['PATH'].split(File::PATH_SEPARATOR).any? {|path|
+ File.executable?("#{path}/#{cmd}#{Config::CONFIG['EXEEXT']}")
+ }
+end
+
+main
diff --git a/ext/ripper/lib/ripper.rb b/ext/ripper/lib/ripper.rb
new file mode 100644
index 0000000000..b34147ee2a
--- /dev/null
+++ b/ext/ripper/lib/ripper.rb
@@ -0,0 +1,945 @@
+#
+# This file is automatically generated from ripper.rb.in and parse.y.
+# DO NOT MODIFY!!!!!!
+#
+
+#
+# ripper.rb
+#
+# Copyright (C) 2003,2004 Minero Aoki
+#
+
+require 'ripper.so'
+
+class Ripper
+ def Ripper.parse(str, *args)
+ new.parse(str, *args)
+ end
+
+ private
+
+ def warn(fmt, *args)
+ end
+
+ def warning(fmt, *args)
+ end
+
+ def compile_error(msg)
+ end
+
+ PARSER_EVENTS = [
+ :BEGIN,
+ :END,
+ :alias,
+ :alias_error,
+ :aref,
+ :aref_field,
+ :arg_ambiguous,
+ :arg_paren,
+ :arglist_add,
+ :arglist_add_block,
+ :arglist_add_star,
+ :arglist_new,
+ :arglist_prepend,
+ :array,
+ :assign,
+ :assign_error,
+ :assoc_new,
+ :assoclist_from_args,
+ :bare_assoc_hash,
+ :begin,
+ :binary,
+ :blockvar_add_block,
+ :blockvar_add_star,
+ :blockvar_new,
+ :bodystmt,
+ :brace_block,
+ :break,
+ :call,
+ :case,
+ :class,
+ :class_name_error,
+ :command,
+ :command_call,
+ :const_ref,
+ :constpath_field,
+ :constpath_ref,
+ :def,
+ :defined,
+ :defs,
+ :do_block,
+ :dot2,
+ :dot3,
+ :dyna_symbol,
+ :else,
+ :elsif,
+ :ensure,
+ :fcall,
+ :field,
+ :for,
+ :hash,
+ :if,
+ :if_mod,
+ :ifop,
+ :iter_block,
+ :massign,
+ :method_add_arg,
+ :mlhs_add,
+ :mlhs_add_star,
+ :mlhs_new,
+ :mlhs_paren,
+ :module,
+ :mrhs_add,
+ :mrhs_add_star,
+ :mrhs_new,
+ :mrhs_new_from_arglist,
+ :next,
+ :opassign,
+ :param_error,
+ :params,
+ :paren,
+ :parse_error,
+ :program,
+ :qwords_add,
+ :qwords_new,
+ :redo,
+ :regexp_literal,
+ :rescue,
+ :rescue_mod,
+ :restparam,
+ :retry,
+ :return,
+ :return0,
+ :sclass,
+ :space,
+ :stmts_add,
+ :stmts_new,
+ :string_add,
+ :string_concat,
+ :string_content,
+ :string_dvar,
+ :string_embexpr,
+ :string_literal,
+ :super,
+ :symbol,
+ :symbol_literal,
+ :topconst_field,
+ :topconst_ref,
+ :unary,
+ :undef,
+ :unless,
+ :unless_mod,
+ :until,
+ :until_mod,
+ :var_alias,
+ :var_field,
+ :var_ref,
+ :void_stmt,
+ :when,
+ :while,
+ :while_mod,
+ :word_add,
+ :word_new,
+ :words_add,
+ :words_new,
+ :xstring_add,
+ :xstring_literal,
+ :xstring_new,
+ :yield,
+ :yield0,
+ :zsuper
+ ]
+
+ SCANNER_EVENTS = [
+ :BEGIN,
+ :END,
+ :alias,
+ :alias_error,
+ :aref,
+ :aref_field,
+ :arg_ambiguous,
+ :arg_paren,
+ :arglist_add,
+ :arglist_add_block,
+ :arglist_add_star,
+ :arglist_new,
+ :arglist_prepend,
+ :array,
+ :assign,
+ :assign_error,
+ :assoc_new,
+ :assoclist_from_args,
+ :bare_assoc_hash,
+ :begin,
+ :binary,
+ :blockvar_add_block,
+ :blockvar_add_star,
+ :blockvar_new,
+ :bodystmt,
+ :brace_block,
+ :break,
+ :call,
+ :case,
+ :class,
+ :class_name_error,
+ :command,
+ :command_call,
+ :const_ref,
+ :constpath_field,
+ :constpath_ref,
+ :def,
+ :defined,
+ :defs,
+ :do_block,
+ :dot2,
+ :dot3,
+ :dyna_symbol,
+ :else,
+ :elsif,
+ :ensure,
+ :fcall,
+ :field,
+ :for,
+ :hash,
+ :if,
+ :if_mod,
+ :ifop,
+ :iter_block,
+ :massign,
+ :method_add_arg,
+ :mlhs_add,
+ :mlhs_add_star,
+ :mlhs_new,
+ :mlhs_paren,
+ :module,
+ :mrhs_add,
+ :mrhs_add_star,
+ :mrhs_new,
+ :mrhs_new_from_arglist,
+ :next,
+ :opassign,
+ :param_error,
+ :params,
+ :paren,
+ :parse_error,
+ :program,
+ :qwords_add,
+ :qwords_new,
+ :redo,
+ :regexp_literal,
+ :rescue,
+ :rescue_mod,
+ :restparam,
+ :retry,
+ :return,
+ :return0,
+ :sclass,
+ :space,
+ :stmts_add,
+ :stmts_new,
+ :string_add,
+ :string_concat,
+ :string_content,
+ :string_dvar,
+ :string_embexpr,
+ :string_literal,
+ :super,
+ :symbol,
+ :symbol_literal,
+ :topconst_field,
+ :topconst_ref,
+ :unary,
+ :undef,
+ :unless,
+ :unless_mod,
+ :until,
+ :until_mod,
+ :var_alias,
+ :var_field,
+ :var_ref,
+ :void_stmt,
+ :when,
+ :while,
+ :while_mod,
+ :word_add,
+ :word_new,
+ :words_add,
+ :words_new,
+ :xstring_add,
+ :xstring_literal,
+ :xstring_new,
+ :yield,
+ :yield0,
+ :zsuper
+ ]
+ LEXER_EVENTS = SCANNER_EVENTS
+
+ EVENTS = PARSER_EVENTS + SCANNER_EVENTS
+
+ #
+ # Parser Events
+ #
+
+ def on__BEGIN(a)
+ a
+ end
+
+ def on__END(a)
+ a
+ end
+
+ def on__alias(a, b)
+ a
+ end
+
+ def on__alias_error(a)
+ a
+ end
+
+ def on__aref(a, b)
+ a
+ end
+
+ def on__aref_field(a, b)
+ a
+ end
+
+ def on__arg_ambiguous()
+ nil
+ end
+
+ def on__arg_paren(a)
+ a
+ end
+
+ def on__arglist_add(a, b)
+ a
+ end
+
+ def on__arglist_add_block(a, b)
+ a
+ end
+
+ def on__arglist_add_star(a, b)
+ a
+ end
+
+ def on__arglist_new()
+ nil
+ end
+
+ def on__arglist_prepend(a, b)
+ a
+ end
+
+ def on__array(a)
+ a
+ end
+
+ def on__assign(a, b)
+ a
+ end
+
+ def on__assign_error(a)
+ a
+ end
+
+ def on__assoc_new(a, b)
+ a
+ end
+
+ def on__assoclist_from_args(a)
+ a
+ end
+
+ def on__bare_assoc_hash(a)
+ a
+ end
+
+ def on__begin(a)
+ a
+ end
+
+ def on__binary(a, b, c)
+ a
+ end
+
+ def on__blockvar_add_block(a, b)
+ a
+ end
+
+ def on__blockvar_add_star(a, b)
+ a
+ end
+
+ def on__blockvar_new(a)
+ a
+ end
+
+ def on__bodystmt(a, b, c, d)
+ a
+ end
+
+ def on__brace_block(a, b)
+ a
+ end
+
+ def on__break(a)
+ a
+ end
+
+ def on__call(a, b, c)
+ a
+ end
+
+ def on__case(a, b)
+ a
+ end
+
+ def on__class(a, b, c)
+ a
+ end
+
+ def on__class_name_error(a)
+ a
+ end
+
+ def on__command(a, b)
+ a
+ end
+
+ def on__command_call(a, b, c, d)
+ a
+ end
+
+ def on__const_ref(a)
+ a
+ end
+
+ def on__constpath_field(a, b)
+ a
+ end
+
+ def on__constpath_ref(a, b)
+ a
+ end
+
+ def on__def(a, b, c)
+ a
+ end
+
+ def on__defined(a)
+ a
+ end
+
+ def on__defs(a, b, c, d, e)
+ a
+ end
+
+ def on__do_block(a, b)
+ a
+ end
+
+ def on__dot2(a, b)
+ a
+ end
+
+ def on__dot3(a, b)
+ a
+ end
+
+ def on__dyna_symbol(a)
+ a
+ end
+
+ def on__else(a)
+ a
+ end
+
+ def on__elsif(a, b, c)
+ a
+ end
+
+ def on__ensure(a)
+ a
+ end
+
+ def on__fcall(a)
+ a
+ end
+
+ def on__field(a, b, c)
+ a
+ end
+
+ def on__for(a, b, c)
+ a
+ end
+
+ def on__hash(a)
+ a
+ end
+
+ def on__if(a, b, c)
+ a
+ end
+
+ def on__if_mod(a, b)
+ a
+ end
+
+ def on__ifop(a, b, c)
+ a
+ end
+
+ def on__iter_block(a, b)
+ a
+ end
+
+ def on__massign(a, b)
+ a
+ end
+
+ def on__method_add_arg(a, b)
+ a
+ end
+
+ def on__mlhs_add(a, b)
+ a
+ end
+
+ def on__mlhs_add_star(a, b)
+ a
+ end
+
+ def on__mlhs_new()
+ nil
+ end
+
+ def on__mlhs_paren(a)
+ a
+ end
+
+ def on__module(a, b)
+ a
+ end
+
+ def on__mrhs_add(a, b)
+ a
+ end
+
+ def on__mrhs_add_star(a, b)
+ a
+ end
+
+ def on__mrhs_new()
+ nil
+ end
+
+ def on__mrhs_new_from_arglist(a)
+ a
+ end
+
+ def on__next(a)
+ a
+ end
+
+ def on__opassign(a, b, c)
+ a
+ end
+
+ def on__param_error(a)
+ a
+ end
+
+ def on__params(a, b, c, d)
+ a
+ end
+
+ def on__paren(a)
+ a
+ end
+
+ def on__parse_error(a)
+ a
+ end
+
+ def on__program(a)
+ a
+ end
+
+ def on__qwords_add(a, b)
+ a
+ end
+
+ def on__qwords_new()
+ nil
+ end
+
+ def on__redo()
+ nil
+ end
+
+ def on__regexp_literal(a)
+ a
+ end
+
+ def on__rescue(a, b, c, d)
+ a
+ end
+
+ def on__rescue_mod(a, b)
+ a
+ end
+
+ def on__restparam(a)
+ a
+ end
+
+ def on__retry()
+ nil
+ end
+
+ def on__return(a)
+ a
+ end
+
+ def on__return0()
+ nil
+ end
+
+ def on__sclass(a, b)
+ a
+ end
+
+ def on__space(a)
+ a
+ end
+
+ def on__stmts_add(a, b)
+ a
+ end
+
+ def on__stmts_new()
+ nil
+ end
+
+ def on__string_add(a, b)
+ a
+ end
+
+ def on__string_concat(a, b)
+ a
+ end
+
+ def on__string_content()
+ nil
+ end
+
+ def on__string_dvar(a)
+ a
+ end
+
+ def on__string_embexpr(a)
+ a
+ end
+
+ def on__string_literal(a)
+ a
+ end
+
+ def on__super(a)
+ a
+ end
+
+ def on__symbol(a)
+ a
+ end
+
+ def on__symbol_literal(a)
+ a
+ end
+
+ def on__topconst_field(a)
+ a
+ end
+
+ def on__topconst_ref(a)
+ a
+ end
+
+ def on__unary(a, b)
+ a
+ end
+
+ def on__undef(a)
+ a
+ end
+
+ def on__unless(a, b, c)
+ a
+ end
+
+ def on__unless_mod(a, b)
+ a
+ end
+
+ def on__until(a, b)
+ a
+ end
+
+ def on__until_mod(a, b)
+ a
+ end
+
+ def on__var_alias(a, b)
+ a
+ end
+
+ def on__var_field(a)
+ a
+ end
+
+ def on__var_ref(a)
+ a
+ end
+
+ def on__void_stmt()
+ nil
+ end
+
+ def on__when(a, b, c)
+ a
+ end
+
+ def on__while(a, b)
+ a
+ end
+
+ def on__while_mod(a, b)
+ a
+ end
+
+ def on__word_add(a, b)
+ a
+ end
+
+ def on__word_new()
+ nil
+ end
+
+ def on__words_add(a, b)
+ a
+ end
+
+ def on__words_new()
+ nil
+ end
+
+ def on__xstring_add(a, b)
+ a
+ end
+
+ def on__xstring_literal(a)
+ a
+ end
+
+ def on__xstring_new()
+ nil
+ end
+
+ def on__yield(a)
+ a
+ end
+
+ def on__yield0()
+ nil
+ end
+
+ def on__zsuper()
+ nil
+ end
+
+ #
+ # Lexer Events
+ #
+
+ def on__scan(event, token)
+ end
+
+ def on__CHAR(token)
+ token
+ end
+
+ def on____end__(token)
+ token
+ end
+
+ def on__backref(token)
+ token
+ end
+
+ def on__backtick(token)
+ token
+ end
+
+ def on__comma(token)
+ token
+ end
+
+ def on__comment(token)
+ token
+ end
+
+ def on__const(token)
+ token
+ end
+
+ def on__cvar(token)
+ token
+ end
+
+ def on__embdoc(token)
+ token
+ end
+
+ def on__embdoc_beg(token)
+ token
+ end
+
+ def on__embdoc_end(token)
+ token
+ end
+
+ def on__embexpr_beg(token)
+ token
+ end
+
+ def on__embexpr_end(token)
+ token
+ end
+
+ def on__embvar(token)
+ token
+ end
+
+ def on__float(token)
+ token
+ end
+
+ def on__gvar(token)
+ token
+ end
+
+ def on__heredoc_beg(token)
+ token
+ end
+
+ def on__heredoc_content(token)
+ token
+ end
+
+ def on__heredoc_end(token)
+ token
+ end
+
+ def on__ident(token)
+ token
+ end
+
+ def on__ignored_nl(token)
+ token
+ end
+
+ def on__int(token)
+ token
+ end
+
+ def on__ivar(token)
+ token
+ end
+
+ def on__kw(token)
+ token
+ end
+
+ def on__lbrace(token)
+ token
+ end
+
+ def on__lbracket(token)
+ token
+ end
+
+ def on__lparen(token)
+ token
+ end
+
+ def on__lwsp(token)
+ token
+ end
+
+ def on__nl(token)
+ token
+ end
+
+ def on__op(token)
+ token
+ end
+
+ def on__period(token)
+ token
+ end
+
+ def on__qwords_beg(token)
+ token
+ end
+
+ def on__rbrace(token)
+ token
+ end
+
+ def on__rbracket(token)
+ token
+ end
+
+ def on__regexp_beg(token)
+ token
+ end
+
+ def on__regexp_end(token)
+ token
+ end
+
+ def on__rparen(token)
+ token
+ end
+
+ def on__semicolon(token)
+ token
+ end
+
+ def on__symbeg(token)
+ token
+ end
+
+ def on__tstring_beg(token)
+ token
+ end
+
+ def on__tstring_content(token)
+ token
+ end
+
+ def on__tstring_end(token)
+ token
+ end
+
+ def on__words_beg(token)
+ token
+ end
+
+ def on__words_sep(token)
+ token
+ end
+end
diff --git a/ext/ripper/ripper.rb.in b/ext/ripper/ripper.rb.in
new file mode 100644
index 0000000000..c6fa554296
--- /dev/null
+++ b/ext/ripper/ripper.rb.in
@@ -0,0 +1,53 @@
+#
+# This file is automatically generated from ripper.rb.in and parse.y.
+# DO NOT MODIFY!!!!!!
+#
+
+#
+# ripper.rb
+#
+# Copyright (C) 2003,2004 Minero Aoki
+#
+
+require 'ripper.so'
+
+class Ripper
+ def Ripper.parse(str, *args)
+ new.parse(str, *args)
+ end
+
+ private
+
+ def warn(fmt, *args)
+ end
+
+ def warning(fmt, *args)
+ end
+
+ def compile_error(msg)
+ end
+
+ PARSER_EVENTS = [
+#include ids1
+ ]
+
+ SCANNER_EVENTS = [
+#include ids2
+ ]
+ LEXER_EVENTS = SCANNER_EVENTS
+
+ EVENTS = PARSER_EVENTS + SCANNER_EVENTS
+
+ #
+ # Parser Events
+ #
+#include handlers1
+
+ #
+ # Lexer Events
+ #
+
+ def on__scan(event, token)
+ end
+#include handlers2
+end
diff --git a/ext/ripper/test/check-event-arity.rb b/ext/ripper/test/check-event-arity.rb
new file mode 100644
index 0000000000..00e1b98dbc
--- /dev/null
+++ b/ext/ripper/test/check-event-arity.rb
@@ -0,0 +1,24 @@
+def main
+ invalid = false
+ table = {}
+ ARGF.each do |line|
+ next if /\A\#\s*define\s+s?dispatch\d/ === line
+ next if /ripper_dispatch\d/ === line
+ line.scan(/dispatch(\d)\((\w+)/) do |num, ev|
+ num = num.to_i
+ if data = table[ev]
+ locations, arity = data
+ unless num == arity
+ invalid = true
+ puts "arity differ [#{ev}]: #{ARGF.lineno}->#{num}; #{locations.join(',')}->#{arity}"
+ end
+ locations.push ARGF.lineno
+ else
+ table[ev] = [[ARGF.lineno], num.to_i]
+ end
+ end
+ end
+ exit 1 if invalid
+end
+
+main
diff --git a/ext/ripper/test/check-event-coverage.sh b/ext/ripper/test/check-event-coverage.sh
new file mode 100755
index 0000000000..727c3b9837
--- /dev/null
+++ b/ext/ripper/test/check-event-coverage.sh
@@ -0,0 +1,15 @@
+# $Id$
+
+RUBY=${RUBY:-ruby}
+status=0
+
+$RUBY tools/list-parse-event-ids.rb parse.y | awk '{print "on__" $1}' > list_a
+$RUBY test/list-called-events.rb | sort -u > list_b
+diff -u list_a list_b | grep '^-on' | sed 's/^-on__//' > list_diff
+if [ -s list_diff ]
+then
+ cat list_diff
+ status=1
+fi
+rm -f list_a list_b list_diff
+exit $status
diff --git a/ext/ripper/test/check-scanner-event-coverage.rb b/ext/ripper/test/check-scanner-event-coverage.rb
new file mode 100644
index 0000000000..4de01e35f4
--- /dev/null
+++ b/ext/ripper/test/check-scanner-event-coverage.rb
@@ -0,0 +1,18 @@
+def main
+ not_tested = eventids() - tested_ids()
+ unless not_tested.empty?
+ puts not_tested
+ exit 1
+ end
+ exit 0
+end
+
+def eventids
+ File.read('eventids2.c').scan(/on__(\w+)/).flatten.uniq
+end
+
+def tested_ids
+ File.read('test/test_scanner_events.rb').scan(/def test_(\S+)/).flatten.uniq
+end
+
+main
diff --git a/ext/ripper/test/list-called-events.rb b/ext/ripper/test/list-called-events.rb
new file mode 100644
index 0000000000..d799efc979
--- /dev/null
+++ b/ext/ripper/test/list-called-events.rb
@@ -0,0 +1,12 @@
+require 'ripper.so'
+
+class R < Ripper
+ def method_missing(mid, *args)
+ puts mid
+ args[0]
+ end
+ undef :warn
+end
+
+fname = (ARGV[0] || 'test/src_rb')
+R.new(File.read(fname), fname, 1).parse
diff --git a/ext/ripper/test/src_rb b/ext/ripper/test/src_rb
new file mode 100644
index 0000000000..e7894362f3
--- /dev/null
+++ b/ext/ripper/test/src_rb
@@ -0,0 +1,363 @@
+# comment
+=begin
+ embedded document
+=end
+
+# literal
+1
+1000000000000000000000
+1.0
+1.234e5
+1..2
+1...3
+:symbol
+:"dynamic #{sym_embexpr} symbol"
+[1,2,3]
+{1 => 2}
+'string'
+"string"
+"before #{str_embexpr} after"
+"str #@ivar str"
+"str #$gvar str"
+"string" "concat"
+`/bin/true`
+{1, 2, 3, 4}
+/regexp/
+/regexp_with_opt/mioe
+/regexp #{regexp_embexpr} after/
+%q[string]
+%Q[str#{str_embexpr}ing]
+%r[regexp]
+%w( a b c )
+%W( a#{w_emb}b c d )
+<<HERE
+heredoc line 1
+heredoc line 2
+heredoc line 3
+HERE
+
+# special variables
+true
+false
+nil
+self
+
+# def
+def a
+end
+def b()
+end
+def c(a)
+end
+def d(a,*rest)
+end
+def e(a,&block)
+end
+def f(a,*rest,&block)
+end
+def g(*rest)
+end
+def h(&block)
+end
+def i(*rest,&block)
+end
+def j(CONST)
+end
+def k(@ivar)
+end
+def l($gvar)
+end
+def n(@@cvar)
+end
+
+# alias
+alias x b
+alias $rest $' # error
+alias $nth $1 # error
+
+# undef
+undef warn
+
+# class, module
+class C
+end
+module M
+end
+class cname
+end
+class << Object.new
+ def self.a
+ end
+end
+
+# field
+$a = 1
+$' = 0 # error
+$1 = 0 # error
+@a = 2
+@@a = 3
+a = 4
+a += 1
+a -= 1
+a *= 1
+a /= 1
+a &&= 1
+a ||= 1
+m.a = 5
+m.a += 1
+m.a &&= 1
+m.a ||= 1
+a[1] = 2
+a[1] += 1
+a[1] &&= 1
+a[1] ||= 1
+C = 1
+C::C = 1
+::C = 1
+def m
+ C = 1 # dynamic const assignment
+ C::C = 1 # dynamic const assignment
+ ::C = 1 # dynamic const assignment
+end
+
+# ref
+lvar = $a
+lvar = @a
+lvar = @@a
+lvar = Object
+lvar = C
+lvar = C::C
+lvar = ::C
+lvar = a[1]
+
+# unary operator
++1
+-1
+not 1
+!1
+~str
+
+# binary operator
+1 + 1
+1 - 1
+1 * 1
+1 / 1
+1 ** 1
+1 ^ 1
+1 & 1
+1 | 1
+1 && 1
+1 || 1
+
+# mlhs, mrhs
+a, b, c = list
+a, = list
+a, * = list
+a, *b = list
+a, (b, c), d, *e = list
+mlhs = 1, 2
+mlhs = 1, 2, 3, *list
+mlhs = *list
+
+# method call
+m
+m()
+m(a)
+m(a,a)
+m(*a)
+m(&b)
+m(a,*a)
+m(a,&b)
+m(a,*a,&b)
+m(1=>2)
+m(1=>2,*a)
+m(1=>2,&b)
+m(1=>2,*a,&b)
+m ()
+m (a)
+m (a,a)
+m (*a)
+m (&b)
+m (a,*a)
+m (a,&b)
+m (a,*a,&b)
+m (1=>2)
+m (1=>2,*a)
+m (1=>2,&b)
+m (1=>2,*a,&b)
+m a
+m a,a
+m *a
+m &b
+m a,*a
+m a,&b
+m a,*a,&b
+m 1=>2
+m 1=>2,*a
+m 1=>2,&b
+m 1=>2,*a,&b
+obj.m
+obj.m()
+obj.m(a)
+obj.m(a,a)
+obj.m(*a)
+obj.m(&b)
+obj.m(a,*a)
+obj.m(a,&b)
+obj.m(a,*a,&b)
+obj.m(1=>2)
+obj.m(1=>2,*a)
+obj.m(1=>2,&b)
+obj.m(1=>2,*a,&b)
+obj.m ()
+obj.m (a)
+obj.m (a,a)
+obj.m (*a)
+obj.m (&b)
+obj.m (a,*a)
+obj.m (a,&b)
+obj.m (a,*a,&b)
+obj.m (1=>2)
+obj.m (1=>2)
+obj.m (1=>2,*a)
+obj.m (1=>2,&b)
+obj.m (1=>2,*a,&b)
+obj.m a
+obj.m a,a
+obj.m *a
+obj.m &b
+obj.m a,*a
+obj.m a,&b
+obj.m a,*a,&b
+obj.m 1=>2
+obj.m 1=>2,*a
+obj.m 1=>2,&b
+obj.m 1=>2,*a,&b
+
+# ambiguous argument
+m +1
+m /r/
+
+# iterator
+[1,2,3].each do |i|
+ print i
+end
+{1=>true}.each do |k,v|
+ puts k
+end
+[1,2,3].each {|i| print i }
+[1].each {|a,| }
+[1].each {|*b| }
+[1].each {|a,*b| }
+[1].each {|&block| }
+[1].each {|a,&block| }
+[1].each {|a,*b,&block| }
+a = lambda() {|n| n * n }
+a = lambda () {|n| n * n }
+a = lambda (a) {|n| n * n }
+a = lambda (a,b) {|n| n * n }
+
+# BEGIN, END
+BEGIN { }
+END { }
+
+# if, unless
+1 if true
+2 unless false
+if false
+ 5
+elsif false
+ 6
+elsif false then 7
+else
+ 8
+end
+if m
+end
+unless 1
+ 2
+end
+unless m
+end
+0 ? 1 : 2
+
+# case
+case 'a'
+when 'b'
+ ;
+when 'c' then 1
+else
+ 2
+end
+case
+when 1
+when 2
+when 3
+else
+end
+case 1
+else
+end
+case
+else
+end
+
+# while, until, for
+while true
+ break
+ next
+ redo
+end
+begin
+ break
+end while true
+until false
+ break
+ next
+ redo
+end
+begin
+ break
+end until false
+for x in m()
+ break
+ next
+ redo
+end
+0 until true
+1 while false
+
+# begin, rescue, else, ensure
+begin
+ 1
+rescue StandardError => er
+ 2
+rescue => er
+ 3
+ retry
+else
+ 4
+ensure
+ 5
+end
+a = 1 rescue 2
+
+# jumps
+def m
+ redo
+ yield
+ yield nil
+ super
+ super 1
+ return
+ return nil
+end
+
+# defined
+defined? f
+defined?(f)
+
+n = 1 \
++ 1
+
+__END__
diff --git a/ext/ripper/test/validate.rb b/ext/ripper/test/validate.rb
new file mode 100644
index 0000000000..808d321043
--- /dev/null
+++ b/ext/ripper/test/validate.rb
@@ -0,0 +1,68 @@
+require 'ripper.so'
+
+class R < Ripper
+ def initialize(*args)
+ super
+ @lineno = 0
+ end
+
+ def parse
+ result = super
+ puts "#{@lineno}:result: #{rawVALUE(result)}"
+ validate_object result
+ p result
+ result
+ end
+
+ def on__nl(str)
+ @lineno += 1
+ end
+
+ def on__ignored_nl(str)
+ @lineno += 1
+ end
+
+ def on__comment(cmt)
+ @lineno += 1
+ end
+
+ def on__embdoc_beg(str)
+ @lineno += 1
+ end
+
+ def on__embdoc(str)
+ @lineno += 1
+ end
+
+ def on__embdoc_end(str)
+ @lineno += 1
+ end
+
+ def method_missing(mid, *args)
+ puts mid
+ args.each_with_index do |a,idx|
+ puts "#{@lineno}:#{mid}\##{idx+1}: #{rawVALUE(a)}"
+ validate_object a
+ p a
+ end
+ args[0]
+ end
+
+ def warn(*args)
+ end
+
+ def warning(*args)
+ end
+
+ unless respond_to?(:validate_object)
+ def validate_object(x)
+ x
+ end
+ def rawVALUE(x)
+ x.object_id
+ end
+ end
+end
+
+fname = (ARGV[0] || 'test/src_rb')
+R.new(File.read(fname), fname, 1).parse
diff --git a/ext/ripper/tools/generate-eventids1.rb b/ext/ripper/tools/generate-eventids1.rb
new file mode 100755
index 0000000000..de0e27b89c
--- /dev/null
+++ b/ext/ripper/tools/generate-eventids1.rb
@@ -0,0 +1,18 @@
+#
+# generate-eventids1.rb
+#
+
+ids = ARGF.map {|s| s.strip }
+
+ids.each do |id|
+ puts "static ID ripper_id_#{id};"
+end
+
+puts
+puts 'static void'
+puts 'ripper_init_eventids1()'
+puts '{'
+ids.each do |id|
+ puts %Q[ ripper_id_#{id} = rb_intern("on__#{id}");]
+end
+puts '}'
diff --git a/ext/ripper/tools/generate-param-macros.rb b/ext/ripper/tools/generate-param-macros.rb
new file mode 100755
index 0000000000..b19f6e8d5c
--- /dev/null
+++ b/ext/ripper/tools/generate-param-macros.rb
@@ -0,0 +1,14 @@
+off = true
+ARGF.each do |line|
+ case line
+ when /RIPPER_PARAMS_DECL_BEGIN/
+ off = false
+ when /RIPPER_PARAMS_DECL_END/
+ exit
+ when /ripper/
+ next if off
+ var = line.scan(/\w+/).last or next
+ base = var.sub(/ripper_/, '')
+ puts %"\#define #{base}\t\t(parser->ripper_#{base})"
+ end
+end
diff --git a/ext/ripper/tools/generate-ripper_rb.rb b/ext/ripper/tools/generate-ripper_rb.rb
new file mode 100755
index 0000000000..c6b2aea13f
--- /dev/null
+++ b/ext/ripper/tools/generate-ripper_rb.rb
@@ -0,0 +1,44 @@
+#
+# generate-ripper_rb.rb
+# Creates ripper.rb, filling in default event handlers, given a basic
+# template, the list of parser events (ids1), and a list of lexer
+# events (ids2).
+#
+
+def main
+ template, ids1, ids2 = *ARGV
+ File.foreach(template) do |line|
+ case line
+ when /\A\#include handlers1/
+ File.foreach(ids1) do |line|
+ id, arity = line.split
+ arity = arity.to_i
+ puts
+ puts " def on__#{id}(#{argdecl(arity)})"
+ puts " #{arity == 0 ? 'nil' : 'a'}"
+ puts " end"
+ end
+ when /\A\#include handlers2/
+ File.foreach(ids2) do |line|
+ id, arity = line.split
+ arity = arity.to_i
+ puts
+ puts " def on__#{id}(token)"
+ puts " token"
+ puts " end"
+ end
+ when /\A\#include (.*)/
+ raise "unknown operation: #include #{$1}"
+ else
+ print line
+ end
+ end
+end
+
+# Generate generic arg list depending on arity (n)
+# n:: [Integer] arity of method
+def argdecl( n )
+ %w(a b c d e f g h i j k l m)[0, n].join(', ')
+end
+
+main
diff --git a/ext/ripper/tools/list-parse-event-ids.rb b/ext/ripper/tools/list-parse-event-ids.rb
new file mode 100755
index 0000000000..2936f9b092
--- /dev/null
+++ b/ext/ripper/tools/list-parse-event-ids.rb
@@ -0,0 +1,38 @@
+#
+# list-parse-event-ids.rb
+#
+
+require 'getopts'
+
+def usage( status )
+ (status == 0 ? $stdout : $stderr).print(<<EOS)
+Usage: #{File.basename($0)} [-a] filename
+EOS
+ exit status
+end
+
+def main
+ getopts('a') or usage(1)
+ extract_ids(ARGF).each do |id, arity|
+ if $OPT_a
+ then puts "#{id} #{arity}"
+ else puts id
+ end
+ end
+end
+
+def extract_ids( f )
+ results = []
+ f.each do |line|
+ next if /\A\#\s*define\s+s?dispatch/ === line
+ next if /ripper_dispatch/ === line
+ if a = line.scan(/dispatch(\d)\((\w+)/)
+ a.each do |arity, event|
+ results.push [event, arity.to_i]
+ end
+ end
+ end
+ results.uniq.sort
+end
+
+main
diff --git a/ext/ripper/tools/list-scan-event-ids.rb b/ext/ripper/tools/list-scan-event-ids.rb
new file mode 100755
index 0000000000..6f25362b5d
--- /dev/null
+++ b/ext/ripper/tools/list-scan-event-ids.rb
@@ -0,0 +1,32 @@
+#
+# list-scan-event-ids.rb
+#
+
+require 'getopts'
+
+def usage(status)
+ (status == 0 ? $stdout : $stderr).puts(<<EOS)
+Usage: #{File.basename($0)} eventids2.c
+ -a print IDs with arity.
+EOS
+ exit status
+end
+
+def main
+ ok = getopts('a', 'help')
+ usage 0 if $OPT_help
+ usage 1 unless ok
+ extract_ids(ARGF).sort.each do |id|
+ if $OPT_a
+ puts "#{id} 1"
+ else
+ puts id
+ end
+ end
+end
+
+def extract_ids(f)
+ (f.read.scan(/ripper_id_(\w+)/).flatten - ['scan']).uniq
+end
+
+main
diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb
new file mode 100755
index 0000000000..a2dba36e02
--- /dev/null
+++ b/ext/ripper/tools/preproc.rb
@@ -0,0 +1,57 @@
+def main
+ prelude
+ grammar
+ usercode
+end
+
+def prelude
+ while line = ARGF.gets
+ case line
+ when %r</\*%%%\*/>
+ puts '/*'
+ when %r</\*%>
+ puts '*/'
+ when %r<%\*/>
+ puts
+ when /\A%%/
+ puts '%%'
+ return
+ when /\A%token/
+ puts line.sub(/<\w+>/, '<val>')
+ when /\A%type/
+ puts line.sub(/<\w+>/, '<val>')
+ else
+ print line
+ end
+ end
+end
+
+def grammar
+ while line = ARGF.gets
+ case line
+ when %r</\*%%%\*/>
+ puts '#if 0'
+ when %r</\*%c%\*/>
+ puts '/*'
+ when %r</\*%c>
+ puts '*/'
+ when %r</\*%>
+ puts '#endif'
+ when %r<%\*/>
+ puts
+ when /\A%%/
+ puts '%%'
+ return
+ else
+ print line
+ end
+ end
+end
+
+def usercode
+ while line = ARGF.gets
+ print line
+ end
+end
+
+main
diff --git a/ext/ripper/tools/strip.rb b/ext/ripper/tools/strip.rb
new file mode 100755
index 0000000000..99413c361d
--- /dev/null
+++ b/ext/ripper/tools/strip.rb
@@ -0,0 +1,12 @@
+last_is_void = false
+ARGF.each do |line|
+ if line.strip.empty?
+ #puts() unless last_is_void
+ last_is_void = true
+ elsif /\A\#/ === line
+ ;
+ else
+ print line
+ last_is_void = false
+ end
+end
diff --git a/test/ripper/dummyparser.rb b/test/ripper/dummyparser.rb
new file mode 100644
index 0000000000..b95ebdb5d1
--- /dev/null
+++ b/test/ripper/dummyparser.rb
@@ -0,0 +1,526 @@
+#
+# dummyparser.rb
+#
+
+class Node
+
+ def initialize( name, *nodes )
+ @name = name
+ @children = nodes
+ end
+
+ attr_reader :children
+
+ def to_s
+ "#{@name}(#{@children.map {|n| n.to_s }.join(',')})"
+ end
+
+end
+
+
+class NodeList
+
+ def initialize
+ @list = []
+ end
+
+ attr_reader :list
+
+ def push( item )
+ @list.push item
+ self
+ end
+
+ def prepend( items )
+ @list[0,0] = items
+ end
+
+ def to_s
+ '[' + @list.join(',') + ']'
+ end
+
+end
+
+
+class DummyParser < Ripper
+
+ def method_missing( mid, *args )
+ raise NoMethodError, "wrong method: #{mid}" unless /\Aon__/ === mid.to_s
+ args[0]
+ end
+
+ def on__program( stmts )
+ $thru_program = true
+ stmts
+ end
+
+ def on__stmts_new
+ NodeList.new
+ end
+
+ def on__stmts_add( stmts, st )
+ stmts.push st
+ stmts
+ end
+
+ def on__void_stmt
+ Node.new('void')
+ end
+
+ def on__BEGIN( stmts )
+ Node.new('BEGIN', stmts)
+ end
+
+ def on__END( stmts )
+ Node.new('END', stmts)
+ end
+
+ def on__var_ref( name )
+ Node.new('ref', name)
+ end
+
+ def on__alias(a, b)
+ Node.new('alias', a, b)
+ end
+
+ def on__var_alias(a, b)
+ Node.new('valias', a, b)
+ end
+
+ def on__alias_error(a)
+ Node.new('aliaserr', a)
+ end
+
+ def on__aref(a, b)
+ Node.new('aref', a, b)
+ end
+
+ def on__aref_field(a, b)
+ Node.new('aref_field', a, b)
+ end
+
+ def on__arg_ambiguous
+ Node.new('arg_ambiguous')
+ end
+
+ def on__arg_paren( args )
+ args
+ end
+
+ def on__arglist_new
+ NodeList.new
+ end
+
+ def on__arglist_add( list, arg )
+ list.push(arg)
+ end
+
+ def on__arglist_add_block( list, blk )
+ list.push('&' + blk.to_s)
+ end
+
+ def on__arglist_add_star( list, arg )
+ list.push('*' + arg.to_s)
+ end
+
+ def on__arglist_prepend( list, args )
+ list.prepend args
+ list
+ end
+
+ def on__method_add_arg( m, arg )
+ m.children.push arg
+ m
+ end
+
+ def on__assoc_new(a, b)
+ Node.new('assoc', a, b)
+ end
+
+ def on__bare_assoc_hash( assoc_list )
+ Node.new('assocs', *assoc_list)
+ end
+
+ def on__assoclist_from_args(a)
+ Node.new('assocs', *a.list)
+ end
+
+ ######## untested
+
+ def on__array(a)
+ Node.new('array', a)
+ end
+
+ def on__assign(a, b)
+ Node.new('assign', a, b)
+ end
+
+ def on__assign_error(a)
+ Node.new('assign_error', a)
+ end
+
+ def on__begin(a)
+ Node.new('begin', a)
+ end
+
+ def on__binary(a, b, c)
+ Node.new('binary', a, b, c)
+ end
+
+ def on__block_var(a)
+ Node.new('block_var', a)
+ end
+
+ def on__bodystmt(a, b, c, d)
+ Node.new('bodystmt', a, b, c, d)
+ end
+
+ def on__brace_block(a, b)
+ Node.new('brace_block', a, b)
+ end
+
+ def on__break(a)
+ Node.new('break', a)
+ end
+
+ def on__call(a, b, c)
+ Node.new('call', a, b, c)
+ end
+
+ def on__case(a, b)
+ Node.new('case', a, b)
+ end
+
+ def on__class(a, b, c)
+ Node.new('class', a, b, c)
+ end
+
+ def on__class_name_error(a)
+ Node.new('class_name_error', a)
+ end
+
+ def on__command(a, b)
+ Node.new('command', a, b)
+ end
+
+ def on__command_call(a, b, c, d)
+ Node.new('command_call', a, b, c, d)
+ end
+
+ def on__const_ref(a)
+ Node.new('const_ref', a)
+ end
+
+ def on__constpath_field(a, b)
+ Node.new('constpath_field', a, b)
+ end
+
+ def on__constpath_ref(a, b)
+ Node.new('constpath_ref', a, b)
+ end
+
+ def on__def(a, b, c)
+ Node.new('def', a, b, c)
+ end
+
+ def on__defined(a)
+ Node.new('defined', a)
+ end
+
+ def on__defs(a, b, c, d, e)
+ Node.new('defs', a, b, c, d, e)
+ end
+
+ def on__do_block(a, b)
+ Node.new('do_block', a, b)
+ end
+
+ def on__dot2(a, b)
+ Node.new('dot2', a, b)
+ end
+
+ def on__dot3(a, b)
+ Node.new('dot3', a, b)
+ end
+
+ def on__dyna_symbol(a)
+ Node.new('dyna_symbol', a)
+ end
+
+ def on__else(a)
+ Node.new('else', a)
+ end
+
+ def on__elsif(a, b, c)
+ Node.new('elsif', a, b, c)
+ end
+
+ def on__ensure(a)
+ Node.new('ensure', a)
+ end
+
+ def on__fcall(a)
+ Node.new('fcall', a)
+ end
+
+ def on__field(a, b, c)
+ Node.new('field', a, b, c)
+ end
+
+ def on__for(a, b, c)
+ Node.new('for', a, b, c)
+ end
+
+ def on__hash(a)
+ Node.new('hash', a)
+ end
+
+ def on__if(a, b, c)
+ Node.new('if', a, b, c)
+ end
+
+ def on__if_mod(a, b)
+ Node.new('if_mod', a, b)
+ end
+
+ def on__ifop(a, b, c)
+ Node.new('ifop', a, b, c)
+ end
+
+ def on__iter_block(a, b)
+ Node.new('iter_block', a, b)
+ end
+
+ def on__massign(a, b)
+ Node.new('massign', a, b)
+ end
+
+ def on__mlhs_add(a, b)
+ Node.new('mlhs_add', a, b)
+ end
+
+ def on__mlhs_add_star(a, b)
+ Node.new('mlhs_add_star', a, b)
+ end
+
+ def on__mlhs_new
+ Node.new('mlhs_new')
+ end
+
+ def on__mlhs_paren(a)
+ Node.new('mlhs_paren', a)
+ end
+
+ def on__module(a, b)
+ Node.new('module', a, b)
+ end
+
+ def on__mrhs_add(a, b)
+ Node.new('mrhs_add', a, b)
+ end
+
+ def on__mrhs_add_star(a, b)
+ Node.new('mrhs_add_star', a, b)
+ end
+
+ def on__mrhs_new
+ Node.new('mrhs_new')
+ end
+
+ def on__mrhs_new_from_arglist(a)
+ Node.new('mrhs_new_from_arglist', a)
+ end
+
+ def on__next(a)
+ Node.new('next', a)
+ end
+
+ def on__opassign(a, b, c)
+ Node.new('opassign', a, b, c)
+ end
+
+ def on__param_error(a)
+ Node.new('param_error', a)
+ end
+
+ def on__params(a, b, c, d)
+ Node.new('params', a, b, c, d)
+ end
+
+ def on__paren(a)
+ Node.new('paren', a)
+ end
+
+ def on__parse_error(a)
+ Node.new('parse_error', a)
+ end
+
+ def on__qwords_add(a, b)
+ Node.new('qwords_add', a, b)
+ end
+
+ def on__qwords_new
+ Node.new('qwords_new')
+ end
+
+ def on__redo
+ Node.new('redo')
+ end
+
+ def on__regexp_literal(a)
+ Node.new('regexp_literal', a)
+ end
+
+ def on__rescue(a, b, c, d)
+ Node.new('rescue', a, b, c, d)
+ end
+
+ def on__rescue_mod(a, b)
+ Node.new('rescue_mod', a, b)
+ end
+
+ def on__restparam(a)
+ Node.new('restparam', a)
+ end
+
+ def on__retry
+ Node.new('retry')
+ end
+
+ def on__return(a)
+ Node.new('return', a)
+ end
+
+ def on__return0
+ Node.new('return0')
+ end
+
+ def on__sclass(a, b)
+ Node.new('sclass', a, b)
+ end
+
+ def on__space(a)
+ Node.new('space', a)
+ end
+
+ def on__string_add(a, b)
+ Node.new('string_add', a, b)
+ end
+
+ def on__string_concat(a, b)
+ Node.new('string_concat', a, b)
+ end
+
+ def on__string_content
+ Node.new('string_content')
+ end
+
+ def on__string_dvar(a)
+ Node.new('string_dvar', a)
+ end
+
+ def on__string_embexpr(a)
+ Node.new('string_embexpr', a)
+ end
+
+ def on__string_literal(a)
+ Node.new('string_literal', a)
+ end
+
+ def on__super(a)
+ Node.new('super', a)
+ end
+
+ def on__symbol(a)
+ Node.new('symbol', a)
+ end
+
+ def on__symbol_literal(a)
+ Node.new('symbol_literal', a)
+ end
+
+ def on__topconst_field(a)
+ Node.new('topconst_field', a)
+ end
+
+ def on__topconst_ref(a)
+ Node.new('topconst_ref', a)
+ end
+
+ def on__unary(a, b)
+ Node.new('unary', a, b)
+ end
+
+ def on__undef(a)
+ Node.new('undef', a)
+ end
+
+ def on__unless(a, b, c)
+ Node.new('unless', a, b, c)
+ end
+
+ def on__unless_mod(a, b)
+ Node.new('unless_mod', a, b)
+ end
+
+ def on__until_mod(a, b)
+ Node.new('until_mod', a, b)
+ end
+
+ def on__var_field(a)
+ Node.new('var_field', a)
+ end
+
+ def on__when(a, b, c)
+ Node.new('when', a, b, c)
+ end
+
+ def on__while(a, b)
+ Node.new('while', a, b)
+ end
+
+ def on__while_mod(a, b)
+ Node.new('while_mod', a, b)
+ end
+
+ def on__word_add(a, b)
+ Node.new('word_add', a, b)
+ end
+
+ def on__word_new
+ Node.new('word_new')
+ end
+
+ def on__words_add(a, b)
+ Node.new('words_add', a, b)
+ end
+
+ def on__words_new
+ Node.new('words_new')
+ end
+
+ def on__xstring_add(a, b)
+ Node.new('xstring_add', a, b)
+ end
+
+ def on__xstring_literal(a)
+ Node.new('xstring_literal', a)
+ end
+
+ def on__xstring_new
+ Node.new('xstring_new')
+ end
+
+ def on__yield(a)
+ Node.new('yield', a)
+ end
+
+ def on__yield0
+ Node.new('yield0')
+ end
+
+ def on__zsuper
+ Node.new('zsuper')
+ end
+
+end
diff --git a/test/ripper/test_parser_events.rb b/test/ripper/test_parser_events.rb
new file mode 100644
index 0000000000..512cf377e7
--- /dev/null
+++ b/test/ripper/test_parser_events.rb
@@ -0,0 +1,489 @@
+#
+# test_parser_events.rb
+#
+
+require 'ripper.so'
+require 'dummyparser'
+raise "ripper version differ" unless Ripper::Version == '0.1.0'
+require 'test/unit'
+
+
+class TestRipper_ParserEvents < Test::Unit::TestCase
+
+ def parse(str)
+ DummyParser.new(str).parse.to_s
+ end
+
+ $thru_program = false
+
+ def test_program
+ assert_equal '[void()]', parse('')
+ assert_equal true, $thru_program
+ end
+
+ def test_stmts_new
+ assert_equal '[void()]', parse('')
+ end
+
+ def test_stmts_add
+ assert_equal '[ref(nil)]', parse('nil')
+ assert_equal '[ref(nil),ref(nil)]', parse('nil;nil')
+ assert_equal '[ref(nil),ref(nil),ref(nil)]', parse('nil;nil;nil')
+ end
+
+ def test_void_stmt
+ assert_equal '[void()]', parse('')
+ assert_equal '[void()]', parse(';;')
+ end
+
+ def test_var_ref
+ assert_equal '[ref(a)]', parse('a')
+ assert_equal '[ref(nil)]', parse('nil')
+ assert_equal '[ref(true)]', parse('true')
+ end
+
+ def test_BEGIN
+ assert_equal '[BEGIN([void()])]', parse('BEGIN{}')
+ assert_equal '[BEGIN([ref(nil)])]', parse('BEGIN{nil}')
+ end
+
+ def test_END
+ assert_equal '[END([void()])]', parse('END{}')
+ assert_equal '[END([ref(nil)])]', parse('END{nil}')
+ end
+
+ def test_alias
+ assert_equal '[alias(a,b)]', parse('alias a b')
+ end
+
+ def test_var_alias
+ assert_equal '[valias($a,$g)]', parse('alias $a $g')
+ end
+
+ def test_alias_error
+ assert_equal '[aliaserr(valias($a,$1))]', parse('alias $a $1')
+ end
+
+ def test_arglist
+ assert_equal '[fcall(m,[])]', parse('m()')
+ assert_equal '[fcall(m,[1])]', parse('m(1)')
+ assert_equal '[fcall(m,[1,2])]', parse('m(1,2)')
+ assert_equal '[fcall(m,[*ref(r)])]', parse('m(*r)')
+ assert_equal '[fcall(m,[1,*ref(r)])]', parse('m(1,*r)')
+ assert_equal '[fcall(m,[1,2,*ref(r)])]', parse('m(1,2,*r)')
+ assert_equal '[fcall(m,[&ref(r)])]', parse('m(&r)')
+ assert_equal '[fcall(m,[1,&ref(r)])]', parse('m(1,&r)')
+ assert_equal '[fcall(m,[1,2,&ref(r)])]', parse('m(1,2,&r)')
+ assert_equal '[fcall(m,[*ref(a),&ref(b)])]', parse('m(*a,&b)')
+ assert_equal '[fcall(m,[1,*ref(a),&ref(b)])]', parse('m(1,*a,&b)')
+ assert_equal '[fcall(m,[1,2,*ref(a),&ref(b)])]', parse('m(1,2,*a,&b)')
+ end
+
+ def test_arg_paren
+ # FIXME
+ end
+
+ def test_aref
+ assert_equal '[aref(ref(v),[1])]', parse('v[1]')
+ assert_equal '[aref(ref(v),[1,2])]', parse('v[1,2]')
+ end
+
+ def test_assocs
+ assert_equal '[fcall(m,[assocs(assoc(1,2))])]', parse('m(1=>2)')
+ assert_equal '[fcall(m,[assocs(assoc(1,2),assoc(3,4))])]', parse('m(1=>2,3=>4)')
+ assert_equal '[fcall(m,[3,assocs(assoc(1,2))])]', parse('m(3,1=>2)')
+ end
+
+ def test_aref_field
+ assert_equal '[assign(aref_field(ref(a),[1]),2)]', parse('a[1]=2')
+ end
+
+=begin
+ def test_arg_ambiguous
+ assert_equal true, $thru__arg_ambiguous
+ end
+=end
+
+ def test_array # array literal
+ assert_equal '[array([1,2,3])]', parse('[1,2,3]')
+ end
+
+ def test_assign # generic assignment
+ assert_equal '[assign(var_field(v),1)]', parse('v=1')
+ end
+
+=begin
+ def test_assign_error
+ assert_equal true, $thru__assign_error
+ end
+
+ def test_begin
+ assert_equal true, $thru__begin
+ end
+
+ def test_binary
+ assert_equal true, $thru__binary
+ end
+
+ def test_block_var
+ assert_equal true, $thru__block_var
+ end
+
+ def test_bodystmt
+ assert_equal true, $thru__bodystmt
+ end
+
+ def test_brace_block
+ assert_equal true, $thru__brace_block
+ end
+
+ def test_break
+ assert_equal true, $thru__break
+ end
+
+ def test_call
+ assert_equal true, $thru__call
+ end
+
+ def test_case
+ assert_equal true, $thru__case
+ end
+
+ def test_class
+ assert_equal true, $thru__class
+ end
+
+ def test_class_name_error
+ assert_equal true, $thru__class_name_error
+ end
+
+ def test_command
+ assert_equal true, $thru__command
+ end
+
+ def test_command_call
+ assert_equal true, $thru__command_call
+ end
+
+ def test_const_ref
+ assert_equal true, $thru__const_ref
+ end
+
+ def test_constpath_field
+ assert_equal true, $thru__constpath_field
+ end
+
+ def test_constpath_ref
+ assert_equal true, $thru__constpath_ref
+ end
+
+ def test_def
+ assert_equal true, $thru__def
+ end
+
+ def test_defined
+ assert_equal true, $thru__defined
+ end
+
+ def test_defs
+ assert_equal true, $thru__defs
+ end
+
+ def test_do_block
+ assert_equal true, $thru__do_block
+ end
+
+ def test_dot2
+ assert_equal true, $thru__dot2
+ end
+
+ def test_dot3
+ assert_equal true, $thru__dot3
+ end
+
+ def test_dyna_symbol
+ assert_equal true, $thru__dyna_symbol
+ end
+
+ def test_else
+ assert_equal true, $thru__else
+ end
+
+ def test_elsif
+ assert_equal true, $thru__elsif
+ end
+
+ def test_ensure
+ assert_equal true, $thru__ensure
+ end
+
+ def test_fcall
+ assert_equal true, $thru__fcall
+ end
+
+ def test_field
+ assert_equal true, $thru__field
+ end
+
+ def test_for
+ assert_equal true, $thru__for
+ end
+
+ def test_hash
+ assert_equal true, $thru__hash
+ end
+
+ def test_if
+ assert_equal true, $thru__if
+ end
+
+ def test_if_mod
+ assert_equal true, $thru__if_mod
+ end
+
+ def test_ifop
+ assert_equal true, $thru__ifop
+ end
+
+ def test_iter_block
+ assert_equal true, $thru__iter_block
+ end
+
+ def test_massign
+ assert_equal true, $thru__massign
+ end
+
+ def test_method_add_arg
+ assert_equal true, $thru__method_add_arg
+ end
+
+ def test_mlhs_add
+ assert_equal true, $thru__mlhs_add
+ end
+
+ def test_mlhs_add_star
+ assert_equal true, $thru__mlhs_add_star
+ end
+
+ def test_mlhs_new
+ assert_equal true, $thru__mlhs_new
+ end
+
+ def test_mlhs_paren
+ assert_equal true, $thru__mlhs_paren
+ end
+
+ def test_module
+ assert_equal true, $thru__module
+ end
+
+ def test_mrhs_add
+ assert_equal true, $thru__mrhs_add
+ end
+
+ def test_mrhs_add_star
+ assert_equal true, $thru__mrhs_add_star
+ end
+
+ def test_mrhs_new
+ assert_equal true, $thru__mrhs_new
+ end
+
+ def test_mrhs_new_from_arglist
+ assert_equal true, $thru__mrhs_new_from_arglist
+ end
+
+ def test_next
+ assert_equal true, $thru__next
+ end
+
+ def test_opassign
+ assert_equal true, $thru__opassign
+ end
+
+ def test_param_error
+ assert_equal true, $thru__param_error
+ end
+
+ def test_params
+ assert_equal true, $thru__params
+ end
+
+ def test_paren
+ assert_equal true, $thru__paren
+ end
+
+ def test_parse_error
+ assert_equal true, $thru__parse_error
+ end
+
+ def test_qwords_add
+ assert_equal true, $thru__qwords_add
+ end
+
+ def test_qwords_new
+ assert_equal true, $thru__qwords_new
+ end
+
+ def test_redo
+ assert_equal true, $thru__redo
+ end
+
+ def test_regexp_literal
+ assert_equal true, $thru__regexp_literal
+ end
+
+ def test_rescue
+ assert_equal true, $thru__rescue
+ end
+
+ def test_rescue_mod
+ assert_equal true, $thru__rescue_mod
+ end
+
+ def test_restparam
+ assert_equal true, $thru__restparam
+ end
+
+ def test_retry
+ assert_equal true, $thru__retry
+ end
+
+ def test_return
+ assert_equal true, $thru__return
+ end
+
+ def test_return0
+ assert_equal true, $thru__return0
+ end
+
+ def test_sclass
+ assert_equal true, $thru__sclass
+ end
+
+ def test_space
+ assert_equal true, $thru__space
+ end
+
+ def test_string_add
+ assert_equal true, $thru__string_add
+ end
+
+ def test_string_concat
+ assert_equal true, $thru__string_concat
+ end
+
+ def test_string_content
+ assert_equal true, $thru__string_content
+ end
+
+ def test_string_dvar
+ assert_equal true, $thru__string_dvar
+ end
+
+ def test_string_embexpr
+ assert_equal true, $thru__string_embexpr
+ end
+
+ def test_string_literal
+ assert_equal true, $thru__string_literal
+ end
+
+ def test_super
+ assert_equal true, $thru__super
+ end
+
+ def test_symbol
+ assert_equal true, $thru__symbol
+ end
+
+ def test_symbol_literal
+ assert_equal true, $thru__symbol_literal
+ end
+
+ def test_topconst_field
+ assert_equal true, $thru__topconst_field
+ end
+
+ def test_topconst_ref
+ assert_equal true, $thru__topconst_ref
+ end
+
+ def test_unary
+ assert_equal true, $thru__unary
+ end
+
+ def test_undef
+ assert_equal true, $thru__undef
+ end
+
+ def test_unless
+ assert_equal true, $thru__unless
+ end
+
+ def test_unless_mod
+ assert_equal true, $thru__unless_mod
+ end
+
+ def test_until_mod
+ assert_equal true, $thru__until_mod
+ end
+
+ def test_var_field
+ assert_equal true, $thru__var_field
+ end
+
+ def test_when
+ assert_equal true, $thru__when
+ end
+
+ def test_while
+ assert_equal true, $thru__while
+ end
+
+ def test_while_mod
+ assert_equal true, $thru__while_mod
+ end
+
+ def test_word_add
+ assert_equal true, $thru__word_add
+ end
+
+ def test_word_new
+ assert_equal true, $thru__word_new
+ end
+
+ def test_words_add
+ assert_equal true, $thru__words_add
+ end
+
+ def test_words_new
+ assert_equal true, $thru__words_new
+ end
+
+ def test_xstring_add
+ assert_equal true, $thru__xstring_add
+ end
+
+ def test_xstring_literal
+ assert_equal true, $thru__xstring_literal
+ end
+
+ def test_xstring_new
+ assert_equal true, $thru__xstring_new
+ end
+
+ def test_yield
+ assert_equal true, $thru__yield
+ end
+
+ def test_yield0
+ assert_equal true, $thru__yield0
+ end
+
+ def test_zsuper
+ assert_equal true, $thru__zsuper
+ end
+=end
+
+end
diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb
new file mode 100644
index 0000000000..28b65eb1b6
--- /dev/null
+++ b/test/ripper/test_scanner_events.rb
@@ -0,0 +1,744 @@
+#
+# test_scanner_events.rb
+#
+
+require 'ripper.so'
+raise 'ripper version differ' unless Ripper::Version == '0.1.0'
+require 'test/unit'
+
+class TestRipper_ScannerEvents < Test::Unit::TestCase
+
+ class R < Ripper
+ def R.scan(target, src)
+ r = new(src, target)
+ r.parse
+ r.tokens.map {|id, tok| tok }
+ end
+
+ def R.lex(src)
+ r = new(src, 'scan')
+ r.parse
+ r.tokens
+ end
+
+ def initialize(src, target)
+ super src
+ @target = ('on__' + target).intern
+ @tokens = []
+ end
+
+ attr_reader :tokens
+
+ def method_missing(mid, *args)
+ case mid.to_s
+ when /\Aon__scan/
+ if @target == :on__scan
+ @tokens.push args
+ else
+ @tokens.push args if @target == args[0]
+ end
+ when /\Aon__/
+ ;
+ else
+ raise NoMethodError, "no such method: #{mid}"
+ end
+ end
+
+ def warn(fmt, *args)
+ #p [fmt, args]
+ end
+
+ def warning(fmt, *args)
+ #p [fmt, args]
+ end
+
+ def compile_error(msg)
+ end
+ end
+
+ def test_scan
+ assert_equal [],
+ R.scan('scan', '')
+ assert_equal ['a'],
+ R.scan('scan', 'a')
+ assert_equal ['1'],
+ R.scan('scan', '1')
+ assert_equal ['1', ';', 'def', ' ', 'm', '(', 'arg', ')', 'end'],
+ R.scan('scan', "1;def m(arg)end")
+ end
+
+ def test_backref
+ assert_equal ["$`", "$&", "$'", '$1', '$2', '$3'],
+ R.scan('backref', %q[m($~, $`, $&, $', $1, $2, $3)])
+ end
+
+ def test_backtick
+ assert_equal ["`"],
+ R.scan('backtick', %q[p `make all`])
+ end
+
+ def test_comma
+ assert_equal [','] * 6,
+ R.scan('comma', %q[ m(0,1,2,3,4,5,6) ])
+ assert_equal [],
+ R.scan('comma', %q[".,.,.,.,.,.,.."])
+ assert_equal [],
+ R.scan('comma', %Q[<<EOS\n,,,,,,,,,,\nEOS])
+ end
+
+ def test_period
+ assert_equal [],
+ R.scan('period', '')
+ assert_equal ['.'],
+ R.scan('period', 'a.b')
+ assert_equal ['.'],
+ R.scan('period', 'Object.new')
+ assert_equal [],
+ R.scan('period', '"."')
+ assert_equal [],
+ R.scan('period', '1..2')
+ assert_equal [],
+ R.scan('period', '1...3')
+ end
+
+ def test_const
+ assert_equal ['CONST'],
+ R.scan('const', 'CONST')
+ assert_equal ['C'],
+ R.scan('const', 'C')
+ assert_equal ['CONST_A'],
+ R.scan('const', 'CONST_A')
+ assert_equal ['Const', 'Const2', 'Const3'],
+ R.scan('const', 'Const; Const2; Const3')
+ assert_equal ['Const'],
+ R.scan('const', 'Const(a)')
+ assert_equal ['M', 'A', 'A2'],
+ R.scan('const', 'M(A,A2)')
+ assert_equal [],
+ R.scan('const', '')
+ assert_equal [],
+ R.scan('const', 'm(lvar, @ivar, @@cvar, $gvar)')
+ end
+
+ def test_cvar
+ assert_equal [],
+ R.scan('cvar', '')
+ assert_equal ['@@cvar'],
+ R.scan('cvar', '@@cvar')
+ assert_equal ['@@__cvar__'],
+ R.scan('cvar', '@@__cvar__')
+ assert_equal ['@@CVAR'],
+ R.scan('cvar', '@@CVAR')
+ assert_equal ['@@cvar'],
+ R.scan('cvar', ' @@cvar#comment')
+ assert_equal ['@@cvar'],
+ R.scan('cvar', ':@@cvar')
+ assert_equal ['@@cvar'],
+ R.scan('cvar', 'm(lvar, @ivar, @@cvar, $gvar)')
+ assert_equal [],
+ R.scan('cvar', '"@@cvar"')
+ end
+
+ def test_embexpr_beg
+ assert_equal [],
+ R.scan('embexpr_beg', '')
+ assert_equal ['#{'],
+ R.scan('embexpr_beg', '"#{expr}"')
+ assert_equal [],
+ R.scan('embexpr_beg', '%q[#{expr}]')
+ assert_equal ['#{'],
+ R.scan('embexpr_beg', '%Q[#{expr}]')
+ assert_equal ['#{'],
+ R.scan('embexpr_beg', "m(<<EOS)\n\#{expr}\nEOS")
+ end
+
+=begin
+ def test_embexpr_end
+ assert_equal [],
+ R.scan('embexpr_end', '')
+ assert_equal ['}'],
+ R.scan('embexpr_end', '"#{expr}"')
+ assert_equal [],
+ R.scan('embexpr_end', '%q[#{expr}]')
+ assert_equal ['}'],
+ R.scan('embexpr_end', '%Q[#{expr}]')
+ assert_equal ['}'],
+ R.scan('embexpr_end', "m(<<EOS)\n\#{expr}\nEOS")
+ end
+=end
+
+ def test_embvar
+ assert_equal [],
+ R.scan('embvar', '')
+ assert_equal ['#'],
+ R.scan('embvar', '"#$gvar"')
+ assert_equal ['#'],
+ R.scan('embvar', '"#@ivar"')
+ assert_equal ['#'],
+ R.scan('embvar', '"#@@cvar"')
+ assert_equal [],
+ R.scan('embvar', '"#lvar"')
+ assert_equal [],
+ R.scan('embvar', '"#"')
+ assert_equal [],
+ R.scan('embvar', '"\#$gvar"')
+ assert_equal [],
+ R.scan('embvar', '"\#@ivar"')
+ assert_equal [],
+ R.scan('embvar', '%q[#@ivar]')
+ assert_equal ['#'],
+ R.scan('embvar', '%Q[#@ivar]')
+ end
+
+ def test_float
+ assert_equal [],
+ R.scan('float', '')
+ assert_equal ['1.000'],
+ R.scan('float', '1.000')
+ assert_equal ['123.456'],
+ R.scan('float', '123.456')
+ assert_equal ['1.2345678901234567890123456789'],
+ R.scan('float', '1.2345678901234567890123456789')
+ assert_equal ['1.000'],
+ R.scan('float', ' 1.000# comment')
+ assert_equal ['1.234e5'],
+ R.scan('float', '1.234e5')
+ assert_equal ['1.234e1234567890'],
+ R.scan('float', '1.234e1234567890')
+ assert_equal ['1.0'],
+ R.scan('float', 'm(a,b,1.0,c,d)')
+ end
+
+ def test_gvar
+ assert_equal [],
+ R.scan('gvar', '')
+ assert_equal ['$a'],
+ R.scan('gvar', '$a')
+ assert_equal ['$A'],
+ R.scan('gvar', '$A')
+ assert_equal ['$gvar'],
+ R.scan('gvar', 'm(lvar, @ivar, @@cvar, $gvar)')
+ assert_equal %w($_ $~ $* $$ $? $! $@ $/ $\\ $; $, $. $= $: $< $> $"),
+ R.scan('gvar', 'm($_, $~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $")')
+ end
+
+ def test_ident
+ assert_equal [],
+ R.scan('ident', '')
+ assert_equal ['lvar'],
+ R.scan('ident', 'lvar')
+ assert_equal ['m', 'lvar'],
+ R.scan('ident', 'm(lvar, @ivar, @@cvar, $gvar)')
+ end
+
+ def test_int
+ assert_equal [],
+ R.scan('int', '')
+ assert_equal ['1', '10', '100000000000000'],
+ R.scan('int', 'm(1,10,100000000000000)')
+ end
+
+ def test_ivar
+ assert_equal [],
+ R.scan('ivar', '')
+ assert_equal ['@ivar'],
+ R.scan('ivar', '@ivar')
+ assert_equal ['@__ivar__'],
+ R.scan('ivar', '@__ivar__')
+ assert_equal ['@IVAR'],
+ R.scan('ivar', '@IVAR')
+ assert_equal ['@ivar'],
+ R.scan('ivar', 'm(lvar, @ivar, @@cvar, $gvar)')
+ end
+
+ def test_kw
+ assert_equal [],
+ R.scan('kw', '')
+ assert_equal %w(not),
+ R.scan('kw', 'not 1')
+ assert_equal %w(and),
+ R.scan('kw', '1 and 2')
+ assert_equal %w(or),
+ R.scan('kw', '1 or 2')
+ assert_equal %w(if then else end),
+ R.scan('kw', 'if 1 then 2 else 3 end')
+ assert_equal %w(if then elsif else end),
+ R.scan('kw', 'if 1 then 2 elsif 3 else 4 end')
+ assert_equal %w(unless then end),
+ R.scan('kw', 'unless 1 then end')
+ assert_equal %w(if true),
+ R.scan('kw', '1 if true')
+ assert_equal %w(unless false),
+ R.scan('kw', '2 unless false')
+ assert_equal %w(case when when else end),
+ R.scan('kw', 'case n; when 1; when 2; else 3 end')
+ assert_equal %w(while do nil end),
+ R.scan('kw', 'while 1 do nil end')
+ assert_equal %w(until do nil end),
+ R.scan('kw', 'until 1 do nil end')
+ assert_equal %w(while),
+ R.scan('kw', '1 while 2')
+ assert_equal %w(until),
+ R.scan('kw', '1 until 2')
+ assert_equal %w(while break next retry end),
+ R.scan('kw', 'while 1; break; next; retry end')
+ assert_equal %w(for in next break end),
+ R.scan('kw', 'for x in obj; next 1; break 2 end')
+ assert_equal %w(begin rescue retry end),
+ R.scan('kw', 'begin 1; rescue; retry; end')
+ assert_equal %w(rescue),
+ R.scan('kw', '1 rescue 2')
+ assert_equal %w(def redo return end),
+ R.scan('kw', 'def m() redo; return end')
+ assert_equal %w(def yield yield end),
+ R.scan('kw', 'def m() yield; yield 1 end')
+ assert_equal %w(def super super super end),
+ R.scan('kw', 'def m() super; super(); super(1) end')
+ assert_equal %w(alias),
+ R.scan('kw', 'alias a b')
+ assert_equal %w(undef),
+ R.scan('kw', 'undef public')
+ assert_equal %w(class end),
+ R.scan('kw', 'class A < Object; end')
+ assert_equal %w(module end),
+ R.scan('kw', 'module M; end')
+ assert_equal %w(class end),
+ R.scan('kw', 'class << obj; end')
+ assert_equal %w(BEGIN),
+ R.scan('kw', 'BEGIN { }')
+ assert_equal %w(END),
+ R.scan('kw', 'END { }')
+ assert_equal %w(self),
+ R.scan('kw', 'self.class')
+ assert_equal %w(nil true false),
+ R.scan('kw', 'p(nil, true, false)')
+ assert_equal %w(__FILE__ __LINE__),
+ R.scan('kw', 'p __FILE__, __LINE__')
+ assert_equal %w(defined?),
+ R.scan('kw', 'defined?(Object)')
+ end
+
+ def test_lbrace
+ assert_equal [],
+ R.scan('lbrace', '')
+ assert_equal ['{'],
+ R.scan('lbrace', '3.times{ }')
+ assert_equal ['{'],
+ R.scan('lbrace', '3.times { }')
+ assert_equal ['{'],
+ R.scan('lbrace', '3.times{}')
+ assert_equal [],
+ R.scan('lbrace', '"{}"')
+ assert_equal ['{'],
+ R.scan('lbrace', '{1=>2}')
+ end
+
+ def test_rbrace
+ assert_equal [],
+ R.scan('rbrace', '')
+ assert_equal ['}'],
+ R.scan('rbrace', '3.times{ }')
+ assert_equal ['}'],
+ R.scan('rbrace', '3.times { }')
+ assert_equal ['}'],
+ R.scan('rbrace', '3.times{}')
+ assert_equal [],
+ R.scan('rbrace', '"{}"')
+ assert_equal ['}'],
+ R.scan('rbrace', '{1=>2}')
+ end
+
+ def test_lbracket
+ assert_equal [],
+ R.scan('lbracket', '')
+ assert_equal ['['],
+ R.scan('lbracket', '[]')
+ assert_equal ['['],
+ R.scan('lbracket', 'a[1]')
+ assert_equal [],
+ R.scan('lbracket', 'm(%q[])')
+ end
+
+ def test_rbracket
+ assert_equal [],
+ R.scan('rbracket', '')
+ assert_equal [']'],
+ R.scan('rbracket', '[]')
+ assert_equal [']'],
+ R.scan('rbracket', 'a[1]')
+ assert_equal [],
+ R.scan('rbracket', 'm(%q[])')
+ end
+
+ def test_lparen
+ assert_equal [],
+ R.scan('lparen', '')
+ assert_equal ['('],
+ R.scan('lparen', '()')
+ assert_equal ['('],
+ R.scan('lparen', 'm()')
+ assert_equal ['('],
+ R.scan('lparen', 'm (a)')
+ assert_equal [],
+ R.scan('lparen', '"()"')
+ assert_equal [],
+ R.scan('lparen', '"%w()"')
+ end
+
+ def test_rparen
+ assert_equal [],
+ R.scan('rparen', '')
+ assert_equal [')'],
+ R.scan('rparen', '()')
+ assert_equal [')'],
+ R.scan('rparen', 'm()')
+ assert_equal [')'],
+ R.scan('rparen', 'm (a)')
+ assert_equal [],
+ R.scan('rparen', '"()"')
+ assert_equal [],
+ R.scan('rparen', '"%w()"')
+ end
+
+ def test_op
+ assert_equal [],
+ R.scan('op', '')
+ assert_equal ['|'],
+ R.scan('op', '1 | 1')
+ assert_equal ['^'],
+ R.scan('op', '1 ^ 1')
+ assert_equal ['&'],
+ R.scan('op', '1 & 1')
+ assert_equal ['<=>'],
+ R.scan('op', '1 <=> 1')
+ assert_equal ['=='],
+ R.scan('op', '1 == 1')
+ assert_equal ['==='],
+ R.scan('op', '1 === 1')
+ assert_equal ['=~'],
+ R.scan('op', '1 =~ 1')
+ assert_equal ['>'],
+ R.scan('op', '1 > 1')
+ assert_equal ['>='],
+ R.scan('op', '1 >= 1')
+ assert_equal ['<'],
+ R.scan('op', '1 < 1')
+ assert_equal ['<='],
+ R.scan('op', '1 <= 1')
+ assert_equal ['<<'],
+ R.scan('op', '1 << 1')
+ assert_equal ['>>'],
+ R.scan('op', '1 >> 1')
+ assert_equal ['+'],
+ R.scan('op', '1 + 1')
+ assert_equal ['-'],
+ R.scan('op', '1 - 1')
+ assert_equal ['*'],
+ R.scan('op', '1 * 1')
+ assert_equal ['/'],
+ R.scan('op', '1 / 1')
+ assert_equal ['%'],
+ R.scan('op', '1 % 1')
+ assert_equal ['**'],
+ R.scan('op', '1 ** 1')
+ assert_equal ['~'],
+ R.scan('op', '~1')
+ assert_equal ['-'],
+ R.scan('op', '-a')
+ assert_equal ['+'],
+ R.scan('op', '+a')
+ assert_equal ['[]'],
+ R.scan('op', ':[]')
+ assert_equal ['[]='],
+ R.scan('op', ':[]=')
+ assert_equal [],
+ R.scan('op', %q[`make all`])
+ end
+
+ def test_symbeg
+ assert_equal [],
+ R.scan('symbeg', '')
+ assert_equal [':'],
+ R.scan('symbeg', ':sym')
+ assert_equal [':'],
+ R.scan('symbeg', '[1,2,3,:sym]')
+ assert_equal [],
+ R.scan('symbeg', '":sym"')
+ assert_equal [],
+ R.scan('symbeg', 'a ? b : c')
+ end
+
+ def test_tstring_beg
+ assert_equal [],
+ R.scan('tstring_beg', '')
+ assert_equal ['"'],
+ R.scan('tstring_beg', '"abcdef"')
+ assert_equal ['%q['],
+ R.scan('tstring_beg', '%q[abcdef]')
+ assert_equal ['%Q['],
+ R.scan('tstring_beg', '%Q[abcdef]')
+ end
+
+ def test_tstring_content
+ assert_equal [],
+ R.scan('tstring_content', '')
+ assert_equal ['abcdef'],
+ R.scan('tstring_content', '"abcdef"')
+ assert_equal ['abcdef'],
+ R.scan('tstring_content', '%q[abcdef]')
+ assert_equal ['abcdef'],
+ R.scan('tstring_content', '%Q[abcdef]')
+ assert_equal ['abc', 'def'],
+ R.scan('tstring_content', '"abc#{1}def"')
+ assert_equal ['sym'],
+ R.scan('tstring_content', ':"sym"')
+ end
+
+ def test_tstring_end
+ assert_equal [],
+ R.scan('tstring_end', '')
+ assert_equal ['"'],
+ R.scan('tstring_end', '"abcdef"')
+ assert_equal [']'],
+ R.scan('tstring_end', '%q[abcdef]')
+ assert_equal [']'],
+ R.scan('tstring_end', '%Q[abcdef]')
+ end
+
+ def test_regexp_beg
+ assert_equal [],
+ R.scan('regexp_beg', '')
+ assert_equal ['/'],
+ R.scan('regexp_beg', '/re/')
+ assert_equal ['%r<'],
+ R.scan('regexp_beg', '%r<re>')
+ assert_equal [],
+ R.scan('regexp_beg', '5 / 5')
+ end
+
+ def test_regexp_end
+ assert_equal [],
+ R.scan('regexp_end', '')
+ assert_equal ['/'],
+ R.scan('regexp_end', '/re/')
+ assert_equal ['>'],
+ R.scan('regexp_end', '%r<re>')
+ end
+
+ def test_words_beg
+ assert_equal [],
+ R.scan('words_beg', '')
+ assert_equal ['%W('],
+ R.scan('words_beg', '%W()')
+ assert_equal ['%W('],
+ R.scan('words_beg', '%W(w w w)')
+ assert_equal ['%W( '],
+ R.scan('words_beg', '%W( w w w )')
+ end
+
+ def test_qwords_beg
+ assert_equal [],
+ R.scan('qwords_beg', '')
+ assert_equal ['%w('],
+ R.scan('qwords_beg', '%w()')
+ assert_equal ['%w('],
+ R.scan('qwords_beg', '%w(w w w)')
+ assert_equal ['%w( '],
+ R.scan('qwords_beg', '%w( w w w )')
+ end
+
+ # FIXME: Close paren must not present (`words_end' scanner event?).
+ def test_words_sep
+ assert_equal [],
+ R.scan('words_sep', '')
+ assert_equal [')'],
+ R.scan('words_sep', '%w()')
+ assert_equal [' ', ' ', ')'],
+ R.scan('words_sep', '%w(w w w)')
+ assert_equal [' ', ' ', ' )'],
+ R.scan('words_sep', '%w( w w w )')
+ assert_equal ["\n", ' ', ' )'],
+ R.scan('words_sep', "%w( w\nw w )")
+ end
+
+ def test_heredoc_beg
+ assert_equal [],
+ R.scan('heredoc_beg', '')
+ assert_equal ['<<EOS'],
+ R.scan('heredoc_beg', "<<EOS\nheredoc\nEOS")
+ assert_equal ['<<EOS'],
+ R.scan('heredoc_beg', "<<EOS\nheredoc\nEOS\n")
+ assert_equal ['<<EOS'],
+ R.scan('heredoc_beg', "<<EOS\nheredoc\nEOS \n")
+ assert_equal ['<<-EOS'],
+ R.scan('heredoc_beg', "<<-EOS\nheredoc\n\tEOS \n")
+ assert_equal ['<<"EOS"'],
+ R.scan('heredoc_beg', %Q[<<"EOS"\nheredoc\nEOS])
+ assert_equal [%q(<<'EOS')],
+ R.scan('heredoc_beg', "<<'EOS'\nheredoc\nEOS")
+ assert_equal [%q(<<`EOS`)],
+ R.scan('heredoc_beg', "<<`EOS`\nheredoc\nEOS")
+ end
+
+ def test_heredoc_content
+ assert_equal [],
+ R.scan('heredoc_content', '')
+ assert_equal ["heredoc\n"],
+ R.scan('heredoc_content', "<<EOS\nheredoc\nEOS")
+ assert_equal ["heredoc\n"],
+ R.scan('heredoc_content', "<<EOS\nheredoc\nEOS\n")
+ assert_equal ["heredoc \n"],
+ R.scan('heredoc_content', "<<EOS\nheredoc \nEOS \n")
+ assert_equal ["heredoc\n"],
+ R.scan('heredoc_content', "<<-EOS\nheredoc\n\tEOS \n")
+ end
+
+ def test_heredoc_end
+ assert_equal [],
+ R.scan('heredoc_end', '')
+ assert_equal ["EOS"],
+ R.scan('heredoc_end', "<<EOS\nheredoc\nEOS")
+ assert_equal ["EOS\n"],
+ R.scan('heredoc_end', "<<EOS\nheredoc\nEOS\n")
+ assert_equal ["EOS \n"],
+ R.scan('heredoc_end', "<<EOS\nheredoc\nEOS \n")
+ assert_equal ["\tEOS \n"],
+ R.scan('heredoc_end', "<<-EOS\nheredoc\n\tEOS \n")
+ end
+
+ def test_semicolon
+ assert_equal [],
+ R.scan('semicolon', '')
+ assert_equal %w(;),
+ R.scan('semicolon', ';')
+ assert_equal %w(; ;),
+ R.scan('semicolon', ';;')
+ assert_equal %w(; ; ;),
+ R.scan('semicolon', 'nil;nil;nil;')
+ assert_equal %w(; ; ;),
+ R.scan('semicolon', 'nil;nil;nil;nil')
+ assert_equal [],
+ R.scan('semicolon', '";"')
+ assert_equal [],
+ R.scan('semicolon', '%w(;)')
+ assert_equal [],
+ R.scan('semicolon', '/;/')
+ end
+
+ def test_comment
+ assert_equal [],
+ R.scan('comment', '')
+ assert_equal ['# comment'],
+ R.scan('comment', '# comment')
+ assert_equal ["# comment\n"],
+ R.scan('comment', "# comment\n")
+ assert_equal ["# comment\n"],
+ R.scan('comment', "# comment\n1 + 1")
+ assert_equal ["# comment\n"],
+ R.scan('comment', "1 + 1 + 1# comment\n1 + 1")
+ end
+
+ def test_embdoc_beg
+ assert_equal [],
+ R.scan('embdoc_beg', '')
+ assert_equal ["=begin\n"],
+ R.scan('embdoc_beg', "=begin\ndoc\n=end")
+ assert_equal ["=begin \n"],
+ R.scan('embdoc_beg', "=begin \ndoc\n=end\n")
+ assert_equal ["=begin comment\n"],
+ R.scan('embdoc_beg', "=begin comment\ndoc\n=end\n")
+ end
+
+ def test_embdoc
+ assert_equal [],
+ R.scan('embdoc', '')
+ assert_equal ["doc\n"],
+ R.scan('embdoc', "=begin\ndoc\n=end")
+ assert_equal ["doc\n"],
+ R.scan('embdoc', "=begin\ndoc\n=end\n")
+ end
+
+ def test_embdoc_end
+ assert_equal [],
+ R.scan('embdoc_end', '')
+ assert_equal ["=end"],
+ R.scan('embdoc_end', "=begin\ndoc\n=end")
+ assert_equal ["=end\n"],
+ R.scan('embdoc_end', "=begin\ndoc\n=end\n")
+ end
+
+ def test_sp
+ assert_equal [],
+ R.scan('sp', '')
+ assert_equal [' '],
+ R.scan('sp', ' ')
+ assert_equal [' '],
+ R.scan('sp', ' 1')
+ assert_equal [],
+ R.scan('sp', "\n")
+ assert_equal [' '],
+ R.scan('sp', " \n")
+ assert_equal [' ', ' '],
+ R.scan('sp', "1 + 1")
+ assert_equal [],
+ R.scan('sp', "' '")
+ assert_equal [],
+ R.scan('sp', "%w( )")
+ assert_equal [],
+ R.scan('sp', "%w( w )")
+ assert_equal [],
+ R.scan('sp', "p(/ /)")
+ end
+
+ # `nl' event always means End-Of-Statement.
+ def test_nl
+ assert_equal [],
+ R.scan('nl', '')
+ assert_equal [],
+ R.scan('nl', "\n")
+ assert_equal ["\n"],
+ R.scan('nl', "1 + 1\n")
+ assert_equal ["\n", "\n"],
+ R.scan('nl', "1 + 1\n2 + 2\n")
+ assert_equal [],
+ R.scan('nl', "1 +\n1")
+ assert_equal [],
+ R.scan('nl', "1;\n")
+ assert_equal ["\n"],
+ R.scan('nl', "1 + 1\r\n")
+ assert_equal [],
+ R.scan('nl', "1;\r\n")
+ end
+
+ def test_ignored_nl
+ assert_equal [],
+ R.scan('ignored_nl', '')
+ assert_equal ["\n"],
+ R.scan('ignored_nl', "\n")
+ assert_equal [],
+ R.scan('ignored_nl', "1 + 1\n")
+ assert_equal [],
+ R.scan('ignored_nl', "1 + 1\n2 + 2\n")
+ assert_equal ["\n"],
+ R.scan('ignored_nl', "1 +\n1")
+ assert_equal ["\n"],
+ R.scan('ignored_nl', "1;\n")
+ assert_equal [],
+ R.scan('ignored_nl', "1 + 1\r\n")
+ assert_equal ["\n"],
+ R.scan('ignored_nl', "1;\r\n")
+ end
+
+ def test___end__
+ end
+
+ def test_CHAR
+ assert_equal [],
+ R.scan('CHAR', "")
+ assert_equal ["@"],
+ R.scan('CHAR', "@")
+ assert_equal [],
+ R.scan('CHAR', "@ivar")
+ end
+
+end