From c971589817150d1850b06a49e11d11edfc489bbf Mon Sep 17 00:00:00 2001
From: aamine <aamine@b2dd03c8-39d4-4d8f-98ff-823fe69b080e>
Date: Sun, 12 Sep 2004 17:34:30 +0000
Subject: * ext/ripper: ripper extention added. * ext/ripper/MANIFEST: new
 file. * ext/ripper/README: new file. * ext/ripper/depend: new file. *
 ext/ripper/extconf.rb: new file. * ext/ripper/eventids2.c: new file. *
 ext/ripper/ripper.rb.in: new file. * ext/ripper/lib/ripper.rb: new file. *
 ext/ripper/test/check-event-arity.rb: new file. *
 ext/ripper/test/check-event-coverage.sh: new file. *
 ext/ripper/test/check-scanner-event-coverage.rb: new file. *
 ext/ripper/test/list-called-events.rb: new file. * ext/ripper/test/src_rb:
 new file. * ext/ripper/test/validate.rb: new file. *
 ext/ripper/tools/generate-eventids1.rb: new file. *
 ext/ripper/tools/generate-param-macros.rb: new file. *
 ext/ripper/tools/generate-ripper_rb.rb: new file. *
 ext/ripper/tools/list-parse-event-ids.rb: new file. *
 ext/ripper/tools/list-scan-event-ids.rb: new file. *
 ext/ripper/tools/preproc.rb: new file. * ext/ripper/tools/strip.rb: new file.
 * test/ripper: ripper tests added. * test/ripper/dummyparser.rb: new file. *
 test/ripper/test_parser_events.rb: new file. *
 test/ripper/test_scanner_events.rb: new file.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@6891 b2dd03c8-39d4-4d8f-98ff-823fe69b080e
---
 ChangeLog                                       |  52 ++
 MANIFEST                                        |   3 +
 ext/ripper/MANIFEST                             |  20 +
 ext/ripper/README                               |  31 +
 ext/ripper/depend                               |  37 +
 ext/ripper/eventids2.c                          | 272 +++++++
 ext/ripper/extconf.rb                           |  24 +
 ext/ripper/lib/ripper.rb                        | 945 ++++++++++++++++++++++++
 ext/ripper/ripper.rb.in                         |  53 ++
 ext/ripper/test/check-event-arity.rb            |  24 +
 ext/ripper/test/check-event-coverage.sh         |  15 +
 ext/ripper/test/check-scanner-event-coverage.rb |  18 +
 ext/ripper/test/list-called-events.rb           |  12 +
 ext/ripper/test/src_rb                          | 363 +++++++++
 ext/ripper/test/validate.rb                     |  68 ++
 ext/ripper/tools/generate-eventids1.rb          |  18 +
 ext/ripper/tools/generate-param-macros.rb       |  14 +
 ext/ripper/tools/generate-ripper_rb.rb          |  44 ++
 ext/ripper/tools/list-parse-event-ids.rb        |  38 +
 ext/ripper/tools/list-scan-event-ids.rb         |  32 +
 ext/ripper/tools/preproc.rb                     |  57 ++
 ext/ripper/tools/strip.rb                       |  12 +
 test/ripper/dummyparser.rb                      | 526 +++++++++++++
 test/ripper/test_parser_events.rb               | 489 ++++++++++++
 test/ripper/test_scanner_events.rb              | 744 +++++++++++++++++++
 25 files changed, 3911 insertions(+)
 create mode 100644 ext/ripper/MANIFEST
 create mode 100644 ext/ripper/README
 create mode 100644 ext/ripper/depend
 create mode 100644 ext/ripper/eventids2.c
 create mode 100644 ext/ripper/extconf.rb
 create mode 100644 ext/ripper/lib/ripper.rb
 create mode 100644 ext/ripper/ripper.rb.in
 create mode 100644 ext/ripper/test/check-event-arity.rb
 create mode 100755 ext/ripper/test/check-event-coverage.sh
 create mode 100644 ext/ripper/test/check-scanner-event-coverage.rb
 create mode 100644 ext/ripper/test/list-called-events.rb
 create mode 100644 ext/ripper/test/src_rb
 create mode 100644 ext/ripper/test/validate.rb
 create mode 100755 ext/ripper/tools/generate-eventids1.rb
 create mode 100755 ext/ripper/tools/generate-param-macros.rb
 create mode 100755 ext/ripper/tools/generate-ripper_rb.rb
 create mode 100755 ext/ripper/tools/list-parse-event-ids.rb
 create mode 100755 ext/ripper/tools/list-scan-event-ids.rb
 create mode 100755 ext/ripper/tools/preproc.rb
 create mode 100755 ext/ripper/tools/strip.rb
 create mode 100644 test/ripper/dummyparser.rb
 create mode 100644 test/ripper/test_parser_events.rb
 create mode 100644 test/ripper/test_scanner_events.rb

diff --git a/ChangeLog b/ChangeLog
index 552684784d..41f4e7c0bb 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,55 @@
+Mon Sep 13 02:26:31 2004  Minero Aoki  <aamine@loveruby.net>
+
+	* ext/ripper: ripper extention added.
+
+	* ext/ripper/MANIFEST: new file.
+
+	* ext/ripper/README: new file.
+
+	* ext/ripper/depend: new file.
+
+	* ext/ripper/extconf.rb: new file.
+
+	* ext/ripper/eventids2.c: new file.
+
+	* ext/ripper/ripper.rb.in: new file.
+
+	* ext/ripper/lib/ripper.rb: new file.
+
+	* ext/ripper/test/check-event-arity.rb: new file.
+
+	* ext/ripper/test/check-event-coverage.sh: new file.
+
+	* ext/ripper/test/check-scanner-event-coverage.rb: new file.
+
+	* ext/ripper/test/list-called-events.rb: new file.
+
+	* ext/ripper/test/src_rb: new file.
+
+	* ext/ripper/test/validate.rb: new file.
+
+	* ext/ripper/tools/generate-eventids1.rb: new file.
+
+	* ext/ripper/tools/generate-param-macros.rb: new file.
+
+	* ext/ripper/tools/generate-ripper_rb.rb: new file.
+
+	* ext/ripper/tools/list-parse-event-ids.rb: new file.
+
+	* ext/ripper/tools/list-scan-event-ids.rb: new file.
+
+	* ext/ripper/tools/preproc.rb: new file.
+
+	* ext/ripper/tools/strip.rb: new file.
+
+	* test/ripper: ripper tests added.
+
+	* test/ripper/dummyparser.rb: new file.
+
+	* test/ripper/test_parser_events.rb: new file.
+
+	* test/ripper/test_scanner_events.rb: new file.
+
 Mon Sep 13 01:03:02 2004  Hidetoshi NAGAI  <nagai@ai.kyutech.ac.jp>
 
 	* ext/tcltklib/tcltklib.c: improve control of preserv/release tcltkip
diff --git a/MANIFEST b/MANIFEST
index eaa424ffd3..47072cd648 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -878,6 +878,9 @@ test/optparse/test_placearg.rb
 test/optparse/test_reqarg.rb
 test/ostruct/test_ostruct.rb
 test/rinda/test_rinda.rb
+test/ripper/dummyparser.rb
+test/ripper/test_parser_events.rb
+test/ripper/test_scanner_events.rb
 test/rss/rss-assertions.rb
 test/rss/rss-testcase.rb
 test/rss/test_1.0.rb
diff --git a/ext/ripper/MANIFEST b/ext/ripper/MANIFEST
new file mode 100644
index 0000000000..fe1e25e620
--- /dev/null
+++ b/ext/ripper/MANIFEST
@@ -0,0 +1,20 @@
+MANIFEST
+README
+depend
+eventids2.c
+extconf.rb
+ripper.rb.in
+lib/ripper.rb
+test/check-event-arity.rb
+test/check-event-coverage.sh
+test/check-scanner-event-coverage.rb
+test/list-called-events.rb
+test/src.rb
+test/validate.rb
+tools/generate-eventids1.rb
+tools/generate-param-macros.rb
+tools/generate-ripper_rb.rb
+tools/list-parse-event-ids.rb
+tools/list-scan-event-ids.rb
+tools/preproc.rb
+tools/strip.rb
diff --git a/ext/ripper/README b/ext/ripper/README
new file mode 100644
index 0000000000..5242c4256d
--- /dev/null
+++ b/ext/ripper/README
@@ -0,0 +1,31 @@
+Ripper README
+=============
+
+  Ripper is a Ruby script parser.  You can get information
+  by event-based style from the parser.
+
+  !! WARNING !!
+
+  Ripper is still early-alpha version.
+  I never assure any kind of backward compatibility.
+  
+Requirements
+------------
+
+  * ruby 1.9 (support CVS HEAD only)
+  * bison 1.28 or later (You MUST use bison)
+
+Usage
+-----
+
+  See test/ripper/* and sample/ripper/*.
+  To know thrown events, see lib/ripper.rb.
+
+License
+-------
+
+  Ruby License.
+
+                                                Minero Aoki
+                                        aamine@loveruby.net
+                                      http://i.loveruby.net
diff --git a/ext/ripper/depend b/ext/ripper/depend
new file mode 100644
index 0000000000..b2333a2fdf
--- /dev/null
+++ b/ext/ripper/depend
@@ -0,0 +1,37 @@
+ripper.o: ripper.c lex.c eventids1.c eventids2.c
+	$(CC) $(CFLAGS) $(CPPFLAGS) -c ripper.c
+
+ripper.c: ripper.y
+	bison -t -v -o$@ ripper.y
+
+ripper.y: $(hdrdir)/parse.y $(srcdir)/tools/preproc.rb
+	$(RUBY) $(srcdir)/tools/preproc.rb $(hdrdir)/parse.y > $@
+
+eventids1.c: $(hdrdir)/parse.y $(srcdir)/tools/list-parse-event-ids.rb $(srcdir)/tools/generate-eventids1.rb
+	$(RUBY) $(srcdir)/tools/list-parse-event-ids.rb $(hdrdir)/parse.y | $(RUBY) $(srcdir)/tools/generate-eventids1.rb > $@
+
+lex.c:
+	cp $(hdrdir)/lex.c . || cp $(srcdir)/lex.c .
+
+#
+# Entries for ripper developpers (no srcdir/objdir support)
+#
+lib/ripper.rb: ripper.rb.in ids1 ids2 tools/generate-ripper_rb.rb
+	$(RUBY) tools/generate-ripper_rb.rb ripper.rb.in ids1 ids2 > $@
+
+ids1: tools/list-parse-event-ids.rb ../../parse.y
+	$(RUBY) tools/list-parse-event-ids.rb -a ../../parse.y > $@
+
+ids2: $(srcdir)/tools/list-scan-event-ids.rb eventids2.c
+	$(RUBY) tools/list-scan-event-ids.rb -a eventids2.c > $@
+
+test: ripper.so test/*.*
+	$(RUBY) test/check-event-arity.rb $(hdrdir)/parse.y
+	sh test/check-event-coverage.sh
+	$(RUBY) test/check-scanner-event-coverage.rb
+	$(RUBY) test/validate.rb > /dev/null
+	@echo OK
+
+preproc: ripper.E
+ripper.E: ripper.c
+	$(CC) -E $(CPPFLAGS) ripper.c | ruby tools/strip.rb > $@
diff --git a/ext/ripper/eventids2.c b/ext/ripper/eventids2.c
new file mode 100644
index 0000000000..28519eb8e6
--- /dev/null
+++ b/ext/ripper/eventids2.c
@@ -0,0 +1,272 @@
+#define tIGNORED_NL (tLAST_TOKEN + 1)
+#define tCOMMENT    (tLAST_TOKEN + 2)
+#define tEMBDOC_BEG (tLAST_TOKEN + 3)
+#define tEMBDOC     (tLAST_TOKEN + 4)
+#define tEMBDOC_END (tLAST_TOKEN + 5)
+#define tLWSP       (tLAST_TOKEN + 6)
+#define tHEREDOC_BEG (tLAST_TOKEN + 7)
+#define tHEREDOC_CONTENT (tLAST_TOKEN + 8)
+#define tHEREDOC_END (tLAST_TOKEN + 9)
+#define k__END__    (tLAST_TOKEN + 10)
+#define tCHAR       (tLAST_TOKEN + 11)
+
+static ID ripper_id_scan;
+
+static ID ripper_id_backref;
+static ID ripper_id_backtick;
+static ID ripper_id_comma;
+static ID ripper_id_const;
+static ID ripper_id_cvar;
+static ID ripper_id_embexpr_beg;
+static ID ripper_id_embexpr_end;
+static ID ripper_id_embvar;
+static ID ripper_id_float;
+static ID ripper_id_gvar;
+static ID ripper_id_ident;
+static ID ripper_id_int;
+static ID ripper_id_ivar;
+static ID ripper_id_kw;
+static ID ripper_id_lbrace;
+static ID ripper_id_lbracket;
+static ID ripper_id_lparen;
+static ID ripper_id_nl;
+static ID ripper_id_op;
+static ID ripper_id_period;
+static ID ripper_id_rbrace;
+static ID ripper_id_rbracket;
+static ID ripper_id_rparen;
+static ID ripper_id_semicolon;
+static ID ripper_id_symbeg;
+static ID ripper_id_tstring_beg;
+static ID ripper_id_tstring_content;
+static ID ripper_id_tstring_end;
+static ID ripper_id_words_beg;
+static ID ripper_id_qwords_beg;
+static ID ripper_id_words_sep;
+static ID ripper_id_regexp_beg;
+static ID ripper_id_regexp_end;
+
+static ID ripper_id_ignored_nl;
+static ID ripper_id_comment;
+static ID ripper_id_embdoc_beg;
+static ID ripper_id_embdoc;
+static ID ripper_id_embdoc_end;
+static ID ripper_id_lwsp;
+static ID ripper_id_heredoc_beg;
+static ID ripper_id_heredoc_content;
+static ID ripper_id_heredoc_end;
+static ID ripper_id___end__;
+static ID ripper_id_CHAR;
+
+static void
+ripper_init_eventids2()
+{
+    ripper_id_scan = rb_intern("on__scan");
+
+    ripper_id_backref = rb_intern("on__backref");
+    ripper_id_backtick = rb_intern("on__backtick");
+    ripper_id_comma = rb_intern("on__comma");
+    ripper_id_const = rb_intern("on__const");
+    ripper_id_cvar = rb_intern("on__cvar");
+    ripper_id_embexpr_beg = rb_intern("on__embexpr_beg");
+    ripper_id_embexpr_end = rb_intern("on__embexpr_end");
+    ripper_id_embvar = rb_intern("on__embvar");
+    ripper_id_float = rb_intern("on__float");
+    ripper_id_gvar = rb_intern("on__gvar");
+    ripper_id_ident = rb_intern("on__ident");
+    ripper_id_int = rb_intern("on__int");
+    ripper_id_ivar = rb_intern("on__ivar");
+    ripper_id_kw = rb_intern("on__kw");
+    ripper_id_lbrace = rb_intern("on__lbrace");
+    ripper_id_lbracket = rb_intern("on__lbracket");
+    ripper_id_lparen = rb_intern("on__lparen");
+    ripper_id_nl = rb_intern("on__nl");
+    ripper_id_op = rb_intern("on__op");
+    ripper_id_period = rb_intern("on__period");
+    ripper_id_rbrace = rb_intern("on__rbrace");
+    ripper_id_rbracket = rb_intern("on__rbracket");
+    ripper_id_rparen = rb_intern("on__rparen");
+    ripper_id_semicolon = rb_intern("on__semicolon");
+    ripper_id_symbeg = rb_intern("on__symbeg");
+    ripper_id_tstring_beg = rb_intern("on__tstring_beg");
+    ripper_id_tstring_content = rb_intern("on__tstring_content");
+    ripper_id_tstring_end = rb_intern("on__tstring_end");
+    ripper_id_words_beg = rb_intern("on__words_beg");
+    ripper_id_qwords_beg = rb_intern("on__qwords_beg");
+    ripper_id_words_sep = rb_intern("on__words_sep");
+    ripper_id_regexp_beg = rb_intern("on__regexp_beg");
+    ripper_id_regexp_end = rb_intern("on__regexp_end");
+
+    ripper_id_ignored_nl = rb_intern("on__ignored_nl");
+    ripper_id_comment = rb_intern("on__comment");
+    ripper_id_embdoc_beg = rb_intern("on__embdoc_beg");
+    ripper_id_embdoc = rb_intern("on__embdoc");
+    ripper_id_embdoc_end = rb_intern("on__embdoc_end");
+    ripper_id_lwsp = rb_intern("on__sp");
+    ripper_id_heredoc_beg = rb_intern("on__heredoc_beg");
+    ripper_id_heredoc_content = rb_intern("on__heredoc_content");
+    ripper_id_heredoc_end = rb_intern("on__heredoc_end");
+    ripper_id___end__ = rb_intern("on____end__");
+    ripper_id_CHAR = rb_intern("on__CHAR");
+}
+
+static struct token_assoc {
+    int token;
+    ID *id;
+} token_to_eventid[] = {
+    {' ',		&ripper_id_words_sep},
+    {'!',		&ripper_id_op},
+    {'%',		&ripper_id_op},
+    {'&',		&ripper_id_op},
+    {'*',		&ripper_id_op},
+    {'+',		&ripper_id_op},
+    {'-',		&ripper_id_op},
+    {'/',		&ripper_id_op},
+    {'<',		&ripper_id_op},
+    {'=',		&ripper_id_op},
+    {'>',		&ripper_id_op},
+    {'?',		&ripper_id_op},
+    {'^',		&ripper_id_op},
+    {'|',		&ripper_id_op},
+    {'~',		&ripper_id_op},
+    {':',		&ripper_id_op},
+    {',',		&ripper_id_comma},
+    {'.',		&ripper_id_period},
+    {';',		&ripper_id_semicolon},
+    {'`',		&ripper_id_backtick},
+    {'\n',              &ripper_id_nl},
+    {kALIAS,		&ripper_id_kw},
+    {kAND,		&ripper_id_kw},
+    {kBEGIN,		&ripper_id_kw},
+    {kBREAK,		&ripper_id_kw},
+    {kCASE,		&ripper_id_kw},
+    {kCLASS,		&ripper_id_kw},
+    {kDEF,		&ripper_id_kw},
+    {kDEFINED,		&ripper_id_kw},
+    {kDO,		&ripper_id_kw},
+    {kDO_BLOCK,		&ripper_id_kw},
+    {kDO_COND,		&ripper_id_kw},
+    {kELSE,		&ripper_id_kw},
+    {kELSIF,		&ripper_id_kw},
+    {kEND,		&ripper_id_kw},
+    {kENSURE,		&ripper_id_kw},
+    {kFALSE,		&ripper_id_kw},
+    {kFOR,		&ripper_id_kw},
+    {kIF,		&ripper_id_kw},
+    {kIF_MOD,		&ripper_id_kw},
+    {kIN,		&ripper_id_kw},
+    {kMODULE,		&ripper_id_kw},
+    {kNEXT,		&ripper_id_kw},
+    {kNIL,		&ripper_id_kw},
+    {kNOT,		&ripper_id_kw},
+    {kOR,		&ripper_id_kw},
+    {kREDO,		&ripper_id_kw},
+    {kRESCUE,		&ripper_id_kw},
+    {kRESCUE_MOD,	&ripper_id_kw},
+    {kRETRY,		&ripper_id_kw},
+    {kRETURN,		&ripper_id_kw},
+    {kSELF,		&ripper_id_kw},
+    {kSUPER,		&ripper_id_kw},
+    {kTHEN,		&ripper_id_kw},
+    {kTRUE,		&ripper_id_kw},
+    {kUNDEF,		&ripper_id_kw},
+    {kUNLESS,		&ripper_id_kw},
+    {kUNLESS_MOD,	&ripper_id_kw},
+    {kUNTIL,		&ripper_id_kw},
+    {kUNTIL_MOD,	&ripper_id_kw},
+    {kWHEN,		&ripper_id_kw},
+    {kWHILE,		&ripper_id_kw},
+    {kWHILE_MOD,	&ripper_id_kw},
+    {kYIELD,		&ripper_id_kw},
+    {k__FILE__,		&ripper_id_kw},
+    {k__LINE__,		&ripper_id_kw},
+    {klBEGIN,		&ripper_id_kw},
+    {klEND,		&ripper_id_kw},
+    {tAMPER,		&ripper_id_op},
+    {tANDOP,		&ripper_id_op},
+    {tAREF,		&ripper_id_op},
+    {tASET,		&ripper_id_op},
+    {tASSOC,		&ripper_id_op},
+    {tBACK_REF,		&ripper_id_backref},
+    {tCMP,		&ripper_id_op},
+    {tCOLON2,		&ripper_id_op},
+    {tCOLON3,		&ripper_id_op},
+    {tCONSTANT,		&ripper_id_const},
+    {tCVAR,		&ripper_id_cvar},
+    {tDOT2,		&ripper_id_op},
+    {tDOT3,		&ripper_id_op},
+    {tEQ,		&ripper_id_op},
+    {tEQQ,		&ripper_id_op},
+    {tFID,		&ripper_id_ident},
+    {tFLOAT,		&ripper_id_float},
+    {tGEQ,		&ripper_id_op},
+    {tGVAR,		&ripper_id_gvar},
+    {tIDENTIFIER,	&ripper_id_ident},
+    {tINTEGER,		&ripper_id_int},
+    {tIVAR,		&ripper_id_ivar},
+    {tLBRACE,		&ripper_id_lbrace},
+    {tLBRACE_ARG,	&ripper_id_lbrace},
+    {'{',       	&ripper_id_lbrace},
+    {'}',       	&ripper_id_rbrace},
+    {tLBRACK,		&ripper_id_lbracket},
+    {'[',       	&ripper_id_lbracket},
+    {']',       	&ripper_id_rbracket},
+    {tLEQ,		&ripper_id_op},
+    {tLPAREN,		&ripper_id_lparen},
+    {tLPAREN_ARG,	&ripper_id_lparen},
+    {'(',		&ripper_id_lparen},
+    {')',		&ripper_id_rparen},
+    {tLSHFT,		&ripper_id_op},
+    {tMATCH,		&ripper_id_op},
+    {tNEQ,		&ripper_id_op},
+    {tNMATCH,		&ripper_id_op},
+    {tNTH_REF,		&ripper_id_backref},
+    {tOP_ASGN,		&ripper_id_op},
+    {tOROP,		&ripper_id_op},
+    {tPOW,		&ripper_id_op},
+    {tQWORDS_BEG,	&ripper_id_qwords_beg},
+    {tREGEXP_BEG,	&ripper_id_regexp_beg},
+    {tREGEXP_END,	&ripper_id_regexp_end},
+    {tRPAREN,		&ripper_id_rparen},
+    {tRSHFT,		&ripper_id_op},
+    {tSTAR,		&ripper_id_op},
+    {tSTRING_BEG,	&ripper_id_tstring_beg},
+    {tSTRING_CONTENT,	&ripper_id_tstring_content},
+    {tSTRING_DBEG,	&ripper_id_embexpr_beg},
+    {tSTRING_DVAR,	&ripper_id_embvar},
+    {tSTRING_END,	&ripper_id_tstring_end},
+    {tSYMBEG,		&ripper_id_symbeg},
+    {tUMINUS,		&ripper_id_op},
+    {tUMINUS_NUM,	&ripper_id_op},
+    {tUPLUS,		&ripper_id_op},
+    {tWORDS_BEG,	&ripper_id_words_beg},
+    {tXSTRING_BEG,	&ripper_id_backtick},
+/* ripper specific tokens */
+    {tIGNORED_NL,       &ripper_id_ignored_nl},
+    {tCOMMENT,          &ripper_id_comment},
+    {tEMBDOC_BEG,       &ripper_id_embdoc_beg},
+    {tEMBDOC,           &ripper_id_embdoc},
+    {tEMBDOC_END,       &ripper_id_embdoc_end},
+    {tLWSP,             &ripper_id_lwsp},
+    {tHEREDOC_BEG,      &ripper_id_heredoc_beg},
+    {tHEREDOC_CONTENT,  &ripper_id_heredoc_content},
+    {tHEREDOC_END,      &ripper_id_heredoc_end},
+    {k__END__,          &ripper_id___end__},
+    {0, NULL}
+};
+
+static ID
+ripper_token2eventid(tok)
+    int tok;
+{
+    struct token_assoc *a;
+
+    for (a = token_to_eventid; a->id != NULL; a++) {
+        if (a->token == tok)
+            return *a->id;
+    }
+    if (tok < 256) {
+        return ripper_id_CHAR;
+    }
+    rb_raise(rb_eRuntimeError, "[Ripper FATAL] unknown token %d", tok);
+}
diff --git a/ext/ripper/extconf.rb b/ext/ripper/extconf.rb
new file mode 100644
index 0000000000..536e23d3ad
--- /dev/null
+++ b/ext/ripper/extconf.rb
@@ -0,0 +1,24 @@
+#!ruby -s
+
+require 'mkmf'
+require 'rbconfig'
+
+def main
+  unless have_command('bison')
+    $stderr.puts 'Ripper requires bison; abort'
+    exit 1
+  end
+  $objs = %w(ripper.o)
+  $cleanfiles.concat %w(ripper.y ripper.c ripper.output ids1 ids2)
+  $CPPFLAGS += ' -DRIPPER'
+  $CPPFLAGS += ' -DRIPPER_DEBUG' if $debug
+  create_makefile 'ripper'
+end
+
+def have_command(cmd)
+  ENV['PATH'].split(File::PATH_SEPARATOR).any? {|path|
+    File.executable?("#{path}/#{cmd}#{Config::CONFIG['EXEEXT']}")
+  }
+end
+
+main
diff --git a/ext/ripper/lib/ripper.rb b/ext/ripper/lib/ripper.rb
new file mode 100644
index 0000000000..b34147ee2a
--- /dev/null
+++ b/ext/ripper/lib/ripper.rb
@@ -0,0 +1,945 @@
+#
+# This file is automatically generated from ripper.rb.in and parse.y.
+# DO NOT MODIFY!!!!!!
+#
+
+#
+# ripper.rb
+#
+# Copyright (C) 2003,2004 Minero Aoki
+#
+
+require 'ripper.so'
+
+class Ripper
+  def Ripper.parse(str, *args)
+    new.parse(str, *args)
+  end
+
+  private
+
+  def warn(fmt, *args)
+  end
+
+  def warning(fmt, *args)
+  end
+
+  def compile_error(msg)
+  end
+
+  PARSER_EVENTS = [
+    :BEGIN,
+    :END,
+    :alias,
+    :alias_error,
+    :aref,
+    :aref_field,
+    :arg_ambiguous,
+    :arg_paren,
+    :arglist_add,
+    :arglist_add_block,
+    :arglist_add_star,
+    :arglist_new,
+    :arglist_prepend,
+    :array,
+    :assign,
+    :assign_error,
+    :assoc_new,
+    :assoclist_from_args,
+    :bare_assoc_hash,
+    :begin,
+    :binary,
+    :blockvar_add_block,
+    :blockvar_add_star,
+    :blockvar_new,
+    :bodystmt,
+    :brace_block,
+    :break,
+    :call,
+    :case,
+    :class,
+    :class_name_error,
+    :command,
+    :command_call,
+    :const_ref,
+    :constpath_field,
+    :constpath_ref,
+    :def,
+    :defined,
+    :defs,
+    :do_block,
+    :dot2,
+    :dot3,
+    :dyna_symbol,
+    :else,
+    :elsif,
+    :ensure,
+    :fcall,
+    :field,
+    :for,
+    :hash,
+    :if,
+    :if_mod,
+    :ifop,
+    :iter_block,
+    :massign,
+    :method_add_arg,
+    :mlhs_add,
+    :mlhs_add_star,
+    :mlhs_new,
+    :mlhs_paren,
+    :module,
+    :mrhs_add,
+    :mrhs_add_star,
+    :mrhs_new,
+    :mrhs_new_from_arglist,
+    :next,
+    :opassign,
+    :param_error,
+    :params,
+    :paren,
+    :parse_error,
+    :program,
+    :qwords_add,
+    :qwords_new,
+    :redo,
+    :regexp_literal,
+    :rescue,
+    :rescue_mod,
+    :restparam,
+    :retry,
+    :return,
+    :return0,
+    :sclass,
+    :space,
+    :stmts_add,
+    :stmts_new,
+    :string_add,
+    :string_concat,
+    :string_content,
+    :string_dvar,
+    :string_embexpr,
+    :string_literal,
+    :super,
+    :symbol,
+    :symbol_literal,
+    :topconst_field,
+    :topconst_ref,
+    :unary,
+    :undef,
+    :unless,
+    :unless_mod,
+    :until,
+    :until_mod,
+    :var_alias,
+    :var_field,
+    :var_ref,
+    :void_stmt,
+    :when,
+    :while,
+    :while_mod,
+    :word_add,
+    :word_new,
+    :words_add,
+    :words_new,
+    :xstring_add,
+    :xstring_literal,
+    :xstring_new,
+    :yield,
+    :yield0,
+    :zsuper
+  ]
+
+  SCANNER_EVENTS = [
+    :BEGIN,
+    :END,
+    :alias,
+    :alias_error,
+    :aref,
+    :aref_field,
+    :arg_ambiguous,
+    :arg_paren,
+    :arglist_add,
+    :arglist_add_block,
+    :arglist_add_star,
+    :arglist_new,
+    :arglist_prepend,
+    :array,
+    :assign,
+    :assign_error,
+    :assoc_new,
+    :assoclist_from_args,
+    :bare_assoc_hash,
+    :begin,
+    :binary,
+    :blockvar_add_block,
+    :blockvar_add_star,
+    :blockvar_new,
+    :bodystmt,
+    :brace_block,
+    :break,
+    :call,
+    :case,
+    :class,
+    :class_name_error,
+    :command,
+    :command_call,
+    :const_ref,
+    :constpath_field,
+    :constpath_ref,
+    :def,
+    :defined,
+    :defs,
+    :do_block,
+    :dot2,
+    :dot3,
+    :dyna_symbol,
+    :else,
+    :elsif,
+    :ensure,
+    :fcall,
+    :field,
+    :for,
+    :hash,
+    :if,
+    :if_mod,
+    :ifop,
+    :iter_block,
+    :massign,
+    :method_add_arg,
+    :mlhs_add,
+    :mlhs_add_star,
+    :mlhs_new,
+    :mlhs_paren,
+    :module,
+    :mrhs_add,
+    :mrhs_add_star,
+    :mrhs_new,
+    :mrhs_new_from_arglist,
+    :next,
+    :opassign,
+    :param_error,
+    :params,
+    :paren,
+    :parse_error,
+    :program,
+    :qwords_add,
+    :qwords_new,
+    :redo,
+    :regexp_literal,
+    :rescue,
+    :rescue_mod,
+    :restparam,
+    :retry,
+    :return,
+    :return0,
+    :sclass,
+    :space,
+    :stmts_add,
+    :stmts_new,
+    :string_add,
+    :string_concat,
+    :string_content,
+    :string_dvar,
+    :string_embexpr,
+    :string_literal,
+    :super,
+    :symbol,
+    :symbol_literal,
+    :topconst_field,
+    :topconst_ref,
+    :unary,
+    :undef,
+    :unless,
+    :unless_mod,
+    :until,
+    :until_mod,
+    :var_alias,
+    :var_field,
+    :var_ref,
+    :void_stmt,
+    :when,
+    :while,
+    :while_mod,
+    :word_add,
+    :word_new,
+    :words_add,
+    :words_new,
+    :xstring_add,
+    :xstring_literal,
+    :xstring_new,
+    :yield,
+    :yield0,
+    :zsuper
+  ]
+  LEXER_EVENTS = SCANNER_EVENTS
+
+  EVENTS = PARSER_EVENTS + SCANNER_EVENTS
+
+  #
+  # Parser Events
+  #
+
+  def on__BEGIN(a)
+    a
+  end
+
+  def on__END(a)
+    a
+  end
+
+  def on__alias(a, b)
+    a
+  end
+
+  def on__alias_error(a)
+    a
+  end
+
+  def on__aref(a, b)
+    a
+  end
+
+  def on__aref_field(a, b)
+    a
+  end
+
+  def on__arg_ambiguous()
+    nil
+  end
+
+  def on__arg_paren(a)
+    a
+  end
+
+  def on__arglist_add(a, b)
+    a
+  end
+
+  def on__arglist_add_block(a, b)
+    a
+  end
+
+  def on__arglist_add_star(a, b)
+    a
+  end
+
+  def on__arglist_new()
+    nil
+  end
+
+  def on__arglist_prepend(a, b)
+    a
+  end
+
+  def on__array(a)
+    a
+  end
+
+  def on__assign(a, b)
+    a
+  end
+
+  def on__assign_error(a)
+    a
+  end
+
+  def on__assoc_new(a, b)
+    a
+  end
+
+  def on__assoclist_from_args(a)
+    a
+  end
+
+  def on__bare_assoc_hash(a)
+    a
+  end
+
+  def on__begin(a)
+    a
+  end
+
+  def on__binary(a, b, c)
+    a
+  end
+
+  def on__blockvar_add_block(a, b)
+    a
+  end
+
+  def on__blockvar_add_star(a, b)
+    a
+  end
+
+  def on__blockvar_new(a)
+    a
+  end
+
+  def on__bodystmt(a, b, c, d)
+    a
+  end
+
+  def on__brace_block(a, b)
+    a
+  end
+
+  def on__break(a)
+    a
+  end
+
+  def on__call(a, b, c)
+    a
+  end
+
+  def on__case(a, b)
+    a
+  end
+
+  def on__class(a, b, c)
+    a
+  end
+
+  def on__class_name_error(a)
+    a
+  end
+
+  def on__command(a, b)
+    a
+  end
+
+  def on__command_call(a, b, c, d)
+    a
+  end
+
+  def on__const_ref(a)
+    a
+  end
+
+  def on__constpath_field(a, b)
+    a
+  end
+
+  def on__constpath_ref(a, b)
+    a
+  end
+
+  def on__def(a, b, c)
+    a
+  end
+
+  def on__defined(a)
+    a
+  end
+
+  def on__defs(a, b, c, d, e)
+    a
+  end
+
+  def on__do_block(a, b)
+    a
+  end
+
+  def on__dot2(a, b)
+    a
+  end
+
+  def on__dot3(a, b)
+    a
+  end
+
+  def on__dyna_symbol(a)
+    a
+  end
+
+  def on__else(a)
+    a
+  end
+
+  def on__elsif(a, b, c)
+    a
+  end
+
+  def on__ensure(a)
+    a
+  end
+
+  def on__fcall(a)
+    a
+  end
+
+  def on__field(a, b, c)
+    a
+  end
+
+  def on__for(a, b, c)
+    a
+  end
+
+  def on__hash(a)
+    a
+  end
+
+  def on__if(a, b, c)
+    a
+  end
+
+  def on__if_mod(a, b)
+    a
+  end
+
+  def on__ifop(a, b, c)
+    a
+  end
+
+  def on__iter_block(a, b)
+    a
+  end
+
+  def on__massign(a, b)
+    a
+  end
+
+  def on__method_add_arg(a, b)
+    a
+  end
+
+  def on__mlhs_add(a, b)
+    a
+  end
+
+  def on__mlhs_add_star(a, b)
+    a
+  end
+
+  def on__mlhs_new()
+    nil
+  end
+
+  def on__mlhs_paren(a)
+    a
+  end
+
+  def on__module(a, b)
+    a
+  end
+
+  def on__mrhs_add(a, b)
+    a
+  end
+
+  def on__mrhs_add_star(a, b)
+    a
+  end
+
+  def on__mrhs_new()
+    nil
+  end
+
+  def on__mrhs_new_from_arglist(a)
+    a
+  end
+
+  def on__next(a)
+    a
+  end
+
+  def on__opassign(a, b, c)
+    a
+  end
+
+  def on__param_error(a)
+    a
+  end
+
+  def on__params(a, b, c, d)
+    a
+  end
+
+  def on__paren(a)
+    a
+  end
+
+  def on__parse_error(a)
+    a
+  end
+
+  def on__program(a)
+    a
+  end
+
+  def on__qwords_add(a, b)
+    a
+  end
+
+  def on__qwords_new()
+    nil
+  end
+
+  def on__redo()
+    nil
+  end
+
+  def on__regexp_literal(a)
+    a
+  end
+
+  def on__rescue(a, b, c, d)
+    a
+  end
+
+  def on__rescue_mod(a, b)
+    a
+  end
+
+  def on__restparam(a)
+    a
+  end
+
+  def on__retry()
+    nil
+  end
+
+  def on__return(a)
+    a
+  end
+
+  def on__return0()
+    nil
+  end
+
+  def on__sclass(a, b)
+    a
+  end
+
+  def on__space(a)
+    a
+  end
+
+  def on__stmts_add(a, b)
+    a
+  end
+
+  def on__stmts_new()
+    nil
+  end
+
+  def on__string_add(a, b)
+    a
+  end
+
+  def on__string_concat(a, b)
+    a
+  end
+
+  def on__string_content()
+    nil
+  end
+
+  def on__string_dvar(a)
+    a
+  end
+
+  def on__string_embexpr(a)
+    a
+  end
+
+  def on__string_literal(a)
+    a
+  end
+
+  def on__super(a)
+    a
+  end
+
+  def on__symbol(a)
+    a
+  end
+
+  def on__symbol_literal(a)
+    a
+  end
+
+  def on__topconst_field(a)
+    a
+  end
+
+  def on__topconst_ref(a)
+    a
+  end
+
+  def on__unary(a, b)
+    a
+  end
+
+  def on__undef(a)
+    a
+  end
+
+  def on__unless(a, b, c)
+    a
+  end
+
+  def on__unless_mod(a, b)
+    a
+  end
+
+  def on__until(a, b)
+    a
+  end
+
+  def on__until_mod(a, b)
+    a
+  end
+
+  def on__var_alias(a, b)
+    a
+  end
+
+  def on__var_field(a)
+    a
+  end
+
+  def on__var_ref(a)
+    a
+  end
+
+  def on__void_stmt()
+    nil
+  end
+
+  def on__when(a, b, c)
+    a
+  end
+
+  def on__while(a, b)
+    a
+  end
+
+  def on__while_mod(a, b)
+    a
+  end
+
+  def on__word_add(a, b)
+    a
+  end
+
+  def on__word_new()
+    nil
+  end
+
+  def on__words_add(a, b)
+    a
+  end
+
+  def on__words_new()
+    nil
+  end
+
+  def on__xstring_add(a, b)
+    a
+  end
+
+  def on__xstring_literal(a)
+    a
+  end
+
+  def on__xstring_new()
+    nil
+  end
+
+  def on__yield(a)
+    a
+  end
+
+  def on__yield0()
+    nil
+  end
+
+  def on__zsuper()
+    nil
+  end
+
+  #
+  # Lexer Events
+  #
+
+  def on__scan(event, token)
+  end
+
+  def on__CHAR(token)
+    token
+  end
+
+  def on____end__(token)
+    token
+  end
+
+  def on__backref(token)
+    token
+  end
+
+  def on__backtick(token)
+    token
+  end
+
+  def on__comma(token)
+    token
+  end
+
+  def on__comment(token)
+    token
+  end
+
+  def on__const(token)
+    token
+  end
+
+  def on__cvar(token)
+    token
+  end
+
+  def on__embdoc(token)
+    token
+  end
+
+  def on__embdoc_beg(token)
+    token
+  end
+
+  def on__embdoc_end(token)
+    token
+  end
+
+  def on__embexpr_beg(token)
+    token
+  end
+
+  def on__embexpr_end(token)
+    token
+  end
+
+  def on__embvar(token)
+    token
+  end
+
+  def on__float(token)
+    token
+  end
+
+  def on__gvar(token)
+    token
+  end
+
+  def on__heredoc_beg(token)
+    token
+  end
+
+  def on__heredoc_content(token)
+    token
+  end
+
+  def on__heredoc_end(token)
+    token
+  end
+
+  def on__ident(token)
+    token
+  end
+
+  def on__ignored_nl(token)
+    token
+  end
+
+  def on__int(token)
+    token
+  end
+
+  def on__ivar(token)
+    token
+  end
+
+  def on__kw(token)
+    token
+  end
+
+  def on__lbrace(token)
+    token
+  end
+
+  def on__lbracket(token)
+    token
+  end
+
+  def on__lparen(token)
+    token
+  end
+
+  def on__lwsp(token)
+    token
+  end
+
+  def on__nl(token)
+    token
+  end
+
+  def on__op(token)
+    token
+  end
+
+  def on__period(token)
+    token
+  end
+
+  def on__qwords_beg(token)
+    token
+  end
+
+  def on__rbrace(token)
+    token
+  end
+
+  def on__rbracket(token)
+    token
+  end
+
+  def on__regexp_beg(token)
+    token
+  end
+
+  def on__regexp_end(token)
+    token
+  end
+
+  def on__rparen(token)
+    token
+  end
+
+  def on__semicolon(token)
+    token
+  end
+
+  def on__symbeg(token)
+    token
+  end
+
+  def on__tstring_beg(token)
+    token
+  end
+
+  def on__tstring_content(token)
+    token
+  end
+
+  def on__tstring_end(token)
+    token
+  end
+
+  def on__words_beg(token)
+    token
+  end
+
+  def on__words_sep(token)
+    token
+  end
+end
diff --git a/ext/ripper/ripper.rb.in b/ext/ripper/ripper.rb.in
new file mode 100644
index 0000000000..c6fa554296
--- /dev/null
+++ b/ext/ripper/ripper.rb.in
@@ -0,0 +1,53 @@
+#
+# This file is automatically generated from ripper.rb.in and parse.y.
+# DO NOT MODIFY!!!!!!
+#
+
+#
+# ripper.rb
+#
+# Copyright (C) 2003,2004 Minero Aoki
+#
+
+require 'ripper.so'
+
+class Ripper
+  def Ripper.parse(str, *args)
+    new.parse(str, *args)
+  end
+
+  private
+
+  def warn(fmt, *args)
+  end
+
+  def warning(fmt, *args)
+  end
+
+  def compile_error(msg)
+  end
+
+  PARSER_EVENTS = [
+#include ids1
+  ]
+
+  SCANNER_EVENTS = [
+#include ids2
+  ]
+  LEXER_EVENTS = SCANNER_EVENTS
+
+  EVENTS = PARSER_EVENTS + SCANNER_EVENTS
+
+  #
+  # Parser Events
+  #
+#include handlers1
+
+  #
+  # Lexer Events
+  #
+
+  def on__scan(event, token)
+  end
+#include handlers2
+end
diff --git a/ext/ripper/test/check-event-arity.rb b/ext/ripper/test/check-event-arity.rb
new file mode 100644
index 0000000000..00e1b98dbc
--- /dev/null
+++ b/ext/ripper/test/check-event-arity.rb
@@ -0,0 +1,24 @@
+def main
+  invalid = false
+  table = {}
+  ARGF.each do |line|
+    next if /\A\#\s*define\s+s?dispatch\d/ === line
+    next if /ripper_dispatch\d/ === line
+    line.scan(/dispatch(\d)\((\w+)/) do |num, ev|
+      num = num.to_i
+      if data = table[ev]
+        locations, arity = data
+        unless num == arity
+          invalid = true
+          puts "arity differ [#{ev}]: #{ARGF.lineno}->#{num}; #{locations.join(',')}->#{arity}"
+        end
+        locations.push ARGF.lineno
+      else
+        table[ev] = [[ARGF.lineno], num.to_i]
+      end
+    end
+  end
+  exit 1 if invalid
+end
+
+main
diff --git a/ext/ripper/test/check-event-coverage.sh b/ext/ripper/test/check-event-coverage.sh
new file mode 100755
index 0000000000..727c3b9837
--- /dev/null
+++ b/ext/ripper/test/check-event-coverage.sh
@@ -0,0 +1,15 @@
+# $Id$
+
+RUBY=${RUBY:-ruby}
+status=0
+
+$RUBY tools/list-parse-event-ids.rb parse.y | awk '{print "on__" $1}' > list_a
+$RUBY test/list-called-events.rb | sort -u > list_b
+diff -u list_a list_b | grep '^-on' | sed 's/^-on__//' > list_diff
+if [ -s list_diff ]
+then
+    cat list_diff
+    status=1
+fi
+rm -f list_a list_b list_diff
+exit $status
diff --git a/ext/ripper/test/check-scanner-event-coverage.rb b/ext/ripper/test/check-scanner-event-coverage.rb
new file mode 100644
index 0000000000..4de01e35f4
--- /dev/null
+++ b/ext/ripper/test/check-scanner-event-coverage.rb
@@ -0,0 +1,18 @@
+def main
+  not_tested = eventids() - tested_ids()
+  unless not_tested.empty?
+    puts not_tested
+    exit 1
+  end
+  exit 0
+end
+
+def eventids
+  File.read('eventids2.c').scan(/on__(\w+)/).flatten.uniq
+end
+
+def tested_ids
+  File.read('test/test_scanner_events.rb').scan(/def test_(\S+)/).flatten.uniq
+end
+
+main
diff --git a/ext/ripper/test/list-called-events.rb b/ext/ripper/test/list-called-events.rb
new file mode 100644
index 0000000000..d799efc979
--- /dev/null
+++ b/ext/ripper/test/list-called-events.rb
@@ -0,0 +1,12 @@
+require 'ripper.so'
+
+class R < Ripper
+  def method_missing(mid, *args)
+    puts mid
+    args[0]
+  end
+  undef :warn
+end
+
+fname = (ARGV[0] || 'test/src_rb')
+R.new(File.read(fname), fname, 1).parse
diff --git a/ext/ripper/test/src_rb b/ext/ripper/test/src_rb
new file mode 100644
index 0000000000..e7894362f3
--- /dev/null
+++ b/ext/ripper/test/src_rb
@@ -0,0 +1,363 @@
+# comment
+=begin
+  embedded document
+=end
+
+# literal
+1
+1000000000000000000000
+1.0
+1.234e5
+1..2
+1...3
+:symbol
+:"dynamic #{sym_embexpr} symbol"
+[1,2,3]
+{1 => 2}
+'string'
+"string"
+"before #{str_embexpr} after"
+"str #@ivar str"
+"str #$gvar str"
+"string" "concat"
+`/bin/true`
+{1, 2, 3, 4}
+/regexp/
+/regexp_with_opt/mioe
+/regexp #{regexp_embexpr} after/
+%q[string]
+%Q[str#{str_embexpr}ing]
+%r[regexp]
+%w( a b c )
+%W( a#{w_emb}b c d )
+<<HERE
+heredoc line 1
+heredoc line 2
+heredoc line 3
+HERE
+
+# special variables
+true
+false
+nil
+self
+
+# def
+def a
+end
+def b()
+end
+def c(a)
+end
+def d(a,*rest)
+end
+def e(a,&block)
+end
+def f(a,*rest,&block)
+end
+def g(*rest)
+end
+def h(&block)
+end
+def i(*rest,&block)
+end
+def j(CONST)
+end
+def k(@ivar)
+end
+def l($gvar)
+end
+def n(@@cvar)
+end
+
+# alias
+alias x b
+alias $rest $'     # error
+alias $nth $1      # error
+
+# undef
+undef warn
+
+# class, module
+class C
+end
+module M
+end
+class cname
+end
+class << Object.new
+  def self.a
+  end
+end
+
+# field
+$a = 1
+$' = 0   # error
+$1 = 0   # error
+@a = 2
+@@a = 3
+a = 4
+a += 1
+a -= 1
+a *= 1
+a /= 1
+a &&= 1
+a ||= 1
+m.a = 5
+m.a += 1
+m.a &&= 1
+m.a ||= 1
+a[1] = 2
+a[1] += 1
+a[1] &&= 1
+a[1] ||= 1
+C = 1
+C::C = 1
+::C = 1
+def m
+  C = 1      # dynamic const assignment
+  C::C = 1   # dynamic const assignment
+  ::C = 1    # dynamic const assignment
+end
+
+# ref
+lvar = $a
+lvar = @a
+lvar = @@a
+lvar = Object
+lvar = C
+lvar = C::C
+lvar = ::C
+lvar = a[1]
+
+# unary operator
++1
+-1
+not 1
+!1
+~str
+
+# binary operator
+1 + 1
+1 - 1
+1 * 1
+1 / 1
+1 ** 1
+1 ^ 1
+1 & 1
+1 | 1
+1 && 1
+1 || 1
+
+# mlhs, mrhs
+a, b, c = list
+a, = list
+a, * = list
+a, *b = list
+a, (b, c), d, *e = list
+mlhs = 1, 2
+mlhs = 1, 2, 3, *list
+mlhs = *list
+
+# method call
+m
+m()
+m(a)
+m(a,a)
+m(*a)
+m(&b)
+m(a,*a)
+m(a,&b)
+m(a,*a,&b)
+m(1=>2)
+m(1=>2,*a)
+m(1=>2,&b)
+m(1=>2,*a,&b)
+m ()
+m (a)
+m (a,a)
+m (*a)
+m (&b)
+m (a,*a)
+m (a,&b)
+m (a,*a,&b)
+m (1=>2)
+m (1=>2,*a)
+m (1=>2,&b)
+m (1=>2,*a,&b)
+m a
+m a,a
+m *a
+m &b
+m a,*a
+m a,&b
+m a,*a,&b
+m 1=>2
+m 1=>2,*a
+m 1=>2,&b
+m 1=>2,*a,&b
+obj.m
+obj.m()
+obj.m(a)
+obj.m(a,a)
+obj.m(*a)
+obj.m(&b)
+obj.m(a,*a)
+obj.m(a,&b)
+obj.m(a,*a,&b)
+obj.m(1=>2)
+obj.m(1=>2,*a)
+obj.m(1=>2,&b)
+obj.m(1=>2,*a,&b)
+obj.m ()
+obj.m (a)
+obj.m (a,a)
+obj.m (*a)
+obj.m (&b)
+obj.m (a,*a)
+obj.m (a,&b)
+obj.m (a,*a,&b)
+obj.m (1=>2)
+obj.m (1=>2)
+obj.m (1=>2,*a)
+obj.m (1=>2,&b)
+obj.m (1=>2,*a,&b)
+obj.m a
+obj.m a,a
+obj.m *a
+obj.m &b
+obj.m a,*a
+obj.m a,&b
+obj.m a,*a,&b
+obj.m 1=>2
+obj.m 1=>2,*a
+obj.m 1=>2,&b
+obj.m 1=>2,*a,&b
+
+# ambiguous argument
+m +1
+m /r/
+
+# iterator
+[1,2,3].each do |i|
+  print i
+end
+{1=>true}.each do |k,v|
+  puts k
+end
+[1,2,3].each {|i| print i }
+[1].each {|a,| }
+[1].each {|*b| }
+[1].each {|a,*b| }
+[1].each {|&block| }
+[1].each {|a,&block| }
+[1].each {|a,*b,&block| }
+a = lambda() {|n| n * n }
+a = lambda () {|n| n * n }
+a = lambda (a) {|n| n * n }
+a = lambda (a,b) {|n| n * n }
+
+# BEGIN, END
+BEGIN { }
+END { }
+
+# if, unless
+1 if true
+2 unless false
+if false
+  5
+elsif false
+  6
+elsif false then 7
+else
+  8
+end
+if m
+end
+unless 1
+  2
+end
+unless m
+end
+0 ? 1 : 2
+
+# case
+case 'a'
+when 'b'
+  ;
+when 'c' then 1
+else
+  2
+end
+case
+when 1
+when 2
+when 3
+else
+end
+case 1
+else
+end
+case
+else
+end
+
+# while, until, for
+while true
+  break
+  next
+  redo
+end
+begin
+  break
+end while true
+until false
+  break
+  next
+  redo
+end
+begin
+  break
+end until false
+for x in m()
+  break
+  next
+  redo
+end
+0 until true
+1 while false
+
+# begin, rescue, else, ensure
+begin
+  1
+rescue StandardError => er
+  2
+rescue => er
+  3
+  retry
+else
+  4
+ensure
+  5
+end
+a = 1 rescue 2
+
+# jumps
+def m
+  redo
+  yield
+  yield nil
+  super
+  super 1
+  return
+  return nil
+end
+
+# defined
+defined? f
+defined?(f)
+
+n = 1 \
++ 1
+
+__END__
diff --git a/ext/ripper/test/validate.rb b/ext/ripper/test/validate.rb
new file mode 100644
index 0000000000..808d321043
--- /dev/null
+++ b/ext/ripper/test/validate.rb
@@ -0,0 +1,68 @@
+require 'ripper.so'
+
+class R < Ripper
+  def initialize(*args)
+    super
+    @lineno = 0
+  end
+
+  def parse
+    result = super
+    puts "#{@lineno}:result: #{rawVALUE(result)}"
+    validate_object result
+    p result
+    result
+  end
+
+  def on__nl(str)
+    @lineno += 1
+  end
+
+  def on__ignored_nl(str)
+    @lineno += 1
+  end
+
+  def on__comment(cmt)
+    @lineno += 1
+  end
+
+  def on__embdoc_beg(str)
+    @lineno += 1
+  end
+
+  def on__embdoc(str)
+    @lineno += 1
+  end
+
+  def on__embdoc_end(str)
+    @lineno += 1
+  end
+
+  def method_missing(mid, *args)
+    puts mid
+    args.each_with_index do |a,idx|
+      puts "#{@lineno}:#{mid}\##{idx+1}: #{rawVALUE(a)}"
+      validate_object a
+      p a
+    end
+    args[0]
+  end
+
+  def warn(*args)
+  end
+
+  def warning(*args)
+  end
+
+  unless respond_to?(:validate_object)
+    def validate_object(x)
+      x
+    end
+    def rawVALUE(x)
+      x.object_id
+    end
+  end
+end
+
+fname = (ARGV[0] || 'test/src_rb')
+R.new(File.read(fname), fname, 1).parse
diff --git a/ext/ripper/tools/generate-eventids1.rb b/ext/ripper/tools/generate-eventids1.rb
new file mode 100755
index 0000000000..de0e27b89c
--- /dev/null
+++ b/ext/ripper/tools/generate-eventids1.rb
@@ -0,0 +1,18 @@
+#
+# generate-eventids1.rb
+#
+
+ids = ARGF.map {|s| s.strip }
+
+ids.each do |id|
+  puts "static ID ripper_id_#{id};"
+end
+
+puts
+puts 'static void'
+puts 'ripper_init_eventids1()'
+puts '{'
+ids.each do |id|
+  puts %Q[    ripper_id_#{id} = rb_intern("on__#{id}");]
+end
+puts '}'
diff --git a/ext/ripper/tools/generate-param-macros.rb b/ext/ripper/tools/generate-param-macros.rb
new file mode 100755
index 0000000000..b19f6e8d5c
--- /dev/null
+++ b/ext/ripper/tools/generate-param-macros.rb
@@ -0,0 +1,14 @@
+off = true
+ARGF.each do |line|
+  case line
+  when /RIPPER_PARAMS_DECL_BEGIN/
+    off = false
+  when /RIPPER_PARAMS_DECL_END/
+    exit
+  when /ripper/
+    next if off
+    var = line.scan(/\w+/).last or next
+    base = var.sub(/ripper_/, '')
+    puts %"\#define #{base}\t\t(parser->ripper_#{base})"
+  end
+end
diff --git a/ext/ripper/tools/generate-ripper_rb.rb b/ext/ripper/tools/generate-ripper_rb.rb
new file mode 100755
index 0000000000..c6b2aea13f
--- /dev/null
+++ b/ext/ripper/tools/generate-ripper_rb.rb
@@ -0,0 +1,44 @@
+#
+# generate-ripper_rb.rb
+# Creates ripper.rb, filling in default event handlers, given a basic 
+# template, the list of parser events (ids1), and a list of lexer 
+# events (ids2).
+#
+
+def main
+  template, ids1, ids2 = *ARGV
+  File.foreach(template) do |line|
+    case line
+    when /\A\#include handlers1/
+      File.foreach(ids1) do |line|
+        id, arity = line.split
+        arity = arity.to_i
+        puts
+        puts "  def on__#{id}(#{argdecl(arity)})"
+        puts "    #{arity == 0 ? 'nil' : 'a'}"
+        puts "  end"
+      end
+    when /\A\#include handlers2/
+      File.foreach(ids2) do |line|
+        id, arity = line.split
+        arity = arity.to_i
+        puts
+        puts "  def on__#{id}(token)"
+        puts "    token"
+        puts "  end"
+      end
+    when /\A\#include (.*)/
+      raise "unknown operation: #include #{$1}"
+    else
+      print line
+    end
+  end
+end
+
+# Generate generic arg list depending on arity (n)
+# n:: [Integer] arity of method
+def argdecl( n )
+  %w(a b c d e f g h i j k l m)[0, n].join(', ')
+end
+
+main
diff --git a/ext/ripper/tools/list-parse-event-ids.rb b/ext/ripper/tools/list-parse-event-ids.rb
new file mode 100755
index 0000000000..2936f9b092
--- /dev/null
+++ b/ext/ripper/tools/list-parse-event-ids.rb
@@ -0,0 +1,38 @@
+#
+# list-parse-event-ids.rb
+#
+
+require 'getopts'
+
+def usage( status )
+  (status == 0 ? $stdout : $stderr).print(<<EOS)
+Usage: #{File.basename($0)} [-a] filename
+EOS
+  exit status
+end
+
+def main
+  getopts('a') or usage(1)
+  extract_ids(ARGF).each do |id, arity|
+    if $OPT_a
+    then puts "#{id} #{arity}"
+    else puts id
+    end
+  end
+end
+
+def extract_ids( f )
+  results = []
+  f.each do |line|
+    next if /\A\#\s*define\s+s?dispatch/ === line
+    next if /ripper_dispatch/ === line
+    if a = line.scan(/dispatch(\d)\((\w+)/)
+      a.each do |arity, event|
+        results.push [event, arity.to_i]
+      end
+    end
+  end
+  results.uniq.sort
+end
+
+main
diff --git a/ext/ripper/tools/list-scan-event-ids.rb b/ext/ripper/tools/list-scan-event-ids.rb
new file mode 100755
index 0000000000..6f25362b5d
--- /dev/null
+++ b/ext/ripper/tools/list-scan-event-ids.rb
@@ -0,0 +1,32 @@
+#
+# list-scan-event-ids.rb
+#
+
+require 'getopts'
+
+def usage(status)
+  (status == 0 ? $stdout : $stderr).puts(<<EOS)
+Usage: #{File.basename($0)} eventids2.c
+    -a    print IDs with arity.
+EOS
+  exit status
+end
+
+def main
+  ok = getopts('a', 'help')
+  usage 0 if $OPT_help
+  usage 1 unless ok
+  extract_ids(ARGF).sort.each do |id|
+    if $OPT_a
+      puts "#{id} 1"
+    else
+      puts id
+    end
+  end
+end
+
+def extract_ids(f)
+  (f.read.scan(/ripper_id_(\w+)/).flatten - ['scan']).uniq
+end
+
+main
diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb
new file mode 100755
index 0000000000..a2dba36e02
--- /dev/null
+++ b/ext/ripper/tools/preproc.rb
@@ -0,0 +1,57 @@
+def main
+  prelude
+  grammar
+  usercode
+end
+
+def prelude
+  while line = ARGF.gets
+    case line
+    when %r</\*%%%\*/>
+      puts '/*'
+    when %r</\*%>
+      puts '*/'
+    when %r<%\*/>
+      puts
+    when /\A%%/
+      puts '%%'
+      return
+    when /\A%token/
+      puts line.sub(/<\w+>/, '<val>')
+    when /\A%type/
+      puts line.sub(/<\w+>/, '<val>')
+    else
+      print line
+    end
+  end
+end
+
+def grammar
+  while line = ARGF.gets
+    case line
+    when %r</\*%%%\*/>
+      puts '#if 0'
+    when %r</\*%c%\*/>
+      puts '/*'
+    when %r</\*%c>
+      puts '*/'
+    when %r</\*%>
+      puts '#endif'
+    when %r<%\*/>
+      puts
+    when /\A%%/
+      puts '%%'
+      return
+    else
+      print line
+    end
+  end
+end
+
+def usercode
+  while line = ARGF.gets
+    print line
+  end
+end
+
+main
diff --git a/ext/ripper/tools/strip.rb b/ext/ripper/tools/strip.rb
new file mode 100755
index 0000000000..99413c361d
--- /dev/null
+++ b/ext/ripper/tools/strip.rb
@@ -0,0 +1,12 @@
+last_is_void = false
+ARGF.each do |line|
+  if line.strip.empty?
+    #puts() unless last_is_void
+    last_is_void = true
+  elsif /\A\#/ === line
+    ;
+  else
+    print line
+    last_is_void = false
+  end
+end
diff --git a/test/ripper/dummyparser.rb b/test/ripper/dummyparser.rb
new file mode 100644
index 0000000000..b95ebdb5d1
--- /dev/null
+++ b/test/ripper/dummyparser.rb
@@ -0,0 +1,526 @@
+#
+# dummyparser.rb
+#
+
+class Node
+
+  def initialize( name, *nodes )
+    @name = name
+    @children = nodes
+  end
+
+  attr_reader :children
+
+  def to_s
+    "#{@name}(#{@children.map {|n| n.to_s }.join(',')})"
+  end
+
+end
+
+
+class NodeList
+
+  def initialize
+    @list = []
+  end
+
+  attr_reader :list
+
+  def push( item )
+    @list.push item
+    self
+  end
+
+  def prepend( items )
+    @list[0,0] = items
+  end
+
+  def to_s
+    '[' + @list.join(',') + ']'
+  end
+
+end
+
+
+class DummyParser < Ripper
+
+  def method_missing( mid, *args )
+    raise NoMethodError, "wrong method: #{mid}" unless /\Aon__/ === mid.to_s
+    args[0]
+  end
+
+  def on__program( stmts )
+    $thru_program = true
+    stmts
+  end
+
+  def on__stmts_new
+    NodeList.new
+  end
+
+  def on__stmts_add( stmts, st )
+    stmts.push st
+    stmts
+  end
+
+  def on__void_stmt
+    Node.new('void')
+  end
+
+  def on__BEGIN( stmts )
+    Node.new('BEGIN', stmts)
+  end
+
+  def on__END( stmts )
+    Node.new('END', stmts)
+  end
+
+  def on__var_ref( name )
+    Node.new('ref', name)
+  end
+
+  def on__alias(a, b)
+    Node.new('alias', a, b)
+  end
+
+  def on__var_alias(a, b)
+    Node.new('valias', a, b)
+  end
+
+  def on__alias_error(a)
+    Node.new('aliaserr', a)
+  end
+
+  def on__aref(a, b)
+    Node.new('aref', a, b)
+  end
+
+  def on__aref_field(a, b)
+    Node.new('aref_field', a, b)
+  end
+
+  def on__arg_ambiguous
+    Node.new('arg_ambiguous')
+  end
+
+  def on__arg_paren( args )
+    args
+  end
+
+  def on__arglist_new
+    NodeList.new
+  end
+
+  def on__arglist_add( list, arg )
+    list.push(arg)
+  end
+
+  def on__arglist_add_block( list, blk )
+    list.push('&' + blk.to_s)
+  end
+
+  def on__arglist_add_star( list, arg )
+    list.push('*' + arg.to_s)
+  end
+
+  def on__arglist_prepend( list, args )
+    list.prepend args
+    list
+  end
+
+  def on__method_add_arg( m, arg )
+    m.children.push arg
+    m
+  end
+
+  def on__assoc_new(a, b)
+    Node.new('assoc', a, b)
+  end
+
+  def on__bare_assoc_hash( assoc_list )
+    Node.new('assocs', *assoc_list)
+  end
+
+  def on__assoclist_from_args(a)
+    Node.new('assocs', *a.list)
+  end
+
+  ######## untested
+
+  def on__array(a)
+    Node.new('array', a)
+  end
+
+  def on__assign(a, b)
+    Node.new('assign', a, b)
+  end
+
+  def on__assign_error(a)
+    Node.new('assign_error', a)
+  end
+
+  def on__begin(a)
+    Node.new('begin', a)
+  end
+
+  def on__binary(a, b, c)
+    Node.new('binary', a, b, c)
+  end
+
+  def on__block_var(a)
+    Node.new('block_var', a)
+  end
+
+  def on__bodystmt(a, b, c, d)
+    Node.new('bodystmt', a, b, c, d)
+  end
+
+  def on__brace_block(a, b)
+    Node.new('brace_block', a, b)
+  end
+
+  def on__break(a)
+    Node.new('break', a)
+  end
+
+  def on__call(a, b, c)
+    Node.new('call', a, b, c)
+  end
+
+  def on__case(a, b)
+    Node.new('case', a, b)
+  end
+
+  def on__class(a, b, c)
+    Node.new('class', a, b, c)
+  end
+
+  def on__class_name_error(a)
+    Node.new('class_name_error', a)
+  end
+
+  def on__command(a, b)
+    Node.new('command', a, b)
+  end
+
+  def on__command_call(a, b, c, d)
+    Node.new('command_call', a, b, c, d)
+  end
+
+  def on__const_ref(a)
+    Node.new('const_ref', a)
+  end
+
+  def on__constpath_field(a, b)
+    Node.new('constpath_field', a, b)
+  end
+
+  def on__constpath_ref(a, b)
+    Node.new('constpath_ref', a, b)
+  end
+
+  def on__def(a, b, c)
+    Node.new('def', a, b, c)
+  end
+
+  def on__defined(a)
+    Node.new('defined', a)
+  end
+
+  def on__defs(a, b, c, d, e)
+    Node.new('defs', a, b, c, d, e)
+  end
+
+  def on__do_block(a, b)
+    Node.new('do_block', a, b)
+  end
+
+  def on__dot2(a, b)
+    Node.new('dot2', a, b)
+  end
+
+  def on__dot3(a, b)
+    Node.new('dot3', a, b)
+  end
+
+  def on__dyna_symbol(a)
+    Node.new('dyna_symbol', a)
+  end
+
+  def on__else(a)
+    Node.new('else', a)
+  end
+
+  def on__elsif(a, b, c)
+    Node.new('elsif', a, b, c)
+  end
+
+  def on__ensure(a)
+    Node.new('ensure', a)
+  end
+
+  def on__fcall(a)
+    Node.new('fcall', a)
+  end
+
+  def on__field(a, b, c)
+    Node.new('field', a, b, c)
+  end
+
+  def on__for(a, b, c)
+    Node.new('for', a, b, c)
+  end
+
+  def on__hash(a)
+    Node.new('hash', a)
+  end
+
+  def on__if(a, b, c)
+    Node.new('if', a, b, c)
+  end
+
+  def on__if_mod(a, b)
+    Node.new('if_mod', a, b)
+  end
+
+  def on__ifop(a, b, c)
+    Node.new('ifop', a, b, c)
+  end
+
+  def on__iter_block(a, b)
+    Node.new('iter_block', a, b)
+  end
+
+  def on__massign(a, b)
+    Node.new('massign', a, b)
+  end
+
+  def on__mlhs_add(a, b)
+    Node.new('mlhs_add', a, b)
+  end
+
+  def on__mlhs_add_star(a, b)
+    Node.new('mlhs_add_star', a, b)
+  end
+
+  def on__mlhs_new
+    Node.new('mlhs_new')
+  end
+
+  def on__mlhs_paren(a)
+    Node.new('mlhs_paren', a)
+  end
+
+  def on__module(a, b)
+    Node.new('module', a, b)
+  end
+
+  def on__mrhs_add(a, b)
+    Node.new('mrhs_add', a, b)
+  end
+
+  def on__mrhs_add_star(a, b)
+    Node.new('mrhs_add_star', a, b)
+  end
+
+  def on__mrhs_new
+    Node.new('mrhs_new')
+  end
+
+  def on__mrhs_new_from_arglist(a)
+    Node.new('mrhs_new_from_arglist', a)
+  end
+
+  def on__next(a)
+    Node.new('next', a)
+  end
+
+  def on__opassign(a, b, c)
+    Node.new('opassign', a, b, c)
+  end
+
+  def on__param_error(a)
+    Node.new('param_error', a)
+  end
+
+  def on__params(a, b, c, d)
+    Node.new('params', a, b, c, d)
+  end
+
+  def on__paren(a)
+    Node.new('paren', a)
+  end
+
+  def on__parse_error(a)
+    Node.new('parse_error', a)
+  end
+
+  def on__qwords_add(a, b)
+    Node.new('qwords_add', a, b)
+  end
+
+  def on__qwords_new
+    Node.new('qwords_new')
+  end
+
+  def on__redo
+    Node.new('redo')
+  end
+
+  def on__regexp_literal(a)
+    Node.new('regexp_literal', a)
+  end
+
+  def on__rescue(a, b, c, d)
+    Node.new('rescue', a, b, c, d)
+  end
+
+  def on__rescue_mod(a, b)
+    Node.new('rescue_mod', a, b)
+  end
+
+  def on__restparam(a)
+    Node.new('restparam', a)
+  end
+
+  def on__retry
+    Node.new('retry')
+  end
+
+  def on__return(a)
+    Node.new('return', a)
+  end
+
+  def on__return0
+    Node.new('return0')
+  end
+
+  def on__sclass(a, b)
+    Node.new('sclass', a, b)
+  end
+
+  def on__space(a)
+    Node.new('space', a)
+  end
+
+  def on__string_add(a, b)
+    Node.new('string_add', a, b)
+  end
+
+  def on__string_concat(a, b)
+    Node.new('string_concat', a, b)
+  end
+
+  def on__string_content
+    Node.new('string_content')
+  end
+
+  def on__string_dvar(a)
+    Node.new('string_dvar', a)
+  end
+
+  def on__string_embexpr(a)
+    Node.new('string_embexpr', a)
+  end
+
+  def on__string_literal(a)
+    Node.new('string_literal', a)
+  end
+
+  def on__super(a)
+    Node.new('super', a)
+  end
+
+  def on__symbol(a)
+    Node.new('symbol', a)
+  end
+
+  def on__symbol_literal(a)
+    Node.new('symbol_literal', a)
+  end
+
+  def on__topconst_field(a)
+    Node.new('topconst_field', a)
+  end
+
+  def on__topconst_ref(a)
+    Node.new('topconst_ref', a)
+  end
+
+  def on__unary(a, b)
+    Node.new('unary', a, b)
+  end
+
+  def on__undef(a)
+    Node.new('undef', a)
+  end
+
+  def on__unless(a, b, c)
+    Node.new('unless', a, b, c)
+  end
+
+  def on__unless_mod(a, b)
+    Node.new('unless_mod', a, b)
+  end
+
+  def on__until_mod(a, b)
+    Node.new('until_mod', a, b)
+  end
+
+  def on__var_field(a)
+    Node.new('var_field', a)
+  end
+
+  def on__when(a, b, c)
+    Node.new('when', a, b, c)
+  end
+
+  def on__while(a, b)
+    Node.new('while', a, b)
+  end
+
+  def on__while_mod(a, b)
+    Node.new('while_mod', a, b)
+  end
+
+  def on__word_add(a, b)
+    Node.new('word_add', a, b)
+  end
+
+  def on__word_new
+    Node.new('word_new')
+  end
+
+  def on__words_add(a, b)
+    Node.new('words_add', a, b)
+  end
+
+  def on__words_new
+    Node.new('words_new')
+  end
+
+  def on__xstring_add(a, b)
+    Node.new('xstring_add', a, b)
+  end
+
+  def on__xstring_literal(a)
+    Node.new('xstring_literal', a)
+  end
+
+  def on__xstring_new
+    Node.new('xstring_new')
+  end
+
+  def on__yield(a)
+    Node.new('yield', a)
+  end
+
+  def on__yield0
+    Node.new('yield0')
+  end
+
+  def on__zsuper
+    Node.new('zsuper')
+  end
+
+end
diff --git a/test/ripper/test_parser_events.rb b/test/ripper/test_parser_events.rb
new file mode 100644
index 0000000000..512cf377e7
--- /dev/null
+++ b/test/ripper/test_parser_events.rb
@@ -0,0 +1,489 @@
+#
+# test_parser_events.rb
+#
+
+require 'ripper.so'
+require 'dummyparser'
+raise "ripper version differ" unless Ripper::Version == '0.1.0'
+require 'test/unit'
+
+
+class TestRipper_ParserEvents < Test::Unit::TestCase
+
+  def parse(str)
+    DummyParser.new(str).parse.to_s
+  end
+
+  $thru_program = false
+
+  def test_program
+    assert_equal '[void()]', parse('')
+    assert_equal true, $thru_program
+  end
+
+  def test_stmts_new
+    assert_equal '[void()]', parse('')
+  end
+
+  def test_stmts_add
+    assert_equal '[ref(nil)]', parse('nil')
+    assert_equal '[ref(nil),ref(nil)]', parse('nil;nil')
+    assert_equal '[ref(nil),ref(nil),ref(nil)]', parse('nil;nil;nil')
+  end
+
+  def test_void_stmt
+    assert_equal '[void()]', parse('')
+    assert_equal '[void()]', parse(';;')
+  end
+
+  def test_var_ref
+    assert_equal '[ref(a)]', parse('a')
+    assert_equal '[ref(nil)]', parse('nil')
+    assert_equal '[ref(true)]', parse('true')
+  end
+
+  def test_BEGIN
+    assert_equal '[BEGIN([void()])]', parse('BEGIN{}')
+    assert_equal '[BEGIN([ref(nil)])]', parse('BEGIN{nil}')
+  end
+
+  def test_END
+    assert_equal '[END([void()])]', parse('END{}')
+    assert_equal '[END([ref(nil)])]', parse('END{nil}')
+  end
+
+  def test_alias
+    assert_equal '[alias(a,b)]', parse('alias a b')
+  end
+
+  def test_var_alias
+    assert_equal '[valias($a,$g)]', parse('alias $a $g')
+  end
+
+  def test_alias_error
+    assert_equal '[aliaserr(valias($a,$1))]', parse('alias $a $1')
+  end
+
+  def test_arglist
+    assert_equal '[fcall(m,[])]', parse('m()')
+    assert_equal '[fcall(m,[1])]', parse('m(1)')
+    assert_equal '[fcall(m,[1,2])]', parse('m(1,2)')
+    assert_equal '[fcall(m,[*ref(r)])]', parse('m(*r)')
+    assert_equal '[fcall(m,[1,*ref(r)])]', parse('m(1,*r)')
+    assert_equal '[fcall(m,[1,2,*ref(r)])]', parse('m(1,2,*r)')
+    assert_equal '[fcall(m,[&ref(r)])]', parse('m(&r)')
+    assert_equal '[fcall(m,[1,&ref(r)])]', parse('m(1,&r)')
+    assert_equal '[fcall(m,[1,2,&ref(r)])]', parse('m(1,2,&r)')
+    assert_equal '[fcall(m,[*ref(a),&ref(b)])]', parse('m(*a,&b)')
+    assert_equal '[fcall(m,[1,*ref(a),&ref(b)])]', parse('m(1,*a,&b)')
+    assert_equal '[fcall(m,[1,2,*ref(a),&ref(b)])]', parse('m(1,2,*a,&b)')
+  end
+
+  def test_arg_paren
+    # FIXME
+  end
+
+  def test_aref
+    assert_equal '[aref(ref(v),[1])]', parse('v[1]')
+    assert_equal '[aref(ref(v),[1,2])]', parse('v[1,2]')
+  end
+
+  def test_assocs
+    assert_equal '[fcall(m,[assocs(assoc(1,2))])]', parse('m(1=>2)')
+    assert_equal '[fcall(m,[assocs(assoc(1,2),assoc(3,4))])]', parse('m(1=>2,3=>4)')
+    assert_equal '[fcall(m,[3,assocs(assoc(1,2))])]', parse('m(3,1=>2)')
+  end
+
+  def test_aref_field
+    assert_equal '[assign(aref_field(ref(a),[1]),2)]', parse('a[1]=2')
+  end
+
+=begin
+  def test_arg_ambiguous
+    assert_equal true, $thru__arg_ambiguous
+  end
+=end
+
+  def test_array   # array literal
+    assert_equal '[array([1,2,3])]', parse('[1,2,3]')
+  end
+
+  def test_assign   # generic assignment
+    assert_equal '[assign(var_field(v),1)]', parse('v=1')
+  end
+
+=begin
+  def test_assign_error
+    assert_equal true, $thru__assign_error
+  end
+
+  def test_begin
+    assert_equal true, $thru__begin
+  end
+
+  def test_binary
+    assert_equal true, $thru__binary
+  end
+
+  def test_block_var
+    assert_equal true, $thru__block_var
+  end
+
+  def test_bodystmt
+    assert_equal true, $thru__bodystmt
+  end
+
+  def test_brace_block
+    assert_equal true, $thru__brace_block
+  end
+
+  def test_break
+    assert_equal true, $thru__break
+  end
+
+  def test_call
+    assert_equal true, $thru__call
+  end
+
+  def test_case
+    assert_equal true, $thru__case
+  end
+
+  def test_class
+    assert_equal true, $thru__class
+  end
+
+  def test_class_name_error
+    assert_equal true, $thru__class_name_error
+  end
+
+  def test_command
+    assert_equal true, $thru__command
+  end
+
+  def test_command_call
+    assert_equal true, $thru__command_call
+  end
+
+  def test_const_ref
+    assert_equal true, $thru__const_ref
+  end
+
+  def test_constpath_field
+    assert_equal true, $thru__constpath_field
+  end
+
+  def test_constpath_ref
+    assert_equal true, $thru__constpath_ref
+  end
+
+  def test_def
+    assert_equal true, $thru__def
+  end
+
+  def test_defined
+    assert_equal true, $thru__defined
+  end
+
+  def test_defs
+    assert_equal true, $thru__defs
+  end
+
+  def test_do_block
+    assert_equal true, $thru__do_block
+  end
+
+  def test_dot2
+    assert_equal true, $thru__dot2
+  end
+
+  def test_dot3
+    assert_equal true, $thru__dot3
+  end
+
+  def test_dyna_symbol
+    assert_equal true, $thru__dyna_symbol
+  end
+
+  def test_else
+    assert_equal true, $thru__else
+  end
+
+  def test_elsif
+    assert_equal true, $thru__elsif
+  end
+
+  def test_ensure
+    assert_equal true, $thru__ensure
+  end
+
+  def test_fcall
+    assert_equal true, $thru__fcall
+  end
+
+  def test_field
+    assert_equal true, $thru__field
+  end
+
+  def test_for
+    assert_equal true, $thru__for
+  end
+
+  def test_hash
+    assert_equal true, $thru__hash
+  end
+
+  def test_if
+    assert_equal true, $thru__if
+  end
+
+  def test_if_mod
+    assert_equal true, $thru__if_mod
+  end
+
+  def test_ifop
+    assert_equal true, $thru__ifop
+  end
+
+  def test_iter_block
+    assert_equal true, $thru__iter_block
+  end
+
+  def test_massign
+    assert_equal true, $thru__massign
+  end
+
+  def test_method_add_arg
+    assert_equal true, $thru__method_add_arg
+  end
+
+  def test_mlhs_add
+    assert_equal true, $thru__mlhs_add
+  end
+
+  def test_mlhs_add_star
+    assert_equal true, $thru__mlhs_add_star
+  end
+
+  def test_mlhs_new
+    assert_equal true, $thru__mlhs_new
+  end
+
+  def test_mlhs_paren
+    assert_equal true, $thru__mlhs_paren
+  end
+
+  def test_module
+    assert_equal true, $thru__module
+  end
+
+  def test_mrhs_add
+    assert_equal true, $thru__mrhs_add
+  end
+
+  def test_mrhs_add_star
+    assert_equal true, $thru__mrhs_add_star
+  end
+
+  def test_mrhs_new
+    assert_equal true, $thru__mrhs_new
+  end
+
+  def test_mrhs_new_from_arglist
+    assert_equal true, $thru__mrhs_new_from_arglist
+  end
+
+  def test_next
+    assert_equal true, $thru__next
+  end
+
+  def test_opassign
+    assert_equal true, $thru__opassign
+  end
+
+  def test_param_error
+    assert_equal true, $thru__param_error
+  end
+
+  def test_params
+    assert_equal true, $thru__params
+  end
+
+  def test_paren
+    assert_equal true, $thru__paren
+  end
+
+  def test_parse_error
+    assert_equal true, $thru__parse_error
+  end
+
+  def test_qwords_add
+    assert_equal true, $thru__qwords_add
+  end
+
+  def test_qwords_new
+    assert_equal true, $thru__qwords_new
+  end
+
+  def test_redo
+    assert_equal true, $thru__redo
+  end
+
+  def test_regexp_literal
+    assert_equal true, $thru__regexp_literal
+  end
+
+  def test_rescue
+    assert_equal true, $thru__rescue
+  end
+
+  def test_rescue_mod
+    assert_equal true, $thru__rescue_mod
+  end
+
+  def test_restparam
+    assert_equal true, $thru__restparam
+  end
+
+  def test_retry
+    assert_equal true, $thru__retry
+  end
+
+  def test_return
+    assert_equal true, $thru__return
+  end
+
+  def test_return0
+    assert_equal true, $thru__return0
+  end
+
+  def test_sclass
+    assert_equal true, $thru__sclass
+  end
+
+  def test_space
+    assert_equal true, $thru__space
+  end
+
+  def test_string_add
+    assert_equal true, $thru__string_add
+  end
+
+  def test_string_concat
+    assert_equal true, $thru__string_concat
+  end
+
+  def test_string_content
+    assert_equal true, $thru__string_content
+  end
+
+  def test_string_dvar
+    assert_equal true, $thru__string_dvar
+  end
+
+  def test_string_embexpr
+    assert_equal true, $thru__string_embexpr
+  end
+
+  def test_string_literal
+    assert_equal true, $thru__string_literal
+  end
+
+  def test_super
+    assert_equal true, $thru__super
+  end
+
+  def test_symbol
+    assert_equal true, $thru__symbol
+  end
+
+  def test_symbol_literal
+    assert_equal true, $thru__symbol_literal
+  end
+
+  def test_topconst_field
+    assert_equal true, $thru__topconst_field
+  end
+
+  def test_topconst_ref
+    assert_equal true, $thru__topconst_ref
+  end
+
+  def test_unary
+    assert_equal true, $thru__unary
+  end
+
+  def test_undef
+    assert_equal true, $thru__undef
+  end
+
+  def test_unless
+    assert_equal true, $thru__unless
+  end
+
+  def test_unless_mod
+    assert_equal true, $thru__unless_mod
+  end
+
+  def test_until_mod
+    assert_equal true, $thru__until_mod
+  end
+
+  def test_var_field
+    assert_equal true, $thru__var_field
+  end
+
+  def test_when
+    assert_equal true, $thru__when
+  end
+
+  def test_while
+    assert_equal true, $thru__while
+  end
+
+  def test_while_mod
+    assert_equal true, $thru__while_mod
+  end
+
+  def test_word_add
+    assert_equal true, $thru__word_add
+  end
+
+  def test_word_new
+    assert_equal true, $thru__word_new
+  end
+
+  def test_words_add
+    assert_equal true, $thru__words_add
+  end
+
+  def test_words_new
+    assert_equal true, $thru__words_new
+  end
+
+  def test_xstring_add
+    assert_equal true, $thru__xstring_add
+  end
+
+  def test_xstring_literal
+    assert_equal true, $thru__xstring_literal
+  end
+
+  def test_xstring_new
+    assert_equal true, $thru__xstring_new
+  end
+
+  def test_yield
+    assert_equal true, $thru__yield
+  end
+
+  def test_yield0
+    assert_equal true, $thru__yield0
+  end
+
+  def test_zsuper
+    assert_equal true, $thru__zsuper
+  end
+=end
+
+end
diff --git a/test/ripper/test_scanner_events.rb b/test/ripper/test_scanner_events.rb
new file mode 100644
index 0000000000..28b65eb1b6
--- /dev/null
+++ b/test/ripper/test_scanner_events.rb
@@ -0,0 +1,744 @@
+#
+# test_scanner_events.rb
+#
+
+require 'ripper.so'
+raise 'ripper version differ' unless Ripper::Version == '0.1.0'
+require 'test/unit'
+
+class TestRipper_ScannerEvents < Test::Unit::TestCase
+
+  class R < Ripper
+    def R.scan(target, src)
+      r = new(src, target)
+      r.parse
+      r.tokens.map {|id, tok| tok }
+    end
+
+    def R.lex(src)
+      r = new(src, 'scan')
+      r.parse
+      r.tokens
+    end
+
+    def initialize(src, target)
+      super src
+      @target = ('on__' + target).intern
+      @tokens = []
+    end
+
+    attr_reader :tokens
+
+    def method_missing(mid, *args)
+      case mid.to_s
+      when /\Aon__scan/
+        if @target == :on__scan
+          @tokens.push args
+        else
+          @tokens.push args if @target == args[0]
+        end
+      when /\Aon__/
+        ;
+      else
+        raise NoMethodError, "no such method: #{mid}"
+      end
+    end
+
+    def warn(fmt, *args)
+      #p [fmt, args]
+    end
+
+    def warning(fmt, *args)
+      #p [fmt, args]
+    end
+
+    def compile_error(msg)
+    end
+  end
+
+  def test_scan
+    assert_equal [],
+                 R.scan('scan', '')
+    assert_equal ['a'],
+                 R.scan('scan', 'a')
+    assert_equal ['1'],
+                 R.scan('scan', '1')
+    assert_equal ['1', ';', 'def', ' ', 'm', '(', 'arg', ')', 'end'],
+                 R.scan('scan', "1;def m(arg)end")
+  end
+
+  def test_backref
+    assert_equal ["$`", "$&", "$'", '$1', '$2', '$3'],
+                 R.scan('backref', %q[m($~, $`, $&, $', $1, $2, $3)])
+  end
+
+  def test_backtick
+    assert_equal ["`"],
+                 R.scan('backtick', %q[p `make all`])
+  end
+
+  def test_comma
+    assert_equal [','] * 6,
+                 R.scan('comma', %q[ m(0,1,2,3,4,5,6) ])
+    assert_equal [],
+                 R.scan('comma', %q[".,.,.,.,.,.,.."])
+    assert_equal [],
+                 R.scan('comma', %Q[<<EOS\n,,,,,,,,,,\nEOS])
+  end
+
+  def test_period
+    assert_equal [],
+                 R.scan('period', '')
+    assert_equal ['.'],
+                 R.scan('period', 'a.b')
+    assert_equal ['.'],
+                 R.scan('period', 'Object.new')
+    assert_equal [],
+                 R.scan('period', '"."')
+    assert_equal [],
+                 R.scan('period', '1..2')
+    assert_equal [],
+                 R.scan('period', '1...3')
+  end
+
+  def test_const
+    assert_equal ['CONST'],
+                 R.scan('const', 'CONST')
+    assert_equal ['C'],
+                 R.scan('const', 'C')
+    assert_equal ['CONST_A'],
+                 R.scan('const', 'CONST_A')
+    assert_equal ['Const', 'Const2', 'Const3'],
+                 R.scan('const', 'Const; Const2; Const3')
+    assert_equal ['Const'],
+                 R.scan('const', 'Const(a)')
+    assert_equal ['M', 'A', 'A2'],
+                 R.scan('const', 'M(A,A2)')
+    assert_equal [],
+                 R.scan('const', '')
+    assert_equal [],
+                 R.scan('const', 'm(lvar, @ivar, @@cvar, $gvar)')
+  end
+
+  def test_cvar
+    assert_equal [],
+                 R.scan('cvar', '')
+    assert_equal ['@@cvar'],
+                 R.scan('cvar', '@@cvar')
+    assert_equal ['@@__cvar__'],
+                 R.scan('cvar', '@@__cvar__')
+    assert_equal ['@@CVAR'],
+                 R.scan('cvar', '@@CVAR')
+    assert_equal ['@@cvar'],
+                 R.scan('cvar', '   @@cvar#comment')
+    assert_equal ['@@cvar'],
+                 R.scan('cvar', ':@@cvar')
+    assert_equal ['@@cvar'],
+                 R.scan('cvar', 'm(lvar, @ivar, @@cvar, $gvar)')
+    assert_equal [],
+                 R.scan('cvar', '"@@cvar"')
+  end
+
+  def test_embexpr_beg
+    assert_equal [],
+                 R.scan('embexpr_beg', '')
+    assert_equal ['#{'],
+                 R.scan('embexpr_beg', '"#{expr}"')
+    assert_equal [],
+                 R.scan('embexpr_beg', '%q[#{expr}]')
+    assert_equal ['#{'],
+                 R.scan('embexpr_beg', '%Q[#{expr}]')
+    assert_equal ['#{'],
+                 R.scan('embexpr_beg', "m(<<EOS)\n\#{expr}\nEOS")
+  end
+
+=begin
+  def test_embexpr_end
+    assert_equal [],
+                 R.scan('embexpr_end', '')
+    assert_equal ['}'],
+                 R.scan('embexpr_end', '"#{expr}"')
+    assert_equal [],
+                 R.scan('embexpr_end', '%q[#{expr}]')
+    assert_equal ['}'],
+                 R.scan('embexpr_end', '%Q[#{expr}]')
+    assert_equal ['}'],
+                 R.scan('embexpr_end', "m(<<EOS)\n\#{expr}\nEOS")
+  end
+=end
+
+  def test_embvar
+    assert_equal [],
+                 R.scan('embvar', '')
+    assert_equal ['#'],
+                 R.scan('embvar', '"#$gvar"')
+    assert_equal ['#'],
+                 R.scan('embvar', '"#@ivar"')
+    assert_equal ['#'],
+                 R.scan('embvar', '"#@@cvar"')
+    assert_equal [],
+                 R.scan('embvar', '"#lvar"')
+    assert_equal [],
+                 R.scan('embvar', '"#"')
+    assert_equal [],
+                 R.scan('embvar', '"\#$gvar"')
+    assert_equal [],
+                 R.scan('embvar', '"\#@ivar"')
+    assert_equal [],
+                 R.scan('embvar', '%q[#@ivar]')
+    assert_equal ['#'],
+                 R.scan('embvar', '%Q[#@ivar]')
+  end
+
+  def test_float
+    assert_equal [],
+                 R.scan('float', '')
+    assert_equal ['1.000'],
+                 R.scan('float', '1.000')
+    assert_equal ['123.456'],
+                 R.scan('float', '123.456')
+    assert_equal ['1.2345678901234567890123456789'],
+                 R.scan('float', '1.2345678901234567890123456789')
+    assert_equal ['1.000'],
+                 R.scan('float', '   1.000# comment')
+    assert_equal ['1.234e5'],
+                 R.scan('float', '1.234e5')
+    assert_equal ['1.234e1234567890'],
+                 R.scan('float', '1.234e1234567890')
+    assert_equal ['1.0'],
+                 R.scan('float', 'm(a,b,1.0,c,d)')
+  end
+
+  def test_gvar
+    assert_equal [],
+                 R.scan('gvar', '')
+    assert_equal ['$a'],
+                 R.scan('gvar', '$a')
+    assert_equal ['$A'],
+                 R.scan('gvar', '$A')
+    assert_equal ['$gvar'],
+                 R.scan('gvar', 'm(lvar, @ivar, @@cvar, $gvar)')
+    assert_equal %w($_ $~ $* $$ $? $! $@ $/ $\\ $; $, $. $= $: $< $> $"),
+                 R.scan('gvar', 'm($_, $~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $")')
+  end
+
+  def test_ident
+    assert_equal [],
+                 R.scan('ident', '')
+    assert_equal ['lvar'],
+                 R.scan('ident', 'lvar')
+    assert_equal ['m', 'lvar'],
+                 R.scan('ident', 'm(lvar, @ivar, @@cvar, $gvar)')
+  end
+
+  def test_int
+    assert_equal [],
+                 R.scan('int', '')
+    assert_equal ['1', '10', '100000000000000'],
+                 R.scan('int', 'm(1,10,100000000000000)')
+  end
+
+  def test_ivar
+    assert_equal [],
+                 R.scan('ivar', '')
+    assert_equal ['@ivar'],
+                 R.scan('ivar', '@ivar')
+    assert_equal ['@__ivar__'],
+                 R.scan('ivar', '@__ivar__')
+    assert_equal ['@IVAR'],
+                 R.scan('ivar', '@IVAR')
+    assert_equal ['@ivar'],
+                 R.scan('ivar', 'm(lvar, @ivar, @@cvar, $gvar)')
+  end
+
+  def test_kw
+    assert_equal [],
+                 R.scan('kw', '')
+    assert_equal %w(not),
+                 R.scan('kw', 'not 1')
+    assert_equal %w(and),
+                 R.scan('kw', '1 and 2')
+    assert_equal %w(or),
+                 R.scan('kw', '1 or 2')
+    assert_equal %w(if then else end),
+                 R.scan('kw', 'if 1 then 2 else 3 end')
+    assert_equal %w(if then elsif else end),
+                 R.scan('kw', 'if 1 then 2 elsif 3 else 4 end')
+    assert_equal %w(unless then end),
+                 R.scan('kw', 'unless 1 then end')
+    assert_equal %w(if true),
+                 R.scan('kw', '1 if true')
+    assert_equal %w(unless false),
+                 R.scan('kw', '2 unless false')
+    assert_equal %w(case when when else end),
+                 R.scan('kw', 'case n; when 1; when 2; else 3 end')
+    assert_equal %w(while do nil end),
+                 R.scan('kw', 'while 1 do nil end')
+    assert_equal %w(until do nil end),
+                 R.scan('kw', 'until 1 do nil end')
+    assert_equal %w(while),
+                 R.scan('kw', '1 while 2')
+    assert_equal %w(until),
+                 R.scan('kw', '1 until 2')
+    assert_equal %w(while break next retry end),
+                 R.scan('kw', 'while 1; break; next; retry end')
+    assert_equal %w(for in next break end),
+                 R.scan('kw', 'for x in obj; next 1; break 2 end')
+    assert_equal %w(begin rescue retry end),
+                 R.scan('kw', 'begin 1; rescue; retry; end')
+    assert_equal %w(rescue),
+                 R.scan('kw', '1 rescue 2')
+    assert_equal %w(def redo return end),
+                 R.scan('kw', 'def m() redo; return end')
+    assert_equal %w(def yield yield end),
+                 R.scan('kw', 'def m() yield; yield 1 end')
+    assert_equal %w(def super super super end),
+                 R.scan('kw', 'def m() super; super(); super(1) end')
+    assert_equal %w(alias),
+                 R.scan('kw', 'alias a b')
+    assert_equal %w(undef),
+                 R.scan('kw', 'undef public')
+    assert_equal %w(class end),
+                 R.scan('kw', 'class A < Object; end')
+    assert_equal %w(module end),
+                 R.scan('kw', 'module M; end')
+    assert_equal %w(class end),
+                 R.scan('kw', 'class << obj; end')
+    assert_equal %w(BEGIN),
+                 R.scan('kw', 'BEGIN { }')
+    assert_equal %w(END),
+                 R.scan('kw', 'END { }')
+    assert_equal %w(self),
+                 R.scan('kw', 'self.class')
+    assert_equal %w(nil true false),
+                 R.scan('kw', 'p(nil, true, false)')
+    assert_equal %w(__FILE__ __LINE__),
+                 R.scan('kw', 'p __FILE__, __LINE__')
+    assert_equal %w(defined?),
+                 R.scan('kw', 'defined?(Object)')
+  end
+
+  def test_lbrace
+    assert_equal [],
+                 R.scan('lbrace', '')
+    assert_equal ['{'],
+                 R.scan('lbrace', '3.times{ }')
+    assert_equal ['{'],
+                 R.scan('lbrace', '3.times  { }')
+    assert_equal ['{'],
+                 R.scan('lbrace', '3.times{}')
+    assert_equal [],
+                 R.scan('lbrace', '"{}"')
+    assert_equal ['{'],
+                 R.scan('lbrace', '{1=>2}')
+  end
+
+  def test_rbrace
+    assert_equal [],
+                 R.scan('rbrace', '')
+    assert_equal ['}'],
+                 R.scan('rbrace', '3.times{ }')
+    assert_equal ['}'],
+                 R.scan('rbrace', '3.times  { }')
+    assert_equal ['}'],
+                 R.scan('rbrace', '3.times{}')
+    assert_equal [],
+                 R.scan('rbrace', '"{}"')
+    assert_equal ['}'],
+                 R.scan('rbrace', '{1=>2}')
+  end
+
+  def test_lbracket
+    assert_equal [],
+                 R.scan('lbracket', '')
+    assert_equal ['['],
+                 R.scan('lbracket', '[]')
+    assert_equal ['['],
+                 R.scan('lbracket', 'a[1]')
+    assert_equal [],
+                 R.scan('lbracket', 'm(%q[])')
+  end
+
+  def test_rbracket
+    assert_equal [],
+                 R.scan('rbracket', '')
+    assert_equal [']'],
+                 R.scan('rbracket', '[]')
+    assert_equal [']'],
+                 R.scan('rbracket', 'a[1]')
+    assert_equal [],
+                 R.scan('rbracket', 'm(%q[])')
+  end
+
+  def test_lparen
+    assert_equal [],
+                 R.scan('lparen', '')
+    assert_equal ['('],
+                 R.scan('lparen', '()')
+    assert_equal ['('],
+                 R.scan('lparen', 'm()')
+    assert_equal ['('],
+                 R.scan('lparen', 'm (a)')
+    assert_equal [],
+                 R.scan('lparen', '"()"')
+    assert_equal [],
+                 R.scan('lparen', '"%w()"')
+  end
+
+  def test_rparen
+    assert_equal [],
+                 R.scan('rparen', '')
+    assert_equal [')'],
+                 R.scan('rparen', '()')
+    assert_equal [')'],
+                 R.scan('rparen', 'm()')
+    assert_equal [')'],
+                 R.scan('rparen', 'm (a)')
+    assert_equal [],
+                 R.scan('rparen', '"()"')
+    assert_equal [],
+                 R.scan('rparen', '"%w()"')
+  end
+
+  def test_op
+    assert_equal [],
+                 R.scan('op', '')
+    assert_equal ['|'],
+                 R.scan('op', '1 | 1')
+    assert_equal ['^'],
+                 R.scan('op', '1 ^ 1')
+    assert_equal ['&'],
+                 R.scan('op', '1 & 1')
+    assert_equal ['<=>'],
+                 R.scan('op', '1 <=> 1')
+    assert_equal ['=='],
+                 R.scan('op', '1 == 1')
+    assert_equal ['==='],
+                 R.scan('op', '1 === 1')
+    assert_equal ['=~'],
+                 R.scan('op', '1 =~ 1')
+    assert_equal ['>'],
+                 R.scan('op', '1 > 1')
+    assert_equal ['>='],
+                 R.scan('op', '1 >= 1')
+    assert_equal ['<'],
+                 R.scan('op', '1 < 1')
+    assert_equal ['<='],
+                 R.scan('op', '1 <= 1')
+    assert_equal ['<<'],
+                 R.scan('op', '1 << 1')
+    assert_equal ['>>'],
+                 R.scan('op', '1 >> 1')
+    assert_equal ['+'],
+                 R.scan('op', '1 + 1')
+    assert_equal ['-'],
+                 R.scan('op', '1 - 1')
+    assert_equal ['*'],
+                 R.scan('op', '1 * 1')
+    assert_equal ['/'],
+                 R.scan('op', '1 / 1')
+    assert_equal ['%'],
+                 R.scan('op', '1 % 1')
+    assert_equal ['**'],
+                 R.scan('op', '1 ** 1')
+    assert_equal ['~'],
+                 R.scan('op', '~1')
+    assert_equal ['-'],
+                 R.scan('op', '-a')
+    assert_equal ['+'],
+                 R.scan('op', '+a')
+    assert_equal ['[]'],
+                 R.scan('op', ':[]')
+    assert_equal ['[]='],
+                 R.scan('op', ':[]=')
+    assert_equal [],
+                 R.scan('op', %q[`make all`])
+  end
+
+  def test_symbeg
+    assert_equal [],
+                 R.scan('symbeg', '')
+    assert_equal [':'],
+                 R.scan('symbeg', ':sym')
+    assert_equal [':'],
+                 R.scan('symbeg', '[1,2,3,:sym]')
+    assert_equal [],
+                 R.scan('symbeg', '":sym"')
+    assert_equal [],
+                 R.scan('symbeg', 'a ? b : c')
+  end
+
+  def test_tstring_beg
+    assert_equal [],
+                 R.scan('tstring_beg', '')
+    assert_equal ['"'],
+                 R.scan('tstring_beg', '"abcdef"')
+    assert_equal ['%q['],
+                 R.scan('tstring_beg', '%q[abcdef]')
+    assert_equal ['%Q['],
+                 R.scan('tstring_beg', '%Q[abcdef]')
+  end
+
+  def test_tstring_content
+    assert_equal [],
+                 R.scan('tstring_content', '')
+    assert_equal ['abcdef'],
+                 R.scan('tstring_content', '"abcdef"')
+    assert_equal ['abcdef'],
+                 R.scan('tstring_content', '%q[abcdef]')
+    assert_equal ['abcdef'],
+                 R.scan('tstring_content', '%Q[abcdef]')
+    assert_equal ['abc', 'def'],
+                 R.scan('tstring_content', '"abc#{1}def"')
+    assert_equal ['sym'],
+                 R.scan('tstring_content', ':"sym"')
+  end
+
+  def test_tstring_end
+    assert_equal [],
+                 R.scan('tstring_end', '')
+    assert_equal ['"'],
+                 R.scan('tstring_end', '"abcdef"')
+    assert_equal [']'],
+                 R.scan('tstring_end', '%q[abcdef]')
+    assert_equal [']'],
+                 R.scan('tstring_end', '%Q[abcdef]')
+  end
+
+  def test_regexp_beg
+    assert_equal [],
+                 R.scan('regexp_beg', '')
+    assert_equal ['/'],
+                 R.scan('regexp_beg', '/re/')
+    assert_equal ['%r<'],
+                 R.scan('regexp_beg', '%r<re>')
+    assert_equal [],
+                 R.scan('regexp_beg', '5 / 5')
+  end
+
+  def test_regexp_end
+    assert_equal [],
+                 R.scan('regexp_end', '')
+    assert_equal ['/'],
+                 R.scan('regexp_end', '/re/')
+    assert_equal ['>'],
+                 R.scan('regexp_end', '%r<re>')
+  end
+
+  def test_words_beg
+    assert_equal [],
+                 R.scan('words_beg', '')
+    assert_equal ['%W('],
+                 R.scan('words_beg', '%W()')
+    assert_equal ['%W('],
+                 R.scan('words_beg', '%W(w w w)')
+    assert_equal ['%W( '],
+                 R.scan('words_beg', '%W( w w w )')
+  end
+
+  def test_qwords_beg
+    assert_equal [],
+                 R.scan('qwords_beg', '')
+    assert_equal ['%w('],
+                 R.scan('qwords_beg', '%w()')
+    assert_equal ['%w('],
+                 R.scan('qwords_beg', '%w(w w w)')
+    assert_equal ['%w( '],
+                 R.scan('qwords_beg', '%w( w w w )')
+  end
+
+  # FIXME: Close paren must not present (`words_end' scanner event?).
+  def test_words_sep
+    assert_equal [],
+                 R.scan('words_sep', '')
+    assert_equal [')'],
+                 R.scan('words_sep', '%w()')
+    assert_equal [' ', ' ', ')'],
+                 R.scan('words_sep', '%w(w w w)')
+    assert_equal [' ', ' ', ' )'],
+                 R.scan('words_sep', '%w( w w w )')
+    assert_equal ["\n", ' ', ' )'],
+                 R.scan('words_sep', "%w( w\nw w )")
+  end
+
+  def test_heredoc_beg
+    assert_equal [],
+                 R.scan('heredoc_beg', '')
+    assert_equal ['<<EOS'],
+                 R.scan('heredoc_beg', "<<EOS\nheredoc\nEOS")
+    assert_equal ['<<EOS'],
+                 R.scan('heredoc_beg', "<<EOS\nheredoc\nEOS\n")
+    assert_equal ['<<EOS'],
+                 R.scan('heredoc_beg', "<<EOS\nheredoc\nEOS \n")
+    assert_equal ['<<-EOS'],
+                 R.scan('heredoc_beg', "<<-EOS\nheredoc\n\tEOS \n")
+    assert_equal ['<<"EOS"'],
+                 R.scan('heredoc_beg', %Q[<<"EOS"\nheredoc\nEOS])
+    assert_equal [%q(<<'EOS')],
+                 R.scan('heredoc_beg', "<<'EOS'\nheredoc\nEOS")
+    assert_equal [%q(<<`EOS`)],
+                 R.scan('heredoc_beg', "<<`EOS`\nheredoc\nEOS")
+  end
+
+  def test_heredoc_content
+    assert_equal [],
+                 R.scan('heredoc_content', '')
+    assert_equal ["heredoc\n"],
+                 R.scan('heredoc_content', "<<EOS\nheredoc\nEOS")
+    assert_equal ["heredoc\n"],
+                 R.scan('heredoc_content', "<<EOS\nheredoc\nEOS\n")
+    assert_equal ["heredoc \n"],
+                 R.scan('heredoc_content', "<<EOS\nheredoc \nEOS \n")
+    assert_equal ["heredoc\n"],
+                 R.scan('heredoc_content', "<<-EOS\nheredoc\n\tEOS \n")
+  end
+
+  def test_heredoc_end
+    assert_equal [],
+                 R.scan('heredoc_end', '')
+    assert_equal ["EOS"],
+                 R.scan('heredoc_end', "<<EOS\nheredoc\nEOS")
+    assert_equal ["EOS\n"],
+                 R.scan('heredoc_end', "<<EOS\nheredoc\nEOS\n")
+    assert_equal ["EOS \n"],
+                 R.scan('heredoc_end', "<<EOS\nheredoc\nEOS \n")
+    assert_equal ["\tEOS \n"],
+                 R.scan('heredoc_end', "<<-EOS\nheredoc\n\tEOS \n")
+  end
+
+  def test_semicolon
+    assert_equal [],
+                 R.scan('semicolon', '')
+    assert_equal %w(;),
+                 R.scan('semicolon', ';')
+    assert_equal %w(; ;),
+                 R.scan('semicolon', ';;')
+    assert_equal %w(; ; ;),
+                 R.scan('semicolon', 'nil;nil;nil;')
+    assert_equal %w(; ; ;),
+                 R.scan('semicolon', 'nil;nil;nil;nil')
+    assert_equal [],
+                 R.scan('semicolon', '";"')
+    assert_equal [],
+                 R.scan('semicolon', '%w(;)')
+    assert_equal [],
+                 R.scan('semicolon', '/;/')
+  end
+
+  def test_comment
+    assert_equal [],
+                 R.scan('comment', '')
+    assert_equal ['# comment'],
+                 R.scan('comment', '# comment')
+    assert_equal ["# comment\n"],
+                 R.scan('comment', "# comment\n")
+    assert_equal ["# comment\n"],
+                 R.scan('comment', "# comment\n1 + 1")
+    assert_equal ["# comment\n"],
+                 R.scan('comment', "1 + 1 + 1# comment\n1 + 1")
+  end
+
+  def test_embdoc_beg
+    assert_equal [],
+                 R.scan('embdoc_beg', '')
+    assert_equal ["=begin\n"],
+                 R.scan('embdoc_beg', "=begin\ndoc\n=end")
+    assert_equal ["=begin \n"],
+                 R.scan('embdoc_beg', "=begin \ndoc\n=end\n")
+    assert_equal ["=begin comment\n"],
+                 R.scan('embdoc_beg', "=begin comment\ndoc\n=end\n")
+  end
+
+  def test_embdoc
+    assert_equal [],
+                 R.scan('embdoc', '')
+    assert_equal ["doc\n"],
+                 R.scan('embdoc', "=begin\ndoc\n=end")
+    assert_equal ["doc\n"],
+                 R.scan('embdoc', "=begin\ndoc\n=end\n")
+  end
+
+  def test_embdoc_end
+    assert_equal [],
+                 R.scan('embdoc_end', '')
+    assert_equal ["=end"],
+                 R.scan('embdoc_end', "=begin\ndoc\n=end")
+    assert_equal ["=end\n"],
+                 R.scan('embdoc_end', "=begin\ndoc\n=end\n")
+  end
+
+  def test_sp
+    assert_equal [],
+                 R.scan('sp', '')
+    assert_equal [' '],
+                 R.scan('sp', ' ')
+    assert_equal [' '],
+                 R.scan('sp', ' 1')
+    assert_equal [],
+                 R.scan('sp', "\n")
+    assert_equal [' '],
+                 R.scan('sp', " \n")
+    assert_equal [' ', ' '],
+                 R.scan('sp', "1 + 1")
+    assert_equal [],
+                 R.scan('sp', "' '")
+    assert_equal [],
+                 R.scan('sp', "%w(  )")
+    assert_equal [],
+                 R.scan('sp', "%w(  w  )")
+    assert_equal [],
+                 R.scan('sp', "p(/ /)")
+  end
+
+  # `nl' event always means End-Of-Statement.
+  def test_nl
+    assert_equal [],
+                 R.scan('nl', '')
+    assert_equal [],
+                 R.scan('nl', "\n")
+    assert_equal ["\n"],
+                 R.scan('nl', "1 + 1\n")
+    assert_equal ["\n", "\n"],
+                 R.scan('nl', "1 + 1\n2 + 2\n")
+    assert_equal [],
+                 R.scan('nl', "1 +\n1")
+    assert_equal [],
+                 R.scan('nl', "1;\n")
+    assert_equal ["\n"],
+                 R.scan('nl', "1 + 1\r\n")
+    assert_equal [],
+                 R.scan('nl', "1;\r\n")
+  end
+
+  def test_ignored_nl
+    assert_equal [],
+                 R.scan('ignored_nl', '')
+    assert_equal ["\n"],
+                 R.scan('ignored_nl', "\n")
+    assert_equal [],
+                 R.scan('ignored_nl', "1 + 1\n")
+    assert_equal [],
+                 R.scan('ignored_nl', "1 + 1\n2 + 2\n")
+    assert_equal ["\n"],
+                 R.scan('ignored_nl', "1 +\n1")
+    assert_equal ["\n"],
+                 R.scan('ignored_nl', "1;\n")
+    assert_equal [],
+                 R.scan('ignored_nl', "1 + 1\r\n")
+    assert_equal ["\n"],
+                 R.scan('ignored_nl', "1;\r\n")
+  end
+
+  def test___end__
+  end
+
+  def test_CHAR
+    assert_equal [],
+                 R.scan('CHAR', "")
+    assert_equal ["@"],
+                 R.scan('CHAR', "@")
+    assert_equal [],
+                 R.scan('CHAR', "@ivar")
+  end
+
+end
-- 
cgit v1.2.3