20 files changed, 2097 insertions, 0 deletions

diff --git a/ext/ripper/MANIFEST b/ext/ripper/MANIFEST
new file mode 100644
index 0000000000..fe1e25e620
--- /dev/null
+++ b/ext/ripper/MANIFEST
@@ -0,0 +1,20 @@
+MANIFEST
+README
+depend
+eventids2.c
+extconf.rb
+ripper.rb.in
+lib/ripper.rb
+test/check-event-arity.rb
+test/check-event-coverage.sh
+test/check-scanner-event-coverage.rb
+test/list-called-events.rb
+test/src.rb
+test/validate.rb
+tools/generate-eventids1.rb
+tools/generate-param-macros.rb
+tools/generate-ripper_rb.rb
+tools/list-parse-event-ids.rb
+tools/list-scan-event-ids.rb
+tools/preproc.rb
+tools/strip.rb
diff --git a/ext/ripper/README b/ext/ripper/README
new file mode 100644
index 0000000000..5242c4256d
--- /dev/null
+++ b/ext/ripper/README
@@ -0,0 +1,31 @@
+Ripper README
+=============
+
+  Ripper is a Ruby script parser.  You can get information
+  by event-based style from the parser.
+
+  !! WARNING !!
+
+  Ripper is still early-alpha version.
+  I never assure any kind of backward compatibility.
+  
+Requirements
+------------
+
+  * ruby 1.9 (support CVS HEAD only)
+  * bison 1.28 or later (You MUST use bison)
+
+Usage
+-----
+
+  See test/ripper/* and sample/ripper/*.
+  To know thrown events, see lib/ripper.rb.
+
+License
+-------
+
+  Ruby License.
+
+                                                Minero Aoki
+                                        aamine@loveruby.net
+                                      http://i.loveruby.net
diff --git a/ext/ripper/depend b/ext/ripper/depend
new file mode 100644
index 0000000000..b2333a2fdf
--- /dev/null
+++ b/ext/ripper/depend
@@ -0,0 +1,37 @@
+ripper.o: ripper.c lex.c eventids1.c eventids2.c
+	$(CC) $(CFLAGS) $(CPPFLAGS) -c ripper.c
+
+ripper.c: ripper.y
+	bison -t -v -o$@ ripper.y
+
+ripper.y: $(hdrdir)/parse.y $(srcdir)/tools/preproc.rb
+	$(RUBY) $(srcdir)/tools/preproc.rb $(hdrdir)/parse.y > $@
+
+eventids1.c: $(hdrdir)/parse.y $(srcdir)/tools/list-parse-event-ids.rb $(srcdir)/tools/generate-eventids1.rb
+	$(RUBY) $(srcdir)/tools/list-parse-event-ids.rb $(hdrdir)/parse.y | $(RUBY) $(srcdir)/tools/generate-eventids1.rb > $@
+
+lex.c:
+	cp $(hdrdir)/lex.c . || cp $(srcdir)/lex.c .
+
+#
+# Entries for ripper developpers (no srcdir/objdir support)
+#
+lib/ripper.rb: ripper.rb.in ids1 ids2 tools/generate-ripper_rb.rb
+	$(RUBY) tools/generate-ripper_rb.rb ripper.rb.in ids1 ids2 > $@
+
+ids1: tools/list-parse-event-ids.rb ../../parse.y
+	$(RUBY) tools/list-parse-event-ids.rb -a ../../parse.y > $@
+
+ids2: $(srcdir)/tools/list-scan-event-ids.rb eventids2.c
+	$(RUBY) tools/list-scan-event-ids.rb -a eventids2.c > $@
+
+test: ripper.so test/*.*
+	$(RUBY) test/check-event-arity.rb $(hdrdir)/parse.y
+	sh test/check-event-coverage.sh
+	$(RUBY) test/check-scanner-event-coverage.rb
+	$(RUBY) test/validate.rb > /dev/null
+	@echo OK
+
+preproc: ripper.E
+ripper.E: ripper.c
+	$(CC) -E $(CPPFLAGS) ripper.c | ruby tools/strip.rb > $@
diff --git a/ext/ripper/eventids2.c b/ext/ripper/eventids2.c
new file mode 100644
index 0000000000..28519eb8e6
--- /dev/null
+++ b/ext/ripper/eventids2.c
@@ -0,0 +1,272 @@
+#define tIGNORED_NL (tLAST_TOKEN + 1)
+#define tCOMMENT    (tLAST_TOKEN + 2)
+#define tEMBDOC_BEG (tLAST_TOKEN + 3)
+#define tEMBDOC     (tLAST_TOKEN + 4)
+#define tEMBDOC_END (tLAST_TOKEN + 5)
+#define tLWSP       (tLAST_TOKEN + 6)
+#define tHEREDOC_BEG (tLAST_TOKEN + 7)
+#define tHEREDOC_CONTENT (tLAST_TOKEN + 8)
+#define tHEREDOC_END (tLAST_TOKEN + 9)
+#define k__END__    (tLAST_TOKEN + 10)
+#define tCHAR       (tLAST_TOKEN + 11)
+
+static ID ripper_id_scan;
+
+static ID ripper_id_backref;
+static ID ripper_id_backtick;
+static ID ripper_id_comma;
+static ID ripper_id_const;
+static ID ripper_id_cvar;
+static ID ripper_id_embexpr_beg;
+static ID ripper_id_embexpr_end;
+static ID ripper_id_embvar;
+static ID ripper_id_float;
+static ID ripper_id_gvar;
+static ID ripper_id_ident;
+static ID ripper_id_int;
+static ID ripper_id_ivar;
+static ID ripper_id_kw;
+static ID ripper_id_lbrace;
+static ID ripper_id_lbracket;
+static ID ripper_id_lparen;
+static ID ripper_id_nl;
+static ID ripper_id_op;
+static ID ripper_id_period;
+static ID ripper_id_rbrace;
+static ID ripper_id_rbracket;
+static ID ripper_id_rparen;
+static ID ripper_id_semicolon;
+static ID ripper_id_symbeg;
+static ID ripper_id_tstring_beg;
+static ID ripper_id_tstring_content;
+static ID ripper_id_tstring_end;
+static ID ripper_id_words_beg;
+static ID ripper_id_qwords_beg;
+static ID ripper_id_words_sep;
+static ID ripper_id_regexp_beg;
+static ID ripper_id_regexp_end;
+
+static ID ripper_id_ignored_nl;
+static ID ripper_id_comment;
+static ID ripper_id_embdoc_beg;
+static ID ripper_id_embdoc;
+static ID ripper_id_embdoc_end;
+static ID ripper_id_lwsp;
+static ID ripper_id_heredoc_beg;
+static ID ripper_id_heredoc_content;
+static ID ripper_id_heredoc_end;
+static ID ripper_id___end__;
+static ID ripper_id_CHAR;
+
+static void
+ripper_init_eventids2()
+{
+    ripper_id_scan = rb_intern("on__scan");
+
+    ripper_id_backref = rb_intern("on__backref");
+    ripper_id_backtick = rb_intern("on__backtick");
+    ripper_id_comma = rb_intern("on__comma");
+    ripper_id_const = rb_intern("on__const");
+    ripper_id_cvar = rb_intern("on__cvar");
+    ripper_id_embexpr_beg = rb_intern("on__embexpr_beg");
+    ripper_id_embexpr_end = rb_intern("on__embexpr_end");
+    ripper_id_embvar = rb_intern("on__embvar");
+    ripper_id_float = rb_intern("on__float");
+    ripper_id_gvar = rb_intern("on__gvar");
+    ripper_id_ident = rb_intern("on__ident");
+    ripper_id_int = rb_intern("on__int");
+    ripper_id_ivar = rb_intern("on__ivar");
+    ripper_id_kw = rb_intern("on__kw");
+    ripper_id_lbrace = rb_intern("on__lbrace");
+    ripper_id_lbracket = rb_intern("on__lbracket");
+    ripper_id_lparen = rb_intern("on__lparen");
+    ripper_id_nl = rb_intern("on__nl");
+    ripper_id_op = rb_intern("on__op");
+    ripper_id_period = rb_intern("on__period");
+    ripper_id_rbrace = rb_intern("on__rbrace");
+    ripper_id_rbracket = rb_intern("on__rbracket");
+    ripper_id_rparen = rb_intern("on__rparen");
+    ripper_id_semicolon = rb_intern("on__semicolon");
+    ripper_id_symbeg = rb_intern("on__symbeg");
+    ripper_id_tstring_beg = rb_intern("on__tstring_beg");
+    ripper_id_tstring_content = rb_intern("on__tstring_content");
+    ripper_id_tstring_end = rb_intern("on__tstring_end");
+    ripper_id_words_beg = rb_intern("on__words_beg");
+    ripper_id_qwords_beg = rb_intern("on__qwords_beg");
+    ripper_id_words_sep = rb_intern("on__words_sep");
+    ripper_id_regexp_beg = rb_intern("on__regexp_beg");
+    ripper_id_regexp_end = rb_intern("on__regexp_end");
+
+    ripper_id_ignored_nl = rb_intern("on__ignored_nl");
+    ripper_id_comment = rb_intern("on__comment");
+    ripper_id_embdoc_beg = rb_intern("on__embdoc_beg");
+    ripper_id_embdoc = rb_intern("on__embdoc");
+    ripper_id_embdoc_end = rb_intern("on__embdoc_end");
+    ripper_id_lwsp = rb_intern("on__sp");
+    ripper_id_heredoc_beg = rb_intern("on__heredoc_beg");
+    ripper_id_heredoc_content = rb_intern("on__heredoc_content");
+    ripper_id_heredoc_end = rb_intern("on__heredoc_end");
+    ripper_id___end__ = rb_intern("on____end__");
+    ripper_id_CHAR = rb_intern("on__CHAR");
+}
+
+static struct token_assoc {
+    int token;
+    ID *id;
+} token_to_eventid[] = {
+    {' ',		&ripper_id_words_sep},
+    {'!',		&ripper_id_op},
+    {'%',		&ripper_id_op},
+    {'&',		&ripper_id_op},
+    {'*',		&ripper_id_op},
+    {'+',		&ripper_id_op},
+    {'-',		&ripper_id_op},
+    {'/',		&ripper_id_op},
+    {'<',		&ripper_id_op},
+    {'=',		&ripper_id_op},
+    {'>',		&ripper_id_op},
+    {'?',		&ripper_id_op},
+    {'^',		&ripper_id_op},
+    {'|',		&ripper_id_op},
+    {'~',		&ripper_id_op},
+    {':',		&ripper_id_op},
+    {',',		&ripper_id_comma},
+    {'.',		&ripper_id_period},
+    {';',		&ripper_id_semicolon},
+    {'`',		&ripper_id_backtick},
+    {'\n',              &ripper_id_nl},
+    {kALIAS,		&ripper_id_kw},
+    {kAND,		&ripper_id_kw},
+    {kBEGIN,		&ripper_id_kw},
+    {kBREAK,		&ripper_id_kw},
+    {kCASE,		&ripper_id_kw},
+    {kCLASS,		&ripper_id_kw},
+    {kDEF,		&ripper_id_kw},
+    {kDEFINED,		&ripper_id_kw},
+    {kDO,		&ripper_id_kw},
+    {kDO_BLOCK,		&ripper_id_kw},
+    {kDO_COND,		&ripper_id_kw},
+    {kELSE,		&ripper_id_kw},
+    {kELSIF,		&ripper_id_kw},
+    {kEND,		&ripper_id_kw},
+    {kENSURE,		&ripper_id_kw},
+    {kFALSE,		&ripper_id_kw},
+    {kFOR,		&ripper_id_kw},
+    {kIF,		&ripper_id_kw},
+    {kIF_MOD,		&ripper_id_kw},
+    {kIN,		&ripper_id_kw},
+    {kMODULE,		&ripper_id_kw},
+    {kNEXT,		&ripper_id_kw},
+    {kNIL,		&ripper_id_kw},
+    {kNOT,		&ripper_id_kw},
+    {kOR,		&ripper_id_kw},
+    {kREDO,		&ripper_id_kw},
+    {kRESCUE,		&ripper_id_kw},
+    {kRESCUE_MOD,	&ripper_id_kw},
+    {kRETRY,		&ripper_id_kw},
+    {kRETURN,		&ripper_id_kw},
+    {kSELF,		&ripper_id_kw},
+    {kSUPER,		&ripper_id_kw},
+    {kTHEN,		&ripper_id_kw},
+    {kTRUE,		&ripper_id_kw},
+    {kUNDEF,		&ripper_id_kw},
+    {kUNLESS,		&ripper_id_kw},
+    {kUNLESS_MOD,	&ripper_id_kw},
+    {kUNTIL,		&ripper_id_kw},
+    {kUNTIL_MOD,	&ripper_id_kw},
+    {kWHEN,		&ripper_id_kw},
+    {kWHILE,		&ripper_id_kw},
+    {kWHILE_MOD,	&ripper_id_kw},
+    {kYIELD,		&ripper_id_kw},
+    {k__FILE__,		&ripper_id_kw},
+    {k__LINE__,		&ripper_id_kw},
+    {klBEGIN,		&ripper_id_kw},
+    {klEND,		&ripper_id_kw},
+    {tAMPER,		&ripper_id_op},
+    {tANDOP,		&ripper_id_op},
+    {tAREF,		&ripper_id_op},
+    {tASET,		&ripper_id_op},
+    {tASSOC,		&ripper_id_op},
+    {tBACK_REF,		&ripper_id_backref},
+    {tCMP,		&ripper_id_op},
+    {tCOLON2,		&ripper_id_op},
+    {tCOLON3,		&ripper_id_op},
+    {tCONSTANT,		&ripper_id_const},
+    {tCVAR,		&ripper_id_cvar},
+    {tDOT2,		&ripper_id_op},
+    {tDOT3,		&ripper_id_op},
+    {tEQ,		&ripper_id_op},
+    {tEQQ,		&ripper_id_op},
+    {tFID,		&ripper_id_ident},
+    {tFLOAT,		&ripper_id_float},
+    {tGEQ,		&ripper_id_op},
+    {tGVAR,		&ripper_id_gvar},
+    {tIDENTIFIER,	&ripper_id_ident},
+    {tINTEGER,		&ripper_id_int},
+    {tIVAR,		&ripper_id_ivar},
+    {tLBRACE,		&ripper_id_lbrace},
+    {tLBRACE_ARG,	&ripper_id_lbrace},
+    {'{',       	&ripper_id_lbrace},
+    {'}',       	&ripper_id_rbrace},
+    {tLBRACK,		&ripper_id_lbracket},
+    {'[',       	&ripper_id_lbracket},
+    {']',       	&ripper_id_rbracket},
+    {tLEQ,		&ripper_id_op},
+    {tLPAREN,		&ripper_id_lparen},
+    {tLPAREN_ARG,	&ripper_id_lparen},
+    {'(',		&ripper_id_lparen},
+    {')',		&ripper_id_rparen},
+    {tLSHFT,		&ripper_id_op},
+    {tMATCH,		&ripper_id_op},
+    {tNEQ,		&ripper_id_op},
+    {tNMATCH,		&ripper_id_op},
+    {tNTH_REF,		&ripper_id_backref},
+    {tOP_ASGN,		&ripper_id_op},
+    {tOROP,		&ripper_id_op},
+    {tPOW,		&ripper_id_op},
+    {tQWORDS_BEG,	&ripper_id_qwords_beg},
+    {tREGEXP_BEG,	&ripper_id_regexp_beg},
+    {tREGEXP_END,	&ripper_id_regexp_end},
+    {tRPAREN,		&ripper_id_rparen},
+    {tRSHFT,		&ripper_id_op},
+    {tSTAR,		&ripper_id_op},
+    {tSTRING_BEG,	&ripper_id_tstring_beg},
+    {tSTRING_CONTENT,	&ripper_id_tstring_content},
+    {tSTRING_DBEG,	&ripper_id_embexpr_beg},
+    {tSTRING_DVAR,	&ripper_id_embvar},
+    {tSTRING_END,	&ripper_id_tstring_end},
+    {tSYMBEG,		&ripper_id_symbeg},
+    {tUMINUS,		&ripper_id_op},
+    {tUMINUS_NUM,	&ripper_id_op},
+    {tUPLUS,		&ripper_id_op},
+    {tWORDS_BEG,	&ripper_id_words_beg},
+    {tXSTRING_BEG,	&ripper_id_backtick},
+/* ripper specific tokens */
+    {tIGNORED_NL,       &ripper_id_ignored_nl},
+    {tCOMMENT,          &ripper_id_comment},
+    {tEMBDOC_BEG,       &ripper_id_embdoc_beg},
+    {tEMBDOC,           &ripper_id_embdoc},
+    {tEMBDOC_END,       &ripper_id_embdoc_end},
+    {tLWSP,             &ripper_id_lwsp},
+    {tHEREDOC_BEG,      &ripper_id_heredoc_beg},
+    {tHEREDOC_CONTENT,  &ripper_id_heredoc_content},
+    {tHEREDOC_END,      &ripper_id_heredoc_end},
+    {k__END__,          &ripper_id___end__},
+    {0, NULL}
+};
+
+static ID
+ripper_token2eventid(tok)
+    int tok;
+{
+    struct token_assoc *a;
+
+    for (a = token_to_eventid; a->id != NULL; a++) {
+        if (a->token == tok)
+            return *a->id;
+    }
+    if (tok < 256) {
+        return ripper_id_CHAR;
+    }
+    rb_raise(rb_eRuntimeError, "[Ripper FATAL] unknown token %d", tok);
+}
diff --git a/ext/ripper/extconf.rb b/ext/ripper/extconf.rb
new file mode 100644
index 0000000000..536e23d3ad
--- /dev/null
+++ b/ext/ripper/extconf.rb
@@ -0,0 +1,24 @@
+#!ruby -s
+
+require 'mkmf'
+require 'rbconfig'
+
+def main
+  unless have_command('bison')
+    $stderr.puts 'Ripper requires bison; abort'
+    exit 1
+  end
+  $objs = %w(ripper.o)
+  $cleanfiles.concat %w(ripper.y ripper.c ripper.output ids1 ids2)
+  $CPPFLAGS += ' -DRIPPER'
+  $CPPFLAGS += ' -DRIPPER_DEBUG' if $debug
+  create_makefile 'ripper'
+end
+
+def have_command(cmd)
+  ENV['PATH'].split(File::PATH_SEPARATOR).any? {|path|
+    File.executable?("#{path}/#{cmd}#{Config::CONFIG['EXEEXT']}")
+  }
+end
+
+main
diff --git a/ext/ripper/lib/ripper.rb b/ext/ripper/lib/ripper.rb
new file mode 100644
index 0000000000..b34147ee2a
--- /dev/null
+++ b/ext/ripper/lib/ripper.rb
@@ -0,0 +1,945 @@
+#
+# This file is automatically generated from ripper.rb.in and parse.y.
+# DO NOT MODIFY!!!!!!
+#
+
+#
+# ripper.rb
+#
+# Copyright (C) 2003,2004 Minero Aoki
+#
+
+require 'ripper.so'
+
+class Ripper
+  def Ripper.parse(str, *args)
+    new.parse(str, *args)
+  end
+
+  private
+
+  def warn(fmt, *args)
+  end
+
+  def warning(fmt, *args)
+  end
+
+  def compile_error(msg)
+  end
+
+  PARSER_EVENTS = [
+    :BEGIN,
+    :END,
+    :alias,
+    :alias_error,
+    :aref,
+    :aref_field,
+    :arg_ambiguous,
+    :arg_paren,
+    :arglist_add,
+    :arglist_add_block,
+    :arglist_add_star,
+    :arglist_new,
+    :arglist_prepend,
+    :array,
+    :assign,
+    :assign_error,
+    :assoc_new,
+    :assoclist_from_args,
+    :bare_assoc_hash,
+    :begin,
+    :binary,
+    :blockvar_add_block,
+    :blockvar_add_star,
+    :blockvar_new,
+    :bodystmt,
+    :brace_block,
+    :break,
+    :call,
+    :case,
+    :class,
+    :class_name_error,
+    :command,
+    :command_call,
+    :const_ref,
+    :constpath_field,
+    :constpath_ref,
+    :def,
+    :defined,
+    :defs,
+    :do_block,
+    :dot2,
+    :dot3,
+    :dyna_symbol,
+    :else,
+    :elsif,
+    :ensure,
+    :fcall,
+    :field,
+    :for,
+    :hash,
+    :if,
+    :if_mod,
+    :ifop,
+    :iter_block,
+    :massign,
+    :method_add_arg,
+    :mlhs_add,
+    :mlhs_add_star,
+    :mlhs_new,
+    :mlhs_paren,
+    :module,
+    :mrhs_add,
+    :mrhs_add_star,
+    :mrhs_new,
+    :mrhs_new_from_arglist,
+    :next,
+    :opassign,
+    :param_error,
+    :params,
+    :paren,
+    :parse_error,
+    :program,
+    :qwords_add,
+    :qwords_new,
+    :redo,
+    :regexp_literal,
+    :rescue,
+    :rescue_mod,
+    :restparam,
+    :retry,
+    :return,
+    :return0,
+    :sclass,
+    :space,
+    :stmts_add,
+    :stmts_new,
+    :string_add,
+    :string_concat,
+    :string_content,
+    :string_dvar,
+    :string_embexpr,
+    :string_literal,
+    :super,
+    :symbol,
+    :symbol_literal,
+    :topconst_field,
+    :topconst_ref,
+    :unary,
+    :undef,
+    :unless,
+    :unless_mod,
+    :until,
+    :until_mod,
+    :var_alias,
+    :var_field,
+    :var_ref,
+    :void_stmt,
+    :when,
+    :while,
+    :while_mod,
+    :word_add,
+    :word_new,
+    :words_add,
+    :words_new,
+    :xstring_add,
+    :xstring_literal,
+    :xstring_new,
+    :yield,
+    :yield0,
+    :zsuper
+  ]
+
+  SCANNER_EVENTS = [
+    :BEGIN,
+    :END,
+    :alias,
+    :alias_error,
+    :aref,
+    :aref_field,
+    :arg_ambiguous,
+    :arg_paren,
+    :arglist_add,
+    :arglist_add_block,
+    :arglist_add_star,
+    :arglist_new,
+    :arglist_prepend,
+    :array,
+    :assign,
+    :assign_error,
+    :assoc_new,
+    :assoclist_from_args,
+    :bare_assoc_hash,
+    :begin,
+    :binary,
+    :blockvar_add_block,
+    :blockvar_add_star,
+    :blockvar_new,
+    :bodystmt,
+    :brace_block,
+    :break,
+    :call,
+    :case,
+    :class,
+    :class_name_error,
+    :command,
+    :command_call,
+    :const_ref,
+    :constpath_field,
+    :constpath_ref,
+    :def,
+    :defined,
+    :defs,
+    :do_block,
+    :dot2,
+    :dot3,
+    :dyna_symbol,
+    :else,
+    :elsif,
+    :ensure,
+    :fcall,
+    :field,
+    :for,
+    :hash,
+    :if,
+    :if_mod,
+    :ifop,
+    :iter_block,
+    :massign,
+    :method_add_arg,
+    :mlhs_add,
+    :mlhs_add_star,
+    :mlhs_new,
+    :mlhs_paren,
+    :module,
+    :mrhs_add,
+    :mrhs_add_star,
+    :mrhs_new,
+    :mrhs_new_from_arglist,
+    :next,
+    :opassign,
+    :param_error,
+    :params,
+    :paren,
+    :parse_error,
+    :program,
+    :qwords_add,
+    :qwords_new,
+    :redo,
+    :regexp_literal,
+    :rescue,
+    :rescue_mod,
+    :restparam,
+    :retry,
+    :return,
+    :return0,
+    :sclass,
+    :space,
+    :stmts_add,
+    :stmts_new,
+    :string_add,
+    :string_concat,
+    :string_content,
+    :string_dvar,
+    :string_embexpr,
+    :string_literal,
+    :super,
+    :symbol,
+    :symbol_literal,
+    :topconst_field,
+    :topconst_ref,
+    :unary,
+    :undef,
+    :unless,
+    :unless_mod,
+    :until,
+    :until_mod,
+    :var_alias,
+    :var_field,
+    :var_ref,
+    :void_stmt,
+    :when,
+    :while,
+    :while_mod,
+    :word_add,
+    :word_new,
+    :words_add,
+    :words_new,
+    :xstring_add,
+    :xstring_literal,
+    :xstring_new,
+    :yield,
+    :yield0,
+    :zsuper
+  ]
+  LEXER_EVENTS = SCANNER_EVENTS
+
+  EVENTS = PARSER_EVENTS + SCANNER_EVENTS
+
+  #
+  # Parser Events
+  #
+
+  def on__BEGIN(a)
+    a
+  end
+
+  def on__END(a)
+    a
+  end
+
+  def on__alias(a, b)
+    a
+  end
+
+  def on__alias_error(a)
+    a
+  end
+
+  def on__aref(a, b)
+    a
+  end
+
+  def on__aref_field(a, b)
+    a
+  end
+
+  def on__arg_ambiguous()
+    nil
+  end
+
+  def on__arg_paren(a)
+    a
+  end
+
+  def on__arglist_add(a, b)
+    a
+  end
+
+  def on__arglist_add_block(a, b)
+    a
+  end
+
+  def on__arglist_add_star(a, b)
+    a
+  end
+
+  def on__arglist_new()
+    nil
+  end
+
+  def on__arglist_prepend(a, b)
+    a
+  end
+
+  def on__array(a)
+    a
+  end
+
+  def on__assign(a, b)
+    a
+  end
+
+  def on__assign_error(a)
+    a
+  end
+
+  def on__assoc_new(a, b)
+    a
+  end
+
+  def on__assoclist_from_args(a)
+    a
+  end
+
+  def on__bare_assoc_hash(a)
+    a
+  end
+
+  def on__begin(a)
+    a
+  end
+
+  def on__binary(a, b, c)
+    a
+  end
+
+  def on__blockvar_add_block(a, b)
+    a
+  end
+
+  def on__blockvar_add_star(a, b)
+    a
+  end
+
+  def on__blockvar_new(a)
+    a
+  end
+
+  def on__bodystmt(a, b, c, d)
+    a
+  end
+
+  def on__brace_block(a, b)
+    a
+  end
+
+  def on__break(a)
+    a
+  end
+
+  def on__call(a, b, c)
+    a
+  end
+
+  def on__case(a, b)
+    a
+  end
+
+  def on__class(a, b, c)
+    a
+  end
+
+  def on__class_name_error(a)
+    a
+  end
+
+  def on__command(a, b)
+    a
+  end
+
+  def on__command_call(a, b, c, d)
+    a
+  end
+
+  def on__const_ref(a)
+    a
+  end
+
+  def on__constpath_field(a, b)
+    a
+  end
+
+  def on__constpath_ref(a, b)
+    a
+  end
+
+  def on__def(a, b, c)
+    a
+  end
+
+  def on__defined(a)
+    a
+  end
+
+  def on__defs(a, b, c, d, e)
+    a
+  end
+
+  def on__do_block(a, b)
+    a
+  end
+
+  def on__dot2(a, b)
+    a
+  end
+
+  def on__dot3(a, b)
+    a
+  end
+
+  def on__dyna_symbol(a)
+    a
+  end
+
+  def on__else(a)
+    a
+  end
+
+  def on__elsif(a, b, c)
+    a
+  end
+
+  def on__ensure(a)
+    a
+  end
+
+  def on__fcall(a)
+    a
+  end
+
+  def on__field(a, b, c)
+    a
+  end
+
+  def on__for(a, b, c)
+    a
+  end
+
+  def on__hash(a)
+    a
+  end
+
+  def on__if(a, b, c)
+    a
+  end
+
+  def on__if_mod(a, b)
+    a
+  end
+
+  def on__ifop(a, b, c)
+    a
+  end
+
+  def on__iter_block(a, b)
+    a
+  end
+
+  def on__massign(a, b)
+    a
+  end
+
+  def on__method_add_arg(a, b)
+    a
+  end
+
+  def on__mlhs_add(a, b)
+    a
+  end
+
+  def on__mlhs_add_star(a, b)
+    a
+  end
+
+  def on__mlhs_new()
+    nil
+  end
+
+  def on__mlhs_paren(a)
+    a
+  end
+
+  def on__module(a, b)
+    a
+  end
+
+  def on__mrhs_add(a, b)
+    a
+  end
+
+  def on__mrhs_add_star(a, b)
+    a
+  end
+
+  def on__mrhs_new()
+    nil
+  end
+
+  def on__mrhs_new_from_arglist(a)
+    a
+  end
+
+  def on__next(a)
+    a
+  end
+
+  def on__opassign(a, b, c)
+    a
+  end
+
+  def on__param_error(a)
+    a
+  end
+
+  def on__params(a, b, c, d)
+    a
+  end
+
+  def on__paren(a)
+    a
+  end
+
+  def on__parse_error(a)
+    a
+  end
+
+  def on__program(a)
+    a
+  end
+
+  def on__qwords_add(a, b)
+    a
+  end
+
+  def on__qwords_new()
+    nil
+  end
+
+  def on__redo()
+    nil
+  end
+
+  def on__regexp_literal(a)
+    a
+  end
+
+  def on__rescue(a, b, c, d)
+    a
+  end
+
+  def on__rescue_mod(a, b)
+    a
+  end
+
+  def on__restparam(a)
+    a
+  end
+
+  def on__retry()
+    nil
+  end
+
+  def on__return(a)
+    a
+  end
+
+  def on__return0()
+    nil
+  end
+
+  def on__sclass(a, b)
+    a
+  end
+
+  def on__space(a)
+    a
+  end
+
+  def on__stmts_add(a, b)
+    a
+  end
+
+  def on__stmts_new()
+    nil
+  end
+
+  def on__string_add(a, b)
+    a
+  end
+
+  def on__string_concat(a, b)
+    a
+  end
+
+  def on__string_content()
+    nil
+  end
+
+  def on__string_dvar(a)
+    a
+  end
+
+  def on__string_embexpr(a)
+    a
+  end
+
+  def on__string_literal(a)
+    a
+  end
+
+  def on__super(a)
+    a
+  end
+
+  def on__symbol(a)
+    a
+  end
+
+  def on__symbol_literal(a)
+    a
+  end
+
+  def on__topconst_field(a)
+    a
+  end
+
+  def on__topconst_ref(a)
+    a
+  end
+
+  def on__unary(a, b)
+    a
+  end
+
+  def on__undef(a)
+    a
+  end
+
+  def on__unless(a, b, c)
+    a
+  end
+
+  def on__unless_mod(a, b)
+    a
+  end
+
+  def on__until(a, b)
+    a
+  end
+
+  def on__until_mod(a, b)
+    a
+  end
+
+  def on__var_alias(a, b)
+    a
+  end
+
+  def on__var_field(a)
+    a
+  end
+
+  def on__var_ref(a)
+    a
+  end
+
+  def on__void_stmt()
+    nil
+  end
+
+  def on__when(a, b, c)
+    a
+  end
+
+  def on__while(a, b)
+    a
+  end
+
+  def on__while_mod(a, b)
+    a
+  end
+
+  def on__word_add(a, b)
+    a
+  end
+
+  def on__word_new()
+    nil
+  end
+
+  def on__words_add(a, b)
+    a
+  end
+
+  def on__words_new()
+    nil
+  end
+
+  def on__xstring_add(a, b)
+    a
+  end
+
+  def on__xstring_literal(a)
+    a
+  end
+
+  def on__xstring_new()
+    nil
+  end
+
+  def on__yield(a)
+    a
+  end
+
+  def on__yield0()
+    nil
+  end
+
+  def on__zsuper()
+    nil
+  end
+
+  #
+  # Lexer Events
+  #
+
+  def on__scan(event, token)
+  end
+
+  def on__CHAR(token)
+    token
+  end
+
+  def on____end__(token)
+    token
+  end
+
+  def on__backref(token)
+    token
+  end
+
+  def on__backtick(token)
+    token
+  end
+
+  def on__comma(token)
+    token
+  end
+
+  def on__comment(token)
+    token
+  end
+
+  def on__const(token)
+    token
+  end
+
+  def on__cvar(token)
+    token
+  end
+
+  def on__embdoc(token)
+    token
+  end
+
+  def on__embdoc_beg(token)
+    token
+  end
+
+  def on__embdoc_end(token)
+    token
+  end
+
+  def on__embexpr_beg(token)
+    token
+  end
+
+  def on__embexpr_end(token)
+    token
+  end
+
+  def on__embvar(token)
+    token
+  end
+
+  def on__float(token)
+    token
+  end
+
+  def on__gvar(token)
+    token
+  end
+
+  def on__heredoc_beg(token)
+    token
+  end
+
+  def on__heredoc_content(token)
+    token
+  end
+
+  def on__heredoc_end(token)
+    token
+  end
+
+  def on__ident(token)
+    token
+  end
+
+  def on__ignored_nl(token)
+    token
+  end
+
+  def on__int(token)
+    token
+  end
+
+  def on__ivar(token)
+    token
+  end
+
+  def on__kw(token)
+    token
+  end
+
+  def on__lbrace(token)
+    token
+  end
+
+  def on__lbracket(token)
+    token
+  end
+
+  def on__lparen(token)
+    token
+  end
+
+  def on__lwsp(token)
+    token
+  end
+
+  def on__nl(token)
+    token
+  end
+
+  def on__op(token)
+    token
+  end
+
+  def on__period(token)
+    token
+  end
+
+  def on__qwords_beg(token)
+    token
+  end
+
+  def on__rbrace(token)
+    token
+  end
+
+  def on__rbracket(token)
+    token
+  end
+
+  def on__regexp_beg(token)
+    token
+  end
+
+  def on__regexp_end(token)
+    token
+  end
+
+  def on__rparen(token)
+    token
+  end
+
+  def on__semicolon(token)
+    token
+  end
+
+  def on__symbeg(token)
+    token
+  end
+
+  def on__tstring_beg(token)
+    token
+  end
+
+  def on__tstring_content(token)
+    token
+  end
+
+  def on__tstring_end(token)
+    token
+  end
+
+  def on__words_beg(token)
+    token
+  end
+
+  def on__words_sep(token)
+    token
+  end
+end
diff --git a/ext/ripper/ripper.rb.in b/ext/ripper/ripper.rb.in
new file mode 100644
index 0000000000..c6fa554296
--- /dev/null
+++ b/ext/ripper/ripper.rb.in
@@ -0,0 +1,53 @@
+#
+# This file is automatically generated from ripper.rb.in and parse.y.
+# DO NOT MODIFY!!!!!!
+#
+
+#
+# ripper.rb
+#
+# Copyright (C) 2003,2004 Minero Aoki
+#
+
+require 'ripper.so'
+
+class Ripper
+  def Ripper.parse(str, *args)
+    new.parse(str, *args)
+  end
+
+  private
+
+  def warn(fmt, *args)
+  end
+
+  def warning(fmt, *args)
+  end
+
+  def compile_error(msg)
+  end
+
+  PARSER_EVENTS = [
+#include ids1
+  ]
+
+  SCANNER_EVENTS = [
+#include ids2
+  ]
+  LEXER_EVENTS = SCANNER_EVENTS
+
+  EVENTS = PARSER_EVENTS + SCANNER_EVENTS
+
+  #
+  # Parser Events
+  #
+#include handlers1
+
+  #
+  # Lexer Events
+  #
+
+  def on__scan(event, token)
+  end
+#include handlers2
+end
diff --git a/ext/ripper/test/check-event-arity.rb b/ext/ripper/test/check-event-arity.rb
new file mode 100644
index 0000000000..00e1b98dbc
--- /dev/null
+++ b/ext/ripper/test/check-event-arity.rb
@@ -0,0 +1,24 @@
+def main
+  invalid = false
+  table = {}
+  ARGF.each do |line|
+    next if /\A\#\s*define\s+s?dispatch\d/ === line
+    next if /ripper_dispatch\d/ === line
+    line.scan(/dispatch(\d)\((\w+)/) do |num, ev|
+      num = num.to_i
+      if data = table[ev]
+        locations, arity = data
+        unless num == arity
+          invalid = true
+          puts "arity differ [#{ev}]: #{ARGF.lineno}->#{num}; #{locations.join(',')}->#{arity}"
+        end
+        locations.push ARGF.lineno
+      else
+        table[ev] = [[ARGF.lineno], num.to_i]
+      end
+    end
+  end
+  exit 1 if invalid
+end
+
+main
diff --git a/ext/ripper/test/check-event-coverage.sh b/ext/ripper/test/check-event-coverage.sh
new file mode 100755
index 0000000000..727c3b9837
--- /dev/null
+++ b/ext/ripper/test/check-event-coverage.sh
@@ -0,0 +1,15 @@
+# $Id$
+
+RUBY=${RUBY:-ruby}
+status=0
+
+$RUBY tools/list-parse-event-ids.rb parse.y | awk '{print "on__" $1}' > list_a
+$RUBY test/list-called-events.rb | sort -u > list_b
+diff -u list_a list_b | grep '^-on' | sed 's/^-on__//' > list_diff
+if [ -s list_diff ]
+then
+    cat list_diff
+    status=1
+fi
+rm -f list_a list_b list_diff
+exit $status
diff --git a/ext/ripper/test/check-scanner-event-coverage.rb b/ext/ripper/test/check-scanner-event-coverage.rb
new file mode 100644
index 0000000000..4de01e35f4
--- /dev/null
+++ b/ext/ripper/test/check-scanner-event-coverage.rb
@@ -0,0 +1,18 @@
+def main
+  not_tested = eventids() - tested_ids()
+  unless not_tested.empty?
+    puts not_tested
+    exit 1
+  end
+  exit 0
+end
+
+def eventids
+  File.read('eventids2.c').scan(/on__(\w+)/).flatten.uniq
+end
+
+def tested_ids
+  File.read('test/test_scanner_events.rb').scan(/def test_(\S+)/).flatten.uniq
+end
+
+main
diff --git a/ext/ripper/test/list-called-events.rb b/ext/ripper/test/list-called-events.rb
new file mode 100644
index 0000000000..d799efc979
--- /dev/null
+++ b/ext/ripper/test/list-called-events.rb
@@ -0,0 +1,12 @@
+require 'ripper.so'
+
+class R < Ripper
+  def method_missing(mid, *args)
+    puts mid
+    args[0]
+  end
+  undef :warn
+end
+
+fname = (ARGV[0] || 'test/src_rb')
+R.new(File.read(fname), fname, 1).parse
diff --git a/ext/ripper/test/src_rb b/ext/ripper/test/src_rb
new file mode 100644
index 0000000000..e7894362f3
--- /dev/null
+++ b/ext/ripper/test/src_rb
@@ -0,0 +1,363 @@
+# comment
+=begin
+  embedded document
+=end
+
+# literal
+1
+1000000000000000000000
+1.0
+1.234e5
+1..2
+1...3
+:symbol
+:"dynamic #{sym_embexpr} symbol"
+[1,2,3]
+{1 => 2}
+'string'
+"string"
+"before #{str_embexpr} after"
+"str #@ivar str"
+"str #$gvar str"
+"string" "concat"
+`/bin/true`
+{1, 2, 3, 4}
+/regexp/
+/regexp_with_opt/mioe
+/regexp #{regexp_embexpr} after/
+%q[string]
+%Q[str#{str_embexpr}ing]
+%r[regexp]
+%w( a b c )
+%W( a#{w_emb}b c d )
+<<HERE
+heredoc line 1
+heredoc line 2
+heredoc line 3
+HERE
+
+# special variables
+true
+false
+nil
+self
+
+# def
+def a
+end
+def b()
+end
+def c(a)
+end
+def d(a,*rest)
+end
+def e(a,&block)
+end
+def f(a,*rest,&block)
+end
+def g(*rest)
+end
+def h(&block)
+end
+def i(*rest,&block)
+end
+def j(CONST)
+end
+def k(@ivar)
+end
+def l($gvar)
+end
+def n(@@cvar)
+end
+
+# alias
+alias x b
+alias $rest $'     # error
+alias $nth $1      # error
+
+# undef
+undef warn
+
+# class, module
+class C
+end
+module M
+end
+class cname
+end
+class << Object.new
+  def self.a
+  end
+end
+
+# field
+$a = 1
+$' = 0   # error
+$1 = 0   # error
+@a = 2
+@@a = 3
+a = 4
+a += 1
+a -= 1
+a *= 1
+a /= 1
+a &&= 1
+a ||= 1
+m.a = 5
+m.a += 1
+m.a &&= 1
+m.a ||= 1
+a[1] = 2
+a[1] += 1
+a[1] &&= 1
+a[1] ||= 1
+C = 1
+C::C = 1
+::C = 1
+def m
+  C = 1      # dynamic const assignment
+  C::C = 1   # dynamic const assignment
+  ::C = 1    # dynamic const assignment
+end
+
+# ref
+lvar = $a
+lvar = @a
+lvar = @@a
+lvar = Object
+lvar = C
+lvar = C::C
+lvar = ::C
+lvar = a[1]
+
+# unary operator
++1
+-1
+not 1
+!1
+~str
+
+# binary operator
+1 + 1
+1 - 1
+1 * 1
+1 / 1
+1 ** 1
+1 ^ 1
+1 & 1
+1 | 1
+1 && 1
+1 || 1
+
+# mlhs, mrhs
+a, b, c = list
+a, = list
+a, * = list
+a, *b = list
+a, (b, c), d, *e = list
+mlhs = 1, 2
+mlhs = 1, 2, 3, *list
+mlhs = *list
+
+# method call
+m
+m()
+m(a)
+m(a,a)
+m(*a)
+m(&b)
+m(a,*a)
+m(a,&b)
+m(a,*a,&b)
+m(1=>2)
+m(1=>2,*a)
+m(1=>2,&b)
+m(1=>2,*a,&b)
+m ()
+m (a)
+m (a,a)
+m (*a)
+m (&b)
+m (a,*a)
+m (a,&b)
+m (a,*a,&b)
+m (1=>2)
+m (1=>2,*a)
+m (1=>2,&b)
+m (1=>2,*a,&b)
+m a
+m a,a
+m *a
+m &b
+m a,*a
+m a,&b
+m a,*a,&b
+m 1=>2
+m 1=>2,*a
+m 1=>2,&b
+m 1=>2,*a,&b
+obj.m
+obj.m()
+obj.m(a)
+obj.m(a,a)
+obj.m(*a)
+obj.m(&b)
+obj.m(a,*a)
+obj.m(a,&b)
+obj.m(a,*a,&b)
+obj.m(1=>2)
+obj.m(1=>2,*a)
+obj.m(1=>2,&b)
+obj.m(1=>2,*a,&b)
+obj.m ()
+obj.m (a)
+obj.m (a,a)
+obj.m (*a)
+obj.m (&b)
+obj.m (a,*a)
+obj.m (a,&b)
+obj.m (a,*a,&b)
+obj.m (1=>2)
+obj.m (1=>2)
+obj.m (1=>2,*a)
+obj.m (1=>2,&b)
+obj.m (1=>2,*a,&b)
+obj.m a
+obj.m a,a
+obj.m *a
+obj.m &b
+obj.m a,*a
+obj.m a,&b
+obj.m a,*a,&b
+obj.m 1=>2
+obj.m 1=>2,*a
+obj.m 1=>2,&b
+obj.m 1=>2,*a,&b
+
+# ambiguous argument
+m +1
+m /r/
+
+# iterator
+[1,2,3].each do |i|
+  print i
+end
+{1=>true}.each do |k,v|
+  puts k
+end
+[1,2,3].each {|i| print i }
+[1].each {|a,| }
+[1].each {|*b| }
+[1].each {|a,*b| }
+[1].each {|&block| }
+[1].each {|a,&block| }
+[1].each {|a,*b,&block| }
+a = lambda() {|n| n * n }
+a = lambda () {|n| n * n }
+a = lambda (a) {|n| n * n }
+a = lambda (a,b) {|n| n * n }
+
+# BEGIN, END
+BEGIN { }
+END { }
+
+# if, unless
+1 if true
+2 unless false
+if false
+  5
+elsif false
+  6
+elsif false then 7
+else
+  8
+end
+if m
+end
+unless 1
+  2
+end
+unless m
+end
+0 ? 1 : 2
+
+# case
+case 'a'
+when 'b'
+  ;
+when 'c' then 1
+else
+  2
+end
+case
+when 1
+when 2
+when 3
+else
+end
+case 1
+else
+end
+case
+else
+end
+
+# while, until, for
+while true
+  break
+  next
+  redo
+end
+begin
+  break
+end while true
+until false
+  break
+  next
+  redo
+end
+begin
+  break
+end until false
+for x in m()
+  break
+  next
+  redo
+end
+0 until true
+1 while false
+
+# begin, rescue, else, ensure
+begin
+  1
+rescue StandardError => er
+  2
+rescue => er
+  3
+  retry
+else
+  4
+ensure
+  5
+end
+a = 1 rescue 2
+
+# jumps
+def m
+  redo
+  yield
+  yield nil
+  super
+  super 1
+  return
+  return nil
+end
+
+# defined
+defined? f
+defined?(f)
+
+n = 1 \
++ 1
+
+__END__
diff --git a/ext/ripper/test/validate.rb b/ext/ripper/test/validate.rb
new file mode 100644
index 0000000000..808d321043
--- /dev/null
+++ b/ext/ripper/test/validate.rb
@@ -0,0 +1,68 @@
+require 'ripper.so'
+
+class R < Ripper
+  def initialize(*args)
+    super
+    @lineno = 0
+  end
+
+  def parse
+    result = super
+    puts "#{@lineno}:result: #{rawVALUE(result)}"
+    validate_object result
+    p result
+    result
+  end
+
+  def on__nl(str)
+    @lineno += 1
+  end
+
+  def on__ignored_nl(str)
+    @lineno += 1
+  end
+
+  def on__comment(cmt)
+    @lineno += 1
+  end
+
+  def on__embdoc_beg(str)
+    @lineno += 1
+  end
+
+  def on__embdoc(str)
+    @lineno += 1
+  end
+
+  def on__embdoc_end(str)
+    @lineno += 1
+  end
+
+  def method_missing(mid, *args)
+    puts mid
+    args.each_with_index do |a,idx|
+      puts "#{@lineno}:#{mid}\##{idx+1}: #{rawVALUE(a)}"
+      validate_object a
+      p a
+    end
+    args[0]
+  end
+
+  def warn(*args)
+  end
+
+  def warning(*args)
+  end
+
+  unless respond_to?(:validate_object)
+    def validate_object(x)
+      x
+    end
+    def rawVALUE(x)
+      x.object_id
+    end
+  end
+end
+
+fname = (ARGV[0] || 'test/src_rb')
+R.new(File.read(fname), fname, 1).parse
diff --git a/ext/ripper/tools/generate-eventids1.rb b/ext/ripper/tools/generate-eventids1.rb
new file mode 100755
index 0000000000..de0e27b89c
--- /dev/null
+++ b/ext/ripper/tools/generate-eventids1.rb
@@ -0,0 +1,18 @@
+#
+# generate-eventids1.rb
+#
+
+ids = ARGF.map {|s| s.strip }
+
+ids.each do |id|
+  puts "static ID ripper_id_#{id};"
+end
+
+puts
+puts 'static void'
+puts 'ripper_init_eventids1()'
+puts '{'
+ids.each do |id|
+  puts %Q[    ripper_id_#{id} = rb_intern("on__#{id}");]
+end
+puts '}'
diff --git a/ext/ripper/tools/generate-param-macros.rb b/ext/ripper/tools/generate-param-macros.rb
new file mode 100755
index 0000000000..b19f6e8d5c
--- /dev/null
+++ b/ext/ripper/tools/generate-param-macros.rb
@@ -0,0 +1,14 @@
+off = true
+ARGF.each do |line|
+  case line
+  when /RIPPER_PARAMS_DECL_BEGIN/
+    off = false
+  when /RIPPER_PARAMS_DECL_END/
+    exit
+  when /ripper/
+    next if off
+    var = line.scan(/\w+/).last or next
+    base = var.sub(/ripper_/, '')
+    puts %"\#define #{base}\t\t(parser->ripper_#{base})"
+  end
+end
diff --git a/ext/ripper/tools/generate-ripper_rb.rb b/ext/ripper/tools/generate-ripper_rb.rb
new file mode 100755
index 0000000000..c6b2aea13f
--- /dev/null
+++ b/ext/ripper/tools/generate-ripper_rb.rb
@@ -0,0 +1,44 @@
+#
+# generate-ripper_rb.rb
+# Creates ripper.rb, filling in default event handlers, given a basic 
+# template, the list of parser events (ids1), and a list of lexer 
+# events (ids2).
+#
+
+def main
+  template, ids1, ids2 = *ARGV
+  File.foreach(template) do |line|
+    case line
+    when /\A\#include handlers1/
+      File.foreach(ids1) do |line|
+        id, arity = line.split
+        arity = arity.to_i
+        puts
+        puts "  def on__#{id}(#{argdecl(arity)})"
+        puts "    #{arity == 0 ? 'nil' : 'a'}"
+        puts "  end"
+      end
+    when /\A\#include handlers2/
+      File.foreach(ids2) do |line|
+        id, arity = line.split
+        arity = arity.to_i
+        puts
+        puts "  def on__#{id}(token)"
+        puts "    token"
+        puts "  end"
+      end
+    when /\A\#include (.*)/
+      raise "unknown operation: #include #{$1}"
+    else
+      print line
+    end
+  end
+end
+
+# Generate generic arg list depending on arity (n)
+# n:: [Integer] arity of method
+def argdecl( n )
+  %w(a b c d e f g h i j k l m)[0, n].join(', ')
+end
+
+main
diff --git a/ext/ripper/tools/list-parse-event-ids.rb b/ext/ripper/tools/list-parse-event-ids.rb
new file mode 100755
index 0000000000..2936f9b092
--- /dev/null
+++ b/ext/ripper/tools/list-parse-event-ids.rb
@@ -0,0 +1,38 @@
+#
+# list-parse-event-ids.rb
+#
+
+require 'getopts'
+
+def usage( status )
+  (status == 0 ? $stdout : $stderr).print(<<EOS)
+Usage: #{File.basename($0)} [-a] filename
+EOS
+  exit status
+end
+
+def main
+  getopts('a') or usage(1)
+  extract_ids(ARGF).each do |id, arity|
+    if $OPT_a
+    then puts "#{id} #{arity}"
+    else puts id
+    end
+  end
+end
+
+def extract_ids( f )
+  results = []
+  f.each do |line|
+    next if /\A\#\s*define\s+s?dispatch/ === line
+    next if /ripper_dispatch/ === line
+    if a = line.scan(/dispatch(\d)\((\w+)/)
+      a.each do |arity, event|
+        results.push [event, arity.to_i]
+      end
+    end
+  end
+  results.uniq.sort
+end
+
+main
diff --git a/ext/ripper/tools/list-scan-event-ids.rb b/ext/ripper/tools/list-scan-event-ids.rb
new file mode 100755
index 0000000000..6f25362b5d
--- /dev/null
+++ b/ext/ripper/tools/list-scan-event-ids.rb
@@ -0,0 +1,32 @@
+#
+# list-scan-event-ids.rb
+#
+
+require 'getopts'
+
+def usage(status)
+  (status == 0 ? $stdout : $stderr).puts(<<EOS)
+Usage: #{File.basename($0)} eventids2.c
+    -a    print IDs with arity.
+EOS
+  exit status
+end
+
+def main
+  ok = getopts('a', 'help')
+  usage 0 if $OPT_help
+  usage 1 unless ok
+  extract_ids(ARGF).sort.each do |id|
+    if $OPT_a
+      puts "#{id} 1"
+    else
+      puts id
+    end
+  end
+end
+
+def extract_ids(f)
+  (f.read.scan(/ripper_id_(\w+)/).flatten - ['scan']).uniq
+end
+
+main
diff --git a/ext/ripper/tools/preproc.rb b/ext/ripper/tools/preproc.rb
new file mode 100755
index 0000000000..a2dba36e02
--- /dev/null
+++ b/ext/ripper/tools/preproc.rb
@@ -0,0 +1,57 @@
+def main
+  prelude
+  grammar
+  usercode
+end
+
+def prelude
+  while line = ARGF.gets
+    case line
+    when %r</\*%%%\*/>
+      puts '/*'
+    when %r</\*%>
+      puts '*/'
+    when %r<%\*/>
+      puts
+    when /\A%%/
+      puts '%%'
+      return
+    when /\A%token/
+      puts line.sub(/<\w+>/, '<val>')
+    when /\A%type/
+      puts line.sub(/<\w+>/, '<val>')
+    else
+      print line
+    end
+  end
+end
+
+def grammar
+  while line = ARGF.gets
+    case line
+    when %r</\*%%%\*/>
+      puts '#if 0'
+    when %r</\*%c%\*/>
+      puts '/*'
+    when %r</\*%c>
+      puts '*/'
+    when %r</\*%>
+      puts '#endif'
+    when %r<%\*/>
+      puts
+    when /\A%%/
+      puts '%%'
+      return
+    else
+      print line
+    end
+  end
+end
+
+def usercode
+  while line = ARGF.gets
+    print line
+  end
+end
+
+main
diff --git a/ext/ripper/tools/strip.rb b/ext/ripper/tools/strip.rb
new file mode 100755
index 0000000000..99413c361d
--- /dev/null
+++ b/ext/ripper/tools/strip.rb
@@ -0,0 +1,12 @@
+last_is_void = false
+ARGF.each do |line|
+  if line.strip.empty?
+    #puts() unless last_is_void
+    last_is_void = true
+  elsif /\A\#/ === line
+    ;
+  else
+    print line
+    last_is_void = false
+  end
+end