4 files changed, 1901 insertions, 0 deletions
diff --git a/trunk/test/ripper/dummyparser.rb b/trunk/test/ripper/dummyparser.rb
new file mode 100644
index 0000000000..483ac0d013
--- /dev/null
+++ b/trunk/test/ripper/dummyparser.rb
@@ -0,0 +1,571 @@
+#
+# dummyparser.rb
+#
+
+require 'ripper'
+
+class Node
+  def initialize(name, *nodes)
+    @name = name
+    @children = nodes
+  end
+
+  attr_reader :children
+
+  def to_s
+    "#{@name}(#{@children.map {|n| n.to_s }.join(',')})"
+  end
+end
+
+class NodeList
+  def initialize
+    @list = []
+  end
+
+  attr_reader :list
+
+  def push(item)
+    @list.push item
+    self
+  end
+
+  def prepend(items)
+    @list.unshift items
+  end
+
+  def to_s
+    '[' + @list.join(',') + ']'
+  end
+end
+
+class DummyParser < Ripper
+
+  def on_program(stmts)
+    $thru_program = true
+    stmts
+  end
+
+  def on_stmts_new
+    NodeList.new
+  end
+
+  def on_stmts_add(stmts, st)
+    stmts.push st
+    stmts
+  end
+
+  def on_void_stmt
+    Node.new('void')
+  end
+
+  def on_BEGIN(stmts)
+    Node.new('BEGIN', stmts)
+  end
+
+  def on_END(stmts)
+    Node.new('END', stmts)
+  end
+
+  def on_var_ref(name)
+    Node.new('ref', name)
+  end
+
+  def on_alias(a, b)
+    Node.new('alias', a, b)
+  end
+
+  def on_var_alias(a, b)
+    Node.new('valias', a, b)
+  end
+
+  def on_alias_error(a)
+    Node.new('aliaserr', a)
+  end
+
+  def on_aref(a, b)
+    Node.new('aref', a, b)
+  end
+
+  def on_aref_field(a, b)
+    Node.new('aref_field', a, b)
+  end
+
+  def on_arg_ambiguous
+    Node.new('arg_ambiguous')
+  end
+
+  def on_arg_paren(args)
+    args
+  end
+
+  def on_args_new
+    NodeList.new
+  end
+
+  def on_args_add(list, arg)
+    list.push(arg)
+  end
+
+  def on_args_add_block(list, blk)
+    if blk
+      list.push('&' + blk.to_s)
+    else
+      list
+    end
+  end
+
+  def on_args_add_star(list, arg)
+    list.push('*' + arg.to_s)
+  end
+
+  def on_args_prepend(list, args)
+    list.prepend args
+    list
+  end
+
+  def on_method_add_arg(m, arg)
+    if arg == nil
+      arg = on_args_new
+    end
+    m.children.push arg
+    m
+  end
+
+  def on_method_add_block(m, b)
+    on_args_add_block(m.children, b)
+    m
+  end
+
+  def on_assoc_new(a, b)
+    Node.new('assoc', a, b)
+  end
+
+  def on_bare_assoc_hash(assoc_list)
+    Node.new('assocs', *assoc_list)
+  end
+
+  def on_assoclist_from_args(a)
+    Node.new('assocs', *a.list)
+  end
+
+  ######## untested
+
+  def on_array(a)
+    Node.new('array', a)
+  end
+
+  def on_assign(a, b)
+    Node.new('assign', a, b)
+  end
+
+  def on_assign_error(a)
+    Node.new('assign_error', a)
+  end
+
+  def on_begin(a)
+    Node.new('begin', a)
+  end
+
+  def on_binary(a, b, c)
+    Node.new('binary', a, b, c)
+  end
+
+  def on_block_var(a)
+    Node.new('block_var', a)
+  end
+
+  def on_bodystmt(a, b, c, d)
+    Node.new('bodystmt', a, b, c, d)
+  end
+
+  def on_brace_block(a, b)
+    Node.new('brace_block', a, b)
+  end
+
+  def on_break(a)
+    Node.new('break', a)
+  end
+
+  def on_call(a, b, c)
+    Node.new('call', a, b, c)
+  end
+
+  def on_case(a, b)
+    Node.new('case', a, b)
+  end
+
+  def on_class(a, b, c)
+    Node.new('class', a, b, c)
+  end
+
+  def on_class_name_error(a)
+    Node.new('class_name_error', a)
+  end
+
+  def on_command(a, b)
+    Node.new('command', a, b)
+  end
+
+  def on_command_call(a, b, c, d)
+    Node.new('command_call', a, b, c, d)
+  end
+
+  def on_const_ref(a)
+    Node.new('const_ref', a)
+  end
+
+  def on_constpath_field(a, b)
+    Node.new('constpath_field', a, b)
+  end
+
+  def on_constpath_ref(a, b)
+    Node.new('constpath_ref', a, b)
+  end
+
+  def on_def(a, b, c)
+    Node.new('def', a, b, c)
+  end
+
+  def on_defined(a)
+    Node.new('defined', a)
+  end
+
+  def on_defs(a, b, c, d, e)
+    Node.new('defs', a, b, c, d, e)
+  end
+
+  def on_do_block(a, b)
+    Node.new('do_block', a, b)
+  end
+
+  def on_dot2(a, b)
+    Node.new('dot2', a, b)
+  end
+
+  def on_dot3(a, b)
+    Node.new('dot3', a, b)
+  end
+
+  def on_dyna_symbol(a)
+    Node.new('dyna_symbol', a)
+  end
+
+  def on_else(a)
+    Node.new('else', a)
+  end
+
+  def on_elsif(a, b, c)
+    Node.new('elsif', a, b, c)
+  end
+
+  def on_ensure(a)
+    Node.new('ensure', a)
+  end
+
+  def on_fcall(a)
+    Node.new('fcall', a)
+  end
+
+  def on_field(a, b, c)
+    Node.new('field', a, b, c)
+  end
+
+  def on_for(a, b, c)
+    Node.new('for', a, b, c)
+  end
+
+  def on_hash(a)
+    Node.new('hash', a)
+  end
+
+  def on_if(a, b, c)
+    Node.new('if', a, b, c)
+  end
+
+  def on_if_mod(a, b)
+    Node.new('if_mod', a, b)
+  end
+
+  def on_ifop(a, b, c)
+    Node.new('ifop', a, b, c)
+  end
+
+  def on_iter_block(a, b)
+    Node.new('iter_block', a, b)
+  end
+
+  def on_massign(a, b)
+    Node.new('massign', a, b)
+  end
+
+  def on_mlhs_add(a, b)
+    Node.new('mlhs_add', a, b)
+  end
+
+  def on_mlhs_add_star(a, b)
+    Node.new('mlhs_add_star', a, b)
+  end
+
+  def on_mlhs_new
+    Node.new('mlhs_new')
+  end
+
+  def on_mlhs_paren(a)
+    Node.new('mlhs_paren', a)
+  end
+
+  def on_module(a, b)
+    Node.new('module', a, b)
+  end
+
+  def on_mrhs_add(a, b)
+    Node.new('mrhs_add', a, b)
+  end
+
+  def on_mrhs_add_star(a, b)
+    Node.new('mrhs_add_star', a, b)
+  end
+
+  def on_mrhs_new
+    Node.new('mrhs_new')
+  end
+
+  def on_mrhs_new_from_arglist(a)
+    Node.new('mrhs_new_from_arglist', a)
+  end
+
+  def on_next(a)
+    Node.new('next', a)
+  end
+
+  def on_opassign(a, b, c)
+    Node.new('opassign', a, b, c)
+  end
+
+  def on_param_error(a)
+    Node.new('param_error', a)
+  end
+
+  def on_params(a, b, c, d)
+    Node.new('params', a, b, c, d)
+  end
+
+  def on_paren(a)
+    Node.new('paren', a)
+  end
+
+  def on_parse_error(a)
+    Node.new('parse_error', a)
+  end
+
+  def on_qwords_add(a, b)
+    Node.new('qwords_add', a, b)
+  end
+
+  def on_qwords_new
+    Node.new('qwords_new')
+  end
+
+  def on_redo
+    Node.new('redo')
+  end
+
+  def on_regexp_literal(a)
+    Node.new('regexp_literal', a)
+  end
+
+  def on_rescue(a, b, c, d)
+    Node.new('rescue', a, b, c, d)
+  end
+
+  def on_rescue_mod(a, b)
+    Node.new('rescue_mod', a, b)
+  end
+
+  def on_restparam(a)
+    Node.new('restparam', a)
+  end
+
+  def on_retry
+    Node.new('retry')
+  end
+
+  def on_return(a)
+    Node.new('return', a)
+  end
+
+  def on_return0
+    Node.new('return0')
+  end
+
+  def on_sclass(a, b)
+    Node.new('sclass', a, b)
+  end
+
+  def on_sp(a)
+    Node.new('space', a)
+  end
+
+  def on_string_add(a, b)
+    Node.new('string_add', a, b)
+  end
+
+  def on_string_concat(a, b)
+    Node.new('string_concat', a, b)
+  end
+
+  def on_string_content
+    Node.new('string_content')
+  end
+
+  def on_string_dvar(a)
+    Node.new('string_dvar', a)
+  end
+
+  def on_string_embexpr(a)
+    Node.new('string_embexpr', a)
+  end
+
+  def on_string_literal(a)
+    Node.new('string_literal', a)
+  end
+
+  def on_super(a)
+    Node.new('super', a)
+  end
+
+  def on_symbol(a)
+    Node.new('symbol', a)
+  end
+
+  def on_symbol_literal(a)
+    Node.new('symbol_literal', a)
+  end
+
+  def on_topconst_field(a)
+    Node.new('topconst_field', a)
+  end
+
+  def on_topconst_ref(a)
+    Node.new('topconst_ref', a)
+  end
+
+  def on_unary(a, b)
+    Node.new('unary', a, b)
+  end
+
+  def on_undef(a)
+    Node.new('undef', a)
+  end
+
+  def on_unless(a, b, c)
+    Node.new('unless', a, b, c)
+  end
+
+  def on_unless_mod(a, b)
+    Node.new('unless_mod', a, b)
+  end
+
+  def on_until_mod(a, b)
+    Node.new('until_mod', a, b)
+  end
+
+  def on_var_field(a)
+    Node.new('var_field', a)
+  end
+
+  def on_when(a, b, c)
+    Node.new('when', a, b, c)
+  end
+
+  def on_while(a, b)
+    Node.new('while', a, b)
+  end
+
+  def on_while_mod(a, b)
+    Node.new('while_mod', a, b)
+  end
+
+  def on_word_add(a, b)
+    Node.new('word_add', a, b)
+  end
+
+  def on_word_new
+    Node.new('word_new')
+  end
+
+  def on_words_add(a, b)
+    Node.new('words_add', a, b)
+  end
+
+  def on_words_new
+    Node.new('words_new')
+  end
+
+  def on_xstring_add(a, b)
+    Node.new('xstring_add', a, b)
+  end
+
+  def on_xstring_literal(a)
+    Node.new('xstring_literal', a)
+  end
+
+  def on_xstring_new
+    Node.new('xstring_new')
+  end
+
+  def on_yield(a)
+    Node.new('yield', a)
+  end
+
+  def on_yield0
+    Node.new('yield0')
+  end
+
+  def on_zsuper
+    Node.new('zsuper')
+  end
+
+  def on_backref(a)
+    a
+  end
+  def on_comma(a)
+    a
+  end
+  def on_gvar(a)
+    a
+  end
+  def on_ident(a)
+    a
+  end
+  def on_int(a)
+    a
+  end
+  def on_kw(a)
+    a
+  end
+  def on_lbrace(a)
+    a
+  end
+  def on_rbrace(a)
+    a
+  end
+  def on_lbracket(a)
+    a
+  end
+  def on_rbracket(a)
+    a
+  end
+  def on_lparen(a)
+    a
+  end
+  def on_rparen(a)
+    a
+  end
+  def on_op(a)
+    a
+  end
+  def on_semicolon(a)
+    a
+  end
+end
diff --git a/trunk/test/ripper/test_files.rb b/trunk/test/ripper/test_files.rb
new file mode 100644
index 0000000000..c2d7a50a3f
--- /dev/null
+++ b/trunk/test/ripper/test_files.rb
@@ -0,0 +1,25 @@
+begin
+
+require 'ripper'
+require 'find'
+require 'test/unit'
+
+class TestRipper_Generic < Test::Unit::TestCase
+  SRCDIR = File.dirname(File.dirname(File.dirname(File.expand_path(__FILE__))))
+
+  class Parser < Ripper
+    PARSER_EVENTS.each {|n| eval "def on_#{n}(*args) r = [:#{n}, *args]; r.inspect; Object.new end" }
+    SCANNER_EVENTS.each {|n| eval "def on_#{n}(*args) r = [:#{n}, *args]; r.inspect; Object.new end" }
+  end
+
+  def test_parse_files
+    Find.find("#{SRCDIR}/lib", "#{SRCDIR}/ext", "#{SRCDIR}/sample", "#{SRCDIR}/test") {|n|
+      next if /\.rb\z/ !~ n || !File.file?(n)
+      assert_nothing_raised("ripper failed to parse: #{n.inspect}") { Parser.new(File.read(n)).parse }
+    }
+  end
+end
+
+rescue LoadError
+end
+  
diff --git a/trunk/test/ripper/test_parser_events.rb b/trunk/test/ripper/test_parser_events.rb
new file mode 100644
index 0000000000..fa640a37ec
--- /dev/null
+++ b/trunk/test/ripper/test_parser_events.rb
@@ -0,0 +1,498 @@
+begin
+
+require 'dummyparser'
+require 'test/unit'
+
+class TestRipper_ParserEvents < Test::Unit::TestCase
+
+  # should be enabled
+=begin
+  def test_event_coverage
+    dispatched = Ripper::PARSER_EVENTS.map {|event,*| event }
+    dispatched.each do |e|
+      assert_equal true, respond_to?("test_#{e}", true),
+                   "event not tested: #{e.inspect}"
+    end
+  end
+=end
+
+  def parse(str)
+    DummyParser.new(str).parse.to_s
+  end
+
+  $thru_program = false
+
+  def test_program
+    assert_equal '[void()]', parse('')
+    assert_equal true, $thru_program
+  end
+
+  def test_stmts_new
+    assert_equal '[void()]', parse('')
+  end
+
+  def test_stmts_add
+    assert_equal '[ref(nil)]', parse('nil')
+    assert_equal '[ref(nil),ref(nil)]', parse('nil;nil')
+    assert_equal '[ref(nil),ref(nil),ref(nil)]', parse('nil;nil;nil')
+  end
+
+  def test_void_stmt
+    assert_equal '[void()]', parse('')
+    assert_equal '[void()]', parse('; ;')
+  end
+
+  def test_var_ref
+    assert_equal '[ref(a)]', parse('a')
+    assert_equal '[ref(nil)]', parse('nil')
+    assert_equal '[ref(true)]', parse('true')
+  end
+
+  def test_BEGIN
+    assert_equal '[BEGIN([void()])]', parse('BEGIN{}')
+    assert_equal '[BEGIN([ref(nil)])]', parse('BEGIN{nil}')
+  end
+
+  def test_END
+    assert_equal '[END([void()])]', parse('END{}')
+    assert_equal '[END([ref(nil)])]', parse('END{nil}')
+  end
+
+  def test_alias
+    assert_equal '[alias(symbol_literal(a),symbol_literal(b))]', parse('alias a b')
+  end
+
+  def test_var_alias
+    assert_equal '[valias($a,$g)]', parse('alias $a $g')
+  end
+
+  def test_alias_error
+    assert_equal '[aliaserr(valias($a,$1))]', parse('alias $a $1')
+  end
+
+  def test_arglist
+    assert_equal '[fcall(m,[])]', parse('m()')
+    assert_equal '[fcall(m,[1])]', parse('m(1)')
+    assert_equal '[fcall(m,[1,2])]', parse('m(1,2)')
+    assert_equal '[fcall(m,[*ref(r)])]', parse('m(*r)')
+    assert_equal '[fcall(m,[1,*ref(r)])]', parse('m(1,*r)')
+    assert_equal '[fcall(m,[1,2,*ref(r)])]', parse('m(1,2,*r)')
+    assert_equal '[fcall(m,[&ref(r)])]', parse('m(&r)')
+    assert_equal '[fcall(m,[1,&ref(r)])]', parse('m(1,&r)')
+    assert_equal '[fcall(m,[1,2,&ref(r)])]', parse('m(1,2,&r)')
+    assert_equal '[fcall(m,[*ref(a),&ref(b)])]', parse('m(*a,&b)')
+    assert_equal '[fcall(m,[1,*ref(a),&ref(b)])]', parse('m(1,*a,&b)')
+    assert_equal '[fcall(m,[1,2,*ref(a),&ref(b)])]', parse('m(1,2,*a,&b)')
+  end
+
+  def test_arg_paren
+    # FIXME
+  end
+
+  def test_aref
+    assert_equal '[aref(ref(v),[1])]', parse('v[1]')
+    assert_equal '[aref(ref(v),[1,2])]', parse('v[1,2]')
+  end
+
+  def test_assocs
+    assert_equal '[fcall(m,[assocs(assoc(1,2))])]', parse('m(1=>2)')
+    assert_equal '[fcall(m,[assocs(assoc(1,2),assoc(3,4))])]', parse('m(1=>2,3=>4)')
+    assert_equal '[fcall(m,[3,assocs(assoc(1,2))])]', parse('m(3,1=>2)')
+  end
+
+  def test_aref_field
+    assert_equal '[assign(aref_field(ref(a),[1]),2)]', parse('a[1]=2')
+  end
+
+=begin
+  def test_arg_ambiguous
+    assert_equal true, $thru__arg_ambiguous
+  end
+=end
+
+  def test_array   # array literal
+    assert_equal '[array([1,2,3])]', parse('[1,2,3]')
+  end
+
+  def test_assign   # generic assignment
+    assert_equal '[assign(var_field(v),1)]', parse('v=1')
+  end
+
+=begin
+  def test_assign_error
+    assert_equal true, $thru__assign_error
+  end
+
+  def test_begin
+    assert_equal true, $thru__begin
+  end
+
+  def test_binary
+    assert_equal true, $thru__binary
+  end
+
+  def test_block_var
+    assert_equal true, $thru__block_var
+  end
+
+  def test_bodystmt
+    assert_equal true, $thru__bodystmt
+  end
+
+  def test_brace_block
+    assert_equal true, $thru__brace_block
+  end
+
+  def test_break
+    assert_equal true, $thru__break
+  end
+
+  def test_call
+    assert_equal true, $thru__call
+  end
+
+  def test_case
+    assert_equal true, $thru__case
+  end
+
+  def test_class
+    assert_equal true, $thru__class
+  end
+
+  def test_class_name_error
+    assert_equal true, $thru__class_name_error
+  end
+
+  def test_command
+    assert_equal true, $thru__command
+  end
+
+  def test_command_call
+    assert_equal true, $thru__command_call
+  end
+
+  def test_const_ref
+    assert_equal true, $thru__const_ref
+  end
+
+  def test_constpath_field
+    assert_equal true, $thru__constpath_field
+  end
+
+  def test_constpath_ref
+    assert_equal true, $thru__constpath_ref
+  end
+
+  def test_def
+    assert_equal true, $thru__def
+  end
+
+  def test_defined
+    assert_equal true, $thru__defined
+  end
+
+  def test_defs
+    assert_equal true, $thru__defs
+  end
+
+  def test_do_block
+    assert_equal true, $thru__do_block
+  end
+
+  def test_dot2
+    assert_equal true, $thru__dot2
+  end
+
+  def test_dot3
+    assert_equal true, $thru__dot3
+  end
+
+  def test_dyna_symbol
+    assert_equal true, $thru__dyna_symbol
+  end
+
+  def test_else
+    assert_equal true, $thru__else
+  end
+
+  def test_elsif
+    assert_equal true, $thru__elsif
+  end
+
+  def test_ensure
+    assert_equal true, $thru__ensure
+  end
+
+  def test_fcall
+    assert_equal true, $thru__fcall
+  end
+
+  def test_field
+    assert_equal true, $thru__field
+  end
+
+  def test_for
+    assert_equal true, $thru__for
+  end
+
+  def test_hash
+    assert_equal true, $thru__hash
+  end
+
+  def test_if
+    assert_equal true, $thru__if
+  end
+
+  def test_if_mod
+    assert_equal true, $thru__if_mod
+  end
+
+  def test_ifop
+    assert_equal true, $thru__ifop
+  end
+
+  def test_iter_block
+    assert_equal true, $thru__iter_block
+  end
+
+  def test_massign
+    assert_equal true, $thru__massign
+  end
+
+  def test_method_add_arg
+    assert_equal true, $thru__method_add_arg
+  end
+
+  def test_mlhs_add
+    assert_equal true, $thru__mlhs_add
+  end
+
+  def test_mlhs_add_star
+    assert_equal true, $thru__mlhs_add_star
+  end
+
+  def test_mlhs_new
+    assert_equal true, $thru__mlhs_new
+  end
+
+  def test_mlhs_paren
+    assert_equal true, $thru__mlhs_paren
+  end
+
+  def test_module
+    assert_equal true, $thru__module
+  end
+
+  def test_mrhs_add
+    assert_equal true, $thru__mrhs_add
+  end
+
+  def test_mrhs_add_star
+    assert_equal true, $thru__mrhs_add_star
+  end
+
+  def test_mrhs_new
+    assert_equal true, $thru__mrhs_new
+  end
+
+  def test_mrhs_new_from_arglist
+    assert_equal true, $thru__mrhs_new_from_arglist
+  end
+
+  def test_next
+    assert_equal true, $thru__next
+  end
+
+  def test_opassign
+    assert_equal true, $thru__opassign
+  end
+
+  def test_param_error
+    assert_equal true, $thru__param_error
+  end
+
+  def test_params
+    assert_equal true, $thru__params
+  end
+
+  def test_paren
+    assert_equal true, $thru__paren
+  end
+
+  def test_parse_error
+    assert_equal true, $thru__parse_error
+  end
+
+  def test_qwords_add
+    assert_equal true, $thru__qwords_add
+  end
+
+  def test_qwords_new
+    assert_equal true, $thru__qwords_new
+  end
+
+  def test_redo
+    assert_equal true, $thru__redo
+  end
+
+  def test_regexp_literal
+    assert_equal true, $thru__regexp_literal
+  end
+
+  def test_rescue
+    assert_equal true, $thru__rescue
+  end
+
+  def test_rescue_mod
+    assert_equal true, $thru__rescue_mod
+  end
+
+  def test_restparam
+    assert_equal true, $thru__restparam
+  end
+
+  def test_retry
+    assert_equal true, $thru__retry
+  end
+
+  def test_return
+    assert_equal true, $thru__return
+  end
+
+  def test_return0
+    assert_equal true, $thru__return0
+  end
+
+  def test_sclass
+    assert_equal true, $thru__sclass
+  end
+
+  def test_space
+    assert_equal true, $thru__space
+  end
+
+  def test_string_add
+    assert_equal true, $thru__string_add
+  end
+
+  def test_string_concat
+    assert_equal true, $thru__string_concat
+  end
+
+  def test_string_content
+    assert_equal true, $thru__string_content
+  end
+
+  def test_string_dvar
+    assert_equal true, $thru__string_dvar
+  end
+
+  def test_string_embexpr
+    assert_equal true, $thru__string_embexpr
+  end
+
+  def test_string_literal
+    assert_equal true, $thru__string_literal
+  end
+
+  def test_super
+    assert_equal true, $thru__super
+  end
+
+  def test_symbol
+    assert_equal true, $thru__symbol
+  end
+
+  def test_symbol_literal
+    assert_equal true, $thru__symbol_literal
+  end
+
+  def test_topconst_field
+    assert_equal true, $thru__topconst_field
+  end
+
+  def test_topconst_ref
+    assert_equal true, $thru__topconst_ref
+  end
+
+  def test_unary
+    assert_equal true, $thru__unary
+  end
+
+  def test_undef
+    assert_equal true, $thru__undef
+  end
+
+  def test_unless
+    assert_equal true, $thru__unless
+  end
+
+  def test_unless_mod
+    assert_equal true, $thru__unless_mod
+  end
+
+  def test_until_mod
+    assert_equal true, $thru__until_mod
+  end
+
+  def test_var_field
+    assert_equal true, $thru__var_field
+  end
+
+  def test_when
+    assert_equal true, $thru__when
+  end
+
+  def test_while
+    assert_equal true, $thru__while
+  end
+
+  def test_while_mod
+    assert_equal true, $thru__while_mod
+  end
+
+  def test_word_add
+    assert_equal true, $thru__word_add
+  end
+
+  def test_word_new
+    assert_equal true, $thru__word_new
+  end
+
+  def test_words_add
+    assert_equal true, $thru__words_add
+  end
+
+  def test_words_new
+    assert_equal true, $thru__words_new
+  end
+
+  def test_xstring_add
+    assert_equal true, $thru__xstring_add
+  end
+
+  def test_xstring_literal
+    assert_equal true, $thru__xstring_literal
+  end
+
+  def test_xstring_new
+    assert_equal true, $thru__xstring_new
+  end
+
+  def test_yield
+    assert_equal true, $thru__yield
+  end
+
+  def test_yield0
+    assert_equal true, $thru__yield0
+  end
+
+  def test_zsuper
+    assert_equal true, $thru__zsuper
+  end
+=end
+
+end
+
+rescue LoadError
+end
diff --git a/trunk/test/ripper/test_scanner_events.rb b/trunk/test/ripper/test_scanner_events.rb
new file mode 100644
index 0000000000..4389946104
--- /dev/null
+++ b/trunk/test/ripper/test_scanner_events.rb
@@ -0,0 +1,807 @@
+#
+# test_scanner_events.rb
+#
+begin
+
+require 'ripper'
+require 'test/unit'
+
+class TestRipper_ScannerEvents < Test::Unit::TestCase
+
+  def test_event_coverage
+    dispatched = Ripper::SCANNER_EVENTS.map {|event,_| event }
+    dispatched.each do |e|
+      assert_equal true, respond_to?("test_#{e}", true), "event not tested: #{e}"
+    end
+  end
+
+  def scan(target, str)
+    sym = "on_#{target}".intern
+    Ripper.lex(str).select {|_1,type,_2| type == sym }.map {|_1,_2,tok| tok }
+  end
+
+  def test_tokenize
+    assert_equal [],
+                 Ripper.tokenize('')
+    assert_equal ['a'],
+                 Ripper.tokenize('a')
+    assert_equal ['1'],
+                 Ripper.tokenize('1')
+    assert_equal ['1', ';', 'def', ' ', 'm', '(', 'arg', ')', 'end'],
+                 Ripper.tokenize("1;def m(arg)end")
+    assert_equal ['print', '(', '<<EOS', ')', "\n", "heredoc\n", "EOS\n"],
+                 Ripper.tokenize("print(<<EOS)\nheredoc\nEOS\n")
+    assert_equal ['print', '(', ' ', '<<EOS', ')', "\n", "heredoc\n", "EOS\n"],
+                 Ripper.tokenize("print( <<EOS)\nheredoc\nEOS\n")
+    assert_equal ["\#\n", "\n", "\#\n", "\n", "nil", "\n"],
+                 Ripper.tokenize("\#\n\n\#\n\nnil\n")
+  end
+
+  def test_lex
+    assert_equal [],
+                 Ripper.lex('')
+    assert_equal [[[1,0], :on_ident, "a"]],
+                 Ripper.lex('a')
+    assert_equal [[[1, 0], :on_kw, "nil"]],
+                 Ripper.lex("nil")
+    assert_equal [[[1, 0], :on_kw, "def"],
+                  [[1, 3], :on_sp, " "],
+                  [[1, 4], :on_ident, "m"],
+                  [[1, 5], :on_lparen, "("],
+                  [[1, 6], :on_ident, "a"],
+                  [[1, 7], :on_rparen, ")"],
+                  [[1, 8], :on_kw, "end"]],
+                 Ripper.lex("def m(a)end")
+    assert_equal [[[1, 0], :on_int, "1"],
+                  [[1, 1], :on_nl, "\n"],
+                  [[2, 0], :on_int, "2"],
+                  [[2, 1], :on_nl, "\n"],
+                  [[3, 0], :on_int, "3"]],
+                 Ripper.lex("1\n2\n3")
+    assert_equal [[[1, 0], :on_heredoc_beg, "<<EOS"],
+                  [[1, 5], :on_nl, "\n"],
+                  [[2, 0], :on_tstring_content, "heredoc\n"],
+                  [[3, 0], :on_heredoc_end, "EOS"]],
+                 Ripper.lex("<<EOS\nheredoc\nEOS")
+  end
+
+  def test_location
+    validate_location ""
+    validate_location " "
+    validate_location "@"
+    validate_location "\n"
+    validate_location "\r\n"
+    validate_location "\n\n\n\n\n\r\n\n\n"
+    validate_location "\n;\n;\n;\n;\n"
+    validate_location "nil"
+    validate_location "@ivar"
+    validate_location "1;2;3"
+    validate_location "1\n2\n3"
+    validate_location "1\n2\n3\n"
+    validate_location "def m(a) nil end"
+    validate_location "if true then false else nil end"
+    validate_location "BEGIN{print nil}"
+    validate_location "%w(a b\nc\r\nd \ne )"
+    validate_location %Q["a\nb\r\nc"]
+    validate_location "print(<<EOS)\nheredoc\nEOS\n"
+    validate_location %Q[print(<<-"EOS")\nheredoc\n     EOS\n]
+  end
+
+  def validate_location(src)
+    buf = ''
+    Ripper.lex(src).each do |pos, type, tok|
+      line, col = *pos
+      assert_equal buf.count("\n") + 1, line,
+          "wrong lineno: #{tok.inspect} (#{type}) [#{line}:#{col}]"
+      assert_equal buf.sub(/\A.*\n/m, '').size, col,
+          "wrong column: #{tok.inspect} (#{type}) [#{line}:#{col}]"
+      buf << tok
+    end
+    assert_equal src, buf
+  end
+
+  def test_backref
+    assert_equal ["$`", "$&", "$'", '$1', '$2', '$3'],
+                 scan('backref', %q[m($~, $`, $&, $', $1, $2, $3)])
+  end
+
+  def test_backtick
+    assert_equal ["`"],
+                 scan('backtick', %q[p `make all`])
+  end
+
+  def test_comma
+    assert_equal [','] * 6,
+                 scan('comma', %q[ m(0,1,2,3,4,5,6) ])
+    assert_equal [],
+                 scan('comma', %q[".,.,.,.,.,.,.."])
+    assert_equal [],
+                 scan('comma', %Q[<<EOS\n,,,,,,,,,,\nEOS])
+  end
+
+  def test_period
+    assert_equal [],
+                 scan('period', '')
+    assert_equal ['.'],
+                 scan('period', 'a.b')
+    assert_equal ['.'],
+                 scan('period', 'Object.new')
+    assert_equal [],
+                 scan('period', '"."')
+    assert_equal [],
+                 scan('period', '1..2')
+    assert_equal [],
+                 scan('period', '1...3')
+  end
+
+  def test_const
+    assert_equal ['CONST'],
+                 scan('const', 'CONST')
+    assert_equal ['C'],
+                 scan('const', 'C')
+    assert_equal ['CONST_A'],
+                 scan('const', 'CONST_A')
+    assert_equal ['Const', 'Const2', 'Const3'],
+                 scan('const', 'Const; Const2; Const3')
+    assert_equal ['Const'],
+                 scan('const', 'Const(a)')
+    assert_equal ['M', 'A', 'A2'],
+                 scan('const', 'M(A,A2)')
+    assert_equal [],
+                 scan('const', '')
+    assert_equal [],
+                 scan('const', 'm(lvar, @ivar, @@cvar, $gvar)')
+  end
+
+  def test_cvar
+    assert_equal [],
+                 scan('cvar', '')
+    assert_equal ['@@cvar'],
+                 scan('cvar', '@@cvar')
+    assert_equal ['@@__cvar__'],
+                 scan('cvar', '@@__cvar__')
+    assert_equal ['@@CVAR'],
+                 scan('cvar', '@@CVAR')
+    assert_equal ['@@cvar'],
+                 scan('cvar', '   @@cvar#comment')
+    assert_equal ['@@cvar'],
+                 scan('cvar', ':@@cvar')
+    assert_equal ['@@cvar'],
+                 scan('cvar', 'm(lvar, @ivar, @@cvar, $gvar)')
+    assert_equal [],
+                 scan('cvar', '"@@cvar"')
+  end
+
+  def test_embexpr_beg
+    assert_equal [],
+                 scan('embexpr_beg', '')
+    assert_equal ['#{'],
+                 scan('embexpr_beg', '"#{expr}"')
+    assert_equal [],
+                 scan('embexpr_beg', '%q[#{expr}]')
+    assert_equal ['#{'],
+                 scan('embexpr_beg', '%Q[#{expr}]')
+    assert_equal ['#{'],
+                 scan('embexpr_beg', "m(<<EOS)\n\#{expr}\nEOS")
+  end
+
+  def test_embexpr_end
+=begin
+    # currently detected as "rbrace"
+    assert_equal [],
+                 scan('embexpr_end', '')
+    assert_equal ['}'],
+                 scan('embexpr_end', '"#{expr}"')
+    assert_equal [],
+                 scan('embexpr_end', '%q[#{expr}]')
+    assert_equal ['}'],
+                 scan('embexpr_end', '%Q[#{expr}]')
+    assert_equal ['}'],
+                 scan('embexpr_end', "m(<<EOS)\n\#{expr}\nEOS")
+=end
+  end
+
+  def test_embvar
+    assert_equal [],
+                 scan('embvar', '')
+    assert_equal ['#'],
+                 scan('embvar', '"#$gvar"')
+    assert_equal ['#'],
+                 scan('embvar', '"#@ivar"')
+    assert_equal ['#'],
+                 scan('embvar', '"#@@cvar"')
+    assert_equal [],
+                 scan('embvar', '"#lvar"')
+    assert_equal [],
+                 scan('embvar', '"#"')
+    assert_equal [],
+                 scan('embvar', '"\#$gvar"')
+    assert_equal [],
+                 scan('embvar', '"\#@ivar"')
+    assert_equal [],
+                 scan('embvar', '%q[#@ivar]')
+    assert_equal ['#'],
+                 scan('embvar', '%Q[#@ivar]')
+  end
+
+  def test_float
+    assert_equal [],
+                 scan('float', '')
+    assert_equal ['1.000'],
+                 scan('float', '1.000')
+    assert_equal ['123.456'],
+                 scan('float', '123.456')
+    assert_equal ['1.2345678901234567890123456789'],
+                 scan('float', '1.2345678901234567890123456789')
+    assert_equal ['1.000'],
+                 scan('float', '   1.000# comment')
+    assert_equal ['1.234e5'],
+                 scan('float', '1.234e5')
+    assert_equal ['1.234e1234567890'],
+                 scan('float', '1.234e1234567890')
+    assert_equal ['1.0'],
+                 scan('float', 'm(a,b,1.0,c,d)')
+  end
+
+  def test_gvar
+    assert_equal [],
+                 scan('gvar', '')
+    assert_equal ['$a'],
+                 scan('gvar', '$a')
+    assert_equal ['$A'],
+                 scan('gvar', '$A')
+    assert_equal ['$gvar'],
+                 scan('gvar', 'm(lvar, @ivar, @@cvar, $gvar)')
+    assert_equal %w($_ $~ $* $$ $? $! $@ $/ $\\ $; $, $. $= $: $< $> $"),
+                 scan('gvar', 'm($_, $~, $*, $$, $?, $!, $@, $/, $\\, $;, $,, $., $=, $:, $<, $>, $")')
+  end
+
+  def test_ident
+    assert_equal [],
+                 scan('ident', '')
+    assert_equal ['lvar'],
+                 scan('ident', 'lvar')
+    assert_equal ['m', 'lvar'],
+                 scan('ident', 'm(lvar, @ivar, @@cvar, $gvar)')
+  end
+
+  def test_int
+    assert_equal [],
+                 scan('int', '')
+    assert_equal ['1', '10', '100000000000000'],
+                 scan('int', 'm(1,10,100000000000000)')
+  end
+
+  def test_ivar
+    assert_equal [],
+                 scan('ivar', '')
+    assert_equal ['@ivar'],
+                 scan('ivar', '@ivar')
+    assert_equal ['@__ivar__'],
+                 scan('ivar', '@__ivar__')
+    assert_equal ['@IVAR'],
+                 scan('ivar', '@IVAR')
+    assert_equal ['@ivar'],
+                 scan('ivar', 'm(lvar, @ivar, @@cvar, $gvar)')
+  end
+
+  def test_kw
+    assert_equal [],
+                 scan('kw', '')
+    assert_equal %w(not),
+                 scan('kw', 'not 1')
+    assert_equal %w(and),
+                 scan('kw', '1 and 2')
+    assert_equal %w(or),
+                 scan('kw', '1 or 2')
+    assert_equal %w(if then else end),
+                 scan('kw', 'if 1 then 2 else 3 end')
+    assert_equal %w(if then elsif else end),
+                 scan('kw', 'if 1 then 2 elsif 3 else 4 end')
+    assert_equal %w(unless then end),
+                 scan('kw', 'unless 1 then end')
+    assert_equal %w(if true),
+                 scan('kw', '1 if true')
+    assert_equal %w(unless false),
+                 scan('kw', '2 unless false')
+    assert_equal %w(case when when else end),
+                 scan('kw', 'case n; when 1; when 2; else 3 end')
+    assert_equal %w(while do nil end),
+                 scan('kw', 'while 1 do nil end')
+    assert_equal %w(until do nil end),
+                 scan('kw', 'until 1 do nil end')
+    assert_equal %w(while),
+                 scan('kw', '1 while 2')
+    assert_equal %w(until),
+                 scan('kw', '1 until 2')
+    assert_equal %w(while break next retry end),
+                 scan('kw', 'while 1; break; next; retry end')
+    assert_equal %w(for in next break end),
+                 scan('kw', 'for x in obj; next 1; break 2 end')
+    assert_equal %w(begin rescue retry end),
+                 scan('kw', 'begin 1; rescue; retry; end')
+    assert_equal %w(rescue),
+                 scan('kw', '1 rescue 2')
+    assert_equal %w(def redo return end),
+                 scan('kw', 'def m() redo; return end')
+    assert_equal %w(def yield yield end),
+                 scan('kw', 'def m() yield; yield 1 end')
+    assert_equal %w(def super super super end),
+                 scan('kw', 'def m() super; super(); super(1) end')
+    assert_equal %w(alias),
+                 scan('kw', 'alias a b')
+    assert_equal %w(undef),
+                 scan('kw', 'undef public')
+    assert_equal %w(class end),
+                 scan('kw', 'class A < Object; end')
+    assert_equal %w(module end),
+                 scan('kw', 'module M; end')
+    assert_equal %w(class end),
+                 scan('kw', 'class << obj; end')
+    assert_equal %w(BEGIN),
+                 scan('kw', 'BEGIN { }')
+    assert_equal %w(END),
+                 scan('kw', 'END { }')
+    assert_equal %w(self),
+                 scan('kw', 'self.class')
+    assert_equal %w(nil true false),
+                 scan('kw', 'p(nil, true, false)')
+    assert_equal %w(__FILE__ __LINE__),
+                 scan('kw', 'p __FILE__, __LINE__')
+    assert_equal %w(defined?),
+                 scan('kw', 'defined?(Object)')
+  end
+
+  def test_lbrace
+    assert_equal [],
+                 scan('lbrace', '')
+    assert_equal ['{'],
+                 scan('lbrace', '3.times{ }')
+    assert_equal ['{'],
+                 scan('lbrace', '3.times  { }')
+    assert_equal ['{'],
+                 scan('lbrace', '3.times{}')
+    assert_equal [],
+                 scan('lbrace', '"{}"')
+    assert_equal ['{'],
+                 scan('lbrace', '{1=>2}')
+  end
+
+  def test_rbrace
+    assert_equal [],
+                 scan('rbrace', '')
+    assert_equal ['}'],
+                 scan('rbrace', '3.times{ }')
+    assert_equal ['}'],
+                 scan('rbrace', '3.times  { }')
+    assert_equal ['}'],
+                 scan('rbrace', '3.times{}')
+    assert_equal [],
+                 scan('rbrace', '"{}"')
+    assert_equal ['}'],
+                 scan('rbrace', '{1=>2}')
+  end
+
+  def test_lbracket
+    assert_equal [],
+                 scan('lbracket', '')
+    assert_equal ['['],
+                 scan('lbracket', '[]')
+    assert_equal ['['],
+                 scan('lbracket', 'a[1]')
+    assert_equal [],
+                 scan('lbracket', 'm(%q[])')
+  end
+
+  def test_rbracket
+    assert_equal [],
+                 scan('rbracket', '')
+    assert_equal [']'],
+                 scan('rbracket', '[]')
+    assert_equal [']'],
+                 scan('rbracket', 'a[1]')
+    assert_equal [],
+                 scan('rbracket', 'm(%q[])')
+  end
+
+  def test_lparen
+    assert_equal [],
+                 scan('lparen', '')
+    assert_equal ['('],
+                 scan('lparen', '()')
+    assert_equal ['('],
+                 scan('lparen', 'm()')
+    assert_equal ['('],
+                 scan('lparen', 'm (a)')
+    assert_equal [],
+                 scan('lparen', '"()"')
+    assert_equal [],
+                 scan('lparen', '"%w()"')
+  end
+
+  def test_rparen
+    assert_equal [],
+                 scan('rparen', '')
+    assert_equal [')'],
+                 scan('rparen', '()')
+    assert_equal [')'],
+                 scan('rparen', 'm()')
+    assert_equal [')'],
+                 scan('rparen', 'm (a)')
+    assert_equal [],
+                 scan('rparen', '"()"')
+    assert_equal [],
+                 scan('rparen', '"%w()"')
+  end
+
+  def test_op
+    assert_equal [],
+                 scan('op', '')
+    assert_equal ['|'],
+                 scan('op', '1 | 1')
+    assert_equal ['^'],
+                 scan('op', '1 ^ 1')
+    assert_equal ['&'],
+                 scan('op', '1 & 1')
+    assert_equal ['<=>'],
+                 scan('op', '1 <=> 1')
+    assert_equal ['=='],
+                 scan('op', '1 == 1')
+    assert_equal ['==='],
+                 scan('op', '1 === 1')
+    assert_equal ['=~'],
+                 scan('op', '1 =~ 1')
+    assert_equal ['>'],
+                 scan('op', '1 > 1')
+    assert_equal ['>='],
+                 scan('op', '1 >= 1')
+    assert_equal ['<'],
+                 scan('op', '1 < 1')
+    assert_equal ['<='],
+                 scan('op', '1 <= 1')
+    assert_equal ['<<'],
+                 scan('op', '1 << 1')
+    assert_equal ['>>'],
+                 scan('op', '1 >> 1')
+    assert_equal ['+'],
+                 scan('op', '1 + 1')
+    assert_equal ['-'],
+                 scan('op', '1 - 1')
+    assert_equal ['*'],
+                 scan('op', '1 * 1')
+    assert_equal ['/'],
+                 scan('op', '1 / 1')
+    assert_equal ['%'],
+                 scan('op', '1 % 1')
+    assert_equal ['**'],
+                 scan('op', '1 ** 1')
+    assert_equal ['~'],
+                 scan('op', '~1')
+    assert_equal ['-'],
+                 scan('op', '-a')
+    assert_equal ['+'],
+                 scan('op', '+a')
+    assert_equal ['[]'],
+                 scan('op', ':[]')
+    assert_equal ['[]='],
+                 scan('op', ':[]=')
+    assert_equal [],
+                 scan('op', %q[`make all`])
+  end
+
+  def test_symbeg
+    assert_equal [],
+                 scan('symbeg', '')
+    assert_equal [':'],
+                 scan('symbeg', ':sym')
+    assert_equal [':'],
+                 scan('symbeg', '[1,2,3,:sym]')
+    assert_equal [],
+                 scan('symbeg', '":sym"')
+    assert_equal [],
+                 scan('symbeg', 'a ? b : c')
+  end
+
+  def test_tstring_beg
+    assert_equal [],
+                 scan('tstring_beg', '')
+    assert_equal ['"'],
+                 scan('tstring_beg', '"abcdef"')
+    assert_equal ['%q['],
+                 scan('tstring_beg', '%q[abcdef]')
+    assert_equal ['%Q['],
+                 scan('tstring_beg', '%Q[abcdef]')
+  end
+
+  def test_tstring_content
+    assert_equal [],
+                 scan('tstring_content', '')
+    assert_equal ['abcdef'],
+                 scan('tstring_content', '"abcdef"')
+    assert_equal ['abcdef'],
+                 scan('tstring_content', '%q[abcdef]')
+    assert_equal ['abcdef'],
+                 scan('tstring_content', '%Q[abcdef]')
+    assert_equal ['abc', 'def'],
+                 scan('tstring_content', '"abc#{1}def"')
+    assert_equal ['sym'],
+                 scan('tstring_content', ':"sym"')
+  end
+
+  def test_tstring_end
+    assert_equal [],
+                 scan('tstring_end', '')
+    assert_equal ['"'],
+                 scan('tstring_end', '"abcdef"')
+    assert_equal [']'],
+                 scan('tstring_end', '%q[abcdef]')
+    assert_equal [']'],
+                 scan('tstring_end', '%Q[abcdef]')
+  end
+
+  def test_regexp_beg
+    assert_equal [],
+                 scan('regexp_beg', '')
+    assert_equal ['/'],
+                 scan('regexp_beg', '/re/')
+    assert_equal ['%r<'],
+                 scan('regexp_beg', '%r<re>')
+    assert_equal [],
+                 scan('regexp_beg', '5 / 5')
+  end
+
+  def test_regexp_end
+    assert_equal [],
+                 scan('regexp_end', '')
+    assert_equal ['/'],
+                 scan('regexp_end', '/re/')
+    assert_equal ['>'],
+                 scan('regexp_end', '%r<re>')
+  end
+
+  def test_words_beg
+    assert_equal [],
+                 scan('words_beg', '')
+    assert_equal ['%W('],
+                 scan('words_beg', '%W()')
+    assert_equal ['%W('],
+                 scan('words_beg', '%W(w w w)')
+    assert_equal ['%W( '],
+                 scan('words_beg', '%W( w w w )')
+  end
+
+  def test_qwords_beg
+    assert_equal [],
+                 scan('qwords_beg', '')
+    assert_equal ['%w('],
+                 scan('qwords_beg', '%w()')
+    assert_equal ['%w('],
+                 scan('qwords_beg', '%w(w w w)')
+    assert_equal ['%w( '],
+                 scan('qwords_beg', '%w( w w w )')
+  end
+
+  # FIXME: Close paren must not present (`words_end' scanner event?).
+  def test_words_sep
+    assert_equal [],
+                 scan('words_sep', '')
+    assert_equal [')'],
+                 scan('words_sep', '%w()')
+    assert_equal [' ', ' ', ')'],
+                 scan('words_sep', '%w(w w w)')
+    assert_equal [' ', ' ', ' )'],
+                 scan('words_sep', '%w( w w w )')
+    assert_equal ["\n", ' ', ' )'],
+                 scan('words_sep', "%w( w\nw w )")
+  end
+
+  def test_heredoc_beg
+    assert_equal [],
+                 scan('heredoc_beg', '')
+    assert_equal ['<<EOS'],
+                 scan('heredoc_beg', "<<EOS\nheredoc\nEOS")
+    assert_equal ['<<EOS'],
+                 scan('heredoc_beg', "<<EOS\nheredoc\nEOS\n")
+    assert_equal ['<<EOS'],
+                 scan('heredoc_beg', "<<EOS\nheredoc\nEOS \n")
+    assert_equal ['<<-EOS'],
+                 scan('heredoc_beg', "<<-EOS\nheredoc\n\tEOS \n")
+    assert_equal ['<<"EOS"'],
+                 scan('heredoc_beg', %Q[<<"EOS"\nheredoc\nEOS])
+    assert_equal [%q(<<'EOS')],
+                 scan('heredoc_beg', "<<'EOS'\nheredoc\nEOS")
+    assert_equal [%q(<<`EOS`)],
+                 scan('heredoc_beg', "<<`EOS`\nheredoc\nEOS")
+    assert_equal [%q(<<" ")],
+                 scan('heredoc_beg', %Q[<<" "\nheredoc\nEOS])
+  end
+
+  def test_tstring_content_HEREDOC
+    assert_equal [],
+                 scan('tstring_content', '')
+    assert_equal ["heredoc\n"],
+                 scan('tstring_content', "<<EOS\nheredoc\nEOS")
+    assert_equal ["heredoc\n"],
+                 scan('tstring_content', "<<EOS\nheredoc\nEOS\n")
+    assert_equal ["heredoc \n"],
+                 scan('tstring_content', "<<EOS\nheredoc \nEOS \n")
+    assert_equal ["heredoc\n"],
+                 scan('tstring_content', "<<-EOS\nheredoc\n\tEOS \n")
+  end
+
+  def test_heredoc_end
+    assert_equal [],
+                 scan('heredoc_end', '')
+    assert_equal ["EOS"],
+                 scan('heredoc_end', "<<EOS\nheredoc\nEOS")
+    assert_equal ["EOS\n"],
+                 scan('heredoc_end', "<<EOS\nheredoc\nEOS\n")
+    assert_equal ["EOS \n"],
+                 scan('heredoc_end', "<<EOS\nheredoc\nEOS \n")
+    assert_equal ["\tEOS \n"],
+                 scan('heredoc_end', "<<-EOS\nheredoc\n\tEOS \n")
+  end
+
+  def test_semicolon
+    assert_equal [],
+                 scan('semicolon', '')
+    assert_equal %w(;),
+                 scan('semicolon', ';')
+    assert_equal %w(; ;),
+                 scan('semicolon', '; ;')
+    assert_equal %w(; ; ;),
+                 scan('semicolon', 'nil;nil;nil;')
+    assert_equal %w(; ; ;),
+                 scan('semicolon', 'nil;nil;nil;nil')
+    assert_equal [],
+                 scan('semicolon', '";"')
+    assert_equal [],
+                 scan('semicolon', '%w(;)')
+    assert_equal [],
+                 scan('semicolon', '/;/')
+  end
+
+  def test_comment
+    assert_equal [],
+                 scan('comment', '')
+    assert_equal ['# comment'],
+                 scan('comment', '# comment')
+    assert_equal ["# comment\n"],
+                 scan('comment', "# comment\n")
+    assert_equal ["# comment\n"],
+                 scan('comment', "# comment\n1 + 1")
+    assert_equal ["# comment\n"],
+                 scan('comment', "1 + 1 + 1# comment\n1 + 1")
+  end
+
+  def test_embdoc_beg
+    assert_equal [],
+                 scan('embdoc_beg', '')
+    assert_equal ["=begin\n"],
+                 scan('embdoc_beg', "=begin\ndoc\n=end")
+    assert_equal ["=begin \n"],
+                 scan('embdoc_beg', "=begin \ndoc\n=end\n")
+    assert_equal ["=begin comment\n"],
+                 scan('embdoc_beg', "=begin comment\ndoc\n=end\n")
+  end
+
+  def test_embdoc
+    assert_equal [],
+                 scan('embdoc', '')
+    assert_equal ["doc\n"],
+                 scan('embdoc', "=begin\ndoc\n=end")
+    assert_equal ["doc\n"],
+                 scan('embdoc', "=begin\ndoc\n=end\n")
+  end
+
+  def test_embdoc_end
+    assert_equal [],
+                 scan('embdoc_end', '')
+    assert_equal ["=end"],
+                 scan('embdoc_end', "=begin\ndoc\n=end")
+    assert_equal ["=end\n"],
+                 scan('embdoc_end', "=begin\ndoc\n=end\n")
+  end
+
+  def test_sp
+    assert_equal [],
+                 scan('sp', '')
+    assert_equal [' '],
+                 scan('sp', ' ')
+    assert_equal [' '],
+                 scan('sp', ' 1')
+    assert_equal [],
+                 scan('sp', "\n")
+    assert_equal [' '],
+                 scan('sp', " \n")
+    assert_equal [' ', ' '],
+                 scan('sp', "1 + 1")
+    assert_equal [],
+                 scan('sp', "' '")
+    assert_equal [],
+                 scan('sp', "%w(  )")
+    assert_equal [],
+                 scan('sp', "%w(  w  )")
+    assert_equal [],
+                 scan('sp', "p(/ /)")
+  end
+
+  # `nl' event always means End-Of-Statement.
+  def test_nl
+    assert_equal [],
+                 scan('nl', '')
+    assert_equal [],
+                 scan('nl', "\n")
+    assert_equal ["\n"],
+                 scan('nl', "1 + 1\n")
+    assert_equal ["\n", "\n"],
+                 scan('nl', "1 + 1\n2 + 2\n")
+    assert_equal [],
+                 scan('nl', "1 +\n1")
+    assert_equal [],
+                 scan('nl', "1;\n")
+    assert_equal ["\r\n"],
+                 scan('nl', "1 + 1\r\n")
+    assert_equal [],
+                 scan('nl', "1;\r\n")
+  end
+
+  def test_ignored_nl
+    assert_equal [],
+                 scan('ignored_nl', '')
+    assert_equal ["\n"],
+                 scan('ignored_nl', "\n")
+    assert_equal [],
+                 scan('ignored_nl', "1 + 1\n")
+    assert_equal [],
+                 scan('ignored_nl', "1 + 1\n2 + 2\n")
+    assert_equal ["\n"],
+                 scan('ignored_nl', "1 +\n1")
+    assert_equal ["\n"],
+                 scan('ignored_nl', "1;\n")
+    assert_equal [],
+                 scan('ignored_nl', "1 + 1\r\n")
+    assert_equal ["\r\n"],
+                 scan('ignored_nl', "1;\r\n")
+  end
+
+  def test___end__
+    assert_equal [],
+                 scan('__end__', "")
+    assert_equal ["__END__"],
+                 scan('__end__', "__END__")
+    assert_equal ["__END__\n"],
+                 scan('__end__', "__END__\n")
+    assert_equal ["__END__\n"],
+                 Ripper.tokenize("__END__\njunk junk junk")
+    assert_equal ["__END__"],
+                 scan('__end__', "1\n__END__")
+    assert_equal [],
+                 scan('__end__', "print('__END__')")
+  end
+
+  def test_CHAR
+    assert_equal [],
+                 scan('CHAR', "")
+    assert_equal ["@"],
+                 scan('CHAR', "@")
+    assert_equal [],
+                 scan('CHAR', "@ivar")
+  end
+
+  def test_label
+  end
+
+  def test_tlambda
+  end
+
+  def test_tlambeg
+  end
+
+  def test_tlambda_arg
+  end
+
+end
+
+rescue LoadError
+end