Merge rdoc-6.0.0.beta2 from upstream.

  * This version changed lexer used Ripper from lexer based IRB.
    see details: https://github.com/ruby/rdoc/pull/512

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59845 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

8 files changed, 241 insertions, 1185 deletions

diff --git a/test/rdoc/test_rdoc_any_method.rb b/test/rdoc/test_rdoc_any_method.rb
index fdbb62efa6..fdfb0f3995 100644
--- a/test/rdoc/test_rdoc_any_method.rb
+++ b/test/rdoc/test_rdoc_any_method.rb
@@ -74,7 +74,7 @@ method(a, b) { |c, d| ... }
 
   def test_markup_code
     tokens = [
-      RDoc::RubyToken::TkCONSTANT. new(0, 0, 0, 'CONSTANT'),
+      { :line_no => 0, :char_no => 0, :kind => :on_const, :text => 'CONSTANT' },
     ]
 
     @c2_a.collect_tokens
diff --git a/test/rdoc/test_rdoc_constant.rb b/test/rdoc/test_rdoc_constant.rb
index c43aa7dc2b..a03ab97a7d 100644
--- a/test/rdoc/test_rdoc_constant.rb
+++ b/test/rdoc/test_rdoc_constant.rb
@@ -86,7 +86,7 @@ class TestRDocConstant < XrefTestCase
     assert_equal top_level,      loaded.file
     assert_equal 'Klass::CONST', loaded.full_name
     assert_equal 'CONST',        loaded.name
-    assert_nil                   loaded.visibility
+    assert_equal :public,        loaded.visibility
     assert_equal cm,             loaded.parent
     assert_equal section,        loaded.section
   end
@@ -114,7 +114,7 @@ class TestRDocConstant < XrefTestCase
     assert_equal top_level,      loaded.file
     assert_equal 'Klass::CONST', loaded.full_name
     assert_equal 'CONST',        loaded.name
-    assert_nil                   loaded.visibility
+    assert_equal :public,        loaded.visibility
     assert_equal cm,             loaded.parent
     assert_equal section,        loaded.section
 
@@ -146,7 +146,7 @@ class TestRDocConstant < XrefTestCase
     assert_equal top_level,      loaded.file
     assert_equal 'Klass::CONST', loaded.full_name
     assert_equal 'CONST',        loaded.name
-    assert_nil                   loaded.visibility
+    assert_equal :public,        loaded.visibility
     assert_equal cm,             loaded.parent
     assert_equal section,        loaded.section
 
diff --git a/test/rdoc/test_rdoc_context.rb b/test/rdoc/test_rdoc_context.rb
index 54e365f7f4..ceac14ac90 100644
--- a/test/rdoc/test_rdoc_context.rb
+++ b/test/rdoc/test_rdoc_context.rb
@@ -719,6 +719,7 @@ class TestRDocContext < XrefTestCase
 
     assert_equal [@pub, @prot, @priv], @vis.method_list
     assert_equal [@apub, @aprot, @apriv], @vis.attributes
+    assert_equal [@cpub, @cpriv], @vis.constants
   end
 
   def test_remove_invisible_nodoc
@@ -728,6 +729,7 @@ class TestRDocContext < XrefTestCase
 
     assert_equal [@pub, @prot, @priv], @vis.method_list
     assert_equal [@apub, @aprot, @apriv], @vis.attributes
+    assert_equal [@cpub, @cpriv], @vis.constants
   end
 
   def test_remove_invisible_protected
@@ -737,6 +739,7 @@ class TestRDocContext < XrefTestCase
 
     assert_equal [@pub, @prot], @vis.method_list
     assert_equal [@apub, @aprot], @vis.attributes
+    assert_equal [@cpub], @vis.constants
   end
 
   def test_remove_invisible_public
@@ -746,6 +749,7 @@ class TestRDocContext < XrefTestCase
 
     assert_equal [@pub], @vis.method_list
     assert_equal [@apub], @vis.attributes
+    assert_equal [@cpub], @vis.constants
   end
 
   def test_remove_invisible_public_force
@@ -755,11 +759,13 @@ class TestRDocContext < XrefTestCase
     @prot.force_documentation = true
     @apriv.force_documentation = true
     @aprot.force_documentation = true
+    @cpriv.force_documentation = true
 
     @vis.remove_invisible :public
 
     assert_equal [@pub, @prot, @priv], @vis.method_list
     assert_equal [@apub, @aprot, @apriv], @vis.attributes
+    assert_equal [@cpub, @cpriv], @vis.constants
   end
 
   def test_remove_invisible_in_protected
@@ -922,6 +928,9 @@ class TestRDocContext < XrefTestCase
     @aprot = RDoc::Attr.new nil, 'prot', 'RW', nil
     @apriv = RDoc::Attr.new nil, 'priv', 'RW', nil
 
+    @cpub  = RDoc::Constant.new 'CONST_PUBLIC', nil, nil
+    @cpriv = RDoc::Constant.new 'CONST_PRIVATE', nil, nil
+
     @vis = RDoc::NormalClass.new 'Vis'
     @vis.add_method @pub
     @vis.add_method @prot
@@ -931,11 +940,16 @@ class TestRDocContext < XrefTestCase
     @vis.add_attribute @aprot
     @vis.add_attribute @apriv
 
+    @vis.add_constant @cpub
+    @vis.add_constant @cpriv
+
     @prot.visibility = :protected
     @priv.visibility = :private
 
     @aprot.visibility = :protected
     @apriv.visibility = :private
+
+    @cpriv.visibility = :private
   end
 
 end
diff --git a/test/rdoc/test_rdoc_markup_to_html.rb b/test/rdoc/test_rdoc_markup_to_html.rb
index fd8a7bb0ca..b5edeb04d2 100644
--- a/test/rdoc/test_rdoc_markup_to_html.rb
+++ b/test/rdoc/test_rdoc_markup_to_html.rb
@@ -452,6 +452,9 @@ class TestRDocMarkupToHtml < RDoc::Markup::FormatterTestCase
   end
 
   def test_accept_verbatim_nl_after_backslash
+    # TODO: Remove "skip" after the issue is resolved: https://github.com/jruby/jruby/issues/4787
+    # This "skip" is for strange behavior around escaped newline on JRuby
+    skip if defined? JRUBY_VERSION
     verb = @RM::Verbatim.new("a = 1 if first_flag_var and \\\n", "  this_is_flag_var\n")
 
     @to.start_accepting
diff --git a/test/rdoc/test_rdoc_parser_c.rb b/test/rdoc/test_rdoc_parser_c.rb
index d0732597dc..5341da16e2 100644
--- a/test/rdoc/test_rdoc_parser_c.rb
+++ b/test/rdoc/test_rdoc_parser_c.rb
@@ -1037,7 +1037,7 @@ Init_Foo(void) {
                  other_function.comment.text
     assert_equal '()', other_function.params
 
-    code = other_function.token_stream.first.text
+    code = other_function.token_stream.first[:text]
 
     assert_equal "VALUE\nother_function() {\n}", code
   end
@@ -1107,7 +1107,7 @@ Init_Foo(void) {
                  other_function.comment.text
     assert_equal '()', other_function.params
 
-    code = other_function.token_stream.first.text
+    code = other_function.token_stream.first[:text]
 
     assert_equal "VALUE\nother_function() {\n}", code
   end
@@ -1141,7 +1141,7 @@ Init_Foo(void) {
     assert_equal '()', other_function.params
     assert_equal 8, other_function.line
 
-    code = other_function.token_stream.first.text
+    code = other_function.token_stream.first[:text]
 
     assert_equal "VALUE\nrb_other_function() {\n}", code
   end
@@ -1174,7 +1174,7 @@ Init_Foo(void) {
     assert_equal '()', other_function.params
     assert_equal 4, other_function.line
 
-    code = other_function.token_stream.first.text
+    code = other_function.token_stream.first[:text]
 
     assert_equal "#define other_function rb_other_function", code
   end
@@ -1314,7 +1314,7 @@ Init_Foo(void) {
                  other_function.comment.text
     assert_equal '()', other_function.params
 
-    code = other_function.token_stream.first.text
+    code = other_function.token_stream.first[:text]
 
     assert_equal "DLL_LOCAL VALUE\nother_function() {\n}", code
   end
diff --git a/test/rdoc/test_rdoc_parser_ruby.rb b/test/rdoc/test_rdoc_parser_ruby.rb
index 55c03d4628..cda407ed51 100644
--- a/test/rdoc/test_rdoc_parser_ruby.rb
+++ b/test/rdoc/test_rdoc_parser_ruby.rb
@@ -74,7 +74,7 @@ class C; end
 
     comment = parser.collect_first_comment
 
-    assert_equal RDoc::Comment.new("=begin\nfirst\n=end\n\n", @top_level), comment
+    assert_equal RDoc::Comment.new("=begin\nfirst\n=end\n", @top_level), comment
   end
 
   def test_get_class_or_module
@@ -84,7 +84,7 @@ class C; end
     cont, name_t, given_name = util_parser('A')    .get_class_or_module ctxt
 
     assert_equal ctxt, cont
-    assert_equal 'A', name_t.text
+    assert_equal 'A', name_t[:text]
     assert_equal 'A', given_name
 
     cont, name_t, given_name = util_parser('B::C') .get_class_or_module ctxt
@@ -92,16 +92,16 @@ class C; end
     b = @store.find_module_named('B')
     assert_equal b, cont
     assert_equal [@top_level], b.in_files
-    assert_equal 'C', name_t.text
+    assert_equal 'C', name_t[:text]
     assert_equal 'B::C', given_name
 
     cont, name_t, given_name = util_parser('D:: E').get_class_or_module ctxt
 
     assert_equal @store.find_module_named('D'), cont
-    assert_equal 'E', name_t.text
+    assert_equal 'E', name_t[:text]
     assert_equal 'D::E', given_name
 
-    assert_raises NoMethodError do
+    assert_raises RDoc::Error do
       util_parser("A::\nB").get_class_or_module ctxt
     end
   end
@@ -1194,10 +1194,12 @@ EOF
     assert_equal klass.current_section, foo.section
 
     stream = [
-      tk(:COMMENT, 0, 1, 1, nil,
-         "# File #{@top_level.relative_name}, line 1"),
-      RDoc::Parser::Ruby::NEWLINE_TOKEN,
-      tk(:SPACE,   0, 1, 1, nil, ''),
+      {
+        :line_no => 1, :char_no => 1, :kind => :on_comment,
+        :text => "# File #{@top_level.relative_name}, line 1"
+      },
+      { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" },
+      { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' }
     ]
 
     assert_equal stream, foo.token_stream
@@ -1358,6 +1360,33 @@ A::B::C = 1
     assert_equal 'comment', c.comment
   end
 
+  def test_parse_constant_with_bracket
+    util_parser <<-RUBY
+class Klass
+end
+
+class Klass2
+  CONSTANT = Klass
+end
+
+class Klass3
+  CONSTANT_2 = {}
+  CONSTANT_2[1] = Klass
+end
+    RUBY
+
+    @parser.scan
+
+    klass = @store.find_class_named 'Klass'
+    klass2 = @store.find_class_named 'Klass2'
+    klass3 = @store.find_class_named 'Klass3'
+    constant = klass2.find_module_named 'CONSTANT'
+    constant2 = klass3.find_module_named 'CONSTANT_2'
+    assert_equal klass, klass2.constants.first.is_alias_for
+    refute_equal klass, klass3.constants.first.is_alias_for
+    assert_nil klass3.find_module_named 'CONSTANT_2'
+  end
+
   def test_parse_extend_or_include_extend
     klass = RDoc::NormalClass.new 'C'
     klass.parent = @top_level
@@ -1434,20 +1463,30 @@ A::B::C = 1
     assert_equal klass.current_section, foo.section
 
     stream = [
-      tk(:COMMENT,     0, 1, 1,  nil,
-         "# File #{@top_level.relative_name}, line 1"),
-      RDoc::Parser::Ruby::NEWLINE_TOKEN,
-      tk(:SPACE,       0, 1, 1,  nil, ''),
-      tk(:IDENTIFIER,  0, 1, 0,  'add_my_method', 'add_my_method'),
-      tk(:SPACE,       0, 1, 13, nil, ' '),
-      tk(:SYMBOL,      0, 1, 14, nil, ':foo'),
-      tk(:COMMA,       0, 1, 18, nil, ','),
-      tk(:SPACE,       0, 1, 19, nil, ' '),
-      tk(:SYMBOL,      0, 1, 20, nil, ':bar'),
-      tk(:NL,          0, 1, 24, nil, "\n"),
+      {
+        :line_no => 1, :char_no => 1, :kind => :on_comment,
+        :text => "# File #{@top_level.relative_name}, line 1"
+      },
+      { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" },
+      { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' },
+      { :line_no => 1, :char_no => 0, :kind => :on_ident, :text => 'add_my_method' },
+      { :line_no => 1, :char_no => 13, :kind => :on_sp, :text => ' ' },
+      { :line_no => 1, :char_no => 14, :kind => :on_symbol, :text => ':foo' },
+      { :line_no => 1, :char_no => 18, :kind => :on_comma, :text => ',' },
+      { :line_no => 1, :char_no => 19, :kind => :on_sp, :text => ' ' },
+      { :line_no => 1, :char_no => 20, :kind => :on_symbol, :text => ':bar' },
+      { :line_no => 1, :char_no => 24, :kind => :on_nl, :text => "\n" }
     ]
+    parsed_stream = foo.token_stream.map { |t|
+      {
+        :line_no => t[:line_no],
+        :char_no => t[:char_no],
+        :kind => t[:kind],
+        :text => t[:text]
+      }
+    }
 
-    assert_equal stream, foo.token_stream
+    assert_equal stream, parsed_stream
   end
 
   def test_parse_meta_method_block
@@ -1468,7 +1507,10 @@ end
 
     @parser.parse_meta_method klass, RDoc::Parser::Ruby::NORMAL, tk, comment
 
-    assert_equal tk(:NL, 0, 3, 3, 3, "\n"), @parser.get_tk
+    rest = { :line_no => 3, :char_no => 3, :kind => :on_nl, :text => "\n" }
+    tk = @parser.get_tk
+    tk = { :line_no => tk[:line_no], :char_no => tk[:char_no], :kind => tk[:kind], :text => tk[:text] }
+    assert_equal rest, tk
   end
 
   def test_parse_meta_method_define_method
@@ -1631,23 +1673,30 @@ end
     assert_equal klass.current_section, foo.section
 
     stream = [
-      tk(:COMMENT,     0, 1, 1,  nil,
-         "# File #{@top_level.relative_name}, line 1"),
-      RDoc::Parser::Ruby::NEWLINE_TOKEN,
-      tk(:SPACE,       0, 1, 1,  nil,   ''),
-      tk(:DEF,         0, 1, 0,  'def', 'def'),
-      tk(:SPACE,       3, 1, 3,  nil,   ' '),
-      tk(:IDENTIFIER,  4, 1, 4,  'foo', 'foo'),
-      tk(:LPAREN,      7, 1, 7,  nil,   '('),
-      tk(:RPAREN,      8, 1, 8,  nil,   ')'),
-      tk(:SPACE,       9, 1, 9,  nil,   ' '),
-      tk(:COLON,      10, 1, 10, nil,   ':'),
-      tk(:IDENTIFIER, 11, 1, 11, 'bar', 'bar'),
-      tk(:SPACE,      14, 1, 14, nil,   ' '),
-      tk(:END,        15, 1, 15, 'end', 'end'),
+      {
+        :line_no => 1, :char_no => 1, :kind => :on_comment,
+        :text => "# File #{@top_level.relative_name}, line 1" },
+      { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" },
+      { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' },
+      { :line_no => 1, :char_no => 0, :kind => :on_kw, :text => 'def' },
+      { :line_no => 1, :char_no => 3, :kind => :on_sp, :text => ' ' },
+      { :line_no => 1, :char_no => 4, :kind => :on_ident, :text => 'foo' },
+      { :line_no => 1, :char_no => 7, :kind => :on_lparen, :text => '(' },
+      { :line_no => 1, :char_no => 8, :kind => :on_rparen, :text => ')' },
+      { :line_no => 1, :char_no => 9, :kind => :on_sp, :text => ' ' },
+      { :line_no => 1, :char_no => 10, :kind => :on_symbol, :text => ':bar' },
+      { :line_no => 1, :char_no => 14, :kind => :on_sp, :text => ' ' },
+      { :line_no => 1, :char_no => 15, :kind => :on_kw, :text => 'end' }
     ]
-
-    assert_equal stream, foo.token_stream
+    parsed_stream = foo.token_stream.map { |t|
+      {
+        :line_no => t[:line_no],
+        :char_no => t[:char_no],
+        :kind => t[:kind],
+        :text => t[:text]
+      }
+    }
+    assert_equal stream, parsed_stream
   end
 
   def test_parse_redefinable_methods
@@ -1664,8 +1713,8 @@ end
     end
 
     klass.method_list.each do |method|
-      assert_kind_of RDoc::RubyToken::TkId, method.token_stream[5]
-      assert_includes redefinable_ops, method.token_stream[5].text
+      assert_equal :on_ident, method.token_stream[5][:kind]
+      assert_includes redefinable_ops, method.token_stream[5][:text]
     end
   end
 
@@ -1909,6 +1958,20 @@ end
     assert_equal '(arg1, arg2, arg3)', foo.params
   end
 
+  def test_parse_method_parameters_with_paren_comment_continue
+    klass = RDoc::NormalClass.new 'Foo'
+    klass.parent = @top_level
+
+    util_parser "def foo(arg1, arg2, # some useful comment\narg3)\nend"
+
+    tk = @parser.get_tk
+
+    @parser.parse_method klass, RDoc::Parser::Ruby::NORMAL, tk, @comment
+
+    foo = klass.method_list.first
+    assert_equal '(arg1, arg2, arg3)', foo.params
+  end
+
   def test_parse_method_star
     klass = RDoc::NormalClass.new 'Foo'
     klass.parent = @top_level
@@ -2091,22 +2154,34 @@ end
     assert_equal 2, x.method_list.length
     a = x.method_list.first
 
+
     expected = [
-      tk(:COMMENT,     0, 2, 1, nil,   "# File #{@filename}, line 2"),
-      tk(:NL,          0, 0, 0, nil,   "\n"),
-      tk(:SPACE,       0, 1, 1, nil,   ''),
-      tk(:DEF,         8, 2, 0, 'def', 'def'),
-      tk(:SPACE,      11, 2, 3, nil,   ' '),
-      tk(:IDENTIFIER, 12, 2, 4, 'a',   'a'),
-      tk(:NL,         13, 2, 5, nil,   "\n"),
-      tk(:REGEXP,     14, 3, 0, nil,   '%r{#}'),
-      tk(:NL,         19, 3, 5, nil,   "\n"),
-      tk(:DREGEXP,    20, 4, 0, nil,   '%r{#{}}'),
-      tk(:NL,         27, 4, 7, nil,   "\n"),
-      tk(:END,        28, 5, 0, 'end', 'end'),
+      {
+        :line_no => 2, :char_no => 1, :kind => :on_comment,
+        :text => "# File #{@filename}, line 2"
+      },
+      { :line_no => 0, :char_no => 0, :kind => :on_nl, :text => "\n" },
+      { :line_no => 1, :char_no => 1, :kind => :on_sp, :text => '' },
+      { :line_no => 2, :char_no => 0, :kind => :on_kw, :text => 'def' },
+      { :line_no => 2, :char_no => 3, :kind => :on_sp, :text => ' ' },
+      { :line_no => 2, :char_no => 4, :kind => :on_ident, :text => 'a' },
+      { :line_no => 2, :char_no => 5, :kind => :on_nl, :text => "\n" },
+      { :line_no => 3, :char_no => 0, :kind => :on_regexp, :text => '%r{#}' },
+      { :line_no => 3, :char_no => 5, :kind => :on_nl, :text => "\n" },
+      { :line_no => 4, :char_no => 0, :kind => :on_regexp, :text => '%r{#{}}' },
+      { :line_no => 4, :char_no => 7, :kind => :on_nl, :text => "\n" },
+      { :line_no => 5, :char_no => 0, :kind => :on_kw, :text => 'end' }
     ]
+    parsed_stream = a.token_stream.map { |tk|
+      {
+        :line_no => tk[:line_no],
+        :char_no => tk[:char_no],
+        :kind => tk[:kind],
+        :text => tk[:text]
+      }
+    }
 
-    assert_equal expected, a.token_stream
+    assert_equal expected, parsed_stream
   end
 
   def test_parse_statements_encoding
@@ -2286,6 +2361,9 @@ class Foo
   SIXTH_CONSTANT = #{sixth_constant}
 
   SEVENTH_CONSTANT = proc { |i| begin i end }
+
+  EIGHTH_CONSTANT = "a" \\
+                    "b"
 end
 EOF
 
@@ -2331,6 +2409,11 @@ EOF
     assert_equal 'SEVENTH_CONSTANT', constant.name
     assert_equal "proc { |i| begin i end }", constant.value
     assert_equal @top_level, constant.file
+
+    constant = constants[7]
+    assert_equal 'EIGHTH_CONSTANT', constant.name
+    assert_equal "\"a\" \\\n\"b\"", constant.value
+    assert_equal @top_level, constant.file
   end
 
   def test_parse_statements_identifier_attr
@@ -2504,7 +2587,7 @@ EXPTECTED
     util_parser <<RUBY
 class Foo
   def blah()
-    <<~EOM if true
+    <<-EOM if true
     EOM
   end
 end
@@ -2512,7 +2595,7 @@ RUBY
 
     expected = <<EXPTECTED
   <span class="ruby-keyword">def</span> <span class="ruby-identifier">blah</span>()
-    <span class="ruby-identifier">&lt;&lt;~EOM</span> <span class="ruby-keyword">if</span> <span class="ruby-keyword">true</span>
+    <span class="ruby-identifier">&lt;&lt;-EOM</span> <span class="ruby-keyword">if</span> <span class="ruby-keyword">true</span>
 <span class="ruby-value"></span><span class="ruby-identifier">    EOM</span>
   <span class="ruby-keyword">end</span>
 EXPTECTED
@@ -2525,7 +2608,7 @@ EXPTECTED
 
     blah = foo.method_list.first
     markup_code = blah.markup_code.sub(/^.*\n/, '')
-    assert_equal markup_code, expected
+    assert_equal expected, markup_code
   end
 
   def test_parse_statements_method_oneliner_with_regexp
@@ -2770,17 +2853,21 @@ RUBY
   end
 
   def test_parse_symbol_in_arg
-    util_parser ':blah "blah" "#{blah}" blah'
+    util_parser '[:blah, "blah", "#{blah}", blah]'
+    @parser.get_tk # skip '['
 
     assert_equal 'blah', @parser.parse_symbol_in_arg
+    @parser.get_tk # skip ','
 
     @parser.skip_tkspace
 
     assert_equal 'blah', @parser.parse_symbol_in_arg
+    @parser.get_tk # skip ','
 
     @parser.skip_tkspace
 
     assert_equal nil, @parser.parse_symbol_in_arg
+    @parser.get_tk # skip ','
 
     @parser.skip_tkspace
 
@@ -2888,7 +2975,7 @@ end
     assert_equal 'category', directive
     assert_equal 'test', value
 
-    assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk
+    assert_equal nil, parser.get_tk
   end
 
   def test_read_directive_allow
@@ -2898,7 +2985,7 @@ end
 
     assert_nil directive
 
-    assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk
+    assert_equal nil, parser.get_tk
   end
 
   def test_read_directive_empty
@@ -2908,7 +2995,7 @@ end
 
     assert_nil directive
 
-    assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk
+    assert_equal nil, parser.get_tk
   end
 
   def test_read_directive_no_comment
@@ -2918,18 +3005,18 @@ end
 
     assert_nil directive
 
-    assert_kind_of RDoc::RubyToken::TkNL, parser.get_tk
+    assert_equal nil, parser.get_tk
   end
 
   def test_read_directive_one_liner
-    parser = util_parser '; end # :category: test'
+    parser = util_parser 'AAA = 1 # :category: test'
 
     directive, value = parser.read_directive %w[category]
 
     assert_equal 'category', directive
     assert_equal 'test', value
 
-    assert_kind_of RDoc::RubyToken::TkSEMICOLON, parser.get_tk
+    assert_equal :on_const, parser.get_tk[:kind]
   end
 
   def test_read_documentation_modifiers
@@ -2974,10 +3061,10 @@ end
 
   def test_sanity_integer
     util_parser '1'
-    assert_equal '1', @parser.get_tk.text
+    assert_equal '1', @parser.get_tk[:text]
 
     util_parser '1.0'
-    assert_equal '1.0', @parser.get_tk.text
+    assert_equal '1.0', @parser.get_tk[:text]
   end
 
   def test_sanity_interpolation
@@ -2986,7 +3073,7 @@ end
 
     while tk = @parser.get_tk do last_tk = tk end
 
-    assert_equal "\n", last_tk.text
+    assert_equal 'end', last_tk[:text]
   end
 
   # If you're writing code like this you're doing it wrong
@@ -2994,15 +3081,15 @@ end
   def test_sanity_interpolation_crazy
     util_parser '"#{"#{"a")}" if b}"'
 
-    assert_equal '"#{"#{"a")}" if b}"', @parser.get_tk.text
-    assert_equal RDoc::RubyToken::TkNL, @parser.get_tk.class
+    assert_equal '"#{"#{"a")}" if b}"', @parser.get_tk[:text]
+    assert_equal nil, @parser.get_tk
   end
 
   def test_sanity_interpolation_curly
     util_parser '%{ #{} }'
 
-    assert_equal '%{ #{} }', @parser.get_tk.text
-    assert_equal RDoc::RubyToken::TkNL, @parser.get_tk.class
+    assert_equal '%{ #{} }', @parser.get_tk[:text]
+    assert_equal nil, @parser.get_tk
   end
 
   def test_sanity_interpolation_format
@@ -3537,6 +3624,53 @@ end
     assert_equal 2, public_method_count
   end
 
+  def test_scan_constant_visibility
+    util_parser <<-RUBY
+class C
+   CONST_A = 123
+
+   CONST_B = 234
+   private_constant :CONST_B
+
+   CONST_C = 345
+   public_constant :CONST_C
+end
+    RUBY
+
+    @parser.scan
+
+    c = @store.find_class_named 'C'
+    const_a, const_b, const_c, const_d = c.constants.sort_by(&:name)
+
+    assert_equal 'CONST_A', const_a.name
+    assert_equal :public, const_a.visibility
+
+    assert_equal 'CONST_B', const_b.name
+    assert_equal :private, const_b.visibility
+
+    assert_equal 'CONST_C', const_c.name
+    assert_equal :public, const_c.visibility
+  end
+
+  def test_document_after_rescue_inside_paren
+    util_parser <<-RUBY
+class C
+  attr_accessor :sample if (1.inexistent_method rescue false)
+  # first
+  # second
+  def a
+  end
+end
+    RUBY
+
+    @parser.scan
+
+    c = @store.find_class_named 'C'
+
+    c_a = c.find_method_named 'a'
+    assert_equal "first\nsecond", c_a.comment.text
+  end
+
   def test_singleton_method_via_eigenclass
     util_parser <<-RUBY
 class C
diff --git a/test/rdoc/test_rdoc_ruby_lex.rb b/test/rdoc/test_rdoc_ruby_lex.rb
deleted file mode 100644
index 9ab5844b54..0000000000
--- a/test/rdoc/test_rdoc_ruby_lex.rb
+++ /dev/null
@@ -1,1095 +0,0 @@
-# coding: UTF-8
-# frozen_string_literal: false
-
-require 'rdoc/test_case'
-
-class TestRDocRubyLex < RDoc::TestCase
-
-  def setup
-    @TK = RDoc::RubyToken
-  end
-
-  def test_token_position
-    tokens = RDoc::RubyLex.tokenize '[ 1, :a, nil ]', nil
-
-    assert_equal '[', tokens[0].text
-    assert_equal 0, tokens[0].seek
-    assert_equal 1, tokens[0].line_no
-    assert_equal 0, tokens[0].char_no
-    assert_equal '1', tokens[2].text
-    assert_equal 2, tokens[2].seek
-    assert_equal 1, tokens[2].line_no
-    assert_equal 2, tokens[2].char_no
-    assert_equal ':a', tokens[5].text
-    assert_equal 5, tokens[5].seek
-    assert_equal 1, tokens[5].line_no
-    assert_equal 5, tokens[5].char_no
-    assert_equal 'nil', tokens[8].text
-    assert_equal 9, tokens[8].seek
-    assert_equal 1, tokens[8].line_no
-    assert_equal 9, tokens[8].char_no
-    assert_equal ']', tokens[10].text
-    assert_equal 13, tokens[10].seek
-    assert_equal 1, tokens[10].line_no
-    assert_equal 13, tokens[10].char_no
-  end
-
-  def test_class_tokenize
-    tokens = RDoc::RubyLex.tokenize "def x() end", nil
-
-    expected = [
-      @TK::TkDEF       .new( 0, 1,  0, "def"),
-      @TK::TkSPACE     .new( 3, 1,  3, " "),
-      @TK::TkIDENTIFIER.new( 4, 1,  4, "x"),
-      @TK::TkLPAREN    .new( 5, 1,  5, "("),
-      @TK::TkRPAREN    .new( 6, 1,  6, ")"),
-      @TK::TkSPACE     .new( 7, 1,  7, " "),
-      @TK::TkEND       .new( 8, 1,  8, "end"),
-      @TK::TkNL        .new(11, 1, 11, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize___END__
-    tokens = RDoc::RubyLex.tokenize '__END__', nil
-
-    expected = [
-      @TK::TkEND_OF_SCRIPT.new(0, 1, 0, '__END__'),
-      @TK::TkNL           .new(7, 1, 7, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize___ENCODING__
-    tokens = RDoc::RubyLex.tokenize '__ENCODING__', nil
-
-    expected = [
-      @TK::Tk__ENCODING__.new( 0, 1,  0, '__ENCODING__'),
-      @TK::TkNL          .new(12, 1, 12, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_character_literal
-    tokens = RDoc::RubyLex.tokenize "?c", nil
-
-    expected = [
-      @TK::TkCHAR.new( 0, 1,  0, "?c"),
-      @TK::TkNL  .new( 2, 1,  2, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_character_literal_with_escape
-    tokens = RDoc::RubyLex.tokenize "?\\s", nil
-
-    expected = [
-      @TK::TkCHAR.new( 0, 1,  0, "?\\s"),
-      @TK::TkNL  .new( 3, 1,  3, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_def_heredoc
-    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-def x
-  <<E
-Line 1
-Line 2
-E
-end
-    RUBY
-
-    expected = [
-      @TK::TkDEF       .new( 0, 1,  0, 'def'),
-      @TK::TkSPACE     .new( 3, 1,  3, ' '),
-      @TK::TkIDENTIFIER.new( 4, 1,  4, 'x'),
-      @TK::TkNL        .new( 5, 1,  5, "\n"),
-      @TK::TkSPACE     .new( 6, 2,  0, '  '),
-
-      @TK::TkHEREDOCBEG.new( 8, 2,  2, '<<E'),
-      @TK::TkNL        .new(11, 2,  6, "\n"),
-      @TK::TkHEREDOC   .new(11, 2,  6, "Line 1\nLine 2\n"),
-      @TK::TkHEREDOCEND.new(27, 5, 26, "E\n"),
-      @TK::TkEND       .new(28, 6,  0, 'end'),
-      @TK::TkNL        .new(31, 6, 28, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_reserved_keyword_with_args
-    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-yield :foo
-super :bar
-defined? :baz
-    RUBY
-
-    expected = [
-      @TK::TkYIELD  .new( 0, 1,  0, "yield"),
-      @TK::TkSPACE  .new( 5, 1,  5, " "),
-      @TK::TkSYMBOL .new( 6, 1,  6, ":foo"),
-      @TK::TkNL     .new(10, 1, 10,  "\n"),
-      @TK::TkSUPER  .new(11, 2,  0, "super"),
-      @TK::TkSPACE  .new(16, 2,  5, " "),
-      @TK::TkSYMBOL .new(17, 2,  6, ":bar"),
-      @TK::TkNL     .new(21, 2, 11,  "\n"),
-      @TK::TkDEFINED.new(22, 3,  0, "defined?"),
-      @TK::TkSPACE  .new(30, 3,  8, " "),
-      @TK::TkSYMBOL .new(31, 3,  9, ":baz"),
-      @TK::TkNL     .new(35, 3, 22,  "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_hash_symbol
-    tokens = RDoc::RubyLex.tokenize '{ class:"foo" }', nil
-
-    expected = [
-      @TK::TkLBRACE.new( 0, 1,  0, '{'),
-      @TK::TkSPACE .new( 1, 1,  1, ' '),
-      @TK::TkSYMBOL.new( 2, 1,  2, 'class:'),
-      @TK::TkSTRING.new( 8, 1,  8, '"foo"'),
-      @TK::TkSPACE .new(13, 1, 13, ' '),
-      @TK::TkRBRACE.new(14, 1, 14, '}'),
-      @TK::TkNL    .new(15, 1, 15, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_double_colon_is_not_hash_symbol
-    tokens = RDoc::RubyLex.tokenize 'self.class::Row', nil
-
-    expected = [
-      @TK::TkSELF      .new( 0, 1,  0, "self"),
-      @TK::TkDOT       .new( 4, 1,  4, "."),
-      @TK::TkIDENTIFIER.new( 5, 1,  5, "class"),
-      @TK::TkCOLON2    .new(10, 1, 10, "::"),
-      @TK::TkCONSTANT  .new(12, 1, 12, "Row"),
-      @TK::TkNL        .new(15, 1, 15, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_safe_nav_operator
-    tokens = RDoc::RubyLex.tokenize 'receiver&.meth', nil
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, "receiver"),
-      @TK::TkSAFENAV   .new( 8, 1,  8, "&."),
-      @TK::TkIDENTIFIER.new(10, 1, 10, "meth"),
-      @TK::TkNL        .new(14, 1, 14, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_hash_rocket
-    tokens = RDoc::RubyLex.tokenize '{ :class => "foo" }', nil
-
-    expected = [
-      @TK::TkLBRACE    .new( 0, 1,  0, '{'),
-      @TK::TkSPACE     .new( 1, 1,  1, ' '),
-      @TK::TkSYMBOL    .new( 2, 1,  2, ':class'),
-      @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkHASHROCKET.new( 9, 1,  9, '=>'),
-      @TK::TkSPACE     .new(11, 1, 11, ' '),
-      @TK::TkSTRING    .new(12, 1, 12, '"foo"'),
-      @TK::TkSPACE     .new(17, 1, 17, ' '),
-      @TK::TkRBRACE    .new(18, 1, 18, '}'),
-      @TK::TkNL        .new(19, 1, 19, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_heredoc_CR_NL
-    tokens = RDoc::RubyLex.tokenize <<-RUBY, nil
-string = <<-STRING\r
-Line 1\r
-Line 2\r
-  STRING\r
-    RUBY
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
-      @TK::TkSPACE     .new( 6, 1,  6, ' '),
-      @TK::TkASSIGN    .new( 7, 1,  7, '='),
-      @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
-      @TK::TkSPACE     .new(18, 1, 18, "\r"),
-      @TK::TkNL        .new(19, 1, 19, "\n"),
-      @TK::TkHEREDOC   .new(19, 1, 19,
-                            %Q{Line 1\nLine 2\n}),
-      @TK::TkHEREDOCEND.new(45, 4, 36, "  STRING\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_opassign
-    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
-a %= b
-a /= b
-a -= b
-a += b
-a |= b
-a &= b
-a >>= b
-a <<= b
-a *= b
-a &&= b
-a ||= b
-a **= b
-RUBY
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1, 0, "a"),
-      @TK::TkSPACE     .new( 1, 1, 1, " "),
-      @TK::TkOPASGN    .new( 2, 1, 2, "%"),
-      @TK::TkSPACE     .new( 4, 1, 4, " "),
-      @TK::TkIDENTIFIER.new( 5, 1, 5, "b"),
-      @TK::TkNL        .new( 6, 1, 6, "\n"),
-      @TK::TkIDENTIFIER.new( 7, 2, 0, "a"),
-      @TK::TkSPACE     .new( 8, 2, 1, " "),
-      @TK::TkOPASGN    .new( 9, 2, 2, "/"),
-      @TK::TkSPACE     .new( 11, 2, 4, " "),
-      @TK::TkIDENTIFIER.new( 12, 2, 5, "b"),
-      @TK::TkNL        .new( 13, 2, 7, "\n"),
-      @TK::TkIDENTIFIER.new( 14, 3, 0, "a"),
-      @TK::TkSPACE     .new( 15, 3, 1, " "),
-      @TK::TkOPASGN    .new( 16, 3, 2, "-"),
-      @TK::TkSPACE     .new( 18, 3, 4, " "),
-      @TK::TkIDENTIFIER.new( 19, 3, 5, "b"),
-      @TK::TkNL        .new( 20, 3, 14, "\n"),
-      @TK::TkIDENTIFIER.new( 21, 4, 0, "a"),
-      @TK::TkSPACE     .new( 22, 4, 1, " "),
-      @TK::TkOPASGN    .new( 23, 4, 2, "+"),
-      @TK::TkSPACE     .new( 25, 4, 4, " "),
-      @TK::TkIDENTIFIER.new( 26, 4, 5, "b"),
-      @TK::TkNL        .new( 27, 4, 21, "\n"),
-      @TK::TkIDENTIFIER.new( 28, 5, 0, "a"),
-      @TK::TkSPACE     .new( 29, 5, 1, " "),
-      @TK::TkOPASGN    .new( 30, 5, 2, "|"),
-      @TK::TkSPACE     .new( 32, 5, 4, " "),
-      @TK::TkIDENTIFIER.new( 33, 5, 5, "b"),
-      @TK::TkNL        .new( 34, 5, 28, "\n"),
-      @TK::TkIDENTIFIER.new( 35, 6, 0, "a"),
-      @TK::TkSPACE     .new( 36, 6, 1, " "),
-      @TK::TkOPASGN    .new( 37, 6, 2, "&"),
-      @TK::TkSPACE     .new( 39, 6, 4, " "),
-      @TK::TkIDENTIFIER.new( 40, 6, 5, "b"),
-      @TK::TkNL        .new( 41, 6, 35, "\n"),
-      @TK::TkIDENTIFIER.new( 42, 7, 0, "a"),
-      @TK::TkSPACE     .new( 43, 7, 1, " "),
-      @TK::TkOPASGN    .new( 44, 7, 2, ">>"),
-      @TK::TkSPACE     .new( 47, 7, 5, " "),
-      @TK::TkIDENTIFIER.new( 48, 7, 6, "b"),
-      @TK::TkNL        .new( 49, 7, 42, "\n"),
-      @TK::TkIDENTIFIER.new( 50, 8, 0, "a"),
-      @TK::TkSPACE     .new( 51, 8, 1, " "),
-      @TK::TkOPASGN    .new( 52, 8, 2, "<<"),
-      @TK::TkSPACE     .new( 55, 8, 5, " "),
-      @TK::TkIDENTIFIER.new( 56, 8, 6, "b"),
-      @TK::TkNL        .new( 57, 8, 50, "\n"),
-      @TK::TkIDENTIFIER.new( 58, 9, 0, "a"),
-      @TK::TkSPACE     .new( 59, 9, 1, " "),
-      @TK::TkOPASGN    .new( 60, 9, 2, "*"),
-      @TK::TkSPACE     .new( 62, 9, 4, " "),
-      @TK::TkIDENTIFIER.new( 63, 9, 5, "b"),
-      @TK::TkNL        .new( 64, 9, 58, "\n"),
-      @TK::TkIDENTIFIER.new( 65, 10, 0, "a"),
-      @TK::TkSPACE     .new( 66, 10, 1, " "),
-      @TK::TkOPASGN    .new( 67, 10, 2, "&&"),
-      @TK::TkSPACE     .new( 70, 10, 5, " "),
-      @TK::TkIDENTIFIER.new( 71, 10, 6, "b"),
-      @TK::TkNL        .new( 72, 10, 65, "\n"),
-      @TK::TkIDENTIFIER.new( 73, 11, 0, "a"),
-      @TK::TkSPACE     .new( 74, 11, 1, " "),
-      @TK::TkOPASGN    .new( 75, 11, 2, "||"),
-      @TK::TkSPACE     .new( 78, 11, 5, " "),
-      @TK::TkIDENTIFIER.new( 79, 11, 6, "b"),
-      @TK::TkNL        .new( 80, 11, 73, "\n"),
-      @TK::TkIDENTIFIER.new( 81, 12, 0, "a"),
-      @TK::TkSPACE     .new( 82, 12, 1, " "),
-      @TK::TkOPASGN    .new( 83, 12, 2, "**"),
-      @TK::TkSPACE     .new( 86, 12, 5, " "),
-      @TK::TkIDENTIFIER.new( 87, 12, 6, "b"),
-      @TK::TkNL        .new( 88, 12, 81, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_heredoc_call
-    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-string = <<-STRING.chomp
-Line 1
-Line 2
-  STRING
-    RUBY
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
-      @TK::TkSPACE     .new( 6, 1,  6, ' '),
-      @TK::TkASSIGN    .new( 7, 1,  7, '='),
-      @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
-      @TK::TkDOT       .new(18, 1, 18, '.'),
-      @TK::TkIDENTIFIER.new(19, 1, 19, 'chomp'),
-      @TK::TkNL        .new(24, 1, 24, "\n"),
-      @TK::TkHEREDOC   .new(24, 1, 24, "Line 1\nLine 2\n"),
-      @TK::TkHEREDOCEND.new(47, 4, 39, "  STRING\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_heredoc_indent
-    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-string = <<-STRING
-Line 1
-Line 2
-  STRING
-    RUBY
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, 'string'),
-      @TK::TkSPACE     .new( 6, 1,  6, ' '),
-      @TK::TkASSIGN    .new( 7, 1,  7, '='),
-      @TK::TkSPACE     .new( 8, 1,  8, ' '),
-
-
-      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
-      @TK::TkNL        .new(18, 1, 18, "\n"),
-      @TK::TkHEREDOC   .new(18, 1, 18, "Line 1\nLine 2\n"),
-      @TK::TkHEREDOCEND.new(41, 4, 33, "  STRING\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_heredoc_missing_end
-    e = assert_raises RDoc::RubyLex::Error do
-      RDoc::RubyLex.tokenize <<-'RUBY', nil
->> string1 = <<-TXT
->" That's swell
->" TXT
-      RUBY
-    end
-
-    assert_equal 'Missing terminating TXT for string', e.message
-  end
-
-  def test_class_tokenize_heredoc_percent_N
-    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
-a b <<-U
-%N
-U
-    RUBY
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, 'a'),
-      @TK::TkSPACE     .new( 1, 1,  1, ' '),
-      @TK::TkIDENTIFIER.new( 2, 1,  2, 'b'),
-      @TK::TkSPACE     .new( 3, 1,  3, ' '),
-      @TK::TkHEREDOCBEG.new( 4, 1,  4, '<<-U'),
-      @TK::TkNL        .new( 8, 1,  8, "\n"),
-      @TK::TkHEREDOC   .new( 8, 1,  8, "%N\n"),
-      @TK::TkHEREDOCEND.new(13, 3, 12, "U\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_identifier_high_unicode
-    tokens = RDoc::RubyLex.tokenize '𝖒', nil
-
-    expected = @TK::TkIDENTIFIER.new(0, 1, 0, '𝖒')
-
-    assert_equal expected, tokens.first
-  end
-
-  def test_class_tokenize_lambda
-    tokens = RDoc::RubyLex.tokenize 'a = -> x, y { x + y }', nil
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, 'a'),
-      @TK::TkSPACE     .new( 1, 1,  1, ' '),
-      @TK::TkASSIGN    .new( 2, 1,  2, '='),
-      @TK::TkSPACE     .new( 3, 1,  3, ' '),
-      @TK::TkLAMBDA    .new( 4, 1,  4, '->'),
-      @TK::TkSPACE     .new( 6, 1,  6, ' '),
-      @TK::TkIDENTIFIER.new( 7, 1,  7, 'x'),
-      @TK::TkCOMMA     .new( 8, 1,  8, ','),
-      @TK::TkSPACE     .new( 9, 1,  9, ' '),
-      @TK::TkIDENTIFIER.new(10, 1, 10, 'y'),
-      @TK::TkSPACE     .new(11, 1, 11, ' '),
-      @TK::TkfLBRACE   .new(12, 1, 12, '{'),
-      @TK::TkSPACE     .new(13, 1, 13, ' '),
-      @TK::TkIDENTIFIER.new(14, 1, 14, 'x'),
-      @TK::TkSPACE     .new(15, 1, 15, ' '),
-      @TK::TkPLUS      .new(16, 1, 16, '+'),
-      @TK::TkSPACE     .new(17, 1, 17, ' '),
-      @TK::TkIDENTIFIER.new(18, 1, 18, 'y'),
-      @TK::TkSPACE     .new(19, 1, 19, ' '),
-      @TK::TkRBRACE    .new(20, 1, 20, '}'),
-      @TK::TkNL        .new(21, 1, 21, "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_1
-    tokens = RDoc::RubyLex.tokenize 'v%10==10', nil
-
-    expected = [
-      @TK::TkIDENTIFIER.new(0, 1, 0, 'v'),
-      @TK::TkMOD.new(       1, 1, 1, '%'),
-      @TK::TkINTEGER.new(   2, 1, 2, '10'),
-      @TK::TkEQ.new(        4, 1, 4, '=='),
-      @TK::TkINTEGER.new(   6, 1, 6, '10'),
-      @TK::TkNL.new(        8, 1, 8, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_r
-    tokens = RDoc::RubyLex.tokenize '%r[hi]', nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, '%r[hi]'),
-      @TK::TkNL    .new( 6, 1, 6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_r_with_slash
-    tokens = RDoc::RubyLex.tokenize '%r/hi/', nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, '%r/hi/'),
-      @TK::TkNL    .new( 6, 1, 6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_large_q
-    tokens = RDoc::RubyLex.tokenize '%Q/hi/', nil
-
-    expected = [
-      @TK::TkSTRING.new( 0, 1,  0, '%Q/hi/'),
-      @TK::TkNL    .new( 6, 1, 6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_large_q_with_double_quote
-    tokens = RDoc::RubyLex.tokenize '%Q"hi"', nil
-
-    expected = [
-      @TK::TkSTRING.new( 0, 1,  0, '%Q"hi"'),
-      @TK::TkNL    .new( 6, 1, 6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_w
-    tokens = RDoc::RubyLex.tokenize '%w[hi]', nil
-
-    expected = [
-      @TK::TkDSTRING.new( 0, 1,  0, '%w[hi]'),
-      @TK::TkNL     .new( 6, 1, 6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_w_quote
-    tokens = RDoc::RubyLex.tokenize '%w"hi"', nil
-
-    expected = [
-      @TK::TkDSTRING.new( 0, 1,  0, '%w"hi"'),
-      @TK::TkNL     .new( 6, 1, 6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_hash_rocket
-    tokens = RDoc::RubyLex.tokenize "{ :foo=> 1 }", nil
-
-    expected = [
-      @TK::TkLBRACE    .new( 0, 1,  0, '{'),
-      @TK::TkSPACE     .new( 1, 1,  1, ' '),
-      @TK::TkSYMBOL    .new( 2, 1,  2, ':foo'),
-      @TK::TkHASHROCKET.new( 6, 1,  6, '=>'),
-      @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkINTEGER   .new( 9, 1,  9, '1'),
-      @TK::TkSPACE     .new(10, 1, 10, ' '),
-      @TK::TkRBRACE    .new(11, 1, 11, '}'),
-      @TK::TkNL        .new(12, 1, 12, "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_percent_sign_quote
-    tokens = RDoc::RubyLex.tokenize '%%hi%', nil
-
-    expected = [
-      @TK::TkSTRING.new( 0, 1, 0, '%%hi%'),
-      @TK::TkNL    .new( 5, 1, 5, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_regexp
-    tokens = RDoc::RubyLex.tokenize "/hay/", nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, "/hay/"),
-      @TK::TkNL    .new( 5, 1,  5, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_regexp_options
-    tokens = RDoc::RubyLex.tokenize "/hAY/i", nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, "/hAY/i"),
-      @TK::TkNL    .new( 6, 1,  6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-
-    tokens = RDoc::RubyLex.tokenize "/hAY/ix", nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, "/hAY/ix"),
-      @TK::TkNL    .new( 7, 1,  7, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_regexp_backref
-    tokens = RDoc::RubyLex.tokenize "/[csh](..) [csh]\\1 in/", nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, "/[csh](..) [csh]\\1 in/"),
-      @TK::TkNL    .new(22, 1, 22, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_regexp_escape
-    tokens = RDoc::RubyLex.tokenize "/\\//", nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, "/\\//"),
-      @TK::TkNL    .new( 4, 1,  4, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_number_with_sign_character
-    tokens = RDoc::RubyLex.tokenize "+3--3r", nil
-
-    expected = [
-      @TK::TkINTEGER .new(0, 1, 0, "+3"),
-      @TK::TkMINUS   .new(2, 1, 2, "-"),
-      @TK::TkRATIONAL.new(3, 1, 3, "-3r"),
-      @TK::TkNL      .new(6, 1, 6, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_regexp_continuing_backslash
-    tokens = RDoc::RubyLex.tokenize "/(?<!\\\\)\\n\z/", nil
-
-    expected = [
-      @TK::TkREGEXP.new( 0, 1,  0, "/(?<!\\\\)\\n\z/"),
-      @TK::TkNL    .new(12, 1, 12, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_single_quote_escape
-    tokens = RDoc::RubyLex.tokenize %q{'\\\\ \\' \\&'}, nil
-
-    expected = [
-      @TK::TkSTRING.new( 0, 1,  0, %q{'\\\\ \\' \\&'}),
-      @TK::TkNL    .new(10, 1, 10, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_string
-    tokens = RDoc::RubyLex.tokenize "'hi'", nil
-
-    expected = [
-      @TK::TkSTRING.new( 0, 1,  0, "'hi'"),
-      @TK::TkNL    .new( 4, 1,  4, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_string_with_escape
-    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
-[
-  '\\',
-  '\'',
-  "'",
-  "\'\"\`",
-  "\#",
-  "\#{}",
-  "#",
-  "#{}",
-  /'"/,
-  /\'\"/,
-  /\//,
-  /\\/,
-  /\#/,
-  /\#{}/,
-  /#/,
-  /#{}/
-]
-RUBY
-
-    expected = [
-      @TK::TkLBRACK .new(  0,  1,   0, "["),
-      @TK::TkNL     .new(  1,  1,   1, "\n"),
-      @TK::TkSPACE  .new(  2,  2,   0, "  "),
-      @TK::TkSTRING .new(  4,  2,   2, "'\\\\'"),
-      @TK::TkCOMMA  .new(  8,  2,   6, ","),
-      @TK::TkNL     .new(  9,  2,   2, "\n"),
-      @TK::TkSPACE  .new( 10,  3,   0, "  "),
-      @TK::TkSTRING .new( 12,  3,   2, "'\\''"),
-      @TK::TkCOMMA  .new( 16,  3,   6, ","),
-      @TK::TkNL     .new( 17,  3,  10, "\n"),
-      @TK::TkSPACE  .new( 18,  4,   0, "  "),
-      @TK::TkSTRING .new( 20,  4,   2, "\"'\""),
-      @TK::TkCOMMA  .new( 23,  4,   5, ","),
-      @TK::TkNL     .new( 24,  4,  18, "\n"),
-      @TK::TkSPACE  .new( 25,  5,   0, "  "),
-      @TK::TkSTRING .new( 27,  5,   2, "\"\\'\\\"\\`\""),
-      @TK::TkCOMMA  .new( 35,  5,  10, ","),
-      @TK::TkNL     .new( 36,  5,  25, "\n"),
-      @TK::TkSPACE  .new( 37,  6,   0, "  "),
-      @TK::TkSTRING .new( 39,  6,   2, "\"\\#\""),
-      @TK::TkCOMMA  .new( 43,  6,   6, ","),
-      @TK::TkNL     .new( 44,  6,  37, "\n"),
-      @TK::TkSPACE  .new( 45,  7,   0, "  "),
-      @TK::TkSTRING .new( 47,  7,   2, "\"\\\#{}\""),
-      @TK::TkCOMMA  .new( 53,  7,   8, ","),
-      @TK::TkNL     .new( 54,  7,  45, "\n"),
-      @TK::TkSPACE  .new( 55,  8,   0, "  "),
-      @TK::TkSTRING .new( 57,  8,   2, "\"#\""),
-      @TK::TkCOMMA  .new( 60,  8,   5, ","),
-      @TK::TkNL     .new( 61,  8,  55, "\n"),
-      @TK::TkSPACE  .new( 62,  9,   0, "  "),
-      @TK::TkDSTRING.new( 64,  9,   2, "\"\#{}\""),
-      @TK::TkCOMMA  .new( 69,  9,   7, ","),
-      @TK::TkNL     .new( 70,  9,  62, "\n"),
-      @TK::TkSPACE  .new( 71, 10,   0, "  "),
-      @TK::TkREGEXP .new( 73, 10,   2, "/'\"/"),
-      @TK::TkCOMMA  .new( 77, 10,   6, ","),
-      @TK::TkNL     .new( 78, 10,  71, "\n"),
-      @TK::TkSPACE  .new( 79, 11,   0, "  "),
-      @TK::TkREGEXP .new( 81, 11,   2, "/\\'\\\"/"),
-      @TK::TkCOMMA  .new( 87, 11,   8, ","),
-      @TK::TkNL     .new( 88, 11,  79, "\n"),
-      @TK::TkSPACE  .new( 89, 12,   0, "  "),
-      @TK::TkREGEXP .new( 91, 12,   2, "/\\//"),
-      @TK::TkCOMMA  .new( 95, 12,   6, ","),
-      @TK::TkNL     .new( 96, 12,  89, "\n"),
-      @TK::TkSPACE  .new( 97, 13,   0, "  "),
-      @TK::TkREGEXP .new( 99, 13,   2, "/\\\\/"),
-      @TK::TkCOMMA  .new(103, 13,   6, ","),
-      @TK::TkNL     .new(104, 13,  97, "\n"),
-      @TK::TkSPACE  .new(105, 14,   0, "  "),
-      @TK::TkREGEXP .new(107, 14,   2, "/\\#/"),
-      @TK::TkCOMMA  .new(111, 14,   6, ","),
-      @TK::TkNL     .new(112, 14, 105, "\n"),
-      @TK::TkSPACE  .new(113, 15,   0, "  "),
-      @TK::TkREGEXP .new(115, 15,   2, "/\\\#{}/"),
-      @TK::TkCOMMA  .new(121, 15,   8, ","),
-      @TK::TkNL     .new(122, 15, 113, "\n"),
-      @TK::TkSPACE  .new(123, 16,   0, "  "),
-      @TK::TkREGEXP .new(125, 16,   2, "/#/"),
-      @TK::TkCOMMA  .new(128, 16,   5, ","),
-      @TK::TkNL     .new(129, 16, 123, "\n"),
-      @TK::TkSPACE  .new(130, 17,   0, "  "),
-      @TK::TkDREGEXP.new(132, 17,   2, "/\#{}/"),
-      @TK::TkNL     .new(137, 17,   7, "\n"),
-      @TK::TkRBRACK .new(138, 18,   0, "]"),
-      @TK::TkNL     .new(139, 18, 138, "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_postfix_if_after_escaped_newline
-    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
-def a
-  1 if true
-  1 \
-    if true
-end
-RUBY
-
-    expected = [
-      @TK::TkDEF       .new( 0, 1, 0,  "def"),
-      @TK::TkSPACE     .new( 3, 1, 3,  " "),
-      @TK::TkIDENTIFIER.new( 4, 1, 4,  "a"),
-      @TK::TkNL        .new( 5, 1, 5,  "\n"),
-      @TK::TkSPACE     .new( 6, 2, 0,  "  "),
-      @TK::TkINTEGER   .new( 8, 2, 2,  "1"),
-      @TK::TkSPACE     .new( 9, 2, 3,  " "),
-      @TK::TkIF_MOD    .new(10, 2, 4,  "if"),
-      @TK::TkSPACE     .new(12, 2, 6,  " "),
-      @TK::TkTRUE      .new(13, 2, 7,  "true"),
-      @TK::TkNL        .new(17, 2, 6,  "\n"),
-      @TK::TkSPACE     .new(18, 3, 0,  "  "),
-      @TK::TkINTEGER   .new(20, 3, 2,  "1"),
-      @TK::TkSPACE     .new(21, 3, 3,  " "),
-      @TK::TkBACKSLASH .new(22, 3, 4,  "\\"),
-      @TK::TkNL        .new(23, 3, 18, "\n"),
-      @TK::TkSPACE     .new(24, 4, 0,  "    "),
-      @TK::TkIF_MOD    .new(28, 4, 4,  "if"),
-      @TK::TkSPACE     .new(30, 4, 6,  " "),
-      @TK::TkTRUE      .new(31, 4, 7,  "true"),
-      @TK::TkNL        .new(35, 4, 24, "\n"),
-      @TK::TkEND       .new(36, 5, 0,  "end"),
-      @TK::TkNL        .new(39, 5, 36, "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_backtick_with_escape
-    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
-[
-  `\\`,
-  `\'\"\``,
-  `\#`,
-  `\#{}`,
-  `#`,
-  `#{}`
-]
-RUBY
-
-    expected = [
-      @TK::TkLBRACK  .new( 0, 1,  0, "["),
-      @TK::TkNL      .new( 1, 1,  1, "\n"),
-      @TK::TkSPACE   .new( 2, 2,  0, "  "),
-      @TK::TkXSTRING .new( 4, 2,  2, "`\\\\`"),
-      @TK::TkCOMMA   .new( 8, 2,  6, ","),
-      @TK::TkNL      .new( 9, 2,  2, "\n"),
-      @TK::TkSPACE   .new(10, 3,  0, "  "),
-      @TK::TkXSTRING .new(12, 3,  2, "`\\'\\\"\\``"),
-      @TK::TkCOMMA   .new(20, 3, 10, ","),
-      @TK::TkNL      .new(21, 3, 10, "\n"),
-      @TK::TkSPACE   .new(22, 4,  0, "  "),
-      @TK::TkXSTRING .new(24, 4,  2, "`\\#`"),
-      @TK::TkCOMMA   .new(28, 4,  6, ","),
-      @TK::TkNL      .new(29, 4, 22, "\n"),
-      @TK::TkSPACE   .new(30, 5,  0, "  "),
-      @TK::TkXSTRING .new(32, 5,  2, "`\\\#{}`"),
-      @TK::TkCOMMA   .new(38, 5,  8, ","),
-      @TK::TkNL      .new(39, 5, 30, "\n"),
-      @TK::TkSPACE   .new(40, 6,  0, "  "),
-      @TK::TkXSTRING .new(42, 6,  2, "`#`"),
-      @TK::TkCOMMA   .new(45, 6,  5, ","),
-      @TK::TkNL      .new(46, 6, 40, "\n"),
-      @TK::TkSPACE   .new(47, 7,  0, "  "),
-      @TK::TkDXSTRING.new(49, 7,  2, "`\#{}`"),
-      @TK::TkNL      .new(54, 7,  7, "\n"),
-      @TK::TkRBRACK  .new(55, 8,  0, "]"),
-      @TK::TkNL      .new(56, 8, 55, "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_string_escape
-    tokens = RDoc::RubyLex.tokenize '"\\n"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\n\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\r"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\r\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\f"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\f\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\\\"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\\\\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\t"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\t\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\v"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\v\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\a"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\a\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\e"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\e\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\b"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\b\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\s"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\s\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\d"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\d\""), tokens.first
-
-  end
-
-  def test_class_tokenize_string_escape_control
-    tokens = RDoc::RubyLex.tokenize '"\\C-a"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\C-a\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\c\\a"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\c\\a\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\C-\\M-a"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\C-\\M-a\""), tokens.first
-  end
-
-  def test_class_tokenize_string_escape_meta
-    tokens = RDoc::RubyLex.tokenize '"\\M-a"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\M-a\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\M-\\C-a"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\M-\\C-a\""), tokens.first
-  end
-
-  def test_class_tokenize_string_escape_hexadecimal
-    tokens = RDoc::RubyLex.tokenize '"\\x0"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\x0\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\x00"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\x00\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\x000"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\x000\""), tokens.first
-  end
-
-  def test_class_tokenize_string_escape_octal
-    tokens = RDoc::RubyLex.tokenize '"\\0"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\0\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\00"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\00\""), tokens.first
-
-    tokens = RDoc::RubyLex.tokenize '"\\000"', nil
-    assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\000\""), tokens.first
-  end
-
-  def test_class_tokenize_symbol
-    tokens = RDoc::RubyLex.tokenize 'scope module: :v1', nil
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, 'scope'),
-      @TK::TkSPACE     .new( 5, 1,  5, ' '),
-      @TK::TkSYMBOL    .new( 6, 1,  6, 'module:'),
-      @TK::TkSPACE     .new(13, 1, 13, ' '),
-      @TK::TkSYMBOL    .new(14, 1, 14, ':v1'),
-      @TK::TkNL        .new(17, 1, 17, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_particular_kind_of_symbols
-    tokens = RDoc::RubyLex.tokenize '{ Thomas: :Thomas, Dave!: :Dave!, undef: :undef }', nil
-
-    expected = [
-      @TK::TkLBRACE.new( 0, 1,  0, "{"),
-      @TK::TkSPACE .new( 1, 1,  1, " "),
-      @TK::TkSYMBOL.new( 2, 1,  2, "Thomas:"),
-      @TK::TkSPACE .new( 9, 1,  9, " "),
-      @TK::TkSYMBOL.new(10, 1, 10, ":Thomas"),
-      @TK::TkCOMMA .new(17, 1, 17, ","),
-      @TK::TkSPACE .new(18, 1, 18, " "),
-      @TK::TkSYMBOL.new(19, 1, 19, "Dave!:"),
-      @TK::TkSPACE .new(25, 1, 25, " "),
-      @TK::TkSYMBOL.new(26, 1, 26, ":Dave!"),
-      @TK::TkCOMMA .new(32, 1, 32, ","),
-      @TK::TkSPACE .new(33, 1, 33, " "),
-      @TK::TkSYMBOL.new(34, 1, 34, "undef:"),
-      @TK::TkSPACE .new(40, 1, 40, " "),
-      @TK::TkSYMBOL.new(41, 1, 41, ":undef"),
-      @TK::TkSPACE .new(47, 1, 47, " "),
-      @TK::TkRBRACE.new(48, 1, 48, "}"),
-      @TK::TkNL    .new(49, 1, 49, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_symbol_for_nested_method
-    tokens = RDoc::RubyLex.tokenize 'return untrace_var :name', nil
-
-    expected = [
-      @TK::TkRETURN    .new( 0, 1,  0, "return"),
-      @TK::TkSPACE     .new( 6, 1,  6, " "),
-      @TK::TkIDENTIFIER.new( 7, 1,  7, "untrace_var"),
-      @TK::TkSPACE     .new(18, 1, 18, " "),
-      @TK::TkSYMBOL    .new(19, 1, 19, ":name"),
-      @TK::TkNL        .new(24, 1, 24, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_symbol_with_quote
-    tokens = RDoc::RubyLex.tokenize <<RUBY, nil
-a.include?()?"a":"b"
-{"t":1,'t2':2}
-RUBY
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1,  0, "a"),
-      @TK::TkDOT       .new( 1, 1,  1, "."),
-      @TK::TkFID       .new( 2, 1,  2, "include?"),
-      @TK::TkLPAREN    .new(10, 1, 10, "("),
-      @TK::TkRPAREN    .new(11, 1, 11, ")"),
-      @TK::TkQUESTION  .new(12, 1, 12, "?"),
-      @TK::TkSTRING    .new(13, 1, 13, "\"a\""),
-      @TK::TkCOLON     .new(16, 1, 16, ":"),
-      @TK::TkSTRING    .new(17, 1, 17, "\"b\""),
-      @TK::TkNL        .new(20, 1, 20, "\n"),
-      @TK::TkLBRACE    .new(21, 2,  0, "{"),
-      @TK::TkSYMBOL    .new(22, 2,  1, "\"t\":"),
-      @TK::TkINTEGER   .new(26, 2,  5, "1"),
-      @TK::TkCOMMA     .new(27, 2,  6, ","),
-      @TK::TkSYMBOL    .new(28, 2,  7, "'t2':"),
-      @TK::TkINTEGER   .new(33, 2, 12, "2"),
-      @TK::TkRBRACE    .new(34, 2, 13, "}"),
-      @TK::TkNL        .new(35, 2, 21, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_unary_minus
-    ruby_lex = RDoc::RubyLex.new("-1", nil)
-    assert_equal("-1", ruby_lex.token.value)
-
-    ruby_lex = RDoc::RubyLex.new("a[-2]", nil)
-    2.times { ruby_lex.token } # skip "a" and "["
-    assert_equal("-2", ruby_lex.token.value)
-
-    ruby_lex = RDoc::RubyLex.new("a[0..-12]", nil)
-    4.times { ruby_lex.token } # skip "a", "[", "0", and ".."
-    assert_equal("-12", ruby_lex.token.value)
-
-    ruby_lex = RDoc::RubyLex.new("0+-0.1", nil)
-    2.times { ruby_lex.token } # skip "0" and "+"
-    assert_equal("-0.1", ruby_lex.token.value)
-  end
-
-  def test_rational_imaginary_tokenize
-    tokens = RDoc::RubyLex.tokenize '1.11r + 2.34i + 5.55ri + 0i', nil
-
-    expected = [
-      @TK::TkRATIONAL .new( 0, 1,  0, '1.11r'),
-      @TK::TkSPACE    .new( 5, 1,  5, ' '),
-      @TK::TkPLUS     .new( 6, 1,  6, '+'),
-      @TK::TkSPACE    .new( 7, 1,  7, ' '),
-      @TK::TkIMAGINARY.new( 8, 1,  8, '2.34i'),
-      @TK::TkSPACE    .new(13, 1, 13, ' '),
-      @TK::TkPLUS     .new(14, 1, 14, '+'),
-      @TK::TkSPACE    .new(15, 1, 15, ' '),
-      @TK::TkIMAGINARY.new(16, 1, 16, '5.55ri'),
-      @TK::TkSPACE    .new(22, 1, 22, ' '),
-      @TK::TkPLUS     .new(23, 1, 23, '+'),
-      @TK::TkSPACE    .new(24, 1, 24, ' '),
-      @TK::TkIMAGINARY.new(25, 1, 25, '0i'),
-      @TK::TkNL       .new(27, 1, 27, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_square_bracket_as_method
-    tokens = RDoc::RubyLex.tokenize "Array.[](1, 2)", nil
-
-    expected = [
-      @TK::TkCONSTANT  .new(0,  1,  0, "Array"),
-      @TK::TkDOT       .new(5,  1,  5, "."),
-      @TK::TkIDENTIFIER.new(6,  1,  6, "[]"),
-      @TK::TkfLPAREN   .new(8,  1,  8, "("),
-      @TK::TkINTEGER   .new(9,  1,  9, "1"),
-      @TK::TkCOMMA     .new(10, 1, 10, ","),
-      @TK::TkSPACE     .new(11, 1, 11, " "),
-      @TK::TkINTEGER   .new(12, 1, 12, "2"),
-      @TK::TkRPAREN    .new(13, 1, 13, ")"),
-      @TK::TkNL        .new(14, 1, 14, "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_constant_with_exclamation
-    tokens = RDoc::RubyLex.tokenize "Hello there, Dave!", nil
-
-    expected = [
-      @TK::TkCONSTANT  .new( 0, 1,  0, "Hello"),
-      @TK::TkSPACE     .new( 5, 1,  5, " "),
-      @TK::TkIDENTIFIER.new( 6, 1,  6, "there"),
-      @TK::TkCOMMA     .new(11, 1, 11, ","),
-      @TK::TkSPACE     .new(12, 1, 12, " "),
-      @TK::TkIDENTIFIER.new(13, 1, 13, "Dave!"),
-      @TK::TkNL        .new(18, 1, 18, "\n")
-    ]
-
-    assert_equal expected, tokens
-  end
-
-  def test_class_tokenize_identifer_not_equal
-    tokens = RDoc::RubyLex.tokenize "foo!=bar\nfoo?=bar", nil
-
-    expected = [
-      @TK::TkIDENTIFIER.new( 0, 1, 0, "foo"),
-      @TK::TkNEQ       .new( 3, 1, 3, "!="),
-      @TK::TkIDENTIFIER.new( 5, 1, 5, "bar"),
-      @TK::TkNL        .new( 8, 1, 8, "\n"),
-      @TK::TkFID       .new( 9, 2, 0, "foo?"),
-      @TK::TkASSIGN    .new(13, 2, 4, "="),
-      @TK::TkIDENTIFIER.new(14, 2, 5, "bar"),
-      @TK::TkNL        .new(17, 2, 9, "\n"),
-    ]
-
-    assert_equal expected, tokens
-  end
-
-end
-
diff --git a/test/rdoc/test_rdoc_token_stream.rb b/test/rdoc/test_rdoc_token_stream.rb
index 5ed7b1d1a9..95c3ffd13e 100644
--- a/test/rdoc/test_rdoc_token_stream.rb
+++ b/test/rdoc/test_rdoc_token_stream.rb
@@ -5,17 +5,17 @@ class TestRDocTokenStream < RDoc::TestCase
 
   def test_class_to_html
     tokens = [
-      RDoc::RubyToken::TkCONSTANT. new(0, 0, 0, 'CONSTANT'),
-      RDoc::RubyToken::TkDEF.      new(0, 0, 0, 'KW'),
-      RDoc::RubyToken::TkIVAR.     new(0, 0, 0, 'IVAR'),
-      RDoc::RubyToken::TkOp.       new(0, 0, 0, 'Op'),
-      RDoc::RubyToken::TkId.       new(0, 0, 0, 'Id'),
-      RDoc::RubyToken::TkNode.     new(0, 0, 0, 'Node'),
-      RDoc::RubyToken::TkCOMMENT.  new(0, 0, 0, 'COMMENT'),
-      RDoc::RubyToken::TkREGEXP.   new(0, 0, 0, 'REGEXP'),
-      RDoc::RubyToken::TkSTRING.   new(0, 0, 0, 'STRING'),
-      RDoc::RubyToken::TkVal.      new(0, 0, 0, 'Val'),
-      RDoc::RubyToken::TkBACKSLASH.new(0, 0, 0, '\\'),
+      { :line_no => 0, :char_no => 0, :kind => :on_const, :text => 'CONSTANT' },
+      { :line_no => 0, :char_no => 0, :kind => :on_kw, :text => 'KW' },
+      { :line_no => 0, :char_no => 0, :kind => :on_ivar, :text => 'IVAR' },
+      { :line_no => 0, :char_no => 0, :kind => :on_op, :text => 'Op' },
+      { :line_no => 0, :char_no => 0, :kind => :on_ident, :text => 'Id' },
+      { :line_no => 0, :char_no => 0, :kind => :on_backref, :text => 'Node' },
+      { :line_no => 0, :char_no => 0, :kind => :on_comment, :text => 'COMMENT' },
+      { :line_no => 0, :char_no => 0, :kind => :on_regexp, :text => 'REGEXP' },
+      { :line_no => 0, :char_no => 0, :kind => :on_tstring, :text => 'STRING' },
+      { :line_no => 0, :char_no => 0, :kind => :on_int, :text => 'Val' },
+      { :line_no => 0, :char_no => 0, :kind => :on_unknown, :text => '\\' }
     ]
 
     expected = [