Merge rdoc-6.0.0.beta1.

  This version fixed strange behavior of ruby code parser.
  We will list all of impromovement to Changelog when 6.0.0 releasing.

git-svn-id: svn+ssh://ci.ruby-lang.org/ruby/trunk@59686 b2dd03c8-39d4-4d8f-98ff-823fe69b080e

9 files changed, 1202 insertions, 61 deletions

diff --git a/test/rdoc/test_rdoc_code_object.rb b/test/rdoc/test_rdoc_code_object.rb
index e0d89e4e67..be4ca268e4 100644
--- a/test/rdoc/test_rdoc_code_object.rb
+++ b/test/rdoc/test_rdoc_code_object.rb
@@ -276,12 +276,6 @@ class TestRDocCodeObject < XrefTestCase
     assert_equal 'not_rdoc', @co.metadata['markup']
   end
 
-  def test_offset
-    @c1_m.offset = 5
-
-    assert_equal 5, @c1_m.offset
-  end
-
   def test_options
     assert_kind_of RDoc::Options, @co.options
 
diff --git a/test/rdoc/test_rdoc_context.rb b/test/rdoc/test_rdoc_context.rb
index 9af1401f94..54e365f7f4 100644
--- a/test/rdoc/test_rdoc_context.rb
+++ b/test/rdoc/test_rdoc_context.rb
@@ -481,6 +481,32 @@ class TestRDocContext < XrefTestCase
     assert_equal expected_attrs, attrs
   end
 
+  def test_each_section_only_display
+    sects  = []
+    consts = []
+    attrs  = []
+
+    @c7.each_section do |section, constants, attributes|
+      sects  << section
+      consts << constants
+      attrs  << attributes
+    end
+
+    assert_equal [nil], sects.map { |section| section.title }
+
+    expected_consts = [
+      @c7.constants.select(&:display?).sort
+    ]
+
+    assert_equal expected_consts, consts
+
+    expected_attrs = [
+      @c7.attributes.select(&:display?).sort
+    ]
+
+    assert_equal expected_attrs, attrs
+  end
+
   def test_each_section_enumerator
     assert_kind_of Enumerator, @c1.each_section
   end
diff --git a/test/rdoc/test_rdoc_markup_to_html.rb b/test/rdoc/test_rdoc_markup_to_html.rb
index 2e5b4b9300..fd8a7bb0ca 100644
--- a/test/rdoc/test_rdoc_markup_to_html.rb
+++ b/test/rdoc/test_rdoc_markup_to_html.rb
@@ -451,6 +451,22 @@ class TestRDocMarkupToHtml < RDoc::Markup::FormatterTestCase
     assert_equal expected, @to.res.join
   end
 
+  def test_accept_verbatim_nl_after_backslash
+    verb = @RM::Verbatim.new("a = 1 if first_flag_var and \\\n", "  this_is_flag_var\n")
+
+    @to.start_accepting
+    @to.accept_verbatim verb
+
+    expected = <<-EXPECTED
+
+<pre class="ruby"><span class="ruby-identifier">a</span> = <span class="ruby-value">1</span> <span class="ruby-keyword">if</span> <span class="ruby-identifier">first_flag_var</span> <span class="ruby-keyword">and</span> \\
+  <span class="ruby-identifier">this_is_flag_var</span>
+</pre>
+    EXPECTED
+
+    assert_equal expected, @to.res.join
+  end
+
   def test_accept_verbatim_pipe
     @options.pipe = true
 
@@ -469,6 +485,106 @@ class TestRDocMarkupToHtml < RDoc::Markup::FormatterTestCase
     assert_equal expected, @to.res.join
   end
 
+  def test_accept_verbatim_escape_in_string
+    code = <<-'RUBY'
+def foo
+  [
+    '\\',
+    '\'',
+    "'",
+    "\'\"\`",
+    "\#",
+    "\#{}",
+    "#",
+    "#{}",
+    /'"/,
+    /\'\"/,
+    /\//,
+    /\\/,
+    /\#/,
+    /\#{}/,
+    /#/,
+    /#{}/
+  ]
+end
+def bar
+end
+    RUBY
+    verb = @RM::Verbatim.new(*code.split(/(?<=\n)/))
+
+    @to.start_accepting
+    @to.accept_verbatim verb
+
+    expected = <<-'EXPECTED'
+
+<pre class="ruby"><span class="ruby-keyword">def</span> <span class="ruby-identifier">foo</span>
+  [
+    <span class="ruby-string">&#39;\\&#39;</span>,
+    <span class="ruby-string">&#39;\&#39;&#39;</span>,
+    <span class="ruby-string">&quot;&#39;&quot;</span>,
+    <span class="ruby-string">&quot;\&#39;\&quot;\`&quot;</span>,
+    <span class="ruby-string">&quot;\#&quot;</span>,
+    <span class="ruby-string">&quot;\#{}&quot;</span>,
+    <span class="ruby-string">&quot;#&quot;</span>,
+    <span class="ruby-node">&quot;#{}&quot;</span>,
+    <span class="ruby-regexp">/&#39;&quot;/</span>,
+    <span class="ruby-regexp">/\&#39;\&quot;/</span>,
+    <span class="ruby-regexp">/\//</span>,
+    <span class="ruby-regexp">/\\/</span>,
+    <span class="ruby-regexp">/\#/</span>,
+    <span class="ruby-regexp">/\#{}/</span>,
+    <span class="ruby-regexp">/#/</span>,
+    <span class="ruby-regexp">/#{}/</span>
+  ]
+<span class="ruby-keyword">end</span>
+<span class="ruby-keyword">def</span> <span class="ruby-identifier">bar</span>
+<span class="ruby-keyword">end</span>
+</pre>
+    EXPECTED
+
+    assert_equal expected, @to.res.join
+  end
+
+  def test_accept_verbatim_escape_in_backtick
+    code = <<-'RUBY'
+def foo
+  [
+    `\\`,
+    `\'\"\``,
+    `\#`,
+    `\#{}`,
+    `#`,
+    `#{}`
+  ]
+end
+def bar
+end
+    RUBY
+    verb = @RM::Verbatim.new(*code.split(/(?<=\n)/))
+
+    @to.start_accepting
+    @to.accept_verbatim verb
+
+    expected = <<-'EXPECTED'
+
+<pre class="ruby"><span class="ruby-keyword">def</span> <span class="ruby-identifier">foo</span>
+  [
+    <span class="ruby-string">`\\`</span>,
+    <span class="ruby-string">`\&#39;\&quot;\``</span>,
+    <span class="ruby-string">`\#`</span>,
+    <span class="ruby-string">`\#{}`</span>,
+    <span class="ruby-string">`#`</span>,
+    <span class="ruby-node">`#{}`</span>
+  ]
+<span class="ruby-keyword">end</span>
+<span class="ruby-keyword">def</span> <span class="ruby-identifier">bar</span>
+<span class="ruby-keyword">end</span>
+</pre>
+    EXPECTED
+
+    assert_equal expected, @to.res.join
+  end
+
   def test_accept_verbatim_ruby
     verb = @RM::Verbatim.new("1 + 1\n")
     verb.format = :ruby
@@ -485,6 +601,36 @@ class TestRDocMarkupToHtml < RDoc::Markup::FormatterTestCase
     assert_equal expected, @to.res.join
   end
 
+  def test_accept_verbatim_redefinable_operators
+    functions = %w[| ^ & <=> == === =~ > >= < <= << >> + - * / % ** ~ +@ -@ [] []= ` !  != !~].map { |redefinable_op|
+      ["def #{redefinable_op}\n", "end\n"]
+    }.flatten
+
+    verb = @RM::Verbatim.new(*functions)
+
+    @to.start_accepting
+    @to.accept_verbatim verb
+
+    expected = <<-EXPECTED
+
+<pre class="ruby">
+    EXPECTED
+    expected = expected.rstrip
+
+    %w[| ^ &amp; &lt;=&gt; == === =~ &gt; &gt;= &lt; &lt;= &lt;&lt; &gt;&gt; + - * / % ** ~ +@ -@ [] []= ` !  != !~].each do |html_escaped_op|
+      expected += <<-EXPECTED
+<span class="ruby-keyword">def</span> <span class="ruby-identifier">#{html_escaped_op}</span>
+<span class="ruby-keyword">end</span>
+      EXPECTED
+    end
+
+    expected += <<-EXPECTED
+</pre>
+EXPECTED
+
+    assert_equal expected, @to.res.join
+  end
+
   def test_convert_string
     assert_equal '&lt;&gt;', @to.convert_string('<>')
   end
diff --git a/test/rdoc/test_rdoc_parser_c.rb b/test/rdoc/test_rdoc_parser_c.rb
index 09d7c4b16d..d0732597dc 100644
--- a/test/rdoc/test_rdoc_parser_c.rb
+++ b/test/rdoc/test_rdoc_parser_c.rb
@@ -1139,7 +1139,6 @@ Init_Foo(void) {
     assert_equal 'my_method', other_function.name
     assert_equal 'a comment for rb_other_function', other_function.comment.text
     assert_equal '()', other_function.params
-    assert_equal 118, other_function.offset
     assert_equal 8, other_function.line
 
     code = other_function.token_stream.first.text
@@ -1173,7 +1172,6 @@ Init_Foo(void) {
     assert_equal 'my_method', other_function.name
     assert_equal 'a comment for other_function', other_function.comment.text
     assert_equal '()', other_function.params
-    assert_equal 39, other_function.offset
     assert_equal 4, other_function.line
 
     code = other_function.token_stream.first.text
@@ -1402,7 +1400,6 @@ rb_m(int argc, VALUE *argv, VALUE obj) {
 
     assert_equal 'm', m.name
     assert_equal @top_level, m.file
-    assert_equal 115, m.offset
     assert_equal 7, m.line
 
     assert_equal '(p1)', m.params
diff --git a/test/rdoc/test_rdoc_parser_ruby.rb b/test/rdoc/test_rdoc_parser_ruby.rb
index 44b38c28aa..55c03d4628 100644
--- a/test/rdoc/test_rdoc_parser_ruby.rb
+++ b/test/rdoc/test_rdoc_parser_ruby.rb
@@ -74,7 +74,7 @@ class C; end
 
     comment = parser.collect_first_comment
 
-    assert_equal RDoc::Comment.new("first\n\n", @top_level), comment
+    assert_equal RDoc::Comment.new("=begin\nfirst\n=end\n\n", @top_level), comment
   end
 
   def test_get_class_or_module
@@ -282,6 +282,31 @@ class C; end
     assert_equal 'blah', @top_level.metadata['unhandled']
   end
 
+  def test_parse_for_in
+    klass = RDoc::NormalClass.new 'Foo'
+    klass.parent = @top_level
+
+    comment = RDoc::Comment.new '', @top_level
+
+    util_parser <<ruby
+def sum(n)
+  result = 0
+  for i in 1..n do
+    result += i
+  end
+  result
+end
+ruby
+
+    tk = @parser.get_tk
+
+    @parser.parse_method klass, RDoc::Parser::Ruby::NORMAL, tk, comment
+
+    sum = klass.method_list.first
+    assert_equal 'sum',      sum.name
+    assert_equal @top_level, sum.file
+  end
+
   def test_parse_alias
     klass = RDoc::NormalClass.new 'Foo'
     klass.parent = @top_level
@@ -297,7 +322,6 @@ class C; end
     assert_equal klass,      alas.parent
     assert_equal 'comment',  alas.comment
     assert_equal @top_level, alas.file
-    assert_equal 0,          alas.offset
     assert_equal 1,          alas.line
   end
 
@@ -365,7 +389,6 @@ class C; end
     assert_equal 'foo', foo.name
     assert_equal 'my attr', foo.comment.text
     assert_equal @top_level, foo.file
-    assert_equal 0, foo.offset
     assert_equal 1, foo.line
   end
 
@@ -404,7 +427,6 @@ class C; end
     assert_equal 'RW', foo.rw
     assert_equal 'my attr', foo.comment.text
     assert_equal @top_level, foo.file
-    assert_equal 0, foo.offset
     assert_equal 1, foo.line
 
     bar = klass.attributes.last
@@ -617,7 +639,6 @@ class C; end
     assert_equal 'Foo', foo.full_name
     assert_equal 'my class', foo.comment.text
     assert_equal [@top_level], foo.in_files
-    assert_equal 0, foo.offset
     assert_equal 1, foo.line
   end
 
@@ -637,7 +658,6 @@ end
 
     c = @top_level.classes.first
     assert_equal 'C', c.full_name
-    assert_equal 0, c.offset
     assert_equal 1, c.line
   end
 
@@ -727,7 +747,6 @@ end
     assert_equal 'Foo', foo.full_name
     assert_empty foo.comment
     assert_equal [@top_level], foo.in_files
-    assert_equal 0, foo.offset
     assert_equal 1, foo.line
   end
 
@@ -758,6 +777,16 @@ end
     assert_empty @top_level.classes.first.comment
   end
 
+  def test_parse_class_lower_name_warning
+    @options.verbosity = 2
+    out, err = capture_io do
+      util_parser "class foo\nend"
+      tk = @parser.get_tk
+      @parser.parse_class @top_level, RDoc::Parser::Ruby::NORMAL, tk, @comment
+    end
+    assert_match /Expected class name or '<<'\. Got/, err
+  end
+
   def test_parse_multi_ghost_methods
     util_parser <<-'CLASS'
 class Foo
@@ -920,7 +949,6 @@ end
     assert_equal %w[A::B A::d], modules.map { |c| c.full_name }
 
     b = modules.first
-    assert_equal 10, b.offset
     assert_equal 2,  b.line
 
     # make sure method/alias was not added to enclosing class/module
@@ -1083,7 +1111,6 @@ EOF
     assert_equal 'RW',       foo.rw
     assert_equal 'my attr',  foo.comment.text
     assert_equal @top_level, foo.file
-    assert_equal 0,          foo.offset
     assert_equal 1,          foo.line
 
     assert_equal nil,        foo.viewer
@@ -1147,7 +1174,6 @@ EOF
     assert_equal 'foo',       foo.name
     assert_equal 'my method', foo.comment.text
     assert_equal @top_level,  foo.file
-    assert_equal 0,           foo.offset
     assert_equal 1,           foo.line
 
     assert_equal [],        foo.aliases
@@ -1223,7 +1249,6 @@ EOF
 
     assert_equal 'A', foo.name
     assert_equal @top_level, foo.file
-    assert_equal 0, foo.offset
     assert_equal 1, foo.line
   end
 
@@ -1389,7 +1414,6 @@ A::B::C = 1
     assert_equal 'foo',       foo.name
     assert_equal 'my method', foo.comment.text
     assert_equal @top_level,  foo.file
-    assert_equal 0,           foo.offset
     assert_equal 1,           foo.line
 
     assert_equal [],      foo.aliases
@@ -1587,7 +1611,6 @@ end
     assert_equal 'foo',       foo.name
     assert_equal 'my method', foo.comment.text
     assert_equal @top_level,  foo.file
-    assert_equal 0,           foo.offset
     assert_equal 1,           foo.line
 
     assert_equal [],        foo.aliases
@@ -1627,6 +1650,47 @@ end
     assert_equal stream, foo.token_stream
   end
 
+  def test_parse_redefinable_methods
+    klass = RDoc::NormalClass.new 'Foo'
+    klass.parent = @top_level
+
+    comment = RDoc::Comment.new "", @top_level
+
+    redefinable_ops = %w[| ^ & <=> == === =~ > >= < <= << >> + - * / % ** ~ +@ -@ [] []= ` !  != !~]
+    redefinable_ops.each do |redefinable_op|
+      util_parser "def #{redefinable_op}\nend\n"
+      tk = @parser.get_tk
+      @parser.parse_method klass, RDoc::Parser::Ruby::NORMAL, tk, comment
+    end
+
+    klass.method_list.each do |method|
+      assert_kind_of RDoc::RubyToken::TkId, method.token_stream[5]
+      assert_includes redefinable_ops, method.token_stream[5].text
+    end
+  end
+
+  def test_parse_method_bracket
+    util_parser <<-RUBY
+class C
+  def [] end
+  def self.[] end
+  def []= end
+  def self.[]= end
+end
+    RUBY
+
+    @parser.scan
+
+    c = @store.find_class_named 'C'
+
+    assert_equal 4, c.method_list.size
+    assert_equal 'C#[]', c.method_list[0].full_name
+    assert_equal 'C::[]', c.method_list[1].full_name
+    assert_equal 'C#[]=', c.method_list[2].full_name
+    assert_equal 'C::[]=', c.method_list[3].full_name
+    assert c.aliases.empty?
+  end
+
   def test_parse_method_alias
     klass = RDoc::NormalClass.new 'Foo'
     klass.parent = @top_level
@@ -1978,6 +2042,30 @@ end
     assert_equal 'Foo#blah', methods.first.full_name
   end
 
+  def test_parse_statements_postfix_if_unless
+    util_parser <<-CODE
+class C
+  def foo
+    1 if nil
+  end
+
+  def bar
+    2 unless nil
+  end
+end
+    CODE
+
+    @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL, nil
+
+    c = @top_level.classes.first
+    assert_equal 'C', c.full_name, 'class C'
+
+    methods = c.method_list
+    assert_equal 2, methods.length
+    assert_equal 'C#foo', methods[0].full_name
+    assert_equal 'C#bar', methods[1].full_name
+  end
+
   def test_parse_statements_class_nested
     comment = RDoc::Comment.new "##\n# my method\n", @top_level
 
@@ -1994,7 +2082,7 @@ end
   end
 
   def test_parse_statements_def_percent_string_pound
-    util_parser "class C\ndef a\n%r{#}\nend\ndef b() end\nend"
+    util_parser "class C\ndef a\n%r{#}\n%r{\#{}}\nend\ndef b() end\nend"
 
     @parser.parse_statements @top_level, RDoc::Parser::Ruby::NORMAL
 
@@ -2011,9 +2099,11 @@ end
       tk(:SPACE,      11, 2, 3, nil,   ' '),
       tk(:IDENTIFIER, 12, 2, 4, 'a',   'a'),
       tk(:NL,         13, 2, 5, nil,   "\n"),
-      tk(:DREGEXP,    14, 3, 0, nil,   '%r{#}'),
+      tk(:REGEXP,     14, 3, 0, nil,   '%r{#}'),
       tk(:NL,         19, 3, 5, nil,   "\n"),
-      tk(:END,        20, 4, 0, 'end', 'end'),
+      tk(:DREGEXP,    20, 4, 0, nil,   '%r{#{}}'),
+      tk(:NL,         27, 4, 7, nil,   "\n"),
+      tk(:END,        28, 5, 0, 'end', 'end'),
     ]
 
     assert_equal expected, a.token_stream
@@ -2377,6 +2467,162 @@ end
     assert_equal :private, date_time_now.visibility, date_time_now.full_name
   end
 
+  def test_parse_statements_complex_condition_in_for
+    util_parser <<RUBY
+class Foo
+  def blah()
+    for i in (k)...n do
+    end
+    for i in (k)...n
+    end
+  end
+end
+RUBY
+
+    expected = <<EXPTECTED
+<span class="ruby-keyword">def</span> <span class="ruby-identifier">blah</span>()
+  <span class="ruby-keyword">for</span> <span class="ruby-identifier">i</span> <span class="ruby-keyword">in</span> (<span class="ruby-identifier">k</span>)<span class="ruby-operator">...</span><span class="ruby-identifier">n</span> <span class="ruby-keyword">do</span>
+  <span class="ruby-keyword">end</span>
+  <span class="ruby-keyword">for</span> <span class="ruby-identifier">i</span> <span class="ruby-keyword">in</span> (<span class="ruby-identifier">k</span>)<span class="ruby-operator">...</span><span class="ruby-identifier">n</span>
+  <span class="ruby-keyword">end</span>
+<span class="ruby-keyword">end</span>
+EXPTECTED
+    expected = expected.rstrip
+
+    @parser.scan
+
+    foo = @top_level.classes.first
+    assert_equal 'Foo', foo.full_name
+
+    blah = foo.method_list.first
+    markup_code = blah.markup_code.sub(/^.*\n/, '')
+    assert_equal markup_code, expected
+  end
+
+  def test_parse_statements_postfix_if_after_heredocbeg
+    @filename = 'file.rb'
+    util_parser <<RUBY
+class Foo
+  def blah()
+    <<~EOM if true
+    EOM
+  end
+end
+RUBY
+
+    expected = <<EXPTECTED
+  <span class="ruby-keyword">def</span> <span class="ruby-identifier">blah</span>()
+    <span class="ruby-identifier">&lt;&lt;~EOM</span> <span class="ruby-keyword">if</span> <span class="ruby-keyword">true</span>
+<span class="ruby-value"></span><span class="ruby-identifier">    EOM</span>
+  <span class="ruby-keyword">end</span>
+EXPTECTED
+    expected = expected.rstrip
+
+    @parser.scan
+
+    foo = @top_level.classes.first
+    assert_equal 'Foo', foo.full_name
+
+    blah = foo.method_list.first
+    markup_code = blah.markup_code.sub(/^.*\n/, '')
+    assert_equal markup_code, expected
+  end
+
+  def test_parse_statements_method_oneliner_with_regexp
+    util_parser <<RUBY
+class Foo
+  def blah() /bar/ end
+end
+RUBY
+
+    expected = <<EXPTECTED
+<span class="ruby-keyword">def</span> <span class="ruby-identifier">blah</span>() <span class="ruby-regexp">/bar/</span> <span class="ruby-keyword">end</span>
+EXPTECTED
+    expected = expected.rstrip
+
+    @parser.scan
+
+    foo = @top_level.classes.first
+    assert_equal 'Foo', foo.full_name
+
+    blah = foo.method_list.first
+    markup_code = blah.markup_code.sub(/^.*\n/, '')
+    assert_equal expected, markup_code
+  end
+
+  def test_parse_statements_embdoc_in_document
+    @filename = 'file.rb'
+    util_parser <<RUBY
+class Foo
+  # doc
+  #
+  #   =begin
+  #   test embdoc
+  #   =end
+  #
+  def blah
+  end
+end
+RUBY
+
+    expected = <<EXPTECTED
+<p>doc
+
+<pre class="ruby"><span class="ruby-comment">=begin
+test embdoc
+=end</span>
+</pre>
+EXPTECTED
+
+    @parser.scan
+
+    foo = @top_level.classes.first
+    assert_equal 'Foo', foo.full_name
+
+    blah = foo.method_list.first
+    markup_comment = blah.search_record[6]
+    assert_equal markup_comment, expected
+  end
+
+  def test_parse_require_dynamic_string
+    content = <<-RUBY
+prefix = 'path'
+require "\#{prefix}/a_library"
+require 'test'
+RUBY
+
+    util_parser content
+
+    @parser.parse_statements @top_level
+
+    assert_equal 1, @top_level.requires.length
+  end
+
+  def test_parse_postfix_nodoc
+    util_parser <<-RUBY
+class A
+end # :nodoc:
+
+class B
+  def a
+  end # :nodoc:
+
+  def b
+  end
+end
+RUBY
+
+    @parser.parse_statements @top_level
+
+    c_a = @top_level.classes.select(&:document_self).first
+    assert_equal 'B', c_a.full_name
+
+    assert_equal 2, @top_level.classes.length
+    assert_equal 1, @top_level.classes.count(&:document_self)
+    assert_equal 1, c_a.method_list.length
+    assert_equal 'B#b', c_a.method_list.first.full_name
+  end
+
   def test_parse_statements_identifier_require
     content = "require 'bar'"
 
@@ -2504,6 +2750,25 @@ end
     assert_equal 'A#b', m_b.full_name
   end
 
+
+  def test_parse_symbol_in_paren_arg
+    util_parser <<RUBY
+class Foo
+  def blah
+  end
+  private(:blah)
+end
+RUBY
+
+    @parser.scan
+
+    foo = @top_level.classes.first
+    assert_equal 'Foo', foo.full_name
+
+    blah = foo.method_list.first
+    assert_equal :private, blah.visibility
+  end
+
   def test_parse_symbol_in_arg
     util_parser ':blah "blah" "#{blah}" blah'
 
@@ -2736,7 +3001,7 @@ end
   def test_sanity_interpolation_curly
     util_parser '%{ #{} }'
 
-    assert_equal '%Q{ #{} }', @parser.get_tk.text
+    assert_equal '%{ #{} }', @parser.get_tk.text
     assert_equal RDoc::RubyToken::TkNL, @parser.get_tk.class
   end
 
@@ -2793,11 +3058,11 @@ end
 
     foo = @top_level.classes.first
 
-    assert_equal 'Foo comment', foo.comment.text
+    assert_equal "=begin rdoc\nFoo comment\n=end", foo.comment.text
 
     m = foo.method_list.first
 
-    assert_equal 'm comment', m.comment.text
+    assert_equal "=begin\nm comment\n=end", m.comment.text
   end
 
   def test_scan_block_comment_nested # Issue #41
@@ -2819,7 +3084,7 @@ end
     foo = @top_level.modules.first
 
     assert_equal 'Foo', foo.full_name
-    assert_equal 'findmeindoc', foo.comment.text
+    assert_equal "=begin rdoc\nfindmeindoc\n=end", foo.comment.text
 
     bar = foo.classes.first
 
@@ -2866,12 +3131,12 @@ end
 
     foo = @top_level.classes.first
 
-    assert_equal "= DESCRIPTION\n\nThis is a simple test class\n\n= RUMPUS\n\nIs a silly word",
+    assert_equal "=begin rdoc\n\n= DESCRIPTION\n\nThis is a simple test class\n\n= RUMPUS\n\nIs a silly word\n\n=end",
       foo.comment.text
 
     m = foo.method_list.first
 
-    assert_equal 'A nice girl', m.comment.text
+    assert_equal "=begin rdoc\nA nice girl\n=end", m.comment.text
   end
 
   def test_scan_class_nested_nodoc
@@ -3245,6 +3510,33 @@ end
     assert c_b.singleton
   end
 
+  def test_scan_visibility_count
+    util_parser <<-RUBY
+class C < Original::Base
+  class C2 < Original::Base
+    def m0() end
+    def m1() end
+
+    private
+
+    def m2() end
+    def m3() end
+    def m4() end
+  end
+end
+    RUBY
+
+    @parser.scan
+
+    c = @store.find_class_named 'C::C2'
+
+    private_method_count = c.method_list.count { |m| :private == m.visibility }
+    assert_equal 3, private_method_count
+
+    public_method_count = c.method_list.count { |m| :public == m.visibility }
+    assert_equal 2, public_method_count
+  end
+
   def test_singleton_method_via_eigenclass
     util_parser <<-RUBY
 class C
diff --git a/test/rdoc/test_rdoc_ruby_lex.rb b/test/rdoc/test_rdoc_ruby_lex.rb
index 6356fa2894..9ab5844b54 100644
--- a/test/rdoc/test_rdoc_ruby_lex.rb
+++ b/test/rdoc/test_rdoc_ruby_lex.rb
@@ -9,6 +9,31 @@ class TestRDocRubyLex < RDoc::TestCase
     @TK = RDoc::RubyToken
   end
 
+  def test_token_position
+    tokens = RDoc::RubyLex.tokenize '[ 1, :a, nil ]', nil
+
+    assert_equal '[', tokens[0].text
+    assert_equal 0, tokens[0].seek
+    assert_equal 1, tokens[0].line_no
+    assert_equal 0, tokens[0].char_no
+    assert_equal '1', tokens[2].text
+    assert_equal 2, tokens[2].seek
+    assert_equal 1, tokens[2].line_no
+    assert_equal 2, tokens[2].char_no
+    assert_equal ':a', tokens[5].text
+    assert_equal 5, tokens[5].seek
+    assert_equal 1, tokens[5].line_no
+    assert_equal 5, tokens[5].char_no
+    assert_equal 'nil', tokens[8].text
+    assert_equal 9, tokens[8].seek
+    assert_equal 1, tokens[8].line_no
+    assert_equal 9, tokens[8].char_no
+    assert_equal ']', tokens[10].text
+    assert_equal 13, tokens[10].seek
+    assert_equal 1, tokens[10].line_no
+    assert_equal 13, tokens[10].char_no
+  end
+
   def test_class_tokenize
     tokens = RDoc::RubyLex.tokenize "def x() end", nil
 
@@ -37,17 +62,39 @@ class TestRDocRubyLex < RDoc::TestCase
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize___ENCODING__
+    tokens = RDoc::RubyLex.tokenize '__ENCODING__', nil
+
+    expected = [
+      @TK::Tk__ENCODING__.new( 0, 1,  0, '__ENCODING__'),
+      @TK::TkNL          .new(12, 1, 12, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_character_literal
-    tokens = RDoc::RubyLex.tokenize "?\\", nil
+    tokens = RDoc::RubyLex.tokenize "?c", nil
 
     expected = [
-      @TK::TkCHAR.new( 0, 1,  0, "?\\"),
+      @TK::TkCHAR.new( 0, 1,  0, "?c"),
       @TK::TkNL  .new( 2, 1,  2, "\n"),
     ]
 
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_character_literal_with_escape
+    tokens = RDoc::RubyLex.tokenize "?\\s", nil
+
+    expected = [
+      @TK::TkCHAR.new( 0, 1,  0, "?\\s"),
+      @TK::TkNL  .new( 3, 1,  3, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_def_heredoc
     tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
 def x
@@ -64,9 +111,11 @@ end
       @TK::TkIDENTIFIER.new( 4, 1,  4, 'x'),
       @TK::TkNL        .new( 5, 1,  5, "\n"),
       @TK::TkSPACE     .new( 6, 2,  0, '  '),
-      @TK::TkHEREDOC   .new( 8, 2,  2,
-                            %Q{<<E\nLine 1\nLine 2\nE}),
-      @TK::TkNL        .new(27, 5, 28, "\n"),
+
+      @TK::TkHEREDOCBEG.new( 8, 2,  2, '<<E'),
+      @TK::TkNL        .new(11, 2,  6, "\n"),
+      @TK::TkHEREDOC   .new(11, 2,  6, "Line 1\nLine 2\n"),
+      @TK::TkHEREDOCEND.new(27, 5, 26, "E\n"),
       @TK::TkEND       .new(28, 6,  0, 'end'),
       @TK::TkNL        .new(31, 6, 28, "\n"),
     ]
@@ -74,18 +123,89 @@ end
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_reserved_keyword_with_args
+    tokens = RDoc::RubyLex.tokenize <<-'RUBY', nil
+yield :foo
+super :bar
+defined? :baz
+    RUBY
+
+    expected = [
+      @TK::TkYIELD  .new( 0, 1,  0, "yield"),
+      @TK::TkSPACE  .new( 5, 1,  5, " "),
+      @TK::TkSYMBOL .new( 6, 1,  6, ":foo"),
+      @TK::TkNL     .new(10, 1, 10,  "\n"),
+      @TK::TkSUPER  .new(11, 2,  0, "super"),
+      @TK::TkSPACE  .new(16, 2,  5, " "),
+      @TK::TkSYMBOL .new(17, 2,  6, ":bar"),
+      @TK::TkNL     .new(21, 2, 11,  "\n"),
+      @TK::TkDEFINED.new(22, 3,  0, "defined?"),
+      @TK::TkSPACE  .new(30, 3,  8, " "),
+      @TK::TkSYMBOL .new(31, 3,  9, ":baz"),
+      @TK::TkNL     .new(35, 3, 22,  "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_hash_symbol
     tokens = RDoc::RubyLex.tokenize '{ class:"foo" }', nil
 
     expected = [
+      @TK::TkLBRACE.new( 0, 1,  0, '{'),
+      @TK::TkSPACE .new( 1, 1,  1, ' '),
+      @TK::TkSYMBOL.new( 2, 1,  2, 'class:'),
+      @TK::TkSTRING.new( 8, 1,  8, '"foo"'),
+      @TK::TkSPACE .new(13, 1, 13, ' '),
+      @TK::TkRBRACE.new(14, 1, 14, '}'),
+      @TK::TkNL    .new(15, 1, 15, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_double_colon_is_not_hash_symbol
+    tokens = RDoc::RubyLex.tokenize 'self.class::Row', nil
+
+    expected = [
+      @TK::TkSELF      .new( 0, 1,  0, "self"),
+      @TK::TkDOT       .new( 4, 1,  4, "."),
+      @TK::TkIDENTIFIER.new( 5, 1,  5, "class"),
+      @TK::TkCOLON2    .new(10, 1, 10, "::"),
+      @TK::TkCONSTANT  .new(12, 1, 12, "Row"),
+      @TK::TkNL        .new(15, 1, 15, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_safe_nav_operator
+    tokens = RDoc::RubyLex.tokenize 'receiver&.meth', nil
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, "receiver"),
+      @TK::TkSAFENAV   .new( 8, 1,  8, "&."),
+      @TK::TkIDENTIFIER.new(10, 1, 10, "meth"),
+      @TK::TkNL        .new(14, 1, 14, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_hash_rocket
+    tokens = RDoc::RubyLex.tokenize '{ :class => "foo" }', nil
+
+    expected = [
       @TK::TkLBRACE    .new( 0, 1,  0, '{'),
       @TK::TkSPACE     .new( 1, 1,  1, ' '),
-      @TK::TkIDENTIFIER.new( 2, 1,  2, 'class'),
-      @TK::TkSYMBEG    .new( 7, 1,  7, ':'),
-      @TK::TkSTRING    .new( 8, 1,  8, '"foo"'),
-      @TK::TkSPACE     .new(13, 1, 13, ' '),
-      @TK::TkRBRACE    .new(14, 1, 14, '}'),
-      @TK::TkNL        .new(15, 1, 15, "\n"),
+      @TK::TkSYMBOL    .new( 2, 1,  2, ':class'),
+      @TK::TkSPACE     .new( 8, 1,  8, ' '),
+      @TK::TkHASHROCKET.new( 9, 1,  9, '=>'),
+      @TK::TkSPACE     .new(11, 1, 11, ' '),
+      @TK::TkSTRING    .new(12, 1, 12, '"foo"'),
+      @TK::TkSPACE     .new(17, 1, 17, ' '),
+      @TK::TkRBRACE    .new(18, 1, 18, '}'),
+      @TK::TkNL        .new(19, 1, 19, "\n"),
     ]
 
     assert_equal expected, tokens
@@ -104,10 +224,106 @@ Line 2\r
       @TK::TkSPACE     .new( 6, 1,  6, ' '),
       @TK::TkASSIGN    .new( 7, 1,  7, '='),
       @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkHEREDOC   .new( 9, 1,  9,
-                            %Q{<<-STRING\nLine 1\nLine 2\n  STRING}),
-      @TK::TkSPACE     .new(44, 4, 45, "\r"),
-      @TK::TkNL        .new(45, 4, 46, "\n"),
+      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
+      @TK::TkSPACE     .new(18, 1, 18, "\r"),
+      @TK::TkNL        .new(19, 1, 19, "\n"),
+      @TK::TkHEREDOC   .new(19, 1, 19,
+                            %Q{Line 1\nLine 2\n}),
+      @TK::TkHEREDOCEND.new(45, 4, 36, "  STRING\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_opassign
+    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
+a %= b
+a /= b
+a -= b
+a += b
+a |= b
+a &= b
+a >>= b
+a <<= b
+a *= b
+a &&= b
+a ||= b
+a **= b
+RUBY
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1, 0, "a"),
+      @TK::TkSPACE     .new( 1, 1, 1, " "),
+      @TK::TkOPASGN    .new( 2, 1, 2, "%"),
+      @TK::TkSPACE     .new( 4, 1, 4, " "),
+      @TK::TkIDENTIFIER.new( 5, 1, 5, "b"),
+      @TK::TkNL        .new( 6, 1, 6, "\n"),
+      @TK::TkIDENTIFIER.new( 7, 2, 0, "a"),
+      @TK::TkSPACE     .new( 8, 2, 1, " "),
+      @TK::TkOPASGN    .new( 9, 2, 2, "/"),
+      @TK::TkSPACE     .new( 11, 2, 4, " "),
+      @TK::TkIDENTIFIER.new( 12, 2, 5, "b"),
+      @TK::TkNL        .new( 13, 2, 7, "\n"),
+      @TK::TkIDENTIFIER.new( 14, 3, 0, "a"),
+      @TK::TkSPACE     .new( 15, 3, 1, " "),
+      @TK::TkOPASGN    .new( 16, 3, 2, "-"),
+      @TK::TkSPACE     .new( 18, 3, 4, " "),
+      @TK::TkIDENTIFIER.new( 19, 3, 5, "b"),
+      @TK::TkNL        .new( 20, 3, 14, "\n"),
+      @TK::TkIDENTIFIER.new( 21, 4, 0, "a"),
+      @TK::TkSPACE     .new( 22, 4, 1, " "),
+      @TK::TkOPASGN    .new( 23, 4, 2, "+"),
+      @TK::TkSPACE     .new( 25, 4, 4, " "),
+      @TK::TkIDENTIFIER.new( 26, 4, 5, "b"),
+      @TK::TkNL        .new( 27, 4, 21, "\n"),
+      @TK::TkIDENTIFIER.new( 28, 5, 0, "a"),
+      @TK::TkSPACE     .new( 29, 5, 1, " "),
+      @TK::TkOPASGN    .new( 30, 5, 2, "|"),
+      @TK::TkSPACE     .new( 32, 5, 4, " "),
+      @TK::TkIDENTIFIER.new( 33, 5, 5, "b"),
+      @TK::TkNL        .new( 34, 5, 28, "\n"),
+      @TK::TkIDENTIFIER.new( 35, 6, 0, "a"),
+      @TK::TkSPACE     .new( 36, 6, 1, " "),
+      @TK::TkOPASGN    .new( 37, 6, 2, "&"),
+      @TK::TkSPACE     .new( 39, 6, 4, " "),
+      @TK::TkIDENTIFIER.new( 40, 6, 5, "b"),
+      @TK::TkNL        .new( 41, 6, 35, "\n"),
+      @TK::TkIDENTIFIER.new( 42, 7, 0, "a"),
+      @TK::TkSPACE     .new( 43, 7, 1, " "),
+      @TK::TkOPASGN    .new( 44, 7, 2, ">>"),
+      @TK::TkSPACE     .new( 47, 7, 5, " "),
+      @TK::TkIDENTIFIER.new( 48, 7, 6, "b"),
+      @TK::TkNL        .new( 49, 7, 42, "\n"),
+      @TK::TkIDENTIFIER.new( 50, 8, 0, "a"),
+      @TK::TkSPACE     .new( 51, 8, 1, " "),
+      @TK::TkOPASGN    .new( 52, 8, 2, "<<"),
+      @TK::TkSPACE     .new( 55, 8, 5, " "),
+      @TK::TkIDENTIFIER.new( 56, 8, 6, "b"),
+      @TK::TkNL        .new( 57, 8, 50, "\n"),
+      @TK::TkIDENTIFIER.new( 58, 9, 0, "a"),
+      @TK::TkSPACE     .new( 59, 9, 1, " "),
+      @TK::TkOPASGN    .new( 60, 9, 2, "*"),
+      @TK::TkSPACE     .new( 62, 9, 4, " "),
+      @TK::TkIDENTIFIER.new( 63, 9, 5, "b"),
+      @TK::TkNL        .new( 64, 9, 58, "\n"),
+      @TK::TkIDENTIFIER.new( 65, 10, 0, "a"),
+      @TK::TkSPACE     .new( 66, 10, 1, " "),
+      @TK::TkOPASGN    .new( 67, 10, 2, "&&"),
+      @TK::TkSPACE     .new( 70, 10, 5, " "),
+      @TK::TkIDENTIFIER.new( 71, 10, 6, "b"),
+      @TK::TkNL        .new( 72, 10, 65, "\n"),
+      @TK::TkIDENTIFIER.new( 73, 11, 0, "a"),
+      @TK::TkSPACE     .new( 74, 11, 1, " "),
+      @TK::TkOPASGN    .new( 75, 11, 2, "||"),
+      @TK::TkSPACE     .new( 78, 11, 5, " "),
+      @TK::TkIDENTIFIER.new( 79, 11, 6, "b"),
+      @TK::TkNL        .new( 80, 11, 73, "\n"),
+      @TK::TkIDENTIFIER.new( 81, 12, 0, "a"),
+      @TK::TkSPACE     .new( 82, 12, 1, " "),
+      @TK::TkOPASGN    .new( 83, 12, 2, "**"),
+      @TK::TkSPACE     .new( 86, 12, 5, " "),
+      @TK::TkIDENTIFIER.new( 87, 12, 6, "b"),
+      @TK::TkNL        .new( 88, 12, 81, "\n"),
     ]
 
     assert_equal expected, tokens
@@ -126,10 +342,12 @@ Line 2
       @TK::TkSPACE     .new( 6, 1,  6, ' '),
       @TK::TkASSIGN    .new( 7, 1,  7, '='),
       @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkSTRING    .new( 9, 1,  9, %Q{"Line 1\nLine 2\n"}),
-      @TK::TkDOT       .new(41, 4, 42, '.'),
-      @TK::TkIDENTIFIER.new(42, 4, 43, 'chomp'),
-      @TK::TkNL        .new(47, 4, 48, "\n"),
+      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
+      @TK::TkDOT       .new(18, 1, 18, '.'),
+      @TK::TkIDENTIFIER.new(19, 1, 19, 'chomp'),
+      @TK::TkNL        .new(24, 1, 24, "\n"),
+      @TK::TkHEREDOC   .new(24, 1, 24, "Line 1\nLine 2\n"),
+      @TK::TkHEREDOCEND.new(47, 4, 39, "  STRING\n"),
     ]
 
     assert_equal expected, tokens
@@ -148,9 +366,12 @@ Line 2
       @TK::TkSPACE     .new( 6, 1,  6, ' '),
       @TK::TkASSIGN    .new( 7, 1,  7, '='),
       @TK::TkSPACE     .new( 8, 1,  8, ' '),
-      @TK::TkHEREDOC   .new( 9, 1,  9,
-                            %Q{<<-STRING\nLine 1\nLine 2\n  STRING}),
-      @TK::TkNL        .new(41, 4, 42, "\n"),
+
+
+      @TK::TkHEREDOCBEG.new( 9, 1,  9, '<<-STRING'),
+      @TK::TkNL        .new(18, 1, 18, "\n"),
+      @TK::TkHEREDOC   .new(18, 1, 18, "Line 1\nLine 2\n"),
+      @TK::TkHEREDOCEND.new(41, 4, 33, "  STRING\n")
     ]
 
     assert_equal expected, tokens
@@ -180,8 +401,10 @@ U
       @TK::TkSPACE     .new( 1, 1,  1, ' '),
       @TK::TkIDENTIFIER.new( 2, 1,  2, 'b'),
       @TK::TkSPACE     .new( 3, 1,  3, ' '),
-      @TK::TkHEREDOC   .new( 4, 1,  4, %Q{<<-U\n%N\nU}),
-      @TK::TkNL        .new(13, 3, 14, "\n"),
+      @TK::TkHEREDOCBEG.new( 4, 1,  4, '<<-U'),
+      @TK::TkNL        .new( 8, 1,  8, "\n"),
+      @TK::TkHEREDOC   .new( 8, 1,  8, "%N\n"),
+      @TK::TkHEREDOCEND.new(13, 3, 12, "U\n")
     ]
 
     assert_equal expected, tokens
@@ -195,6 +418,36 @@ U
     assert_equal expected, tokens.first
   end
 
+  def test_class_tokenize_lambda
+    tokens = RDoc::RubyLex.tokenize 'a = -> x, y { x + y }', nil
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, 'a'),
+      @TK::TkSPACE     .new( 1, 1,  1, ' '),
+      @TK::TkASSIGN    .new( 2, 1,  2, '='),
+      @TK::TkSPACE     .new( 3, 1,  3, ' '),
+      @TK::TkLAMBDA    .new( 4, 1,  4, '->'),
+      @TK::TkSPACE     .new( 6, 1,  6, ' '),
+      @TK::TkIDENTIFIER.new( 7, 1,  7, 'x'),
+      @TK::TkCOMMA     .new( 8, 1,  8, ','),
+      @TK::TkSPACE     .new( 9, 1,  9, ' '),
+      @TK::TkIDENTIFIER.new(10, 1, 10, 'y'),
+      @TK::TkSPACE     .new(11, 1, 11, ' '),
+      @TK::TkfLBRACE   .new(12, 1, 12, '{'),
+      @TK::TkSPACE     .new(13, 1, 13, ' '),
+      @TK::TkIDENTIFIER.new(14, 1, 14, 'x'),
+      @TK::TkSPACE     .new(15, 1, 15, ' '),
+      @TK::TkPLUS      .new(16, 1, 16, '+'),
+      @TK::TkSPACE     .new(17, 1, 17, ' '),
+      @TK::TkIDENTIFIER.new(18, 1, 18, 'y'),
+      @TK::TkSPACE     .new(19, 1, 19, ' '),
+      @TK::TkRBRACE    .new(20, 1, 20, '}'),
+      @TK::TkNL        .new(21, 1, 21, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_percent_1
     tokens = RDoc::RubyLex.tokenize 'v%10==10', nil
 
@@ -221,6 +474,39 @@ U
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_percent_r_with_slash
+    tokens = RDoc::RubyLex.tokenize '%r/hi/', nil
+
+    expected = [
+      @TK::TkREGEXP.new( 0, 1,  0, '%r/hi/'),
+      @TK::TkNL    .new( 6, 1, 6, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_percent_large_q
+    tokens = RDoc::RubyLex.tokenize '%Q/hi/', nil
+
+    expected = [
+      @TK::TkSTRING.new( 0, 1,  0, '%Q/hi/'),
+      @TK::TkNL    .new( 6, 1, 6, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_percent_large_q_with_double_quote
+    tokens = RDoc::RubyLex.tokenize '%Q"hi"', nil
+
+    expected = [
+      @TK::TkSTRING.new( 0, 1,  0, '%Q"hi"'),
+      @TK::TkNL    .new( 6, 1, 6, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_percent_w
     tokens = RDoc::RubyLex.tokenize '%w[hi]', nil
 
@@ -243,6 +529,35 @@ U
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_hash_rocket
+    tokens = RDoc::RubyLex.tokenize "{ :foo=> 1 }", nil
+
+    expected = [
+      @TK::TkLBRACE    .new( 0, 1,  0, '{'),
+      @TK::TkSPACE     .new( 1, 1,  1, ' '),
+      @TK::TkSYMBOL    .new( 2, 1,  2, ':foo'),
+      @TK::TkHASHROCKET.new( 6, 1,  6, '=>'),
+      @TK::TkSPACE     .new( 8, 1,  8, ' '),
+      @TK::TkINTEGER   .new( 9, 1,  9, '1'),
+      @TK::TkSPACE     .new(10, 1, 10, ' '),
+      @TK::TkRBRACE    .new(11, 1, 11, '}'),
+      @TK::TkNL        .new(12, 1, 12, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_percent_sign_quote
+    tokens = RDoc::RubyLex.tokenize '%%hi%', nil
+
+    expected = [
+      @TK::TkSTRING.new( 0, 1, 0, '%%hi%'),
+      @TK::TkNL    .new( 5, 1, 5, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_regexp
     tokens = RDoc::RubyLex.tokenize "/hay/", nil
 
@@ -296,6 +611,41 @@ U
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_number_with_sign_character
+    tokens = RDoc::RubyLex.tokenize "+3--3r", nil
+
+    expected = [
+      @TK::TkINTEGER .new(0, 1, 0, "+3"),
+      @TK::TkMINUS   .new(2, 1, 2, "-"),
+      @TK::TkRATIONAL.new(3, 1, 3, "-3r"),
+      @TK::TkNL      .new(6, 1, 6, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_regexp_continuing_backslash
+    tokens = RDoc::RubyLex.tokenize "/(?<!\\\\)\\n\z/", nil
+
+    expected = [
+      @TK::TkREGEXP.new( 0, 1,  0, "/(?<!\\\\)\\n\z/"),
+      @TK::TkNL    .new(12, 1, 12, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_single_quote_escape
+    tokens = RDoc::RubyLex.tokenize %q{'\\\\ \\' \\&'}, nil
+
+    expected = [
+      @TK::TkSTRING.new( 0, 1,  0, %q{'\\\\ \\' \\&'}),
+      @TK::TkNL    .new(10, 1, 10, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_string
     tokens = RDoc::RubyLex.tokenize "'hi'", nil
 
@@ -307,6 +657,184 @@ U
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_string_with_escape
+    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
+[
+  '\\',
+  '\'',
+  "'",
+  "\'\"\`",
+  "\#",
+  "\#{}",
+  "#",
+  "#{}",
+  /'"/,
+  /\'\"/,
+  /\//,
+  /\\/,
+  /\#/,
+  /\#{}/,
+  /#/,
+  /#{}/
+]
+RUBY
+
+    expected = [
+      @TK::TkLBRACK .new(  0,  1,   0, "["),
+      @TK::TkNL     .new(  1,  1,   1, "\n"),
+      @TK::TkSPACE  .new(  2,  2,   0, "  "),
+      @TK::TkSTRING .new(  4,  2,   2, "'\\\\'"),
+      @TK::TkCOMMA  .new(  8,  2,   6, ","),
+      @TK::TkNL     .new(  9,  2,   2, "\n"),
+      @TK::TkSPACE  .new( 10,  3,   0, "  "),
+      @TK::TkSTRING .new( 12,  3,   2, "'\\''"),
+      @TK::TkCOMMA  .new( 16,  3,   6, ","),
+      @TK::TkNL     .new( 17,  3,  10, "\n"),
+      @TK::TkSPACE  .new( 18,  4,   0, "  "),
+      @TK::TkSTRING .new( 20,  4,   2, "\"'\""),
+      @TK::TkCOMMA  .new( 23,  4,   5, ","),
+      @TK::TkNL     .new( 24,  4,  18, "\n"),
+      @TK::TkSPACE  .new( 25,  5,   0, "  "),
+      @TK::TkSTRING .new( 27,  5,   2, "\"\\'\\\"\\`\""),
+      @TK::TkCOMMA  .new( 35,  5,  10, ","),
+      @TK::TkNL     .new( 36,  5,  25, "\n"),
+      @TK::TkSPACE  .new( 37,  6,   0, "  "),
+      @TK::TkSTRING .new( 39,  6,   2, "\"\\#\""),
+      @TK::TkCOMMA  .new( 43,  6,   6, ","),
+      @TK::TkNL     .new( 44,  6,  37, "\n"),
+      @TK::TkSPACE  .new( 45,  7,   0, "  "),
+      @TK::TkSTRING .new( 47,  7,   2, "\"\\\#{}\""),
+      @TK::TkCOMMA  .new( 53,  7,   8, ","),
+      @TK::TkNL     .new( 54,  7,  45, "\n"),
+      @TK::TkSPACE  .new( 55,  8,   0, "  "),
+      @TK::TkSTRING .new( 57,  8,   2, "\"#\""),
+      @TK::TkCOMMA  .new( 60,  8,   5, ","),
+      @TK::TkNL     .new( 61,  8,  55, "\n"),
+      @TK::TkSPACE  .new( 62,  9,   0, "  "),
+      @TK::TkDSTRING.new( 64,  9,   2, "\"\#{}\""),
+      @TK::TkCOMMA  .new( 69,  9,   7, ","),
+      @TK::TkNL     .new( 70,  9,  62, "\n"),
+      @TK::TkSPACE  .new( 71, 10,   0, "  "),
+      @TK::TkREGEXP .new( 73, 10,   2, "/'\"/"),
+      @TK::TkCOMMA  .new( 77, 10,   6, ","),
+      @TK::TkNL     .new( 78, 10,  71, "\n"),
+      @TK::TkSPACE  .new( 79, 11,   0, "  "),
+      @TK::TkREGEXP .new( 81, 11,   2, "/\\'\\\"/"),
+      @TK::TkCOMMA  .new( 87, 11,   8, ","),
+      @TK::TkNL     .new( 88, 11,  79, "\n"),
+      @TK::TkSPACE  .new( 89, 12,   0, "  "),
+      @TK::TkREGEXP .new( 91, 12,   2, "/\\//"),
+      @TK::TkCOMMA  .new( 95, 12,   6, ","),
+      @TK::TkNL     .new( 96, 12,  89, "\n"),
+      @TK::TkSPACE  .new( 97, 13,   0, "  "),
+      @TK::TkREGEXP .new( 99, 13,   2, "/\\\\/"),
+      @TK::TkCOMMA  .new(103, 13,   6, ","),
+      @TK::TkNL     .new(104, 13,  97, "\n"),
+      @TK::TkSPACE  .new(105, 14,   0, "  "),
+      @TK::TkREGEXP .new(107, 14,   2, "/\\#/"),
+      @TK::TkCOMMA  .new(111, 14,   6, ","),
+      @TK::TkNL     .new(112, 14, 105, "\n"),
+      @TK::TkSPACE  .new(113, 15,   0, "  "),
+      @TK::TkREGEXP .new(115, 15,   2, "/\\\#{}/"),
+      @TK::TkCOMMA  .new(121, 15,   8, ","),
+      @TK::TkNL     .new(122, 15, 113, "\n"),
+      @TK::TkSPACE  .new(123, 16,   0, "  "),
+      @TK::TkREGEXP .new(125, 16,   2, "/#/"),
+      @TK::TkCOMMA  .new(128, 16,   5, ","),
+      @TK::TkNL     .new(129, 16, 123, "\n"),
+      @TK::TkSPACE  .new(130, 17,   0, "  "),
+      @TK::TkDREGEXP.new(132, 17,   2, "/\#{}/"),
+      @TK::TkNL     .new(137, 17,   7, "\n"),
+      @TK::TkRBRACK .new(138, 18,   0, "]"),
+      @TK::TkNL     .new(139, 18, 138, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_postfix_if_after_escaped_newline
+    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
+def a
+  1 if true
+  1 \
+    if true
+end
+RUBY
+
+    expected = [
+      @TK::TkDEF       .new( 0, 1, 0,  "def"),
+      @TK::TkSPACE     .new( 3, 1, 3,  " "),
+      @TK::TkIDENTIFIER.new( 4, 1, 4,  "a"),
+      @TK::TkNL        .new( 5, 1, 5,  "\n"),
+      @TK::TkSPACE     .new( 6, 2, 0,  "  "),
+      @TK::TkINTEGER   .new( 8, 2, 2,  "1"),
+      @TK::TkSPACE     .new( 9, 2, 3,  " "),
+      @TK::TkIF_MOD    .new(10, 2, 4,  "if"),
+      @TK::TkSPACE     .new(12, 2, 6,  " "),
+      @TK::TkTRUE      .new(13, 2, 7,  "true"),
+      @TK::TkNL        .new(17, 2, 6,  "\n"),
+      @TK::TkSPACE     .new(18, 3, 0,  "  "),
+      @TK::TkINTEGER   .new(20, 3, 2,  "1"),
+      @TK::TkSPACE     .new(21, 3, 3,  " "),
+      @TK::TkBACKSLASH .new(22, 3, 4,  "\\"),
+      @TK::TkNL        .new(23, 3, 18, "\n"),
+      @TK::TkSPACE     .new(24, 4, 0,  "    "),
+      @TK::TkIF_MOD    .new(28, 4, 4,  "if"),
+      @TK::TkSPACE     .new(30, 4, 6,  " "),
+      @TK::TkTRUE      .new(31, 4, 7,  "true"),
+      @TK::TkNL        .new(35, 4, 24, "\n"),
+      @TK::TkEND       .new(36, 5, 0,  "end"),
+      @TK::TkNL        .new(39, 5, 36, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_backtick_with_escape
+    tokens = RDoc::RubyLex.tokenize <<'RUBY', nil
+[
+  `\\`,
+  `\'\"\``,
+  `\#`,
+  `\#{}`,
+  `#`,
+  `#{}`
+]
+RUBY
+
+    expected = [
+      @TK::TkLBRACK  .new( 0, 1,  0, "["),
+      @TK::TkNL      .new( 1, 1,  1, "\n"),
+      @TK::TkSPACE   .new( 2, 2,  0, "  "),
+      @TK::TkXSTRING .new( 4, 2,  2, "`\\\\`"),
+      @TK::TkCOMMA   .new( 8, 2,  6, ","),
+      @TK::TkNL      .new( 9, 2,  2, "\n"),
+      @TK::TkSPACE   .new(10, 3,  0, "  "),
+      @TK::TkXSTRING .new(12, 3,  2, "`\\'\\\"\\``"),
+      @TK::TkCOMMA   .new(20, 3, 10, ","),
+      @TK::TkNL      .new(21, 3, 10, "\n"),
+      @TK::TkSPACE   .new(22, 4,  0, "  "),
+      @TK::TkXSTRING .new(24, 4,  2, "`\\#`"),
+      @TK::TkCOMMA   .new(28, 4,  6, ","),
+      @TK::TkNL      .new(29, 4, 22, "\n"),
+      @TK::TkSPACE   .new(30, 5,  0, "  "),
+      @TK::TkXSTRING .new(32, 5,  2, "`\\\#{}`"),
+      @TK::TkCOMMA   .new(38, 5,  8, ","),
+      @TK::TkNL      .new(39, 5, 30, "\n"),
+      @TK::TkSPACE   .new(40, 6,  0, "  "),
+      @TK::TkXSTRING .new(42, 6,  2, "`#`"),
+      @TK::TkCOMMA   .new(45, 6,  5, ","),
+      @TK::TkNL      .new(46, 6, 40, "\n"),
+      @TK::TkSPACE   .new(47, 7,  0, "  "),
+      @TK::TkDXSTRING.new(49, 7,  2, "`\#{}`"),
+      @TK::TkNL      .new(54, 7,  7, "\n"),
+      @TK::TkRBRACK  .new(55, 8,  0, "]"),
+      @TK::TkNL      .new(56, 8, 55, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_class_tokenize_string_escape
     tokens = RDoc::RubyLex.tokenize '"\\n"', nil
     assert_equal @TK::TkSTRING.new( 0, 1,  0, "\"\\n\""), tokens.first
@@ -390,17 +918,87 @@ U
     expected = [
       @TK::TkIDENTIFIER.new( 0, 1,  0, 'scope'),
       @TK::TkSPACE     .new( 5, 1,  5, ' '),
-      @TK::TkIDENTIFIER.new( 6, 1,  6, 'module'),
-      @TK::TkCOLON     .new(12, 1, 12, ':'),
+      @TK::TkSYMBOL    .new( 6, 1,  6, 'module:'),
       @TK::TkSPACE     .new(13, 1, 13, ' '),
-      @TK::TkSYMBEG    .new(14, 1, 14, ':'),
-      @TK::TkIDENTIFIER.new(15, 1, 15, 'v1'),
+      @TK::TkSYMBOL    .new(14, 1, 14, ':v1'),
       @TK::TkNL        .new(17, 1, 17, "\n"),
     ]
 
     assert_equal expected, tokens
   end
 
+  def test_class_tokenize_particular_kind_of_symbols
+    tokens = RDoc::RubyLex.tokenize '{ Thomas: :Thomas, Dave!: :Dave!, undef: :undef }', nil
+
+    expected = [
+      @TK::TkLBRACE.new( 0, 1,  0, "{"),
+      @TK::TkSPACE .new( 1, 1,  1, " "),
+      @TK::TkSYMBOL.new( 2, 1,  2, "Thomas:"),
+      @TK::TkSPACE .new( 9, 1,  9, " "),
+      @TK::TkSYMBOL.new(10, 1, 10, ":Thomas"),
+      @TK::TkCOMMA .new(17, 1, 17, ","),
+      @TK::TkSPACE .new(18, 1, 18, " "),
+      @TK::TkSYMBOL.new(19, 1, 19, "Dave!:"),
+      @TK::TkSPACE .new(25, 1, 25, " "),
+      @TK::TkSYMBOL.new(26, 1, 26, ":Dave!"),
+      @TK::TkCOMMA .new(32, 1, 32, ","),
+      @TK::TkSPACE .new(33, 1, 33, " "),
+      @TK::TkSYMBOL.new(34, 1, 34, "undef:"),
+      @TK::TkSPACE .new(40, 1, 40, " "),
+      @TK::TkSYMBOL.new(41, 1, 41, ":undef"),
+      @TK::TkSPACE .new(47, 1, 47, " "),
+      @TK::TkRBRACE.new(48, 1, 48, "}"),
+      @TK::TkNL    .new(49, 1, 49, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_symbol_for_nested_method
+    tokens = RDoc::RubyLex.tokenize 'return untrace_var :name', nil
+
+    expected = [
+      @TK::TkRETURN    .new( 0, 1,  0, "return"),
+      @TK::TkSPACE     .new( 6, 1,  6, " "),
+      @TK::TkIDENTIFIER.new( 7, 1,  7, "untrace_var"),
+      @TK::TkSPACE     .new(18, 1, 18, " "),
+      @TK::TkSYMBOL    .new(19, 1, 19, ":name"),
+      @TK::TkNL        .new(24, 1, 24, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_symbol_with_quote
+    tokens = RDoc::RubyLex.tokenize <<RUBY, nil
+a.include?()?"a":"b"
+{"t":1,'t2':2}
+RUBY
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1,  0, "a"),
+      @TK::TkDOT       .new( 1, 1,  1, "."),
+      @TK::TkFID       .new( 2, 1,  2, "include?"),
+      @TK::TkLPAREN    .new(10, 1, 10, "("),
+      @TK::TkRPAREN    .new(11, 1, 11, ")"),
+      @TK::TkQUESTION  .new(12, 1, 12, "?"),
+      @TK::TkSTRING    .new(13, 1, 13, "\"a\""),
+      @TK::TkCOLON     .new(16, 1, 16, ":"),
+      @TK::TkSTRING    .new(17, 1, 17, "\"b\""),
+      @TK::TkNL        .new(20, 1, 20, "\n"),
+      @TK::TkLBRACE    .new(21, 2,  0, "{"),
+      @TK::TkSYMBOL    .new(22, 2,  1, "\"t\":"),
+      @TK::TkINTEGER   .new(26, 2,  5, "1"),
+      @TK::TkCOMMA     .new(27, 2,  6, ","),
+      @TK::TkSYMBOL    .new(28, 2,  7, "'t2':"),
+      @TK::TkINTEGER   .new(33, 2, 12, "2"),
+      @TK::TkRBRACE    .new(34, 2, 13, "}"),
+      @TK::TkNL        .new(35, 2, 21, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
   def test_unary_minus
     ruby_lex = RDoc::RubyLex.new("-1", nil)
     assert_equal("-1", ruby_lex.token.value)
@@ -418,5 +1016,80 @@ U
     assert_equal("-0.1", ruby_lex.token.value)
   end
 
+  def test_rational_imaginary_tokenize
+    tokens = RDoc::RubyLex.tokenize '1.11r + 2.34i + 5.55ri + 0i', nil
+
+    expected = [
+      @TK::TkRATIONAL .new( 0, 1,  0, '1.11r'),
+      @TK::TkSPACE    .new( 5, 1,  5, ' '),
+      @TK::TkPLUS     .new( 6, 1,  6, '+'),
+      @TK::TkSPACE    .new( 7, 1,  7, ' '),
+      @TK::TkIMAGINARY.new( 8, 1,  8, '2.34i'),
+      @TK::TkSPACE    .new(13, 1, 13, ' '),
+      @TK::TkPLUS     .new(14, 1, 14, '+'),
+      @TK::TkSPACE    .new(15, 1, 15, ' '),
+      @TK::TkIMAGINARY.new(16, 1, 16, '5.55ri'),
+      @TK::TkSPACE    .new(22, 1, 22, ' '),
+      @TK::TkPLUS     .new(23, 1, 23, '+'),
+      @TK::TkSPACE    .new(24, 1, 24, ' '),
+      @TK::TkIMAGINARY.new(25, 1, 25, '0i'),
+      @TK::TkNL       .new(27, 1, 27, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_square_bracket_as_method
+    tokens = RDoc::RubyLex.tokenize "Array.[](1, 2)", nil
+
+    expected = [
+      @TK::TkCONSTANT  .new(0,  1,  0, "Array"),
+      @TK::TkDOT       .new(5,  1,  5, "."),
+      @TK::TkIDENTIFIER.new(6,  1,  6, "[]"),
+      @TK::TkfLPAREN   .new(8,  1,  8, "("),
+      @TK::TkINTEGER   .new(9,  1,  9, "1"),
+      @TK::TkCOMMA     .new(10, 1, 10, ","),
+      @TK::TkSPACE     .new(11, 1, 11, " "),
+      @TK::TkINTEGER   .new(12, 1, 12, "2"),
+      @TK::TkRPAREN    .new(13, 1, 13, ")"),
+      @TK::TkNL        .new(14, 1, 14, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_constant_with_exclamation
+    tokens = RDoc::RubyLex.tokenize "Hello there, Dave!", nil
+
+    expected = [
+      @TK::TkCONSTANT  .new( 0, 1,  0, "Hello"),
+      @TK::TkSPACE     .new( 5, 1,  5, " "),
+      @TK::TkIDENTIFIER.new( 6, 1,  6, "there"),
+      @TK::TkCOMMA     .new(11, 1, 11, ","),
+      @TK::TkSPACE     .new(12, 1, 12, " "),
+      @TK::TkIDENTIFIER.new(13, 1, 13, "Dave!"),
+      @TK::TkNL        .new(18, 1, 18, "\n")
+    ]
+
+    assert_equal expected, tokens
+  end
+
+  def test_class_tokenize_identifer_not_equal
+    tokens = RDoc::RubyLex.tokenize "foo!=bar\nfoo?=bar", nil
+
+    expected = [
+      @TK::TkIDENTIFIER.new( 0, 1, 0, "foo"),
+      @TK::TkNEQ       .new( 3, 1, 3, "!="),
+      @TK::TkIDENTIFIER.new( 5, 1, 5, "bar"),
+      @TK::TkNL        .new( 8, 1, 8, "\n"),
+      @TK::TkFID       .new( 9, 2, 0, "foo?"),
+      @TK::TkASSIGN    .new(13, 2, 4, "="),
+      @TK::TkIDENTIFIER.new(14, 2, 5, "bar"),
+      @TK::TkNL        .new(17, 2, 9, "\n"),
+    ]
+
+    assert_equal expected, tokens
+  end
+
 end
 
diff --git a/test/rdoc/test_rdoc_store.rb b/test/rdoc/test_rdoc_store.rb
index 2c52d109ff..d32eab3ec1 100644
--- a/test/rdoc/test_rdoc_store.rb
+++ b/test/rdoc/test_rdoc_store.rb
@@ -162,7 +162,7 @@ class TestRDocStore < XrefTestCase
 
   def test_all_classes_and_modules
     expected = %w[
-      C1 C2 C2::C3 C2::C3::H1 C3 C3::H1 C3::H2 C4 C4::C4 C5 C5::C1 C6
+      C1 C2 C2::C3 C2::C3::H1 C3 C3::H1 C3::H2 C4 C4::C4 C5 C5::C1 C6 C7
       Child
       M1 M1::M2
       Parent
@@ -213,7 +213,7 @@ class TestRDocStore < XrefTestCase
 
   def test_classes
     expected = %w[
-      C1 C2 C2::C3 C2::C3::H1 C3 C3::H1 C3::H2 C4 C4::C4 C5 C5::C1 C6
+      C1 C2 C2::C3 C2::C3::H1 C3 C3::H1 C3::H2 C4 C4::C4 C5 C5::C1 C6 C7
       Child
       Parent
     ]
diff --git a/test/rdoc/xref_data.rb b/test/rdoc/xref_data.rb
index 44d44956ac..c712ab8891 100644
--- a/test/rdoc/xref_data.rb
+++ b/test/rdoc/xref_data.rb
@@ -82,6 +82,18 @@ class C6
   def prot6() end
 end
 
+class C7
+  attr_reader :attr_reader
+  attr_reader :attr_reader_nodoc # :nodoc:
+  attr_writer :attr_writer
+  attr_writer :attr_writer_nodoc # :nodoc:
+  attr_accessor :attr_accessor
+  attr_accessor :attr_accessor_nodoc # :nodoc:
+
+  CONST = :const
+  CONST_NODOC = :const_nodoc # :nodoc:
+end
+
 module M1
   def m
   end
diff --git a/test/rdoc/xref_test_case.rb b/test/rdoc/xref_test_case.rb
index ea7fe705e3..9b2f876e5a 100644
--- a/test/rdoc/xref_test_case.rb
+++ b/test/rdoc/xref_test_case.rb
@@ -52,6 +52,7 @@ class XrefTestCase < RDoc::TestCase
     @c3_h1 = @xref_data.find_module_named 'C3::H1'
     @c3_h2 = @xref_data.find_module_named 'C3::H2'
     @c6    = @xref_data.find_module_named 'C6'
+    @c7    = @xref_data.find_module_named 'C7'
 
     @m1    = @xref_data.find_module_named 'M1'
     @m1_m  = @m1.method_list.first